1 /*
   2  * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
  25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
  26 
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  29 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
  30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  31 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  32 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  34 
  35 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
  36   return ShenandoahForwarding::get_forwardee(p);
  37 }
  38 
  39 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
  40   if (((HeapWord*) p) != NULL) {
  41     return resolve_forwarded_not_null(p);
  42   } else {
  43     return p;
  44   }
  45 }
  46 
  47 template <DecoratorSet decorators, typename BarrierSetT>
  48 template <typename T>
  49 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
  50   oop value = Raw::oop_load_in_heap(addr);
  51   value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
  52   keep_alive_if_weak(decorators, value);
  53   return value;
  54 }
  55 
  56 template <DecoratorSet decorators, typename BarrierSetT>
  57 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
  58   oop value = Raw::oop_load_in_heap_at(base, offset);
  59   value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
  60   keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
  61   return value;
  62 }
  63 
  64 template <DecoratorSet decorators, typename BarrierSetT>
  65 template <typename T>
  66 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
  67   oop value = Raw::oop_load_not_in_heap(addr);
  68   value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
  69   keep_alive_if_weak(decorators, value);
  70   return value;
  71 }
  72 
  73 template <DecoratorSet decorators, typename BarrierSetT>
  74 template <typename T>
  75 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) {
  76   ShenandoahBarrierSet::barrier_set()->storeval_barrier(value);
  77   const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
  78   if (keep_alive) {
  79     ShenandoahBarrierSet::barrier_set()->write_ref_field_pre_work(addr, value);
  80   }
  81   Raw::oop_store_in_heap(addr, value);
  82 }
  83 
  84 template <DecoratorSet decorators, typename BarrierSetT>
  85 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
  86   oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value);
  87 }
  88 
  89 template <DecoratorSet decorators, typename BarrierSetT>
  90 template <typename T>
  91 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) {
  92   oop res;
  93   oop expected = compare_value;
  94   do {
  95     compare_value = expected;
  96     res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
  97     expected = res;
  98   } while (compare_value != expected && resolve_forwarded(compare_value) == resolve_forwarded(expected));
  99   if (res != NULL) {
 100     return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res);
 101   } else {
 102     return res;
 103   }
 104 }
 105 
 106 template <DecoratorSet decorators, typename BarrierSetT>
 107 template <typename T>
 108 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_impl(oop new_value, T* addr, oop compare_value) {
 109   ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
 110   oop result = oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value);
 111   const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
 112   if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) &&
 113       result == compare_value &&
 114       ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
 115     ShenandoahBarrierSet::barrier_set()->enqueue(result);
 116   }
 117   return result;
 118 }
 119 
 120 template <DecoratorSet decorators, typename BarrierSetT>
 121 template <typename T>
 122 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
 123   oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, addr, compare_value);
 124   keep_alive_if_weak(decorators, result);
 125   return result;
 126 }
 127 
 128 template <DecoratorSet decorators, typename BarrierSetT>
 129 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
 130   oop result = oop_atomic_cmpxchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
 131   keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result);
 132   return result;
 133 }
 134 
 135 template <DecoratorSet decorators, typename BarrierSetT>
 136 template <typename T>
 137 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) {
 138   oop previous = Raw::oop_atomic_xchg(new_value, addr);
 139   if (previous != NULL) {
 140     return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous);
 141   } else {
 142     return previous;
 143   }
 144 }
 145 
 146 template <DecoratorSet decorators, typename BarrierSetT>
 147 template <typename T>
 148 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_impl(oop new_value, T* addr) {
 149   ShenandoahBarrierSet::barrier_set()->storeval_barrier(new_value);
 150   oop result = oop_atomic_xchg_not_in_heap(new_value, addr);
 151   const bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0;
 152   if (keep_alive && ShenandoahSATBBarrier && !CompressedOops::is_null(result) &&
 153       ShenandoahHeap::heap()->is_concurrent_mark_in_progress()) {
 154     ShenandoahBarrierSet::barrier_set()->enqueue(result);
 155   }
 156   return result;
 157 }
 158 
 159 template <DecoratorSet decorators, typename BarrierSetT>
 160 template <typename T>
 161 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
 162   oop result = oop_atomic_xchg_in_heap_impl(new_value, addr);
 163   keep_alive_if_weak(addr, result);
 164   return result;
 165 }
 166 
 167 template <DecoratorSet decorators, typename BarrierSetT>
 168 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) {
 169   oop result = oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset));
 170   keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), result);
 171   return result;
 172 }
 173 
 174 template <typename T>
 175 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound,
 176                                             bool checkcast, bool satb, bool disjoint,
 177                                             ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
 178   if (checkcast) {
 179     return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, disjoint, storeval_mode);
 180   } else {
 181     return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, disjoint, storeval_mode);
 182   }
 183 }
 184 
 185 template <typename T, bool CHECKCAST>
 186 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound,
 187                                             bool satb, bool disjoint,
 188                                             ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
 189   if (satb) {
 190     return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, disjoint, storeval_mode);
 191   } else {
 192     return arraycopy_loop_3<T, CHECKCAST, false>(src, dst, length, bound, disjoint, storeval_mode);
 193   }
 194 }
 195 
 196 template <typename T, bool CHECKCAST, bool SATB>
 197 bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, bool disjoint,
 198                                             ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {
 199   switch (storeval_mode) {
 200     case NONE:
 201       return arraycopy_loop<T, CHECKCAST, SATB, NONE>(src, dst, length, bound, disjoint);
 202     case RESOLVE_BARRIER:
 203       return arraycopy_loop<T, CHECKCAST, SATB, RESOLVE_BARRIER>(src, dst, length, bound, disjoint);
 204     case EVAC_BARRIER:
 205       return arraycopy_loop<T, CHECKCAST, SATB, EVAC_BARRIER>(src, dst, length, bound, disjoint);
 206     default:
 207       ShouldNotReachHere();
 208       return true; // happy compiler
 209   }
 210 }
 211 
 212 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE>
 213 bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint) {
 214   Thread* thread = Thread::current();
 215   ShenandoahMarkingContext* ctx = _heap->marking_context();
 216   ShenandoahEvacOOMScope oom_evac_scope;
 217 
 218   // We need to handle four cases:
 219   //
 220   // a) src < dst, conjoint, can only copy backward only
 221   //   [...src...]
 222   //         [...dst...]
 223   //
 224   // b) src < dst, disjoint, can only copy forward, because types may mismatch
 225   //   [...src...]
 226   //              [...dst...]
 227   //
 228   // c) src > dst, conjoint, can copy forward only
 229   //         [...src...]
 230   //   [...dst...]
 231   //
 232   // d) src > dst, disjoint, can only copy forward, because types may mismatch
 233   //              [...src...]
 234   //   [...dst...]
 235   //
 236   if (src > dst || disjoint) {
 237     // copy forward:
 238     T* cur_src = src;
 239     T* cur_dst = dst;
 240     T* src_end = src + length;
 241     for (; cur_src < src_end; cur_src++, cur_dst++) {
 242       if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread, ctx)) {
 243         return false;
 244       }
 245     }
 246   } else {
 247     // copy backward:
 248     T* cur_src = src + length - 1;
 249     T* cur_dst = dst + length - 1;
 250     for (; cur_src >= src; cur_src--, cur_dst--) {
 251       if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread, ctx)) {
 252         return false;
 253       }
 254     }
 255   }
 256   return true;
 257 }
 258 
 259 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE>
 260 bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* const thread, ShenandoahMarkingContext* const ctx) {
 261   T o = RawAccess<>::oop_load(cur_src);
 262 
 263   if (SATB) {
 264     assert(ShenandoahThreadLocalData::satb_mark_queue(thread).is_active(), "Shouldn't be here otherwise");
 265     T prev = RawAccess<>::oop_load(cur_dst);
 266     if (!CompressedOops::is_null(prev)) {
 267       oop prev_obj = CompressedOops::decode_not_null(prev);
 268       switch (STOREVAL_MODE) {
 269       case NONE:
 270         break;
 271       case RESOLVE_BARRIER:
 272       case EVAC_BARRIER:
 273         // The evac-barrier case cannot really happen. It's traversal-only and traversal
 274         // doesn't currently use SATB. And even if it did, it would not be fatal to just do the normal resolve here.
 275         prev_obj = ShenandoahBarrierSet::resolve_forwarded_not_null(prev_obj);
 276       }
 277       if (!ctx->is_marked(prev_obj)) {
 278         ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(prev_obj);
 279       }
 280     }
 281   }
 282 
 283   if (!CompressedOops::is_null(o)) {
 284     oop obj = CompressedOops::decode_not_null(o);
 285 
 286     if (CHECKCAST) {
 287       assert(bound != NULL, "need element klass for checkcast");
 288       if (!oopDesc::is_instanceof_or_null(obj, bound)) {
 289         return false;
 290       }
 291     }
 292 
 293     switch (STOREVAL_MODE) {
 294     case NONE:
 295       break;
 296     case RESOLVE_BARRIER:
 297       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 298       break;
 299     case EVAC_BARRIER:
 300       if (_heap->in_collection_set(obj)) {
 301         oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 302         if (forw == obj) {
 303           forw = _heap->evacuate_object(forw, thread);
 304         }
 305         obj = forw;
 306       }
 307       enqueue(obj);
 308       break;
 309     default:
 310       ShouldNotReachHere();
 311     }
 312 
 313     RawAccess<IS_NOT_NULL>::oop_store(cur_dst, obj);
 314   } else {
 315     // Store null.
 316     RawAccess<>::oop_store(cur_dst, o);
 317   }
 318   return true;
 319 }
 320 
 321 // Clone barrier support
 322 template <DecoratorSet decorators, typename BarrierSetT>
 323 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
 324   Raw::clone(src, dst, size);
 325   ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));
 326 }
 327 
 328 template <DecoratorSet decorators, typename BarrierSetT>
 329 template <typename T>
 330 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 331                                                                                          arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 332                                                                                          size_t length) {
 333   ShenandoahHeap* heap = ShenandoahHeap::heap();
 334   bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();
 335   bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;
 336   bool disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
 337   ArrayCopyStoreValMode storeval_mode;
 338   if (heap->has_forwarded_objects()) {
 339     if (heap->is_concurrent_traversal_in_progress()) {
 340       storeval_mode = EVAC_BARRIER;
 341     } else if (heap->is_update_refs_in_progress()) {
 342       storeval_mode = RESOLVE_BARRIER;
 343     } else {
 344       assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress");
 345       storeval_mode = NONE; // E.g. during evac or outside cycle
 346     }
 347   } else {
 348     assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress");
 349     storeval_mode = NONE;
 350   }
 351 
 352   if (!satb && !checkcast && storeval_mode == NONE) {
 353     // Short-circuit to bulk copy.
 354     return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
 355   }
 356 
 357   src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
 358   dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
 359 
 360   Klass* bound = objArrayOop(dst_obj)->element_klass();
 361   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 362   return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, disjoint, storeval_mode);
 363 }
 364 
 365 #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP