< prev index next >

src/hotspot/share/gc/z/zBarrierSet.inline.hpp

Print this page

 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
 25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
 26 
 27 #include "gc/z/zBarrierSet.hpp"
 28 
 29 #include "gc/shared/accessBarrierSupport.inline.hpp"
 30 #include "gc/z/zAddress.inline.hpp"
 31 #include "gc/z/zBarrier.inline.hpp"
 32 #include "gc/z/zIterator.inline.hpp"
 33 #include "gc/z/zNMethod.hpp"
 34 #include "memory/iterator.inline.hpp"

 35 #include "utilities/debug.hpp"
 36 
 37 template <DecoratorSet decorators, typename BarrierSetT>
 38 template <DecoratorSet expected>
 39 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
 40   if ((decorators & expected) == 0) {
 41     fatal("Using unsupported access decorators");
 42   }
 43 }
 44 
 45 template <DecoratorSet decorators, typename BarrierSetT>
 46 template <DecoratorSet expected>
 47 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
 48   if ((decorators & expected) != 0) {
 49     fatal("Using unsupported access decorators");
 50   }
 51 }
 52 
 53 template <DecoratorSet decorators, typename BarrierSetT>
 54 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {

312   verify_decorators_absent<AS_NO_KEEPALIVE>();
313 
314   zpointer* const p = field_addr(base, offset);
315 
316   store_barrier_heap_with_healing(p);
317 
318   const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
319   assert_is_valid(o);
320 
321   return to_oop(ZPointer::uncolor_store_good(o));
322 }
323 
324 template <DecoratorSet decorators, typename BarrierSetT>
325 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
326   store_barrier_heap_without_healing(dst);
327 
328   return ZBarrier::load_barrier_on_oop_field(src);
329 }
330 
331 template <DecoratorSet decorators, typename BarrierSetT>
332 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
333   const zaddress obj = oop_copy_one_barriers(dst, src);
334 




335   Atomic::store(dst, ZAddress::store_good(obj));

336 }
337 
338 template <DecoratorSet decorators, typename BarrierSetT>
339 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
340   const zaddress obj = oop_copy_one_barriers(dst, src);

341 
342   if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {



343     // Check cast failed
344     return false;
345   }
346 
347   Atomic::store(dst, ZAddress::store_good(obj));
348 
349   return true;
350 }
351 
352 
353 template <DecoratorSet decorators, typename BarrierSetT>
354 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
355   // Check cast and copy each elements
356   for (const zpointer* const end = src + length; src < end; src++, dst++) {
357     if (!oop_copy_one_check_cast(dst, src, dst_klass)) {
358       // Check cast failed
359       return false;
360     }
361   }
362 
363   return true;
364 }
365 
366 template <DecoratorSet decorators, typename BarrierSetT>
367 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
368   const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
369 
370   if (is_disjoint || src > dst) {
371     for (const zpointer* const end = src + length; src < end; src++, dst++) {
372       oop_copy_one(dst, src);
373     }
374     return true;
375   }
376 
377   if (src < dst) {
378     const zpointer* const end = src;
379     src += length - 1;
380     dst += length - 1;
381     for ( ; src >= end; src--, dst--) {
382       oop_copy_one(dst, src);
383     }
384     return true;
385   }
386 
387   // src and dst are the same; nothing to do
388   return true;
389 }
390 
391 template <DecoratorSet decorators, typename BarrierSetT>
392 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
393                                                                                        arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
394                                                                                        size_t length) {
395   zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
396   zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);

397 
398   if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
399     Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
400     return oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);


401   }
402 
403   return oop_arraycopy_in_heap_no_check_cast(dst, src, length);












404 }
405 
406 class ZStoreBarrierOopClosure : public BasicOopIterateClosure {
407 public:
408   virtual void do_oop(oop* p_) {
409     volatile zpointer* const p = (volatile zpointer*)p_;
410     const zpointer ptr = ZBarrier::load_atomic(p);
411     const zaddress addr = ZPointer::uncolor(ptr);
412     ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
413     *p = ZAddress::store_good(addr);
414   }
415 
416   virtual void do_oop(narrowOop* p) {
417     ShouldNotReachHere();
418   }
419 };
420 
421 class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
422 public:
423   virtual void do_oop(oop* p) {

430 };
431 
432 template <DecoratorSet decorators, typename BarrierSetT>
433 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
434   assert_is_valid(to_zaddress(src));
435 
436   // Fix the oops
437   ZLoadBarrierOopClosure cl;
438   ZIterator::oop_iterate(src, &cl);
439 
440   // Clone the object
441   Raw::clone_in_heap(src, dst, size);
442 
443   assert(ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
444 
445   // Color store good before handing out
446   ZStoreBarrierOopClosure cl_sg;
447   ZIterator::oop_iterate(dst, &cl_sg);
448 }
449 




















450 //
451 // Not in heap
452 //
453 template <DecoratorSet decorators, typename BarrierSetT>
454 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
455   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
456 
457   const zpointer o = Raw::template load<zpointer>(p);
458   assert_is_valid(o);
459   return to_oop(load_barrier(p, o));
460 }
461 
462 template <DecoratorSet decorators, typename BarrierSetT>
463 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
464   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
465 
466   if (HasDecorator<decorators, IN_NMETHOD>::value) {
467     return ZNMethod::load_oop(p, decorators);
468   } else {
469     return oop_load_not_in_heap((zpointer*)p);

 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
 25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
 26 
 27 #include "gc/z/zBarrierSet.hpp"
 28 
 29 #include "gc/shared/accessBarrierSupport.inline.hpp"
 30 #include "gc/z/zAddress.inline.hpp"
 31 #include "gc/z/zBarrier.inline.hpp"
 32 #include "gc/z/zIterator.inline.hpp"
 33 #include "gc/z/zNMethod.hpp"
 34 #include "memory/iterator.inline.hpp"
 35 #include "oops/inlineKlass.inline.hpp"
 36 #include "utilities/debug.hpp"
 37 
 38 template <DecoratorSet decorators, typename BarrierSetT>
 39 template <DecoratorSet expected>
 40 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
 41   if ((decorators & expected) == 0) {
 42     fatal("Using unsupported access decorators");
 43   }
 44 }
 45 
 46 template <DecoratorSet decorators, typename BarrierSetT>
 47 template <DecoratorSet expected>
 48 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
 49   if ((decorators & expected) != 0) {
 50     fatal("Using unsupported access decorators");
 51   }
 52 }
 53 
 54 template <DecoratorSet decorators, typename BarrierSetT>
 55 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {

313   verify_decorators_absent<AS_NO_KEEPALIVE>();
314 
315   zpointer* const p = field_addr(base, offset);
316 
317   store_barrier_heap_with_healing(p);
318 
319   const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
320   assert_is_valid(o);
321 
322   return to_oop(ZPointer::uncolor_store_good(o));
323 }
324 
325 template <DecoratorSet decorators, typename BarrierSetT>
326 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
327   store_barrier_heap_without_healing(dst);
328 
329   return ZBarrier::load_barrier_on_oop_field(src);
330 }
331 
332 template <DecoratorSet decorators, typename BarrierSetT>
333 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
334   const zaddress obj = oop_copy_one_barriers(dst, src);
335 
336   if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) {
337     return oop_copy_check_null;
338   }
339 
340   Atomic::store(dst, ZAddress::store_good(obj));
341   return oop_copy_check_ok;
342 }
343 
344 template <DecoratorSet decorators, typename BarrierSetT>
345 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
346   const zaddress obj = oop_copy_one_barriers(dst, src);
347   const bool null_check = HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value;
348 
349   if (null_check && is_null(obj)) {
350     return oop_copy_check_null;
351   }
352   else if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
353     // Check cast failed
354     return oop_copy_check_class_cast;
355   }
356 
357   Atomic::store(dst, ZAddress::store_good(obj));
358 
359   return oop_copy_check_ok;
360 }
361 
362 
363 template <DecoratorSet decorators, typename BarrierSetT>
364 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
365   // Check cast and copy each elements
366   OopCopyCheckStatus check_status = oop_copy_check_ok;
367   for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
368     check_status = oop_copy_one_check_cast(dst, src, dst_klass);


369   }
370   return check_status;

371 }
372 
373 template <DecoratorSet decorators, typename BarrierSetT>
374 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
375   const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
376   OopCopyCheckStatus check_status = oop_copy_check_ok;
377   if (is_disjoint || src > dst) {
378     for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
379       check_status = oop_copy_one(dst, src);
380     }
381     return check_status;
382   }
383 
384   if (src < dst) {
385     const zpointer* const end = src;
386     src += length - 1;
387     dst += length - 1;
388     for ( ; (check_status == oop_copy_check_ok) && (src >= end); src--, dst--) {
389       check_status = oop_copy_one(dst, src);
390     }
391     return check_status;
392   }
393 
394   // src and dst are the same; nothing to do
395   return check_status;
396 }
397 
398 template <DecoratorSet decorators, typename BarrierSetT>
399 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
400                                                                                        arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
401                                                                                        size_t length) {
402   zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
403   zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
404   OopCopyCheckStatus check_status;
405 
406   if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
407     Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
408     check_status = oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
409   } else {
410     check_status = oop_arraycopy_in_heap_no_check_cast(dst, src, length);
411   }
412 
413   switch (check_status) {
414   case oop_copy_check_ok:
415     return;
416   case oop_copy_check_class_cast:
417     throw_array_store_exception(src_obj, dst_obj, JavaThread::current());
418     break;
419   case oop_copy_check_null:
420     throw_array_null_pointer_store_exception(src_obj, dst_obj, JavaThread::current());
421     break;
422   default:
423     ShouldNotReachHere();
424     return;
425   }
426 }
427 
428 class ZStoreBarrierOopClosure : public BasicOopIterateClosure {
429 public:
430   virtual void do_oop(oop* p_) {
431     volatile zpointer* const p = (volatile zpointer*)p_;
432     const zpointer ptr = ZBarrier::load_atomic(p);
433     const zaddress addr = ZPointer::uncolor(ptr);
434     ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
435     *p = ZAddress::store_good(addr);
436   }
437 
438   virtual void do_oop(narrowOop* p) {
439     ShouldNotReachHere();
440   }
441 };
442 
443 class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
444 public:
445   virtual void do_oop(oop* p) {

452 };
453 
454 template <DecoratorSet decorators, typename BarrierSetT>
455 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
456   assert_is_valid(to_zaddress(src));
457 
458   // Fix the oops
459   ZLoadBarrierOopClosure cl;
460   ZIterator::oop_iterate(src, &cl);
461 
462   // Clone the object
463   Raw::clone_in_heap(src, dst, size);
464 
465   assert(ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
466 
467   // Color store good before handing out
468   ZStoreBarrierOopClosure cl_sg;
469   ZIterator::oop_iterate(dst, &cl_sg);
470 }
471 
472 template <DecoratorSet decorators, typename BarrierSetT>
473 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(void* src, void* dst, InlineKlass* md) {
474   if (md->contains_oops()) {
475     // src/dst aren't oops, need offset to adjust oop map offset
476     const address src_oop_addr_offset = ((address) src) - md->first_field_offset();
477 
478     OopMapBlock* map = md->start_of_nonstatic_oop_maps();
479     const OopMapBlock* const end = map + md->nonstatic_oop_map_count();
480     while (map != end) {
481       const address soop_address = src_oop_addr_offset + map->offset();
482       zpointer *p = (zpointer*) soop_address;
483       for (const zpointer* const end = p + map->count(); p < end; p++) {
484         ZBarrier::load_barrier_on_oop_field(p);
485       }
486       map++;
487     }
488   }
489   Raw::value_copy_in_heap(src, dst, md);
490 }
491 
492 //
493 // Not in heap
494 //
495 template <DecoratorSet decorators, typename BarrierSetT>
496 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
497   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
498 
499   const zpointer o = Raw::template load<zpointer>(p);
500   assert_is_valid(o);
501   return to_oop(load_barrier(p, o));
502 }
503 
504 template <DecoratorSet decorators, typename BarrierSetT>
505 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
506   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
507 
508   if (HasDecorator<decorators, IN_NMETHOD>::value) {
509     return ZNMethod::load_oop(p, decorators);
510   } else {
511     return oop_load_not_in_heap((zpointer*)p);
< prev index next >