475 // Hot code path, called from compiler/runtime. Make sure fast path is fast.
476
477 // Fix up src before doing the copy, if needed.
478 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
479 if (gc_state != 0 && ShenandoahCloneBarrier) {
480 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
481 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
482 bs->clone_work<true>(src);
483 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
484 bs->clone_work<false>(src);
485 }
486 }
487
488 Raw::clone(src, dst, size);
489
490 // Current allocator never allocates in old, so clone destination is guaranteed to be in young.
491 // Otherwise we need card barriers.
492 shenandoah_assert_in_young_if(nullptr, dst, ShenandoahCardBarrier);
493 }
494
495 template <DecoratorSet decorators, typename BarrierSetT>
496 template <typename T>
497 OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
498 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
499 size_t length) {
500 T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
501 T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
502
503 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
504 bs->arraycopy_barrier(src, dst, length);
505 OopCopyResult result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
506 if (ShenandoahCardBarrier) {
507 bs->write_ref_array((HeapWord*) dst, length);
508 }
509 return result;
510 }
511
512 template <class T>
513 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
514 if (count == 0) {
515 // No elements to copy, no need for barrier
516 return;
517 }
518
519 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
520 if ((gc_state & ShenandoahHeap::MARKING) != 0) {
521 // If marking old or young, we must evaluate the SATB barrier. This will be the only
522 // action if we are not marking old. If we are marking old, we must still evaluate the
523 // load reference barrier for a young collection.
524 if (_heap->mode()->is_generational()) {
525 arraycopy_marking<true>(dst, count);
526 } else {
527 arraycopy_marking<false>(dst, count);
528 }
529 }
530
531 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
532 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during evacuation");
533 arraycopy_evacuation(src, count);
534 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
535 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during update-refs");
536 arraycopy_update(src, count);
537 }
538 }
539
540 template <bool IS_GENERATIONAL, class T>
|
475 // Hot code path, called from compiler/runtime. Make sure fast path is fast.
476
477 // Fix up src before doing the copy, if needed.
478 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
479 if (gc_state != 0 && ShenandoahCloneBarrier) {
480 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
481 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
482 bs->clone_work<true>(src);
483 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
484 bs->clone_work<false>(src);
485 }
486 }
487
488 Raw::clone(src, dst, size);
489
490 // Current allocator never allocates in old, so clone destination is guaranteed to be in young.
491 // Otherwise we need card barriers.
492 shenandoah_assert_in_young_if(nullptr, dst, ShenandoahCardBarrier);
493 }
494
495 template <DecoratorSet decorators, typename BarrierSetT>
496 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(const ValuePayload& src, const ValuePayload& dst) {
497 precond(src.klass() == dst.klass());
498
499 const InlineKlass* md = src.klass();
500 if (!md->contains_oops()) {
501 // If we do not have oops in the flat array, we can just do a raw copy.
502 Raw::value_copy(src, dst);
503 } else {
504 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
505 // addr() points at the payload start, the oop map offset are relative to
506 // the object header, adjust address to account for this discrepancy.
507 const address oop_map_adjusted_src_addr = src.addr() - md->payload_offset();
508 const address oop_map_adjusted_dst_addr = dst.addr() - md->payload_offset();
509 typedef typename ValueOopType<decorators>::type OopType;
510
511 // Oop maps tell us where the array-like structures are in the value payload.
512 // For simplicity, apply arraycopy barriers over them.
513 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
514 {
515 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
516 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
517
518 while (map != end) {
519 address src_oop_address = oop_map_adjusted_src_addr + map->offset();
520 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
521 bs->arraycopy_barrier((OopType*) src_oop_address, (OopType*) dst_oop_address, map->count(), dest_uninit);
522 map++;
523 }
524 }
525
526 Raw::value_copy(src, dst);
527
528 // Similarly, we can ask for ref array store helper for post-barriers.
529 if (ShenandoahCardBarrier) {
530 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
531 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
532 while (map != end) {
533 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
534 bs->write_ref_array((HeapWord*) dst_oop_address, map->count());
535 map++;
536 }
537 }
538 }
539 }
540
541 template <DecoratorSet decorators, typename BarrierSetT>
542 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_store_null_in_heap(const ValuePayload& dst) {
543 const InlineKlass* md = dst.klass();
544 if (!md->contains_oops()) {
545 // If we do not have oops in the flat array, we can just do a raw clear.
546 Raw::value_store_null(dst);
547 } else {
548 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
549 // addr() points at the payload start, the oop map offset are relative to
550 // the object header, adjust address to account for this discrepancy.
551 const address oop_map_adjusted_dst_addr = dst.addr() - md->payload_offset();
552 typedef typename ValueOopType<decorators>::type OopType;
553
554 // Oop maps tell us where the array-like structures are in the value payload.
555 // For simplicity, apply arraycopy barriers over them.
556 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
557 if (!dest_uninit) {
558 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
559 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
560 while (map != end) {
561 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
562 bs->arraycopy_barrier((OopType*) dst_oop_address, (OopType*) dst_oop_address, map->count(), dest_uninit);
563 map++;
564 }
565 }
566
567 Raw::value_store_null(dst);
568
569 // Storing null does not require post-barriers
570 }
571 }
572
573 template <DecoratorSet decorators, typename BarrierSetT>
574 template <typename T>
575 OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
576 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
577 size_t length) {
578 T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
579 T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
580 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
581
582 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
583 bs->arraycopy_barrier(src, dst, length, dest_uninit);
584 OopCopyResult result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
585 if (ShenandoahCardBarrier) {
586 bs->write_ref_array((HeapWord*) dst, length);
587 }
588 return result;
589 }
590
591 template <class T>
592 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count, bool dest_uninit) {
593 if (count == 0) {
594 // No elements to copy, no need for barrier
595 return;
596 }
597
598 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
599 if (!dest_uninit && (gc_state & ShenandoahHeap::MARKING) != 0) {
600 // If marking old or young, we must evaluate the SATB barrier. This will be the only
601 // action if we are not marking old. If we are marking old, we must still evaluate the
602 // load reference barrier for a young collection.
603 if (_heap->mode()->is_generational()) {
604 arraycopy_marking<true>(dst, count);
605 } else {
606 arraycopy_marking<false>(dst, count);
607 }
608 }
609
610 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
611 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during evacuation");
612 arraycopy_evacuation(src, count);
613 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
614 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during update-refs");
615 arraycopy_update(src, count);
616 }
617 }
618
619 template <bool IS_GENERATIONAL, class T>
|