490 // Hot code path, called from compiler/runtime. Make sure fast path is fast.
491
492 // Fix up src before doing the copy, if needed.
493 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
494 if (gc_state != 0 && ShenandoahCloneBarrier) {
495 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
496 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
497 bs->clone_evacuation(src);
498 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
499 bs->clone_update(src);
500 }
501 }
502
503 Raw::clone(src, dst, size);
504
505 // Current allocator never allocates in old, so clone destination is guaranteed to be in young.
506 // Otherwise we need card barriers.
507 shenandoah_assert_in_young_if(nullptr, dst, ShenandoahCardBarrier);
508 }
509
510 template <DecoratorSet decorators, typename BarrierSetT>
511 template <typename T>
512 OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
513 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
514 size_t length) {
515 T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
516 T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
517
518 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
519 bs->arraycopy_barrier(src, dst, length);
520 OopCopyResult result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
521 if (ShenandoahCardBarrier) {
522 bs->write_ref_array((HeapWord*) dst, length);
523 }
524 return result;
525 }
526
527 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
528 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
529 // Young cycles are allowed to run when old marking is in progress. When old marking is in progress,
530 // this barrier will be called with ENQUEUE=true and HAS_FWD=false, even though the young generation
531 // may have forwarded objects.
532 assert(HAS_FWD == _heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded object status is sane");
533 // This function cannot be called to handle marking and evacuation at the same time (they operate on
534 // different sides of the copy).
535 static_assert((HAS_FWD || EVAC) != ENQUEUE, "Cannot evacuate and mark both sides of copy.");
536
537 Thread* thread = Thread::current();
538 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
539 ShenandoahMarkingContext* ctx = _heap->marking_context();
542 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) {
543 T o = RawAccess<>::oop_load(elem_ptr);
544 if (!CompressedOops::is_null(o)) {
545 oop obj = CompressedOops::decode_not_null(o);
546 if (HAS_FWD && cset->is_in(obj)) {
547 oop fwd = resolve_forwarded_not_null(obj);
548 if (EVAC && obj == fwd) {
549 fwd = _heap->evacuate_object(obj, thread);
550 }
551 shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc());
552 ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o);
553 }
554 if (ENQUEUE && !ctx->is_marked_strong(obj)) {
555 _satb_mark_queue_set.enqueue_known_active(queue, obj);
556 }
557 }
558 }
559 }
560
561 template <class T>
562 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
563 if (count == 0) {
564 // No elements to copy, no need for barrier
565 return;
566 }
567
568 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
569 if ((gc_state & ShenandoahHeap::MARKING) != 0) {
570 // If marking old or young, we must evaluate the SATB barrier. This will be the only
571 // action if we are not marking old. If we are marking old, we must still evaluate the
572 // load reference barrier for a young collection.
573 if (_heap->mode()->is_generational()) {
574 arraycopy_marking<true>(dst, count);
575 } else {
576 arraycopy_marking<false>(dst, count);
577 }
578 }
579
580 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
581 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during evacuation");
582 arraycopy_evacuation(src, count);
583 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
584 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during update-refs");
585 arraycopy_update(src, count);
586 }
587 }
588
589 template <bool IS_GENERATIONAL, class T>
|
490 // Hot code path, called from compiler/runtime. Make sure fast path is fast.
491
492 // Fix up src before doing the copy, if needed.
493 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
494 if (gc_state != 0 && ShenandoahCloneBarrier) {
495 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
496 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
497 bs->clone_evacuation(src);
498 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
499 bs->clone_update(src);
500 }
501 }
502
503 Raw::clone(src, dst, size);
504
505 // Current allocator never allocates in old, so clone destination is guaranteed to be in young.
506 // Otherwise we need card barriers.
507 shenandoah_assert_in_young_if(nullptr, dst, ShenandoahCardBarrier);
508 }
509
510 template <DecoratorSet decorators, typename BarrierSetT>
511 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(const ValuePayload& src, const ValuePayload& dst) {
512 precond(src.klass() == dst.klass());
513
514 const InlineKlass* md = src.klass();
515 if (!md->contains_oops()) {
516 // If we do not have oops in the flat array, we can just do a raw copy.
517 Raw::value_copy(src, dst);
518 } else {
519 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
520 // addr() points at the payload start, the oop map offset are relative to
521 // the object header, adjust address to account for this discrepancy.
522 const address oop_map_adjusted_src_addr = src.addr() - md->payload_offset();
523 const address oop_map_adjusted_dst_addr = dst.addr() - md->payload_offset();
524 typedef typename ValueOopType<decorators>::type OopType;
525
526 // Oop maps tell us where the array-like structures are in the value payload.
527 // For simplicity, apply arraycopy barriers over them.
528 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
529 {
530 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
531 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
532
533 while (map != end) {
534 address src_oop_address = oop_map_adjusted_src_addr + map->offset();
535 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
536 bs->arraycopy_barrier((OopType*) src_oop_address, (OopType*) dst_oop_address, map->count(), dest_uninit);
537 map++;
538 }
539 }
540
541 Raw::value_copy(src, dst);
542
543 // Similarly, we can ask for ref array store helper for post-barriers.
544 if (ShenandoahCardBarrier) {
545 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
546 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
547 while (map != end) {
548 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
549 bs->write_ref_array((HeapWord*) dst_oop_address, map->count());
550 map++;
551 }
552 }
553 }
554 }
555
556 template <DecoratorSet decorators, typename BarrierSetT>
557 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_store_null_in_heap(const ValuePayload& dst) {
558 const InlineKlass* md = dst.klass();
559 if (!md->contains_oops()) {
560 // If we do not have oops in the flat array, we can just do a raw clear.
561 Raw::value_store_null(dst);
562 } else {
563 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
564 // addr() points at the payload start, the oop map offset are relative to
565 // the object header, adjust address to account for this discrepancy.
566 const address oop_map_adjusted_dst_addr = dst.addr() - md->payload_offset();
567 typedef typename ValueOopType<decorators>::type OopType;
568
569 // Oop maps tell us where the array-like structures are in the value payload.
570 // For simplicity, apply arraycopy barriers over them.
571 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
572 if (!dest_uninit) {
573 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
574 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
575 while (map != end) {
576 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
577 bs->arraycopy_barrier((OopType*) dst_oop_address, (OopType*) dst_oop_address, map->count(), dest_uninit);
578 map++;
579 }
580 }
581
582 Raw::value_store_null(dst);
583
584 // Storing null does not require post-barriers
585 }
586 }
587
588 template <DecoratorSet decorators, typename BarrierSetT>
589 template <typename T>
590 OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
591 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
592 size_t length) {
593 T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
594 T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
595 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
596
597 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
598 bs->arraycopy_barrier(src, dst, length, dest_uninit);
599 OopCopyResult result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
600 if (ShenandoahCardBarrier) {
601 bs->write_ref_array((HeapWord*) dst, length);
602 }
603 return result;
604 }
605
606 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
607 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
608 // Young cycles are allowed to run when old marking is in progress. When old marking is in progress,
609 // this barrier will be called with ENQUEUE=true and HAS_FWD=false, even though the young generation
610 // may have forwarded objects.
611 assert(HAS_FWD == _heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded object status is sane");
612 // This function cannot be called to handle marking and evacuation at the same time (they operate on
613 // different sides of the copy).
614 static_assert((HAS_FWD || EVAC) != ENQUEUE, "Cannot evacuate and mark both sides of copy.");
615
616 Thread* thread = Thread::current();
617 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
618 ShenandoahMarkingContext* ctx = _heap->marking_context();
621 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) {
622 T o = RawAccess<>::oop_load(elem_ptr);
623 if (!CompressedOops::is_null(o)) {
624 oop obj = CompressedOops::decode_not_null(o);
625 if (HAS_FWD && cset->is_in(obj)) {
626 oop fwd = resolve_forwarded_not_null(obj);
627 if (EVAC && obj == fwd) {
628 fwd = _heap->evacuate_object(obj, thread);
629 }
630 shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc());
631 ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o);
632 }
633 if (ENQUEUE && !ctx->is_marked_strong(obj)) {
634 _satb_mark_queue_set.enqueue_known_active(queue, obj);
635 }
636 }
637 }
638 }
639
640 template <class T>
641 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count, bool dest_uninit) {
642 if (count == 0) {
643 // No elements to copy, no need for barrier
644 return;
645 }
646
647 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
648 if (!dest_uninit && (gc_state & ShenandoahHeap::MARKING) != 0) {
649 // If marking old or young, we must evaluate the SATB barrier. This will be the only
650 // action if we are not marking old. If we are marking old, we must still evaluate the
651 // load reference barrier for a young collection.
652 if (_heap->mode()->is_generational()) {
653 arraycopy_marking<true>(dst, count);
654 } else {
655 arraycopy_marking<false>(dst, count);
656 }
657 }
658
659 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
660 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during evacuation");
661 arraycopy_evacuation(src, count);
662 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
663 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during update-refs");
664 arraycopy_update(src, count);
665 }
666 }
667
668 template <bool IS_GENERATIONAL, class T>
|