470 // Hot code path, called from compiler/runtime. Make sure fast path is fast.
471
472 // Fix up src before doing the copy, if needed.
473 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
474 if (gc_state != 0 && ShenandoahCloneBarrier) {
475 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
476 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
477 bs->clone_work<true>(src);
478 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
479 bs->clone_work<false>(src);
480 }
481 }
482
483 Raw::clone(src, dst, size);
484
485 // Current allocator never allocates in old, so clone destination is guaranteed to be in young.
486 // Otherwise we need card barriers.
487 shenandoah_assert_in_young_if(nullptr, dst, ShenandoahCardBarrier);
488 }
489
490 template <DecoratorSet decorators, typename BarrierSetT>
491 template <typename T>
492 OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
493 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
494 size_t length) {
495 T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
496 T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
497
498 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
499 bs->arraycopy_barrier(src, dst, length);
500 OopCopyResult result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
501 if (ShenandoahCardBarrier) {
502 bs->write_ref_array((HeapWord*) dst, length);
503 }
504 return result;
505 }
506
507 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
508 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
509 // Young cycles are allowed to run when old marking is in progress. When old marking is in progress,
510 // this barrier will be called with ENQUEUE=true and HAS_FWD=false, even though the young generation
511 // may have forwarded objects.
512 assert(HAS_FWD == _heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded object status is sane");
513 // This function cannot be called to handle marking and evacuation at the same time (they operate on
514 // different sides of the copy).
515 static_assert((HAS_FWD || EVAC) != ENQUEUE, "Cannot evacuate and mark both sides of copy.");
516
517 Thread* thread = Thread::current();
518 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
519 ShenandoahMarkingContext* ctx = _heap->marking_context();
522 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) {
523 T o = RawAccess<>::oop_load(elem_ptr);
524 if (!CompressedOops::is_null(o)) {
525 oop obj = CompressedOops::decode_not_null(o);
526 if (HAS_FWD && cset->is_in(obj)) {
527 oop fwd = ShenandoahForwarding::get_forwardee(obj);
528 if (EVAC && obj == fwd) {
529 fwd = _heap->evacuate_object(obj, thread);
530 }
531 shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc());
532 ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o);
533 }
534 if (ENQUEUE && !ctx->is_marked_strong(obj)) {
535 _satb_mark_queue_set.enqueue_known_active(queue, obj);
536 }
537 }
538 }
539 }
540
541 template <class T>
542 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
543 if (count == 0) {
544 // No elements to copy, no need for barrier
545 return;
546 }
547
548 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
549 if ((gc_state & ShenandoahHeap::MARKING) != 0) {
550 // If marking old or young, we must evaluate the SATB barrier. This will be the only
551 // action if we are not marking old. If we are marking old, we must still evaluate the
552 // load reference barrier for a young collection.
553 if (_heap->mode()->is_generational()) {
554 arraycopy_marking<true>(dst, count);
555 } else {
556 arraycopy_marking<false>(dst, count);
557 }
558 }
559
560 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
561 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during evacuation");
562 arraycopy_evacuation(src, count);
563 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
564 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during update-refs");
565 arraycopy_update(src, count);
566 }
567 }
568
569 template <bool IS_GENERATIONAL, class T>
|
470 // Hot code path, called from compiler/runtime. Make sure fast path is fast.
471
472 // Fix up src before doing the copy, if needed.
473 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
474 if (gc_state != 0 && ShenandoahCloneBarrier) {
475 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
476 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
477 bs->clone_work<true>(src);
478 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
479 bs->clone_work<false>(src);
480 }
481 }
482
483 Raw::clone(src, dst, size);
484
485 // Current allocator never allocates in old, so clone destination is guaranteed to be in young.
486 // Otherwise we need card barriers.
487 shenandoah_assert_in_young_if(nullptr, dst, ShenandoahCardBarrier);
488 }
489
490 template <DecoratorSet decorators, typename BarrierSetT>
491 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(const ValuePayload& src, const ValuePayload& dst) {
492 precond(src.klass() == dst.klass());
493
494 const InlineKlass* md = src.klass();
495 if (!md->contains_oops()) {
496 // If we do not have oops in the flat array, we can just do a raw copy.
497 Raw::value_copy(src, dst);
498 } else {
499 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
500 // addr() points at the payload start, the oop map offset are relative to
501 // the object header, adjust address to account for this discrepancy.
502 const address oop_map_adjusted_src_addr = src.addr() - md->payload_offset();
503 const address oop_map_adjusted_dst_addr = dst.addr() - md->payload_offset();
504 typedef typename ValueOopType<decorators>::type OopType;
505
506 // Oop maps tell us where the array-like structures are in the value payload.
507 // For simplicity, apply arraycopy barriers over them.
508 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
509 {
510 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
511 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
512
513 while (map != end) {
514 address src_oop_address = oop_map_adjusted_src_addr + map->offset();
515 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
516 bs->arraycopy_barrier((OopType*) src_oop_address, (OopType*) dst_oop_address, map->count(), dest_uninit);
517 map++;
518 }
519 }
520
521 Raw::value_copy(src, dst);
522
523 // Similarly, we can ask for ref array store helper for post-barriers.
524 if (ShenandoahCardBarrier) {
525 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
526 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
527 while (map != end) {
528 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
529 bs->write_ref_array((HeapWord*) dst_oop_address, map->count());
530 map++;
531 }
532 }
533 }
534 }
535
536 template <DecoratorSet decorators, typename BarrierSetT>
537 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_store_null_in_heap(const ValuePayload& dst) {
538 const InlineKlass* md = dst.klass();
539 if (!md->contains_oops()) {
540 // If we do not have oops in the flat array, we can just do a raw clear.
541 Raw::value_store_null(dst);
542 } else {
543 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
544 // addr() points at the payload start, the oop map offset are relative to
545 // the object header, adjust address to account for this discrepancy.
546 const address oop_map_adjusted_dst_addr = dst.addr() - md->payload_offset();
547 typedef typename ValueOopType<decorators>::type OopType;
548
549 // Oop maps tell us where the array-like structures are in the value payload.
550 // For simplicity, apply arraycopy barriers over them.
551 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
552 if (!dest_uninit) {
553 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
554 OopMapBlock* const end = map + md->nonstatic_oop_map_count();
555 while (map != end) {
556 address dst_oop_address = oop_map_adjusted_dst_addr + map->offset();
557 bs->arraycopy_barrier((OopType*) dst_oop_address, (OopType*) dst_oop_address, map->count(), dest_uninit);
558 map++;
559 }
560 }
561
562 Raw::value_store_null(dst);
563
564 // Storing null does not require post-barriers
565 }
566 }
567
568 template <DecoratorSet decorators, typename BarrierSetT>
569 template <typename T>
570 OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
571 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
572 size_t length) {
573 T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
574 T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
575 bool dest_uninit = HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value;
576
577 ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
578 bs->arraycopy_barrier(src, dst, length, dest_uninit);
579 OopCopyResult result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
580 if (ShenandoahCardBarrier) {
581 bs->write_ref_array((HeapWord*) dst, length);
582 }
583 return result;
584 }
585
586 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
587 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
588 // Young cycles are allowed to run when old marking is in progress. When old marking is in progress,
589 // this barrier will be called with ENQUEUE=true and HAS_FWD=false, even though the young generation
590 // may have forwarded objects.
591 assert(HAS_FWD == _heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded object status is sane");
592 // This function cannot be called to handle marking and evacuation at the same time (they operate on
593 // different sides of the copy).
594 static_assert((HAS_FWD || EVAC) != ENQUEUE, "Cannot evacuate and mark both sides of copy.");
595
596 Thread* thread = Thread::current();
597 SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
598 ShenandoahMarkingContext* ctx = _heap->marking_context();
601 for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) {
602 T o = RawAccess<>::oop_load(elem_ptr);
603 if (!CompressedOops::is_null(o)) {
604 oop obj = CompressedOops::decode_not_null(o);
605 if (HAS_FWD && cset->is_in(obj)) {
606 oop fwd = ShenandoahForwarding::get_forwardee(obj);
607 if (EVAC && obj == fwd) {
608 fwd = _heap->evacuate_object(obj, thread);
609 }
610 shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc());
611 ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o);
612 }
613 if (ENQUEUE && !ctx->is_marked_strong(obj)) {
614 _satb_mark_queue_set.enqueue_known_active(queue, obj);
615 }
616 }
617 }
618 }
619
620 template <class T>
621 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count, bool dest_uninit) {
622 if (count == 0) {
623 // No elements to copy, no need for barrier
624 return;
625 }
626
627 const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
628 if (!dest_uninit && (gc_state & ShenandoahHeap::MARKING) != 0) {
629 // If marking old or young, we must evaluate the SATB barrier. This will be the only
630 // action if we are not marking old. If we are marking old, we must still evaluate the
631 // load reference barrier for a young collection.
632 if (_heap->mode()->is_generational()) {
633 arraycopy_marking<true>(dst, count);
634 } else {
635 arraycopy_marking<false>(dst, count);
636 }
637 }
638
639 if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
640 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during evacuation");
641 arraycopy_evacuation(src, count);
642 } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
643 assert((gc_state & ShenandoahHeap::YOUNG_MARKING) == 0, "Cannot be marking young during update-refs");
644 arraycopy_update(src, count);
645 }
646 }
647
648 template <bool IS_GENERATIONAL, class T>
|