13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #include "opto/output.hpp"
47 #endif
48
49 #define __ masm->
50
51 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
52 Register src, Register dst, Register count, RegSet saved_regs) {
53 if (is_oop) {
54 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
55 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
56
631 __ ld(tmp1, curr_ct_holder_addr);
632
633 // tmp1 = effective address
634 __ la(tmp2, address);
635
636 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
637 __ srli(tmp2, tmp2, CardTable::card_shift());
638 __ add(tmp2, tmp2, tmp1);
639
640 if (UseCondCardMark) {
641 Label L_already_dirty;
642 __ lbu(tmp1, Address(tmp2));
643 __ beqz(tmp1, L_already_dirty);
644 __ sb(zr, Address(tmp2));
645 __ bind(L_already_dirty);
646 } else {
647 __ sb(zr, Address(tmp2));
648 }
649 }
650
651 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
652 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
653
654 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
655 __ lbu(tmp, gc_state_fast);
656 __ beqz(tmp, *continuation());
657 __ j(*entry());
658
659 // This is were the slowpath stub will return to or the code above will
660 // jump to if the checks are false
661 __ bind(*continuation());
662 }
663
664 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
665 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
666 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
667
668 __ bind(*entry());
669
670 // If we need to load ourselves, do it here.
671 if (_do_load) {
672 if (_narrow) {
673 __ lwu(_obj, _addr);
674 } else {
675 __ ld(_obj, _addr);
676 }
677 }
678
679 // If the object is null, there is no point in applying barriers.
680 maybe_far_jump_if_zero(masm, _obj);
681
695 keepalive(masm, continuation());
696 } else if (_needs_load_ref_barrier) {
697 lrb(masm);
698 } else {
699 ShouldNotReachHere();
700 }
701 }
702
703 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
704 Label L_short_jump;
705 __ bnez(reg, L_short_jump);
706 __ j(*continuation());
707 __ bind(L_short_jump);
708 }
709
710 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
711 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
712 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
713 Label L_through, L_slowpath;
714
715 // If another barrier is enabled as well, do a runtime check for a specific barrier.
716 if (_needs_load_ref_barrier) {
717 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
718 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
719 __ lbu(_tmp1, gc_state_fast);
720 __ beqz(_tmp1, L_through);
721 }
722
723 // Fast-path: put object into buffer.
724 // If buffer is already full, go slow.
725 __ ld(_tmp1, index);
726 __ beqz(_tmp1, L_slowpath);
727 __ subi(_tmp1, _tmp1, wordSize);
728 __ sd(_tmp1, index);
729 __ ld(_tmp2, buffer);
730
731 // Store the object in queue.
732 // If object is narrow, we need to decode it before inserting.
733 __ add(_tmp1, _tmp1, _tmp2);
734 if (_narrow) {
735 __ decode_heap_oop_not_null(_tmp2, _obj);
736 __ sd(_tmp2, Address(_tmp1));
737 } else {
738 __ sd(_obj, Address(_tmp1));
739 }
740
749 __ bind(L_slowpath);
750
751 {
752 SaveLiveRegisters slr(&masm, this);
753
754 // Go to runtime and handle the rest there.
755 __ mv(c_rarg0, _obj);
756 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
757 __ jalr(ra);
758 }
759 if (L_done != nullptr) {
760 __ j(*L_done);
761 } else {
762 __ bind(L_through);
763 }
764 }
765
766 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
767 Label L_slow;
768
769 // If another barrier is enabled as well, do a runtime check for a specific barrier.
770 if (_needs_keep_alive_barrier) {
771 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
772 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
773 __ lbu(_tmp1, gc_state_fast);
774 maybe_far_jump_if_zero(masm, _tmp1);
775 }
776
777 // If weak references are being processed, weak/phantom loads need to go slow,
778 // regardless of their cset status.
779 if (_needs_load_ref_weak_barrier) {
780 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
781 __ lbu(_tmp1, gc_state_fast);
782 __ bnez(_tmp1, L_slow);
783 }
784
785 // Cset-check. Fall-through to slow if in collection set.
786 if (_narrow) {
787 __ decode_heap_oop_not_null(_tmp2, _obj);
788 } else {
789 __ mv(_tmp2, _obj);
790 }
791
792 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
793 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
794 __ add(_tmp1, _tmp1, _tmp2);
795 __ lbu(_tmp1, Address(_tmp1, 0));
796 maybe_far_jump_if_zero(masm, _tmp1);
797
798 // Slow path
799 __ bind(L_slow);
800
801 // Obj is the result, need to temporarily stop preserving it.
802 bool is_obj_preserved = is_preserved(_obj);
|
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "nativeInst_riscv.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
54 Register src, Register dst, Register count, RegSet saved_regs) {
55 if (is_oop) {
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
58
633 __ ld(tmp1, curr_ct_holder_addr);
634
635 // tmp1 = effective address
636 __ la(tmp2, address);
637
638 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
639 __ srli(tmp2, tmp2, CardTable::card_shift());
640 __ add(tmp2, tmp2, tmp1);
641
642 if (UseCondCardMark) {
643 Label L_already_dirty;
644 __ lbu(tmp1, Address(tmp2));
645 __ beqz(tmp1, L_already_dirty);
646 __ sb(zr, Address(tmp2));
647 __ bind(L_already_dirty);
648 } else {
649 __ sb(zr, Address(tmp2));
650 }
651 }
652
653 void ShenandoahBarrierStubC2::patchable_jump_if_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
654 // Emit the unconditional branch in the first version of the method.
655 // Let the rest of runtime figure out how to manage it.
656 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, false)));
657 __ j(*L_target);
658 }
659
660 void ShenandoahBarrierStubC2::patchable_jump_if_not_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
661 // Emit the unconditional branch in the first version of the method.
662 // Let the rest of runtime figure out how to manage it.
663 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, true)));
664 __ j(*L_target);
665 }
666
667 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
668 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
669 patchable_jump_if_gc_state(masm, test_state, entry());
670 __ bind(*continuation());
671 }
672
673 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
674 NativeInstruction* ni = nativeInstruction_at(pc);
675 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
676 NativeJump* jmp = nativeJump_at(pc);
677 return jmp->jump_destination();
678 }
679
680 static bool is_patchable_nop(address pc) {
681 if (*(pc + 0) != 0x13) return false;
682 if (*(pc + 1) != 0x00) return false;
683 if (*(pc + 2) != 0x00) return false;
684 if (*(pc + 3) != 0x00) return false;
685 return true;
686 }
687
688 static void insert_patchable_nop(address pc) {
689 *reinterpret_cast<int32_t*>(pc) = 0x00000013;
690 }
691
692 static void check_at(bool cond, address pc, const char* msg) {
693 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x",
694 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3));
695 }
696
697 bool ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
698 NativeInstruction* ni = nativeInstruction_at(pc);
699 bool patching = ni->is_jump();
700 if (patching) {
701 insert_patchable_nop(pc);
702 }
703
704 check_at(is_patchable_nop(pc), pc, "Should be nop");
705 return patching;
706 }
707
708 bool ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
709 bool patching = is_patchable_nop(pc);
710 if (patching) {
711 NativeJump::insert(pc, stub_addr);
712 }
713
714 NativeInstruction* ni = nativeInstruction_at(pc);
715 check_at(ni->is_jump(), pc, "Should already be jump");
716 check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
717 return patching;
718 }
719
720 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
721 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
722 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
723
724 __ bind(*entry());
725
726 // If we need to load ourselves, do it here.
727 if (_do_load) {
728 if (_narrow) {
729 __ lwu(_obj, _addr);
730 } else {
731 __ ld(_obj, _addr);
732 }
733 }
734
735 // If the object is null, there is no point in applying barriers.
736 maybe_far_jump_if_zero(masm, _obj);
737
751 keepalive(masm, continuation());
752 } else if (_needs_load_ref_barrier) {
753 lrb(masm);
754 } else {
755 ShouldNotReachHere();
756 }
757 }
758
759 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
760 Label L_short_jump;
761 __ bnez(reg, L_short_jump);
762 __ j(*continuation());
763 __ bind(L_short_jump);
764 }
765
766 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
767 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
768 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
769 Label L_through, L_slowpath;
770
771 // If another barrier is enabled as well, do a check for a specific barrier.
772 if (_needs_load_ref_barrier) {
773 assert(L_done == nullptr, "Should be");
774 char state_to_check = ShenandoahHeap::MARKING;
775 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
776 }
777
778 // Fast-path: put object into buffer.
779 // If buffer is already full, go slow.
780 __ ld(_tmp1, index);
781 __ beqz(_tmp1, L_slowpath);
782 __ subi(_tmp1, _tmp1, wordSize);
783 __ sd(_tmp1, index);
784 __ ld(_tmp2, buffer);
785
786 // Store the object in queue.
787 // If object is narrow, we need to decode it before inserting.
788 __ add(_tmp1, _tmp1, _tmp2);
789 if (_narrow) {
790 __ decode_heap_oop_not_null(_tmp2, _obj);
791 __ sd(_tmp2, Address(_tmp1));
792 } else {
793 __ sd(_obj, Address(_tmp1));
794 }
795
804 __ bind(L_slowpath);
805
806 {
807 SaveLiveRegisters slr(&masm, this);
808
809 // Go to runtime and handle the rest there.
810 __ mv(c_rarg0, _obj);
811 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
812 __ jalr(ra);
813 }
814 if (L_done != nullptr) {
815 __ j(*L_done);
816 } else {
817 __ bind(L_through);
818 }
819 }
820
821 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
822 Label L_slow;
823
824 // If another barrier is enabled as well, do a check for a specific barrier.
825 if (_needs_keep_alive_barrier) {
826 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
827 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
828 }
829
830 // If weak references are being processed, weak/phantom loads need to go slow,
831 // regardless of their cset status.
832 if (_needs_load_ref_weak_barrier) {
833 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
834 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
835 }
836
837 // Cset-check. Fall-through to slow if in collection set.
838 if (_narrow) {
839 __ decode_heap_oop_not_null(_tmp2, _obj);
840 } else {
841 __ mv(_tmp2, _obj);
842 }
843
844 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
845 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
846 __ add(_tmp1, _tmp1, _tmp2);
847 __ lbu(_tmp1, Address(_tmp1, 0));
848 maybe_far_jump_if_zero(masm, _tmp1);
849
850 // Slow path
851 __ bind(L_slow);
852
853 // Obj is the result, need to temporarily stop preserving it.
854 bool is_obj_preserved = is_preserved(_obj);
|