13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #include "opto/output.hpp"
47 #endif
48
49 #define __ masm->
50
51 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
52 Register src, Register dst, Register count, RegSet saved_regs) {
53 if (is_oop) {
54 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
55 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
56
734 __ la(tmp2, address);
735
736 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
737 __ srli(tmp2, tmp2, CardTable::card_shift());
738 __ add(tmp2, tmp2, tmp1);
739
740 if (UseCondCardMark) {
741 Label L_already_dirty;
742 __ lbu(tmp1, Address(tmp2));
743 __ beqz(tmp1, L_already_dirty);
744 __ sb(zr, Address(tmp2));
745 __ bind(L_already_dirty);
746 } else {
747 __ sb(zr, Address(tmp2));
748 }
749 }
750
751 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
752 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
753
754 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
755 __ lbu(tmp, gc_state_fast);
756 __ beqz(tmp, *continuation());
757 __ j(*entry());
758
759 // This is were the slowpath stub will return to or the code above will
760 // jump to if the checks are false
761 __ bind(*continuation());
762 }
763
764 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
765 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
766 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
767
768 __ bind(*entry());
769
770 // If we need to load ourselves, do it here.
771 if (_do_load) {
772 if (_narrow) {
773 __ lwu(_obj, _addr);
774 } else {
775 __ ld(_obj, _addr);
776 }
777 }
778
779 // If the object is null, there is no point in applying barriers.
780 maybe_far_jump_if_zero(masm, _obj);
781
782 // We need to make sure that loads done by callers survive across slow-path calls.
783 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
795 keepalive(masm, continuation());
796 } else if (_needs_load_ref_barrier) {
797 lrb(masm);
798 } else {
799 ShouldNotReachHere();
800 }
801 }
802
803 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
804 Label L_short_jump;
805 __ bnez(reg, L_short_jump);
806 __ j(*continuation());
807 __ bind(L_short_jump);
808 }
809
810 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
811 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
812 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
813 Label L_through, L_slowpath;
814
815 // If another barrier is enabled as well, do a runtime check for a specific barrier.
816 if (_needs_load_ref_barrier) {
817 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
818 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
819 __ lbu(_tmp1, gc_state_fast);
820 __ beqz(_tmp1, L_through);
821 }
822
823 // Fast-path: put object into buffer.
824 // If buffer is already full, go slow.
825 __ ld(_tmp1, index);
826 __ beqz(_tmp1, L_slowpath);
827 __ subi(_tmp1, _tmp1, wordSize);
828 __ sd(_tmp1, index);
829 __ ld(_tmp2, buffer);
830
831 // Store the object in queue.
832 // If object is narrow, we need to decode it before inserting.
833 __ add(_tmp1, _tmp1, _tmp2);
834 if (_narrow) {
835 __ decode_heap_oop_not_null(_tmp2, _obj);
836 __ sd(_tmp2, Address(_tmp1));
837 } else {
838 __ sd(_obj, Address(_tmp1));
839 }
840
849 __ bind(L_slowpath);
850
851 {
852 SaveLiveRegisters slr(&masm, this);
853
854 // Go to runtime and handle the rest there.
855 __ mv(c_rarg0, _obj);
856 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
857 __ jalr(ra);
858 }
859 if (L_done != nullptr) {
860 __ j(*L_done);
861 } else {
862 __ bind(L_through);
863 }
864 }
865
866 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
867 Label L_slow;
868
869 // If another barrier is enabled as well, do a runtime check for a specific barrier.
870 if (_needs_keep_alive_barrier) {
871 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
872 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
873 __ lbu(_tmp1, gc_state_fast);
874 maybe_far_jump_if_zero(masm, _tmp1);
875 }
876
877 // If weak references are being processed, weak/phantom loads need to go slow,
878 // regardless of their cset status.
879 if (_needs_load_ref_weak_barrier) {
880 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
881 __ lbu(_tmp1, gc_state_fast);
882 __ bnez(_tmp1, L_slow);
883 }
884
885 // Cset-check. Fall-through to slow if in collection set.
886 if (_narrow) {
887 __ decode_heap_oop_not_null(_tmp2, _obj);
888 } else {
889 __ mv(_tmp2, _obj);
890 }
891
892 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
893 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
894 __ add(_tmp1, _tmp1, _tmp2);
895 __ lbu(_tmp1, Address(_tmp1, 0));
896 maybe_far_jump_if_zero(masm, _tmp1);
897
898 // Slow path
899 __ bind(L_slow);
900
901 // Obj is the result, need to temporarily stop preserving it.
902 bool is_obj_preserved = is_preserved(_obj);
|
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "nativeInst_riscv.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
54 Register src, Register dst, Register count, RegSet saved_regs) {
55 if (is_oop) {
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
58
736 __ la(tmp2, address);
737
738 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
739 __ srli(tmp2, tmp2, CardTable::card_shift());
740 __ add(tmp2, tmp2, tmp1);
741
742 if (UseCondCardMark) {
743 Label L_already_dirty;
744 __ lbu(tmp1, Address(tmp2));
745 __ beqz(tmp1, L_already_dirty);
746 __ sb(zr, Address(tmp2));
747 __ bind(L_already_dirty);
748 } else {
749 __ sb(zr, Address(tmp2));
750 }
751 }
752
753 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
754 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
755
756 // Emit the unconditional branch in the first version of the method.
757 // Let the rest of runtime figure out how to manage it.
758 __ relocate(barrier_Relocation::spec(), (int)ShenandoahNMethod::gc_state_to_reloc(test_state));
759 __ j(*entry());
760
761 // This is were the slowpath stub will return to or the code above will
762 // jump to if the checks are false
763 __ bind(*continuation());
764 }
765
766 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
767 NativeInstruction* ni = nativeInstruction_at(pc);
768 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
769 NativeJump* jmp = nativeJump_at(pc);
770 return jmp->jump_destination();
771 }
772
773 static bool is_nop(address pc) {
774 if (*(pc + 0) != 0x13) return false;
775 if (*(pc + 1) != 0x00) return false;
776 if (*(pc + 2) != 0x00) return false;
777 if (*(pc + 3) != 0x00) return false;
778 return true;
779 }
780
781 static void insert_nop(address pc) {
782 *reinterpret_cast<int32_t*>(pc) = 0x00000013;
783 assert(is_nop(pc), "Should be");
784 ICache::invalidate_range(pc, 4);
785 }
786
787 static void check_at(bool cond, address pc, const char* msg) {
788 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x",
789 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3));
790 }
791
792 bool ShenandoahBarrierSetAssembler::is_active(address pc) {
793 NativeInstruction* ni = nativeInstruction_at(pc);
794 return ni->is_jump();
795 }
796
797 void ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
798 NativeInstruction* ni = nativeInstruction_at(pc);
799 if (ni->is_jump()) {
800 insert_nop(pc);
801 } else {
802 check_at(is_nop(pc), pc, "Should already be nop");
803 }
804 }
805
806 void ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
807 NativeInstruction* ni = nativeInstruction_at(pc);
808 if (is_nop(pc)) {
809 NativeJump::insert(pc, stub_addr);
810 } else {
811 check_at(ni->is_jump(), pc, "Should already be jump");
812 check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
813 }
814 }
815
816 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
817 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
818 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
819
820 __ bind(*entry());
821
822 // If we need to load ourselves, do it here.
823 if (_do_load) {
824 if (_narrow) {
825 __ lwu(_obj, _addr);
826 } else {
827 __ ld(_obj, _addr);
828 }
829 }
830
831 // If the object is null, there is no point in applying barriers.
832 maybe_far_jump_if_zero(masm, _obj);
833
834 // We need to make sure that loads done by callers survive across slow-path calls.
835 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
847 keepalive(masm, continuation());
848 } else if (_needs_load_ref_barrier) {
849 lrb(masm);
850 } else {
851 ShouldNotReachHere();
852 }
853 }
854
855 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
856 Label L_short_jump;
857 __ bnez(reg, L_short_jump);
858 __ j(*continuation());
859 __ bind(L_short_jump);
860 }
861
862 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
863 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
864 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
865 Label L_through, L_slowpath;
866
867 // If another barrier is enabled as well, do a check for a specific barrier.
868 if (_needs_load_ref_barrier) {
869 assert(L_done == nullptr, "Should be");
870 // Emit the unconditional branch in the first version of the method.
871 // Let the rest of runtime figure out how to manage it.
872 // TODO: We could have spared the over-jump if patching knew we need the inverse branch.
873 char state_to_check = ShenandoahHeap::MARKING;
874 Label L_over;
875 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
876 __ j(L_over);
877 __ j(L_through);
878 __ bind(L_over);
879 }
880
881 // Fast-path: put object into buffer.
882 // If buffer is already full, go slow.
883 __ ld(_tmp1, index);
884 __ beqz(_tmp1, L_slowpath);
885 __ subi(_tmp1, _tmp1, wordSize);
886 __ sd(_tmp1, index);
887 __ ld(_tmp2, buffer);
888
889 // Store the object in queue.
890 // If object is narrow, we need to decode it before inserting.
891 __ add(_tmp1, _tmp1, _tmp2);
892 if (_narrow) {
893 __ decode_heap_oop_not_null(_tmp2, _obj);
894 __ sd(_tmp2, Address(_tmp1));
895 } else {
896 __ sd(_obj, Address(_tmp1));
897 }
898
907 __ bind(L_slowpath);
908
909 {
910 SaveLiveRegisters slr(&masm, this);
911
912 // Go to runtime and handle the rest there.
913 __ mv(c_rarg0, _obj);
914 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
915 __ jalr(ra);
916 }
917 if (L_done != nullptr) {
918 __ j(*L_done);
919 } else {
920 __ bind(L_through);
921 }
922 }
923
924 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
925 Label L_slow;
926
927 // If weak references are being processed, weak/phantom loads need to go slow,
928 // regardless of their cset status.
929 if (_needs_load_ref_weak_barrier) {
930 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
931 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
932 __ j(L_slow);
933 }
934
935 if (_needs_keep_alive_barrier) {
936 // Emit the unconditional branch in the first version of the method.
937 // Let the rest of runtime figure out how to manage it.
938 // TODO: We could have spared the over-jump if patching knew we need the inverse branch.
939 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
940 Label L_over;
941 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
942 __ j(L_over);
943 __ j(*continuation());
944 __ bind(L_over);
945 }
946
947 // Cset-check. Fall-through to slow if in collection set.
948 if (_narrow) {
949 __ decode_heap_oop_not_null(_tmp2, _obj);
950 } else {
951 __ mv(_tmp2, _obj);
952 }
953
954 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
955 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
956 __ add(_tmp1, _tmp1, _tmp2);
957 __ lbu(_tmp1, Address(_tmp1, 0));
958 maybe_far_jump_if_zero(masm, _tmp1);
959
960 // Slow path
961 __ bind(L_slow);
962
963 // Obj is the result, need to temporarily stop preserving it.
964 bool is_obj_preserved = is_preserved(_obj);
|