25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
49 Register src, Register dst, Register count, RegSet saved_regs) {
50 if (is_oop) {
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
53
54 Label done;
55
56 // Avoid calling runtime if count == 0
57 __ beqz(count, done);
58
59 // Is GC active?
60 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
61 assert_different_registers(src, dst, count, t0);
62
63 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
64
751 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
752 } else {
753 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
754 }
755 } else {
756 assert(is_phantom, "only remaining strength");
757 assert(is_native, "phantom must only be called off-heap");
758 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
759 }
760 __ rt_call(target);
761 __ mv(t0, x10);
762 __ pop_call_clobbered_registers();
763 __ mv(x10, t0);
764
765 __ epilogue();
766 }
767
768 #undef __
769
770 #endif // COMPILER1
|
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
53 Register src, Register dst, Register count, RegSet saved_regs) {
54 if (is_oop) {
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57
58 Label done;
59
60 // Avoid calling runtime if count == 0
61 __ beqz(count, done);
62
63 // Is GC active?
64 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
65 assert_different_registers(src, dst, count, t0);
66
67 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
68
755 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
756 } else {
757 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
758 }
759 } else {
760 assert(is_phantom, "only remaining strength");
761 assert(is_native, "phantom must only be called off-heap");
762 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
763 }
764 __ rt_call(target);
765 __ mv(t0, x10);
766 __ pop_call_clobbered_registers();
767 __ mv(x10, t0);
768
769 __ epilogue();
770 }
771
772 #undef __
773
774 #endif // COMPILER1
775
776 #ifdef COMPILER2
777
778 #undef __
779 #define __ masm->
780
781 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, Register tmp1, Register tmp2, bool is_narrow) {
782 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
783 if (is_narrow) {
784 __ lwu(dst, src);
785 } else {
786 __ ld(dst, src);
787 }
788
789 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, tmp1, tmp2, is_narrow);
790 }
791
792 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
793 Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
794
795 ShenandoahBarrierStubC2::store_pre(masm, node, tmp1, dst, tmp2, tmp3, dst_narrow);
796
797 // Do the actual store
798 if (dst_narrow) {
799 if (!src_narrow) {
800 // Need to encode into tmp, because we cannot clobber src.
801 assert(tmp1 != noreg, "need temp register");
802 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
803 __ encode_heap_oop(tmp1, src);
804 } else {
805 __ encode_heap_oop_not_null(tmp1, src);
806 }
807 src = tmp1;
808 }
809 __ sw(src, dst);
810 } else {
811 __ sd(src, dst);
812 }
813
814 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp2, tmp3);
815 }
816
817 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
818 Register oldval, Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool is_acquire) {
819 const Assembler::Aqrl acquire = is_acquire ? Assembler::aq : Assembler::relaxed;
820 const Assembler::Aqrl release = Assembler::rl;
821 const Assembler::operand_size size = narrow ? Assembler::uint32 : Assembler::int64;
822
823 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp1, Address(addr), tmp2, tmp3, narrow);
824
825 // CAS!
826 __ cmpxchg(addr, oldval, newval, size, acquire, release, /* result */ res, !exchange /* result_as_bool */);
827
828 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
829 }
830
831 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
832 Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3, bool is_acquire) {
833 const bool is_narrow = node->bottom_type()->isa_narrowoop();
834
835 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp1, Address(addr, 0), tmp2, tmp3, is_narrow);
836
837 if (is_narrow) {
838 if (is_acquire) {
839 __ atomic_xchgalwu(preval, newval, addr);
840 } else {
841 __ atomic_xchgwu(preval, newval, addr);
842 }
843 } else {
844 if (is_acquire) {
845 __ atomic_xchgal(preval, newval, addr);
846 } else {
847 __ atomic_xchg(preval, newval, addr);
848 }
849 }
850
851 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
852 }
853
854 #undef __
855 #define __ masm.
856
857 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
858 assert(CardTable::dirty_card_val() == 0, "must be");
859 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
860
861 // tmp1 = card table base (holder)
862 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
863 __ ld(tmp1, curr_ct_holder_addr);
864
865 // tmp1 = effective address
866 __ la(tmp2, address);
867
868 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
869 __ srli(tmp2, tmp2, CardTable::card_shift());
870 __ add(tmp2, tmp2, tmp1);
871
872 if (UseCondCardMark) {
873 Label L_already_dirty;
874 __ lbu(tmp1, Address(tmp2));
875 __ beqz(tmp1, L_already_dirty);
876 __ sb(zr, Address(tmp2));
877 __ bind(L_already_dirty);
878 } else {
879 __ sb(zr, Address(tmp2));
880 }
881 }
882
883 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
884 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
885
886 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
887 __ lbu(tmp, gc_state_fast);
888 __ beqz(tmp, *continuation());
889 __ j(*entry());
890
891 // This is were the slowpath stub will return to or the code above will
892 // jump to if the checks are false
893 __ bind(*continuation());
894 }
895
896 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
897 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
898 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
899
900 __ bind(*entry());
901
902 // If we need to load ourselves, do it here.
903 if (_do_load) {
904 if (_narrow) {
905 __ lwu(_obj, _addr);
906 } else {
907 __ ld(_obj, _addr);
908 }
909 }
910
911 // If the object is null, there is no point in applying barriers.
912 maybe_far_jump_if_zero(masm, _obj);
913
914 // We need to make sure that loads done by callers survive across slow-path calls.
915 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
916 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
917 if (!_do_load || needs_both_barriers) {
918 preserve(_obj);
919 }
920
921 // Go for barriers. Barriers can return straight to continuation, as long
922 // as another barrier is not needed and we can reach the fastpath.
923 if (needs_both_barriers) {
924 keepalive(masm, nullptr);
925 lrb(masm);
926 } else if (_needs_keep_alive_barrier) {
927 keepalive(masm, continuation());
928 } else if (_needs_load_ref_barrier) {
929 lrb(masm);
930 } else {
931 ShouldNotReachHere();
932 }
933 }
934
935 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
936 Label L_short_jump;
937 __ bnez(reg, L_short_jump);
938 __ j(*continuation());
939 __ bind(L_short_jump);
940 }
941
942 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
943 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
944 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
945 Label L_through, L_slowpath;
946
947 // If another barrier is enabled as well, do a runtime check for a specific barrier.
948 if (_needs_load_ref_barrier) {
949 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
950 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
951 __ lbu(_tmp1, gc_state_fast);
952 __ beqz(_tmp1, L_through);
953 }
954
955 // Fast-path: put object into buffer.
956 // If buffer is already full, go slow.
957 __ ld(_tmp1, index);
958 __ beqz(_tmp1, L_slowpath);
959 __ subi(_tmp1, _tmp1, wordSize);
960 __ sd(_tmp1, index);
961 __ ld(_tmp2, buffer);
962
963 // Store the object in queue.
964 // If object is narrow, we need to decode it before inserting.
965 __ add(_tmp1, _tmp1, _tmp2);
966 if (_narrow) {
967 __ decode_heap_oop_not_null(_tmp2, _obj);
968 __ sd(_tmp2, Address(_tmp1));
969 } else {
970 __ sd(_obj, Address(_tmp1));
971 }
972
973 // Fast-path exits here.
974 if (L_done != nullptr) {
975 __ j(*L_done);
976 } else {
977 __ j(L_through);
978 }
979
980 // Slow-path: call runtime to handle.
981 __ bind(L_slowpath);
982
983 {
984 SaveLiveRegisters slr(&masm, this);
985
986 // Go to runtime and handle the rest there.
987 __ mv(c_rarg0, _obj);
988 __ rt_call(keepalive_runtime_entry_addr());
989 }
990 if (L_done != nullptr) {
991 __ j(*L_done);
992 } else {
993 __ bind(L_through);
994 }
995 }
996
997 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
998 Label L_slow;
999
1000 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1001 if (_needs_keep_alive_barrier) {
1002 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1003 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
1004 __ lbu(_tmp1, gc_state_fast);
1005 maybe_far_jump_if_zero(masm, _tmp1);
1006 }
1007
1008 // If weak references are being processed, weak/phantom loads need to go slow,
1009 // regardless of their cset status.
1010 if (_needs_load_ref_weak_barrier) {
1011 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
1012 __ lbu(_tmp1, gc_state_fast);
1013 __ bnez(_tmp1, L_slow);
1014 }
1015
1016 // Cset-check. Fall-through to slow if in collection set.
1017 if (_narrow) {
1018 __ decode_heap_oop_not_null(_tmp2, _obj);
1019 } else {
1020 __ mv(_tmp2, _obj);
1021 }
1022
1023 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
1024 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1025 __ add(_tmp1, _tmp1, _tmp2);
1026 __ lbu(_tmp1, Address(_tmp1, 0));
1027 maybe_far_jump_if_zero(masm, _tmp1);
1028
1029 // Slow path
1030 __ bind(L_slow);
1031
1032 // Obj is the result, need to temporarily stop preserving it.
1033 bool is_obj_preserved = is_preserved(_obj);
1034 if (is_obj_preserved) {
1035 dont_preserve(_obj);
1036 }
1037 {
1038 SaveLiveRegisters slr(&masm, this);
1039
1040 // Shuffle in the arguments. The end result should be:
1041 // c_rarg0 <- obj
1042 // c_rarg1 <- lea(addr)
1043 if (c_rarg0 == _obj) {
1044 __ la(c_rarg1, _addr);
1045 } else if (c_rarg1 == _obj) {
1046 // Set up arguments in reverse, and then flip them
1047 __ la(c_rarg0, _addr);
1048 // flip them
1049 __ mv(_tmp1, c_rarg0);
1050 __ mv(c_rarg0, c_rarg1);
1051 __ mv(c_rarg1, _tmp1);
1052 } else {
1053 assert_different_registers(c_rarg1, _obj);
1054 __ la(c_rarg1, _addr);
1055 __ mv(c_rarg0, _obj);
1056 }
1057
1058 // Go to runtime and handle the rest there.
1059 __ rt_call(lrb_runtime_entry_addr());
1060
1061 // Save the result where needed. Narrow entries return narrowOop (32 bits)
1062 // we need to zero the upper 32 bits of x10.
1063 if (_narrow) {
1064 __ zext_w(_obj, x10);
1065 } else {
1066 __ mv(_obj, x10);
1067 }
1068 }
1069 if (is_obj_preserved) {
1070 preserve(_obj);
1071 }
1072
1073 __ j(*continuation());
1074 }
1075
1076 int ShenandoahBarrierStubC2::available_gp_registers() {
1077 Unimplemented(); // Not used
1078 return 0;
1079 }
1080
1081 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1082 Unimplemented(); // Not used
1083 return true;
1084 }
1085
1086 void ShenandoahBarrierStubC2::post_init() {
1087 // Do nothing.
1088 }
1089
1090 #endif // COMPILER2
|