25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
49 Register src, Register dst, Register count, RegSet saved_regs) {
50 if (is_oop) {
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
53
54 Label done;
55
56 // Avoid calling runtime if count == 0
57 __ beqz(count, done);
58
59 // Is GC active?
60 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
61 assert_different_registers(src, dst, count, t0);
62
63 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
64
751 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
752 } else {
753 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
754 }
755 } else {
756 assert(is_phantom, "only remaining strength");
757 assert(is_native, "phantom must only be called off-heap");
758 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
759 }
760 __ rt_call(target);
761 __ mv(t0, x10);
762 __ pop_call_clobbered_registers();
763 __ mv(x10, t0);
764
765 __ epilogue();
766 }
767
768 #undef __
769
770 #endif // COMPILER1
|
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #include "utilities/population_count.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #endif
51
52 #define __ masm->
53
54 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
55 Register src, Register dst, Register count, RegSet saved_regs) {
56 if (is_oop) {
57 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
58 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
59
60 Label done;
61
62 // Avoid calling runtime if count == 0
63 __ beqz(count, done);
64
65 // Is GC active?
66 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
67 assert_different_registers(src, dst, count, t0);
68
69 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
70
757 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
758 } else {
759 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
760 }
761 } else {
762 assert(is_phantom, "only remaining strength");
763 assert(is_native, "phantom must only be called off-heap");
764 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
765 }
766 __ rt_call(target);
767 __ mv(t0, x10);
768 __ pop_call_clobbered_registers();
769 __ mv(x10, t0);
770
771 __ epilogue();
772 }
773
774 #undef __
775
776 #endif // COMPILER1
777
778 #ifdef COMPILER2
779
780 #undef __
781 #define __ masm.
782
783 int ShenandoahBarrierStubC2::available_gp_registers() {
784 return Register::number_of_registers;
785 }
786
787 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
788 return r == fp || r == sp ||
789 r == xheapbase || r == xthread ||
790 r == t0 || r == t1 || r == zr;
791 }
792
793 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
794 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
795
796 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
797 __ lbu(tmp, gc_state_fast);
798 __ beqz(tmp, *continuation());
799 __ j(*entry());
800
801 // This is were the slowpath stub will return to or the code above will
802 // jump to if the checks are false
803 __ bind(*continuation());
804 }
805
806 #undef __
807 #define __ masm->
808
809 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
810 Register oldval, Register newval, Register tmp, bool exchange, bool narrow, bool is_acquire) {
811 const Assembler::Aqrl acquire = is_acquire ? Assembler::aq : Assembler::relaxed;
812 const Assembler::Aqrl release = Assembler::rl;
813
814 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp, Address(addr), t0, t1, narrow);
815
816 // Existing RISCV cmpxchg_oop already handles Shenandoah forwarded-value retry logic.
817 // FIXME: Why? Pre-barrier already obviates the need for retry. This is an awkward dependency on SBSA. Emit the plain cmpxchg.
818 // It returns:
819 // - boolean 0/1 for CAS (!exchange)
820 // - loaded/current value for CAE (exchange)
821 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, addr, oldval, newval, acquire, release, exchange /* is_cae */, res);
822
823 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), t0, t1);
824 }
825
826 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
827 Register newval, Register addr, Register tmp, bool is_acquire) {
828 const bool is_narrow = node->bottom_type()->isa_narrowoop();
829
830 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp, Address(addr, 0), t0, t1, is_narrow);
831
832 if (is_narrow) {
833 if (is_acquire) {
834 __ atomic_xchgalwu(preval, newval, addr);
835 } else {
836 __ atomic_xchgwu(preval, newval, addr);
837 }
838 } else {
839 if (is_acquire) {
840 __ atomic_xchgal(preval, newval, addr);
841 } else {
842 __ atomic_xchg(preval, newval, addr);
843 }
844 }
845
846 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), t0, t1);
847 }
848
849 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
850 Register src, bool src_narrow, Register tmp) {
851
852 ShenandoahBarrierStubC2::store_pre(masm, node, tmp, dst, t0, t1, dst_narrow);
853
854 // Do the actual store
855 if (dst_narrow) {
856 if (!src_narrow) {
857 // Need to encode into tmp, because we cannot clobber src.
858 assert(tmp != noreg, "need temp register");
859 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
860 __ encode_heap_oop(tmp, src);
861 } else {
862 __ encode_heap_oop_not_null(tmp, src);
863 }
864 src = tmp;
865 }
866 __ sw(src, dst);
867 } else {
868 __ sd(src, dst);
869 }
870
871 ShenandoahBarrierStubC2::store_post(masm, node, dst, t0, t1);
872 }
873
874 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, bool is_narrow) {
875 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
876 if (is_narrow) {
877 __ lwu(dst, src);
878 } else {
879 __ ld(dst, src);
880 }
881
882 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, t0, t1, is_narrow);
883 }
884
885 void ShenandoahBarrierStubC2::store_post(MacroAssembler* masm, const MachNode* node, Address address, Register tmp1, Register tmp2) {
886 if (!needs_card_barrier(node)) {
887 return;
888 }
889
890 assert(CardTable::dirty_card_val() == 0, "must be");
891 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
892
893 // tmp1 = card table base (holder)
894 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
895 __ ld(tmp1, curr_ct_holder_addr);
896
897 // tmp1 = effective address
898 __ la(tmp2, address);
899
900 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
901 __ srli(tmp2, tmp2, CardTable::card_shift());
902 __ add(tmp2, tmp2, tmp1);
903
904 if (UseCondCardMark) {
905 Label L_already_dirty;
906 __ lbu(tmp1, Address(tmp2));
907 __ beqz(tmp1, L_already_dirty);
908 __ sb(zr, Address(tmp2));
909 __ bind(L_already_dirty);
910 } else {
911 __ sb(zr, Address(tmp2));
912 }
913 }
914
915 void ShenandoahBarrierStubC2::load_store_post(MacroAssembler* masm, const MachNode* node, Address address, Register tmp1, Register tmp2) {
916 store_post(masm, node, address, tmp1, tmp2);
917 }
918
919 #undef __
920 #define __ masm.
921
922 void ShenandoahBarrierStubC2::post_init() {
923 // Do nothing.
924 }
925
926 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
927 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
928 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
929
930 __ bind(*entry());
931
932 // If we need to load ourselves, do it here.
933 if (_do_load) {
934 if (_narrow) {
935 __ lwu(_obj, _addr);
936 } else {
937 __ ld(_obj, _addr);
938 }
939 }
940
941 // If the object is null, there is no point in applying barriers.
942 maybe_far_jump_if_zero(masm, _obj, continuation());
943
944 // Go for barriers. Barriers can return straight to continuation, as long
945 // as another barrier is not needed and we can reach the fastpath.
946 if (_needs_keep_alive_barrier && _needs_load_ref_barrier) {
947 keepalive(masm, nullptr);
948 lrb(masm);
949 } else if (_needs_keep_alive_barrier) {
950 keepalive(masm, continuation());
951 } else if (_needs_load_ref_barrier) {
952 lrb(masm);
953 } else {
954 ShouldNotReachHere();
955 }
956 }
957
958 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg, Label* L_done) {
959 Label L_short_jump;
960 __ bnez(reg, L_short_jump);
961 __ j(*L_done);
962 __ bind(L_short_jump);
963 }
964
965 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
966 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
967 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
968 Label L_through, L_slowpath;
969
970 // If another barrier is enabled as well, do a runtime check for a specific barrier.
971 if (_needs_load_ref_barrier) {
972 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
973 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
974 __ lbu(_tmp1, gc_state_fast);
975 __ beqz(_tmp1, L_through);
976 }
977
978 // Fast-path: put object into buffer.
979 // If buffer is already full, go slow.
980 __ ld(_tmp1, index);
981 __ beqz(_tmp1, L_slowpath);
982 __ subi(_tmp1, _tmp1, wordSize);
983 __ sd(_tmp1, index);
984 __ ld(_tmp2, buffer);
985
986 // If object is narrow, we need to unpack it before inserting into buffer.
987 __ add(_tmp1, _tmp1, _tmp2);
988 if (_narrow) {
989 __ decode_heap_oop_not_null(_tmp2, _obj);
990 __ sd(_tmp2, Address(_tmp1));
991 } else {
992 __ sd(_obj, Address(_tmp1));
993 }
994
995 // Fast-path exits here.
996 if (L_done != nullptr) {
997 __ j(*L_done);
998 } else {
999 __ j(L_through);
1000 }
1001
1002 // Slow-path: call runtime to handle.
1003 __ bind(L_slowpath);
1004
1005 // If this stub also supports LRB then we need to preserve _obj to use it there.
1006 if (_needs_load_ref_barrier) {
1007 preserve(_obj);
1008 } else {
1009 dont_preserve(_obj);
1010 }
1011
1012 {
1013 SaveLiveRegisters slr(&masm, this);
1014
1015 // Go to runtime and handle the rest there.
1016 __ mv(c_rarg0, _obj);
1017 __ rt_call(keepalive_runtime_entry_addr());
1018 }
1019
1020 if (L_done != nullptr) {
1021 __ j(*L_done);
1022 } else {
1023 __ bind(L_through);
1024 }
1025 }
1026
1027 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1028 Label L_slow;
1029
1030 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1031 if (_needs_keep_alive_barrier) {
1032 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1033 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
1034 __ lbu(_tmp1, gc_state_fast);
1035 maybe_far_jump_if_zero(masm, _tmp1, continuation());
1036 }
1037
1038 // If weak references are being processed, weak/phantom loads need to go slow,
1039 // regardless of their cset status.
1040 if (_needs_load_ref_weak_barrier) {
1041 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
1042 __ lbu(_tmp1, gc_state_fast);
1043 __ bnez(_tmp1, L_slow);
1044 }
1045
1046 // Cset-check. Fall-through to slow if in collection set.
1047 if (_narrow) {
1048 __ decode_heap_oop_not_null(_tmp2, _obj);
1049 } else {
1050 __ mv(_tmp2, _obj);
1051 }
1052
1053 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
1054 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1055 __ add(_tmp1, _tmp1, _tmp2);
1056 __ lbu(_tmp1, Address(_tmp1, 0));
1057 maybe_far_jump_if_zero(masm, _tmp1, continuation());
1058
1059 // Slow path
1060 __ bind(L_slow);
1061
1062 // Obj is the result, need to temporarily stop preserving it.
1063 dont_preserve(_obj);
1064 {
1065 SaveLiveRegisters slr(&masm, this);
1066
1067 // Shuffle in the arguments. The end result should be:
1068 // c_rarg0 <- obj
1069 // c_rarg1 <- lea(addr)
1070 if (c_rarg0 == _obj) {
1071 __ la(c_rarg1, _addr);
1072 } else if (c_rarg1 == _obj) {
1073 // Set up arguments in reverse, and then flip them
1074 __ la(c_rarg0, _addr);
1075 // flip them
1076 __ mv(t0, c_rarg0);
1077 __ mv(c_rarg0, c_rarg1);
1078 __ mv(c_rarg1, t0);
1079 } else {
1080 assert_different_registers(c_rarg1, _obj);
1081 __ la(c_rarg1, _addr);
1082 __ mv(c_rarg0, _obj);
1083 }
1084
1085 // Go to runtime and handle the rest there.
1086 __ rt_call(lrb_runtime_entry_addr());
1087
1088 // Save the result where needed.
1089 if (_narrow) {
1090 __ zext_w(_obj, x10);
1091 } else {
1092 __ mv(_obj, x10);
1093 }
1094 }
1095 preserve(_obj);
1096
1097 __ j(*continuation());
1098 }
1099
1100 #endif // COMPILER2
|