892 // jump to if the checks are false
893 __ bind(*continuation());
894 }
895
896 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
897 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
898 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
899
900 __ bind(*entry());
901
902 // If we need to load ourselves, do it here.
903 if (_do_load) {
904 if (_narrow) {
905 __ lwu(_obj, _addr);
906 } else {
907 __ ld(_obj, _addr);
908 }
909 }
910
911 // If the object is null, there is no point in applying barriers.
912 maybe_far_jump_if_zero(masm, _obj);
913
914 // We need to make sure that loads done by callers survive across slow-path calls.
915 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
916 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
917 if (!_do_load || needs_both_barriers) {
918 preserve(_obj);
919 }
920
921 // Go for barriers. Barriers can return straight to continuation, as long
922 // as another barrier is not needed and we can reach the fastpath.
923 if (needs_both_barriers) {
924 keepalive(masm, nullptr);
925 lrb(masm);
926 } else if (_needs_keep_alive_barrier) {
927 keepalive(masm, continuation());
928 } else if (_needs_load_ref_barrier) {
929 lrb(masm);
930 } else {
931 ShouldNotReachHere();
932 }
933 }
934
935 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
936 Label L_short_jump;
937 __ bnez(reg, L_short_jump);
938 __ j(*continuation());
939 __ bind(L_short_jump);
940 }
941
942 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
943 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
944 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
945 Label L_through, L_slowpath;
946
947 // If another barrier is enabled as well, do a runtime check for a specific barrier.
948 if (_needs_load_ref_barrier) {
949 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
950 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
951 __ lbu(_tmp1, gc_state_fast);
952 __ beqz(_tmp1, L_through);
953 }
954
955 // Fast-path: put object into buffer.
956 // If buffer is already full, go slow.
957 __ ld(_tmp1, index);
958 __ beqz(_tmp1, L_slowpath);
959 __ subi(_tmp1, _tmp1, wordSize);
960 __ sd(_tmp1, index);
961 __ ld(_tmp2, buffer);
962
963 // Store the object in queue.
964 // If object is narrow, we need to decode it before inserting.
965 __ add(_tmp1, _tmp1, _tmp2);
966 if (_narrow) {
967 __ decode_heap_oop_not_null(_tmp2, _obj);
968 __ sd(_tmp2, Address(_tmp1));
969 } else {
970 __ sd(_obj, Address(_tmp1));
971 }
981 __ bind(L_slowpath);
982
983 {
984 SaveLiveRegisters slr(&masm, this);
985
986 // Go to runtime and handle the rest there.
987 __ mv(c_rarg0, _obj);
988 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
989 __ jalr(ra);
990 }
991 if (L_done != nullptr) {
992 __ j(*L_done);
993 } else {
994 __ bind(L_through);
995 }
996 }
997
998 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
999 Label L_slow;
1000
1001 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1002 if (_needs_keep_alive_barrier) {
1003 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1004 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
1005 __ lbu(_tmp1, gc_state_fast);
1006 maybe_far_jump_if_zero(masm, _tmp1);
1007 }
1008
1009 // If weak references are being processed, weak/phantom loads need to go slow,
1010 // regardless of their cset status.
1011 if (_needs_load_ref_weak_barrier) {
1012 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
1013 __ lbu(_tmp1, gc_state_fast);
1014 __ bnez(_tmp1, L_slow);
1015 }
1016
1017 // Cset-check. Fall-through to slow if in collection set.
1018 if (_narrow) {
1019 __ decode_heap_oop_not_null(_tmp2, _obj);
1020 } else {
1021 __ mv(_tmp2, _obj);
1022 }
1023
1024 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
1025 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1026 __ add(_tmp1, _tmp1, _tmp2);
1027 __ lbu(_tmp1, Address(_tmp1, 0));
1028 maybe_far_jump_if_zero(masm, _tmp1);
1029
1030 // Slow path
1031 __ bind(L_slow);
1032
1033 // Obj is the result, need to temporarily stop preserving it.
1034 bool is_obj_preserved = is_preserved(_obj);
1035 if (is_obj_preserved) {
1036 dont_preserve(_obj);
1037 }
1038 {
1039 SaveLiveRegisters slr(&masm, this);
1040
1041 // Shuffle in the arguments. The end result should be:
1042 // c_rarg0 <- obj
1043 // c_rarg1 <- lea(addr)
1044 if (c_rarg0 == _obj) {
1045 __ la(c_rarg1, _addr);
1046 } else if (c_rarg1 == _obj) {
1047 // Set up arguments in reverse, and then flip them
1048 __ la(c_rarg0, _addr);
|
892 // jump to if the checks are false
893 __ bind(*continuation());
894 }
895
896 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
897 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
898 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
899
900 __ bind(*entry());
901
902 // If we need to load ourselves, do it here.
903 if (_do_load) {
904 if (_narrow) {
905 __ lwu(_obj, _addr);
906 } else {
907 __ ld(_obj, _addr);
908 }
909 }
910
911 // If the object is null, there is no point in applying barriers.
912 maybe_far_jump_if_zero(masm, _obj, continuation());
913
914 // We need to make sure that loads done by callers survive across slow-path calls.
915 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
916 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
917 if (!_do_load || needs_both_barriers) {
918 preserve(_obj);
919 }
920
921 // Go for barriers. Barriers can return straight to continuation, as long
922 // as another barrier is not needed and we can reach the fastpath.
923 if (needs_both_barriers) {
924 keepalive(masm, nullptr);
925 lrb(masm);
926 } else if (_needs_keep_alive_barrier) {
927 keepalive(masm, continuation());
928 } else if (_needs_load_ref_barrier) {
929 lrb(masm);
930 } else {
931 ShouldNotReachHere();
932 }
933 }
934
935 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg, Label* L_target) {
936 Label L_short_jump;
937 __ bnez(reg, L_short_jump);
938 __ j(*L_target);
939 __ bind(L_short_jump);
940 }
941
942 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
943 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
944 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
945 Label L_through, L_slowpath;
946
947 // Hotpatched GC checks only care about idle/non-idle state, so we need to check again here.
948 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
949 __ lbu(_tmp1, gc_state_fast);
950 if (L_done != nullptr) {
951 maybe_far_jump_if_zero(masm, _tmp1, L_done);
952 } else {
953 __ beqz(_tmp1, L_through);
954 }
955
956 // Fast-path: put object into buffer.
957 // If buffer is already full, go slow.
958 __ ld(_tmp1, index);
959 __ beqz(_tmp1, L_slowpath);
960 __ subi(_tmp1, _tmp1, wordSize);
961 __ sd(_tmp1, index);
962 __ ld(_tmp2, buffer);
963
964 // Store the object in queue.
965 // If object is narrow, we need to decode it before inserting.
966 __ add(_tmp1, _tmp1, _tmp2);
967 if (_narrow) {
968 __ decode_heap_oop_not_null(_tmp2, _obj);
969 __ sd(_tmp2, Address(_tmp1));
970 } else {
971 __ sd(_obj, Address(_tmp1));
972 }
982 __ bind(L_slowpath);
983
984 {
985 SaveLiveRegisters slr(&masm, this);
986
987 // Go to runtime and handle the rest there.
988 __ mv(c_rarg0, _obj);
989 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
990 __ jalr(ra);
991 }
992 if (L_done != nullptr) {
993 __ j(*L_done);
994 } else {
995 __ bind(L_through);
996 }
997 }
998
999 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1000 Label L_slow;
1001
1002 // Hotpatched GC checks only care about idle/non-idle state, so we need to check again here.
1003 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1004 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
1005 __ lbu(_tmp1, gc_state_fast);
1006 maybe_far_jump_if_zero(masm, _tmp1, continuation());
1007
1008 // If weak references are being processed, weak/phantom loads need to go slow,
1009 // regardless of their cset status.
1010 if (_needs_load_ref_weak_barrier) {
1011 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
1012 __ lbu(_tmp1, gc_state_fast);
1013 __ bnez(_tmp1, L_slow);
1014 }
1015
1016 // Cset-check. Fall-through to slow if in collection set.
1017 if (_narrow) {
1018 __ decode_heap_oop_not_null(_tmp2, _obj);
1019 } else {
1020 __ mv(_tmp2, _obj);
1021 }
1022
1023 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
1024 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1025 __ add(_tmp1, _tmp1, _tmp2);
1026 __ lbu(_tmp1, Address(_tmp1, 0));
1027 maybe_far_jump_if_zero(masm, _tmp1, continuation());
1028
1029 // Slow path
1030 __ bind(L_slow);
1031
1032 // Obj is the result, need to temporarily stop preserving it.
1033 bool is_obj_preserved = is_preserved(_obj);
1034 if (is_obj_preserved) {
1035 dont_preserve(_obj);
1036 }
1037 {
1038 SaveLiveRegisters slr(&masm, this);
1039
1040 // Shuffle in the arguments. The end result should be:
1041 // c_rarg0 <- obj
1042 // c_rarg1 <- lea(addr)
1043 if (c_rarg0 == _obj) {
1044 __ la(c_rarg1, _addr);
1045 } else if (c_rarg1 == _obj) {
1046 // Set up arguments in reverse, and then flip them
1047 __ la(c_rarg0, _addr);
|