17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahRuntime.hpp"
38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
39 #include "interpreter/interpreter.hpp"
40 #include "macroAssembler_ppc.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/globalDefinitions.hpp"
44 #include "vm_version_ppc.hpp"
45 #ifdef COMPILER1
46 #include "c1/c1_LIRAssembler.hpp"
47 #include "c1/c1_MacroAssembler.hpp"
48 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
49 #endif
50 #ifdef COMPILER2
51 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
52 #endif
53
54 #define __ masm->
55
56 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
57 Register base, RegisterOrConstant ind_or_offs,
58 Register tmp1, Register tmp2, Register tmp3,
59 MacroAssembler::PreservationLevel preservation_level,
60 int extra_stack_space) {
912 #define __ masm.
913
914 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
915 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
916 assert_different_registers(tmp1, tmp2, address.index(), address.base());
917
918 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
919 if (address.index() == noreg) {
920 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
921 } else {
922 __ add(tmp2, address.index(), address.base());
923 if (address.disp() != 0) {
924 __ addi(tmp2, tmp2, address.disp());
925 }
926 }
927 __ srdi(tmp2, tmp2, CardTable::card_shift());
928 __ li(R0, CardTable::dirty_card_val());
929 __ stbx(R0, tmp2, tmp1);
930 }
931
932 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
933 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
934
935 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)), R16_thread);
936 __ cmpdi(CR0, tmp, 0);
937 // Branch to entry if not equal
938 __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *entry());
939 // This is were the slowpath stub will return to
940 __ bind(*continuation());
941 }
942
943 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
944 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
945 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
946
947 __ bind(*entry());
948
949 // If we need to load ourselves, do it here.
950 if (_do_load) {
951 if (_narrow) {
952 __ lwz(_obj, _addr.disp(), _addr.base());
953 } else {
954 __ ld(_obj, _addr.disp(), _addr.base());
955 }
956 }
957
958 // If the object is null, there is no point in applying barriers.
959 maybe_far_jump_if_zero(masm, _obj);
960
961 // We need to make sure that loads done by callers survive across slow-path calls.
962 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
969 // as another barrier is not needed and we can reach the fastpath.
970 if (needs_both_barriers) {
971 keepalive(masm, nullptr);
972 lrb(masm);
973 } else if (_needs_keep_alive_barrier) {
974 keepalive(masm, continuation());
975 } else if (_needs_load_ref_barrier) {
976 lrb(masm);
977 } else {
978 ShouldNotReachHere();
979 }
980 }
981
982 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
983 __ cmpdi(CR0, reg, 0);
984 // Branch to continuation if equal
985 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
986 }
987
988 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
989 const int gcstate_offset = in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING));
990 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
991 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
992 Label L_through, L_slowpath;
993
994 // If another barrier is enabled as well, do a runtime check for a specific barrier.
995 if (_needs_load_ref_barrier) {
996 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
997 __ lbz(_tmp1, gcstate_offset, R16_thread);
998 __ cmpdi(CR0, _tmp1, 0);
999 __ beq(CR0, L_through);
1000 }
1001
1002 // Fast-path: put object into buffer.
1003 // If buffer is already full, go slow.
1004 __ ld(_tmp1, index_offset, R16_thread);
1005 __ cmpdi(CR0, _tmp1, 0);
1006 __ beq(CR0, L_slowpath);
1007 __ addi(_tmp1, _tmp1, -wordSize);
1008 __ std(_tmp1, index_offset, R16_thread);
1009 __ ld(_tmp2, buffer_offset, R16_thread);
1010
1011 // Store the object in queue.
1012 // If object is narrow, we need to decode it before inserting.
1013 if (_narrow) {
1014 __ add(_tmp2, _tmp2, _tmp1);
1015 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1016 __ stdx(decoded, _tmp2);
1017 } else {
1018 __ stdx(_obj, _tmp2, _tmp1);
1019 }
1028 // Slow-path: call runtime to handle.
1029 __ bind(L_slowpath);
1030
1031 {
1032 SaveLiveRegisters slr(&masm, this);
1033
1034 // Go to runtime and handle the rest there.
1035 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1036 }
1037
1038 if (L_done != nullptr) {
1039 __ b(*L_done);
1040 } else {
1041 __ bind(L_through);
1042 }
1043 }
1044
1045 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1046 Label L_slow;
1047
1048 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1049 if (_needs_keep_alive_barrier) {
1050 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1051 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)), R16_thread);
1052 maybe_far_jump_if_zero(masm, _tmp1);
1053 }
1054
1055 // If weak references are being processed, weak/phantom loads need to go slow,
1056 // regardless of their cset status.
1057 if (_needs_load_ref_weak_barrier) {
1058 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)), R16_thread);
1059 __ cmpdi(CR0, _tmp1, 0);
1060 __ bne(CR0, L_slow);
1061 }
1062
1063 // Cset-check. Fall-through to slow if in collection set.
1064 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1065 if (_narrow) {
1066 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1067 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1068 } else {
1069 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1070 }
1071 __ lbzx(_tmp2, _tmp2, _tmp1);
1072 maybe_far_jump_if_zero(masm, _tmp2);
1073
1074 // Slow path
1075 __ bind(L_slow);
1076
1077 // Obj is the result, need to temporarily stop preserving it.
1078 bool is_obj_preserved = is_preserved(_obj);
1079 if (is_obj_preserved) {
1080 dont_preserve(_obj);
|
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "nativeInst_ppc.hpp"
43 #include "runtime/javaThread.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "utilities/globalDefinitions.hpp"
46 #include "vm_version_ppc.hpp"
47 #ifdef COMPILER1
48 #include "c1/c1_LIRAssembler.hpp"
49 #include "c1/c1_MacroAssembler.hpp"
50 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
51 #endif
52 #ifdef COMPILER2
53 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
54 #endif
55
56 #define __ masm->
57
58 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
59 Register base, RegisterOrConstant ind_or_offs,
60 Register tmp1, Register tmp2, Register tmp3,
61 MacroAssembler::PreservationLevel preservation_level,
62 int extra_stack_space) {
914 #define __ masm.
915
916 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
917 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
918 assert_different_registers(tmp1, tmp2, address.index(), address.base());
919
920 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
921 if (address.index() == noreg) {
922 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
923 } else {
924 __ add(tmp2, address.index(), address.base());
925 if (address.disp() != 0) {
926 __ addi(tmp2, tmp2, address.disp());
927 }
928 }
929 __ srdi(tmp2, tmp2, CardTable::card_shift());
930 __ li(R0, CardTable::dirty_card_val());
931 __ stbx(R0, tmp2, tmp1);
932 }
933
934 void ShenandoahBarrierStubC2::patchable_jump_if_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
935 // Emit the unconditional branch in the first version of the method.
936 // Let the rest of runtime figure out how to manage it.
937 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, false)));
938 __ b(*L_target);
939 }
940
941 void ShenandoahBarrierStubC2::patchable_jump_if_not_gc_state(MacroAssembler& masm, const char test_state, Label* L_target) {
942 // Emit the unconditional branch in the first version of the method.
943 // Let the rest of runtime figure out how to manage it.
944 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(test_state, true)));
945 __ b(*L_target);
946 }
947
948 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
949 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
950 patchable_jump_if_gc_state(masm, test_state, entry());
951 __ bind(*continuation());
952 }
953
954 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
955 NativeInstruction* ni = nativeInstruction_at(pc);
956 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
957 NativeGeneralJump* jmp = nativeGeneralJump_at(pc);
958 return jmp->jump_destination();
959 }
960
961 static void check_at(bool cond, address pc, const char* msg) {
962 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x",
963 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3));
964 }
965
966 static bool is_patchable_nop(address pc) {
967 if (*(pc + 0) != 0x00) return false;
968 if (*(pc + 1) != 0x00) return false;
969 if (*(pc + 2) != 0x00) return false;
970 if (*(pc + 3) != 0x60) return false;
971 return true;
972 }
973
974 static void insert_patchable_nop(address pc) {
975 *reinterpret_cast<int32_t*>(pc) = 0x60000000;
976 }
977
978 bool ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
979 NativeInstruction* ni = nativeInstruction_at(pc);
980 bool patching = ni->is_jump();
981 if (patching) {
982 insert_patchable_nop(pc);
983 }
984 check_at(is_patchable_nop(pc), pc, "Should be nop");
985 return patching;
986 }
987
988 bool ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
989 bool patching = is_patchable_nop(pc);
990 if (patching) {
991 NativeGeneralJump::insert_unconditional(pc, stub_addr);
992 }
993
994 NativeInstruction* ni = nativeInstruction_at(pc);
995 check_at(ni->is_jump(), pc, "Should be jump");
996 check_at(nativeGeneralJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
997 return patching;
998 }
999
1000 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1001 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1002 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1003
1004 __ bind(*entry());
1005
1006 // If we need to load ourselves, do it here.
1007 if (_do_load) {
1008 if (_narrow) {
1009 __ lwz(_obj, _addr.disp(), _addr.base());
1010 } else {
1011 __ ld(_obj, _addr.disp(), _addr.base());
1012 }
1013 }
1014
1015 // If the object is null, there is no point in applying barriers.
1016 maybe_far_jump_if_zero(masm, _obj);
1017
1018 // We need to make sure that loads done by callers survive across slow-path calls.
1019 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1026 // as another barrier is not needed and we can reach the fastpath.
1027 if (needs_both_barriers) {
1028 keepalive(masm, nullptr);
1029 lrb(masm);
1030 } else if (_needs_keep_alive_barrier) {
1031 keepalive(masm, continuation());
1032 } else if (_needs_load_ref_barrier) {
1033 lrb(masm);
1034 } else {
1035 ShouldNotReachHere();
1036 }
1037 }
1038
1039 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1040 __ cmpdi(CR0, reg, 0);
1041 // Branch to continuation if equal
1042 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1043 }
1044
1045 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1046 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1047 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1048 Label L_through, L_slowpath;
1049
1050 // If another barrier is enabled as well, do a check for a specific barrier.
1051 if (_needs_load_ref_barrier) {
1052 assert(L_done == nullptr, "Should be");
1053 char state_to_check = ShenandoahHeap::MARKING;
1054 patchable_jump_if_not_gc_state(masm, state_to_check, &L_through);
1055 }
1056
1057 // Fast-path: put object into buffer.
1058 // If buffer is already full, go slow.
1059 __ ld(_tmp1, index_offset, R16_thread);
1060 __ cmpdi(CR0, _tmp1, 0);
1061 __ beq(CR0, L_slowpath);
1062 __ addi(_tmp1, _tmp1, -wordSize);
1063 __ std(_tmp1, index_offset, R16_thread);
1064 __ ld(_tmp2, buffer_offset, R16_thread);
1065
1066 // Store the object in queue.
1067 // If object is narrow, we need to decode it before inserting.
1068 if (_narrow) {
1069 __ add(_tmp2, _tmp2, _tmp1);
1070 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1071 __ stdx(decoded, _tmp2);
1072 } else {
1073 __ stdx(_obj, _tmp2, _tmp1);
1074 }
1083 // Slow-path: call runtime to handle.
1084 __ bind(L_slowpath);
1085
1086 {
1087 SaveLiveRegisters slr(&masm, this);
1088
1089 // Go to runtime and handle the rest there.
1090 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1091 }
1092
1093 if (L_done != nullptr) {
1094 __ b(*L_done);
1095 } else {
1096 __ bind(L_through);
1097 }
1098 }
1099
1100 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1101 Label L_slow;
1102
1103 // If another barrier is enabled as well, do a check for a specific barrier.
1104 if (_needs_keep_alive_barrier) {
1105 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1106 patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
1107 }
1108
1109 // If weak references are being processed, weak/phantom loads need to go slow,
1110 // regardless of their cset status.
1111 if (_needs_load_ref_weak_barrier) {
1112 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
1113 patchable_jump_if_gc_state(masm, state_to_check, &L_slow);
1114 }
1115
1116 // Cset-check. Fall-through to slow if in collection set.
1117 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1118 if (_narrow) {
1119 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1120 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1121 } else {
1122 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1123 }
1124 __ lbzx(_tmp2, _tmp2, _tmp1);
1125 maybe_far_jump_if_zero(masm, _tmp2);
1126
1127 // Slow path
1128 __ bind(L_slow);
1129
1130 // Obj is the result, need to temporarily stop preserving it.
1131 bool is_obj_preserved = is_preserved(_obj);
1132 if (is_obj_preserved) {
1133 dont_preserve(_obj);
|