17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahRuntime.hpp"
38 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
39 #include "interpreter/interpreter.hpp"
40 #include "macroAssembler_ppc.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/globalDefinitions.hpp"
44 #include "vm_version_ppc.hpp"
45 #ifdef COMPILER1
46 #include "c1/c1_LIRAssembler.hpp"
47 #include "c1/c1_MacroAssembler.hpp"
48 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
49 #endif
50 #ifdef COMPILER2
51 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
52 #endif
53
54 #define __ masm->
55
56 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
57 Register base, RegisterOrConstant ind_or_offs,
58 Register tmp1, Register tmp2, Register tmp3,
59 MacroAssembler::PreservationLevel preservation_level) {
1035 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1036 assert_different_registers(tmp1, tmp2, address.index(), address.base());
1037
1038 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
1039 if (address.index() == noreg) {
1040 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
1041 } else {
1042 __ add(tmp2, address.index(), address.base());
1043 if (address.disp() != 0) {
1044 __ addi(tmp2, tmp2, address.disp());
1045 }
1046 }
1047 __ srdi(tmp2, tmp2, CardTable::card_shift());
1048 __ li(R0, CardTable::dirty_card_val());
1049 __ stbx(R0, tmp2, tmp1);
1050 }
1051
1052 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1053 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1054
1055 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)), R16_thread);
1056 __ cmpdi(CR0, tmp, 0);
1057 // Branch to entry if not equal
1058 __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *entry());
1059 // This is were the slowpath stub will return to
1060 __ bind(*continuation());
1061 }
1062
1063 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1064 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1065 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1066
1067 __ bind(*entry());
1068
1069 // If we need to load ourselves, do it here.
1070 if (_do_load) {
1071 if (_narrow) {
1072 __ lwz(_obj, _addr.disp(), _addr.base());
1073 } else {
1074 __ ld(_obj, _addr.disp(), _addr.base());
1075 }
1076 }
1077
1078 // If the object is null, there is no point in applying barriers.
1079 maybe_far_jump_if_zero(masm, _obj);
1080
1081 // We need to make sure that loads done by callers survive across slow-path calls.
1082 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1084 if (!_do_load || needs_both_barriers) {
1085 preserve(_obj);
1086 }
1087
1088 // Go for barriers. Barriers can return straight to continuation, as long
1089 // as another barrier is not needed and we can reach the fastpath.
1090 if (needs_both_barriers) {
1091 keepalive(masm, nullptr);
1092 lrb(masm);
1093 } else if (_needs_keep_alive_barrier) {
1094 keepalive(masm, continuation());
1095 } else if (_needs_load_ref_barrier) {
1096 lrb(masm);
1097 } else {
1098 ShouldNotReachHere();
1099 }
1100 }
1101
1102 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1103 __ cmpdi(CR0, reg, 0);
1104 // Branch to continuation if equal
1105 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1106 }
1107
1108 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1109 const int gcstate_offset = in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING));
1110 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1111 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1112 Label L_through, L_slowpath;
1113
1114 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1115 if (_needs_load_ref_barrier) {
1116 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
1117 __ lbz(_tmp1, gcstate_offset, R16_thread);
1118 __ cmpdi(CR0, _tmp1, 0);
1119 __ beq(CR0, L_through);
1120 }
1121
1122 // Fast-path: put object into buffer.
1123 // If buffer is already full, go slow.
1124 __ ld(_tmp1, index_offset, R16_thread);
1125 __ cmpdi(CR0, _tmp1, 0);
1126 __ beq(CR0, L_slowpath);
1127 __ addi(_tmp1, _tmp1, -wordSize);
1128 __ std(_tmp1, index_offset, R16_thread);
1129 __ ld(_tmp2, buffer_offset, R16_thread);
1130
1131 // Store the object in queue.
1132 // If object is narrow, we need to decode it before inserting.
1133 if (_narrow) {
1134 __ add(_tmp2, _tmp2, _tmp1);
1135 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1136 __ stdx(decoded, _tmp2);
1137 } else {
1138 __ stdx(_obj, _tmp2, _tmp1);
1139 }
1148 // Slow-path: call runtime to handle.
1149 __ bind(L_slowpath);
1150
1151 {
1152 SaveLiveRegisters slr(&masm, this);
1153
1154 // Go to runtime and handle the rest there.
1155 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1156 }
1157
1158 if (L_done != nullptr) {
1159 __ b(*L_done);
1160 } else {
1161 __ bind(L_through);
1162 }
1163 }
1164
1165 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1166 Label L_slow;
1167
1168 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1169 if (_needs_keep_alive_barrier) {
1170 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1171 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)), R16_thread);
1172 maybe_far_jump_if_zero(masm, _tmp1);
1173 }
1174
1175 // If weak references are being processed, weak/phantom loads need to go slow,
1176 // regardless of their cset status.
1177 if (_needs_load_ref_weak_barrier) {
1178 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)), R16_thread);
1179 __ cmpdi(CR0, _tmp1, 0);
1180 __ bne(CR0, L_slow);
1181 }
1182
1183 // Cset-check. Fall-through to slow if in collection set.
1184 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1185 if (_narrow) {
1186 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1187 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1188 } else {
1189 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1190 }
1191 __ lbzx(_tmp2, _tmp2, _tmp1);
1192 maybe_far_jump_if_zero(masm, _tmp2);
1193
1194 // Slow path
1195 __ bind(L_slow);
1196
1197 // Obj is the result, need to temporarily stop preserving it.
1198 bool is_obj_preserved = is_preserved(_obj);
1199 if (is_obj_preserved) {
1200 dont_preserve(_obj);
|
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "gc/shared/gcArguments.hpp"
30 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahHeap.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
37 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "nativeInst_ppc.hpp"
42 #include "macroAssembler_ppc.hpp"
43 #include "runtime/javaThread.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "utilities/globalDefinitions.hpp"
46 #include "vm_version_ppc.hpp"
47 #ifdef COMPILER1
48 #include "c1/c1_LIRAssembler.hpp"
49 #include "c1/c1_MacroAssembler.hpp"
50 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
51 #endif
52 #ifdef COMPILER2
53 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
54 #endif
55
56 #define __ masm->
57
58 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
59 Register base, RegisterOrConstant ind_or_offs,
60 Register tmp1, Register tmp2, Register tmp3,
61 MacroAssembler::PreservationLevel preservation_level) {
1037 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1038 assert_different_registers(tmp1, tmp2, address.index(), address.base());
1039
1040 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
1041 if (address.index() == noreg) {
1042 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
1043 } else {
1044 __ add(tmp2, address.index(), address.base());
1045 if (address.disp() != 0) {
1046 __ addi(tmp2, tmp2, address.disp());
1047 }
1048 }
1049 __ srdi(tmp2, tmp2, CardTable::card_shift());
1050 __ li(R0, CardTable::dirty_card_val());
1051 __ stbx(R0, tmp2, tmp1);
1052 }
1053
1054 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1055 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1056
1057 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(test_state));
1058 __ b(*entry());
1059
1060 // This is were the slowpath stub will return to
1061 __ bind(*continuation());
1062 }
1063
1064 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
1065 NativeInstruction* ni = nativeInstruction_at(pc);
1066 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
1067 NativeGeneralJump* jmp = nativeGeneralJump_at(pc);
1068 return jmp->jump_destination();
1069 }
1070
1071 static void check_at(bool cond, address pc, const char* msg) {
1072 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x",
1073 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3));
1074 }
1075
1076 static bool is_nop(address pc) {
1077 if (*(pc + 0) != 0x00) return false;
1078 if (*(pc + 1) != 0x00) return false;
1079 if (*(pc + 2) != 0x00) return false;
1080 if (*(pc + 3) != 0x60) return false;
1081 return true;
1082 }
1083
1084 static void insert_nop(address pc) {
1085 *reinterpret_cast<int32_t*>(pc) = 0x60000000;
1086 check_at(is_nop(pc), pc, "Should be nop");
1087 ICache::invalidate_range(pc, 4);
1088 }
1089
1090 bool ShenandoahBarrierSetAssembler::is_active(address pc) {
1091 NativeInstruction* ni = nativeInstruction_at(pc);
1092 return ni->is_jump();
1093 }
1094
1095 void ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
1096 NativeInstruction* ni = nativeInstruction_at(pc);
1097 if (ni->is_jump()) {
1098 insert_nop(pc);
1099 } else {
1100 check_at(is_nop(pc), pc, "Should already be nop");
1101 }
1102 }
1103
1104 void ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
1105 NativeInstruction* ni = nativeInstruction_at(pc);
1106 if (is_nop(pc)) {
1107 NativeGeneralJump::insert_unconditional(pc, stub_addr);
1108 } else {
1109 check_at(ni->is_jump(), pc, "Should already be jump");
1110 check_at(nativeGeneralJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
1111 }
1112 }
1113
1114 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1115 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1116 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1117
1118 __ bind(*entry());
1119
1120 // If we need to load ourselves, do it here.
1121 if (_do_load) {
1122 if (_narrow) {
1123 __ lwz(_obj, _addr.disp(), _addr.base());
1124 } else {
1125 __ ld(_obj, _addr.disp(), _addr.base());
1126 }
1127 }
1128
1129 // If the object is null, there is no point in applying barriers.
1130 maybe_far_jump_if_zero(masm, _obj);
1131
1132 // We need to make sure that loads done by callers survive across slow-path calls.
1133 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1135 if (!_do_load || needs_both_barriers) {
1136 preserve(_obj);
1137 }
1138
1139 // Go for barriers. Barriers can return straight to continuation, as long
1140 // as another barrier is not needed and we can reach the fastpath.
1141 if (needs_both_barriers) {
1142 keepalive(masm, nullptr);
1143 lrb(masm);
1144 } else if (_needs_keep_alive_barrier) {
1145 keepalive(masm, continuation());
1146 } else if (_needs_load_ref_barrier) {
1147 lrb(masm);
1148 } else {
1149 ShouldNotReachHere();
1150 }
1151 }
1152
1153 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1154 __ cmpdi(CR0, reg, 0);
1155 // Branch to target if equal
1156 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1157 }
1158
1159 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1160 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1161 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1162 Label L_through, L_slowpath;
1163
1164 // If another barrier is enabled as well, do a check for a specific barrier.
1165 if (_needs_load_ref_barrier) {
1166 assert(L_done == nullptr, "Should be");
1167 // Emit the unconditional branch in the first version of the method.
1168 // Let the rest of runtime figure out how to manage it.
1169 // TODO: We could have spared the over-jump if patching knew we need the inverse branch.
1170 char state_to_check = ShenandoahHeap::MARKING;
1171 Label L_over;
1172 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
1173 __ b(L_over);
1174 __ b(L_through);
1175 __ bind(L_over);
1176 }
1177
1178 // Fast-path: put object into buffer.
1179 // If buffer is already full, go slow.
1180 __ ld(_tmp1, index_offset, R16_thread);
1181 __ cmpdi(CR0, _tmp1, 0);
1182 __ beq(CR0, L_slowpath);
1183 __ addi(_tmp1, _tmp1, -wordSize);
1184 __ std(_tmp1, index_offset, R16_thread);
1185 __ ld(_tmp2, buffer_offset, R16_thread);
1186
1187 // Store the object in queue.
1188 // If object is narrow, we need to decode it before inserting.
1189 if (_narrow) {
1190 __ add(_tmp2, _tmp2, _tmp1);
1191 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1192 __ stdx(decoded, _tmp2);
1193 } else {
1194 __ stdx(_obj, _tmp2, _tmp1);
1195 }
1204 // Slow-path: call runtime to handle.
1205 __ bind(L_slowpath);
1206
1207 {
1208 SaveLiveRegisters slr(&masm, this);
1209
1210 // Go to runtime and handle the rest there.
1211 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1212 }
1213
1214 if (L_done != nullptr) {
1215 __ b(*L_done);
1216 } else {
1217 __ bind(L_through);
1218 }
1219 }
1220
1221 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1222 Label L_slow;
1223
1224 // If weak references are being processed, weak/phantom loads need to go slow,
1225 // regardless of their cset status.
1226 if (_needs_load_ref_weak_barrier) {
1227 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
1228 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
1229 __ b(L_slow);
1230 }
1231
1232 if (_needs_keep_alive_barrier) {
1233 // Emit the unconditional branch in the first version of the method.
1234 // Let the rest of runtime figure out how to manage it.
1235 // TODO: We could have spared the over-jump if patching knew we need the inverse branch.
1236 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1237 Label L_over;
1238 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
1239 __ b(L_over);
1240 __ b(*continuation());
1241 __ bind(L_over);
1242 }
1243
1244 // Cset-check. Fall-through to slow if in collection set.
1245 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1246 if (_narrow) {
1247 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1248 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1249 } else {
1250 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1251 }
1252 __ lbzx(_tmp2, _tmp2, _tmp1);
1253 maybe_far_jump_if_zero(masm, _tmp2);
1254
1255 // Slow path
1256 __ bind(L_slow);
1257
1258 // Obj is the result, need to temporarily stop preserving it.
1259 bool is_obj_preserved = is_preserved(_obj);
1260 if (is_obj_preserved) {
1261 dont_preserve(_obj);
|