25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
49 if (handle_gpr) {
50 __ push_IU_state();
51 }
52
53 if (handle_fp) {
54 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
55 assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call");
56
57 const int xmm_size = wordSize * 2;
58 __ subptr(rsp, xmm_size * 8);
59 __ movdbl(Address(rsp, xmm_size * 0), xmm0);
60 __ movdbl(Address(rsp, xmm_size * 1), xmm1);
61 __ movdbl(Address(rsp, xmm_size * 2), xmm2);
62 __ movdbl(Address(rsp, xmm_size * 3), xmm3);
63 __ movdbl(Address(rsp, xmm_size * 4), xmm4);
64 __ movdbl(Address(rsp, xmm_size * 5), xmm5);
996 assert(!is_native, "weak must not be called off-heap");
997 if (UseCompressedOops) {
998 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
999 } else {
1000 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
1001 }
1002 } else {
1003 assert(is_phantom, "only remaining strength");
1004 assert(is_native, "phantom must only be called off-heap");
1005 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
1006 }
1007
1008 __ restore_live_registers_except_rax(true);
1009
1010 __ epilogue();
1011 }
1012
1013 #undef __
1014
1015 #endif // COMPILER1
|
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #endif
48
49 #define __ masm->
50
51 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
52 if (handle_gpr) {
53 __ push_IU_state();
54 }
55
56 if (handle_fp) {
57 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
58 assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call");
59
60 const int xmm_size = wordSize * 2;
61 __ subptr(rsp, xmm_size * 8);
62 __ movdbl(Address(rsp, xmm_size * 0), xmm0);
63 __ movdbl(Address(rsp, xmm_size * 1), xmm1);
64 __ movdbl(Address(rsp, xmm_size * 2), xmm2);
65 __ movdbl(Address(rsp, xmm_size * 3), xmm3);
66 __ movdbl(Address(rsp, xmm_size * 4), xmm4);
67 __ movdbl(Address(rsp, xmm_size * 5), xmm5);
999 assert(!is_native, "weak must not be called off-heap");
1000 if (UseCompressedOops) {
1001 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
1002 } else {
1003 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
1004 }
1005 } else {
1006 assert(is_phantom, "only remaining strength");
1007 assert(is_native, "phantom must only be called off-heap");
1008 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
1009 }
1010
1011 __ restore_live_registers_except_rax(true);
1012
1013 __ epilogue();
1014 }
1015
1016 #undef __
1017
1018 #endif // COMPILER1
1019
1020 #ifdef COMPILER2
1021
1022 #undef __
1023 #define __ masm->
1024
1025 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, bool narrow) {
1026 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
1027 if (narrow) {
1028 __ movl(dst, src);
1029 } else {
1030 __ movq(dst, src);
1031 }
1032
1033 // Post-barrier: LRB
1034 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1035 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, false);
1036 char check = 0;
1037 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1038 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1039 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
1040 stub->enter_if_gc_state(*masm, check);
1041 }
1042 }
1043
1044 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
1045 Address dst, bool dst_narrow,
1046 Register src, bool src_narrow,
1047 Register tmp) {
1048
1049 // Pre-barrier: SATB, keep-alive the current memory value.
1050 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1051 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier(node), "Should not be required for stores");
1052 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, dst, dst_narrow, true);
1053 stub->enter_if_gc_state(*masm, ShenandoahHeap::MARKING);
1054 }
1055
1056 // Need to encode into tmp, because we cannot clobber src.
1057 // TODO: Maybe there is a matcher way to test that src is unused after this?
1058 if (dst_narrow && !src_narrow) {
1059 __ movq(tmp, src);
1060 if (ShenandoahBarrierStubC2::maybe_null(node)) {
1061 __ encode_heap_oop(tmp);
1062 } else {
1063 __ encode_heap_oop_not_null(tmp);
1064 }
1065 src = tmp;
1066 }
1067
1068 // Do the actual store
1069 if (dst_narrow) {
1070 __ movl(dst, src);
1071 } else {
1072 __ movq(dst, src);
1073 }
1074
1075 // Post-barrier: card updates.
1076 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1077 card_barrier_c2(masm, dst, tmp);
1078 }
1079 }
1080
1081 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm,
1082 Register res, Address addr,
1083 Register oldval, Register newval, Register tmp,
1084 bool narrow) {
1085
1086 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
1087
1088 // Oldval and newval can be in the same register, but all other registers should be
1089 // distinct for extra safety, as we shuffle register values around.
1090 assert_different_registers(oldval, tmp, addr.base(), addr.index());
1091 assert_different_registers(newval, tmp, addr.base(), addr.index());
1092
1093 // Pre-barrier covers several things:
1094 // a. Avoids false positives from CAS encountering to-space memory values.
1095 // b. Satisfies the need for LRB for the CAE result.
1096 // c. Records old value for the sake of SATB.
1097 //
1098 // (a) and (b) are covered because load barrier does memory location fixup.
1099 // (c) is covered by KA on the current memory value.
1100 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1101 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, true);
1102 char check = 0;
1103 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1104 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1105 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS");
1106 stub->enter_if_gc_state(*masm, check);
1107 }
1108
1109 // CAS!
1110 __ lock();
1111 if (narrow) {
1112 __ cmpxchgl(newval, addr);
1113 } else {
1114 __ cmpxchgptr(newval, addr);
1115 }
1116
1117 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
1118 if (res != noreg) {
1119 __ setcc(Assembler::equal, res);
1120 }
1121
1122 // Post-barrier deals with card updates.
1123 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1124 card_barrier_c2(masm, addr, tmp);
1125 }
1126 }
1127
1128 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register newval, Address addr, Register tmp, bool narrow) {
1129 assert_different_registers(newval, tmp, addr.base(), addr.index());
1130
1131 // Pre-barrier covers several things:
1132 // a. Satisfies the need for LRB for the GAS result.
1133 // b. Records old value for the sake of SATB.
1134 //
1135 // (a) is covered because load barrier does memory location fixup.
1136 // (b) is covered by KA on the current memory value.
1137 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1138 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, true);
1139 char check = 0;
1140 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1141 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1142 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for GAS");
1143 stub->enter_if_gc_state(*masm, check);
1144 }
1145
1146 if (narrow) {
1147 __ xchgl(newval, addr);
1148 } else {
1149 __ xchgq(newval, addr);
1150 }
1151
1152 // Post-barrier deals with card updates.
1153 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1154 card_barrier_c2(masm, addr, tmp);
1155 }
1156 }
1157
1158 void ShenandoahBarrierSetAssembler::card_barrier_c2(MacroAssembler* masm, Address dst, Register tmp) {
1159 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
1160
1161 // TODO: Might be a good place to implement some filters here.
1162 // For example, G1 only flips card marks for stores within a single region.
1163
1164 __ lea(tmp, dst);
1165 __ shrptr(tmp, CardTable::card_shift());
1166 __ addptr(tmp, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
1167 Address card_address(tmp, 0);
1168
1169 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
1170 Label L_done;
1171 if (UseCondCardMark) {
1172 __ cmpb(card_address, 0);
1173 __ jccb(Assembler::equal, L_done);
1174 }
1175 if (UseCompressedOops && CompressedOops::base() == nullptr) {
1176 __ movb(card_address, r12);
1177 } else {
1178 __ movb(card_address, 0);
1179 }
1180 __ bind(L_done);
1181 }
1182
1183 #undef __
1184 #define __ masm.
1185
1186 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
1187 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1188
1189 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
1190 __ testb(gc_state_fast, ShenandoahThreadLocalData::gc_state_to_fast(test_state));
1191 __ jcc(Assembler::notZero, *entry());
1192 __ bind(*continuation());
1193 }
1194
1195 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1196 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1197
1198 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1199
1200 __ align(InteriorEntryAlignment);
1201 __ bind(*entry());
1202
1203 // If we need to load ourselves, do it here.
1204 if (_do_load) {
1205 if (_narrow) {
1206 __ movl(_obj, _addr);
1207 } else {
1208 __ movq(_obj, _addr);
1209 }
1210 }
1211
1212 // If the object is null, there is no point in applying barriers.
1213 if (_narrow) {
1214 __ testl(_obj, _obj);
1215 } else {
1216 __ testptr(_obj, _obj);
1217 }
1218 __ jcc(Assembler::zero, *continuation());
1219
1220 // Barriers need temp to work, allocate one now.
1221 bool tmp_live;
1222 Register tmp = select_temp_register(tmp_live, _addr, _obj);
1223 if (tmp_live) {
1224 push_save_register(masm, tmp);
1225 }
1226
1227 // Go for barriers. Barriers can return straight to continuation, as long
1228 // as another barrier is not needed and tmp is not live.
1229 if (_needs_keep_alive_barrier) {
1230 keepalive(masm, _obj, tmp, (_needs_load_ref_barrier || tmp_live) ? nullptr : continuation());
1231 }
1232 if (_needs_load_ref_barrier) {
1233 lrb(masm, _obj, _addr, tmp, (tmp_live) ? nullptr : continuation());
1234 }
1235
1236 if (tmp_live) {
1237 pop_save_register(masm, tmp);
1238 __ jmp(*continuation());
1239 } else {
1240 #ifdef ASSERT
1241 __ hlt(); // Should not reach here
1242 #endif
1243 }
1244 }
1245
1246 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Register obj, Register tmp, Label* L_done) {
1247 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1248 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1249
1250 Label L_through, L_fast, L_pack_and_done;
1251
1252 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1253 if (_needs_load_ref_barrier) {
1254 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1255 __ testb(gc_state, ShenandoahHeap::MARKING);
1256 __ jcc(Assembler::zero, (L_done != nullptr) ? *L_done : L_through);
1257 }
1258
1259 // If object is narrow, we need to unpack it before inserting into buffer.
1260 if (_narrow) {
1261 __ decode_heap_oop_not_null(obj);
1262 }
1263
1264 // Check if buffer is already full. Go slow, if so.
1265 __ movptr(tmp, index);
1266 __ testptr(tmp, tmp);
1267 __ jcc(Assembler::notZero, L_fast);
1268
1269 // Slow-path: call runtime to handle.
1270 {
1271 // Shuffle in the arguments. The end result should be:
1272 // c_rarg0 <-- obj
1273 //
1274 // Save clobbered registers before overwriting them.
1275 bool clobbered_c_rarg0 = false;
1276 if (c_rarg0 != obj) {
1277 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1278 __ mov(c_rarg0, obj);
1279 }
1280
1281 // Handle the rest there.
1282 if (!ShenandoahFasterRuntimeStubs || has_live_vector_registers()) {
1283 __ call(RuntimeAddress(keepalive_runtime_entry_addr(SaveMode::All)));
1284 } else if (has_save_space_for_live_gp_registers(clobbered_c_rarg0, false, false)) {
1285 save_live_gp_regs(masm, clobbered_c_rarg0, false, false);
1286 __ call(RuntimeAddress(keepalive_runtime_entry_addr(SaveMode::Nothing)));
1287 restore_live_gp_regs(masm, clobbered_c_rarg0, false, false);
1288 } else {
1289 __ call(RuntimeAddress(keepalive_runtime_entry_addr(SaveMode::GP)));
1290 }
1291
1292 // Restore the clobbered registers.
1293 if (clobbered_c_rarg0) {
1294 pop_save_register(masm, c_rarg0);
1295 }
1296 __ jmpb(L_pack_and_done);
1297 }
1298
1299 // Fast-path: put object into buffer.
1300 __ bind(L_fast);
1301 __ subptr(tmp, wordSize);
1302 __ movptr(index, tmp);
1303 __ addptr(tmp, buffer);
1304 __ movptr(Address(tmp, 0), obj);
1305
1306 // Exit here.
1307 __ bind(L_pack_and_done);
1308
1309 // Pack the object back if needed. We can skip this if we performed
1310 // the load ourselves: the value is not used by the caller.
1311 if (_narrow && !_do_load) {
1312 __ encode_heap_oop_not_null(obj);
1313 }
1314 if (L_done != nullptr) {
1315 __ jmp(*L_done);
1316 } else {
1317 // Fall-through
1318 __ bind(L_through);
1319 }
1320 }
1321
1322 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm, Register obj, Address addr, Register tmp, Label* L_done) {
1323 Label L_through, L_slow;
1324
1325 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1326 if (_needs_keep_alive_barrier) {
1327 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1328 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
1329 __ jcc(Assembler::zero, (L_done != nullptr) ? *L_done : L_through);
1330 }
1331
1332 // If weak references are being processed, weak/phantom loads need to go slow,
1333 // regadless of their cset status.
1334 if (_needs_load_ref_weak_barrier) {
1335 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1336 __ testb(gc_state, ShenandoahHeap::WEAK_ROOTS);
1337 __ jccb(Assembler::notZero, L_slow);
1338 }
1339
1340 // Compute the cset bitmap index
1341 if (_narrow) {
1342 __ decode_heap_oop_not_null(tmp, obj);
1343 } else {
1344 __ movptr(tmp, obj);
1345 }
1346 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1347
1348 // If cset address is in good spot to just use it as offset. It almost always is.
1349 Address cset_addr_arg;
1350 intptr_t cset_addr = (intptr_t) ShenandoahHeap::in_cset_fast_test_addr();
1351 if ((cset_addr >> 3) < INT32_MAX) {
1352 assert(is_aligned(cset_addr, 8), "Sanity");
1353 cset_addr_arg = Address(tmp, checked_cast<int>(cset_addr >> 3), Address::times_8);
1354 } else {
1355 __ addptr(tmp, cset_addr);
1356 cset_addr_arg = Address(tmp, 0);
1357 }
1358
1359 // Cset-check. Fall-through to slow if in collection set.
1360 __ cmpb(cset_addr_arg, 0);
1361 __ jcc(Assembler::equal, (L_done != nullptr) ? *L_done : L_through);
1362
1363 // Slow path
1364 __ bind(L_slow);
1365 {
1366 assert_different_registers(rax, c_rarg0, c_rarg1);
1367
1368 // Shuffle in the arguments. The end result should be:
1369 // c_rarg0 <-- obj
1370 // c_rarg1 <-- lea(addr)
1371 //
1372 // Save clobbered registers before overwriting them, unless they
1373 // carry obj, which would be overwritten on return.
1374 bool clobbered_c_rarg0 = false;
1375 bool clobbered_c_rarg1 = false;
1376 bool clobbered_rax = false;
1377
1378 if (obj == c_rarg0) {
1379 clobbered_c_rarg1 = push_save_register_if_live(masm, c_rarg1);
1380 __ lea(c_rarg1, addr);
1381 } else if (obj == c_rarg1) {
1382 // Set up arguments in reverse, and then flip them
1383 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1384 __ lea(c_rarg0, addr);
1385 __ xchgptr(c_rarg0, c_rarg1);
1386 } else {
1387 assert_different_registers(obj, c_rarg0, c_rarg1);
1388 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1389 clobbered_c_rarg1 = push_save_register_if_live(masm, c_rarg1);
1390 __ lea(c_rarg1, addr);
1391 __ movptr(c_rarg0, obj);
1392 }
1393 if (obj != rax) {
1394 clobbered_rax = push_save_register_if_live(masm, rax);
1395 }
1396
1397 // Decode if needed.
1398 if (_narrow) {
1399 __ decode_heap_oop_not_null(c_rarg0);
1400 }
1401
1402 // Go to runtime stub and handle the rest there.
1403 if (!ShenandoahFasterRuntimeStubs || has_live_vector_registers()) {
1404 __ call(RuntimeAddress(lrb_runtime_entry_addr(SaveMode::All)));
1405 } else if (has_save_space_for_live_gp_registers(clobbered_c_rarg0, clobbered_c_rarg1, true)) {
1406 save_live_gp_regs(masm, clobbered_c_rarg0, clobbered_c_rarg1, true);
1407 __ call(RuntimeAddress(lrb_runtime_entry_addr(SaveMode::Nothing)));
1408 restore_live_gp_regs(masm, clobbered_c_rarg0, clobbered_c_rarg1, true);
1409 } else {
1410 __ call(RuntimeAddress(lrb_runtime_entry_addr(SaveMode::GP)));
1411 }
1412
1413 // Save the result where needed and restore the clobbered registers.
1414 if (obj != rax) {
1415 __ movptr(obj, rax);
1416 }
1417 // If object is narrow, we need to encode it before exiting.
1418 // For encoding, dst can only turn null if we are dealing with weak loads.
1419 // Otherwise, we have already null-checked. We can skip this if we performed
1420 // the load ourselves: the value is not used by the caller.
1421 if (_narrow && !_do_load) {
1422 if (_needs_load_ref_weak_barrier) {
1423 __ encode_heap_oop(obj);
1424 } else {
1425 __ encode_heap_oop_not_null(obj);
1426 }
1427 }
1428 if (clobbered_rax) {
1429 pop_save_register(masm, rax);
1430 }
1431 if (clobbered_c_rarg1) {
1432 pop_save_register(masm, c_rarg1);
1433 }
1434 if (clobbered_c_rarg0) {
1435 pop_save_register(masm, c_rarg0);
1436 }
1437 }
1438
1439 // Exit here
1440 if (L_done != nullptr) {
1441 __ jmp(*L_done);
1442 } else {
1443 // Fall-through.
1444 __ bind(L_through);
1445 }
1446 }
1447
1448 bool ShenandoahBarrierStubC2::has_live_vector_registers() {
1449 return _has_live_vector_registers;
1450 }
1451
1452 bool ShenandoahBarrierStubC2::is_live(Register reg) {
1453 return _live_gp.contains(reg);
1454 }
1455
1456 bool ShenandoahBarrierStubC2::has_save_space_for_live_gp_registers(bool skip_crarg0, bool skip_crarg1, bool skip_rax) {
1457 int c = 0;
1458 for (int i = 0; i < _live_gp.length(); i++) {
1459 Register r = _live_gp.at(i);
1460 if (skip_rax && (r == rax)) continue;
1461 if (skip_crarg0 && (r == c_rarg0)) continue;
1462 if (skip_crarg1 && (r == c_rarg1)) continue;
1463 c++;
1464 }
1465 return c <= (ShenandoahBarrierSetC2::bsc2()->reserved_slots() - _save_slots_idx);
1466 }
1467
1468 void ShenandoahBarrierStubC2::save_live_gp_regs(MacroAssembler& masm, bool skip_crarg0, bool skip_crarg1, bool skip_rax) {
1469 for (int i = 0; i < _live_gp.length(); i++) {
1470 Register r = _live_gp.at(i);
1471 if (skip_rax && (r == rax)) continue;
1472 if (skip_crarg0 && (r == c_rarg0)) continue;
1473 if (skip_crarg1 && (r == c_rarg1)) continue;
1474 push_save_register(masm, r);
1475 }
1476 }
1477
1478 void ShenandoahBarrierStubC2::restore_live_gp_regs(MacroAssembler& masm, bool skip_crarg0, bool skip_crarg1, bool skip_rax) {
1479 for (int i = _live_gp.length() - 1; i >= 0; i--) {
1480 Register r = _live_gp.at(i);
1481 if (skip_rax && (r == rax)) continue;
1482 if (skip_crarg0 && (r == c_rarg0)) continue;
1483 if (skip_crarg1 && (r == c_rarg1)) continue;
1484 pop_save_register(masm, r);
1485 }
1486 }
1487
1488 bool ShenandoahBarrierStubC2::push_save_register_if_live(MacroAssembler& masm, Register reg) {
1489 if (is_live(reg)) {
1490 push_save_register(masm, reg);
1491 return true;
1492 } else {
1493 return false;
1494 }
1495 }
1496
1497 void ShenandoahBarrierStubC2::push_save_register(MacroAssembler& masm, Register reg) {
1498 __ movptr(Address(rsp, push_save_slot()), reg);
1499 }
1500
1501 void ShenandoahBarrierStubC2::pop_save_register(MacroAssembler& masm, Register reg) {
1502 __ movptr(reg, Address(rsp, pop_save_slot()));
1503 }
1504
1505 Register ShenandoahBarrierStubC2::select_temp_register(bool& selected_live, Address addr, Register reg1) {
1506 Register tmp = noreg;
1507 Register fallback_live = noreg;
1508
1509 // Try to select non-live first:
1510 for (int i = 0; i < Register::available_gp_registers(); i++) {
1511 Register r = as_Register(i);
1512 if (r != rsp && r != rbp && r != r12_heapbase && r != r15_thread &&
1513 r != reg1 && r != addr.base() && r != addr.index()) {
1514 if (!is_live(r)) {
1515 tmp = r;
1516 break;
1517 } else if (fallback_live == noreg) {
1518 fallback_live = r;
1519 }
1520 }
1521 }
1522
1523 // If we could not find a non-live register, select the live fallback:
1524 if (tmp == noreg) {
1525 tmp = fallback_live;
1526 selected_live = true;
1527 } else {
1528 selected_live = false;
1529 }
1530
1531 assert(tmp != noreg, "successfully selected");
1532 assert_different_registers(tmp, reg1);
1533 assert_different_registers(tmp, addr.base());
1534 assert_different_registers(tmp, addr.index());
1535 return tmp;
1536 }
1537
1538 void ShenandoahBarrierStubC2::post_init(int offset) {
1539 // Precompute live registers.
1540 assert(_live_gp.is_empty(), "sanity: initial state");
1541 assert(!_has_live_vector_registers, "sanity: initial state");
1542 RegMaskIterator rmi(preserve_set());
1543 while (rmi.has_next()) {
1544 const OptoReg::Name opto_reg = rmi.next();
1545 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
1546 if (vm_reg->is_Register()) {
1547 Register r = vm_reg->as_Register();
1548 if (r == rsp) continue;
1549 _live_gp.append_if_missing(r);
1550 } else if (vm_reg->is_KRegister()) {
1551 _has_live_vector_registers = true;
1552 } else if (vm_reg->is_XMMRegister()) {
1553 _has_live_vector_registers = true;
1554 } else {
1555 fatal("Unexpected register type");
1556 }
1557 }
1558 }
1559 #undef __
1560 #endif
|