25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
49 Register src, Register dst, Register count) {
50
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52
53 if (is_reference_type(type)) {
54 if (ShenandoahCardBarrier) {
55 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
56 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
57 bool obj_int = (type == T_OBJECT) && UseCompressedOops;
58
59 // We need to save the original element count because the array copy stub
60 // will destroy the value and we need it for the card marking barrier.
61 if (!checkcast) {
62 if (!obj_int) {
63 // Save count for barrier
64 __ movptr(r11, count);
884 assert(!is_native, "weak must not be called off-heap");
885 if (UseCompressedOops) {
886 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
887 } else {
888 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
889 }
890 } else {
891 assert(is_phantom, "only remaining strength");
892 assert(is_native, "phantom must only be called off-heap");
893 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
894 }
895
896 __ restore_live_registers_except_rax(true);
897
898 __ epilogue();
899 }
900
901 #undef __
902
903 #endif // COMPILER1
|
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #endif
48
49 #define __ masm->
50
51 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
52 Register src, Register dst, Register count) {
53
54 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
55
56 if (is_reference_type(type)) {
57 if (ShenandoahCardBarrier) {
58 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
59 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
60 bool obj_int = (type == T_OBJECT) && UseCompressedOops;
61
62 // We need to save the original element count because the array copy stub
63 // will destroy the value and we need it for the card marking barrier.
64 if (!checkcast) {
65 if (!obj_int) {
66 // Save count for barrier
67 __ movptr(r11, count);
887 assert(!is_native, "weak must not be called off-heap");
888 if (UseCompressedOops) {
889 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
890 } else {
891 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
892 }
893 } else {
894 assert(is_phantom, "only remaining strength");
895 assert(is_native, "phantom must only be called off-heap");
896 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
897 }
898
899 __ restore_live_registers_except_rax(true);
900
901 __ epilogue();
902 }
903
904 #undef __
905
906 #endif // COMPILER1
907
908 #ifdef COMPILER2
909
910 #undef __
911 #define __ masm->
912
913 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, bool narrow) {
914 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
915 if (narrow) {
916 __ movl(dst, src);
917 } else {
918 __ movq(dst, src);
919 }
920
921 // Post-barrier: LRB
922 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
923 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, false);
924 char check = 0;
925 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
926 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
927 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
928 stub->enter_if_gc_state(*masm, check);
929 }
930 }
931
932 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
933 Address dst, bool dst_narrow,
934 Register src, bool src_narrow,
935 Register tmp) {
936
937 // Pre-barrier: SATB, keep-alive the current memory value.
938 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
939 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier(node), "Should not be required for stores");
940 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, dst, dst_narrow, true);
941 stub->enter_if_gc_state(*masm, ShenandoahHeap::MARKING);
942 }
943
944 // Need to encode into tmp, because we cannot clobber src.
945 if (dst_narrow && !src_narrow) {
946 __ movq(tmp, src);
947 if (ShenandoahBarrierStubC2::maybe_null(node)) {
948 __ encode_heap_oop(tmp);
949 } else {
950 __ encode_heap_oop_not_null(tmp);
951 }
952 src = tmp;
953 }
954
955 // Do the actual store
956 if (dst_narrow) {
957 __ movl(dst, src);
958 } else {
959 __ movq(dst, src);
960 }
961
962 // Post-barrier: card updates.
963 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
964 card_barrier_c2(masm, dst, tmp);
965 }
966 }
967
968 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm,
969 Register res, Address addr,
970 Register oldval, Register newval, Register tmp,
971 bool narrow) {
972
973 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
974
975 // Oldval and newval can be in the same register, but all other registers should be
976 // distinct for extra safety, as we shuffle register values around.
977 assert_different_registers(oldval, tmp, addr.base(), addr.index());
978 assert_different_registers(newval, tmp, addr.base(), addr.index());
979
980 // Pre-barrier covers several things:
981 // a. Avoids false positives from CAS encountering to-space memory values.
982 // b. Satisfies the need for LRB for the CAE result.
983 // c. Records old value for the sake of SATB.
984 //
985 // (a) and (b) are covered because load barrier does memory location fixup.
986 // (c) is covered by KA on the current memory value.
987 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
988 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, true);
989 char check = 0;
990 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
991 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
992 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS");
993 stub->enter_if_gc_state(*masm, check);
994 }
995
996 // CAS!
997 __ lock();
998 if (narrow) {
999 __ cmpxchgl(newval, addr);
1000 } else {
1001 __ cmpxchgptr(newval, addr);
1002 }
1003
1004 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
1005 if (res != noreg) {
1006 __ setcc(Assembler::equal, res);
1007 }
1008
1009 // Post-barrier deals with card updates.
1010 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1011 card_barrier_c2(masm, addr, tmp);
1012 }
1013 }
1014
1015 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register newval, Address addr, Register tmp, bool narrow) {
1016 assert_different_registers(newval, tmp, addr.base(), addr.index());
1017
1018 // Pre-barrier covers several things:
1019 // a. Satisfies the need for LRB for the GAS result.
1020 // b. Records old value for the sake of SATB.
1021 //
1022 // (a) is covered because load barrier does memory location fixup.
1023 // (b) is covered by KA on the current memory value.
1024 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1025 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, true);
1026 char check = 0;
1027 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1028 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1029 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for GAS");
1030 stub->enter_if_gc_state(*masm, check);
1031 }
1032
1033 if (narrow) {
1034 __ xchgl(newval, addr);
1035 } else {
1036 __ xchgq(newval, addr);
1037 }
1038
1039 // Post-barrier deals with card updates.
1040 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1041 card_barrier_c2(masm, addr, tmp);
1042 }
1043 }
1044
1045 void ShenandoahBarrierSetAssembler::card_barrier_c2(MacroAssembler* masm, Address dst, Register tmp) {
1046 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
1047
1048 __ lea(tmp, dst);
1049 __ shrptr(tmp, CardTable::card_shift());
1050 __ addptr(tmp, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
1051 Address card_address(tmp, 0);
1052
1053 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
1054 Label L_done;
1055 if (UseCondCardMark) {
1056 __ cmpb(card_address, 0);
1057 __ jccb(Assembler::equal, L_done);
1058 }
1059 if (UseCompressedOops && CompressedOops::base() == nullptr) {
1060 __ movb(card_address, r12);
1061 } else {
1062 __ movb(card_address, 0);
1063 }
1064 __ bind(L_done);
1065 }
1066
1067 #undef __
1068 #define __ masm.
1069
1070 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1071 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1072
1073 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
1074 __ cmpb(gc_state_fast, 0);
1075 __ jcc(Assembler::notEqual, *entry());
1076 __ bind(*continuation());
1077 }
1078
1079 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1080 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1081
1082 __ align(InteriorEntryAlignment);
1083 __ bind(*entry());
1084
1085 // If we need to load ourselves, do it here.
1086 if (_do_load) {
1087 if (_narrow) {
1088 __ movl(_obj, _addr);
1089 } else {
1090 __ movq(_obj, _addr);
1091 }
1092 }
1093
1094 // If the object is null, there is no point in applying barriers.
1095 if (_narrow) {
1096 __ testl(_obj, _obj);
1097 } else {
1098 __ testq(_obj, _obj);
1099 }
1100 __ jcc(Assembler::zero, *continuation());
1101
1102 // We need to make sure that loads done by callers survive across slow-path calls.
1103 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1104 if (!_do_load || (_needs_keep_alive_barrier && _needs_load_ref_barrier)) {
1105 preserve(_obj);
1106 }
1107
1108 // Go for barriers. If both KA and LRB are needed (rare), do additional gc-state
1109 // checks to verify which one is currently needed. Note that KA and LRB are *not*
1110 // exclusive, because we can have an overlapping marking/evac in generational mode.
1111 if (_needs_keep_alive_barrier && _needs_load_ref_barrier) {
1112 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1113
1114 Label L_skip_keepalive;
1115 __ testb(gc_state, ShenandoahHeap::MARKING);
1116 __ jcc(Assembler::zero, L_skip_keepalive);
1117 keepalive(masm, nullptr);
1118 __ bind(L_skip_keepalive);
1119
1120 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
1121 __ jcc(Assembler::zero, *continuation());
1122 lrb(masm, continuation());
1123 } else if (_needs_keep_alive_barrier) {
1124 keepalive(masm, continuation());
1125 } else if (_needs_load_ref_barrier) {
1126 lrb(masm, continuation());
1127 } else {
1128 ShouldNotReachHere();
1129 }
1130 }
1131
1132 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1133 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1134 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1135
1136 Label L_pop_and_slow;
1137
1138 // Need temp to work, allocate one now.
1139 bool tmp_live;
1140 Register tmp = select_temp_register(tmp_live);
1141 if (tmp_live) {
1142 __ push(tmp);
1143 }
1144
1145 // Fast-path: put object into buffer.
1146 // If buffer is already full, go slow.
1147 __ movptr(tmp, index);
1148 __ subptr(tmp, wordSize);
1149 __ jccb(Assembler::below, L_pop_and_slow);
1150 __ movptr(index, tmp);
1151 __ addptr(tmp, buffer);
1152
1153 // If object is narrow, we need to unpack it before inserting into buffer,
1154 // and pack it back. We can skip the unpack if we know that object is not preserved.
1155 if (_narrow) {
1156 __ decode_heap_oop_not_null(_obj);
1157 }
1158 __ movptr(Address(tmp, 0), _obj);
1159 if (_narrow && is_preserved(_obj)) {
1160 __ encode_heap_oop_not_null(_obj);
1161 }
1162
1163 // Fast-path exits here.
1164 if (tmp_live) {
1165 __ pop(tmp);
1166 }
1167
1168 Label L_fallthrough;
1169 if (L_done != nullptr) {
1170 __ jmp(*L_done);
1171 } else {
1172 __ jmp(L_fallthrough);
1173 }
1174
1175 // Slow-path: call runtime to handle.
1176 // Need to pop tmp immediately for stack to remain aligned.
1177 __ bind(L_pop_and_slow);
1178 if (tmp_live) {
1179 __ pop(tmp);
1180 }
1181 {
1182 SaveLiveRegisters slr(&masm, this);
1183
1184 // Shuffle in the arguments. The end result should be:
1185 // c_rarg0 <-- obj
1186 if (c_rarg0 != _obj) {
1187 __ mov(c_rarg0, _obj);
1188 }
1189
1190 // Go to runtime and handle the rest there.
1191 __ call(RuntimeAddress(keepalive_runtime_entry_addr()));
1192 }
1193 if (L_done != nullptr) {
1194 __ jmp(*L_done);
1195 } else {
1196 __ bind(L_fallthrough);
1197 }
1198 }
1199
1200 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm, Label* L_done) {
1201 assert(L_done != nullptr, "Must be set");
1202
1203 Label L_pop_and_slow, L_slow;
1204
1205 // If weak references are being processed, weak/phantom loads need to go slow,
1206 // regardless of their cset status.
1207 if (_needs_load_ref_weak_barrier) {
1208 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1209 __ testb(gc_state, ShenandoahHeap::WEAK_ROOTS);
1210 __ jccb(Assembler::notZero, L_slow);
1211 }
1212
1213 // Need temp to work, allocate one now.
1214 bool tmp_live;
1215 Register tmp = select_temp_register(tmp_live);
1216 if (tmp_live) {
1217 __ push(tmp);
1218 }
1219
1220 // Compute the cset bitmap index
1221 if (_narrow) {
1222 __ decode_heap_oop_not_null(tmp, _obj);
1223 } else {
1224 __ movptr(tmp, _obj);
1225 }
1226 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1227
1228 Address cset_addr_arg;
1229 intptr_t cset_addr = reinterpret_cast<intptr_t>(ShenandoahHeap::in_cset_fast_test_addr());
1230 if ((cset_addr >> 3) < INT32_MAX) {
1231 // Cset bitmap is at easily encodeable address. Just use it as offset.
1232 assert(is_aligned(cset_addr, 8), "Sanity");
1233 cset_addr_arg = Address(tmp, cset_addr >> 3, Address::times_8);
1234 } else {
1235 // Cset bitmap is way further than our encoding limit. Add its address fully.
1236 bool tmp2_live;
1237 Register tmp2 = select_temp_register(tmp2_live, /* skip_reg1 = */ tmp);
1238 if (tmp2_live) {
1239 __ push(tmp2);
1240 }
1241 __ movptr(tmp2, cset_addr);
1242 __ addptr(tmp, tmp2);
1243 if (tmp2_live) {
1244 __ pop(tmp2);
1245 }
1246 cset_addr_arg = Address(tmp, 0);
1247 }
1248
1249 // Cset-check. Fall-through to slow if in collection set.
1250 __ cmpb(cset_addr_arg, 0);
1251 if (tmp_live) {
1252 __ jccb(Assembler::notEqual, L_pop_and_slow);
1253 __ pop(tmp);
1254 __ jmp(*L_done);
1255 } else {
1256 // Nothing else to do, jump back
1257 __ jcc(Assembler::equal, *L_done);
1258 }
1259
1260 // Slow path
1261 __ bind(L_pop_and_slow);
1262 // Need to pop tmp immediately for stack to remain aligned.
1263 if (tmp_live) {
1264 __ pop(tmp);
1265 }
1266 __ bind(L_slow);
1267
1268 // Obj is the result, need to temporarily stop preserving it.
1269 bool is_obj_preserved = is_preserved(_obj);
1270 if (is_obj_preserved) {
1271 dont_preserve(_obj);
1272 }
1273 {
1274 SaveLiveRegisters slr(&masm, this);
1275
1276 assert_different_registers(rax, c_rarg0, c_rarg1);
1277
1278 // Shuffle in the arguments. The end result should be:
1279 // c_rarg0 <-- obj
1280 // c_rarg1 <-- lea(addr)
1281 if (_obj == c_rarg0) {
1282 __ lea(c_rarg1, _addr);
1283 } else if (_obj == c_rarg1) {
1284 // Set up arguments in reverse, and then flip them
1285 __ lea(c_rarg0, _addr);
1286 __ xchgptr(c_rarg0, c_rarg1);
1287 } else {
1288 assert_different_registers(_obj, c_rarg0, c_rarg1);
1289 __ lea(c_rarg1, _addr);
1290 __ movptr(c_rarg0, _obj);
1291 }
1292
1293 // Go to runtime and handle the rest there.
1294 __ call(RuntimeAddress(lrb_runtime_entry_addr()));
1295
1296 // Save the result where needed.
1297 if (_obj != rax) {
1298 __ movptr(_obj, rax);
1299 }
1300 }
1301 if (is_obj_preserved) {
1302 preserve(_obj);
1303 }
1304
1305 __ jmp(*L_done);
1306 }
1307
1308 int ShenandoahBarrierStubC2::available_gp_registers() {
1309 return Register::available_gp_registers();
1310 }
1311
1312 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1313 return r == rsp || r == rbp || r == r12_heapbase || r == r15_thread;
1314 }
1315
1316 void ShenandoahBarrierStubC2::post_init() {
1317 // Do nothing.
1318 }
1319 #undef __
1320 #endif
|