< prev index next >

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

Print this page

 932 #endif
 933 
 934   xorl(tmpReg, tmpReg); // Set ZF == 1
 935 
 936   bind(NO_COUNT);
 937 }
 938 
 939 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register rax_reg,
 940                                               Register t, Register thread) {
 941   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 942   assert(rax_reg == rax, "Used for CAS");
 943   assert_different_registers(obj, box, rax_reg, t, thread);
 944 
 945   // Handle inflated monitor.
 946   Label inflated;
 947   // Finish fast lock successfully. ZF value is irrelevant.
 948   Label locked;
 949   // Finish fast lock unsuccessfully. MUST jump with ZF == 0
 950   Label slow_path;
 951 



 952   if (DiagnoseSyncOnValueBasedClasses != 0) {
 953     load_klass(rax_reg, obj, t);
 954     movl(rax_reg, Address(rax_reg, Klass::access_flags_offset()));
 955     testl(rax_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
 956     jcc(Assembler::notZero, slow_path);
 957   }
 958 
 959   const Register mark = t;
 960 
 961   { // Lightweight Lock
 962 
 963     Label push;
 964 
 965     const Register top = box;
 966 
 967     // Load the mark.
 968     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 969 
 970     // Prefetch top.
 971     movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
 972 
 973     // Check for monitor (0b10).
 974     testptr(mark, markWord::monitor_value);
 975     jcc(Assembler::notZero, inflated);
 976 
 977     // Check if lock-stack is full.
 978     cmpl(top, LockStack::end_offset() - 1);
 979     jcc(Assembler::greater, slow_path);
 980 
 981     // Check if recursive.
 982     cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
 983     jccb(Assembler::equal, push);
 984 
 985     // Try to lock. Transition lock bits 0b01 => 0b00
 986     movptr(rax_reg, mark);
 987     orptr(rax_reg, markWord::unlocked_value);
 988     andptr(mark, ~(int32_t)markWord::unlocked_value);
 989     lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 990     jcc(Assembler::notEqual, slow_path);
 991 
 992     bind(push);
 993     // After successful lock, push object on lock-stack.


 994     movptr(Address(thread, top), obj);
 995     addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
 996     jmpb(locked);
 997   }
 998 
 999   { // Handle inflated monitor.
1000     bind(inflated);
1001 
1002     const Register tagged_monitor = mark;





1003 
1004     // CAS owner (null => current thread).
1005     xorptr(rax_reg, rax_reg);
1006     lock(); cmpxchgptr(thread, Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1007     jccb(Assembler::equal, locked);
1008 
1009     // Check if recursive.
1010     cmpptr(thread, rax_reg);
1011     jccb(Assembler::notEqual, slow_path);
















1012 
1013     // Recursive.
1014     increment(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));


















































1015   }
1016 
1017   bind(locked);
1018   increment(Address(thread, JavaThread::held_monitor_count_offset()));
1019   // Set ZF = 1
1020   xorl(rax_reg, rax_reg);
1021 
1022 #ifdef ASSERT
1023   // Check that locked label is reached with ZF set.
1024   Label zf_correct;
1025   Label zf_bad_zero;
1026   jcc(Assembler::zero, zf_correct);
1027   jmp(zf_bad_zero);
1028 #endif
1029 
1030   bind(slow_path);
1031 #ifdef ASSERT
1032   // Check that slow_path label is reached with ZF not set.
1033   jcc(Assembler::notZero, zf_correct);
1034   stop("Fast Lock ZF != 0");

1036   stop("Fast Lock ZF != 1");
1037   bind(zf_correct);
1038 #endif
1039   // C2 uses the value of ZF to determine the continuation.
1040 }
1041 
1042 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread) {
1043   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
1044   assert(reg_rax == rax, "Used for CAS");
1045   assert_different_registers(obj, reg_rax, t);
1046 
1047   // Handle inflated monitor.
1048   Label inflated, inflated_check_lock_stack;
1049   // Finish fast unlock successfully.  MUST jump with ZF == 1
1050   Label unlocked;
1051 
1052   // Assume success.
1053   decrement(Address(thread, JavaThread::held_monitor_count_offset()));
1054 
1055   const Register mark = t;
1056   const Register top = reg_rax;


1057 
1058   Label dummy;
1059   C2FastUnlockLightweightStub* stub = nullptr;
1060 
1061   if (!Compile::current()->output()->in_scratch_emit_size()) {
1062     stub = new (Compile::current()->comp_arena()) C2FastUnlockLightweightStub(obj, mark, reg_rax, thread);
1063     Compile::current()->output()->add_stub(stub);
1064   }
1065 
1066   Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
1067   Label& check_successor = stub == nullptr ? dummy : stub->check_successor();

1068 
1069   { // Lightweight Unlock
1070 
1071     // Load top.
1072     movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
1073 
1074     // Prefetch mark.
1075     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
1076 
1077     // Check if obj is top of lock-stack.
1078     cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
1079     // Top of lock stack was not obj. Must be monitor.
1080     jcc(Assembler::notEqual, inflated_check_lock_stack);
1081 
1082     // Pop lock-stack.
1083     DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
1084     subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
1085 
1086     // Check if recursive.
1087     cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
1088     jcc(Assembler::equal, unlocked);
1089 
1090     // We elide the monitor check, let the CAS fail instead.
1091 



1092     // Try to unlock. Transition lock bits 0b00 => 0b01
1093     movptr(reg_rax, mark);
1094     andptr(reg_rax, ~(int32_t)markWord::lock_mask);
1095     orptr(mark, markWord::unlocked_value);
1096     lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
1097     jcc(Assembler::notEqual, push_and_slow_path);
1098     jmp(unlocked);
1099   }
1100 
1101 
1102   { // Handle inflated monitor.
1103     bind(inflated_check_lock_stack);
1104 #ifdef ASSERT
1105     Label check_done;
1106     subl(top, oopSize);
1107     cmpl(top, in_bytes(JavaThread::lock_stack_base_offset()));
1108     jcc(Assembler::below, check_done);
1109     cmpptr(obj, Address(thread, top));
1110     jccb(Assembler::notEqual, inflated_check_lock_stack);
1111     stop("Fast Unlock lock on stack");
1112     bind(check_done);


1113     testptr(mark, markWord::monitor_value);
1114     jccb(Assembler::notZero, inflated);
1115     stop("Fast Unlock not monitor");
1116 #endif
1117 
1118     bind(inflated);
1119 
1120     // mark contains the tagged ObjectMonitor*.
1121     const Register monitor = mark;
1122 









1123 #ifndef _LP64
1124     // Check if recursive.
1125     xorptr(reg_rax, reg_rax);
1126     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
1127     jcc(Assembler::notZero, check_successor);











1128 
1129     // Check if the entry lists are empty.
1130     movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
1131     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
1132     jcc(Assembler::notZero, check_successor);
1133 
1134     // Release lock.
1135     movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
1136 #else // _LP64
1137     Label recursive;
1138 
1139     // Check if recursive.
1140     cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 0);
1141     jccb(Assembler::notEqual, recursive);
1142 
1143     // Check if the entry lists are empty.
1144     movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
1145     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
1146     jcc(Assembler::notZero, check_successor);
1147 
1148     // Release lock.
1149     movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
1150     jmpb(unlocked);
1151 
1152     // Recursive unlock.
1153     bind(recursive);
1154     decrement(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
1155     xorl(t, t);
1156 #endif

1157   }
1158 
1159   bind(unlocked);
1160   if (stub != nullptr) {
1161     bind(stub->unlocked_continuation());
1162   }
1163 
1164 #ifdef ASSERT
1165   // Check that unlocked label is reached with ZF set.
1166   Label zf_correct;
1167   jcc(Assembler::zero, zf_correct);
1168   stop("Fast Unlock ZF != 1");
1169 #endif
1170 
1171   if (stub != nullptr) {
1172     bind(stub->slow_path_continuation());
1173   }
1174 #ifdef ASSERT
1175   // Check that stub->continuation() label is reached with ZF not set.
1176   jccb(Assembler::notZero, zf_correct);

6480   // Perform above steps with lane comparison expression as INDEX >= 48 && INDEX < 64
6481   // and broadcasting third 128 bit lane.
6482   evpcmpb(ktmp, k0, shuffle,  xtmp1, Assembler::nlt, true, vlen_enc);
6483   vpsllq(xtmp2, xtmp2, 0x1, vlen_enc);
6484   evpcmpb(ktmp, ktmp, shuffle,  xtmp2, Assembler::lt, true, vlen_enc);
6485   evshufi64x2(xtmp3, src, src, 0xFF, vlen_enc);
6486   evpshufb(dst, ktmp, xtmp3, shuffle, true, vlen_enc);
6487 }
6488 
6489 void C2_MacroAssembler::vector_rearrange_int_float(BasicType bt, XMMRegister dst,
6490                                                    XMMRegister shuffle, XMMRegister src, int vlen_enc) {
6491   if (vlen_enc == AVX_128bit) {
6492     vpermilps(dst, src, shuffle, vlen_enc);
6493   } else if (bt == T_INT) {
6494     vpermd(dst, shuffle, src, vlen_enc);
6495   } else {
6496     assert(bt == T_FLOAT, "");
6497     vpermps(dst, shuffle, src, vlen_enc);
6498   }
6499 }

















 932 #endif
 933 
 934   xorl(tmpReg, tmpReg); // Set ZF == 1
 935 
 936   bind(NO_COUNT);
 937 }
 938 
 939 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register rax_reg,
 940                                               Register t, Register thread) {
 941   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
 942   assert(rax_reg == rax, "Used for CAS");
 943   assert_different_registers(obj, box, rax_reg, t, thread);
 944 
 945   // Handle inflated monitor.
 946   Label inflated;
 947   // Finish fast lock successfully. ZF value is irrelevant.
 948   Label locked;
 949   // Finish fast lock unsuccessfully. MUST jump with ZF == 0
 950   Label slow_path;
 951 
 952   // Clear box. TODO[OMWorld]: Is this necessary? May also defer this to not write twice.
 953   movptr(Address(box, BasicLock::displaced_header_offset_in_bytes()), 0);
 954 
 955   if (DiagnoseSyncOnValueBasedClasses != 0) {
 956     load_klass(rax_reg, obj, t);
 957     movl(rax_reg, Address(rax_reg, Klass::access_flags_offset()));
 958     testl(rax_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
 959     jcc(Assembler::notZero, slow_path);
 960   }
 961 
 962   const Register mark = t;
 963 
 964   { // Lightweight Lock
 965 
 966     Label push;
 967 
 968     const Register top = rax_reg;
 969 
 970     // Load the mark.
 971     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 972 
 973     // Prefetch top.
 974     movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
 975 
 976     // Check for monitor (0b10).
 977     testptr(mark, markWord::monitor_value);
 978     jcc(Assembler::notZero, inflated);
 979 
 980     // Check if lock-stack is full.
 981     cmpl(top, LockStack::end_offset() - 1);
 982     jcc(Assembler::greater, slow_path);
 983 
 984     // Check if recursive.
 985     cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
 986     jccb(Assembler::equal, push);
 987 
 988     // Try to lock. Transition lock bits 0b01 => 0b00
 989     movptr(rax_reg, mark);
 990     orptr(rax_reg, markWord::unlocked_value);
 991     andptr(mark, ~(int32_t)markWord::unlocked_value);
 992     lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
 993     jcc(Assembler::notEqual, slow_path);
 994 
 995     bind(push);
 996     // After successful lock, push object on lock-stack.
 997     // TODO[OMWorld]: Was prepush better?
 998     movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
 999     movptr(Address(thread, top), obj);
1000     addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
1001     jmpb(locked);
1002   }
1003 
1004   { // Handle inflated monitor.
1005     bind(inflated);
1006 
1007     const Register monitor = t;
1008 
1009     if (!OMUseC2Cache) {
1010       jmp(slow_path);
1011     } else {
1012       if (OMCacheHitRate) increment(Address(thread, JavaThread::lock_lookup_offset()));
1013 
1014       // Fetch ObjectMonitor* from the cache or take the slow-path.
1015       Label monitor_found;


1016 
1017       // Load cache address
1018       lea(t, Address(thread, JavaThread::om_cache_oops_offset()));
1019 
1020       const int num_unrolled = MIN2(OMC2UnrollCacheEntries, OMCacheSize);
1021       for (int i = 0; i < num_unrolled; i++) {
1022         cmpptr(obj, Address(t));
1023         jccb(Assembler::equal, monitor_found);
1024         if (i + 1 != num_unrolled) {
1025           increment(t, in_bytes(OMCache::oop_to_oop_difference()));
1026         }
1027       }
1028 
1029       if (num_unrolled == 0 || (OMC2UnrollCacheLookupLoopTail && num_unrolled != OMCacheSize)) {
1030         if (num_unrolled != 0) {
1031           // Loop after unrolling, advance iterator.
1032           increment(t, in_bytes(OMCache::oop_to_oop_difference()));
1033         }
1034 
1035         Label loop;
1036 
1037         // Search for obj in cache.
1038         bind(loop);
1039 
1040         // Check for match.
1041         cmpptr(obj, Address(t));
1042         jccb(Assembler::equal, monitor_found);
1043 
1044         // Search until null encountered, guaranteed _null_sentinel at end.
1045         cmpptr(Address(t), 1);
1046         jcc(Assembler::below, slow_path); // 0 check, but with ZF=0 when *t == 0
1047         increment(t, in_bytes(OMCache::oop_to_oop_difference()));
1048         jmpb(loop);
1049       } else {
1050         jmp(slow_path);
1051       }
1052 
1053       // Cache hit.
1054       bind(monitor_found);
1055       movptr(monitor, Address(t, OMCache::oop_to_monitor_difference()));
1056       if (OMCacheHitRate) increment(Address(thread, JavaThread::lock_hit_offset()));
1057 
1058       Label monitor_locked;
1059       // Lock the monitor.
1060       Label recursion;
1061       if (OMRecursiveFastPath) {
1062         // Check owner for recursion first.
1063         cmpptr(thread, Address(monitor, ObjectMonitor::owner_offset()));
1064         jccb(Assembler::equal, recursion);
1065       }
1066 
1067       // CAS owner (null => current thread).
1068       xorptr(rax, rax);
1069       lock(); cmpxchgptr(thread, Address(monitor, ObjectMonitor::owner_offset()));
1070       jccb(Assembler::equal, monitor_locked);
1071 
1072       if (OMRecursiveFastPath) {
1073         // Recursion already checked.
1074         jmpb(slow_path);
1075       } else {
1076         // Check if recursive.
1077         cmpptr(thread, rax);
1078         jccb(Assembler::notEqual, slow_path);
1079       }
1080 
1081       // Recursive.
1082       bind(recursion);
1083       increment(Address(monitor, ObjectMonitor::recursions_offset()));
1084 
1085       bind(monitor_locked);
1086       // Cache the monitor for unlock
1087       movptr(Address(box, BasicLock::displaced_header_offset_in_bytes()), monitor);
1088     }
1089   }
1090 
1091   bind(locked);
1092   increment(Address(thread, JavaThread::held_monitor_count_offset()));
1093   // Set ZF = 1
1094   xorl(rax_reg, rax_reg);
1095 
1096 #ifdef ASSERT
1097   // Check that locked label is reached with ZF set.
1098   Label zf_correct;
1099   Label zf_bad_zero;
1100   jcc(Assembler::zero, zf_correct);
1101   jmp(zf_bad_zero);
1102 #endif
1103 
1104   bind(slow_path);
1105 #ifdef ASSERT
1106   // Check that slow_path label is reached with ZF not set.
1107   jcc(Assembler::notZero, zf_correct);
1108   stop("Fast Lock ZF != 0");

1110   stop("Fast Lock ZF != 1");
1111   bind(zf_correct);
1112 #endif
1113   // C2 uses the value of ZF to determine the continuation.
1114 }
1115 
1116 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread) {
1117   assert(LockingMode == LM_LIGHTWEIGHT, "must be");
1118   assert(reg_rax == rax, "Used for CAS");
1119   assert_different_registers(obj, reg_rax, t);
1120 
1121   // Handle inflated monitor.
1122   Label inflated, inflated_check_lock_stack;
1123   // Finish fast unlock successfully.  MUST jump with ZF == 1
1124   Label unlocked;
1125 
1126   // Assume success.
1127   decrement(Address(thread, JavaThread::held_monitor_count_offset()));
1128 
1129   const Register mark = t;
1130   const Register monitor = t;
1131   const Register top = t;
1132   const Register box = reg_rax;
1133 
1134   Label dummy;
1135   C2FastUnlockLightweightStub* stub = nullptr;
1136 
1137   if (!Compile::current()->output()->in_scratch_emit_size()) {
1138     stub = new (Compile::current()->comp_arena()) C2FastUnlockLightweightStub(obj, mark, reg_rax, thread);
1139     Compile::current()->output()->add_stub(stub);
1140   }
1141 
1142   Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
1143   Label& check_successor = stub == nullptr ? dummy : stub->check_successor();
1144   Label& slow_path = stub == nullptr ? dummy : stub->slow_path();
1145 
1146   { // Lightweight Unlock
1147 
1148     // Load top.
1149     movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
1150 



1151     // Check if obj is top of lock-stack.
1152     cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
1153     // Top of lock stack was not obj. Must be monitor.
1154     jcc(Assembler::notEqual, inflated_check_lock_stack);
1155 
1156     // Pop lock-stack.
1157     DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
1158     subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
1159 
1160     // Check if recursive.
1161     cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
1162     jcc(Assembler::equal, unlocked);
1163 
1164     // We elide the monitor check, let the CAS fail instead.
1165 
1166     // Load mark.
1167     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
1168 
1169     // Try to unlock. Transition lock bits 0b00 => 0b01
1170     movptr(reg_rax, mark);
1171     andptr(reg_rax, ~(int32_t)markWord::lock_mask);
1172     orptr(mark, markWord::unlocked_value);
1173     lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
1174     jcc(Assembler::notEqual, push_and_slow_path);
1175     jmp(unlocked);
1176   }
1177 
1178 
1179   { // Handle inflated monitor.
1180     bind(inflated_check_lock_stack);
1181 #ifdef ASSERT
1182     Label check_done;
1183     subl(top, oopSize);
1184     cmpl(top, in_bytes(JavaThread::lock_stack_base_offset()));
1185     jcc(Assembler::below, check_done);
1186     cmpptr(obj, Address(thread, top));
1187     jccb(Assembler::notEqual, inflated_check_lock_stack);
1188     stop("Fast Unlock lock on stack");
1189     bind(check_done);
1190     const Register mark = t;
1191     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
1192     testptr(mark, markWord::monitor_value);
1193     jccb(Assembler::notZero, inflated);
1194     stop("Fast Unlock not monitor");
1195 #endif
1196 
1197     bind(inflated);
1198 
1199     if (!OMUseC2Cache) {
1200       jmp(slow_path);
1201     } else {
1202       if (OMCacheHitRate) increment(Address(thread, JavaThread::unlock_lookup_offset()));
1203       movptr(monitor, Address(box, BasicLock::displaced_header_offset_in_bytes()));
1204       // TODO[OMWorld]: Figure out the correctness surrounding the owner field here. Obj is not on the lock stack
1205       //                but this means this thread must have locked on the inflated monitor at some point. So it
1206       //                should not be anonymous.
1207       cmpptr(monitor, 2);
1208       jcc(Assembler::below, slow_path);
1209 
1210       if (OMCacheHitRate) increment(Address(thread, JavaThread::unlock_hit_offset()));
1211 #ifndef _LP64
1212         // TODO[OMWorld]: Unify 32 with 64. Should just be a straight up use 64 on 32. We have the registers here.
1213         // Check if recursive.
1214         xorptr(reg_rax, reg_rax);
1215         orptr(reg_rax, Address(monitor, ObjectMonitor::recursions_offset()));
1216         jcc(Assembler::notZero, check_successor);
1217 
1218         // Check if the entry lists are empty.
1219         movptr(reg_rax, Address(monitor, ObjectMonitor::EntryList_offset()));
1220         orptr(reg_rax, Address(monitor, ObjectMonitor::cxq_offset()));
1221         jcc(Assembler::notZero, check_successor);
1222 
1223         // Release lock.
1224         movptr(Address(monitor, ObjectMonitor::owner_offset()), NULL_WORD);
1225 #else // _LP64
1226         Label recursive;
1227 
1228         // Check if recursive.
1229         cmpptr(Address(monitor,ObjectMonitor::recursions_offset()),0);
1230         jccb(Assembler::notEqual, recursive);

1231 
1232         // Check if the entry lists are empty.
1233         movptr(reg_rax, Address(monitor, ObjectMonitor::cxq_offset()));
1234         orptr(reg_rax, Address(monitor, ObjectMonitor::EntryList_offset()));
1235         jcc(Assembler::notZero, check_successor);
1236 
1237         // Release lock.
1238         movptr(Address(monitor, ObjectMonitor::owner_offset()), NULL_WORD);
1239         jmpb(unlocked);
1240 
1241         // Recursive unlock.
1242         bind(recursive);
1243         decrement(Address(monitor, ObjectMonitor::recursions_offset()));
1244         xorl(t, t);









1245 #endif
1246     }
1247   }
1248 
1249   bind(unlocked);
1250   if (stub != nullptr) {
1251     bind(stub->unlocked_continuation());
1252   }
1253 
1254 #ifdef ASSERT
1255   // Check that unlocked label is reached with ZF set.
1256   Label zf_correct;
1257   jcc(Assembler::zero, zf_correct);
1258   stop("Fast Unlock ZF != 1");
1259 #endif
1260 
1261   if (stub != nullptr) {
1262     bind(stub->slow_path_continuation());
1263   }
1264 #ifdef ASSERT
1265   // Check that stub->continuation() label is reached with ZF not set.
1266   jccb(Assembler::notZero, zf_correct);

6570   // Perform above steps with lane comparison expression as INDEX >= 48 && INDEX < 64
6571   // and broadcasting third 128 bit lane.
6572   evpcmpb(ktmp, k0, shuffle,  xtmp1, Assembler::nlt, true, vlen_enc);
6573   vpsllq(xtmp2, xtmp2, 0x1, vlen_enc);
6574   evpcmpb(ktmp, ktmp, shuffle,  xtmp2, Assembler::lt, true, vlen_enc);
6575   evshufi64x2(xtmp3, src, src, 0xFF, vlen_enc);
6576   evpshufb(dst, ktmp, xtmp3, shuffle, true, vlen_enc);
6577 }
6578 
6579 void C2_MacroAssembler::vector_rearrange_int_float(BasicType bt, XMMRegister dst,
6580                                                    XMMRegister shuffle, XMMRegister src, int vlen_enc) {
6581   if (vlen_enc == AVX_128bit) {
6582     vpermilps(dst, src, shuffle, vlen_enc);
6583   } else if (bt == T_INT) {
6584     vpermd(dst, shuffle, src, vlen_enc);
6585   } else {
6586     assert(bt == T_FLOAT, "");
6587     vpermps(dst, shuffle, src, vlen_enc);
6588   }
6589 }
6590 
6591 #ifdef _LP64
6592 void C2_MacroAssembler::load_nklass_compact_c2(Register dst, Register obj, Register index, Address::ScaleFactor scale, int disp) {
6593   // Note: Don't clobber obj anywhere in that method!
6594 
6595   // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
6596   // obj-start, so that we can load from the object's mark-word instead. Usually the address
6597   // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
6598   // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
6599   // then passes that register as obj and 0 in disp. The following code extracts the base
6600   // and offset to load the mark-word.
6601   int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
6602   movq(dst, Address(obj, index, scale, offset));
6603   shrq(dst, markWord::klass_shift);
6604 }
6605 #endif
< prev index next >