14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "code/aotCodeCache.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_x86.inline.hpp"
47
48
49 // These masks are used to provide 128-bit aligned bitmasks to the XMM
50 // instructions, to allow sign-masking or sign-bit flipping. They allow
51 // fast versions of NegF/NegD and AbsF/AbsD.
52
53 // Note: 'double' and 'long long' have 32-bits alignment on x86.
54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
55 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
56 // of 128-bits operands for SSE instructions.
57 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
58 // Store the value to a 128-bits operand.
59 operand[0] = lo;
395 // Perform needed unlocking
396 MonitorExitStub* stub = nullptr;
397 if (method()->is_synchronized()) {
398 monitor_address(0, FrameMap::rax_opr);
399 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
400 __ unlock_object(rdi, rsi, rax, *stub->entry());
401 __ bind(*stub->continuation());
402 }
403
404 if (compilation()->env()->dtrace_method_probes()) {
405 __ mov(rdi, r15_thread);
406 __ mov_metadata(rsi, method()->constant_encoding());
407 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
408 }
409
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(rax, rbx); // Restore the exception
412 }
413
414 // remove the activation and dispatch to the unwind handler
415 __ remove_frame(initial_frame_size_in_bytes());
416 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
417
418 // Emit the slow path assembly
419 if (stub != nullptr) {
420 stub->emit_code(this);
421 }
422
423 return offset;
424 }
425
426
427 int LIR_Assembler::emit_deopt_handler() {
428 // generate code for exception handler
429 address handler_base = __ start_a_stub(deopt_handler_size());
430 if (handler_base == nullptr) {
431 // not enough space left for the handler
432 bailout("deopt handler overflow");
433 return -1;
434 }
435
440
441 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
442
443 int entry_offset = __ offset();
444
445 __ jmp(start);
446
447 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
448 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
449 "out of bounds read in post-call NOP check");
450 __ end_a_stub();
451
452 return entry_offset;
453 }
454
455 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
456 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
457 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
458 assert(result->fpu() == 0, "result must already be on TOS");
459 }
460
461 // Pop the stack before the safepoint code
462 __ remove_frame(initial_frame_size_in_bytes());
463
464 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
465 __ reserved_stack_check();
466 }
467
468 // Note: we do not need to round double result; float result has the right precision
469 // the poll sets the condition code, but no data registers
470
471 code_stub->set_safepoint_offset(__ offset());
472 __ relocate(relocInfo::poll_return_type);
473 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
474 __ ret(0);
475 }
476
477
478 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
479 guarantee(info != nullptr, "Shouldn't be null");
480 int offset = __ offset();
481 const Register poll_addr = rscratch1;
482 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
483 add_debug_info_for_branch(info);
484 __ relocate(relocInfo::poll_type);
485 address pre_pc = __ pc();
486 __ testl(rax, Address(poll_addr, 0));
487 address post_pc = __ pc();
488 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
489 return offset;
490 }
491
492
493 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
494 if (from_reg != to_reg) __ mov(to_reg, from_reg);
495 }
496
497 void LIR_Assembler::swap_reg(Register a, Register b) {
1205 // init_state needs acquire, but x86 is TSO, and so we are already good.
1206 __ cmpb(Address(op->klass()->as_register(),
1207 InstanceKlass::init_state_offset()),
1208 InstanceKlass::fully_initialized);
1209 __ jcc(Assembler::notEqual, *op->stub()->entry());
1210 }
1211 __ allocate_object(op->obj()->as_register(),
1212 op->tmp1()->as_register(),
1213 op->tmp2()->as_register(),
1214 op->header_size(),
1215 op->object_size(),
1216 op->klass()->as_register(),
1217 *op->stub()->entry());
1218 __ bind(*op->stub()->continuation());
1219 }
1220
1221 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1222 Register len = op->len()->as_register();
1223 __ movslq(len, len);
1224
1225 if (UseSlowPath ||
1226 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1227 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1228 __ jmp(*op->stub()->entry());
1229 } else {
1230 Register tmp1 = op->tmp1()->as_register();
1231 Register tmp2 = op->tmp2()->as_register();
1232 Register tmp3 = op->tmp3()->as_register();
1233 if (len == tmp1) {
1234 tmp1 = tmp3;
1235 } else if (len == tmp2) {
1236 tmp2 = tmp3;
1237 } else if (len == tmp3) {
1238 // everything is ok
1239 } else {
1240 __ mov(tmp3, len);
1241 }
1242 __ allocate_array(op->obj()->as_register(),
1243 len,
1244 tmp1,
1245 tmp2,
1280 int bci = op->profiled_bci();
1281 md = method->method_data_or_null();
1282 assert(md != nullptr, "Sanity");
1283 data = md->bci_to_data(bci);
1284 assert(data != nullptr, "need data for type check");
1285 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1286 }
1287 Label* success_target = success;
1288 Label* failure_target = failure;
1289
1290 if (obj == k_RInfo) {
1291 k_RInfo = dst;
1292 } else if (obj == klass_RInfo) {
1293 klass_RInfo = dst;
1294 }
1295 Rtmp1 = op->tmp3()->as_register();
1296 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1297
1298 assert_different_registers(obj, k_RInfo, klass_RInfo);
1299
1300 __ testptr(obj, obj);
1301 if (op->should_profile()) {
1302 Label not_null;
1303 Register mdo = klass_RInfo;
1304 __ mov_metadata(mdo, md->constant_encoding());
1305 __ jccb(Assembler::notEqual, not_null);
1306 // Object is null; update MDO and exit
1307 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1308 int header_bits = BitData::null_seen_byte_constant();
1309 __ orb(data_addr, header_bits);
1310 __ jmp(*obj_is_null);
1311 __ bind(not_null);
1312
1313 Register recv = k_RInfo;
1314 __ load_klass(recv, obj, tmp_load_klass);
1315 type_profile_helper(mdo, md, data, recv);
1316 } else {
1317 __ jcc(Assembler::equal, *obj_is_null);
1318 }
1319
1320 if (!k->is_loaded()) {
1321 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1322 } else {
1323 __ mov_metadata(k_RInfo, k->constant_encoding());
1324 }
1325 __ verify_oop(obj);
1326
1327 if (op->fast_check()) {
1328 // get object class
1329 // not a safepoint as obj null check happens earlier
1330 __ load_klass(Rtmp1, obj, tmp_load_klass);
1331 __ cmpptr(k_RInfo, Rtmp1);
1332 __ jcc(Assembler::notEqual, *failure_target);
1333 // successful cast, fall through to profile or jump
1334 } else {
1335 // get object class
1336 // not a safepoint as obj null check happens earlier
1337 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1338 if (k->is_loaded()) {
1339 // See if we get an immediate positive hit
1340 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1341 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1342 __ jcc(Assembler::notEqual, *failure_target);
1343 // successful cast, fall through to profile or jump
1344 } else {
1345 // See if we get an immediate positive hit
1346 __ jcc(Assembler::equal, *success_target);
1347 // check for self
1348 __ cmpptr(klass_RInfo, k_RInfo);
1349 __ jcc(Assembler::equal, *success_target);
1350
1351 __ push_ppx(klass_RInfo);
1352 __ push_ppx(k_RInfo);
1353 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1354 __ pop_ppx(klass_RInfo);
1355 __ pop_ppx(klass_RInfo);
1356 // result is a boolean
1357 __ testl(klass_RInfo, klass_RInfo);
1358 __ jcc(Assembler::equal, *failure_target);
1359 // successful cast, fall through to profile or jump
1360 }
1361 } else {
1362 // perform the fast part of the checking logic
1363 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1364 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1365 __ push_ppx(klass_RInfo);
1366 __ push_ppx(k_RInfo);
1367 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1368 __ pop_ppx(klass_RInfo);
1458 __ mov(dst, obj);
1459 }
1460 } else
1461 if (code == lir_instanceof) {
1462 Register obj = op->object()->as_register();
1463 Register dst = op->result_opr()->as_register();
1464 Label success, failure, done;
1465 emit_typecheck_helper(op, &success, &failure, &failure);
1466 __ bind(failure);
1467 __ xorptr(dst, dst);
1468 __ jmpb(done);
1469 __ bind(success);
1470 __ movptr(dst, 1);
1471 __ bind(done);
1472 } else {
1473 ShouldNotReachHere();
1474 }
1475
1476 }
1477
1478
1479 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1480 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1481 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1482 Register newval = op->new_value()->as_register();
1483 Register cmpval = op->cmp_value()->as_register();
1484 assert(cmpval == rax, "wrong register");
1485 assert(newval != noreg, "new val must be register");
1486 assert(cmpval != newval, "cmp and new values must be in different registers");
1487 assert(cmpval != addr, "cmp and addr must be in different registers");
1488 assert(newval != addr, "new value and addr must be in different registers");
1489
1490 if (op->code() == lir_cas_obj) {
1491 if (UseCompressedOops) {
1492 __ encode_heap_oop(cmpval);
1493 __ mov(rscratch1, newval);
1494 __ encode_heap_oop(rscratch1);
1495 __ lock();
1496 // cmpval (rax) is implicitly used by this instruction
1497 __ cmpxchgl(rscratch1, Address(addr, 0));
1503 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1504 __ lock();
1505 __ cmpxchgl(newval, Address(addr, 0));
1506 }
1507 } else if (op->code() == lir_cas_long) {
1508 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1509 Register newval = op->new_value()->as_register_lo();
1510 Register cmpval = op->cmp_value()->as_register_lo();
1511 assert(cmpval == rax, "wrong register");
1512 assert(newval != noreg, "new val must be register");
1513 assert(cmpval != newval, "cmp and new values must be in different registers");
1514 assert(cmpval != addr, "cmp and addr must be in different registers");
1515 assert(newval != addr, "new value and addr must be in different registers");
1516 __ lock();
1517 __ cmpxchgq(newval, Address(addr, 0));
1518 } else {
1519 Unimplemented();
1520 }
1521 }
1522
1523 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1524 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1525 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1526
1527 Assembler::Condition acond, ncond;
1528 switch (condition) {
1529 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1530 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1531 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1532 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1533 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1534 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1535 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1536 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1537 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1538 ShouldNotReachHere();
1539 }
1540
1541 if (opr1->is_cpu_register()) {
1542 reg2reg(opr1, result);
2113 int offset = __ offset();
2114 switch (code) {
2115 case lir_static_call:
2116 case lir_optvirtual_call:
2117 case lir_dynamic_call:
2118 offset += NativeCall::displacement_offset;
2119 break;
2120 case lir_icvirtual_call:
2121 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2122 break;
2123 default: ShouldNotReachHere();
2124 }
2125 __ align(BytesPerWord, offset);
2126 }
2127
2128
2129 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2130 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2131 "must be aligned");
2132 __ call(AddressLiteral(op->addr(), rtype));
2133 add_call_info(code_offset(), op->info());
2134 __ post_call_nop();
2135 }
2136
2137
2138 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2139 __ ic_call(op->addr());
2140 add_call_info(code_offset(), op->info());
2141 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2142 "must be aligned");
2143 __ post_call_nop();
2144 }
2145
2146
2147 void LIR_Assembler::emit_static_call_stub() {
2148 address call_pc = __ pc();
2149 address stub = __ start_a_stub(call_stub_size());
2150 if (stub == nullptr) {
2151 bailout("static call stub overflow");
2152 return;
2153 }
2154
2155 int start = __ offset();
2156
2157 // make sure that the displacement word of the call ends up word aligned
2158 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2159 __ relocate(static_stub_Relocation::spec(call_pc));
2160 __ mov_metadata(rbx, (Metadata*)nullptr);
2287 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2288 }
2289
2290
2291 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2292 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2293 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2294 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2295 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2296 }
2297
2298
2299 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2300 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2301 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2302 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2303 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2304 }
2305
2306
2307 // This code replaces a call to arraycopy; no exception may
2308 // be thrown in this code, they must be thrown in the System.arraycopy
2309 // activation frame; we could save some checks if this would not be the case
2310 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2311 ciArrayKlass* default_type = op->expected_type();
2312 Register src = op->src()->as_register();
2313 Register dst = op->dst()->as_register();
2314 Register src_pos = op->src_pos()->as_register();
2315 Register dst_pos = op->dst_pos()->as_register();
2316 Register length = op->length()->as_register();
2317 Register tmp = op->tmp()->as_register();
2318 Register tmp_load_klass = rscratch1;
2319 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2320
2321 CodeStub* stub = op->stub();
2322 int flags = op->flags();
2323 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2324 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2325
2326 // if we don't know anything, just go through the generic arraycopy
2327 if (default_type == nullptr) {
2328 // save outgoing arguments on stack in case call to System.arraycopy is needed
2329 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2330 // for interpreter calling conventions. Now we have to do it in new style conventions.
2331 // For the moment until C1 gets the new register allocator I just force all the
2332 // args to the right place (except the register args) and then on the back side
2333 // reload the register args properly if we go slow path. Yuck
2334
2335 // These are proper for the calling convention
2336 store_parameter(length, 2);
2337 store_parameter(dst_pos, 1);
2338 store_parameter(dst, 0);
2339
2340 // these are just temporary placements until we need to reload
2341 store_parameter(src_pos, 3);
2342 store_parameter(src, 4);
2343
2344 address copyfunc_addr = StubRoutines::generic_arraycopy();
2345 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2382 __ mov(tmp, rax);
2383 __ xorl(tmp, -1);
2384
2385 // Reload values from the stack so they are where the stub
2386 // expects them.
2387 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2388 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2389 __ movptr (length, Address(rsp, 2*BytesPerWord));
2390 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2391 __ movptr (src, Address(rsp, 4*BytesPerWord));
2392
2393 __ subl(length, tmp);
2394 __ addl(src_pos, tmp);
2395 __ addl(dst_pos, tmp);
2396 __ jmp(*stub->entry());
2397
2398 __ bind(*stub->continuation());
2399 return;
2400 }
2401
2402 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2403
2404 int elem_size = type2aelembytes(basic_type);
2405 Address::ScaleFactor scale;
2406
2407 switch (elem_size) {
2408 case 1 :
2409 scale = Address::times_1;
2410 break;
2411 case 2 :
2412 scale = Address::times_2;
2413 break;
2414 case 4 :
2415 scale = Address::times_4;
2416 break;
2417 case 8 :
2418 scale = Address::times_8;
2419 break;
2420 default:
2421 scale = Address::no_scale;
2919 // first time here. Set profile type.
2920 __ movptr(mdo_addr, tmp);
2921 #ifdef ASSERT
2922 __ andptr(tmp, TypeEntries::type_klass_mask);
2923 __ verify_klass_ptr(tmp);
2924 #endif
2925 } else {
2926 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2927 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2928
2929 __ testptr(mdo_addr, TypeEntries::type_unknown);
2930 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2931
2932 __ orptr(mdo_addr, TypeEntries::type_unknown);
2933 }
2934 }
2935 }
2936 __ bind(next);
2937 }
2938
2939 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2940 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2941 }
2942
2943
2944 void LIR_Assembler::align_backward_branch_target() {
2945 __ align(BytesPerWord);
2946 }
2947
2948
2949 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2950 if (left->is_single_cpu()) {
2951 __ negl(left->as_register());
2952 move_regs(left->as_register(), dest->as_register());
2953
2954 } else if (left->is_double_cpu()) {
2955 Register lo = left->as_register_lo();
2956 Register dst = dest->as_register_lo();
2957 __ movptr(dst, lo);
2958 __ negptr(dst);
3104 }
3105
3106 void LIR_Assembler::membar_loadstore() {
3107 // no-op
3108 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3109 }
3110
3111 void LIR_Assembler::membar_storeload() {
3112 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3113 }
3114
3115 void LIR_Assembler::on_spin_wait() {
3116 __ pause ();
3117 }
3118
3119 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3120 assert(result_reg->is_register(), "check");
3121 __ mov(result_reg->as_register(), r15_thread);
3122 }
3123
3124
3125 void LIR_Assembler::peephole(LIR_List*) {
3126 // do nothing for now
3127 }
3128
3129 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3130 assert(data == dest, "xchg/xadd uses only 2 operands");
3131
3132 if (data->type() == T_INT) {
3133 if (code == lir_xadd) {
3134 __ lock();
3135 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3136 } else {
3137 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3138 }
3139 } else if (data->is_oop()) {
3140 assert (code == lir_xchg, "xadd for oops");
3141 Register obj = data->as_register();
3142 if (UseCompressedOops) {
3143 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "utilities/powerOfTwo.hpp"
49 #include "vmreg_x86.inline.hpp"
50
51
52 // These masks are used to provide 128-bit aligned bitmasks to the XMM
53 // instructions, to allow sign-masking or sign-bit flipping. They allow
54 // fast versions of NegF/NegD and AbsF/AbsD.
55
56 // Note: 'double' and 'long long' have 32-bits alignment on x86.
57 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
58 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
59 // of 128-bits operands for SSE instructions.
60 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
61 // Store the value to a 128-bits operand.
62 operand[0] = lo;
398 // Perform needed unlocking
399 MonitorExitStub* stub = nullptr;
400 if (method()->is_synchronized()) {
401 monitor_address(0, FrameMap::rax_opr);
402 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
403 __ unlock_object(rdi, rsi, rax, *stub->entry());
404 __ bind(*stub->continuation());
405 }
406
407 if (compilation()->env()->dtrace_method_probes()) {
408 __ mov(rdi, r15_thread);
409 __ mov_metadata(rsi, method()->constant_encoding());
410 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
411 }
412
413 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
414 __ mov(rax, rbx); // Restore the exception
415 }
416
417 // remove the activation and dispatch to the unwind handler
418 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
419 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
420
421 // Emit the slow path assembly
422 if (stub != nullptr) {
423 stub->emit_code(this);
424 }
425
426 return offset;
427 }
428
429
430 int LIR_Assembler::emit_deopt_handler() {
431 // generate code for exception handler
432 address handler_base = __ start_a_stub(deopt_handler_size());
433 if (handler_base == nullptr) {
434 // not enough space left for the handler
435 bailout("deopt handler overflow");
436 return -1;
437 }
438
443
444 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
445
446 int entry_offset = __ offset();
447
448 __ jmp(start);
449
450 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
451 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
452 "out of bounds read in post-call NOP check");
453 __ end_a_stub();
454
455 return entry_offset;
456 }
457
458 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
459 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
460 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
461 assert(result->fpu() == 0, "result must already be on TOS");
462 }
463 if (InlineTypeReturnedAsFields) {
464 #ifndef _LP64
465 Unimplemented();
466 #endif
467 // Check if we are returning an non-null inline type and load its fields into registers
468 ciType* return_type = compilation()->method()->return_type();
469 if (return_type->is_inlinetype()) {
470 ciInlineKlass* vk = return_type->as_inline_klass();
471 if (vk->can_be_returned_as_fields()) {
472 address unpack_handler = vk->unpack_handler();
473 assert(unpack_handler != nullptr, "must be");
474 __ call(RuntimeAddress(unpack_handler));
475 }
476 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
477 Label skip;
478 Label not_null;
479 __ testptr(rax, rax);
480 __ jcc(Assembler::notZero, not_null);
481 // Returned value is null, zero all return registers because they may belong to oop fields
482 __ xorq(j_rarg1, j_rarg1);
483 __ xorq(j_rarg2, j_rarg2);
484 __ xorq(j_rarg3, j_rarg3);
485 __ xorq(j_rarg4, j_rarg4);
486 __ xorq(j_rarg5, j_rarg5);
487 __ jmp(skip);
488 __ bind(not_null);
489
490 // Check if we are returning an non-null inline type and load its fields into registers
491 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
492
493 // Load fields from a buffered value with an inline class specific handler
494 __ load_klass(rdi, rax, rscratch1);
495 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
496 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
497 // Unpack handler can be null if inline type is not scalarizable in returns
498 __ testptr(rdi, rdi);
499 __ jcc(Assembler::zero, skip);
500 __ call(rdi);
501
502 __ bind(skip);
503 }
504 // At this point, rax points to the value object (for interpreter or C1 caller).
505 // The fields of the object are copied into registers (for C2 caller).
506 }
507
508 // Pop the stack before the safepoint code
509 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
510
511 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
512 __ reserved_stack_check();
513 }
514
515 // Note: we do not need to round double result; float result has the right precision
516 // the poll sets the condition code, but no data registers
517
518 code_stub->set_safepoint_offset(__ offset());
519 __ relocate(relocInfo::poll_return_type);
520 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
521 __ ret(0);
522 }
523
524
525 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
526 return (__ store_inline_type_fields_to_buf(vk, false));
527 }
528
529 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
530 guarantee(info != nullptr, "Shouldn't be null");
531 int offset = __ offset();
532 const Register poll_addr = rscratch1;
533 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
534 add_debug_info_for_branch(info);
535 __ relocate(relocInfo::poll_type);
536 address pre_pc = __ pc();
537 __ testl(rax, Address(poll_addr, 0));
538 address post_pc = __ pc();
539 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
540 return offset;
541 }
542
543
544 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
545 if (from_reg != to_reg) __ mov(to_reg, from_reg);
546 }
547
548 void LIR_Assembler::swap_reg(Register a, Register b) {
1256 // init_state needs acquire, but x86 is TSO, and so we are already good.
1257 __ cmpb(Address(op->klass()->as_register(),
1258 InstanceKlass::init_state_offset()),
1259 InstanceKlass::fully_initialized);
1260 __ jcc(Assembler::notEqual, *op->stub()->entry());
1261 }
1262 __ allocate_object(op->obj()->as_register(),
1263 op->tmp1()->as_register(),
1264 op->tmp2()->as_register(),
1265 op->header_size(),
1266 op->object_size(),
1267 op->klass()->as_register(),
1268 *op->stub()->entry());
1269 __ bind(*op->stub()->continuation());
1270 }
1271
1272 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1273 Register len = op->len()->as_register();
1274 __ movslq(len, len);
1275
1276 if (UseSlowPath || op->always_slow_path() ||
1277 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1278 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1279 __ jmp(*op->stub()->entry());
1280 } else {
1281 Register tmp1 = op->tmp1()->as_register();
1282 Register tmp2 = op->tmp2()->as_register();
1283 Register tmp3 = op->tmp3()->as_register();
1284 if (len == tmp1) {
1285 tmp1 = tmp3;
1286 } else if (len == tmp2) {
1287 tmp2 = tmp3;
1288 } else if (len == tmp3) {
1289 // everything is ok
1290 } else {
1291 __ mov(tmp3, len);
1292 }
1293 __ allocate_array(op->obj()->as_register(),
1294 len,
1295 tmp1,
1296 tmp2,
1331 int bci = op->profiled_bci();
1332 md = method->method_data_or_null();
1333 assert(md != nullptr, "Sanity");
1334 data = md->bci_to_data(bci);
1335 assert(data != nullptr, "need data for type check");
1336 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1337 }
1338 Label* success_target = success;
1339 Label* failure_target = failure;
1340
1341 if (obj == k_RInfo) {
1342 k_RInfo = dst;
1343 } else if (obj == klass_RInfo) {
1344 klass_RInfo = dst;
1345 }
1346 Rtmp1 = op->tmp3()->as_register();
1347 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1348
1349 assert_different_registers(obj, k_RInfo, klass_RInfo);
1350
1351 if (op->need_null_check()) {
1352 __ testptr(obj, obj);
1353 if (op->should_profile()) {
1354 Label not_null;
1355 Register mdo = klass_RInfo;
1356 __ mov_metadata(mdo, md->constant_encoding());
1357 __ jccb(Assembler::notEqual, not_null);
1358 // Object is null; update MDO and exit
1359 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1360 int header_bits = BitData::null_seen_byte_constant();
1361 __ orb(data_addr, header_bits);
1362 __ jmp(*obj_is_null);
1363 __ bind(not_null);
1364
1365 Register recv = k_RInfo;
1366 __ load_klass(recv, obj, tmp_load_klass);
1367 type_profile_helper(mdo, md, data, recv);
1368 } else {
1369 __ jcc(Assembler::equal, *obj_is_null);
1370 }
1371 }
1372
1373 if (!k->is_loaded()) {
1374 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1375 } else {
1376 __ mov_metadata(k_RInfo, k->constant_encoding());
1377 }
1378 __ verify_oop(obj);
1379
1380 if (op->fast_check()) {
1381 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1382 // get object class
1383 // not a safepoint as obj null check happens earlier
1384 __ load_klass(Rtmp1, obj, tmp_load_klass);
1385 __ cmpptr(k_RInfo, Rtmp1);
1386 __ jcc(Assembler::notEqual, *failure_target);
1387 // successful cast, fall through to profile or jump
1388 } else {
1389 // get object class
1390 // not a safepoint as obj null check happens earlier
1391 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1392 if (k->is_loaded()) {
1393 // See if we get an immediate positive hit
1394 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1395 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1396 __ jcc(Assembler::notEqual, *failure_target);
1397 // successful cast, fall through to profile or jump
1398 } else {
1399 // See if we get an immediate positive hit
1400 __ jcc(Assembler::equal, *success_target);
1401 // check for self
1402 if (k->is_loaded() && k->is_obj_array_klass()) {
1403 // For a direct pointer comparison, we need the refined array klass pointer
1404 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1405 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1406 __ cmpptr(klass_RInfo, tmp_load_klass);
1407 } else {
1408 __ cmpptr(klass_RInfo, k_RInfo);
1409 }
1410 __ jcc(Assembler::equal, *success_target);
1411
1412 __ push_ppx(klass_RInfo);
1413 __ push_ppx(k_RInfo);
1414 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1415 __ pop_ppx(klass_RInfo);
1416 __ pop_ppx(klass_RInfo);
1417 // result is a boolean
1418 __ testl(klass_RInfo, klass_RInfo);
1419 __ jcc(Assembler::equal, *failure_target);
1420 // successful cast, fall through to profile or jump
1421 }
1422 } else {
1423 // perform the fast part of the checking logic
1424 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1425 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1426 __ push_ppx(klass_RInfo);
1427 __ push_ppx(k_RInfo);
1428 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1429 __ pop_ppx(klass_RInfo);
1519 __ mov(dst, obj);
1520 }
1521 } else
1522 if (code == lir_instanceof) {
1523 Register obj = op->object()->as_register();
1524 Register dst = op->result_opr()->as_register();
1525 Label success, failure, done;
1526 emit_typecheck_helper(op, &success, &failure, &failure);
1527 __ bind(failure);
1528 __ xorptr(dst, dst);
1529 __ jmpb(done);
1530 __ bind(success);
1531 __ movptr(dst, 1);
1532 __ bind(done);
1533 } else {
1534 ShouldNotReachHere();
1535 }
1536
1537 }
1538
1539 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1540 // We are loading/storing from/to an array that *may* be a flat array (the
1541 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1542 // If this array is a flat array, take the slow path.
1543 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1544 if (!op->value()->is_illegal()) {
1545 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1546 // The array is not a flat array, but it might be null-free. If we are storing
1547 // a null into a null-free array, take the slow path (which will throw NPE).
1548 Label skip;
1549 __ cmpptr(op->value()->as_register(), NULL_WORD);
1550 __ jcc(Assembler::notEqual, skip);
1551 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1552 __ bind(skip);
1553 }
1554 }
1555
1556 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1557 // We are storing into an array that *may* be null-free (the declared type is
1558 // Object[], abstract[], interface[] or VT.ref[]).
1559 Label test_mark_word;
1560 Register tmp = op->tmp()->as_register();
1561 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1562 __ testl(tmp, markWord::unlocked_value);
1563 __ jccb(Assembler::notZero, test_mark_word);
1564 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1565 __ bind(test_mark_word);
1566 __ testl(tmp, markWord::null_free_array_bit_in_place);
1567 }
1568
1569 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1570 Label L_oops_equal;
1571 Label L_oops_not_equal;
1572 Label L_end;
1573
1574 Register left = op->left()->as_register();
1575 Register right = op->right()->as_register();
1576
1577 __ cmpptr(left, right);
1578 __ jcc(Assembler::equal, L_oops_equal);
1579
1580 // (1) Null check -- if one of the operands is null, the other must not be null (because
1581 // the two references are not equal), so they are not substitutable,
1582 // FIXME: do null check only if the operand is nullable
1583 __ testptr(left, right);
1584 __ jcc(Assembler::zero, L_oops_not_equal);
1585
1586 ciKlass* left_klass = op->left_klass();
1587 ciKlass* right_klass = op->right_klass();
1588
1589 // (2) Inline type check -- if either of the operands is not a inline type,
1590 // they are not substitutable. We do this only if we are not sure that the
1591 // operands are inline type
1592 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1593 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1594 Register tmp = op->tmp1()->as_register();
1595 __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1596 __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1597 __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1598 __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1599 __ jcc(Assembler::notEqual, L_oops_not_equal);
1600 }
1601
1602 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1603 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1604 // No need to load klass -- the operands are statically known to be the same inline klass.
1605 __ jmp(*op->stub()->entry());
1606 } else {
1607 Register tmp1 = op->tmp1()->as_register();
1608 Register tmp2 = op->tmp2()->as_register();
1609 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1610 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1611 // fall through to L_oops_not_equal
1612 }
1613
1614 __ bind(L_oops_not_equal);
1615 move(op->not_equal_result(), op->result_opr());
1616 __ jmp(L_end);
1617
1618 __ bind(L_oops_equal);
1619 move(op->equal_result(), op->result_opr());
1620 __ jmp(L_end);
1621
1622 // We've returned from the stub. RAX contains 0x0 IFF the two
1623 // operands are not substitutable. (Don't compare against 0x1 in case the
1624 // C compiler is naughty)
1625 __ bind(*op->stub()->continuation());
1626 __ cmpl(rax, 0);
1627 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1628 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1629 // fall-through
1630 __ bind(L_end);
1631 }
1632
1633 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1634 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1635 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1636 Register newval = op->new_value()->as_register();
1637 Register cmpval = op->cmp_value()->as_register();
1638 assert(cmpval == rax, "wrong register");
1639 assert(newval != noreg, "new val must be register");
1640 assert(cmpval != newval, "cmp and new values must be in different registers");
1641 assert(cmpval != addr, "cmp and addr must be in different registers");
1642 assert(newval != addr, "new value and addr must be in different registers");
1643
1644 if (op->code() == lir_cas_obj) {
1645 if (UseCompressedOops) {
1646 __ encode_heap_oop(cmpval);
1647 __ mov(rscratch1, newval);
1648 __ encode_heap_oop(rscratch1);
1649 __ lock();
1650 // cmpval (rax) is implicitly used by this instruction
1651 __ cmpxchgl(rscratch1, Address(addr, 0));
1657 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1658 __ lock();
1659 __ cmpxchgl(newval, Address(addr, 0));
1660 }
1661 } else if (op->code() == lir_cas_long) {
1662 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1663 Register newval = op->new_value()->as_register_lo();
1664 Register cmpval = op->cmp_value()->as_register_lo();
1665 assert(cmpval == rax, "wrong register");
1666 assert(newval != noreg, "new val must be register");
1667 assert(cmpval != newval, "cmp and new values must be in different registers");
1668 assert(cmpval != addr, "cmp and addr must be in different registers");
1669 assert(newval != addr, "new value and addr must be in different registers");
1670 __ lock();
1671 __ cmpxchgq(newval, Address(addr, 0));
1672 } else {
1673 Unimplemented();
1674 }
1675 }
1676
1677 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1678 assert(dst->is_cpu_register(), "must be");
1679 assert(dst->type() == src->type(), "must be");
1680
1681 if (src->is_cpu_register()) {
1682 reg2reg(src, dst);
1683 } else if (src->is_stack()) {
1684 stack2reg(src, dst, dst->type());
1685 } else if (src->is_constant()) {
1686 const2reg(src, dst, lir_patch_none, nullptr);
1687 } else {
1688 ShouldNotReachHere();
1689 }
1690 }
1691
1692 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1693 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1694 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1695
1696 Assembler::Condition acond, ncond;
1697 switch (condition) {
1698 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1699 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1700 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1701 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1702 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1703 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1704 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1705 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1706 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1707 ShouldNotReachHere();
1708 }
1709
1710 if (opr1->is_cpu_register()) {
1711 reg2reg(opr1, result);
2282 int offset = __ offset();
2283 switch (code) {
2284 case lir_static_call:
2285 case lir_optvirtual_call:
2286 case lir_dynamic_call:
2287 offset += NativeCall::displacement_offset;
2288 break;
2289 case lir_icvirtual_call:
2290 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2291 break;
2292 default: ShouldNotReachHere();
2293 }
2294 __ align(BytesPerWord, offset);
2295 }
2296
2297
2298 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2299 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2300 "must be aligned");
2301 __ call(AddressLiteral(op->addr(), rtype));
2302 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2303 __ post_call_nop();
2304 }
2305
2306
2307 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2308 __ ic_call(op->addr());
2309 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2310 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2311 "must be aligned");
2312 __ post_call_nop();
2313 }
2314
2315
2316 void LIR_Assembler::emit_static_call_stub() {
2317 address call_pc = __ pc();
2318 address stub = __ start_a_stub(call_stub_size());
2319 if (stub == nullptr) {
2320 bailout("static call stub overflow");
2321 return;
2322 }
2323
2324 int start = __ offset();
2325
2326 // make sure that the displacement word of the call ends up word aligned
2327 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2328 __ relocate(static_stub_Relocation::spec(call_pc));
2329 __ mov_metadata(rbx, (Metadata*)nullptr);
2456 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2457 }
2458
2459
2460 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2461 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2462 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2463 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2464 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2465 }
2466
2467
2468 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2469 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2470 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2471 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2472 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2473 }
2474
2475
2476 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2477 if (null_check) {
2478 __ testptr(obj, obj);
2479 __ jcc(Assembler::zero, *slow_path->entry());
2480 }
2481 if (is_dest) {
2482 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2483 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2484 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2485 } else {
2486 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2487 }
2488 }
2489
2490
2491 // This code replaces a call to arraycopy; no exception may
2492 // be thrown in this code, they must be thrown in the System.arraycopy
2493 // activation frame; we could save some checks if this would not be the case
2494 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2495 ciArrayKlass* default_type = op->expected_type();
2496 Register src = op->src()->as_register();
2497 Register dst = op->dst()->as_register();
2498 Register src_pos = op->src_pos()->as_register();
2499 Register dst_pos = op->dst_pos()->as_register();
2500 Register length = op->length()->as_register();
2501 Register tmp = op->tmp()->as_register();
2502 Register tmp_load_klass = rscratch1;
2503 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2504
2505 CodeStub* stub = op->stub();
2506 int flags = op->flags();
2507 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2508 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2509
2510 if (flags & LIR_OpArrayCopy::always_slow_path) {
2511 __ jmp(*stub->entry());
2512 __ bind(*stub->continuation());
2513 return;
2514 }
2515
2516 // if we don't know anything, just go through the generic arraycopy
2517 if (default_type == nullptr) {
2518 // save outgoing arguments on stack in case call to System.arraycopy is needed
2519 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2520 // for interpreter calling conventions. Now we have to do it in new style conventions.
2521 // For the moment until C1 gets the new register allocator I just force all the
2522 // args to the right place (except the register args) and then on the back side
2523 // reload the register args properly if we go slow path. Yuck
2524
2525 // These are proper for the calling convention
2526 store_parameter(length, 2);
2527 store_parameter(dst_pos, 1);
2528 store_parameter(dst, 0);
2529
2530 // these are just temporary placements until we need to reload
2531 store_parameter(src_pos, 3);
2532 store_parameter(src, 4);
2533
2534 address copyfunc_addr = StubRoutines::generic_arraycopy();
2535 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2572 __ mov(tmp, rax);
2573 __ xorl(tmp, -1);
2574
2575 // Reload values from the stack so they are where the stub
2576 // expects them.
2577 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2578 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2579 __ movptr (length, Address(rsp, 2*BytesPerWord));
2580 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2581 __ movptr (src, Address(rsp, 4*BytesPerWord));
2582
2583 __ subl(length, tmp);
2584 __ addl(src_pos, tmp);
2585 __ addl(dst_pos, tmp);
2586 __ jmp(*stub->entry());
2587
2588 __ bind(*stub->continuation());
2589 return;
2590 }
2591
2592 // Handle inline type arrays
2593 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2594 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2595 }
2596 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2597 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2598 }
2599
2600 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2601
2602 int elem_size = type2aelembytes(basic_type);
2603 Address::ScaleFactor scale;
2604
2605 switch (elem_size) {
2606 case 1 :
2607 scale = Address::times_1;
2608 break;
2609 case 2 :
2610 scale = Address::times_2;
2611 break;
2612 case 4 :
2613 scale = Address::times_4;
2614 break;
2615 case 8 :
2616 scale = Address::times_8;
2617 break;
2618 default:
2619 scale = Address::no_scale;
3117 // first time here. Set profile type.
3118 __ movptr(mdo_addr, tmp);
3119 #ifdef ASSERT
3120 __ andptr(tmp, TypeEntries::type_klass_mask);
3121 __ verify_klass_ptr(tmp);
3122 #endif
3123 } else {
3124 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3125 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3126
3127 __ testptr(mdo_addr, TypeEntries::type_unknown);
3128 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3129
3130 __ orptr(mdo_addr, TypeEntries::type_unknown);
3131 }
3132 }
3133 }
3134 __ bind(next);
3135 }
3136
3137 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3138 Register obj = op->obj()->as_register();
3139 Register tmp = op->tmp()->as_pointer_register();
3140 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3141 bool not_null = op->not_null();
3142 int flag = op->flag();
3143
3144 Label not_inline_type;
3145 if (!not_null) {
3146 __ testptr(obj, obj);
3147 __ jccb(Assembler::zero, not_inline_type);
3148 }
3149
3150 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3151
3152 __ orb(mdo_addr, flag);
3153
3154 __ bind(not_inline_type);
3155 }
3156
3157
3158 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3159 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3160 }
3161
3162
3163 void LIR_Assembler::align_backward_branch_target() {
3164 __ align(BytesPerWord);
3165 }
3166
3167
3168 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3169 if (left->is_single_cpu()) {
3170 __ negl(left->as_register());
3171 move_regs(left->as_register(), dest->as_register());
3172
3173 } else if (left->is_double_cpu()) {
3174 Register lo = left->as_register_lo();
3175 Register dst = dest->as_register_lo();
3176 __ movptr(dst, lo);
3177 __ negptr(dst);
3323 }
3324
3325 void LIR_Assembler::membar_loadstore() {
3326 // no-op
3327 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3328 }
3329
3330 void LIR_Assembler::membar_storeload() {
3331 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3332 }
3333
3334 void LIR_Assembler::on_spin_wait() {
3335 __ pause ();
3336 }
3337
3338 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3339 assert(result_reg->is_register(), "check");
3340 __ mov(result_reg->as_register(), r15_thread);
3341 }
3342
3343 void LIR_Assembler::check_orig_pc() {
3344 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3345 }
3346
3347 void LIR_Assembler::peephole(LIR_List*) {
3348 // do nothing for now
3349 }
3350
3351 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3352 assert(data == dest, "xchg/xadd uses only 2 operands");
3353
3354 if (data->type() == T_INT) {
3355 if (code == lir_xadd) {
3356 __ lock();
3357 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3358 } else {
3359 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3360 }
3361 } else if (data->is_oop()) {
3362 assert (code == lir_xchg, "xadd for oops");
3363 Register obj = data->as_register();
3364 if (UseCompressedOops) {
3365 __ encode_heap_oop(obj);
|