14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
411 // Perform needed unlocking
412 MonitorExitStub* stub = nullptr;
413 if (method()->is_synchronized()) {
414 monitor_address(0, FrameMap::rax_opr);
415 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
416 __ unlock_object(rdi, rsi, rax, *stub->entry());
417 __ bind(*stub->continuation());
418 }
419
420 if (compilation()->env()->dtrace_method_probes()) {
421 __ mov(rdi, r15_thread);
422 __ mov_metadata(rsi, method()->constant_encoding());
423 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
424 }
425
426 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
427 __ mov(rax, rbx); // Restore the exception
428 }
429
430 // remove the activation and dispatch to the unwind handler
431 __ remove_frame(initial_frame_size_in_bytes());
432 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
433
434 // Emit the slow path assembly
435 if (stub != nullptr) {
436 stub->emit_code(this);
437 }
438
439 return offset;
440 }
441
442
443 int LIR_Assembler::emit_deopt_handler() {
444 // generate code for exception handler
445 address handler_base = __ start_a_stub(deopt_handler_size());
446 if (handler_base == nullptr) {
447 // not enough space left for the handler
448 bailout("deopt handler overflow");
449 return -1;
450 }
451
456
457 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
458
459 int entry_offset = __ offset();
460
461 __ jmp(start);
462
463 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
464 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
465 "out of bounds read in post-call NOP check");
466 __ end_a_stub();
467
468 return entry_offset;
469 }
470
471 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
472 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
473 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
474 assert(result->fpu() == 0, "result must already be on TOS");
475 }
476
477 // Pop the stack before the safepoint code
478 __ remove_frame(initial_frame_size_in_bytes());
479
480 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
481 __ reserved_stack_check();
482 }
483
484 // Note: we do not need to round double result; float result has the right precision
485 // the poll sets the condition code, but no data registers
486
487 code_stub->set_safepoint_offset(__ offset());
488 __ relocate(relocInfo::poll_return_type);
489 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
490 __ ret(0);
491 }
492
493
494 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
495 guarantee(info != nullptr, "Shouldn't be null");
496 int offset = __ offset();
497 const Register poll_addr = rscratch1;
498 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
499 add_debug_info_for_branch(info);
500 __ relocate(relocInfo::poll_type);
501 address pre_pc = __ pc();
502 __ testl(rax, Address(poll_addr, 0));
503 address post_pc = __ pc();
504 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
505 return offset;
506 }
507
508
509 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
510 if (from_reg != to_reg) __ mov(to_reg, from_reg);
511 }
512
513 void LIR_Assembler::swap_reg(Register a, Register b) {
1212 // init_state needs acquire, but x86 is TSO, and so we are already good.
1213 __ cmpb(Address(op->klass()->as_register(),
1214 InstanceKlass::init_state_offset()),
1215 InstanceKlass::fully_initialized);
1216 __ jcc(Assembler::notEqual, *op->stub()->entry());
1217 }
1218 __ allocate_object(op->obj()->as_register(),
1219 op->tmp1()->as_register(),
1220 op->tmp2()->as_register(),
1221 op->header_size(),
1222 op->object_size(),
1223 op->klass()->as_register(),
1224 *op->stub()->entry());
1225 __ bind(*op->stub()->continuation());
1226 }
1227
1228 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1229 Register len = op->len()->as_register();
1230 __ movslq(len, len);
1231
1232 if (UseSlowPath ||
1233 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1234 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1235 __ jmp(*op->stub()->entry());
1236 } else {
1237 Register tmp1 = op->tmp1()->as_register();
1238 Register tmp2 = op->tmp2()->as_register();
1239 Register tmp3 = op->tmp3()->as_register();
1240 if (len == tmp1) {
1241 tmp1 = tmp3;
1242 } else if (len == tmp2) {
1243 tmp2 = tmp3;
1244 } else if (len == tmp3) {
1245 // everything is ok
1246 } else {
1247 __ mov(tmp3, len);
1248 }
1249 __ allocate_array(op->obj()->as_register(),
1250 len,
1251 tmp1,
1252 tmp2,
1291 assert(data != nullptr, "need data for type check");
1292 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1293 }
1294 Label* success_target = success;
1295 Label* failure_target = failure;
1296
1297 if (obj == k_RInfo) {
1298 k_RInfo = dst;
1299 } else if (obj == klass_RInfo) {
1300 klass_RInfo = dst;
1301 }
1302 if (k->is_loaded() && !UseCompressedClassPointers) {
1303 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1304 } else {
1305 Rtmp1 = op->tmp3()->as_register();
1306 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1307 }
1308
1309 assert_different_registers(obj, k_RInfo, klass_RInfo);
1310
1311 __ testptr(obj, obj);
1312 if (op->should_profile()) {
1313 Label not_null;
1314 Register mdo = klass_RInfo;
1315 __ mov_metadata(mdo, md->constant_encoding());
1316 __ jccb(Assembler::notEqual, not_null);
1317 // Object is null; update MDO and exit
1318 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1319 int header_bits = BitData::null_seen_byte_constant();
1320 __ orb(data_addr, header_bits);
1321 __ jmp(*obj_is_null);
1322 __ bind(not_null);
1323
1324 Register recv = k_RInfo;
1325 __ load_klass(recv, obj, tmp_load_klass);
1326 type_profile_helper(mdo, md, data, recv);
1327 } else {
1328 __ jcc(Assembler::equal, *obj_is_null);
1329 }
1330
1331 if (!k->is_loaded()) {
1332 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1333 } else {
1334 __ mov_metadata(k_RInfo, k->constant_encoding());
1335 }
1336 __ verify_oop(obj);
1337
1338 if (op->fast_check()) {
1339 // get object class
1340 // not a safepoint as obj null check happens earlier
1341 if (UseCompressedClassPointers) {
1342 __ load_klass(Rtmp1, obj, tmp_load_klass);
1343 __ cmpptr(k_RInfo, Rtmp1);
1344 } else {
1345 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1346 }
1347 __ jcc(Assembler::notEqual, *failure_target);
1348 // successful cast, fall through to profile or jump
1349 } else {
1350 // get object class
1351 // not a safepoint as obj null check happens earlier
1352 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1353 if (k->is_loaded()) {
1354 // See if we get an immediate positive hit
1355 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1356 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1357 __ jcc(Assembler::notEqual, *failure_target);
1358 // successful cast, fall through to profile or jump
1359 } else {
1360 // See if we get an immediate positive hit
1361 __ jcc(Assembler::equal, *success_target);
1362 // check for self
1363 __ cmpptr(klass_RInfo, k_RInfo);
1364 __ jcc(Assembler::equal, *success_target);
1365
1366 __ push_ppx(klass_RInfo);
1367 __ push_ppx(k_RInfo);
1368 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1369 __ pop_ppx(klass_RInfo);
1370 __ pop_ppx(klass_RInfo);
1371 // result is a boolean
1372 __ testl(klass_RInfo, klass_RInfo);
1373 __ jcc(Assembler::equal, *failure_target);
1374 // successful cast, fall through to profile or jump
1375 }
1376 } else {
1377 // perform the fast part of the checking logic
1378 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1379 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1380 __ push_ppx(klass_RInfo);
1381 __ push_ppx(k_RInfo);
1382 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1383 __ pop_ppx(klass_RInfo);
1473 __ mov(dst, obj);
1474 }
1475 } else
1476 if (code == lir_instanceof) {
1477 Register obj = op->object()->as_register();
1478 Register dst = op->result_opr()->as_register();
1479 Label success, failure, done;
1480 emit_typecheck_helper(op, &success, &failure, &failure);
1481 __ bind(failure);
1482 __ xorptr(dst, dst);
1483 __ jmpb(done);
1484 __ bind(success);
1485 __ movptr(dst, 1);
1486 __ bind(done);
1487 } else {
1488 ShouldNotReachHere();
1489 }
1490
1491 }
1492
1493
1494 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1495 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1496 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1497 Register newval = op->new_value()->as_register();
1498 Register cmpval = op->cmp_value()->as_register();
1499 assert(cmpval == rax, "wrong register");
1500 assert(newval != noreg, "new val must be register");
1501 assert(cmpval != newval, "cmp and new values must be in different registers");
1502 assert(cmpval != addr, "cmp and addr must be in different registers");
1503 assert(newval != addr, "new value and addr must be in different registers");
1504
1505 if (op->code() == lir_cas_obj) {
1506 if (UseCompressedOops) {
1507 __ encode_heap_oop(cmpval);
1508 __ mov(rscratch1, newval);
1509 __ encode_heap_oop(rscratch1);
1510 __ lock();
1511 // cmpval (rax) is implicitly used by this instruction
1512 __ cmpxchgl(rscratch1, Address(addr, 0));
1518 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1519 __ lock();
1520 __ cmpxchgl(newval, Address(addr, 0));
1521 }
1522 } else if (op->code() == lir_cas_long) {
1523 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1524 Register newval = op->new_value()->as_register_lo();
1525 Register cmpval = op->cmp_value()->as_register_lo();
1526 assert(cmpval == rax, "wrong register");
1527 assert(newval != noreg, "new val must be register");
1528 assert(cmpval != newval, "cmp and new values must be in different registers");
1529 assert(cmpval != addr, "cmp and addr must be in different registers");
1530 assert(newval != addr, "new value and addr must be in different registers");
1531 __ lock();
1532 __ cmpxchgq(newval, Address(addr, 0));
1533 } else {
1534 Unimplemented();
1535 }
1536 }
1537
1538 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1539 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1540 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1541
1542 Assembler::Condition acond, ncond;
1543 switch (condition) {
1544 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1545 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1546 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1547 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1548 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1549 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1550 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1551 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1552 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1553 ShouldNotReachHere();
1554 }
1555
1556 if (opr1->is_cpu_register()) {
1557 reg2reg(opr1, result);
2128 int offset = __ offset();
2129 switch (code) {
2130 case lir_static_call:
2131 case lir_optvirtual_call:
2132 case lir_dynamic_call:
2133 offset += NativeCall::displacement_offset;
2134 break;
2135 case lir_icvirtual_call:
2136 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2137 break;
2138 default: ShouldNotReachHere();
2139 }
2140 __ align(BytesPerWord, offset);
2141 }
2142
2143
2144 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2145 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2146 "must be aligned");
2147 __ call(AddressLiteral(op->addr(), rtype));
2148 add_call_info(code_offset(), op->info());
2149 __ post_call_nop();
2150 }
2151
2152
2153 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2154 __ ic_call(op->addr());
2155 add_call_info(code_offset(), op->info());
2156 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2157 "must be aligned");
2158 __ post_call_nop();
2159 }
2160
2161
2162 void LIR_Assembler::emit_static_call_stub() {
2163 address call_pc = __ pc();
2164 address stub = __ start_a_stub(call_stub_size());
2165 if (stub == nullptr) {
2166 bailout("static call stub overflow");
2167 return;
2168 }
2169
2170 int start = __ offset();
2171
2172 // make sure that the displacement word of the call ends up word aligned
2173 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2174 __ relocate(static_stub_Relocation::spec(call_pc));
2175 __ mov_metadata(rbx, (Metadata*)nullptr);
2302 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2303 }
2304
2305
2306 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2307 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2308 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2309 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2310 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2311 }
2312
2313
2314 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2315 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2316 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2317 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2318 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2319 }
2320
2321
2322 // This code replaces a call to arraycopy; no exception may
2323 // be thrown in this code, they must be thrown in the System.arraycopy
2324 // activation frame; we could save some checks if this would not be the case
2325 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2326 ciArrayKlass* default_type = op->expected_type();
2327 Register src = op->src()->as_register();
2328 Register dst = op->dst()->as_register();
2329 Register src_pos = op->src_pos()->as_register();
2330 Register dst_pos = op->dst_pos()->as_register();
2331 Register length = op->length()->as_register();
2332 Register tmp = op->tmp()->as_register();
2333 Register tmp_load_klass = rscratch1;
2334 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2335
2336 CodeStub* stub = op->stub();
2337 int flags = op->flags();
2338 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2339 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2340
2341 // if we don't know anything, just go through the generic arraycopy
2342 if (default_type == nullptr) {
2343 // save outgoing arguments on stack in case call to System.arraycopy is needed
2344 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2345 // for interpreter calling conventions. Now we have to do it in new style conventions.
2346 // For the moment until C1 gets the new register allocator I just force all the
2347 // args to the right place (except the register args) and then on the back side
2348 // reload the register args properly if we go slow path. Yuck
2349
2350 // These are proper for the calling convention
2351 store_parameter(length, 2);
2352 store_parameter(dst_pos, 1);
2353 store_parameter(dst, 0);
2354
2355 // these are just temporary placements until we need to reload
2356 store_parameter(src_pos, 3);
2357 store_parameter(src, 4);
2358
2359 address copyfunc_addr = StubRoutines::generic_arraycopy();
2360 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2397 __ mov(tmp, rax);
2398 __ xorl(tmp, -1);
2399
2400 // Reload values from the stack so they are where the stub
2401 // expects them.
2402 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2403 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2404 __ movptr (length, Address(rsp, 2*BytesPerWord));
2405 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2406 __ movptr (src, Address(rsp, 4*BytesPerWord));
2407
2408 __ subl(length, tmp);
2409 __ addl(src_pos, tmp);
2410 __ addl(dst_pos, tmp);
2411 __ jmp(*stub->entry());
2412
2413 __ bind(*stub->continuation());
2414 return;
2415 }
2416
2417 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2418
2419 int elem_size = type2aelembytes(basic_type);
2420 Address::ScaleFactor scale;
2421
2422 switch (elem_size) {
2423 case 1 :
2424 scale = Address::times_1;
2425 break;
2426 case 2 :
2427 scale = Address::times_2;
2428 break;
2429 case 4 :
2430 scale = Address::times_4;
2431 break;
2432 case 8 :
2433 scale = Address::times_8;
2434 break;
2435 default:
2436 scale = Address::no_scale;
2623 }
2624
2625 __ jmp(*stub->entry());
2626
2627 __ bind(cont);
2628 __ pop(dst);
2629 __ pop(src);
2630 }
2631 }
2632
2633 #ifdef ASSERT
2634 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2635 // Sanity check the known type with the incoming class. For the
2636 // primitive case the types must match exactly with src.klass and
2637 // dst.klass each exactly matching the default type. For the
2638 // object array case, if no type check is needed then either the
2639 // dst type is exactly the expected type and the src type is a
2640 // subtype which we can't check or src is the same array as dst
2641 // but not necessarily exactly of type default_type.
2642 Label known_ok, halt;
2643 __ mov_metadata(tmp, default_type->constant_encoding());
2644 if (UseCompressedClassPointers) {
2645 __ encode_klass_not_null(tmp, rscratch1);
2646 }
2647
2648 if (basic_type != T_OBJECT) {
2649 __ cmp_klass(tmp, dst, tmp2);
2650 __ jcc(Assembler::notEqual, halt);
2651 __ cmp_klass(tmp, src, tmp2);
2652 __ jcc(Assembler::equal, known_ok);
2653 } else {
2654 __ cmp_klass(tmp, dst, tmp2);
2655 __ jcc(Assembler::equal, known_ok);
2656 __ cmpptr(src, dst);
2657 __ jcc(Assembler::equal, known_ok);
2658 }
2659 __ bind(halt);
2660 __ stop("incorrect type information in arraycopy");
2661 __ bind(known_ok);
2662 }
2936 // first time here. Set profile type.
2937 __ movptr(mdo_addr, tmp);
2938 #ifdef ASSERT
2939 __ andptr(tmp, TypeEntries::type_klass_mask);
2940 __ verify_klass_ptr(tmp);
2941 #endif
2942 } else {
2943 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2944 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2945
2946 __ testptr(mdo_addr, TypeEntries::type_unknown);
2947 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2948
2949 __ orptr(mdo_addr, TypeEntries::type_unknown);
2950 }
2951 }
2952 }
2953 __ bind(next);
2954 }
2955
2956 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2957 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2958 }
2959
2960
2961 void LIR_Assembler::align_backward_branch_target() {
2962 __ align(BytesPerWord);
2963 }
2964
2965
2966 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2967 if (left->is_single_cpu()) {
2968 __ negl(left->as_register());
2969 move_regs(left->as_register(), dest->as_register());
2970
2971 } else if (left->is_double_cpu()) {
2972 Register lo = left->as_register_lo();
2973 Register dst = dest->as_register_lo();
2974 __ movptr(dst, lo);
2975 __ negptr(dst);
3121 }
3122
3123 void LIR_Assembler::membar_loadstore() {
3124 // no-op
3125 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3126 }
3127
3128 void LIR_Assembler::membar_storeload() {
3129 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3130 }
3131
3132 void LIR_Assembler::on_spin_wait() {
3133 __ pause ();
3134 }
3135
3136 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3137 assert(result_reg->is_register(), "check");
3138 __ mov(result_reg->as_register(), r15_thread);
3139 }
3140
3141
3142 void LIR_Assembler::peephole(LIR_List*) {
3143 // do nothing for now
3144 }
3145
3146 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3147 assert(data == dest, "xchg/xadd uses only 2 operands");
3148
3149 if (data->type() == T_INT) {
3150 if (code == lir_xadd) {
3151 __ lock();
3152 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3153 } else {
3154 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3155 }
3156 } else if (data->is_oop()) {
3157 assert (code == lir_xchg, "xadd for oops");
3158 Register obj = data->as_register();
3159 if (UseCompressedOops) {
3160 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
459
460 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
461
462 int entry_offset = __ offset();
463
464 __ jmp(start);
465
466 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
467 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
468 "out of bounds read in post-call NOP check");
469 __ end_a_stub();
470
471 return entry_offset;
472 }
473
474 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
475 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
476 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
477 assert(result->fpu() == 0, "result must already be on TOS");
478 }
479 if (InlineTypeReturnedAsFields) {
480 #ifndef _LP64
481 Unimplemented();
482 #endif
483 // Check if we are returning an non-null inline type and load its fields into registers
484 ciType* return_type = compilation()->method()->return_type();
485 if (return_type->is_inlinetype()) {
486 ciInlineKlass* vk = return_type->as_inline_klass();
487 if (vk->can_be_returned_as_fields()) {
488 address unpack_handler = vk->unpack_handler();
489 assert(unpack_handler != nullptr, "must be");
490 __ call(RuntimeAddress(unpack_handler));
491 }
492 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
493 Label skip;
494 Label not_null;
495 __ testptr(rax, rax);
496 __ jcc(Assembler::notZero, not_null);
497 // Returned value is null, zero all return registers because they may belong to oop fields
498 __ xorq(j_rarg1, j_rarg1);
499 __ xorq(j_rarg2, j_rarg2);
500 __ xorq(j_rarg3, j_rarg3);
501 __ xorq(j_rarg4, j_rarg4);
502 __ xorq(j_rarg5, j_rarg5);
503 __ jmp(skip);
504 __ bind(not_null);
505
506 // Check if we are returning an non-null inline type and load its fields into registers
507 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
508
509 // Load fields from a buffered value with an inline class specific handler
510 __ load_klass(rdi, rax, rscratch1);
511 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
512 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
513 // Unpack handler can be null if inline type is not scalarizable in returns
514 __ testptr(rdi, rdi);
515 __ jcc(Assembler::zero, skip);
516 __ call(rdi);
517
518 __ bind(skip);
519 }
520 // At this point, rax points to the value object (for interpreter or C1 caller).
521 // The fields of the object are copied into registers (for C2 caller).
522 }
523
524 // Pop the stack before the safepoint code
525 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
526
527 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
528 __ reserved_stack_check();
529 }
530
531 // Note: we do not need to round double result; float result has the right precision
532 // the poll sets the condition code, but no data registers
533
534 code_stub->set_safepoint_offset(__ offset());
535 __ relocate(relocInfo::poll_return_type);
536 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
537 __ ret(0);
538 }
539
540
541 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
542 return (__ store_inline_type_fields_to_buf(vk, false));
543 }
544
545 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
546 guarantee(info != nullptr, "Shouldn't be null");
547 int offset = __ offset();
548 const Register poll_addr = rscratch1;
549 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
550 add_debug_info_for_branch(info);
551 __ relocate(relocInfo::poll_type);
552 address pre_pc = __ pc();
553 __ testl(rax, Address(poll_addr, 0));
554 address post_pc = __ pc();
555 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
556 return offset;
557 }
558
559
560 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
561 if (from_reg != to_reg) __ mov(to_reg, from_reg);
562 }
563
564 void LIR_Assembler::swap_reg(Register a, Register b) {
1263 // init_state needs acquire, but x86 is TSO, and so we are already good.
1264 __ cmpb(Address(op->klass()->as_register(),
1265 InstanceKlass::init_state_offset()),
1266 InstanceKlass::fully_initialized);
1267 __ jcc(Assembler::notEqual, *op->stub()->entry());
1268 }
1269 __ allocate_object(op->obj()->as_register(),
1270 op->tmp1()->as_register(),
1271 op->tmp2()->as_register(),
1272 op->header_size(),
1273 op->object_size(),
1274 op->klass()->as_register(),
1275 *op->stub()->entry());
1276 __ bind(*op->stub()->continuation());
1277 }
1278
1279 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1280 Register len = op->len()->as_register();
1281 __ movslq(len, len);
1282
1283 if (UseSlowPath || op->always_slow_path() ||
1284 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1285 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1286 __ jmp(*op->stub()->entry());
1287 } else {
1288 Register tmp1 = op->tmp1()->as_register();
1289 Register tmp2 = op->tmp2()->as_register();
1290 Register tmp3 = op->tmp3()->as_register();
1291 if (len == tmp1) {
1292 tmp1 = tmp3;
1293 } else if (len == tmp2) {
1294 tmp2 = tmp3;
1295 } else if (len == tmp3) {
1296 // everything is ok
1297 } else {
1298 __ mov(tmp3, len);
1299 }
1300 __ allocate_array(op->obj()->as_register(),
1301 len,
1302 tmp1,
1303 tmp2,
1342 assert(data != nullptr, "need data for type check");
1343 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1344 }
1345 Label* success_target = success;
1346 Label* failure_target = failure;
1347
1348 if (obj == k_RInfo) {
1349 k_RInfo = dst;
1350 } else if (obj == klass_RInfo) {
1351 klass_RInfo = dst;
1352 }
1353 if (k->is_loaded() && !UseCompressedClassPointers) {
1354 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1355 } else {
1356 Rtmp1 = op->tmp3()->as_register();
1357 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1358 }
1359
1360 assert_different_registers(obj, k_RInfo, klass_RInfo);
1361
1362 if (op->need_null_check()) {
1363 __ testptr(obj, obj);
1364 if (op->should_profile()) {
1365 Label not_null;
1366 Register mdo = klass_RInfo;
1367 __ mov_metadata(mdo, md->constant_encoding());
1368 __ jccb(Assembler::notEqual, not_null);
1369 // Object is null; update MDO and exit
1370 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1371 int header_bits = BitData::null_seen_byte_constant();
1372 __ orb(data_addr, header_bits);
1373 __ jmp(*obj_is_null);
1374 __ bind(not_null);
1375
1376 Register recv = k_RInfo;
1377 __ load_klass(recv, obj, tmp_load_klass);
1378 type_profile_helper(mdo, md, data, recv);
1379 } else {
1380 __ jcc(Assembler::equal, *obj_is_null);
1381 }
1382 }
1383
1384 if (!k->is_loaded()) {
1385 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1386 } else {
1387 __ mov_metadata(k_RInfo, k->constant_encoding());
1388 }
1389 __ verify_oop(obj);
1390
1391 if (op->fast_check()) {
1392 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1393 // get object class
1394 // not a safepoint as obj null check happens earlier
1395 if (UseCompressedClassPointers) {
1396 __ load_klass(Rtmp1, obj, tmp_load_klass);
1397 __ cmpptr(k_RInfo, Rtmp1);
1398 } else {
1399 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1400 }
1401 __ jcc(Assembler::notEqual, *failure_target);
1402 // successful cast, fall through to profile or jump
1403 } else {
1404 // get object class
1405 // not a safepoint as obj null check happens earlier
1406 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1407 if (k->is_loaded()) {
1408 // See if we get an immediate positive hit
1409 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1410 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // See if we get an immediate positive hit
1415 __ jcc(Assembler::equal, *success_target);
1416 // check for self
1417 if (k->is_loaded() && k->is_obj_array_klass()) {
1418 // For a direct pointer comparison, we need the refined array klass pointer
1419 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1420 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1421 __ cmpptr(klass_RInfo, tmp_load_klass);
1422 } else {
1423 __ cmpptr(klass_RInfo, k_RInfo);
1424 }
1425 __ jcc(Assembler::equal, *success_target);
1426
1427 __ push_ppx(klass_RInfo);
1428 __ push_ppx(k_RInfo);
1429 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1430 __ pop_ppx(klass_RInfo);
1431 __ pop_ppx(klass_RInfo);
1432 // result is a boolean
1433 __ testl(klass_RInfo, klass_RInfo);
1434 __ jcc(Assembler::equal, *failure_target);
1435 // successful cast, fall through to profile or jump
1436 }
1437 } else {
1438 // perform the fast part of the checking logic
1439 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1440 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1441 __ push_ppx(klass_RInfo);
1442 __ push_ppx(k_RInfo);
1443 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1444 __ pop_ppx(klass_RInfo);
1534 __ mov(dst, obj);
1535 }
1536 } else
1537 if (code == lir_instanceof) {
1538 Register obj = op->object()->as_register();
1539 Register dst = op->result_opr()->as_register();
1540 Label success, failure, done;
1541 emit_typecheck_helper(op, &success, &failure, &failure);
1542 __ bind(failure);
1543 __ xorptr(dst, dst);
1544 __ jmpb(done);
1545 __ bind(success);
1546 __ movptr(dst, 1);
1547 __ bind(done);
1548 } else {
1549 ShouldNotReachHere();
1550 }
1551
1552 }
1553
1554 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1555 // We are loading/storing from/to an array that *may* be a flat array (the
1556 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1557 // If this array is a flat array, take the slow path.
1558 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1559 if (!op->value()->is_illegal()) {
1560 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1561 // The array is not a flat array, but it might be null-free. If we are storing
1562 // a null into a null-free array, take the slow path (which will throw NPE).
1563 Label skip;
1564 __ cmpptr(op->value()->as_register(), NULL_WORD);
1565 __ jcc(Assembler::notEqual, skip);
1566 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1567 __ bind(skip);
1568 }
1569 }
1570
1571 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1572 // We are storing into an array that *may* be null-free (the declared type is
1573 // Object[], abstract[], interface[] or VT.ref[]).
1574 Label test_mark_word;
1575 Register tmp = op->tmp()->as_register();
1576 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1577 __ testl(tmp, markWord::unlocked_value);
1578 __ jccb(Assembler::notZero, test_mark_word);
1579 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1580 __ bind(test_mark_word);
1581 __ testl(tmp, markWord::null_free_array_bit_in_place);
1582 }
1583
1584 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1585 Label L_oops_equal;
1586 Label L_oops_not_equal;
1587 Label L_end;
1588
1589 Register left = op->left()->as_register();
1590 Register right = op->right()->as_register();
1591
1592 __ cmpptr(left, right);
1593 __ jcc(Assembler::equal, L_oops_equal);
1594
1595 // (1) Null check -- if one of the operands is null, the other must not be null (because
1596 // the two references are not equal), so they are not substitutable,
1597 // FIXME: do null check only if the operand is nullable
1598 __ testptr(left, right);
1599 __ jcc(Assembler::zero, L_oops_not_equal);
1600
1601 ciKlass* left_klass = op->left_klass();
1602 ciKlass* right_klass = op->right_klass();
1603
1604 // (2) Inline type check -- if either of the operands is not a inline type,
1605 // they are not substitutable. We do this only if we are not sure that the
1606 // operands are inline type
1607 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1608 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1609 Register tmp1 = op->tmp1()->as_register();
1610 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1611 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1612 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1613 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1614 __ jcc(Assembler::notEqual, L_oops_not_equal);
1615 }
1616
1617 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1618 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1619 // No need to load klass -- the operands are statically known to be the same inline klass.
1620 __ jmp(*op->stub()->entry());
1621 } else {
1622 Register left_klass_op = op->left_klass_op()->as_register();
1623 Register right_klass_op = op->right_klass_op()->as_register();
1624
1625 if (UseCompressedClassPointers) {
1626 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1627 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1628 __ cmpl(left_klass_op, right_klass_op);
1629 } else {
1630 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1631 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1632 __ cmpptr(left_klass_op, right_klass_op);
1633 }
1634
1635 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1636 // fall through to L_oops_not_equal
1637 }
1638
1639 __ bind(L_oops_not_equal);
1640 move(op->not_equal_result(), op->result_opr());
1641 __ jmp(L_end);
1642
1643 __ bind(L_oops_equal);
1644 move(op->equal_result(), op->result_opr());
1645 __ jmp(L_end);
1646
1647 // We've returned from the stub. RAX contains 0x0 IFF the two
1648 // operands are not substitutable. (Don't compare against 0x1 in case the
1649 // C compiler is naughty)
1650 __ bind(*op->stub()->continuation());
1651 __ cmpl(rax, 0);
1652 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1653 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1654 // fall-through
1655 __ bind(L_end);
1656 }
1657
1658 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1659 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1660 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1661 Register newval = op->new_value()->as_register();
1662 Register cmpval = op->cmp_value()->as_register();
1663 assert(cmpval == rax, "wrong register");
1664 assert(newval != noreg, "new val must be register");
1665 assert(cmpval != newval, "cmp and new values must be in different registers");
1666 assert(cmpval != addr, "cmp and addr must be in different registers");
1667 assert(newval != addr, "new value and addr must be in different registers");
1668
1669 if (op->code() == lir_cas_obj) {
1670 if (UseCompressedOops) {
1671 __ encode_heap_oop(cmpval);
1672 __ mov(rscratch1, newval);
1673 __ encode_heap_oop(rscratch1);
1674 __ lock();
1675 // cmpval (rax) is implicitly used by this instruction
1676 __ cmpxchgl(rscratch1, Address(addr, 0));
1682 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1683 __ lock();
1684 __ cmpxchgl(newval, Address(addr, 0));
1685 }
1686 } else if (op->code() == lir_cas_long) {
1687 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1688 Register newval = op->new_value()->as_register_lo();
1689 Register cmpval = op->cmp_value()->as_register_lo();
1690 assert(cmpval == rax, "wrong register");
1691 assert(newval != noreg, "new val must be register");
1692 assert(cmpval != newval, "cmp and new values must be in different registers");
1693 assert(cmpval != addr, "cmp and addr must be in different registers");
1694 assert(newval != addr, "new value and addr must be in different registers");
1695 __ lock();
1696 __ cmpxchgq(newval, Address(addr, 0));
1697 } else {
1698 Unimplemented();
1699 }
1700 }
1701
1702 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1703 assert(dst->is_cpu_register(), "must be");
1704 assert(dst->type() == src->type(), "must be");
1705
1706 if (src->is_cpu_register()) {
1707 reg2reg(src, dst);
1708 } else if (src->is_stack()) {
1709 stack2reg(src, dst, dst->type());
1710 } else if (src->is_constant()) {
1711 const2reg(src, dst, lir_patch_none, nullptr);
1712 } else {
1713 ShouldNotReachHere();
1714 }
1715 }
1716
1717 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1718 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1719 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1720
1721 Assembler::Condition acond, ncond;
1722 switch (condition) {
1723 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1724 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1725 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1726 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1727 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1728 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1729 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1730 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1731 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1732 ShouldNotReachHere();
1733 }
1734
1735 if (opr1->is_cpu_register()) {
1736 reg2reg(opr1, result);
2307 int offset = __ offset();
2308 switch (code) {
2309 case lir_static_call:
2310 case lir_optvirtual_call:
2311 case lir_dynamic_call:
2312 offset += NativeCall::displacement_offset;
2313 break;
2314 case lir_icvirtual_call:
2315 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2316 break;
2317 default: ShouldNotReachHere();
2318 }
2319 __ align(BytesPerWord, offset);
2320 }
2321
2322
2323 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2324 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2325 "must be aligned");
2326 __ call(AddressLiteral(op->addr(), rtype));
2327 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2328 __ post_call_nop();
2329 }
2330
2331
2332 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2333 __ ic_call(op->addr());
2334 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2335 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2336 "must be aligned");
2337 __ post_call_nop();
2338 }
2339
2340
2341 void LIR_Assembler::emit_static_call_stub() {
2342 address call_pc = __ pc();
2343 address stub = __ start_a_stub(call_stub_size());
2344 if (stub == nullptr) {
2345 bailout("static call stub overflow");
2346 return;
2347 }
2348
2349 int start = __ offset();
2350
2351 // make sure that the displacement word of the call ends up word aligned
2352 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2353 __ relocate(static_stub_Relocation::spec(call_pc));
2354 __ mov_metadata(rbx, (Metadata*)nullptr);
2481 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2482 }
2483
2484
2485 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2486 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2487 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2488 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2489 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2490 }
2491
2492
2493 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2494 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2495 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2496 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2497 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2498 }
2499
2500
2501 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2502 if (null_check) {
2503 __ testptr(obj, obj);
2504 __ jcc(Assembler::zero, *slow_path->entry());
2505 }
2506 if (is_dest) {
2507 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2508 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2509 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2510 } else {
2511 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2512 }
2513 }
2514
2515
2516 // This code replaces a call to arraycopy; no exception may
2517 // be thrown in this code, they must be thrown in the System.arraycopy
2518 // activation frame; we could save some checks if this would not be the case
2519 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2520 ciArrayKlass* default_type = op->expected_type();
2521 Register src = op->src()->as_register();
2522 Register dst = op->dst()->as_register();
2523 Register src_pos = op->src_pos()->as_register();
2524 Register dst_pos = op->dst_pos()->as_register();
2525 Register length = op->length()->as_register();
2526 Register tmp = op->tmp()->as_register();
2527 Register tmp_load_klass = rscratch1;
2528 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2529
2530 CodeStub* stub = op->stub();
2531 int flags = op->flags();
2532 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2533 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2534
2535 if (flags & LIR_OpArrayCopy::always_slow_path) {
2536 __ jmp(*stub->entry());
2537 __ bind(*stub->continuation());
2538 return;
2539 }
2540
2541 // if we don't know anything, just go through the generic arraycopy
2542 if (default_type == nullptr) {
2543 // save outgoing arguments on stack in case call to System.arraycopy is needed
2544 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2545 // for interpreter calling conventions. Now we have to do it in new style conventions.
2546 // For the moment until C1 gets the new register allocator I just force all the
2547 // args to the right place (except the register args) and then on the back side
2548 // reload the register args properly if we go slow path. Yuck
2549
2550 // These are proper for the calling convention
2551 store_parameter(length, 2);
2552 store_parameter(dst_pos, 1);
2553 store_parameter(dst, 0);
2554
2555 // these are just temporary placements until we need to reload
2556 store_parameter(src_pos, 3);
2557 store_parameter(src, 4);
2558
2559 address copyfunc_addr = StubRoutines::generic_arraycopy();
2560 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2597 __ mov(tmp, rax);
2598 __ xorl(tmp, -1);
2599
2600 // Reload values from the stack so they are where the stub
2601 // expects them.
2602 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2603 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2604 __ movptr (length, Address(rsp, 2*BytesPerWord));
2605 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2606 __ movptr (src, Address(rsp, 4*BytesPerWord));
2607
2608 __ subl(length, tmp);
2609 __ addl(src_pos, tmp);
2610 __ addl(dst_pos, tmp);
2611 __ jmp(*stub->entry());
2612
2613 __ bind(*stub->continuation());
2614 return;
2615 }
2616
2617 // Handle inline type arrays
2618 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2619 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2620 }
2621 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2622 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2623 }
2624
2625 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2626
2627 int elem_size = type2aelembytes(basic_type);
2628 Address::ScaleFactor scale;
2629
2630 switch (elem_size) {
2631 case 1 :
2632 scale = Address::times_1;
2633 break;
2634 case 2 :
2635 scale = Address::times_2;
2636 break;
2637 case 4 :
2638 scale = Address::times_4;
2639 break;
2640 case 8 :
2641 scale = Address::times_8;
2642 break;
2643 default:
2644 scale = Address::no_scale;
2831 }
2832
2833 __ jmp(*stub->entry());
2834
2835 __ bind(cont);
2836 __ pop(dst);
2837 __ pop(src);
2838 }
2839 }
2840
2841 #ifdef ASSERT
2842 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2843 // Sanity check the known type with the incoming class. For the
2844 // primitive case the types must match exactly with src.klass and
2845 // dst.klass each exactly matching the default type. For the
2846 // object array case, if no type check is needed then either the
2847 // dst type is exactly the expected type and the src type is a
2848 // subtype which we can't check or src is the same array as dst
2849 // but not necessarily exactly of type default_type.
2850 Label known_ok, halt;
2851
2852 __ mov_metadata(tmp, default_type->constant_encoding());
2853 if (UseCompressedClassPointers) {
2854 __ encode_klass_not_null(tmp, rscratch1);
2855 }
2856
2857 if (basic_type != T_OBJECT) {
2858 __ cmp_klass(tmp, dst, tmp2);
2859 __ jcc(Assembler::notEqual, halt);
2860 __ cmp_klass(tmp, src, tmp2);
2861 __ jcc(Assembler::equal, known_ok);
2862 } else {
2863 __ cmp_klass(tmp, dst, tmp2);
2864 __ jcc(Assembler::equal, known_ok);
2865 __ cmpptr(src, dst);
2866 __ jcc(Assembler::equal, known_ok);
2867 }
2868 __ bind(halt);
2869 __ stop("incorrect type information in arraycopy");
2870 __ bind(known_ok);
2871 }
3145 // first time here. Set profile type.
3146 __ movptr(mdo_addr, tmp);
3147 #ifdef ASSERT
3148 __ andptr(tmp, TypeEntries::type_klass_mask);
3149 __ verify_klass_ptr(tmp);
3150 #endif
3151 } else {
3152 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3153 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3154
3155 __ testptr(mdo_addr, TypeEntries::type_unknown);
3156 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3157
3158 __ orptr(mdo_addr, TypeEntries::type_unknown);
3159 }
3160 }
3161 }
3162 __ bind(next);
3163 }
3164
3165 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3166 Register obj = op->obj()->as_register();
3167 Register tmp = op->tmp()->as_pointer_register();
3168 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3169 bool not_null = op->not_null();
3170 int flag = op->flag();
3171
3172 Label not_inline_type;
3173 if (!not_null) {
3174 __ testptr(obj, obj);
3175 __ jccb(Assembler::zero, not_inline_type);
3176 }
3177
3178 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3179
3180 __ orb(mdo_addr, flag);
3181
3182 __ bind(not_inline_type);
3183 }
3184
3185
3186 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3187 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3188 }
3189
3190
3191 void LIR_Assembler::align_backward_branch_target() {
3192 __ align(BytesPerWord);
3193 }
3194
3195
3196 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3197 if (left->is_single_cpu()) {
3198 __ negl(left->as_register());
3199 move_regs(left->as_register(), dest->as_register());
3200
3201 } else if (left->is_double_cpu()) {
3202 Register lo = left->as_register_lo();
3203 Register dst = dest->as_register_lo();
3204 __ movptr(dst, lo);
3205 __ negptr(dst);
3351 }
3352
3353 void LIR_Assembler::membar_loadstore() {
3354 // no-op
3355 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3356 }
3357
3358 void LIR_Assembler::membar_storeload() {
3359 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3360 }
3361
3362 void LIR_Assembler::on_spin_wait() {
3363 __ pause ();
3364 }
3365
3366 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3367 assert(result_reg->is_register(), "check");
3368 __ mov(result_reg->as_register(), r15_thread);
3369 }
3370
3371 void LIR_Assembler::check_orig_pc() {
3372 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3373 }
3374
3375 void LIR_Assembler::peephole(LIR_List*) {
3376 // do nothing for now
3377 }
3378
3379 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3380 assert(data == dest, "xchg/xadd uses only 2 operands");
3381
3382 if (data->type() == T_INT) {
3383 if (code == lir_xadd) {
3384 __ lock();
3385 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3386 } else {
3387 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3388 }
3389 } else if (data->is_oop()) {
3390 assert (code == lir_xchg, "xadd for oops");
3391 Register obj = data->as_register();
3392 if (UseCompressedOops) {
3393 __ encode_heap_oop(obj);
|