14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
415 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
416 if (LockingMode == LM_MONITOR) {
417 __ jmp(*stub->entry());
418 } else {
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 }
421 __ bind(*stub->continuation());
422 }
423
424 if (compilation()->env()->dtrace_method_probes()) {
425 __ mov(rdi, r15_thread);
426 __ mov_metadata(rsi, method()->constant_encoding());
427 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
428 }
429
430 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
431 __ mov(rax, rbx); // Restore the exception
432 }
433
434 // remove the activation and dispatch to the unwind handler
435 __ remove_frame(initial_frame_size_in_bytes());
436 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
437
438 // Emit the slow path assembly
439 if (stub != nullptr) {
440 stub->emit_code(this);
441 }
442
443 return offset;
444 }
445
446
447 int LIR_Assembler::emit_deopt_handler() {
448 // generate code for exception handler
449 address handler_base = __ start_a_stub(deopt_handler_size());
450 if (handler_base == nullptr) {
451 // not enough space left for the handler
452 bailout("deopt handler overflow");
453 return -1;
454 }
455
456 int offset = code_offset();
457 InternalAddress here(__ pc());
458
459 __ pushptr(here.addr(), rscratch1);
460 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
461 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
462 __ end_a_stub();
463
464 return offset;
465 }
466
467 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
468 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
469 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
470 assert(result->fpu() == 0, "result must already be on TOS");
471 }
472
473 // Pop the stack before the safepoint code
474 __ remove_frame(initial_frame_size_in_bytes());
475
476 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
477 __ reserved_stack_check();
478 }
479
480 // Note: we do not need to round double result; float result has the right precision
481 // the poll sets the condition code, but no data registers
482
483 code_stub->set_safepoint_offset(__ offset());
484 __ relocate(relocInfo::poll_return_type);
485 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
486 __ ret(0);
487 }
488
489
490 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
491 guarantee(info != nullptr, "Shouldn't be null");
492 int offset = __ offset();
493 const Register poll_addr = rscratch1;
494 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
495 add_debug_info_for_branch(info);
496 __ relocate(relocInfo::poll_type);
497 address pre_pc = __ pc();
498 __ testl(rax, Address(poll_addr, 0));
499 address post_pc = __ pc();
500 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
501 return offset;
502 }
503
504
505 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
506 if (from_reg != to_reg) __ mov(to_reg, from_reg);
507 }
508
509 void LIR_Assembler::swap_reg(Register a, Register b) {
1208 // init_state needs acquire, but x86 is TSO, and so we are already good.
1209 __ cmpb(Address(op->klass()->as_register(),
1210 InstanceKlass::init_state_offset()),
1211 InstanceKlass::fully_initialized);
1212 __ jcc(Assembler::notEqual, *op->stub()->entry());
1213 }
1214 __ allocate_object(op->obj()->as_register(),
1215 op->tmp1()->as_register(),
1216 op->tmp2()->as_register(),
1217 op->header_size(),
1218 op->object_size(),
1219 op->klass()->as_register(),
1220 *op->stub()->entry());
1221 __ bind(*op->stub()->continuation());
1222 }
1223
1224 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1225 Register len = op->len()->as_register();
1226 __ movslq(len, len);
1227
1228 if (UseSlowPath ||
1229 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1230 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1231 __ jmp(*op->stub()->entry());
1232 } else {
1233 Register tmp1 = op->tmp1()->as_register();
1234 Register tmp2 = op->tmp2()->as_register();
1235 Register tmp3 = op->tmp3()->as_register();
1236 if (len == tmp1) {
1237 tmp1 = tmp3;
1238 } else if (len == tmp2) {
1239 tmp2 = tmp3;
1240 } else if (len == tmp3) {
1241 // everything is ok
1242 } else {
1243 __ mov(tmp3, len);
1244 }
1245 __ allocate_array(op->obj()->as_register(),
1246 len,
1247 tmp1,
1248 tmp2,
1307 assert(data != nullptr, "need data for type check");
1308 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1309 }
1310 Label* success_target = success;
1311 Label* failure_target = failure;
1312
1313 if (obj == k_RInfo) {
1314 k_RInfo = dst;
1315 } else if (obj == klass_RInfo) {
1316 klass_RInfo = dst;
1317 }
1318 if (k->is_loaded() && !UseCompressedClassPointers) {
1319 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1320 } else {
1321 Rtmp1 = op->tmp3()->as_register();
1322 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1323 }
1324
1325 assert_different_registers(obj, k_RInfo, klass_RInfo);
1326
1327 __ testptr(obj, obj);
1328 if (op->should_profile()) {
1329 Label not_null;
1330 Register mdo = klass_RInfo;
1331 __ mov_metadata(mdo, md->constant_encoding());
1332 __ jccb(Assembler::notEqual, not_null);
1333 // Object is null; update MDO and exit
1334 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1335 int header_bits = BitData::null_seen_byte_constant();
1336 __ orb(data_addr, header_bits);
1337 __ jmp(*obj_is_null);
1338 __ bind(not_null);
1339
1340 Label update_done;
1341 Register recv = k_RInfo;
1342 __ load_klass(recv, obj, tmp_load_klass);
1343 type_profile_helper(mdo, md, data, recv, &update_done);
1344
1345 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1346 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1347
1348 __ bind(update_done);
1349 } else {
1350 __ jcc(Assembler::equal, *obj_is_null);
1351 }
1352
1353 if (!k->is_loaded()) {
1354 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1355 } else {
1356 __ mov_metadata(k_RInfo, k->constant_encoding());
1357 }
1358 __ verify_oop(obj);
1359
1360 if (op->fast_check()) {
1361 // get object class
1362 // not a safepoint as obj null check happens earlier
1363 if (UseCompressedClassPointers) {
1364 __ load_klass(Rtmp1, obj, tmp_load_klass);
1365 __ cmpptr(k_RInfo, Rtmp1);
1366 } else {
1367 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1368 }
1369 __ jcc(Assembler::notEqual, *failure_target);
1370 // successful cast, fall through to profile or jump
1500 __ mov(dst, obj);
1501 }
1502 } else
1503 if (code == lir_instanceof) {
1504 Register obj = op->object()->as_register();
1505 Register dst = op->result_opr()->as_register();
1506 Label success, failure, done;
1507 emit_typecheck_helper(op, &success, &failure, &failure);
1508 __ bind(failure);
1509 __ xorptr(dst, dst);
1510 __ jmpb(done);
1511 __ bind(success);
1512 __ movptr(dst, 1);
1513 __ bind(done);
1514 } else {
1515 ShouldNotReachHere();
1516 }
1517
1518 }
1519
1520
1521 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1522 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1523 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1524 Register newval = op->new_value()->as_register();
1525 Register cmpval = op->cmp_value()->as_register();
1526 assert(cmpval == rax, "wrong register");
1527 assert(newval != noreg, "new val must be register");
1528 assert(cmpval != newval, "cmp and new values must be in different registers");
1529 assert(cmpval != addr, "cmp and addr must be in different registers");
1530 assert(newval != addr, "new value and addr must be in different registers");
1531
1532 if (op->code() == lir_cas_obj) {
1533 if (UseCompressedOops) {
1534 __ encode_heap_oop(cmpval);
1535 __ mov(rscratch1, newval);
1536 __ encode_heap_oop(rscratch1);
1537 __ lock();
1538 // cmpval (rax) is implicitly used by this instruction
1539 __ cmpxchgl(rscratch1, Address(addr, 0));
1545 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1546 __ lock();
1547 __ cmpxchgl(newval, Address(addr, 0));
1548 }
1549 } else if (op->code() == lir_cas_long) {
1550 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1551 Register newval = op->new_value()->as_register_lo();
1552 Register cmpval = op->cmp_value()->as_register_lo();
1553 assert(cmpval == rax, "wrong register");
1554 assert(newval != noreg, "new val must be register");
1555 assert(cmpval != newval, "cmp and new values must be in different registers");
1556 assert(cmpval != addr, "cmp and addr must be in different registers");
1557 assert(newval != addr, "new value and addr must be in different registers");
1558 __ lock();
1559 __ cmpxchgq(newval, Address(addr, 0));
1560 } else {
1561 Unimplemented();
1562 }
1563 }
1564
1565 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1566 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1567 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1568
1569 Assembler::Condition acond, ncond;
1570 switch (condition) {
1571 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1572 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1573 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1574 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1575 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1576 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1577 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1578 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1579 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1580 ShouldNotReachHere();
1581 }
1582
1583 if (opr1->is_cpu_register()) {
1584 reg2reg(opr1, result);
2155 int offset = __ offset();
2156 switch (code) {
2157 case lir_static_call:
2158 case lir_optvirtual_call:
2159 case lir_dynamic_call:
2160 offset += NativeCall::displacement_offset;
2161 break;
2162 case lir_icvirtual_call:
2163 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2164 break;
2165 default: ShouldNotReachHere();
2166 }
2167 __ align(BytesPerWord, offset);
2168 }
2169
2170
2171 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2172 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2173 "must be aligned");
2174 __ call(AddressLiteral(op->addr(), rtype));
2175 add_call_info(code_offset(), op->info());
2176 __ post_call_nop();
2177 }
2178
2179
2180 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2181 __ ic_call(op->addr());
2182 add_call_info(code_offset(), op->info());
2183 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2184 "must be aligned");
2185 __ post_call_nop();
2186 }
2187
2188
2189 void LIR_Assembler::emit_static_call_stub() {
2190 address call_pc = __ pc();
2191 address stub = __ start_a_stub(call_stub_size());
2192 if (stub == nullptr) {
2193 bailout("static call stub overflow");
2194 return;
2195 }
2196
2197 int start = __ offset();
2198
2199 // make sure that the displacement word of the call ends up word aligned
2200 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2201 __ relocate(static_stub_Relocation::spec(call_pc));
2202 __ mov_metadata(rbx, (Metadata*)nullptr);
2329 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2330 }
2331
2332
2333 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2334 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2335 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2336 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2337 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2338 }
2339
2340
2341 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2342 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2343 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2344 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2345 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2346 }
2347
2348
2349 // This code replaces a call to arraycopy; no exception may
2350 // be thrown in this code, they must be thrown in the System.arraycopy
2351 // activation frame; we could save some checks if this would not be the case
2352 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2353 ciArrayKlass* default_type = op->expected_type();
2354 Register src = op->src()->as_register();
2355 Register dst = op->dst()->as_register();
2356 Register src_pos = op->src_pos()->as_register();
2357 Register dst_pos = op->dst_pos()->as_register();
2358 Register length = op->length()->as_register();
2359 Register tmp = op->tmp()->as_register();
2360 Register tmp_load_klass = rscratch1;
2361 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2362
2363 CodeStub* stub = op->stub();
2364 int flags = op->flags();
2365 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2366 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2367
2368 // if we don't know anything, just go through the generic arraycopy
2369 if (default_type == nullptr) {
2370 // save outgoing arguments on stack in case call to System.arraycopy is needed
2371 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2372 // for interpreter calling conventions. Now we have to do it in new style conventions.
2373 // For the moment until C1 gets the new register allocator I just force all the
2374 // args to the right place (except the register args) and then on the back side
2375 // reload the register args properly if we go slow path. Yuck
2376
2377 // These are proper for the calling convention
2378 store_parameter(length, 2);
2379 store_parameter(dst_pos, 1);
2380 store_parameter(dst, 0);
2381
2382 // these are just temporary placements until we need to reload
2383 store_parameter(src_pos, 3);
2384 store_parameter(src, 4);
2385
2386 address copyfunc_addr = StubRoutines::generic_arraycopy();
2387 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2424 __ mov(tmp, rax);
2425 __ xorl(tmp, -1);
2426
2427 // Reload values from the stack so they are where the stub
2428 // expects them.
2429 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2430 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2431 __ movptr (length, Address(rsp, 2*BytesPerWord));
2432 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2433 __ movptr (src, Address(rsp, 4*BytesPerWord));
2434
2435 __ subl(length, tmp);
2436 __ addl(src_pos, tmp);
2437 __ addl(dst_pos, tmp);
2438 __ jmp(*stub->entry());
2439
2440 __ bind(*stub->continuation());
2441 return;
2442 }
2443
2444 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2445
2446 int elem_size = type2aelembytes(basic_type);
2447 Address::ScaleFactor scale;
2448
2449 switch (elem_size) {
2450 case 1 :
2451 scale = Address::times_1;
2452 break;
2453 case 2 :
2454 scale = Address::times_2;
2455 break;
2456 case 4 :
2457 scale = Address::times_4;
2458 break;
2459 case 8 :
2460 scale = Address::times_8;
2461 break;
2462 default:
2463 scale = Address::no_scale;
2994 // first time here. Set profile type.
2995 __ movptr(mdo_addr, tmp);
2996 #ifdef ASSERT
2997 __ andptr(tmp, TypeEntries::type_klass_mask);
2998 __ verify_klass_ptr(tmp);
2999 #endif
3000 } else {
3001 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3002 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3003
3004 __ testptr(mdo_addr, TypeEntries::type_unknown);
3005 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3006
3007 __ orptr(mdo_addr, TypeEntries::type_unknown);
3008 }
3009 }
3010 }
3011 __ bind(next);
3012 }
3013
3014 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3015 Unimplemented();
3016 }
3017
3018
3019 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3020 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3021 }
3022
3023
3024 void LIR_Assembler::align_backward_branch_target() {
3025 __ align(BytesPerWord);
3026 }
3027
3028
3029 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3030 if (left->is_single_cpu()) {
3031 __ negl(left->as_register());
3032 move_regs(left->as_register(), dest->as_register());
3033
3184 }
3185
3186 void LIR_Assembler::membar_loadstore() {
3187 // no-op
3188 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3189 }
3190
3191 void LIR_Assembler::membar_storeload() {
3192 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3193 }
3194
3195 void LIR_Assembler::on_spin_wait() {
3196 __ pause ();
3197 }
3198
3199 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3200 assert(result_reg->is_register(), "check");
3201 __ mov(result_reg->as_register(), r15_thread);
3202 }
3203
3204
3205 void LIR_Assembler::peephole(LIR_List*) {
3206 // do nothing for now
3207 }
3208
3209 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3210 assert(data == dest, "xchg/xadd uses only 2 operands");
3211
3212 if (data->type() == T_INT) {
3213 if (code == lir_xadd) {
3214 __ lock();
3215 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3216 } else {
3217 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3218 }
3219 } else if (data->is_oop()) {
3220 assert (code == lir_xchg, "xadd for oops");
3221 Register obj = data->as_register();
3222 if (UseCompressedOops) {
3223 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
60 operand[0] = lo;
417 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
418 if (LockingMode == LM_MONITOR) {
419 __ jmp(*stub->entry());
420 } else {
421 __ unlock_object(rdi, rsi, rax, *stub->entry());
422 }
423 __ bind(*stub->continuation());
424 }
425
426 if (compilation()->env()->dtrace_method_probes()) {
427 __ mov(rdi, r15_thread);
428 __ mov_metadata(rsi, method()->constant_encoding());
429 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
430 }
431
432 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
433 __ mov(rax, rbx); // Restore the exception
434 }
435
436 // remove the activation and dispatch to the unwind handler
437 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
438 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
439
440 // Emit the slow path assembly
441 if (stub != nullptr) {
442 stub->emit_code(this);
443 }
444
445 return offset;
446 }
447
448
449 int LIR_Assembler::emit_deopt_handler() {
450 // generate code for exception handler
451 address handler_base = __ start_a_stub(deopt_handler_size());
452 if (handler_base == nullptr) {
453 // not enough space left for the handler
454 bailout("deopt handler overflow");
455 return -1;
456 }
457
458 int offset = code_offset();
459 InternalAddress here(__ pc());
460
461 __ pushptr(here.addr(), rscratch1);
462 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
463 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
464 __ end_a_stub();
465
466 return offset;
467 }
468
469 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
470 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
471 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
472 assert(result->fpu() == 0, "result must already be on TOS");
473 }
474 if (InlineTypeReturnedAsFields) {
475 #ifndef _LP64
476 Unimplemented();
477 #endif
478 // Check if we are returning an non-null inline type and load its fields into registers
479 ciType* return_type = compilation()->method()->return_type();
480 if (return_type->is_inlinetype()) {
481 ciInlineKlass* vk = return_type->as_inline_klass();
482 if (vk->can_be_returned_as_fields()) {
483 address unpack_handler = vk->unpack_handler();
484 assert(unpack_handler != nullptr, "must be");
485 __ call(RuntimeAddress(unpack_handler));
486 }
487 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
488 Label skip;
489 __ test_oop_is_not_inline_type(rax, rscratch1, skip);
490
491 // Load fields from a buffered value with an inline class specific handler
492 __ load_klass(rdi, rax, rscratch1);
493 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
494 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
495 // Unpack handler can be null if inline type is not scalarizable in returns
496 __ testptr(rdi, rdi);
497 __ jcc(Assembler::zero, skip);
498 __ call(rdi);
499
500 __ bind(skip);
501 }
502 // At this point, rax points to the value object (for interpreter or C1 caller).
503 // The fields of the object are copied into registers (for C2 caller).
504 }
505
506 // Pop the stack before the safepoint code
507 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
508
509 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
510 __ reserved_stack_check();
511 }
512
513 // Note: we do not need to round double result; float result has the right precision
514 // the poll sets the condition code, but no data registers
515
516 code_stub->set_safepoint_offset(__ offset());
517 __ relocate(relocInfo::poll_return_type);
518 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
519 __ ret(0);
520 }
521
522
523 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
524 return (__ store_inline_type_fields_to_buf(vk, false));
525 }
526
527 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
528 guarantee(info != nullptr, "Shouldn't be null");
529 int offset = __ offset();
530 const Register poll_addr = rscratch1;
531 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
532 add_debug_info_for_branch(info);
533 __ relocate(relocInfo::poll_type);
534 address pre_pc = __ pc();
535 __ testl(rax, Address(poll_addr, 0));
536 address post_pc = __ pc();
537 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
538 return offset;
539 }
540
541
542 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
543 if (from_reg != to_reg) __ mov(to_reg, from_reg);
544 }
545
546 void LIR_Assembler::swap_reg(Register a, Register b) {
1245 // init_state needs acquire, but x86 is TSO, and so we are already good.
1246 __ cmpb(Address(op->klass()->as_register(),
1247 InstanceKlass::init_state_offset()),
1248 InstanceKlass::fully_initialized);
1249 __ jcc(Assembler::notEqual, *op->stub()->entry());
1250 }
1251 __ allocate_object(op->obj()->as_register(),
1252 op->tmp1()->as_register(),
1253 op->tmp2()->as_register(),
1254 op->header_size(),
1255 op->object_size(),
1256 op->klass()->as_register(),
1257 *op->stub()->entry());
1258 __ bind(*op->stub()->continuation());
1259 }
1260
1261 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1262 Register len = op->len()->as_register();
1263 __ movslq(len, len);
1264
1265 if (UseSlowPath || op->is_null_free() ||
1266 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1267 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1268 __ jmp(*op->stub()->entry());
1269 } else {
1270 Register tmp1 = op->tmp1()->as_register();
1271 Register tmp2 = op->tmp2()->as_register();
1272 Register tmp3 = op->tmp3()->as_register();
1273 if (len == tmp1) {
1274 tmp1 = tmp3;
1275 } else if (len == tmp2) {
1276 tmp2 = tmp3;
1277 } else if (len == tmp3) {
1278 // everything is ok
1279 } else {
1280 __ mov(tmp3, len);
1281 }
1282 __ allocate_array(op->obj()->as_register(),
1283 len,
1284 tmp1,
1285 tmp2,
1344 assert(data != nullptr, "need data for type check");
1345 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1346 }
1347 Label* success_target = success;
1348 Label* failure_target = failure;
1349
1350 if (obj == k_RInfo) {
1351 k_RInfo = dst;
1352 } else if (obj == klass_RInfo) {
1353 klass_RInfo = dst;
1354 }
1355 if (k->is_loaded() && !UseCompressedClassPointers) {
1356 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1357 } else {
1358 Rtmp1 = op->tmp3()->as_register();
1359 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1360 }
1361
1362 assert_different_registers(obj, k_RInfo, klass_RInfo);
1363
1364 if (op->need_null_check()) {
1365 __ testptr(obj, obj);
1366 if (op->should_profile()) {
1367 Label not_null;
1368 Register mdo = klass_RInfo;
1369 __ mov_metadata(mdo, md->constant_encoding());
1370 __ jccb(Assembler::notEqual, not_null);
1371 // Object is null; update MDO and exit
1372 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1373 int header_bits = BitData::null_seen_byte_constant();
1374 __ orb(data_addr, header_bits);
1375 __ jmp(*obj_is_null);
1376 __ bind(not_null);
1377
1378 Label update_done;
1379 Register recv = k_RInfo;
1380 __ load_klass(recv, obj, tmp_load_klass);
1381 type_profile_helper(mdo, md, data, recv, &update_done);
1382
1383 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1384 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1385
1386 __ bind(update_done);
1387 } else {
1388 __ jcc(Assembler::equal, *obj_is_null);
1389 }
1390 }
1391
1392 if (!k->is_loaded()) {
1393 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1394 } else {
1395 __ mov_metadata(k_RInfo, k->constant_encoding());
1396 }
1397 __ verify_oop(obj);
1398
1399 if (op->fast_check()) {
1400 // get object class
1401 // not a safepoint as obj null check happens earlier
1402 if (UseCompressedClassPointers) {
1403 __ load_klass(Rtmp1, obj, tmp_load_klass);
1404 __ cmpptr(k_RInfo, Rtmp1);
1405 } else {
1406 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1407 }
1408 __ jcc(Assembler::notEqual, *failure_target);
1409 // successful cast, fall through to profile or jump
1539 __ mov(dst, obj);
1540 }
1541 } else
1542 if (code == lir_instanceof) {
1543 Register obj = op->object()->as_register();
1544 Register dst = op->result_opr()->as_register();
1545 Label success, failure, done;
1546 emit_typecheck_helper(op, &success, &failure, &failure);
1547 __ bind(failure);
1548 __ xorptr(dst, dst);
1549 __ jmpb(done);
1550 __ bind(success);
1551 __ movptr(dst, 1);
1552 __ bind(done);
1553 } else {
1554 ShouldNotReachHere();
1555 }
1556
1557 }
1558
1559 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1560 // We are loading/storing from/to an array that *may* be a flat array (the
1561 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1562 // If this array is a flat array, take the slow path.
1563 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1564 if (!op->value()->is_illegal()) {
1565 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1566 // The array is not a flat array, but it might be null-free. If we are storing
1567 // a null into a null-free array, take the slow path (which will throw NPE).
1568 Label skip;
1569 __ cmpptr(op->value()->as_register(), NULL_WORD);
1570 __ jcc(Assembler::notEqual, skip);
1571 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1572 __ bind(skip);
1573 }
1574 }
1575
1576 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1577 // We are storing into an array that *may* be null-free (the declared type is
1578 // Object[], abstract[], interface[] or VT.ref[]).
1579 Label test_mark_word;
1580 Register tmp = op->tmp()->as_register();
1581 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1582 __ testl(tmp, markWord::unlocked_value);
1583 __ jccb(Assembler::notZero, test_mark_word);
1584 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1585 __ bind(test_mark_word);
1586 __ testl(tmp, markWord::null_free_array_bit_in_place);
1587 }
1588
1589 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1590 Label L_oops_equal;
1591 Label L_oops_not_equal;
1592 Label L_end;
1593
1594 Register left = op->left()->as_register();
1595 Register right = op->right()->as_register();
1596
1597 __ cmpptr(left, right);
1598 __ jcc(Assembler::equal, L_oops_equal);
1599
1600 // (1) Null check -- if one of the operands is null, the other must not be null (because
1601 // the two references are not equal), so they are not substitutable,
1602 // FIXME: do null check only if the operand is nullable
1603 __ testptr(left, right);
1604 __ jcc(Assembler::zero, L_oops_not_equal);
1605
1606 ciKlass* left_klass = op->left_klass();
1607 ciKlass* right_klass = op->right_klass();
1608
1609 // (2) Inline type check -- if either of the operands is not a inline type,
1610 // they are not substitutable. We do this only if we are not sure that the
1611 // operands are inline type
1612 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1613 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1614 Register tmp1 = op->tmp1()->as_register();
1615 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1616 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1617 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1618 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1619 __ jcc(Assembler::notEqual, L_oops_not_equal);
1620 }
1621
1622 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1623 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1624 // No need to load klass -- the operands are statically known to be the same inline klass.
1625 __ jmp(*op->stub()->entry());
1626 } else {
1627 Register left_klass_op = op->left_klass_op()->as_register();
1628 Register right_klass_op = op->right_klass_op()->as_register();
1629
1630 if (UseCompressedClassPointers) {
1631 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1632 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1633 __ cmpl(left_klass_op, right_klass_op);
1634 } else {
1635 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1636 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1637 __ cmpptr(left_klass_op, right_klass_op);
1638 }
1639
1640 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1641 // fall through to L_oops_not_equal
1642 }
1643
1644 __ bind(L_oops_not_equal);
1645 move(op->not_equal_result(), op->result_opr());
1646 __ jmp(L_end);
1647
1648 __ bind(L_oops_equal);
1649 move(op->equal_result(), op->result_opr());
1650 __ jmp(L_end);
1651
1652 // We've returned from the stub. RAX contains 0x0 IFF the two
1653 // operands are not substitutable. (Don't compare against 0x1 in case the
1654 // C compiler is naughty)
1655 __ bind(*op->stub()->continuation());
1656 __ cmpl(rax, 0);
1657 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1658 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1659 // fall-through
1660 __ bind(L_end);
1661 }
1662
1663 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1664 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1665 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1666 Register newval = op->new_value()->as_register();
1667 Register cmpval = op->cmp_value()->as_register();
1668 assert(cmpval == rax, "wrong register");
1669 assert(newval != noreg, "new val must be register");
1670 assert(cmpval != newval, "cmp and new values must be in different registers");
1671 assert(cmpval != addr, "cmp and addr must be in different registers");
1672 assert(newval != addr, "new value and addr must be in different registers");
1673
1674 if (op->code() == lir_cas_obj) {
1675 if (UseCompressedOops) {
1676 __ encode_heap_oop(cmpval);
1677 __ mov(rscratch1, newval);
1678 __ encode_heap_oop(rscratch1);
1679 __ lock();
1680 // cmpval (rax) is implicitly used by this instruction
1681 __ cmpxchgl(rscratch1, Address(addr, 0));
1687 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1688 __ lock();
1689 __ cmpxchgl(newval, Address(addr, 0));
1690 }
1691 } else if (op->code() == lir_cas_long) {
1692 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1693 Register newval = op->new_value()->as_register_lo();
1694 Register cmpval = op->cmp_value()->as_register_lo();
1695 assert(cmpval == rax, "wrong register");
1696 assert(newval != noreg, "new val must be register");
1697 assert(cmpval != newval, "cmp and new values must be in different registers");
1698 assert(cmpval != addr, "cmp and addr must be in different registers");
1699 assert(newval != addr, "new value and addr must be in different registers");
1700 __ lock();
1701 __ cmpxchgq(newval, Address(addr, 0));
1702 } else {
1703 Unimplemented();
1704 }
1705 }
1706
1707 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1708 assert(dst->is_cpu_register(), "must be");
1709 assert(dst->type() == src->type(), "must be");
1710
1711 if (src->is_cpu_register()) {
1712 reg2reg(src, dst);
1713 } else if (src->is_stack()) {
1714 stack2reg(src, dst, dst->type());
1715 } else if (src->is_constant()) {
1716 const2reg(src, dst, lir_patch_none, nullptr);
1717 } else {
1718 ShouldNotReachHere();
1719 }
1720 }
1721
1722 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1723 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1724 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1725
1726 Assembler::Condition acond, ncond;
1727 switch (condition) {
1728 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1729 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1730 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1731 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1732 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1733 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1734 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1735 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1736 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1737 ShouldNotReachHere();
1738 }
1739
1740 if (opr1->is_cpu_register()) {
1741 reg2reg(opr1, result);
2312 int offset = __ offset();
2313 switch (code) {
2314 case lir_static_call:
2315 case lir_optvirtual_call:
2316 case lir_dynamic_call:
2317 offset += NativeCall::displacement_offset;
2318 break;
2319 case lir_icvirtual_call:
2320 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2321 break;
2322 default: ShouldNotReachHere();
2323 }
2324 __ align(BytesPerWord, offset);
2325 }
2326
2327
2328 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2329 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2330 "must be aligned");
2331 __ call(AddressLiteral(op->addr(), rtype));
2332 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2333 __ post_call_nop();
2334 }
2335
2336
2337 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2338 __ ic_call(op->addr());
2339 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2340 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2341 "must be aligned");
2342 __ post_call_nop();
2343 }
2344
2345
2346 void LIR_Assembler::emit_static_call_stub() {
2347 address call_pc = __ pc();
2348 address stub = __ start_a_stub(call_stub_size());
2349 if (stub == nullptr) {
2350 bailout("static call stub overflow");
2351 return;
2352 }
2353
2354 int start = __ offset();
2355
2356 // make sure that the displacement word of the call ends up word aligned
2357 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2358 __ relocate(static_stub_Relocation::spec(call_pc));
2359 __ mov_metadata(rbx, (Metadata*)nullptr);
2486 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2487 }
2488
2489
2490 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2491 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2492 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2493 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2494 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2495 }
2496
2497
2498 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2499 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2500 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2501 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2502 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2503 }
2504
2505
2506 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2507 if (null_check) {
2508 __ testptr(obj, obj);
2509 __ jcc(Assembler::zero, *slow_path->entry());
2510 }
2511 if (is_dest) {
2512 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2513 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2514 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2515 } else {
2516 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2517 }
2518 }
2519
2520
2521 // This code replaces a call to arraycopy; no exception may
2522 // be thrown in this code, they must be thrown in the System.arraycopy
2523 // activation frame; we could save some checks if this would not be the case
2524 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2525 ciArrayKlass* default_type = op->expected_type();
2526 Register src = op->src()->as_register();
2527 Register dst = op->dst()->as_register();
2528 Register src_pos = op->src_pos()->as_register();
2529 Register dst_pos = op->dst_pos()->as_register();
2530 Register length = op->length()->as_register();
2531 Register tmp = op->tmp()->as_register();
2532 Register tmp_load_klass = rscratch1;
2533 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2534
2535 CodeStub* stub = op->stub();
2536 int flags = op->flags();
2537 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2538 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2539
2540 if (flags & LIR_OpArrayCopy::always_slow_path) {
2541 __ jmp(*stub->entry());
2542 __ bind(*stub->continuation());
2543 return;
2544 }
2545
2546 // if we don't know anything, just go through the generic arraycopy
2547 if (default_type == nullptr) {
2548 // save outgoing arguments on stack in case call to System.arraycopy is needed
2549 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2550 // for interpreter calling conventions. Now we have to do it in new style conventions.
2551 // For the moment until C1 gets the new register allocator I just force all the
2552 // args to the right place (except the register args) and then on the back side
2553 // reload the register args properly if we go slow path. Yuck
2554
2555 // These are proper for the calling convention
2556 store_parameter(length, 2);
2557 store_parameter(dst_pos, 1);
2558 store_parameter(dst, 0);
2559
2560 // these are just temporary placements until we need to reload
2561 store_parameter(src_pos, 3);
2562 store_parameter(src, 4);
2563
2564 address copyfunc_addr = StubRoutines::generic_arraycopy();
2565 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2602 __ mov(tmp, rax);
2603 __ xorl(tmp, -1);
2604
2605 // Reload values from the stack so they are where the stub
2606 // expects them.
2607 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2608 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2609 __ movptr (length, Address(rsp, 2*BytesPerWord));
2610 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2611 __ movptr (src, Address(rsp, 4*BytesPerWord));
2612
2613 __ subl(length, tmp);
2614 __ addl(src_pos, tmp);
2615 __ addl(dst_pos, tmp);
2616 __ jmp(*stub->entry());
2617
2618 __ bind(*stub->continuation());
2619 return;
2620 }
2621
2622 // Handle inline type arrays
2623 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2624 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2625 }
2626 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2627 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2628 }
2629
2630 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2631
2632 int elem_size = type2aelembytes(basic_type);
2633 Address::ScaleFactor scale;
2634
2635 switch (elem_size) {
2636 case 1 :
2637 scale = Address::times_1;
2638 break;
2639 case 2 :
2640 scale = Address::times_2;
2641 break;
2642 case 4 :
2643 scale = Address::times_4;
2644 break;
2645 case 8 :
2646 scale = Address::times_8;
2647 break;
2648 default:
2649 scale = Address::no_scale;
3180 // first time here. Set profile type.
3181 __ movptr(mdo_addr, tmp);
3182 #ifdef ASSERT
3183 __ andptr(tmp, TypeEntries::type_klass_mask);
3184 __ verify_klass_ptr(tmp);
3185 #endif
3186 } else {
3187 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3188 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3189
3190 __ testptr(mdo_addr, TypeEntries::type_unknown);
3191 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3192
3193 __ orptr(mdo_addr, TypeEntries::type_unknown);
3194 }
3195 }
3196 }
3197 __ bind(next);
3198 }
3199
3200 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3201 Register obj = op->obj()->as_register();
3202 Register tmp = op->tmp()->as_pointer_register();
3203 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3204 bool not_null = op->not_null();
3205 int flag = op->flag();
3206
3207 Label not_inline_type;
3208 if (!not_null) {
3209 __ testptr(obj, obj);
3210 __ jccb(Assembler::zero, not_inline_type);
3211 }
3212
3213 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3214
3215 __ orb(mdo_addr, flag);
3216
3217 __ bind(not_inline_type);
3218 }
3219
3220 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3221 Unimplemented();
3222 }
3223
3224
3225 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3226 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3227 }
3228
3229
3230 void LIR_Assembler::align_backward_branch_target() {
3231 __ align(BytesPerWord);
3232 }
3233
3234
3235 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3236 if (left->is_single_cpu()) {
3237 __ negl(left->as_register());
3238 move_regs(left->as_register(), dest->as_register());
3239
3390 }
3391
3392 void LIR_Assembler::membar_loadstore() {
3393 // no-op
3394 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3395 }
3396
3397 void LIR_Assembler::membar_storeload() {
3398 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3399 }
3400
3401 void LIR_Assembler::on_spin_wait() {
3402 __ pause ();
3403 }
3404
3405 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3406 assert(result_reg->is_register(), "check");
3407 __ mov(result_reg->as_register(), r15_thread);
3408 }
3409
3410 void LIR_Assembler::check_orig_pc() {
3411 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3412 }
3413
3414 void LIR_Assembler::peephole(LIR_List*) {
3415 // do nothing for now
3416 }
3417
3418 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3419 assert(data == dest, "xchg/xadd uses only 2 operands");
3420
3421 if (data->type() == T_INT) {
3422 if (code == lir_xadd) {
3423 __ lock();
3424 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3425 } else {
3426 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3427 }
3428 } else if (data->is_oop()) {
3429 assert (code == lir_xchg, "xadd for oops");
3430 Register obj = data->as_register();
3431 if (UseCompressedOops) {
3432 __ encode_heap_oop(obj);
|