14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
411 // Perform needed unlocking
412 MonitorExitStub* stub = nullptr;
413 if (method()->is_synchronized()) {
414 monitor_address(0, FrameMap::rax_opr);
415 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
416 __ unlock_object(rdi, rsi, rax, *stub->entry());
417 __ bind(*stub->continuation());
418 }
419
420 if (compilation()->env()->dtrace_method_probes()) {
421 __ mov(rdi, r15_thread);
422 __ mov_metadata(rsi, method()->constant_encoding());
423 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
424 }
425
426 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
427 __ mov(rax, rbx); // Restore the exception
428 }
429
430 // remove the activation and dispatch to the unwind handler
431 __ remove_frame(initial_frame_size_in_bytes());
432 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
433
434 // Emit the slow path assembly
435 if (stub != nullptr) {
436 stub->emit_code(this);
437 }
438
439 return offset;
440 }
441
442
443 int LIR_Assembler::emit_deopt_handler() {
444 // generate code for exception handler
445 address handler_base = __ start_a_stub(deopt_handler_size());
446 if (handler_base == nullptr) {
447 // not enough space left for the handler
448 bailout("deopt handler overflow");
449 return -1;
450 }
451
452 int offset = code_offset();
453 InternalAddress here(__ pc());
454
455 __ pushptr(here.addr(), rscratch1);
456 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
458 __ end_a_stub();
459
460 return offset;
461 }
462
463 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
464 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
465 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
466 assert(result->fpu() == 0, "result must already be on TOS");
467 }
468
469 // Pop the stack before the safepoint code
470 __ remove_frame(initial_frame_size_in_bytes());
471
472 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
473 __ reserved_stack_check();
474 }
475
476 // Note: we do not need to round double result; float result has the right precision
477 // the poll sets the condition code, but no data registers
478
479 code_stub->set_safepoint_offset(__ offset());
480 __ relocate(relocInfo::poll_return_type);
481 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
482 __ ret(0);
483 }
484
485
486 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
487 guarantee(info != nullptr, "Shouldn't be null");
488 int offset = __ offset();
489 const Register poll_addr = rscratch1;
490 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
491 add_debug_info_for_branch(info);
492 __ relocate(relocInfo::poll_type);
493 address pre_pc = __ pc();
494 __ testl(rax, Address(poll_addr, 0));
495 address post_pc = __ pc();
496 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
497 return offset;
498 }
499
500
501 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
502 if (from_reg != to_reg) __ mov(to_reg, from_reg);
503 }
504
505 void LIR_Assembler::swap_reg(Register a, Register b) {
1204 // init_state needs acquire, but x86 is TSO, and so we are already good.
1205 __ cmpb(Address(op->klass()->as_register(),
1206 InstanceKlass::init_state_offset()),
1207 InstanceKlass::fully_initialized);
1208 __ jcc(Assembler::notEqual, *op->stub()->entry());
1209 }
1210 __ allocate_object(op->obj()->as_register(),
1211 op->tmp1()->as_register(),
1212 op->tmp2()->as_register(),
1213 op->header_size(),
1214 op->object_size(),
1215 op->klass()->as_register(),
1216 *op->stub()->entry());
1217 __ bind(*op->stub()->continuation());
1218 }
1219
1220 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1221 Register len = op->len()->as_register();
1222 __ movslq(len, len);
1223
1224 if (UseSlowPath ||
1225 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1226 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1227 __ jmp(*op->stub()->entry());
1228 } else {
1229 Register tmp1 = op->tmp1()->as_register();
1230 Register tmp2 = op->tmp2()->as_register();
1231 Register tmp3 = op->tmp3()->as_register();
1232 if (len == tmp1) {
1233 tmp1 = tmp3;
1234 } else if (len == tmp2) {
1235 tmp2 = tmp3;
1236 } else if (len == tmp3) {
1237 // everything is ok
1238 } else {
1239 __ mov(tmp3, len);
1240 }
1241 __ allocate_array(op->obj()->as_register(),
1242 len,
1243 tmp1,
1244 tmp2,
1303 assert(data != nullptr, "need data for type check");
1304 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1305 }
1306 Label* success_target = success;
1307 Label* failure_target = failure;
1308
1309 if (obj == k_RInfo) {
1310 k_RInfo = dst;
1311 } else if (obj == klass_RInfo) {
1312 klass_RInfo = dst;
1313 }
1314 if (k->is_loaded() && !UseCompressedClassPointers) {
1315 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1316 } else {
1317 Rtmp1 = op->tmp3()->as_register();
1318 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1319 }
1320
1321 assert_different_registers(obj, k_RInfo, klass_RInfo);
1322
1323 __ testptr(obj, obj);
1324 if (op->should_profile()) {
1325 Label not_null;
1326 Register mdo = klass_RInfo;
1327 __ mov_metadata(mdo, md->constant_encoding());
1328 __ jccb(Assembler::notEqual, not_null);
1329 // Object is null; update MDO and exit
1330 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1331 int header_bits = BitData::null_seen_byte_constant();
1332 __ orb(data_addr, header_bits);
1333 __ jmp(*obj_is_null);
1334 __ bind(not_null);
1335
1336 Label update_done;
1337 Register recv = k_RInfo;
1338 __ load_klass(recv, obj, tmp_load_klass);
1339 type_profile_helper(mdo, md, data, recv, &update_done);
1340
1341 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1342 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1343
1344 __ bind(update_done);
1345 } else {
1346 __ jcc(Assembler::equal, *obj_is_null);
1347 }
1348
1349 if (!k->is_loaded()) {
1350 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1351 } else {
1352 __ mov_metadata(k_RInfo, k->constant_encoding());
1353 }
1354 __ verify_oop(obj);
1355
1356 if (op->fast_check()) {
1357 // get object class
1358 // not a safepoint as obj null check happens earlier
1359 if (UseCompressedClassPointers) {
1360 __ load_klass(Rtmp1, obj, tmp_load_klass);
1361 __ cmpptr(k_RInfo, Rtmp1);
1362 } else {
1363 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1364 }
1365 __ jcc(Assembler::notEqual, *failure_target);
1366 // successful cast, fall through to profile or jump
1367 } else {
1368 // get object class
1369 // not a safepoint as obj null check happens earlier
1370 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1371 if (k->is_loaded()) {
1372 // See if we get an immediate positive hit
1373 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1374 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1375 __ jcc(Assembler::notEqual, *failure_target);
1376 // successful cast, fall through to profile or jump
1496 __ mov(dst, obj);
1497 }
1498 } else
1499 if (code == lir_instanceof) {
1500 Register obj = op->object()->as_register();
1501 Register dst = op->result_opr()->as_register();
1502 Label success, failure, done;
1503 emit_typecheck_helper(op, &success, &failure, &failure);
1504 __ bind(failure);
1505 __ xorptr(dst, dst);
1506 __ jmpb(done);
1507 __ bind(success);
1508 __ movptr(dst, 1);
1509 __ bind(done);
1510 } else {
1511 ShouldNotReachHere();
1512 }
1513
1514 }
1515
1516
1517 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1518 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1519 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1520 Register newval = op->new_value()->as_register();
1521 Register cmpval = op->cmp_value()->as_register();
1522 assert(cmpval == rax, "wrong register");
1523 assert(newval != noreg, "new val must be register");
1524 assert(cmpval != newval, "cmp and new values must be in different registers");
1525 assert(cmpval != addr, "cmp and addr must be in different registers");
1526 assert(newval != addr, "new value and addr must be in different registers");
1527
1528 if (op->code() == lir_cas_obj) {
1529 if (UseCompressedOops) {
1530 __ encode_heap_oop(cmpval);
1531 __ mov(rscratch1, newval);
1532 __ encode_heap_oop(rscratch1);
1533 __ lock();
1534 // cmpval (rax) is implicitly used by this instruction
1535 __ cmpxchgl(rscratch1, Address(addr, 0));
1541 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1542 __ lock();
1543 __ cmpxchgl(newval, Address(addr, 0));
1544 }
1545 } else if (op->code() == lir_cas_long) {
1546 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1547 Register newval = op->new_value()->as_register_lo();
1548 Register cmpval = op->cmp_value()->as_register_lo();
1549 assert(cmpval == rax, "wrong register");
1550 assert(newval != noreg, "new val must be register");
1551 assert(cmpval != newval, "cmp and new values must be in different registers");
1552 assert(cmpval != addr, "cmp and addr must be in different registers");
1553 assert(newval != addr, "new value and addr must be in different registers");
1554 __ lock();
1555 __ cmpxchgq(newval, Address(addr, 0));
1556 } else {
1557 Unimplemented();
1558 }
1559 }
1560
1561 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1562 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1563 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1564
1565 Assembler::Condition acond, ncond;
1566 switch (condition) {
1567 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1568 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1569 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1570 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1571 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1572 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1573 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1574 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1575 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1576 ShouldNotReachHere();
1577 }
1578
1579 if (opr1->is_cpu_register()) {
1580 reg2reg(opr1, result);
2151 int offset = __ offset();
2152 switch (code) {
2153 case lir_static_call:
2154 case lir_optvirtual_call:
2155 case lir_dynamic_call:
2156 offset += NativeCall::displacement_offset;
2157 break;
2158 case lir_icvirtual_call:
2159 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2160 break;
2161 default: ShouldNotReachHere();
2162 }
2163 __ align(BytesPerWord, offset);
2164 }
2165
2166
2167 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2168 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2169 "must be aligned");
2170 __ call(AddressLiteral(op->addr(), rtype));
2171 add_call_info(code_offset(), op->info());
2172 __ post_call_nop();
2173 }
2174
2175
2176 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2177 __ ic_call(op->addr());
2178 add_call_info(code_offset(), op->info());
2179 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2180 "must be aligned");
2181 __ post_call_nop();
2182 }
2183
2184
2185 void LIR_Assembler::emit_static_call_stub() {
2186 address call_pc = __ pc();
2187 address stub = __ start_a_stub(call_stub_size());
2188 if (stub == nullptr) {
2189 bailout("static call stub overflow");
2190 return;
2191 }
2192
2193 int start = __ offset();
2194
2195 // make sure that the displacement word of the call ends up word aligned
2196 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2197 __ relocate(static_stub_Relocation::spec(call_pc));
2198 __ mov_metadata(rbx, (Metadata*)nullptr);
2325 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2326 }
2327
2328
2329 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2330 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2331 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2332 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2333 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2334 }
2335
2336
2337 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2338 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2339 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2340 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2341 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2342 }
2343
2344
2345 // This code replaces a call to arraycopy; no exception may
2346 // be thrown in this code, they must be thrown in the System.arraycopy
2347 // activation frame; we could save some checks if this would not be the case
2348 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2349 ciArrayKlass* default_type = op->expected_type();
2350 Register src = op->src()->as_register();
2351 Register dst = op->dst()->as_register();
2352 Register src_pos = op->src_pos()->as_register();
2353 Register dst_pos = op->dst_pos()->as_register();
2354 Register length = op->length()->as_register();
2355 Register tmp = op->tmp()->as_register();
2356 Register tmp_load_klass = rscratch1;
2357 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2358
2359 CodeStub* stub = op->stub();
2360 int flags = op->flags();
2361 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2362 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2363
2364 // if we don't know anything, just go through the generic arraycopy
2365 if (default_type == nullptr) {
2366 // save outgoing arguments on stack in case call to System.arraycopy is needed
2367 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2368 // for interpreter calling conventions. Now we have to do it in new style conventions.
2369 // For the moment until C1 gets the new register allocator I just force all the
2370 // args to the right place (except the register args) and then on the back side
2371 // reload the register args properly if we go slow path. Yuck
2372
2373 // These are proper for the calling convention
2374 store_parameter(length, 2);
2375 store_parameter(dst_pos, 1);
2376 store_parameter(dst, 0);
2377
2378 // these are just temporary placements until we need to reload
2379 store_parameter(src_pos, 3);
2380 store_parameter(src, 4);
2381
2382 address copyfunc_addr = StubRoutines::generic_arraycopy();
2383 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2420 __ mov(tmp, rax);
2421 __ xorl(tmp, -1);
2422
2423 // Reload values from the stack so they are where the stub
2424 // expects them.
2425 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2426 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2427 __ movptr (length, Address(rsp, 2*BytesPerWord));
2428 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2429 __ movptr (src, Address(rsp, 4*BytesPerWord));
2430
2431 __ subl(length, tmp);
2432 __ addl(src_pos, tmp);
2433 __ addl(dst_pos, tmp);
2434 __ jmp(*stub->entry());
2435
2436 __ bind(*stub->continuation());
2437 return;
2438 }
2439
2440 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2441
2442 int elem_size = type2aelembytes(basic_type);
2443 Address::ScaleFactor scale;
2444
2445 switch (elem_size) {
2446 case 1 :
2447 scale = Address::times_1;
2448 break;
2449 case 2 :
2450 scale = Address::times_2;
2451 break;
2452 case 4 :
2453 scale = Address::times_4;
2454 break;
2455 case 8 :
2456 scale = Address::times_8;
2457 break;
2458 default:
2459 scale = Address::no_scale;
2646 }
2647
2648 __ jmp(*stub->entry());
2649
2650 __ bind(cont);
2651 __ pop(dst);
2652 __ pop(src);
2653 }
2654 }
2655
2656 #ifdef ASSERT
2657 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2658 // Sanity check the known type with the incoming class. For the
2659 // primitive case the types must match exactly with src.klass and
2660 // dst.klass each exactly matching the default type. For the
2661 // object array case, if no type check is needed then either the
2662 // dst type is exactly the expected type and the src type is a
2663 // subtype which we can't check or src is the same array as dst
2664 // but not necessarily exactly of type default_type.
2665 Label known_ok, halt;
2666 __ mov_metadata(tmp, default_type->constant_encoding());
2667 if (UseCompressedClassPointers) {
2668 __ encode_klass_not_null(tmp, rscratch1);
2669 }
2670
2671 if (basic_type != T_OBJECT) {
2672 __ cmp_klass(tmp, dst, tmp2);
2673 __ jcc(Assembler::notEqual, halt);
2674 __ cmp_klass(tmp, src, tmp2);
2675 __ jcc(Assembler::equal, known_ok);
2676 } else {
2677 __ cmp_klass(tmp, dst, tmp2);
2678 __ jcc(Assembler::equal, known_ok);
2679 __ cmpptr(src, dst);
2680 __ jcc(Assembler::equal, known_ok);
2681 }
2682 __ bind(halt);
2683 __ stop("incorrect type information in arraycopy");
2684 __ bind(known_ok);
2685 }
2984 // first time here. Set profile type.
2985 __ movptr(mdo_addr, tmp);
2986 #ifdef ASSERT
2987 __ andptr(tmp, TypeEntries::type_klass_mask);
2988 __ verify_klass_ptr(tmp);
2989 #endif
2990 } else {
2991 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2992 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2993
2994 __ testptr(mdo_addr, TypeEntries::type_unknown);
2995 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2996
2997 __ orptr(mdo_addr, TypeEntries::type_unknown);
2998 }
2999 }
3000 }
3001 __ bind(next);
3002 }
3003
3004 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3005 Unimplemented();
3006 }
3007
3008
3009 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3010 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3011 }
3012
3013
3014 void LIR_Assembler::align_backward_branch_target() {
3015 __ align(BytesPerWord);
3016 }
3017
3018
3019 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3020 if (left->is_single_cpu()) {
3021 __ negl(left->as_register());
3022 move_regs(left->as_register(), dest->as_register());
3023
3174 }
3175
3176 void LIR_Assembler::membar_loadstore() {
3177 // no-op
3178 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3179 }
3180
3181 void LIR_Assembler::membar_storeload() {
3182 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3183 }
3184
3185 void LIR_Assembler::on_spin_wait() {
3186 __ pause ();
3187 }
3188
3189 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3190 assert(result_reg->is_register(), "check");
3191 __ mov(result_reg->as_register(), r15_thread);
3192 }
3193
3194
3195 void LIR_Assembler::peephole(LIR_List*) {
3196 // do nothing for now
3197 }
3198
3199 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3200 assert(data == dest, "xchg/xadd uses only 2 operands");
3201
3202 if (data->type() == T_INT) {
3203 if (code == lir_xadd) {
3204 __ lock();
3205 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3206 } else {
3207 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3208 }
3209 } else if (data->is_oop()) {
3210 assert (code == lir_xchg, "xadd for oops");
3211 Register obj = data->as_register();
3212 if (UseCompressedOops) {
3213 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
60 operand[0] = lo;
413 // Perform needed unlocking
414 MonitorExitStub* stub = nullptr;
415 if (method()->is_synchronized()) {
416 monitor_address(0, FrameMap::rax_opr);
417 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
418 __ unlock_object(rdi, rsi, rax, *stub->entry());
419 __ bind(*stub->continuation());
420 }
421
422 if (compilation()->env()->dtrace_method_probes()) {
423 __ mov(rdi, r15_thread);
424 __ mov_metadata(rsi, method()->constant_encoding());
425 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
426 }
427
428 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
429 __ mov(rax, rbx); // Restore the exception
430 }
431
432 // remove the activation and dispatch to the unwind handler
433 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
434 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
435
436 // Emit the slow path assembly
437 if (stub != nullptr) {
438 stub->emit_code(this);
439 }
440
441 return offset;
442 }
443
444
445 int LIR_Assembler::emit_deopt_handler() {
446 // generate code for exception handler
447 address handler_base = __ start_a_stub(deopt_handler_size());
448 if (handler_base == nullptr) {
449 // not enough space left for the handler
450 bailout("deopt handler overflow");
451 return -1;
452 }
453
454 int offset = code_offset();
455 InternalAddress here(__ pc());
456
457 __ pushptr(here.addr(), rscratch1);
458 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
459 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
460 __ end_a_stub();
461
462 return offset;
463 }
464
465 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
466 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
467 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
468 assert(result->fpu() == 0, "result must already be on TOS");
469 }
470 if (InlineTypeReturnedAsFields) {
471 #ifndef _LP64
472 Unimplemented();
473 #endif
474 // Check if we are returning an non-null inline type and load its fields into registers
475 ciType* return_type = compilation()->method()->return_type();
476 if (return_type->is_inlinetype()) {
477 ciInlineKlass* vk = return_type->as_inline_klass();
478 if (vk->can_be_returned_as_fields()) {
479 address unpack_handler = vk->unpack_handler();
480 assert(unpack_handler != nullptr, "must be");
481 __ call(RuntimeAddress(unpack_handler));
482 }
483 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
484 Label skip;
485 Label not_null;
486 __ testptr(rax, rax);
487 __ jcc(Assembler::notZero, not_null);
488 // Returned value is null, zero all return registers because they may belong to oop fields
489 __ xorq(j_rarg1, j_rarg1);
490 __ xorq(j_rarg2, j_rarg2);
491 __ xorq(j_rarg3, j_rarg3);
492 __ xorq(j_rarg4, j_rarg4);
493 __ xorq(j_rarg5, j_rarg5);
494 __ jmp(skip);
495 __ bind(not_null);
496
497 // Check if we are returning an non-null inline type and load its fields into registers
498 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
499
500 // Load fields from a buffered value with an inline class specific handler
501 __ load_klass(rdi, rax, rscratch1);
502 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
503 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
504 // Unpack handler can be null if inline type is not scalarizable in returns
505 __ testptr(rdi, rdi);
506 __ jcc(Assembler::zero, skip);
507 __ call(rdi);
508
509 __ bind(skip);
510 }
511 // At this point, rax points to the value object (for interpreter or C1 caller).
512 // The fields of the object are copied into registers (for C2 caller).
513 }
514
515 // Pop the stack before the safepoint code
516 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
517
518 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
519 __ reserved_stack_check();
520 }
521
522 // Note: we do not need to round double result; float result has the right precision
523 // the poll sets the condition code, but no data registers
524
525 code_stub->set_safepoint_offset(__ offset());
526 __ relocate(relocInfo::poll_return_type);
527 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
528 __ ret(0);
529 }
530
531
532 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
533 return (__ store_inline_type_fields_to_buf(vk, false));
534 }
535
536 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
537 guarantee(info != nullptr, "Shouldn't be null");
538 int offset = __ offset();
539 const Register poll_addr = rscratch1;
540 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
541 add_debug_info_for_branch(info);
542 __ relocate(relocInfo::poll_type);
543 address pre_pc = __ pc();
544 __ testl(rax, Address(poll_addr, 0));
545 address post_pc = __ pc();
546 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
547 return offset;
548 }
549
550
551 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
552 if (from_reg != to_reg) __ mov(to_reg, from_reg);
553 }
554
555 void LIR_Assembler::swap_reg(Register a, Register b) {
1254 // init_state needs acquire, but x86 is TSO, and so we are already good.
1255 __ cmpb(Address(op->klass()->as_register(),
1256 InstanceKlass::init_state_offset()),
1257 InstanceKlass::fully_initialized);
1258 __ jcc(Assembler::notEqual, *op->stub()->entry());
1259 }
1260 __ allocate_object(op->obj()->as_register(),
1261 op->tmp1()->as_register(),
1262 op->tmp2()->as_register(),
1263 op->header_size(),
1264 op->object_size(),
1265 op->klass()->as_register(),
1266 *op->stub()->entry());
1267 __ bind(*op->stub()->continuation());
1268 }
1269
1270 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1271 Register len = op->len()->as_register();
1272 __ movslq(len, len);
1273
1274 if (UseSlowPath || op->always_slow_path() ||
1275 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1276 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1277 __ jmp(*op->stub()->entry());
1278 } else {
1279 Register tmp1 = op->tmp1()->as_register();
1280 Register tmp2 = op->tmp2()->as_register();
1281 Register tmp3 = op->tmp3()->as_register();
1282 if (len == tmp1) {
1283 tmp1 = tmp3;
1284 } else if (len == tmp2) {
1285 tmp2 = tmp3;
1286 } else if (len == tmp3) {
1287 // everything is ok
1288 } else {
1289 __ mov(tmp3, len);
1290 }
1291 __ allocate_array(op->obj()->as_register(),
1292 len,
1293 tmp1,
1294 tmp2,
1353 assert(data != nullptr, "need data for type check");
1354 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1355 }
1356 Label* success_target = success;
1357 Label* failure_target = failure;
1358
1359 if (obj == k_RInfo) {
1360 k_RInfo = dst;
1361 } else if (obj == klass_RInfo) {
1362 klass_RInfo = dst;
1363 }
1364 if (k->is_loaded() && !UseCompressedClassPointers) {
1365 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1366 } else {
1367 Rtmp1 = op->tmp3()->as_register();
1368 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1369 }
1370
1371 assert_different_registers(obj, k_RInfo, klass_RInfo);
1372
1373 if (op->need_null_check()) {
1374 __ testptr(obj, obj);
1375 if (op->should_profile()) {
1376 Label not_null;
1377 Register mdo = klass_RInfo;
1378 __ mov_metadata(mdo, md->constant_encoding());
1379 __ jccb(Assembler::notEqual, not_null);
1380 // Object is null; update MDO and exit
1381 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1382 int header_bits = BitData::null_seen_byte_constant();
1383 __ orb(data_addr, header_bits);
1384 __ jmp(*obj_is_null);
1385 __ bind(not_null);
1386
1387 Label update_done;
1388 Register recv = k_RInfo;
1389 __ load_klass(recv, obj, tmp_load_klass);
1390 type_profile_helper(mdo, md, data, recv, &update_done);
1391
1392 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1393 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1394
1395 __ bind(update_done);
1396 } else {
1397 __ jcc(Assembler::equal, *obj_is_null);
1398 }
1399 }
1400
1401 if (!k->is_loaded()) {
1402 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1403 } else {
1404 __ mov_metadata(k_RInfo, k->constant_encoding());
1405 }
1406 __ verify_oop(obj);
1407
1408 if (op->fast_check()) {
1409 // TODO 8366668 Is this correct? I don't think so. Probably we now always go to the slow path here. Same on AArch64.
1410 // get object class
1411 // not a safepoint as obj null check happens earlier
1412 if (UseCompressedClassPointers) {
1413 __ load_klass(Rtmp1, obj, tmp_load_klass);
1414 __ cmpptr(k_RInfo, Rtmp1);
1415 } else {
1416 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1417 }
1418 __ jcc(Assembler::notEqual, *failure_target);
1419 // successful cast, fall through to profile or jump
1420 } else {
1421 // get object class
1422 // not a safepoint as obj null check happens earlier
1423 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1424 if (k->is_loaded()) {
1425 // See if we get an immediate positive hit
1426 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1427 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1428 __ jcc(Assembler::notEqual, *failure_target);
1429 // successful cast, fall through to profile or jump
1549 __ mov(dst, obj);
1550 }
1551 } else
1552 if (code == lir_instanceof) {
1553 Register obj = op->object()->as_register();
1554 Register dst = op->result_opr()->as_register();
1555 Label success, failure, done;
1556 emit_typecheck_helper(op, &success, &failure, &failure);
1557 __ bind(failure);
1558 __ xorptr(dst, dst);
1559 __ jmpb(done);
1560 __ bind(success);
1561 __ movptr(dst, 1);
1562 __ bind(done);
1563 } else {
1564 ShouldNotReachHere();
1565 }
1566
1567 }
1568
1569 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1570 // We are loading/storing from/to an array that *may* be a flat array (the
1571 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1572 // If this array is a flat array, take the slow path.
1573 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1574 if (!op->value()->is_illegal()) {
1575 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1576 // The array is not a flat array, but it might be null-free. If we are storing
1577 // a null into a null-free array, take the slow path (which will throw NPE).
1578 Label skip;
1579 __ cmpptr(op->value()->as_register(), NULL_WORD);
1580 __ jcc(Assembler::notEqual, skip);
1581 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1582 __ bind(skip);
1583 }
1584 }
1585
1586 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1587 // We are storing into an array that *may* be null-free (the declared type is
1588 // Object[], abstract[], interface[] or VT.ref[]).
1589 Label test_mark_word;
1590 Register tmp = op->tmp()->as_register();
1591 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1592 __ testl(tmp, markWord::unlocked_value);
1593 __ jccb(Assembler::notZero, test_mark_word);
1594 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1595 __ bind(test_mark_word);
1596 __ testl(tmp, markWord::null_free_array_bit_in_place);
1597 }
1598
1599 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1600 Label L_oops_equal;
1601 Label L_oops_not_equal;
1602 Label L_end;
1603
1604 Register left = op->left()->as_register();
1605 Register right = op->right()->as_register();
1606
1607 __ cmpptr(left, right);
1608 __ jcc(Assembler::equal, L_oops_equal);
1609
1610 // (1) Null check -- if one of the operands is null, the other must not be null (because
1611 // the two references are not equal), so they are not substitutable,
1612 // FIXME: do null check only if the operand is nullable
1613 __ testptr(left, right);
1614 __ jcc(Assembler::zero, L_oops_not_equal);
1615
1616 ciKlass* left_klass = op->left_klass();
1617 ciKlass* right_klass = op->right_klass();
1618
1619 // (2) Inline type check -- if either of the operands is not a inline type,
1620 // they are not substitutable. We do this only if we are not sure that the
1621 // operands are inline type
1622 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1623 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1624 Register tmp1 = op->tmp1()->as_register();
1625 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1626 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1627 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1628 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1629 __ jcc(Assembler::notEqual, L_oops_not_equal);
1630 }
1631
1632 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1633 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1634 // No need to load klass -- the operands are statically known to be the same inline klass.
1635 __ jmp(*op->stub()->entry());
1636 } else {
1637 Register left_klass_op = op->left_klass_op()->as_register();
1638 Register right_klass_op = op->right_klass_op()->as_register();
1639
1640 if (UseCompressedClassPointers) {
1641 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1642 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1643 __ cmpl(left_klass_op, right_klass_op);
1644 } else {
1645 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1646 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1647 __ cmpptr(left_klass_op, right_klass_op);
1648 }
1649
1650 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1651 // fall through to L_oops_not_equal
1652 }
1653
1654 __ bind(L_oops_not_equal);
1655 move(op->not_equal_result(), op->result_opr());
1656 __ jmp(L_end);
1657
1658 __ bind(L_oops_equal);
1659 move(op->equal_result(), op->result_opr());
1660 __ jmp(L_end);
1661
1662 // We've returned from the stub. RAX contains 0x0 IFF the two
1663 // operands are not substitutable. (Don't compare against 0x1 in case the
1664 // C compiler is naughty)
1665 __ bind(*op->stub()->continuation());
1666 __ cmpl(rax, 0);
1667 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1668 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1669 // fall-through
1670 __ bind(L_end);
1671 }
1672
1673 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1674 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1675 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1676 Register newval = op->new_value()->as_register();
1677 Register cmpval = op->cmp_value()->as_register();
1678 assert(cmpval == rax, "wrong register");
1679 assert(newval != noreg, "new val must be register");
1680 assert(cmpval != newval, "cmp and new values must be in different registers");
1681 assert(cmpval != addr, "cmp and addr must be in different registers");
1682 assert(newval != addr, "new value and addr must be in different registers");
1683
1684 if (op->code() == lir_cas_obj) {
1685 if (UseCompressedOops) {
1686 __ encode_heap_oop(cmpval);
1687 __ mov(rscratch1, newval);
1688 __ encode_heap_oop(rscratch1);
1689 __ lock();
1690 // cmpval (rax) is implicitly used by this instruction
1691 __ cmpxchgl(rscratch1, Address(addr, 0));
1697 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1698 __ lock();
1699 __ cmpxchgl(newval, Address(addr, 0));
1700 }
1701 } else if (op->code() == lir_cas_long) {
1702 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1703 Register newval = op->new_value()->as_register_lo();
1704 Register cmpval = op->cmp_value()->as_register_lo();
1705 assert(cmpval == rax, "wrong register");
1706 assert(newval != noreg, "new val must be register");
1707 assert(cmpval != newval, "cmp and new values must be in different registers");
1708 assert(cmpval != addr, "cmp and addr must be in different registers");
1709 assert(newval != addr, "new value and addr must be in different registers");
1710 __ lock();
1711 __ cmpxchgq(newval, Address(addr, 0));
1712 } else {
1713 Unimplemented();
1714 }
1715 }
1716
1717 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1718 assert(dst->is_cpu_register(), "must be");
1719 assert(dst->type() == src->type(), "must be");
1720
1721 if (src->is_cpu_register()) {
1722 reg2reg(src, dst);
1723 } else if (src->is_stack()) {
1724 stack2reg(src, dst, dst->type());
1725 } else if (src->is_constant()) {
1726 const2reg(src, dst, lir_patch_none, nullptr);
1727 } else {
1728 ShouldNotReachHere();
1729 }
1730 }
1731
1732 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1733 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1734 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1735
1736 Assembler::Condition acond, ncond;
1737 switch (condition) {
1738 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1739 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1740 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1741 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1742 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1743 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1744 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1745 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1746 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1747 ShouldNotReachHere();
1748 }
1749
1750 if (opr1->is_cpu_register()) {
1751 reg2reg(opr1, result);
2322 int offset = __ offset();
2323 switch (code) {
2324 case lir_static_call:
2325 case lir_optvirtual_call:
2326 case lir_dynamic_call:
2327 offset += NativeCall::displacement_offset;
2328 break;
2329 case lir_icvirtual_call:
2330 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2331 break;
2332 default: ShouldNotReachHere();
2333 }
2334 __ align(BytesPerWord, offset);
2335 }
2336
2337
2338 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2339 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2340 "must be aligned");
2341 __ call(AddressLiteral(op->addr(), rtype));
2342 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2343 __ post_call_nop();
2344 }
2345
2346
2347 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2348 __ ic_call(op->addr());
2349 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2350 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2351 "must be aligned");
2352 __ post_call_nop();
2353 }
2354
2355
2356 void LIR_Assembler::emit_static_call_stub() {
2357 address call_pc = __ pc();
2358 address stub = __ start_a_stub(call_stub_size());
2359 if (stub == nullptr) {
2360 bailout("static call stub overflow");
2361 return;
2362 }
2363
2364 int start = __ offset();
2365
2366 // make sure that the displacement word of the call ends up word aligned
2367 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2368 __ relocate(static_stub_Relocation::spec(call_pc));
2369 __ mov_metadata(rbx, (Metadata*)nullptr);
2496 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2497 }
2498
2499
2500 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2501 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2502 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2503 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2504 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2505 }
2506
2507
2508 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2509 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2510 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2511 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2512 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2513 }
2514
2515
2516 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2517 if (null_check) {
2518 __ testptr(obj, obj);
2519 __ jcc(Assembler::zero, *slow_path->entry());
2520 }
2521 if (is_dest) {
2522 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2523 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2524 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2525 } else {
2526 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2527 }
2528 }
2529
2530
2531 // This code replaces a call to arraycopy; no exception may
2532 // be thrown in this code, they must be thrown in the System.arraycopy
2533 // activation frame; we could save some checks if this would not be the case
2534 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2535 ciArrayKlass* default_type = op->expected_type();
2536 Register src = op->src()->as_register();
2537 Register dst = op->dst()->as_register();
2538 Register src_pos = op->src_pos()->as_register();
2539 Register dst_pos = op->dst_pos()->as_register();
2540 Register length = op->length()->as_register();
2541 Register tmp = op->tmp()->as_register();
2542 Register tmp_load_klass = rscratch1;
2543 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2544
2545 CodeStub* stub = op->stub();
2546 int flags = op->flags();
2547 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2548 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2549
2550 if (flags & LIR_OpArrayCopy::always_slow_path) {
2551 __ jmp(*stub->entry());
2552 __ bind(*stub->continuation());
2553 return;
2554 }
2555
2556 // if we don't know anything, just go through the generic arraycopy
2557 if (default_type == nullptr) {
2558 // save outgoing arguments on stack in case call to System.arraycopy is needed
2559 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2560 // for interpreter calling conventions. Now we have to do it in new style conventions.
2561 // For the moment until C1 gets the new register allocator I just force all the
2562 // args to the right place (except the register args) and then on the back side
2563 // reload the register args properly if we go slow path. Yuck
2564
2565 // These are proper for the calling convention
2566 store_parameter(length, 2);
2567 store_parameter(dst_pos, 1);
2568 store_parameter(dst, 0);
2569
2570 // these are just temporary placements until we need to reload
2571 store_parameter(src_pos, 3);
2572 store_parameter(src, 4);
2573
2574 address copyfunc_addr = StubRoutines::generic_arraycopy();
2575 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2612 __ mov(tmp, rax);
2613 __ xorl(tmp, -1);
2614
2615 // Reload values from the stack so they are where the stub
2616 // expects them.
2617 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2618 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2619 __ movptr (length, Address(rsp, 2*BytesPerWord));
2620 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2621 __ movptr (src, Address(rsp, 4*BytesPerWord));
2622
2623 __ subl(length, tmp);
2624 __ addl(src_pos, tmp);
2625 __ addl(dst_pos, tmp);
2626 __ jmp(*stub->entry());
2627
2628 __ bind(*stub->continuation());
2629 return;
2630 }
2631
2632 // Handle inline type arrays
2633 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2634 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2635 }
2636 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2637 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2638 }
2639
2640 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2641
2642 int elem_size = type2aelembytes(basic_type);
2643 Address::ScaleFactor scale;
2644
2645 switch (elem_size) {
2646 case 1 :
2647 scale = Address::times_1;
2648 break;
2649 case 2 :
2650 scale = Address::times_2;
2651 break;
2652 case 4 :
2653 scale = Address::times_4;
2654 break;
2655 case 8 :
2656 scale = Address::times_8;
2657 break;
2658 default:
2659 scale = Address::no_scale;
2846 }
2847
2848 __ jmp(*stub->entry());
2849
2850 __ bind(cont);
2851 __ pop(dst);
2852 __ pop(src);
2853 }
2854 }
2855
2856 #ifdef ASSERT
2857 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2858 // Sanity check the known type with the incoming class. For the
2859 // primitive case the types must match exactly with src.klass and
2860 // dst.klass each exactly matching the default type. For the
2861 // object array case, if no type check is needed then either the
2862 // dst type is exactly the expected type and the src type is a
2863 // subtype which we can't check or src is the same array as dst
2864 // but not necessarily exactly of type default_type.
2865 Label known_ok, halt;
2866
2867 __ mov_metadata(tmp, default_type->constant_encoding());
2868 if (UseCompressedClassPointers) {
2869 __ encode_klass_not_null(tmp, rscratch1);
2870 }
2871
2872 if (basic_type != T_OBJECT) {
2873 __ cmp_klass(tmp, dst, tmp2);
2874 __ jcc(Assembler::notEqual, halt);
2875 __ cmp_klass(tmp, src, tmp2);
2876 __ jcc(Assembler::equal, known_ok);
2877 } else {
2878 __ cmp_klass(tmp, dst, tmp2);
2879 __ jcc(Assembler::equal, known_ok);
2880 __ cmpptr(src, dst);
2881 __ jcc(Assembler::equal, known_ok);
2882 }
2883 __ bind(halt);
2884 __ stop("incorrect type information in arraycopy");
2885 __ bind(known_ok);
2886 }
3185 // first time here. Set profile type.
3186 __ movptr(mdo_addr, tmp);
3187 #ifdef ASSERT
3188 __ andptr(tmp, TypeEntries::type_klass_mask);
3189 __ verify_klass_ptr(tmp);
3190 #endif
3191 } else {
3192 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3193 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3194
3195 __ testptr(mdo_addr, TypeEntries::type_unknown);
3196 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3197
3198 __ orptr(mdo_addr, TypeEntries::type_unknown);
3199 }
3200 }
3201 }
3202 __ bind(next);
3203 }
3204
3205 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3206 Register obj = op->obj()->as_register();
3207 Register tmp = op->tmp()->as_pointer_register();
3208 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3209 bool not_null = op->not_null();
3210 int flag = op->flag();
3211
3212 Label not_inline_type;
3213 if (!not_null) {
3214 __ testptr(obj, obj);
3215 __ jccb(Assembler::zero, not_inline_type);
3216 }
3217
3218 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3219
3220 __ orb(mdo_addr, flag);
3221
3222 __ bind(not_inline_type);
3223 }
3224
3225 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3226 Unimplemented();
3227 }
3228
3229
3230 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3231 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3232 }
3233
3234
3235 void LIR_Assembler::align_backward_branch_target() {
3236 __ align(BytesPerWord);
3237 }
3238
3239
3240 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3241 if (left->is_single_cpu()) {
3242 __ negl(left->as_register());
3243 move_regs(left->as_register(), dest->as_register());
3244
3395 }
3396
3397 void LIR_Assembler::membar_loadstore() {
3398 // no-op
3399 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3400 }
3401
3402 void LIR_Assembler::membar_storeload() {
3403 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3404 }
3405
3406 void LIR_Assembler::on_spin_wait() {
3407 __ pause ();
3408 }
3409
3410 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3411 assert(result_reg->is_register(), "check");
3412 __ mov(result_reg->as_register(), r15_thread);
3413 }
3414
3415 void LIR_Assembler::check_orig_pc() {
3416 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3417 }
3418
3419 void LIR_Assembler::peephole(LIR_List*) {
3420 // do nothing for now
3421 }
3422
3423 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3424 assert(data == dest, "xchg/xadd uses only 2 operands");
3425
3426 if (data->type() == T_INT) {
3427 if (code == lir_xadd) {
3428 __ lock();
3429 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3430 } else {
3431 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3432 }
3433 } else if (data->is_oop()) {
3434 assert (code == lir_xchg, "xadd for oops");
3435 Register obj = data->as_register();
3436 if (UseCompressedOops) {
3437 __ encode_heap_oop(obj);
|