14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
415 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
416 if (LockingMode == LM_MONITOR) {
417 __ jmp(*stub->entry());
418 } else {
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 }
421 __ bind(*stub->continuation());
422 }
423
424 if (compilation()->env()->dtrace_method_probes()) {
425 __ mov(rdi, r15_thread);
426 __ mov_metadata(rsi, method()->constant_encoding());
427 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
428 }
429
430 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
431 __ mov(rax, rbx); // Restore the exception
432 }
433
434 // remove the activation and dispatch to the unwind handler
435 __ remove_frame(initial_frame_size_in_bytes());
436 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
437
438 // Emit the slow path assembly
439 if (stub != nullptr) {
440 stub->emit_code(this);
441 }
442
443 return offset;
444 }
445
446
447 int LIR_Assembler::emit_deopt_handler() {
448 // generate code for exception handler
449 address handler_base = __ start_a_stub(deopt_handler_size());
450 if (handler_base == nullptr) {
451 // not enough space left for the handler
452 bailout("deopt handler overflow");
453 return -1;
454 }
455
456 int offset = code_offset();
457 InternalAddress here(__ pc());
458
459 __ pushptr(here.addr(), rscratch1);
460 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
461 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
462 __ end_a_stub();
463
464 return offset;
465 }
466
467 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
468 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
469 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
470 assert(result->fpu() == 0, "result must already be on TOS");
471 }
472
473 // Pop the stack before the safepoint code
474 __ remove_frame(initial_frame_size_in_bytes());
475
476 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
477 __ reserved_stack_check();
478 }
479
480 // Note: we do not need to round double result; float result has the right precision
481 // the poll sets the condition code, but no data registers
482
483 code_stub->set_safepoint_offset(__ offset());
484 __ relocate(relocInfo::poll_return_type);
485 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
486 __ ret(0);
487 }
488
489
490 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
491 guarantee(info != nullptr, "Shouldn't be null");
492 int offset = __ offset();
493 const Register poll_addr = rscratch1;
494 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
495 add_debug_info_for_branch(info);
496 __ relocate(relocInfo::poll_type);
497 address pre_pc = __ pc();
498 __ testl(rax, Address(poll_addr, 0));
499 address post_pc = __ pc();
500 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
501 return offset;
502 }
503
504
505 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
506 if (from_reg != to_reg) __ mov(to_reg, from_reg);
507 }
508
509 void LIR_Assembler::swap_reg(Register a, Register b) {
1208 // init_state needs acquire, but x86 is TSO, and so we are already good.
1209 __ cmpb(Address(op->klass()->as_register(),
1210 InstanceKlass::init_state_offset()),
1211 InstanceKlass::fully_initialized);
1212 __ jcc(Assembler::notEqual, *op->stub()->entry());
1213 }
1214 __ allocate_object(op->obj()->as_register(),
1215 op->tmp1()->as_register(),
1216 op->tmp2()->as_register(),
1217 op->header_size(),
1218 op->object_size(),
1219 op->klass()->as_register(),
1220 *op->stub()->entry());
1221 __ bind(*op->stub()->continuation());
1222 }
1223
1224 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1225 Register len = op->len()->as_register();
1226 __ movslq(len, len);
1227
1228 if (UseSlowPath ||
1229 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1230 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1231 __ jmp(*op->stub()->entry());
1232 } else {
1233 Register tmp1 = op->tmp1()->as_register();
1234 Register tmp2 = op->tmp2()->as_register();
1235 Register tmp3 = op->tmp3()->as_register();
1236 if (len == tmp1) {
1237 tmp1 = tmp3;
1238 } else if (len == tmp2) {
1239 tmp2 = tmp3;
1240 } else if (len == tmp3) {
1241 // everything is ok
1242 } else {
1243 __ mov(tmp3, len);
1244 }
1245 __ allocate_array(op->obj()->as_register(),
1246 len,
1247 tmp1,
1248 tmp2,
1307 assert(data != nullptr, "need data for type check");
1308 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1309 }
1310 Label* success_target = success;
1311 Label* failure_target = failure;
1312
1313 if (obj == k_RInfo) {
1314 k_RInfo = dst;
1315 } else if (obj == klass_RInfo) {
1316 klass_RInfo = dst;
1317 }
1318 if (k->is_loaded() && !UseCompressedClassPointers) {
1319 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1320 } else {
1321 Rtmp1 = op->tmp3()->as_register();
1322 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1323 }
1324
1325 assert_different_registers(obj, k_RInfo, klass_RInfo);
1326
1327 __ testptr(obj, obj);
1328 if (op->should_profile()) {
1329 Label not_null;
1330 Register mdo = klass_RInfo;
1331 __ mov_metadata(mdo, md->constant_encoding());
1332 __ jccb(Assembler::notEqual, not_null);
1333 // Object is null; update MDO and exit
1334 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1335 int header_bits = BitData::null_seen_byte_constant();
1336 __ orb(data_addr, header_bits);
1337 __ jmp(*obj_is_null);
1338 __ bind(not_null);
1339
1340 Label update_done;
1341 Register recv = k_RInfo;
1342 __ load_klass(recv, obj, tmp_load_klass);
1343 type_profile_helper(mdo, md, data, recv, &update_done);
1344
1345 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1346 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1347
1348 __ bind(update_done);
1349 } else {
1350 __ jcc(Assembler::equal, *obj_is_null);
1351 }
1352
1353 if (!k->is_loaded()) {
1354 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1355 } else {
1356 __ mov_metadata(k_RInfo, k->constant_encoding());
1357 }
1358 __ verify_oop(obj);
1359
1360 if (op->fast_check()) {
1361 // get object class
1362 // not a safepoint as obj null check happens earlier
1363 if (UseCompressedClassPointers) {
1364 __ load_klass(Rtmp1, obj, tmp_load_klass);
1365 __ cmpptr(k_RInfo, Rtmp1);
1366 } else {
1367 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1368 }
1369 __ jcc(Assembler::notEqual, *failure_target);
1370 // successful cast, fall through to profile or jump
1371 } else {
1372 // get object class
1373 // not a safepoint as obj null check happens earlier
1374 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1375 if (k->is_loaded()) {
1376 // See if we get an immediate positive hit
1377 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1378 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1379 __ jcc(Assembler::notEqual, *failure_target);
1380 // successful cast, fall through to profile or jump
1500 __ mov(dst, obj);
1501 }
1502 } else
1503 if (code == lir_instanceof) {
1504 Register obj = op->object()->as_register();
1505 Register dst = op->result_opr()->as_register();
1506 Label success, failure, done;
1507 emit_typecheck_helper(op, &success, &failure, &failure);
1508 __ bind(failure);
1509 __ xorptr(dst, dst);
1510 __ jmpb(done);
1511 __ bind(success);
1512 __ movptr(dst, 1);
1513 __ bind(done);
1514 } else {
1515 ShouldNotReachHere();
1516 }
1517
1518 }
1519
1520
1521 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1522 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1523 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1524 Register newval = op->new_value()->as_register();
1525 Register cmpval = op->cmp_value()->as_register();
1526 assert(cmpval == rax, "wrong register");
1527 assert(newval != noreg, "new val must be register");
1528 assert(cmpval != newval, "cmp and new values must be in different registers");
1529 assert(cmpval != addr, "cmp and addr must be in different registers");
1530 assert(newval != addr, "new value and addr must be in different registers");
1531
1532 if (op->code() == lir_cas_obj) {
1533 if (UseCompressedOops) {
1534 __ encode_heap_oop(cmpval);
1535 __ mov(rscratch1, newval);
1536 __ encode_heap_oop(rscratch1);
1537 __ lock();
1538 // cmpval (rax) is implicitly used by this instruction
1539 __ cmpxchgl(rscratch1, Address(addr, 0));
1545 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1546 __ lock();
1547 __ cmpxchgl(newval, Address(addr, 0));
1548 }
1549 } else if (op->code() == lir_cas_long) {
1550 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1551 Register newval = op->new_value()->as_register_lo();
1552 Register cmpval = op->cmp_value()->as_register_lo();
1553 assert(cmpval == rax, "wrong register");
1554 assert(newval != noreg, "new val must be register");
1555 assert(cmpval != newval, "cmp and new values must be in different registers");
1556 assert(cmpval != addr, "cmp and addr must be in different registers");
1557 assert(newval != addr, "new value and addr must be in different registers");
1558 __ lock();
1559 __ cmpxchgq(newval, Address(addr, 0));
1560 } else {
1561 Unimplemented();
1562 }
1563 }
1564
1565 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1566 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1567 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1568
1569 Assembler::Condition acond, ncond;
1570 switch (condition) {
1571 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1572 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1573 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1574 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1575 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1576 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1577 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1578 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1579 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1580 ShouldNotReachHere();
1581 }
1582
1583 if (opr1->is_cpu_register()) {
1584 reg2reg(opr1, result);
2155 int offset = __ offset();
2156 switch (code) {
2157 case lir_static_call:
2158 case lir_optvirtual_call:
2159 case lir_dynamic_call:
2160 offset += NativeCall::displacement_offset;
2161 break;
2162 case lir_icvirtual_call:
2163 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2164 break;
2165 default: ShouldNotReachHere();
2166 }
2167 __ align(BytesPerWord, offset);
2168 }
2169
2170
2171 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2172 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2173 "must be aligned");
2174 __ call(AddressLiteral(op->addr(), rtype));
2175 add_call_info(code_offset(), op->info());
2176 __ post_call_nop();
2177 }
2178
2179
2180 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2181 __ ic_call(op->addr());
2182 add_call_info(code_offset(), op->info());
2183 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2184 "must be aligned");
2185 __ post_call_nop();
2186 }
2187
2188
2189 void LIR_Assembler::emit_static_call_stub() {
2190 address call_pc = __ pc();
2191 address stub = __ start_a_stub(call_stub_size());
2192 if (stub == nullptr) {
2193 bailout("static call stub overflow");
2194 return;
2195 }
2196
2197 int start = __ offset();
2198
2199 // make sure that the displacement word of the call ends up word aligned
2200 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2201 __ relocate(static_stub_Relocation::spec(call_pc));
2202 __ mov_metadata(rbx, (Metadata*)nullptr);
2329 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2330 }
2331
2332
2333 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2334 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2335 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2336 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2337 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2338 }
2339
2340
2341 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2342 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2343 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2344 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2345 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2346 }
2347
2348
2349 // This code replaces a call to arraycopy; no exception may
2350 // be thrown in this code, they must be thrown in the System.arraycopy
2351 // activation frame; we could save some checks if this would not be the case
2352 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2353 ciArrayKlass* default_type = op->expected_type();
2354 Register src = op->src()->as_register();
2355 Register dst = op->dst()->as_register();
2356 Register src_pos = op->src_pos()->as_register();
2357 Register dst_pos = op->dst_pos()->as_register();
2358 Register length = op->length()->as_register();
2359 Register tmp = op->tmp()->as_register();
2360 Register tmp_load_klass = rscratch1;
2361 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2362
2363 CodeStub* stub = op->stub();
2364 int flags = op->flags();
2365 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2366 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2367
2368 // if we don't know anything, just go through the generic arraycopy
2369 if (default_type == nullptr) {
2370 // save outgoing arguments on stack in case call to System.arraycopy is needed
2371 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2372 // for interpreter calling conventions. Now we have to do it in new style conventions.
2373 // For the moment until C1 gets the new register allocator I just force all the
2374 // args to the right place (except the register args) and then on the back side
2375 // reload the register args properly if we go slow path. Yuck
2376
2377 // These are proper for the calling convention
2378 store_parameter(length, 2);
2379 store_parameter(dst_pos, 1);
2380 store_parameter(dst, 0);
2381
2382 // these are just temporary placements until we need to reload
2383 store_parameter(src_pos, 3);
2384 store_parameter(src, 4);
2385
2386 address copyfunc_addr = StubRoutines::generic_arraycopy();
2387 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2424 __ mov(tmp, rax);
2425 __ xorl(tmp, -1);
2426
2427 // Reload values from the stack so they are where the stub
2428 // expects them.
2429 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2430 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2431 __ movptr (length, Address(rsp, 2*BytesPerWord));
2432 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2433 __ movptr (src, Address(rsp, 4*BytesPerWord));
2434
2435 __ subl(length, tmp);
2436 __ addl(src_pos, tmp);
2437 __ addl(dst_pos, tmp);
2438 __ jmp(*stub->entry());
2439
2440 __ bind(*stub->continuation());
2441 return;
2442 }
2443
2444 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2445
2446 int elem_size = type2aelembytes(basic_type);
2447 Address::ScaleFactor scale;
2448
2449 switch (elem_size) {
2450 case 1 :
2451 scale = Address::times_1;
2452 break;
2453 case 2 :
2454 scale = Address::times_2;
2455 break;
2456 case 4 :
2457 scale = Address::times_4;
2458 break;
2459 case 8 :
2460 scale = Address::times_8;
2461 break;
2462 default:
2463 scale = Address::no_scale;
2650 }
2651
2652 __ jmp(*stub->entry());
2653
2654 __ bind(cont);
2655 __ pop(dst);
2656 __ pop(src);
2657 }
2658 }
2659
2660 #ifdef ASSERT
2661 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2662 // Sanity check the known type with the incoming class. For the
2663 // primitive case the types must match exactly with src.klass and
2664 // dst.klass each exactly matching the default type. For the
2665 // object array case, if no type check is needed then either the
2666 // dst type is exactly the expected type and the src type is a
2667 // subtype which we can't check or src is the same array as dst
2668 // but not necessarily exactly of type default_type.
2669 Label known_ok, halt;
2670 __ mov_metadata(tmp, default_type->constant_encoding());
2671 if (UseCompressedClassPointers) {
2672 __ encode_klass_not_null(tmp, rscratch1);
2673 }
2674
2675 if (basic_type != T_OBJECT) {
2676 __ cmp_klass(tmp, dst, tmp2);
2677 __ jcc(Assembler::notEqual, halt);
2678 __ cmp_klass(tmp, src, tmp2);
2679 __ jcc(Assembler::equal, known_ok);
2680 } else {
2681 __ cmp_klass(tmp, dst, tmp2);
2682 __ jcc(Assembler::equal, known_ok);
2683 __ cmpptr(src, dst);
2684 __ jcc(Assembler::equal, known_ok);
2685 }
2686 __ bind(halt);
2687 __ stop("incorrect type information in arraycopy");
2688 __ bind(known_ok);
2689 }
2994 // first time here. Set profile type.
2995 __ movptr(mdo_addr, tmp);
2996 #ifdef ASSERT
2997 __ andptr(tmp, TypeEntries::type_klass_mask);
2998 __ verify_klass_ptr(tmp);
2999 #endif
3000 } else {
3001 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3002 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3003
3004 __ testptr(mdo_addr, TypeEntries::type_unknown);
3005 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3006
3007 __ orptr(mdo_addr, TypeEntries::type_unknown);
3008 }
3009 }
3010 }
3011 __ bind(next);
3012 }
3013
3014 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3015 Unimplemented();
3016 }
3017
3018
3019 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3020 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3021 }
3022
3023
3024 void LIR_Assembler::align_backward_branch_target() {
3025 __ align(BytesPerWord);
3026 }
3027
3028
3029 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3030 if (left->is_single_cpu()) {
3031 __ negl(left->as_register());
3032 move_regs(left->as_register(), dest->as_register());
3033
3184 }
3185
3186 void LIR_Assembler::membar_loadstore() {
3187 // no-op
3188 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3189 }
3190
3191 void LIR_Assembler::membar_storeload() {
3192 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3193 }
3194
3195 void LIR_Assembler::on_spin_wait() {
3196 __ pause ();
3197 }
3198
3199 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3200 assert(result_reg->is_register(), "check");
3201 __ mov(result_reg->as_register(), r15_thread);
3202 }
3203
3204
3205 void LIR_Assembler::peephole(LIR_List*) {
3206 // do nothing for now
3207 }
3208
3209 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3210 assert(data == dest, "xchg/xadd uses only 2 operands");
3211
3212 if (data->type() == T_INT) {
3213 if (code == lir_xadd) {
3214 __ lock();
3215 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3216 } else {
3217 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3218 }
3219 } else if (data->is_oop()) {
3220 assert (code == lir_xchg, "xadd for oops");
3221 Register obj = data->as_register();
3222 if (UseCompressedOops) {
3223 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
60 operand[0] = lo;
417 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
418 if (LockingMode == LM_MONITOR) {
419 __ jmp(*stub->entry());
420 } else {
421 __ unlock_object(rdi, rsi, rax, *stub->entry());
422 }
423 __ bind(*stub->continuation());
424 }
425
426 if (compilation()->env()->dtrace_method_probes()) {
427 __ mov(rdi, r15_thread);
428 __ mov_metadata(rsi, method()->constant_encoding());
429 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
430 }
431
432 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
433 __ mov(rax, rbx); // Restore the exception
434 }
435
436 // remove the activation and dispatch to the unwind handler
437 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
438 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
439
440 // Emit the slow path assembly
441 if (stub != nullptr) {
442 stub->emit_code(this);
443 }
444
445 return offset;
446 }
447
448
449 int LIR_Assembler::emit_deopt_handler() {
450 // generate code for exception handler
451 address handler_base = __ start_a_stub(deopt_handler_size());
452 if (handler_base == nullptr) {
453 // not enough space left for the handler
454 bailout("deopt handler overflow");
455 return -1;
456 }
457
458 int offset = code_offset();
459 InternalAddress here(__ pc());
460
461 __ pushptr(here.addr(), rscratch1);
462 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
463 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
464 __ end_a_stub();
465
466 return offset;
467 }
468
469 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
470 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
471 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
472 assert(result->fpu() == 0, "result must already be on TOS");
473 }
474 if (InlineTypeReturnedAsFields) {
475 #ifndef _LP64
476 Unimplemented();
477 #endif
478 // Check if we are returning an non-null inline type and load its fields into registers
479 ciType* return_type = compilation()->method()->return_type();
480 if (return_type->is_inlinetype()) {
481 ciInlineKlass* vk = return_type->as_inline_klass();
482 if (vk->can_be_returned_as_fields()) {
483 address unpack_handler = vk->unpack_handler();
484 assert(unpack_handler != nullptr, "must be");
485 __ call(RuntimeAddress(unpack_handler));
486 }
487 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
488 Label skip;
489 Label not_null;
490 __ testptr(rax, rax);
491 __ jcc(Assembler::notZero, not_null);
492 // Returned value is null, zero all return registers because they may belong to oop fields
493 __ xorq(j_rarg1, j_rarg1);
494 __ xorq(j_rarg2, j_rarg2);
495 __ xorq(j_rarg3, j_rarg3);
496 __ xorq(j_rarg4, j_rarg4);
497 __ xorq(j_rarg5, j_rarg5);
498 __ jmp(skip);
499 __ bind(not_null);
500
501 // Check if we are returning an non-null inline type and load its fields into registers
502 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
503
504 // Load fields from a buffered value with an inline class specific handler
505 __ load_klass(rdi, rax, rscratch1);
506 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
507 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
508 // Unpack handler can be null if inline type is not scalarizable in returns
509 __ testptr(rdi, rdi);
510 __ jcc(Assembler::zero, skip);
511 __ call(rdi);
512
513 __ bind(skip);
514 }
515 // At this point, rax points to the value object (for interpreter or C1 caller).
516 // The fields of the object are copied into registers (for C2 caller).
517 }
518
519 // Pop the stack before the safepoint code
520 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
521
522 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
523 __ reserved_stack_check();
524 }
525
526 // Note: we do not need to round double result; float result has the right precision
527 // the poll sets the condition code, but no data registers
528
529 code_stub->set_safepoint_offset(__ offset());
530 __ relocate(relocInfo::poll_return_type);
531 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
532 __ ret(0);
533 }
534
535
536 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
537 return (__ store_inline_type_fields_to_buf(vk, false));
538 }
539
540 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
541 guarantee(info != nullptr, "Shouldn't be null");
542 int offset = __ offset();
543 const Register poll_addr = rscratch1;
544 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
545 add_debug_info_for_branch(info);
546 __ relocate(relocInfo::poll_type);
547 address pre_pc = __ pc();
548 __ testl(rax, Address(poll_addr, 0));
549 address post_pc = __ pc();
550 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
551 return offset;
552 }
553
554
555 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
556 if (from_reg != to_reg) __ mov(to_reg, from_reg);
557 }
558
559 void LIR_Assembler::swap_reg(Register a, Register b) {
1258 // init_state needs acquire, but x86 is TSO, and so we are already good.
1259 __ cmpb(Address(op->klass()->as_register(),
1260 InstanceKlass::init_state_offset()),
1261 InstanceKlass::fully_initialized);
1262 __ jcc(Assembler::notEqual, *op->stub()->entry());
1263 }
1264 __ allocate_object(op->obj()->as_register(),
1265 op->tmp1()->as_register(),
1266 op->tmp2()->as_register(),
1267 op->header_size(),
1268 op->object_size(),
1269 op->klass()->as_register(),
1270 *op->stub()->entry());
1271 __ bind(*op->stub()->continuation());
1272 }
1273
1274 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1275 Register len = op->len()->as_register();
1276 __ movslq(len, len);
1277
1278 if (UseSlowPath || op->always_slow_path() ||
1279 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1280 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1281 __ jmp(*op->stub()->entry());
1282 } else {
1283 Register tmp1 = op->tmp1()->as_register();
1284 Register tmp2 = op->tmp2()->as_register();
1285 Register tmp3 = op->tmp3()->as_register();
1286 if (len == tmp1) {
1287 tmp1 = tmp3;
1288 } else if (len == tmp2) {
1289 tmp2 = tmp3;
1290 } else if (len == tmp3) {
1291 // everything is ok
1292 } else {
1293 __ mov(tmp3, len);
1294 }
1295 __ allocate_array(op->obj()->as_register(),
1296 len,
1297 tmp1,
1298 tmp2,
1357 assert(data != nullptr, "need data for type check");
1358 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1359 }
1360 Label* success_target = success;
1361 Label* failure_target = failure;
1362
1363 if (obj == k_RInfo) {
1364 k_RInfo = dst;
1365 } else if (obj == klass_RInfo) {
1366 klass_RInfo = dst;
1367 }
1368 if (k->is_loaded() && !UseCompressedClassPointers) {
1369 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1370 } else {
1371 Rtmp1 = op->tmp3()->as_register();
1372 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1373 }
1374
1375 assert_different_registers(obj, k_RInfo, klass_RInfo);
1376
1377 if (op->need_null_check()) {
1378 __ testptr(obj, obj);
1379 if (op->should_profile()) {
1380 Label not_null;
1381 Register mdo = klass_RInfo;
1382 __ mov_metadata(mdo, md->constant_encoding());
1383 __ jccb(Assembler::notEqual, not_null);
1384 // Object is null; update MDO and exit
1385 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1386 int header_bits = BitData::null_seen_byte_constant();
1387 __ orb(data_addr, header_bits);
1388 __ jmp(*obj_is_null);
1389 __ bind(not_null);
1390
1391 Label update_done;
1392 Register recv = k_RInfo;
1393 __ load_klass(recv, obj, tmp_load_klass);
1394 type_profile_helper(mdo, md, data, recv, &update_done);
1395
1396 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1397 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1398
1399 __ bind(update_done);
1400 } else {
1401 __ jcc(Assembler::equal, *obj_is_null);
1402 }
1403 }
1404
1405 if (!k->is_loaded()) {
1406 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1407 } else {
1408 __ mov_metadata(k_RInfo, k->constant_encoding());
1409 }
1410 __ verify_oop(obj);
1411
1412 if (op->fast_check()) {
1413 // TODO 8366668 Is this correct? I don't think so. Probably we now always go to the slow path here. Same on AArch64.
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 if (UseCompressedClassPointers) {
1417 __ load_klass(Rtmp1, obj, tmp_load_klass);
1418 __ cmpptr(k_RInfo, Rtmp1);
1419 } else {
1420 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1421 }
1422 __ jcc(Assembler::notEqual, *failure_target);
1423 // successful cast, fall through to profile or jump
1424 } else {
1425 // get object class
1426 // not a safepoint as obj null check happens earlier
1427 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1428 if (k->is_loaded()) {
1429 // See if we get an immediate positive hit
1430 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1431 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1432 __ jcc(Assembler::notEqual, *failure_target);
1433 // successful cast, fall through to profile or jump
1553 __ mov(dst, obj);
1554 }
1555 } else
1556 if (code == lir_instanceof) {
1557 Register obj = op->object()->as_register();
1558 Register dst = op->result_opr()->as_register();
1559 Label success, failure, done;
1560 emit_typecheck_helper(op, &success, &failure, &failure);
1561 __ bind(failure);
1562 __ xorptr(dst, dst);
1563 __ jmpb(done);
1564 __ bind(success);
1565 __ movptr(dst, 1);
1566 __ bind(done);
1567 } else {
1568 ShouldNotReachHere();
1569 }
1570
1571 }
1572
1573 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1574 // We are loading/storing from/to an array that *may* be a flat array (the
1575 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1576 // If this array is a flat array, take the slow path.
1577 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1578 if (!op->value()->is_illegal()) {
1579 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1580 // The array is not a flat array, but it might be null-free. If we are storing
1581 // a null into a null-free array, take the slow path (which will throw NPE).
1582 Label skip;
1583 __ cmpptr(op->value()->as_register(), NULL_WORD);
1584 __ jcc(Assembler::notEqual, skip);
1585 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1586 __ bind(skip);
1587 }
1588 }
1589
1590 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1591 // We are storing into an array that *may* be null-free (the declared type is
1592 // Object[], abstract[], interface[] or VT.ref[]).
1593 Label test_mark_word;
1594 Register tmp = op->tmp()->as_register();
1595 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1596 __ testl(tmp, markWord::unlocked_value);
1597 __ jccb(Assembler::notZero, test_mark_word);
1598 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1599 __ bind(test_mark_word);
1600 __ testl(tmp, markWord::null_free_array_bit_in_place);
1601 }
1602
1603 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1604 Label L_oops_equal;
1605 Label L_oops_not_equal;
1606 Label L_end;
1607
1608 Register left = op->left()->as_register();
1609 Register right = op->right()->as_register();
1610
1611 __ cmpptr(left, right);
1612 __ jcc(Assembler::equal, L_oops_equal);
1613
1614 // (1) Null check -- if one of the operands is null, the other must not be null (because
1615 // the two references are not equal), so they are not substitutable,
1616 // FIXME: do null check only if the operand is nullable
1617 __ testptr(left, right);
1618 __ jcc(Assembler::zero, L_oops_not_equal);
1619
1620 ciKlass* left_klass = op->left_klass();
1621 ciKlass* right_klass = op->right_klass();
1622
1623 // (2) Inline type check -- if either of the operands is not a inline type,
1624 // they are not substitutable. We do this only if we are not sure that the
1625 // operands are inline type
1626 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1627 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1628 Register tmp1 = op->tmp1()->as_register();
1629 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1630 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1631 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1632 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1633 __ jcc(Assembler::notEqual, L_oops_not_equal);
1634 }
1635
1636 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1637 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1638 // No need to load klass -- the operands are statically known to be the same inline klass.
1639 __ jmp(*op->stub()->entry());
1640 } else {
1641 Register left_klass_op = op->left_klass_op()->as_register();
1642 Register right_klass_op = op->right_klass_op()->as_register();
1643
1644 if (UseCompressedClassPointers) {
1645 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1646 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1647 __ cmpl(left_klass_op, right_klass_op);
1648 } else {
1649 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1650 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1651 __ cmpptr(left_klass_op, right_klass_op);
1652 }
1653
1654 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1655 // fall through to L_oops_not_equal
1656 }
1657
1658 __ bind(L_oops_not_equal);
1659 move(op->not_equal_result(), op->result_opr());
1660 __ jmp(L_end);
1661
1662 __ bind(L_oops_equal);
1663 move(op->equal_result(), op->result_opr());
1664 __ jmp(L_end);
1665
1666 // We've returned from the stub. RAX contains 0x0 IFF the two
1667 // operands are not substitutable. (Don't compare against 0x1 in case the
1668 // C compiler is naughty)
1669 __ bind(*op->stub()->continuation());
1670 __ cmpl(rax, 0);
1671 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1672 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1673 // fall-through
1674 __ bind(L_end);
1675 }
1676
1677 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1678 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1679 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1680 Register newval = op->new_value()->as_register();
1681 Register cmpval = op->cmp_value()->as_register();
1682 assert(cmpval == rax, "wrong register");
1683 assert(newval != noreg, "new val must be register");
1684 assert(cmpval != newval, "cmp and new values must be in different registers");
1685 assert(cmpval != addr, "cmp and addr must be in different registers");
1686 assert(newval != addr, "new value and addr must be in different registers");
1687
1688 if (op->code() == lir_cas_obj) {
1689 if (UseCompressedOops) {
1690 __ encode_heap_oop(cmpval);
1691 __ mov(rscratch1, newval);
1692 __ encode_heap_oop(rscratch1);
1693 __ lock();
1694 // cmpval (rax) is implicitly used by this instruction
1695 __ cmpxchgl(rscratch1, Address(addr, 0));
1701 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1702 __ lock();
1703 __ cmpxchgl(newval, Address(addr, 0));
1704 }
1705 } else if (op->code() == lir_cas_long) {
1706 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1707 Register newval = op->new_value()->as_register_lo();
1708 Register cmpval = op->cmp_value()->as_register_lo();
1709 assert(cmpval == rax, "wrong register");
1710 assert(newval != noreg, "new val must be register");
1711 assert(cmpval != newval, "cmp and new values must be in different registers");
1712 assert(cmpval != addr, "cmp and addr must be in different registers");
1713 assert(newval != addr, "new value and addr must be in different registers");
1714 __ lock();
1715 __ cmpxchgq(newval, Address(addr, 0));
1716 } else {
1717 Unimplemented();
1718 }
1719 }
1720
1721 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1722 assert(dst->is_cpu_register(), "must be");
1723 assert(dst->type() == src->type(), "must be");
1724
1725 if (src->is_cpu_register()) {
1726 reg2reg(src, dst);
1727 } else if (src->is_stack()) {
1728 stack2reg(src, dst, dst->type());
1729 } else if (src->is_constant()) {
1730 const2reg(src, dst, lir_patch_none, nullptr);
1731 } else {
1732 ShouldNotReachHere();
1733 }
1734 }
1735
1736 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1737 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1738 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1739
1740 Assembler::Condition acond, ncond;
1741 switch (condition) {
1742 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1743 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1744 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1745 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1746 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1747 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1748 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1749 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1750 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1751 ShouldNotReachHere();
1752 }
1753
1754 if (opr1->is_cpu_register()) {
1755 reg2reg(opr1, result);
2326 int offset = __ offset();
2327 switch (code) {
2328 case lir_static_call:
2329 case lir_optvirtual_call:
2330 case lir_dynamic_call:
2331 offset += NativeCall::displacement_offset;
2332 break;
2333 case lir_icvirtual_call:
2334 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2335 break;
2336 default: ShouldNotReachHere();
2337 }
2338 __ align(BytesPerWord, offset);
2339 }
2340
2341
2342 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2343 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2344 "must be aligned");
2345 __ call(AddressLiteral(op->addr(), rtype));
2346 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2347 __ post_call_nop();
2348 }
2349
2350
2351 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2352 __ ic_call(op->addr());
2353 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2354 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2355 "must be aligned");
2356 __ post_call_nop();
2357 }
2358
2359
2360 void LIR_Assembler::emit_static_call_stub() {
2361 address call_pc = __ pc();
2362 address stub = __ start_a_stub(call_stub_size());
2363 if (stub == nullptr) {
2364 bailout("static call stub overflow");
2365 return;
2366 }
2367
2368 int start = __ offset();
2369
2370 // make sure that the displacement word of the call ends up word aligned
2371 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2372 __ relocate(static_stub_Relocation::spec(call_pc));
2373 __ mov_metadata(rbx, (Metadata*)nullptr);
2500 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2501 }
2502
2503
2504 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2505 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2506 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2507 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2508 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2509 }
2510
2511
2512 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2513 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2514 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2515 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2516 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2517 }
2518
2519
2520 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2521 if (null_check) {
2522 __ testptr(obj, obj);
2523 __ jcc(Assembler::zero, *slow_path->entry());
2524 }
2525 if (is_dest) {
2526 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2527 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2528 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2529 } else {
2530 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2531 }
2532 }
2533
2534
2535 // This code replaces a call to arraycopy; no exception may
2536 // be thrown in this code, they must be thrown in the System.arraycopy
2537 // activation frame; we could save some checks if this would not be the case
2538 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2539 ciArrayKlass* default_type = op->expected_type();
2540 Register src = op->src()->as_register();
2541 Register dst = op->dst()->as_register();
2542 Register src_pos = op->src_pos()->as_register();
2543 Register dst_pos = op->dst_pos()->as_register();
2544 Register length = op->length()->as_register();
2545 Register tmp = op->tmp()->as_register();
2546 Register tmp_load_klass = rscratch1;
2547 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2548
2549 CodeStub* stub = op->stub();
2550 int flags = op->flags();
2551 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2552 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2553
2554 if (flags & LIR_OpArrayCopy::always_slow_path) {
2555 __ jmp(*stub->entry());
2556 __ bind(*stub->continuation());
2557 return;
2558 }
2559
2560 // if we don't know anything, just go through the generic arraycopy
2561 if (default_type == nullptr) {
2562 // save outgoing arguments on stack in case call to System.arraycopy is needed
2563 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2564 // for interpreter calling conventions. Now we have to do it in new style conventions.
2565 // For the moment until C1 gets the new register allocator I just force all the
2566 // args to the right place (except the register args) and then on the back side
2567 // reload the register args properly if we go slow path. Yuck
2568
2569 // These are proper for the calling convention
2570 store_parameter(length, 2);
2571 store_parameter(dst_pos, 1);
2572 store_parameter(dst, 0);
2573
2574 // these are just temporary placements until we need to reload
2575 store_parameter(src_pos, 3);
2576 store_parameter(src, 4);
2577
2578 address copyfunc_addr = StubRoutines::generic_arraycopy();
2579 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2616 __ mov(tmp, rax);
2617 __ xorl(tmp, -1);
2618
2619 // Reload values from the stack so they are where the stub
2620 // expects them.
2621 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2622 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2623 __ movptr (length, Address(rsp, 2*BytesPerWord));
2624 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2625 __ movptr (src, Address(rsp, 4*BytesPerWord));
2626
2627 __ subl(length, tmp);
2628 __ addl(src_pos, tmp);
2629 __ addl(dst_pos, tmp);
2630 __ jmp(*stub->entry());
2631
2632 __ bind(*stub->continuation());
2633 return;
2634 }
2635
2636 // Handle inline type arrays
2637 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2638 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2639 }
2640 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2641 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2642 }
2643
2644 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2645
2646 int elem_size = type2aelembytes(basic_type);
2647 Address::ScaleFactor scale;
2648
2649 switch (elem_size) {
2650 case 1 :
2651 scale = Address::times_1;
2652 break;
2653 case 2 :
2654 scale = Address::times_2;
2655 break;
2656 case 4 :
2657 scale = Address::times_4;
2658 break;
2659 case 8 :
2660 scale = Address::times_8;
2661 break;
2662 default:
2663 scale = Address::no_scale;
2850 }
2851
2852 __ jmp(*stub->entry());
2853
2854 __ bind(cont);
2855 __ pop(dst);
2856 __ pop(src);
2857 }
2858 }
2859
2860 #ifdef ASSERT
2861 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2862 // Sanity check the known type with the incoming class. For the
2863 // primitive case the types must match exactly with src.klass and
2864 // dst.klass each exactly matching the default type. For the
2865 // object array case, if no type check is needed then either the
2866 // dst type is exactly the expected type and the src type is a
2867 // subtype which we can't check or src is the same array as dst
2868 // but not necessarily exactly of type default_type.
2869 Label known_ok, halt;
2870
2871 __ mov_metadata(tmp, default_type->constant_encoding());
2872 if (UseCompressedClassPointers) {
2873 __ encode_klass_not_null(tmp, rscratch1);
2874 }
2875
2876 if (basic_type != T_OBJECT) {
2877 __ cmp_klass(tmp, dst, tmp2);
2878 __ jcc(Assembler::notEqual, halt);
2879 __ cmp_klass(tmp, src, tmp2);
2880 __ jcc(Assembler::equal, known_ok);
2881 } else {
2882 __ cmp_klass(tmp, dst, tmp2);
2883 __ jcc(Assembler::equal, known_ok);
2884 __ cmpptr(src, dst);
2885 __ jcc(Assembler::equal, known_ok);
2886 }
2887 __ bind(halt);
2888 __ stop("incorrect type information in arraycopy");
2889 __ bind(known_ok);
2890 }
3195 // first time here. Set profile type.
3196 __ movptr(mdo_addr, tmp);
3197 #ifdef ASSERT
3198 __ andptr(tmp, TypeEntries::type_klass_mask);
3199 __ verify_klass_ptr(tmp);
3200 #endif
3201 } else {
3202 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3203 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3204
3205 __ testptr(mdo_addr, TypeEntries::type_unknown);
3206 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3207
3208 __ orptr(mdo_addr, TypeEntries::type_unknown);
3209 }
3210 }
3211 }
3212 __ bind(next);
3213 }
3214
3215 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3216 Register obj = op->obj()->as_register();
3217 Register tmp = op->tmp()->as_pointer_register();
3218 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3219 bool not_null = op->not_null();
3220 int flag = op->flag();
3221
3222 Label not_inline_type;
3223 if (!not_null) {
3224 __ testptr(obj, obj);
3225 __ jccb(Assembler::zero, not_inline_type);
3226 }
3227
3228 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3229
3230 __ orb(mdo_addr, flag);
3231
3232 __ bind(not_inline_type);
3233 }
3234
3235 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3236 Unimplemented();
3237 }
3238
3239
3240 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3241 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3242 }
3243
3244
3245 void LIR_Assembler::align_backward_branch_target() {
3246 __ align(BytesPerWord);
3247 }
3248
3249
3250 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3251 if (left->is_single_cpu()) {
3252 __ negl(left->as_register());
3253 move_regs(left->as_register(), dest->as_register());
3254
3405 }
3406
3407 void LIR_Assembler::membar_loadstore() {
3408 // no-op
3409 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3410 }
3411
3412 void LIR_Assembler::membar_storeload() {
3413 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3414 }
3415
3416 void LIR_Assembler::on_spin_wait() {
3417 __ pause ();
3418 }
3419
3420 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3421 assert(result_reg->is_register(), "check");
3422 __ mov(result_reg->as_register(), r15_thread);
3423 }
3424
3425 void LIR_Assembler::check_orig_pc() {
3426 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3427 }
3428
3429 void LIR_Assembler::peephole(LIR_List*) {
3430 // do nothing for now
3431 }
3432
3433 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3434 assert(data == dest, "xchg/xadd uses only 2 operands");
3435
3436 if (data->type() == T_INT) {
3437 if (code == lir_xadd) {
3438 __ lock();
3439 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3440 } else {
3441 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3442 }
3443 } else if (data->is_oop()) {
3444 assert (code == lir_xchg, "xadd for oops");
3445 Register obj = data->as_register();
3446 if (UseCompressedOops) {
3447 __ encode_heap_oop(obj);
|