14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "code/aotCodeCache.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "runtime/threadIdentifier.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
407 // Perform needed unlocking
408 MonitorExitStub* stub = nullptr;
409 if (method()->is_synchronized()) {
410 monitor_address(0, FrameMap::rax_opr);
411 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
412 __ unlock_object(rdi, rsi, rax, *stub->entry());
413 __ bind(*stub->continuation());
414 }
415
416 if (compilation()->env()->dtrace_method_probes()) {
417 __ mov(rdi, r15_thread);
418 __ mov_metadata(rsi, method()->constant_encoding());
419 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
420 }
421
422 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
423 __ mov(rax, rbx); // Restore the exception
424 }
425
426 // remove the activation and dispatch to the unwind handler
427 __ remove_frame(initial_frame_size_in_bytes());
428 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
429
430 // Emit the slow path assembly
431 if (stub != nullptr) {
432 stub->emit_code(this);
433 }
434
435 return offset;
436 }
437
438
439 int LIR_Assembler::emit_deopt_handler() {
440 // generate code for exception handler
441 address handler_base = __ start_a_stub(deopt_handler_size());
442 if (handler_base == nullptr) {
443 // not enough space left for the handler
444 bailout("deopt handler overflow");
445 return -1;
446 }
447
452
453 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
454
455 int entry_offset = __ offset();
456
457 __ jmp(start);
458
459 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
460 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
461 "out of bounds read in post-call NOP check");
462 __ end_a_stub();
463
464 return entry_offset;
465 }
466
467 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
468 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
469 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
470 assert(result->fpu() == 0, "result must already be on TOS");
471 }
472
473 // Pop the stack before the safepoint code
474 __ remove_frame(initial_frame_size_in_bytes());
475
476 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
477 __ reserved_stack_check();
478 }
479
480 // Note: we do not need to round double result; float result has the right precision
481 // the poll sets the condition code, but no data registers
482
483 code_stub->set_safepoint_offset(__ offset());
484 __ relocate(relocInfo::poll_return_type);
485 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
486 __ ret(0);
487 }
488
489
490 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
491 guarantee(info != nullptr, "Shouldn't be null");
492 int offset = __ offset();
493 const Register poll_addr = rscratch1;
494 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
495 add_debug_info_for_branch(info);
496 __ relocate(relocInfo::poll_type);
497 address pre_pc = __ pc();
498 __ testl(rax, Address(poll_addr, 0));
499 address post_pc = __ pc();
500 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
501 return offset;
502 }
503
504
505 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
506 if (from_reg != to_reg) __ mov(to_reg, from_reg);
507 }
508
509 void LIR_Assembler::swap_reg(Register a, Register b) {
1232 // init_state needs acquire, but x86 is TSO, and so we are already good.
1233 __ cmpb(Address(op->klass()->as_register(),
1234 InstanceKlass::init_state_offset()),
1235 InstanceKlass::fully_initialized);
1236 __ jcc(Assembler::notEqual, *op->stub()->entry());
1237 }
1238 __ allocate_object(op->obj()->as_register(),
1239 op->tmp1()->as_register(),
1240 op->tmp2()->as_register(),
1241 op->header_size(),
1242 op->object_size(),
1243 op->klass()->as_register(),
1244 *op->stub()->entry());
1245 __ bind(*op->stub()->continuation());
1246 }
1247
1248 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1249 Register len = op->len()->as_register();
1250 __ movslq(len, len);
1251
1252 if (UseSlowPath ||
1253 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1254 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1255 __ jmp(*op->stub()->entry());
1256 } else {
1257 Register tmp1 = op->tmp1()->as_register();
1258 Register tmp2 = op->tmp2()->as_register();
1259 Register tmp3 = op->tmp3()->as_register();
1260 if (len == tmp1) {
1261 tmp1 = tmp3;
1262 } else if (len == tmp2) {
1263 tmp2 = tmp3;
1264 } else if (len == tmp3) {
1265 // everything is ok
1266 } else {
1267 __ mov(tmp3, len);
1268 }
1269 __ allocate_array(op->obj()->as_register(),
1270 len,
1271 tmp1,
1272 tmp2,
1307 int bci = op->profiled_bci();
1308 md = method->method_data_or_null();
1309 assert(md != nullptr, "Sanity");
1310 data = md->bci_to_data(bci);
1311 assert(data != nullptr, "need data for type check");
1312 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1313 }
1314 Label* success_target = success;
1315 Label* failure_target = failure;
1316
1317 if (obj == k_RInfo) {
1318 k_RInfo = dst;
1319 } else if (obj == klass_RInfo) {
1320 klass_RInfo = dst;
1321 }
1322 Rtmp1 = op->tmp3()->as_register();
1323 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1324
1325 assert_different_registers(obj, k_RInfo, klass_RInfo);
1326
1327 __ testptr(obj, obj);
1328 if (op->should_profile()) {
1329 Label not_null;
1330 Register mdo = klass_RInfo;
1331 __ mov_metadata(mdo, md->constant_encoding());
1332 __ jccb(Assembler::notEqual, not_null);
1333 // Object is null; update MDO and exit
1334 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1335 int header_bits = BitData::null_seen_byte_constant();
1336 __ orb(data_addr, header_bits);
1337 __ jmp(*obj_is_null);
1338 __ bind(not_null);
1339
1340 Register recv = k_RInfo;
1341 __ load_klass(recv, obj, tmp_load_klass);
1342 type_profile_helper(mdo, md, data, recv);
1343 } else {
1344 __ jcc(Assembler::equal, *obj_is_null);
1345 }
1346
1347 if (!k->is_loaded()) {
1348 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1349 } else {
1350 __ mov_metadata(k_RInfo, k->constant_encoding());
1351 }
1352 __ verify_oop(obj);
1353
1354 if (op->fast_check()) {
1355 // get object class
1356 // not a safepoint as obj null check happens earlier
1357 __ load_klass(Rtmp1, obj, tmp_load_klass);
1358 __ cmpptr(k_RInfo, Rtmp1);
1359 __ jcc(Assembler::notEqual, *failure_target);
1360 // successful cast, fall through to profile or jump
1361 } else {
1362 // get object class
1363 // not a safepoint as obj null check happens earlier
1364 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1365 if (k->is_loaded()) {
1366 // See if we get an immediate positive hit
1367 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1368 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1369 __ jcc(Assembler::notEqual, *failure_target);
1370 // successful cast, fall through to profile or jump
1371 } else {
1372 // See if we get an immediate positive hit
1373 __ jcc(Assembler::equal, *success_target);
1374 // check for self
1375 __ cmpptr(klass_RInfo, k_RInfo);
1376 __ jcc(Assembler::equal, *success_target);
1377
1378 __ push_ppx(klass_RInfo);
1379 __ push_ppx(k_RInfo);
1380 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1381 __ pop_ppx(klass_RInfo);
1382 __ pop_ppx(klass_RInfo);
1383 // result is a boolean
1384 __ testl(klass_RInfo, klass_RInfo);
1385 __ jcc(Assembler::equal, *failure_target);
1386 // successful cast, fall through to profile or jump
1387 }
1388 } else {
1389 // perform the fast part of the checking logic
1390 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1391 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1392 __ push_ppx(klass_RInfo);
1393 __ push_ppx(k_RInfo);
1394 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1395 __ pop_ppx(klass_RInfo);
1485 __ mov(dst, obj);
1486 }
1487 } else
1488 if (code == lir_instanceof) {
1489 Register obj = op->object()->as_register();
1490 Register dst = op->result_opr()->as_register();
1491 Label success, failure, done;
1492 emit_typecheck_helper(op, &success, &failure, &failure);
1493 __ bind(failure);
1494 __ xorptr(dst, dst);
1495 __ jmpb(done);
1496 __ bind(success);
1497 __ movptr(dst, 1);
1498 __ bind(done);
1499 } else {
1500 ShouldNotReachHere();
1501 }
1502
1503 }
1504
1505
1506 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1507 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1508 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1509 Register newval = op->new_value()->as_register();
1510 Register cmpval = op->cmp_value()->as_register();
1511 assert(cmpval == rax, "wrong register");
1512 assert(newval != noreg, "new val must be register");
1513 assert(cmpval != newval, "cmp and new values must be in different registers");
1514 assert(cmpval != addr, "cmp and addr must be in different registers");
1515 assert(newval != addr, "new value and addr must be in different registers");
1516
1517 if (op->code() == lir_cas_obj) {
1518 if (UseCompressedOops) {
1519 __ encode_heap_oop(cmpval);
1520 __ mov(rscratch1, newval);
1521 __ encode_heap_oop(rscratch1);
1522 __ lock();
1523 // cmpval (rax) is implicitly used by this instruction
1524 __ cmpxchgl(rscratch1, Address(addr, 0));
1530 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1531 __ lock();
1532 __ cmpxchgl(newval, Address(addr, 0));
1533 }
1534 } else if (op->code() == lir_cas_long) {
1535 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1536 Register newval = op->new_value()->as_register_lo();
1537 Register cmpval = op->cmp_value()->as_register_lo();
1538 assert(cmpval == rax, "wrong register");
1539 assert(newval != noreg, "new val must be register");
1540 assert(cmpval != newval, "cmp and new values must be in different registers");
1541 assert(cmpval != addr, "cmp and addr must be in different registers");
1542 assert(newval != addr, "new value and addr must be in different registers");
1543 __ lock();
1544 __ cmpxchgq(newval, Address(addr, 0));
1545 } else {
1546 Unimplemented();
1547 }
1548 }
1549
1550 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1551 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1552 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1553
1554 Assembler::Condition acond, ncond;
1555 switch (condition) {
1556 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1557 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1558 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1559 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1560 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1561 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1562 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1563 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1564 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1565 ShouldNotReachHere();
1566 }
1567
1568 if (opr1->is_cpu_register()) {
1569 reg2reg(opr1, result);
2140 int offset = __ offset();
2141 switch (code) {
2142 case lir_static_call:
2143 case lir_optvirtual_call:
2144 case lir_dynamic_call:
2145 offset += NativeCall::displacement_offset;
2146 break;
2147 case lir_icvirtual_call:
2148 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2149 break;
2150 default: ShouldNotReachHere();
2151 }
2152 __ align(BytesPerWord, offset);
2153 }
2154
2155
2156 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2157 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2158 "must be aligned");
2159 __ call(AddressLiteral(op->addr(), rtype));
2160 add_call_info(code_offset(), op->info());
2161 __ post_call_nop();
2162 }
2163
2164
2165 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2166 __ ic_call(op->addr());
2167 add_call_info(code_offset(), op->info());
2168 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2169 "must be aligned");
2170 __ post_call_nop();
2171 }
2172
2173
2174 void LIR_Assembler::emit_static_call_stub() {
2175 address call_pc = __ pc();
2176 address stub = __ start_a_stub(call_stub_size());
2177 if (stub == nullptr) {
2178 bailout("static call stub overflow");
2179 return;
2180 }
2181
2182 int start = __ offset();
2183
2184 // make sure that the displacement word of the call ends up word aligned
2185 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2186 __ relocate(static_stub_Relocation::spec(call_pc));
2187 __ mov_metadata(rbx, (Metadata*)nullptr);
2314 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2315 }
2316
2317
2318 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2319 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2320 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2321 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2322 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2323 }
2324
2325
2326 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2327 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2328 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2329 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2330 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2331 }
2332
2333
2334 // This code replaces a call to arraycopy; no exception may
2335 // be thrown in this code, they must be thrown in the System.arraycopy
2336 // activation frame; we could save some checks if this would not be the case
2337 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2338 ciArrayKlass* default_type = op->expected_type();
2339 Register src = op->src()->as_register();
2340 Register dst = op->dst()->as_register();
2341 Register src_pos = op->src_pos()->as_register();
2342 Register dst_pos = op->dst_pos()->as_register();
2343 Register length = op->length()->as_register();
2344 Register tmp = op->tmp()->as_register();
2345 Register tmp_load_klass = rscratch1;
2346 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2347
2348 CodeStub* stub = op->stub();
2349 int flags = op->flags();
2350 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2351 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2352
2353 // if we don't know anything, just go through the generic arraycopy
2354 if (default_type == nullptr) {
2355 // save outgoing arguments on stack in case call to System.arraycopy is needed
2356 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2357 // for interpreter calling conventions. Now we have to do it in new style conventions.
2358 // For the moment until C1 gets the new register allocator I just force all the
2359 // args to the right place (except the register args) and then on the back side
2360 // reload the register args properly if we go slow path. Yuck
2361
2362 // These are proper for the calling convention
2363 store_parameter(length, 2);
2364 store_parameter(dst_pos, 1);
2365 store_parameter(dst, 0);
2366
2367 // these are just temporary placements until we need to reload
2368 store_parameter(src_pos, 3);
2369 store_parameter(src, 4);
2370
2371 address copyfunc_addr = StubRoutines::generic_arraycopy();
2372 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2409 __ mov(tmp, rax);
2410 __ xorl(tmp, -1);
2411
2412 // Reload values from the stack so they are where the stub
2413 // expects them.
2414 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2415 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2416 __ movptr (length, Address(rsp, 2*BytesPerWord));
2417 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2418 __ movptr (src, Address(rsp, 4*BytesPerWord));
2419
2420 __ subl(length, tmp);
2421 __ addl(src_pos, tmp);
2422 __ addl(dst_pos, tmp);
2423 __ jmp(*stub->entry());
2424
2425 __ bind(*stub->continuation());
2426 return;
2427 }
2428
2429 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2430
2431 int elem_size = type2aelembytes(basic_type);
2432 Address::ScaleFactor scale;
2433
2434 switch (elem_size) {
2435 case 1 :
2436 scale = Address::times_1;
2437 break;
2438 case 2 :
2439 scale = Address::times_2;
2440 break;
2441 case 4 :
2442 scale = Address::times_4;
2443 break;
2444 case 8 :
2445 scale = Address::times_8;
2446 break;
2447 default:
2448 scale = Address::no_scale;
2946 // first time here. Set profile type.
2947 __ movptr(mdo_addr, tmp);
2948 #ifdef ASSERT
2949 __ andptr(tmp, TypeEntries::type_klass_mask);
2950 __ verify_klass_ptr(tmp);
2951 #endif
2952 } else {
2953 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2954 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2955
2956 __ testptr(mdo_addr, TypeEntries::type_unknown);
2957 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2958
2959 __ orptr(mdo_addr, TypeEntries::type_unknown);
2960 }
2961 }
2962 }
2963 __ bind(next);
2964 }
2965
2966 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2967 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2968 }
2969
2970
2971 void LIR_Assembler::align_backward_branch_target() {
2972 __ align(BytesPerWord);
2973 }
2974
2975
2976 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2977 if (left->is_single_cpu()) {
2978 __ negl(left->as_register());
2979 move_regs(left->as_register(), dest->as_register());
2980
2981 } else if (left->is_double_cpu()) {
2982 Register lo = left->as_register_lo();
2983 Register dst = dest->as_register_lo();
2984 __ movptr(dst, lo);
2985 __ negptr(dst);
3131 }
3132
3133 void LIR_Assembler::membar_loadstore() {
3134 // no-op
3135 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3136 }
3137
3138 void LIR_Assembler::membar_storeload() {
3139 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3140 }
3141
3142 void LIR_Assembler::on_spin_wait() {
3143 __ pause ();
3144 }
3145
3146 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3147 assert(result_reg->is_register(), "check");
3148 __ mov(result_reg->as_register(), r15_thread);
3149 }
3150
3151
3152 void LIR_Assembler::peephole(LIR_List*) {
3153 // do nothing for now
3154 }
3155
3156 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3157 assert(data == dest, "xchg/xadd uses only 2 operands");
3158
3159 if (data->type() == T_INT) {
3160 if (code == lir_xadd) {
3161 __ lock();
3162 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3163 } else {
3164 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3165 }
3166 } else if (data->is_oop()) {
3167 assert (code == lir_xchg, "xadd for oops");
3168 Register obj = data->as_register();
3169 if (UseCompressedOops) {
3170 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/threadIdentifier.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #include "vmreg_x86.inline.hpp"
51
52
53 // These masks are used to provide 128-bit aligned bitmasks to the XMM
54 // instructions, to allow sign-masking or sign-bit flipping. They allow
55 // fast versions of NegF/NegD and AbsF/AbsD.
56
57 // Note: 'double' and 'long long' have 32-bits alignment on x86.
58 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
59 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
60 // of 128-bits operands for SSE instructions.
61 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
62 // Store the value to a 128-bits operand.
410 // Perform needed unlocking
411 MonitorExitStub* stub = nullptr;
412 if (method()->is_synchronized()) {
413 monitor_address(0, FrameMap::rax_opr);
414 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
415 __ unlock_object(rdi, rsi, rax, *stub->entry());
416 __ bind(*stub->continuation());
417 }
418
419 if (compilation()->env()->dtrace_method_probes()) {
420 __ mov(rdi, r15_thread);
421 __ mov_metadata(rsi, method()->constant_encoding());
422 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
423 }
424
425 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
426 __ mov(rax, rbx); // Restore the exception
427 }
428
429 // remove the activation and dispatch to the unwind handler
430 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
431 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
432
433 // Emit the slow path assembly
434 if (stub != nullptr) {
435 stub->emit_code(this);
436 }
437
438 return offset;
439 }
440
441
442 int LIR_Assembler::emit_deopt_handler() {
443 // generate code for exception handler
444 address handler_base = __ start_a_stub(deopt_handler_size());
445 if (handler_base == nullptr) {
446 // not enough space left for the handler
447 bailout("deopt handler overflow");
448 return -1;
449 }
450
455
456 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457
458 int entry_offset = __ offset();
459
460 __ jmp(start);
461
462 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
463 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
464 "out of bounds read in post-call NOP check");
465 __ end_a_stub();
466
467 return entry_offset;
468 }
469
470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
471 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
472 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
473 assert(result->fpu() == 0, "result must already be on TOS");
474 }
475 if (InlineTypeReturnedAsFields) {
476 #ifndef _LP64
477 Unimplemented();
478 #endif
479 // Check if we are returning a non-null inline type and load its fields into registers
480 ciType* return_type = compilation()->method()->return_type();
481 if (return_type->is_inlinetype()) {
482 ciInlineKlass* vk = return_type->as_inline_klass();
483 if (vk->can_be_returned_as_fields()) {
484 address unpack_handler = vk->unpack_handler();
485 assert(unpack_handler != nullptr, "must be");
486 __ call(RuntimeAddress(unpack_handler));
487 }
488 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
489 Label skip;
490 Label not_null;
491 __ testptr(rax, rax);
492 __ jcc(Assembler::notZero, not_null);
493 // Returned value is null, zero all return registers because they may belong to oop fields
494 __ xorq(j_rarg1, j_rarg1);
495 __ xorq(j_rarg2, j_rarg2);
496 __ xorq(j_rarg3, j_rarg3);
497 __ xorq(j_rarg4, j_rarg4);
498 __ xorq(j_rarg5, j_rarg5);
499 __ jmp(skip);
500 __ bind(not_null);
501
502 // Check if we are returning a non-null inline type and load its fields into registers
503 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
504
505 // Load fields from a buffered value with an inline class specific handler
506 __ load_klass(rdi, rax, rscratch1);
507 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
508 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
509 // Unpack handler can be null if inline type is not scalarizable in returns
510 __ testptr(rdi, rdi);
511 __ jcc(Assembler::zero, skip);
512 __ call(rdi);
513
514 __ bind(skip);
515 }
516 // At this point, rax points to the value object (for interpreter or C1 caller).
517 // The fields of the object are copied into registers (for C2 caller).
518 }
519
520 // Pop the stack before the safepoint code
521 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
522
523 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
524 __ reserved_stack_check();
525 }
526
527 // Note: we do not need to round double result; float result has the right precision
528 // the poll sets the condition code, but no data registers
529
530 code_stub->set_safepoint_offset(__ offset());
531 __ relocate(relocInfo::poll_return_type);
532 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
533 __ ret(0);
534 }
535
536
537 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
538 return (__ store_inline_type_fields_to_buf(vk, false));
539 }
540
541 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
542 guarantee(info != nullptr, "Shouldn't be null");
543 int offset = __ offset();
544 const Register poll_addr = rscratch1;
545 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
546 add_debug_info_for_branch(info);
547 __ relocate(relocInfo::poll_type);
548 address pre_pc = __ pc();
549 __ testl(rax, Address(poll_addr, 0));
550 address post_pc = __ pc();
551 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
552 return offset;
553 }
554
555
556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
557 if (from_reg != to_reg) __ mov(to_reg, from_reg);
558 }
559
560 void LIR_Assembler::swap_reg(Register a, Register b) {
1283 // init_state needs acquire, but x86 is TSO, and so we are already good.
1284 __ cmpb(Address(op->klass()->as_register(),
1285 InstanceKlass::init_state_offset()),
1286 InstanceKlass::fully_initialized);
1287 __ jcc(Assembler::notEqual, *op->stub()->entry());
1288 }
1289 __ allocate_object(op->obj()->as_register(),
1290 op->tmp1()->as_register(),
1291 op->tmp2()->as_register(),
1292 op->header_size(),
1293 op->object_size(),
1294 op->klass()->as_register(),
1295 *op->stub()->entry());
1296 __ bind(*op->stub()->continuation());
1297 }
1298
1299 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1300 Register len = op->len()->as_register();
1301 __ movslq(len, len);
1302
1303 if (UseSlowPath || op->always_slow_path() ||
1304 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1305 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1306 __ jmp(*op->stub()->entry());
1307 } else {
1308 Register tmp1 = op->tmp1()->as_register();
1309 Register tmp2 = op->tmp2()->as_register();
1310 Register tmp3 = op->tmp3()->as_register();
1311 if (len == tmp1) {
1312 tmp1 = tmp3;
1313 } else if (len == tmp2) {
1314 tmp2 = tmp3;
1315 } else if (len == tmp3) {
1316 // everything is ok
1317 } else {
1318 __ mov(tmp3, len);
1319 }
1320 __ allocate_array(op->obj()->as_register(),
1321 len,
1322 tmp1,
1323 tmp2,
1358 int bci = op->profiled_bci();
1359 md = method->method_data_or_null();
1360 assert(md != nullptr, "Sanity");
1361 data = md->bci_to_data(bci);
1362 assert(data != nullptr, "need data for type check");
1363 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1364 }
1365 Label* success_target = success;
1366 Label* failure_target = failure;
1367
1368 if (obj == k_RInfo) {
1369 k_RInfo = dst;
1370 } else if (obj == klass_RInfo) {
1371 klass_RInfo = dst;
1372 }
1373 Rtmp1 = op->tmp3()->as_register();
1374 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1375
1376 assert_different_registers(obj, k_RInfo, klass_RInfo);
1377
1378 if (op->need_null_check()) {
1379 __ testptr(obj, obj);
1380 if (op->should_profile()) {
1381 Label not_null;
1382 Register mdo = klass_RInfo;
1383 __ mov_metadata(mdo, md->constant_encoding());
1384 __ jccb(Assembler::notEqual, not_null);
1385 // Object is null; update MDO and exit
1386 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1387 int header_bits = BitData::null_seen_byte_constant();
1388 __ orb(data_addr, header_bits);
1389 __ jmp(*obj_is_null);
1390 __ bind(not_null);
1391
1392 Register recv = k_RInfo;
1393 __ load_klass(recv, obj, tmp_load_klass);
1394 type_profile_helper(mdo, md, data, recv);
1395 } else {
1396 __ jcc(Assembler::equal, *obj_is_null);
1397 }
1398 }
1399
1400 if (!k->is_loaded()) {
1401 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1402 } else {
1403 __ mov_metadata(k_RInfo, k->constant_encoding());
1404 }
1405 __ verify_oop(obj);
1406
1407 if (op->fast_check()) {
1408 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1409 // get object class
1410 // not a safepoint as obj null check happens earlier
1411 __ load_klass(Rtmp1, obj, tmp_load_klass);
1412 __ cmpptr(k_RInfo, Rtmp1);
1413 __ jcc(Assembler::notEqual, *failure_target);
1414 // successful cast, fall through to profile or jump
1415 } else {
1416 // get object class
1417 // not a safepoint as obj null check happens earlier
1418 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1419 if (k->is_loaded()) {
1420 // See if we get an immediate positive hit
1421 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1422 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1423 __ jcc(Assembler::notEqual, *failure_target);
1424 // successful cast, fall through to profile or jump
1425 } else {
1426 // See if we get an immediate positive hit
1427 __ jcc(Assembler::equal, *success_target);
1428 // check for self
1429 if (k->is_loaded() && k->is_obj_array_klass()) {
1430 // For a direct pointer comparison, we need the refined array klass pointer
1431 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1432 if (!k_refined->is_loaded()) {
1433 bailout("encountered unloaded_ciobjarrayklass due to out of memory error");
1434 return;
1435 }
1436 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1437 __ cmpptr(klass_RInfo, tmp_load_klass);
1438 } else {
1439 __ cmpptr(klass_RInfo, k_RInfo);
1440 }
1441 __ jcc(Assembler::equal, *success_target);
1442
1443 __ push_ppx(klass_RInfo);
1444 __ push_ppx(k_RInfo);
1445 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1446 __ pop_ppx(klass_RInfo);
1447 __ pop_ppx(klass_RInfo);
1448 // result is a boolean
1449 __ testl(klass_RInfo, klass_RInfo);
1450 __ jcc(Assembler::equal, *failure_target);
1451 // successful cast, fall through to profile or jump
1452 }
1453 } else {
1454 // perform the fast part of the checking logic
1455 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1456 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1457 __ push_ppx(klass_RInfo);
1458 __ push_ppx(k_RInfo);
1459 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1460 __ pop_ppx(klass_RInfo);
1550 __ mov(dst, obj);
1551 }
1552 } else
1553 if (code == lir_instanceof) {
1554 Register obj = op->object()->as_register();
1555 Register dst = op->result_opr()->as_register();
1556 Label success, failure, done;
1557 emit_typecheck_helper(op, &success, &failure, &failure);
1558 __ bind(failure);
1559 __ xorptr(dst, dst);
1560 __ jmpb(done);
1561 __ bind(success);
1562 __ movptr(dst, 1);
1563 __ bind(done);
1564 } else {
1565 ShouldNotReachHere();
1566 }
1567
1568 }
1569
1570 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1571 // We are loading/storing from/to an array that *may* be a flat array (the
1572 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1573 // If this array is a flat array, take the slow path.
1574 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1575 }
1576
1577 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1578 // We are storing into an array that *may* be null-free (the declared type is
1579 // Object[], abstract[], interface[] or VT.ref[]).
1580 Label test_mark_word;
1581 Register tmp = op->tmp()->as_register();
1582 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1583 __ testl(tmp, markWord::unlocked_value);
1584 __ jccb(Assembler::notZero, test_mark_word);
1585 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1586 __ bind(test_mark_word);
1587 __ testl(tmp, markWord::null_free_array_bit_in_place);
1588 }
1589
1590 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1591 Label L_oops_equal;
1592 Label L_oops_not_equal;
1593 Label L_end;
1594
1595 Register left = op->left()->as_register();
1596 Register right = op->right()->as_register();
1597
1598 __ cmpptr(left, right);
1599 __ jcc(Assembler::equal, L_oops_equal);
1600
1601 // (1) Null check -- if one of the operands is null, the other must not be null (because
1602 // the two references are not equal), so they are not substitutable,
1603 __ testptr(left, left);
1604 __ jcc(Assembler::zero, L_oops_not_equal);
1605 __ testptr(right, right);
1606 __ jcc(Assembler::zero, L_oops_not_equal);
1607
1608 ciKlass* left_klass = op->left_klass();
1609 ciKlass* right_klass = op->right_klass();
1610
1611 // (2) Inline type check -- if either of the operands is not an inline type,
1612 // they are not substitutable. We do this only if we are not sure that the
1613 // operands are inline type
1614 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1615 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1616 Register tmp = op->tmp1()->as_register();
1617 __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1618 __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1619 __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1620 __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1621 __ jcc(Assembler::notEqual, L_oops_not_equal);
1622 }
1623
1624 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1625 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1626 // No need to load klass -- the operands are statically known to be the same inline klass.
1627 __ jmp(*op->stub()->entry());
1628 } else {
1629 Register tmp1 = op->tmp1()->as_register();
1630 Register tmp2 = op->tmp2()->as_register();
1631 if (left == right) { // same operand, so clearly the same klasses, let's save the check
1632 __ jmp (*op->stub()->entry()); // -> do slow check
1633 } else {
1634 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1635 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1636 }
1637 // fall through to L_oops_not_equal
1638 }
1639
1640 __ bind(L_oops_not_equal);
1641 move(op->not_equal_result(), op->result_opr());
1642 __ jmp(L_end);
1643
1644 // We've returned from the stub. RAX contains 0x0 IFF the two
1645 // operands are not substitutable. (Don't compare against 0x1 in case the
1646 // C compiler is naughty)
1647 __ bind(*op->stub()->continuation());
1648 __ cmpl(rax, 0);
1649 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1650
1651 __ bind(L_oops_equal);
1652 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1653 // fall-through
1654 __ bind(L_end);
1655 }
1656
1657 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1658 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1659 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1660 Register newval = op->new_value()->as_register();
1661 Register cmpval = op->cmp_value()->as_register();
1662 assert(cmpval == rax, "wrong register");
1663 assert(newval != noreg, "new val must be register");
1664 assert(cmpval != newval, "cmp and new values must be in different registers");
1665 assert(cmpval != addr, "cmp and addr must be in different registers");
1666 assert(newval != addr, "new value and addr must be in different registers");
1667
1668 if (op->code() == lir_cas_obj) {
1669 if (UseCompressedOops) {
1670 __ encode_heap_oop(cmpval);
1671 __ mov(rscratch1, newval);
1672 __ encode_heap_oop(rscratch1);
1673 __ lock();
1674 // cmpval (rax) is implicitly used by this instruction
1675 __ cmpxchgl(rscratch1, Address(addr, 0));
1681 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1682 __ lock();
1683 __ cmpxchgl(newval, Address(addr, 0));
1684 }
1685 } else if (op->code() == lir_cas_long) {
1686 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1687 Register newval = op->new_value()->as_register_lo();
1688 Register cmpval = op->cmp_value()->as_register_lo();
1689 assert(cmpval == rax, "wrong register");
1690 assert(newval != noreg, "new val must be register");
1691 assert(cmpval != newval, "cmp and new values must be in different registers");
1692 assert(cmpval != addr, "cmp and addr must be in different registers");
1693 assert(newval != addr, "new value and addr must be in different registers");
1694 __ lock();
1695 __ cmpxchgq(newval, Address(addr, 0));
1696 } else {
1697 Unimplemented();
1698 }
1699 }
1700
1701 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1702 assert(dst->is_cpu_register(), "must be");
1703 assert(dst->type() == src->type(), "must be");
1704
1705 if (src->is_cpu_register()) {
1706 reg2reg(src, dst);
1707 } else if (src->is_stack()) {
1708 stack2reg(src, dst, dst->type());
1709 } else if (src->is_constant()) {
1710 const2reg(src, dst, lir_patch_none, nullptr);
1711 } else {
1712 ShouldNotReachHere();
1713 }
1714 }
1715
1716 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1717 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1718 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1719
1720 Assembler::Condition acond, ncond;
1721 switch (condition) {
1722 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1723 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1724 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1725 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1726 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1727 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1728 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1729 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1730 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1731 ShouldNotReachHere();
1732 }
1733
1734 if (opr1->is_cpu_register()) {
1735 reg2reg(opr1, result);
2306 int offset = __ offset();
2307 switch (code) {
2308 case lir_static_call:
2309 case lir_optvirtual_call:
2310 case lir_dynamic_call:
2311 offset += NativeCall::displacement_offset;
2312 break;
2313 case lir_icvirtual_call:
2314 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2315 break;
2316 default: ShouldNotReachHere();
2317 }
2318 __ align(BytesPerWord, offset);
2319 }
2320
2321
2322 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2323 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2324 "must be aligned");
2325 __ call(AddressLiteral(op->addr(), rtype));
2326 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2327 __ post_call_nop();
2328 }
2329
2330
2331 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2332 __ ic_call(op->addr());
2333 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2334 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2335 "must be aligned");
2336 __ post_call_nop();
2337 }
2338
2339
2340 void LIR_Assembler::emit_static_call_stub() {
2341 address call_pc = __ pc();
2342 address stub = __ start_a_stub(call_stub_size());
2343 if (stub == nullptr) {
2344 bailout("static call stub overflow");
2345 return;
2346 }
2347
2348 int start = __ offset();
2349
2350 // make sure that the displacement word of the call ends up word aligned
2351 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2352 __ relocate(static_stub_Relocation::spec(call_pc));
2353 __ mov_metadata(rbx, (Metadata*)nullptr);
2480 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2481 }
2482
2483
2484 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2485 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2486 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2487 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2488 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2489 }
2490
2491
2492 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2493 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2494 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2495 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2496 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2497 }
2498
2499
2500 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2501 if (null_check) {
2502 __ testptr(obj, obj);
2503 __ jcc(Assembler::zero, *slow_path->entry());
2504 }
2505 if (is_dest) {
2506 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2507 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2508 } else {
2509 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2510 }
2511 }
2512
2513
2514 // This code replaces a call to arraycopy; no exception may
2515 // be thrown in this code, they must be thrown in the System.arraycopy
2516 // activation frame; we could save some checks if this would not be the case
2517 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2518 ciArrayKlass* default_type = op->expected_type();
2519 Register src = op->src()->as_register();
2520 Register dst = op->dst()->as_register();
2521 Register src_pos = op->src_pos()->as_register();
2522 Register dst_pos = op->dst_pos()->as_register();
2523 Register length = op->length()->as_register();
2524 Register tmp = op->tmp()->as_register();
2525 Register tmp_load_klass = rscratch1;
2526 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2527
2528 CodeStub* stub = op->stub();
2529 int flags = op->flags();
2530 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2531 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2532
2533 if (flags & LIR_OpArrayCopy::always_slow_path) {
2534 __ jmp(*stub->entry());
2535 __ bind(*stub->continuation());
2536 return;
2537 }
2538
2539 // if we don't know anything, just go through the generic arraycopy
2540 if (default_type == nullptr) {
2541 // save outgoing arguments on stack in case call to System.arraycopy is needed
2542 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2543 // for interpreter calling conventions. Now we have to do it in new style conventions.
2544 // For the moment until C1 gets the new register allocator I just force all the
2545 // args to the right place (except the register args) and then on the back side
2546 // reload the register args properly if we go slow path. Yuck
2547
2548 // These are proper for the calling convention
2549 store_parameter(length, 2);
2550 store_parameter(dst_pos, 1);
2551 store_parameter(dst, 0);
2552
2553 // these are just temporary placements until we need to reload
2554 store_parameter(src_pos, 3);
2555 store_parameter(src, 4);
2556
2557 address copyfunc_addr = StubRoutines::generic_arraycopy();
2558 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2595 __ mov(tmp, rax);
2596 __ xorl(tmp, -1);
2597
2598 // Reload values from the stack so they are where the stub
2599 // expects them.
2600 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2601 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2602 __ movptr (length, Address(rsp, 2*BytesPerWord));
2603 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2604 __ movptr (src, Address(rsp, 4*BytesPerWord));
2605
2606 __ subl(length, tmp);
2607 __ addl(src_pos, tmp);
2608 __ addl(dst_pos, tmp);
2609 __ jmp(*stub->entry());
2610
2611 __ bind(*stub->continuation());
2612 return;
2613 }
2614
2615 // Handle inline type arrays
2616 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2617 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2618 }
2619 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2620 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2621 }
2622
2623 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2624
2625 int elem_size = type2aelembytes(basic_type);
2626 Address::ScaleFactor scale;
2627
2628 switch (elem_size) {
2629 case 1 :
2630 scale = Address::times_1;
2631 break;
2632 case 2 :
2633 scale = Address::times_2;
2634 break;
2635 case 4 :
2636 scale = Address::times_4;
2637 break;
2638 case 8 :
2639 scale = Address::times_8;
2640 break;
2641 default:
2642 scale = Address::no_scale;
3140 // first time here. Set profile type.
3141 __ movptr(mdo_addr, tmp);
3142 #ifdef ASSERT
3143 __ andptr(tmp, TypeEntries::type_klass_mask);
3144 __ verify_klass_ptr(tmp);
3145 #endif
3146 } else {
3147 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3148 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3149
3150 __ testptr(mdo_addr, TypeEntries::type_unknown);
3151 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3152
3153 __ orptr(mdo_addr, TypeEntries::type_unknown);
3154 }
3155 }
3156 }
3157 __ bind(next);
3158 }
3159
3160 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3161 Register obj = op->obj()->as_register();
3162 Register tmp = op->tmp()->as_pointer_register();
3163 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3164 bool not_null = op->not_null();
3165 int flag = op->flag();
3166
3167 Label not_inline_type;
3168 if (!not_null) {
3169 __ testptr(obj, obj);
3170 __ jccb(Assembler::zero, not_inline_type);
3171 }
3172
3173 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3174
3175 __ orb(mdo_addr, flag);
3176
3177 __ bind(not_inline_type);
3178 }
3179
3180
3181 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3182 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3183 }
3184
3185
3186 void LIR_Assembler::align_backward_branch_target() {
3187 __ align(BytesPerWord);
3188 }
3189
3190
3191 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3192 if (left->is_single_cpu()) {
3193 __ negl(left->as_register());
3194 move_regs(left->as_register(), dest->as_register());
3195
3196 } else if (left->is_double_cpu()) {
3197 Register lo = left->as_register_lo();
3198 Register dst = dest->as_register_lo();
3199 __ movptr(dst, lo);
3200 __ negptr(dst);
3346 }
3347
3348 void LIR_Assembler::membar_loadstore() {
3349 // no-op
3350 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3351 }
3352
3353 void LIR_Assembler::membar_storeload() {
3354 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3355 }
3356
3357 void LIR_Assembler::on_spin_wait() {
3358 __ pause ();
3359 }
3360
3361 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3362 assert(result_reg->is_register(), "check");
3363 __ mov(result_reg->as_register(), r15_thread);
3364 }
3365
3366 void LIR_Assembler::check_orig_pc() {
3367 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3368 }
3369
3370 void LIR_Assembler::peephole(LIR_List*) {
3371 // do nothing for now
3372 }
3373
3374 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3375 assert(data == dest, "xchg/xadd uses only 2 operands");
3376
3377 if (data->type() == T_INT) {
3378 if (code == lir_xadd) {
3379 __ lock();
3380 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3381 } else {
3382 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3383 }
3384 } else if (data->is_oop()) {
3385 assert (code == lir_xchg, "xadd for oops");
3386 Register obj = data->as_register();
3387 if (UseCompressedOops) {
3388 __ encode_heap_oop(obj);
|