14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "code/aotCodeCache.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "runtime/threadIdentifier.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
407 // Perform needed unlocking
408 MonitorExitStub* stub = nullptr;
409 if (method()->is_synchronized()) {
410 monitor_address(0, FrameMap::rax_opr);
411 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
412 __ unlock_object(rdi, rsi, rax, *stub->entry());
413 __ bind(*stub->continuation());
414 }
415
416 if (compilation()->env()->dtrace_method_probes()) {
417 __ mov(rdi, r15_thread);
418 __ mov_metadata(rsi, method()->constant_encoding());
419 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
420 }
421
422 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
423 __ mov(rax, rbx); // Restore the exception
424 }
425
426 // remove the activation and dispatch to the unwind handler
427 __ remove_frame(initial_frame_size_in_bytes());
428 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
429
430 // Emit the slow path assembly
431 if (stub != nullptr) {
432 stub->emit_code(this);
433 }
434
435 return offset;
436 }
437
438
439 int LIR_Assembler::emit_deopt_handler() {
440 // generate code for exception handler
441 address handler_base = __ start_a_stub(deopt_handler_size());
442 if (handler_base == nullptr) {
443 // not enough space left for the handler
444 bailout("deopt handler overflow");
445 return -1;
446 }
447
452
453 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
454
455 int entry_offset = __ offset();
456
457 __ jmp(start);
458
459 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
460 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
461 "out of bounds read in post-call NOP check");
462 __ end_a_stub();
463
464 return entry_offset;
465 }
466
467 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
468 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
469 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
470 assert(result->fpu() == 0, "result must already be on TOS");
471 }
472
473 // Pop the stack before the safepoint code
474 __ remove_frame(initial_frame_size_in_bytes());
475
476 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
477 __ reserved_stack_check();
478 }
479
480 // Note: we do not need to round double result; float result has the right precision
481 // the poll sets the condition code, but no data registers
482
483 code_stub->set_safepoint_offset(__ offset());
484 __ relocate(relocInfo::poll_return_type);
485 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
486 __ ret(0);
487 }
488
489
490 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
491 guarantee(info != nullptr, "Shouldn't be null");
492 int offset = __ offset();
493 const Register poll_addr = rscratch1;
494 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
495 add_debug_info_for_branch(info);
496 __ relocate(relocInfo::poll_type);
497 address pre_pc = __ pc();
498 __ testl(rax, Address(poll_addr, 0));
499 address post_pc = __ pc();
500 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
501 return offset;
502 }
503
504
505 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
506 if (from_reg != to_reg) __ mov(to_reg, from_reg);
507 }
508
509 void LIR_Assembler::swap_reg(Register a, Register b) {
1230 // init_state needs acquire, but x86 is TSO, and so we are already good.
1231 __ cmpb(Address(op->klass()->as_register(),
1232 InstanceKlass::init_state_offset()),
1233 InstanceKlass::fully_initialized);
1234 __ jcc(Assembler::notEqual, *op->stub()->entry());
1235 }
1236 __ allocate_object(op->obj()->as_register(),
1237 op->tmp1()->as_register(),
1238 op->tmp2()->as_register(),
1239 op->header_size(),
1240 op->object_size(),
1241 op->klass()->as_register(),
1242 *op->stub()->entry());
1243 __ bind(*op->stub()->continuation());
1244 }
1245
1246 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1247 Register len = op->len()->as_register();
1248 __ movslq(len, len);
1249
1250 if (UseSlowPath ||
1251 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1252 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1253 __ jmp(*op->stub()->entry());
1254 } else {
1255 Register tmp1 = op->tmp1()->as_register();
1256 Register tmp2 = op->tmp2()->as_register();
1257 Register tmp3 = op->tmp3()->as_register();
1258 if (len == tmp1) {
1259 tmp1 = tmp3;
1260 } else if (len == tmp2) {
1261 tmp2 = tmp3;
1262 } else if (len == tmp3) {
1263 // everything is ok
1264 } else {
1265 __ mov(tmp3, len);
1266 }
1267 __ allocate_array(op->obj()->as_register(),
1268 len,
1269 tmp1,
1270 tmp2,
1305 int bci = op->profiled_bci();
1306 md = method->method_data_or_null();
1307 assert(md != nullptr, "Sanity");
1308 data = md->bci_to_data(bci);
1309 assert(data != nullptr, "need data for type check");
1310 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1311 }
1312 Label* success_target = success;
1313 Label* failure_target = failure;
1314
1315 if (obj == k_RInfo) {
1316 k_RInfo = dst;
1317 } else if (obj == klass_RInfo) {
1318 klass_RInfo = dst;
1319 }
1320 Rtmp1 = op->tmp3()->as_register();
1321 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1322
1323 assert_different_registers(obj, k_RInfo, klass_RInfo);
1324
1325 __ testptr(obj, obj);
1326 if (op->should_profile()) {
1327 Label not_null;
1328 Register mdo = klass_RInfo;
1329 __ mov_metadata(mdo, md->constant_encoding());
1330 __ jccb(Assembler::notEqual, not_null);
1331 // Object is null; update MDO and exit
1332 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1333 int header_bits = BitData::null_seen_byte_constant();
1334 __ orb(data_addr, header_bits);
1335 __ jmp(*obj_is_null);
1336 __ bind(not_null);
1337
1338 Register recv = k_RInfo;
1339 __ load_klass(recv, obj, tmp_load_klass);
1340 type_profile_helper(mdo, md, data, recv);
1341 } else {
1342 __ jcc(Assembler::equal, *obj_is_null);
1343 }
1344
1345 if (!k->is_loaded()) {
1346 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1347 } else {
1348 __ mov_metadata(k_RInfo, k->constant_encoding());
1349 }
1350 __ verify_oop(obj);
1351
1352 if (op->fast_check()) {
1353 // get object class
1354 // not a safepoint as obj null check happens earlier
1355 __ load_klass(Rtmp1, obj, tmp_load_klass);
1356 __ cmpptr(k_RInfo, Rtmp1);
1357 __ jcc(Assembler::notEqual, *failure_target);
1358 // successful cast, fall through to profile or jump
1359 } else {
1360 // get object class
1361 // not a safepoint as obj null check happens earlier
1362 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1363 if (k->is_loaded()) {
1364 // See if we get an immediate positive hit
1365 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1366 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1367 __ jcc(Assembler::notEqual, *failure_target);
1368 // successful cast, fall through to profile or jump
1369 } else {
1370 // See if we get an immediate positive hit
1371 __ jcc(Assembler::equal, *success_target);
1372 // check for self
1373 __ cmpptr(klass_RInfo, k_RInfo);
1374 __ jcc(Assembler::equal, *success_target);
1375
1376 __ push_ppx(klass_RInfo);
1377 __ push_ppx(k_RInfo);
1378 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1379 __ pop_ppx(klass_RInfo);
1380 __ pop_ppx(klass_RInfo);
1381 // result is a boolean
1382 __ testl(klass_RInfo, klass_RInfo);
1383 __ jcc(Assembler::equal, *failure_target);
1384 // successful cast, fall through to profile or jump
1385 }
1386 } else {
1387 // perform the fast part of the checking logic
1388 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1389 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1390 __ push_ppx(klass_RInfo);
1391 __ push_ppx(k_RInfo);
1392 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1393 __ pop_ppx(klass_RInfo);
1483 __ mov(dst, obj);
1484 }
1485 } else
1486 if (code == lir_instanceof) {
1487 Register obj = op->object()->as_register();
1488 Register dst = op->result_opr()->as_register();
1489 Label success, failure, done;
1490 emit_typecheck_helper(op, &success, &failure, &failure);
1491 __ bind(failure);
1492 __ xorptr(dst, dst);
1493 __ jmpb(done);
1494 __ bind(success);
1495 __ movptr(dst, 1);
1496 __ bind(done);
1497 } else {
1498 ShouldNotReachHere();
1499 }
1500
1501 }
1502
1503
1504 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1505 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1506 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1507 Register newval = op->new_value()->as_register();
1508 Register cmpval = op->cmp_value()->as_register();
1509 assert(cmpval == rax, "wrong register");
1510 assert(newval != noreg, "new val must be register");
1511 assert(cmpval != newval, "cmp and new values must be in different registers");
1512 assert(cmpval != addr, "cmp and addr must be in different registers");
1513 assert(newval != addr, "new value and addr must be in different registers");
1514
1515 if (op->code() == lir_cas_obj) {
1516 if (UseCompressedOops) {
1517 __ encode_heap_oop(cmpval);
1518 __ mov(rscratch1, newval);
1519 __ encode_heap_oop(rscratch1);
1520 __ lock();
1521 // cmpval (rax) is implicitly used by this instruction
1522 __ cmpxchgl(rscratch1, Address(addr, 0));
1528 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1529 __ lock();
1530 __ cmpxchgl(newval, Address(addr, 0));
1531 }
1532 } else if (op->code() == lir_cas_long) {
1533 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1534 Register newval = op->new_value()->as_register_lo();
1535 Register cmpval = op->cmp_value()->as_register_lo();
1536 assert(cmpval == rax, "wrong register");
1537 assert(newval != noreg, "new val must be register");
1538 assert(cmpval != newval, "cmp and new values must be in different registers");
1539 assert(cmpval != addr, "cmp and addr must be in different registers");
1540 assert(newval != addr, "new value and addr must be in different registers");
1541 __ lock();
1542 __ cmpxchgq(newval, Address(addr, 0));
1543 } else {
1544 Unimplemented();
1545 }
1546 }
1547
1548 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1549 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1550 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1551
1552 Assembler::Condition acond, ncond;
1553 switch (condition) {
1554 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1555 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1556 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1557 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1558 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1559 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1560 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1561 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1562 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1563 ShouldNotReachHere();
1564 }
1565
1566 if (opr1->is_cpu_register()) {
1567 reg2reg(opr1, result);
2138 int offset = __ offset();
2139 switch (code) {
2140 case lir_static_call:
2141 case lir_optvirtual_call:
2142 case lir_dynamic_call:
2143 offset += NativeCall::displacement_offset;
2144 break;
2145 case lir_icvirtual_call:
2146 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2147 break;
2148 default: ShouldNotReachHere();
2149 }
2150 __ align(BytesPerWord, offset);
2151 }
2152
2153
2154 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2155 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2156 "must be aligned");
2157 __ call(AddressLiteral(op->addr(), rtype));
2158 add_call_info(code_offset(), op->info());
2159 __ post_call_nop();
2160 }
2161
2162
2163 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2164 __ ic_call(op->addr());
2165 add_call_info(code_offset(), op->info());
2166 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2167 "must be aligned");
2168 __ post_call_nop();
2169 }
2170
2171
2172 void LIR_Assembler::emit_static_call_stub() {
2173 address call_pc = __ pc();
2174 address stub = __ start_a_stub(call_stub_size());
2175 if (stub == nullptr) {
2176 bailout("static call stub overflow");
2177 return;
2178 }
2179
2180 int start = __ offset();
2181
2182 // make sure that the displacement word of the call ends up word aligned
2183 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2184 __ relocate(static_stub_Relocation::spec(call_pc));
2185 __ mov_metadata(rbx, (Metadata*)nullptr);
2312 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2313 }
2314
2315
2316 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2317 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2318 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2319 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2320 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2321 }
2322
2323
2324 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2325 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2326 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2327 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2328 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2329 }
2330
2331
2332 // This code replaces a call to arraycopy; no exception may
2333 // be thrown in this code, they must be thrown in the System.arraycopy
2334 // activation frame; we could save some checks if this would not be the case
2335 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2336 ciArrayKlass* default_type = op->expected_type();
2337 Register src = op->src()->as_register();
2338 Register dst = op->dst()->as_register();
2339 Register src_pos = op->src_pos()->as_register();
2340 Register dst_pos = op->dst_pos()->as_register();
2341 Register length = op->length()->as_register();
2342 Register tmp = op->tmp()->as_register();
2343 Register tmp_load_klass = rscratch1;
2344 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2345
2346 CodeStub* stub = op->stub();
2347 int flags = op->flags();
2348 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2349 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2350
2351 // if we don't know anything, just go through the generic arraycopy
2352 if (default_type == nullptr) {
2353 // save outgoing arguments on stack in case call to System.arraycopy is needed
2354 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2355 // for interpreter calling conventions. Now we have to do it in new style conventions.
2356 // For the moment until C1 gets the new register allocator I just force all the
2357 // args to the right place (except the register args) and then on the back side
2358 // reload the register args properly if we go slow path. Yuck
2359
2360 // These are proper for the calling convention
2361 store_parameter(length, 2);
2362 store_parameter(dst_pos, 1);
2363 store_parameter(dst, 0);
2364
2365 // these are just temporary placements until we need to reload
2366 store_parameter(src_pos, 3);
2367 store_parameter(src, 4);
2368
2369 address copyfunc_addr = StubRoutines::generic_arraycopy();
2370 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2407 __ mov(tmp, rax);
2408 __ xorl(tmp, -1);
2409
2410 // Reload values from the stack so they are where the stub
2411 // expects them.
2412 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2413 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2414 __ movptr (length, Address(rsp, 2*BytesPerWord));
2415 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2416 __ movptr (src, Address(rsp, 4*BytesPerWord));
2417
2418 __ subl(length, tmp);
2419 __ addl(src_pos, tmp);
2420 __ addl(dst_pos, tmp);
2421 __ jmp(*stub->entry());
2422
2423 __ bind(*stub->continuation());
2424 return;
2425 }
2426
2427 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2428
2429 int elem_size = type2aelembytes(basic_type);
2430 Address::ScaleFactor scale;
2431
2432 switch (elem_size) {
2433 case 1 :
2434 scale = Address::times_1;
2435 break;
2436 case 2 :
2437 scale = Address::times_2;
2438 break;
2439 case 4 :
2440 scale = Address::times_4;
2441 break;
2442 case 8 :
2443 scale = Address::times_8;
2444 break;
2445 default:
2446 scale = Address::no_scale;
2944 // first time here. Set profile type.
2945 __ movptr(mdo_addr, tmp);
2946 #ifdef ASSERT
2947 __ andptr(tmp, TypeEntries::type_klass_mask);
2948 __ verify_klass_ptr(tmp);
2949 #endif
2950 } else {
2951 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2952 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2953
2954 __ testptr(mdo_addr, TypeEntries::type_unknown);
2955 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2956
2957 __ orptr(mdo_addr, TypeEntries::type_unknown);
2958 }
2959 }
2960 }
2961 __ bind(next);
2962 }
2963
2964 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2965 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2966 }
2967
2968
2969 void LIR_Assembler::align_backward_branch_target() {
2970 __ align(BytesPerWord);
2971 }
2972
2973
2974 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2975 if (left->is_single_cpu()) {
2976 __ negl(left->as_register());
2977 move_regs(left->as_register(), dest->as_register());
2978
2979 } else if (left->is_double_cpu()) {
2980 Register lo = left->as_register_lo();
2981 Register dst = dest->as_register_lo();
2982 __ movptr(dst, lo);
2983 __ negptr(dst);
3129 }
3130
3131 void LIR_Assembler::membar_loadstore() {
3132 // no-op
3133 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3134 }
3135
3136 void LIR_Assembler::membar_storeload() {
3137 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3138 }
3139
3140 void LIR_Assembler::on_spin_wait() {
3141 __ pause ();
3142 }
3143
3144 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3145 assert(result_reg->is_register(), "check");
3146 __ mov(result_reg->as_register(), r15_thread);
3147 }
3148
3149
3150 void LIR_Assembler::peephole(LIR_List*) {
3151 // do nothing for now
3152 }
3153
3154 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3155 assert(data == dest, "xchg/xadd uses only 2 operands");
3156
3157 if (data->type() == T_INT) {
3158 if (code == lir_xadd) {
3159 __ lock();
3160 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3161 } else {
3162 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3163 }
3164 } else if (data->is_oop()) {
3165 assert (code == lir_xchg, "xadd for oops");
3166 Register obj = data->as_register();
3167 if (UseCompressedOops) {
3168 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/threadIdentifier.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #include "vmreg_x86.inline.hpp"
51
52
53 // These masks are used to provide 128-bit aligned bitmasks to the XMM
54 // instructions, to allow sign-masking or sign-bit flipping. They allow
55 // fast versions of NegF/NegD and AbsF/AbsD.
56
57 // Note: 'double' and 'long long' have 32-bits alignment on x86.
58 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
59 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
60 // of 128-bits operands for SSE instructions.
61 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
62 // Store the value to a 128-bits operand.
410 // Perform needed unlocking
411 MonitorExitStub* stub = nullptr;
412 if (method()->is_synchronized()) {
413 monitor_address(0, FrameMap::rax_opr);
414 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
415 __ unlock_object(rdi, rsi, rax, *stub->entry());
416 __ bind(*stub->continuation());
417 }
418
419 if (compilation()->env()->dtrace_method_probes()) {
420 __ mov(rdi, r15_thread);
421 __ mov_metadata(rsi, method()->constant_encoding());
422 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
423 }
424
425 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
426 __ mov(rax, rbx); // Restore the exception
427 }
428
429 // remove the activation and dispatch to the unwind handler
430 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
431 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
432
433 // Emit the slow path assembly
434 if (stub != nullptr) {
435 stub->emit_code(this);
436 }
437
438 return offset;
439 }
440
441
442 int LIR_Assembler::emit_deopt_handler() {
443 // generate code for exception handler
444 address handler_base = __ start_a_stub(deopt_handler_size());
445 if (handler_base == nullptr) {
446 // not enough space left for the handler
447 bailout("deopt handler overflow");
448 return -1;
449 }
450
455
456 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457
458 int entry_offset = __ offset();
459
460 __ jmp(start);
461
462 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
463 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
464 "out of bounds read in post-call NOP check");
465 __ end_a_stub();
466
467 return entry_offset;
468 }
469
470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
471 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
472 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
473 assert(result->fpu() == 0, "result must already be on TOS");
474 }
475 if (InlineTypeReturnedAsFields) {
476 #ifndef _LP64
477 Unimplemented();
478 #endif
479 // Check if we are returning an non-null inline type and load its fields into registers
480 ciType* return_type = compilation()->method()->return_type();
481 if (return_type->is_inlinetype()) {
482 ciInlineKlass* vk = return_type->as_inline_klass();
483 if (vk->can_be_returned_as_fields()) {
484 address unpack_handler = vk->unpack_handler();
485 assert(unpack_handler != nullptr, "must be");
486 __ call(RuntimeAddress(unpack_handler));
487 }
488 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
489 Label skip;
490 Label not_null;
491 __ testptr(rax, rax);
492 __ jcc(Assembler::notZero, not_null);
493 // Returned value is null, zero all return registers because they may belong to oop fields
494 __ xorq(j_rarg1, j_rarg1);
495 __ xorq(j_rarg2, j_rarg2);
496 __ xorq(j_rarg3, j_rarg3);
497 __ xorq(j_rarg4, j_rarg4);
498 __ xorq(j_rarg5, j_rarg5);
499 __ jmp(skip);
500 __ bind(not_null);
501
502 // Check if we are returning an non-null inline type and load its fields into registers
503 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
504
505 // Load fields from a buffered value with an inline class specific handler
506 __ load_klass(rdi, rax, rscratch1);
507 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
508 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
509 // Unpack handler can be null if inline type is not scalarizable in returns
510 __ testptr(rdi, rdi);
511 __ jcc(Assembler::zero, skip);
512 __ call(rdi);
513
514 __ bind(skip);
515 }
516 // At this point, rax points to the value object (for interpreter or C1 caller).
517 // The fields of the object are copied into registers (for C2 caller).
518 }
519
520 // Pop the stack before the safepoint code
521 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
522
523 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
524 __ reserved_stack_check();
525 }
526
527 // Note: we do not need to round double result; float result has the right precision
528 // the poll sets the condition code, but no data registers
529
530 code_stub->set_safepoint_offset(__ offset());
531 __ relocate(relocInfo::poll_return_type);
532 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
533 __ ret(0);
534 }
535
536
537 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
538 return (__ store_inline_type_fields_to_buf(vk, false));
539 }
540
541 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
542 guarantee(info != nullptr, "Shouldn't be null");
543 int offset = __ offset();
544 const Register poll_addr = rscratch1;
545 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
546 add_debug_info_for_branch(info);
547 __ relocate(relocInfo::poll_type);
548 address pre_pc = __ pc();
549 __ testl(rax, Address(poll_addr, 0));
550 address post_pc = __ pc();
551 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
552 return offset;
553 }
554
555
556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
557 if (from_reg != to_reg) __ mov(to_reg, from_reg);
558 }
559
560 void LIR_Assembler::swap_reg(Register a, Register b) {
1281 // init_state needs acquire, but x86 is TSO, and so we are already good.
1282 __ cmpb(Address(op->klass()->as_register(),
1283 InstanceKlass::init_state_offset()),
1284 InstanceKlass::fully_initialized);
1285 __ jcc(Assembler::notEqual, *op->stub()->entry());
1286 }
1287 __ allocate_object(op->obj()->as_register(),
1288 op->tmp1()->as_register(),
1289 op->tmp2()->as_register(),
1290 op->header_size(),
1291 op->object_size(),
1292 op->klass()->as_register(),
1293 *op->stub()->entry());
1294 __ bind(*op->stub()->continuation());
1295 }
1296
1297 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1298 Register len = op->len()->as_register();
1299 __ movslq(len, len);
1300
1301 if (UseSlowPath || op->always_slow_path() ||
1302 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1303 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1304 __ jmp(*op->stub()->entry());
1305 } else {
1306 Register tmp1 = op->tmp1()->as_register();
1307 Register tmp2 = op->tmp2()->as_register();
1308 Register tmp3 = op->tmp3()->as_register();
1309 if (len == tmp1) {
1310 tmp1 = tmp3;
1311 } else if (len == tmp2) {
1312 tmp2 = tmp3;
1313 } else if (len == tmp3) {
1314 // everything is ok
1315 } else {
1316 __ mov(tmp3, len);
1317 }
1318 __ allocate_array(op->obj()->as_register(),
1319 len,
1320 tmp1,
1321 tmp2,
1356 int bci = op->profiled_bci();
1357 md = method->method_data_or_null();
1358 assert(md != nullptr, "Sanity");
1359 data = md->bci_to_data(bci);
1360 assert(data != nullptr, "need data for type check");
1361 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1362 }
1363 Label* success_target = success;
1364 Label* failure_target = failure;
1365
1366 if (obj == k_RInfo) {
1367 k_RInfo = dst;
1368 } else if (obj == klass_RInfo) {
1369 klass_RInfo = dst;
1370 }
1371 Rtmp1 = op->tmp3()->as_register();
1372 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1373
1374 assert_different_registers(obj, k_RInfo, klass_RInfo);
1375
1376 if (op->need_null_check()) {
1377 __ testptr(obj, obj);
1378 if (op->should_profile()) {
1379 Label not_null;
1380 Register mdo = klass_RInfo;
1381 __ mov_metadata(mdo, md->constant_encoding());
1382 __ jccb(Assembler::notEqual, not_null);
1383 // Object is null; update MDO and exit
1384 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1385 int header_bits = BitData::null_seen_byte_constant();
1386 __ orb(data_addr, header_bits);
1387 __ jmp(*obj_is_null);
1388 __ bind(not_null);
1389
1390 Register recv = k_RInfo;
1391 __ load_klass(recv, obj, tmp_load_klass);
1392 type_profile_helper(mdo, md, data, recv);
1393 } else {
1394 __ jcc(Assembler::equal, *obj_is_null);
1395 }
1396 }
1397
1398 if (!k->is_loaded()) {
1399 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1400 } else {
1401 __ mov_metadata(k_RInfo, k->constant_encoding());
1402 }
1403 __ verify_oop(obj);
1404
1405 if (op->fast_check()) {
1406 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1407 // get object class
1408 // not a safepoint as obj null check happens earlier
1409 __ load_klass(Rtmp1, obj, tmp_load_klass);
1410 __ cmpptr(k_RInfo, Rtmp1);
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1417 if (k->is_loaded()) {
1418 // See if we get an immediate positive hit
1419 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1420 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1421 __ jcc(Assembler::notEqual, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 } else {
1424 // See if we get an immediate positive hit
1425 __ jcc(Assembler::equal, *success_target);
1426 // check for self
1427 if (k->is_loaded() && k->is_obj_array_klass()) {
1428 // For a direct pointer comparison, we need the refined array klass pointer
1429 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1430 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1431 __ cmpptr(klass_RInfo, tmp_load_klass);
1432 } else {
1433 __ cmpptr(klass_RInfo, k_RInfo);
1434 }
1435 __ jcc(Assembler::equal, *success_target);
1436
1437 __ push_ppx(klass_RInfo);
1438 __ push_ppx(k_RInfo);
1439 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1440 __ pop_ppx(klass_RInfo);
1441 __ pop_ppx(klass_RInfo);
1442 // result is a boolean
1443 __ testl(klass_RInfo, klass_RInfo);
1444 __ jcc(Assembler::equal, *failure_target);
1445 // successful cast, fall through to profile or jump
1446 }
1447 } else {
1448 // perform the fast part of the checking logic
1449 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1450 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1451 __ push_ppx(klass_RInfo);
1452 __ push_ppx(k_RInfo);
1453 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1454 __ pop_ppx(klass_RInfo);
1544 __ mov(dst, obj);
1545 }
1546 } else
1547 if (code == lir_instanceof) {
1548 Register obj = op->object()->as_register();
1549 Register dst = op->result_opr()->as_register();
1550 Label success, failure, done;
1551 emit_typecheck_helper(op, &success, &failure, &failure);
1552 __ bind(failure);
1553 __ xorptr(dst, dst);
1554 __ jmpb(done);
1555 __ bind(success);
1556 __ movptr(dst, 1);
1557 __ bind(done);
1558 } else {
1559 ShouldNotReachHere();
1560 }
1561
1562 }
1563
1564 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1565 // We are loading/storing from/to an array that *may* be a flat array (the
1566 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1567 // If this array is a flat array, take the slow path.
1568 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1569 if (!op->value()->is_illegal()) {
1570 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1571 // The array is not a flat array, but it might be null-free. If we are storing
1572 // a null into a null-free array, take the slow path (which will throw NPE).
1573 Label skip;
1574 __ cmpptr(op->value()->as_register(), NULL_WORD);
1575 __ jcc(Assembler::notEqual, skip);
1576 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1577 __ bind(skip);
1578 }
1579 }
1580
1581 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1582 // We are storing into an array that *may* be null-free (the declared type is
1583 // Object[], abstract[], interface[] or VT.ref[]).
1584 Label test_mark_word;
1585 Register tmp = op->tmp()->as_register();
1586 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1587 __ testl(tmp, markWord::unlocked_value);
1588 __ jccb(Assembler::notZero, test_mark_word);
1589 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1590 __ bind(test_mark_word);
1591 __ testl(tmp, markWord::null_free_array_bit_in_place);
1592 }
1593
1594 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1595 Label L_oops_equal;
1596 Label L_oops_not_equal;
1597 Label L_end;
1598
1599 Register left = op->left()->as_register();
1600 Register right = op->right()->as_register();
1601
1602 __ cmpptr(left, right);
1603 __ jcc(Assembler::equal, L_oops_equal);
1604
1605 // (1) Null check -- if one of the operands is null, the other must not be null (because
1606 // the two references are not equal), so they are not substitutable,
1607 __ testptr(left, left);
1608 __ jcc(Assembler::zero, L_oops_not_equal);
1609 __ testptr(right, right);
1610 __ jcc(Assembler::zero, L_oops_not_equal);
1611
1612 ciKlass* left_klass = op->left_klass();
1613 ciKlass* right_klass = op->right_klass();
1614
1615 // (2) Inline type check -- if either of the operands is not a inline type,
1616 // they are not substitutable. We do this only if we are not sure that the
1617 // operands are inline type
1618 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1619 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1620 Register tmp = op->tmp1()->as_register();
1621 __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1622 __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1623 __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1624 __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1625 __ jcc(Assembler::notEqual, L_oops_not_equal);
1626 }
1627
1628 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1629 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1630 // No need to load klass -- the operands are statically known to be the same inline klass.
1631 __ jmp(*op->stub()->entry());
1632 } else {
1633 Register tmp1 = op->tmp1()->as_register();
1634 Register tmp2 = op->tmp2()->as_register();
1635 if (left == right) { // same operand, so clearly the same klasses, let's save the check
1636 __ jmp (*op->stub()->entry()); // -> do slow check
1637 } else {
1638 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1639 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1640 }
1641 // fall through to L_oops_not_equal
1642 }
1643
1644 __ bind(L_oops_not_equal);
1645 move(op->not_equal_result(), op->result_opr());
1646 __ jmp(L_end);
1647
1648 __ bind(L_oops_equal);
1649 move(op->equal_result(), op->result_opr());
1650 __ jmp(L_end);
1651
1652 // We've returned from the stub. RAX contains 0x0 IFF the two
1653 // operands are not substitutable. (Don't compare against 0x1 in case the
1654 // C compiler is naughty)
1655 __ bind(*op->stub()->continuation());
1656 __ cmpl(rax, 0);
1657 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1658 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1659 // fall-through
1660 __ bind(L_end);
1661 }
1662
1663 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1664 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1665 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1666 Register newval = op->new_value()->as_register();
1667 Register cmpval = op->cmp_value()->as_register();
1668 assert(cmpval == rax, "wrong register");
1669 assert(newval != noreg, "new val must be register");
1670 assert(cmpval != newval, "cmp and new values must be in different registers");
1671 assert(cmpval != addr, "cmp and addr must be in different registers");
1672 assert(newval != addr, "new value and addr must be in different registers");
1673
1674 if (op->code() == lir_cas_obj) {
1675 if (UseCompressedOops) {
1676 __ encode_heap_oop(cmpval);
1677 __ mov(rscratch1, newval);
1678 __ encode_heap_oop(rscratch1);
1679 __ lock();
1680 // cmpval (rax) is implicitly used by this instruction
1681 __ cmpxchgl(rscratch1, Address(addr, 0));
1687 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1688 __ lock();
1689 __ cmpxchgl(newval, Address(addr, 0));
1690 }
1691 } else if (op->code() == lir_cas_long) {
1692 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1693 Register newval = op->new_value()->as_register_lo();
1694 Register cmpval = op->cmp_value()->as_register_lo();
1695 assert(cmpval == rax, "wrong register");
1696 assert(newval != noreg, "new val must be register");
1697 assert(cmpval != newval, "cmp and new values must be in different registers");
1698 assert(cmpval != addr, "cmp and addr must be in different registers");
1699 assert(newval != addr, "new value and addr must be in different registers");
1700 __ lock();
1701 __ cmpxchgq(newval, Address(addr, 0));
1702 } else {
1703 Unimplemented();
1704 }
1705 }
1706
1707 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1708 assert(dst->is_cpu_register(), "must be");
1709 assert(dst->type() == src->type(), "must be");
1710
1711 if (src->is_cpu_register()) {
1712 reg2reg(src, dst);
1713 } else if (src->is_stack()) {
1714 stack2reg(src, dst, dst->type());
1715 } else if (src->is_constant()) {
1716 const2reg(src, dst, lir_patch_none, nullptr);
1717 } else {
1718 ShouldNotReachHere();
1719 }
1720 }
1721
1722 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1723 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1724 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1725
1726 Assembler::Condition acond, ncond;
1727 switch (condition) {
1728 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1729 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1730 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1731 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1732 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1733 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1734 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1735 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1736 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1737 ShouldNotReachHere();
1738 }
1739
1740 if (opr1->is_cpu_register()) {
1741 reg2reg(opr1, result);
2312 int offset = __ offset();
2313 switch (code) {
2314 case lir_static_call:
2315 case lir_optvirtual_call:
2316 case lir_dynamic_call:
2317 offset += NativeCall::displacement_offset;
2318 break;
2319 case lir_icvirtual_call:
2320 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2321 break;
2322 default: ShouldNotReachHere();
2323 }
2324 __ align(BytesPerWord, offset);
2325 }
2326
2327
2328 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2329 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2330 "must be aligned");
2331 __ call(AddressLiteral(op->addr(), rtype));
2332 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2333 __ post_call_nop();
2334 }
2335
2336
2337 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2338 __ ic_call(op->addr());
2339 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2340 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2341 "must be aligned");
2342 __ post_call_nop();
2343 }
2344
2345
2346 void LIR_Assembler::emit_static_call_stub() {
2347 address call_pc = __ pc();
2348 address stub = __ start_a_stub(call_stub_size());
2349 if (stub == nullptr) {
2350 bailout("static call stub overflow");
2351 return;
2352 }
2353
2354 int start = __ offset();
2355
2356 // make sure that the displacement word of the call ends up word aligned
2357 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2358 __ relocate(static_stub_Relocation::spec(call_pc));
2359 __ mov_metadata(rbx, (Metadata*)nullptr);
2486 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2487 }
2488
2489
2490 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2491 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2492 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2493 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2494 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2495 }
2496
2497
2498 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2499 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2500 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2501 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2502 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2503 }
2504
2505
2506 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2507 if (null_check) {
2508 __ testptr(obj, obj);
2509 __ jcc(Assembler::zero, *slow_path->entry());
2510 }
2511 if (is_dest) {
2512 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2513 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2514 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2515 } else {
2516 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2517 }
2518 }
2519
2520
2521 // This code replaces a call to arraycopy; no exception may
2522 // be thrown in this code, they must be thrown in the System.arraycopy
2523 // activation frame; we could save some checks if this would not be the case
2524 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2525 ciArrayKlass* default_type = op->expected_type();
2526 Register src = op->src()->as_register();
2527 Register dst = op->dst()->as_register();
2528 Register src_pos = op->src_pos()->as_register();
2529 Register dst_pos = op->dst_pos()->as_register();
2530 Register length = op->length()->as_register();
2531 Register tmp = op->tmp()->as_register();
2532 Register tmp_load_klass = rscratch1;
2533 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2534
2535 CodeStub* stub = op->stub();
2536 int flags = op->flags();
2537 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2538 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2539
2540 if (flags & LIR_OpArrayCopy::always_slow_path) {
2541 __ jmp(*stub->entry());
2542 __ bind(*stub->continuation());
2543 return;
2544 }
2545
2546 // if we don't know anything, just go through the generic arraycopy
2547 if (default_type == nullptr) {
2548 // save outgoing arguments on stack in case call to System.arraycopy is needed
2549 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2550 // for interpreter calling conventions. Now we have to do it in new style conventions.
2551 // For the moment until C1 gets the new register allocator I just force all the
2552 // args to the right place (except the register args) and then on the back side
2553 // reload the register args properly if we go slow path. Yuck
2554
2555 // These are proper for the calling convention
2556 store_parameter(length, 2);
2557 store_parameter(dst_pos, 1);
2558 store_parameter(dst, 0);
2559
2560 // these are just temporary placements until we need to reload
2561 store_parameter(src_pos, 3);
2562 store_parameter(src, 4);
2563
2564 address copyfunc_addr = StubRoutines::generic_arraycopy();
2565 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2602 __ mov(tmp, rax);
2603 __ xorl(tmp, -1);
2604
2605 // Reload values from the stack so they are where the stub
2606 // expects them.
2607 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2608 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2609 __ movptr (length, Address(rsp, 2*BytesPerWord));
2610 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2611 __ movptr (src, Address(rsp, 4*BytesPerWord));
2612
2613 __ subl(length, tmp);
2614 __ addl(src_pos, tmp);
2615 __ addl(dst_pos, tmp);
2616 __ jmp(*stub->entry());
2617
2618 __ bind(*stub->continuation());
2619 return;
2620 }
2621
2622 // Handle inline type arrays
2623 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2624 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2625 }
2626 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2627 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2628 }
2629
2630 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2631
2632 int elem_size = type2aelembytes(basic_type);
2633 Address::ScaleFactor scale;
2634
2635 switch (elem_size) {
2636 case 1 :
2637 scale = Address::times_1;
2638 break;
2639 case 2 :
2640 scale = Address::times_2;
2641 break;
2642 case 4 :
2643 scale = Address::times_4;
2644 break;
2645 case 8 :
2646 scale = Address::times_8;
2647 break;
2648 default:
2649 scale = Address::no_scale;
3147 // first time here. Set profile type.
3148 __ movptr(mdo_addr, tmp);
3149 #ifdef ASSERT
3150 __ andptr(tmp, TypeEntries::type_klass_mask);
3151 __ verify_klass_ptr(tmp);
3152 #endif
3153 } else {
3154 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3155 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3156
3157 __ testptr(mdo_addr, TypeEntries::type_unknown);
3158 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3159
3160 __ orptr(mdo_addr, TypeEntries::type_unknown);
3161 }
3162 }
3163 }
3164 __ bind(next);
3165 }
3166
3167 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3168 Register obj = op->obj()->as_register();
3169 Register tmp = op->tmp()->as_pointer_register();
3170 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3171 bool not_null = op->not_null();
3172 int flag = op->flag();
3173
3174 Label not_inline_type;
3175 if (!not_null) {
3176 __ testptr(obj, obj);
3177 __ jccb(Assembler::zero, not_inline_type);
3178 }
3179
3180 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3181
3182 __ orb(mdo_addr, flag);
3183
3184 __ bind(not_inline_type);
3185 }
3186
3187
3188 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3189 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3190 }
3191
3192
3193 void LIR_Assembler::align_backward_branch_target() {
3194 __ align(BytesPerWord);
3195 }
3196
3197
3198 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3199 if (left->is_single_cpu()) {
3200 __ negl(left->as_register());
3201 move_regs(left->as_register(), dest->as_register());
3202
3203 } else if (left->is_double_cpu()) {
3204 Register lo = left->as_register_lo();
3205 Register dst = dest->as_register_lo();
3206 __ movptr(dst, lo);
3207 __ negptr(dst);
3353 }
3354
3355 void LIR_Assembler::membar_loadstore() {
3356 // no-op
3357 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3358 }
3359
3360 void LIR_Assembler::membar_storeload() {
3361 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3362 }
3363
3364 void LIR_Assembler::on_spin_wait() {
3365 __ pause ();
3366 }
3367
3368 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3369 assert(result_reg->is_register(), "check");
3370 __ mov(result_reg->as_register(), r15_thread);
3371 }
3372
3373 void LIR_Assembler::check_orig_pc() {
3374 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3375 }
3376
3377 void LIR_Assembler::peephole(LIR_List*) {
3378 // do nothing for now
3379 }
3380
3381 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3382 assert(data == dest, "xchg/xadd uses only 2 operands");
3383
3384 if (data->type() == T_INT) {
3385 if (code == lir_xadd) {
3386 __ lock();
3387 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3388 } else {
3389 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3390 }
3391 } else if (data->is_oop()) {
3392 assert (code == lir_xchg, "xadd for oops");
3393 Register obj = data->as_register();
3394 if (UseCompressedOops) {
3395 __ encode_heap_oop(obj);
|