14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "code/aotCodeCache.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "runtime/threadIdentifier.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
407 // Perform needed unlocking
408 MonitorExitStub* stub = nullptr;
409 if (method()->is_synchronized()) {
410 monitor_address(0, FrameMap::rax_opr);
411 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
412 __ unlock_object(rdi, rsi, rax, *stub->entry());
413 __ bind(*stub->continuation());
414 }
415
416 if (compilation()->env()->dtrace_method_probes()) {
417 __ mov(rdi, r15_thread);
418 __ mov_metadata(rsi, method()->constant_encoding());
419 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
420 }
421
422 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
423 __ mov(rax, rbx); // Restore the exception
424 }
425
426 // remove the activation and dispatch to the unwind handler
427 __ remove_frame(initial_frame_size_in_bytes());
428 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
429
430 // Emit the slow path assembly
431 if (stub != nullptr) {
432 stub->emit_code(this);
433 }
434
435 return offset;
436 }
437
438
439 int LIR_Assembler::emit_deopt_handler() {
440 // generate code for exception handler
441 address handler_base = __ start_a_stub(deopt_handler_size());
442 if (handler_base == nullptr) {
443 // not enough space left for the handler
444 bailout("deopt handler overflow");
445 return -1;
446 }
447
452
453 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
454
455 int entry_offset = __ offset();
456
457 __ jmp(start);
458
459 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
460 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
461 "out of bounds read in post-call NOP check");
462 __ end_a_stub();
463
464 return entry_offset;
465 }
466
467 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
468 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
469 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
470 assert(result->fpu() == 0, "result must already be on TOS");
471 }
472
473 // Pop the stack before the safepoint code
474 __ remove_frame(initial_frame_size_in_bytes());
475
476 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
477 __ reserved_stack_check();
478 }
479
480 // Note: we do not need to round double result; float result has the right precision
481 // the poll sets the condition code, but no data registers
482
483 code_stub->set_safepoint_offset(__ offset());
484 __ relocate(relocInfo::poll_return_type);
485 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
486 __ ret(0);
487 }
488
489
490 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
491 guarantee(info != nullptr, "Shouldn't be null");
492 int offset = __ offset();
493 const Register poll_addr = rscratch1;
494 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
495 add_debug_info_for_branch(info);
496 __ relocate(relocInfo::poll_type);
497 address pre_pc = __ pc();
498 __ testl(rax, Address(poll_addr, 0));
499 address post_pc = __ pc();
500 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
501 return offset;
502 }
503
504
505 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
506 if (from_reg != to_reg) __ mov(to_reg, from_reg);
507 }
508
509 void LIR_Assembler::swap_reg(Register a, Register b) {
1230 // init_state needs acquire, but x86 is TSO, and so we are already good.
1231 __ cmpb(Address(op->klass()->as_register(),
1232 InstanceKlass::init_state_offset()),
1233 InstanceKlass::fully_initialized);
1234 __ jcc(Assembler::notEqual, *op->stub()->entry());
1235 }
1236 __ allocate_object(op->obj()->as_register(),
1237 op->tmp1()->as_register(),
1238 op->tmp2()->as_register(),
1239 op->header_size(),
1240 op->object_size(),
1241 op->klass()->as_register(),
1242 *op->stub()->entry());
1243 __ bind(*op->stub()->continuation());
1244 }
1245
1246 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1247 Register len = op->len()->as_register();
1248 __ movslq(len, len);
1249
1250 if (UseSlowPath ||
1251 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1252 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1253 __ jmp(*op->stub()->entry());
1254 } else {
1255 Register tmp1 = op->tmp1()->as_register();
1256 Register tmp2 = op->tmp2()->as_register();
1257 Register tmp3 = op->tmp3()->as_register();
1258 if (len == tmp1) {
1259 tmp1 = tmp3;
1260 } else if (len == tmp2) {
1261 tmp2 = tmp3;
1262 } else if (len == tmp3) {
1263 // everything is ok
1264 } else {
1265 __ mov(tmp3, len);
1266 }
1267 __ allocate_array(op->obj()->as_register(),
1268 len,
1269 tmp1,
1270 tmp2,
1305 int bci = op->profiled_bci();
1306 md = method->method_data_or_null();
1307 assert(md != nullptr, "Sanity");
1308 data = md->bci_to_data(bci);
1309 assert(data != nullptr, "need data for type check");
1310 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1311 }
1312 Label* success_target = success;
1313 Label* failure_target = failure;
1314
1315 if (obj == k_RInfo) {
1316 k_RInfo = dst;
1317 } else if (obj == klass_RInfo) {
1318 klass_RInfo = dst;
1319 }
1320 Rtmp1 = op->tmp3()->as_register();
1321 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1322
1323 assert_different_registers(obj, k_RInfo, klass_RInfo);
1324
1325 __ testptr(obj, obj);
1326 if (op->should_profile()) {
1327 Label not_null;
1328 Register mdo = klass_RInfo;
1329 __ mov_metadata(mdo, md->constant_encoding());
1330 __ jccb(Assembler::notEqual, not_null);
1331 // Object is null; update MDO and exit
1332 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1333 int header_bits = BitData::null_seen_byte_constant();
1334 __ orb(data_addr, header_bits);
1335 __ jmp(*obj_is_null);
1336 __ bind(not_null);
1337
1338 Register recv = k_RInfo;
1339 __ load_klass(recv, obj, tmp_load_klass);
1340 type_profile_helper(mdo, md, data, recv);
1341 } else {
1342 __ jcc(Assembler::equal, *obj_is_null);
1343 }
1344
1345 if (!k->is_loaded()) {
1346 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1347 } else {
1348 __ mov_metadata(k_RInfo, k->constant_encoding());
1349 }
1350 __ verify_oop(obj);
1351
1352 if (op->fast_check()) {
1353 // get object class
1354 // not a safepoint as obj null check happens earlier
1355 __ load_klass(Rtmp1, obj, tmp_load_klass);
1356 __ cmpptr(k_RInfo, Rtmp1);
1357 __ jcc(Assembler::notEqual, *failure_target);
1358 // successful cast, fall through to profile or jump
1359 } else {
1360 // get object class
1361 // not a safepoint as obj null check happens earlier
1362 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1363 if (k->is_loaded()) {
1364 // See if we get an immediate positive hit
1365 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1366 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1367 __ jcc(Assembler::notEqual, *failure_target);
1368 // successful cast, fall through to profile or jump
1369 } else {
1370 // See if we get an immediate positive hit
1371 __ jcc(Assembler::equal, *success_target);
1372 // check for self
1373 __ cmpptr(klass_RInfo, k_RInfo);
1374 __ jcc(Assembler::equal, *success_target);
1375
1376 __ push_ppx(klass_RInfo);
1377 __ push_ppx(k_RInfo);
1378 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1379 __ pop_ppx(klass_RInfo);
1380 __ pop_ppx(klass_RInfo);
1381 // result is a boolean
1382 __ testl(klass_RInfo, klass_RInfo);
1383 __ jcc(Assembler::equal, *failure_target);
1384 // successful cast, fall through to profile or jump
1385 }
1386 } else {
1387 // perform the fast part of the checking logic
1388 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1389 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1390 __ push_ppx(klass_RInfo);
1391 __ push_ppx(k_RInfo);
1392 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1393 __ pop_ppx(klass_RInfo);
1483 __ mov(dst, obj);
1484 }
1485 } else
1486 if (code == lir_instanceof) {
1487 Register obj = op->object()->as_register();
1488 Register dst = op->result_opr()->as_register();
1489 Label success, failure, done;
1490 emit_typecheck_helper(op, &success, &failure, &failure);
1491 __ bind(failure);
1492 __ xorptr(dst, dst);
1493 __ jmpb(done);
1494 __ bind(success);
1495 __ movptr(dst, 1);
1496 __ bind(done);
1497 } else {
1498 ShouldNotReachHere();
1499 }
1500
1501 }
1502
1503
1504 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1505 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1506 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1507 Register newval = op->new_value()->as_register();
1508 Register cmpval = op->cmp_value()->as_register();
1509 assert(cmpval == rax, "wrong register");
1510 assert(newval != noreg, "new val must be register");
1511 assert(cmpval != newval, "cmp and new values must be in different registers");
1512 assert(cmpval != addr, "cmp and addr must be in different registers");
1513 assert(newval != addr, "new value and addr must be in different registers");
1514
1515 if (op->code() == lir_cas_obj) {
1516 if (UseCompressedOops) {
1517 __ encode_heap_oop(cmpval);
1518 __ mov(rscratch1, newval);
1519 __ encode_heap_oop(rscratch1);
1520 __ lock();
1521 // cmpval (rax) is implicitly used by this instruction
1522 __ cmpxchgl(rscratch1, Address(addr, 0));
1528 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1529 __ lock();
1530 __ cmpxchgl(newval, Address(addr, 0));
1531 }
1532 } else if (op->code() == lir_cas_long) {
1533 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1534 Register newval = op->new_value()->as_register_lo();
1535 Register cmpval = op->cmp_value()->as_register_lo();
1536 assert(cmpval == rax, "wrong register");
1537 assert(newval != noreg, "new val must be register");
1538 assert(cmpval != newval, "cmp and new values must be in different registers");
1539 assert(cmpval != addr, "cmp and addr must be in different registers");
1540 assert(newval != addr, "new value and addr must be in different registers");
1541 __ lock();
1542 __ cmpxchgq(newval, Address(addr, 0));
1543 } else {
1544 Unimplemented();
1545 }
1546 }
1547
1548 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1549 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1550 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1551
1552 Assembler::Condition acond, ncond;
1553 switch (condition) {
1554 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1555 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1556 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1557 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1558 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1559 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1560 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1561 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1562 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1563 ShouldNotReachHere();
1564 }
1565
1566 if (opr1->is_cpu_register()) {
1567 reg2reg(opr1, result);
2138 int offset = __ offset();
2139 switch (code) {
2140 case lir_static_call:
2141 case lir_optvirtual_call:
2142 case lir_dynamic_call:
2143 offset += NativeCall::displacement_offset;
2144 break;
2145 case lir_icvirtual_call:
2146 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2147 break;
2148 default: ShouldNotReachHere();
2149 }
2150 __ align(BytesPerWord, offset);
2151 }
2152
2153
2154 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2155 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2156 "must be aligned");
2157 __ call(AddressLiteral(op->addr(), rtype));
2158 add_call_info(code_offset(), op->info());
2159 __ post_call_nop();
2160 }
2161
2162
2163 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2164 __ ic_call(op->addr());
2165 add_call_info(code_offset(), op->info());
2166 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2167 "must be aligned");
2168 __ post_call_nop();
2169 }
2170
2171
2172 void LIR_Assembler::emit_static_call_stub() {
2173 address call_pc = __ pc();
2174 address stub = __ start_a_stub(call_stub_size());
2175 if (stub == nullptr) {
2176 bailout("static call stub overflow");
2177 return;
2178 }
2179
2180 int start = __ offset();
2181
2182 // make sure that the displacement word of the call ends up word aligned
2183 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2184 __ relocate(static_stub_Relocation::spec(call_pc));
2185 __ mov_metadata(rbx, (Metadata*)nullptr);
2312 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2313 }
2314
2315
2316 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2317 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2318 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2319 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2320 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2321 }
2322
2323
2324 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2325 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2326 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2327 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2328 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2329 }
2330
2331
2332 // This code replaces a call to arraycopy; no exception may
2333 // be thrown in this code, they must be thrown in the System.arraycopy
2334 // activation frame; we could save some checks if this would not be the case
2335 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2336 ciArrayKlass* default_type = op->expected_type();
2337 Register src = op->src()->as_register();
2338 Register dst = op->dst()->as_register();
2339 Register src_pos = op->src_pos()->as_register();
2340 Register dst_pos = op->dst_pos()->as_register();
2341 Register length = op->length()->as_register();
2342 Register tmp = op->tmp()->as_register();
2343 Register tmp_load_klass = rscratch1;
2344 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2345
2346 CodeStub* stub = op->stub();
2347 int flags = op->flags();
2348 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2349 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2350
2351 // if we don't know anything, just go through the generic arraycopy
2352 if (default_type == nullptr) {
2353 // save outgoing arguments on stack in case call to System.arraycopy is needed
2354 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2355 // for interpreter calling conventions. Now we have to do it in new style conventions.
2356 // For the moment until C1 gets the new register allocator I just force all the
2357 // args to the right place (except the register args) and then on the back side
2358 // reload the register args properly if we go slow path. Yuck
2359
2360 // These are proper for the calling convention
2361 store_parameter(length, 2);
2362 store_parameter(dst_pos, 1);
2363 store_parameter(dst, 0);
2364
2365 // these are just temporary placements until we need to reload
2366 store_parameter(src_pos, 3);
2367 store_parameter(src, 4);
2368
2369 address copyfunc_addr = StubRoutines::generic_arraycopy();
2370 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2407 __ mov(tmp, rax);
2408 __ xorl(tmp, -1);
2409
2410 // Reload values from the stack so they are where the stub
2411 // expects them.
2412 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2413 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2414 __ movptr (length, Address(rsp, 2*BytesPerWord));
2415 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2416 __ movptr (src, Address(rsp, 4*BytesPerWord));
2417
2418 __ subl(length, tmp);
2419 __ addl(src_pos, tmp);
2420 __ addl(dst_pos, tmp);
2421 __ jmp(*stub->entry());
2422
2423 __ bind(*stub->continuation());
2424 return;
2425 }
2426
2427 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2428
2429 int elem_size = type2aelembytes(basic_type);
2430 Address::ScaleFactor scale;
2431
2432 switch (elem_size) {
2433 case 1 :
2434 scale = Address::times_1;
2435 break;
2436 case 2 :
2437 scale = Address::times_2;
2438 break;
2439 case 4 :
2440 scale = Address::times_4;
2441 break;
2442 case 8 :
2443 scale = Address::times_8;
2444 break;
2445 default:
2446 scale = Address::no_scale;
2944 // first time here. Set profile type.
2945 __ movptr(mdo_addr, tmp);
2946 #ifdef ASSERT
2947 __ andptr(tmp, TypeEntries::type_klass_mask);
2948 __ verify_klass_ptr(tmp);
2949 #endif
2950 } else {
2951 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2952 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2953
2954 __ testptr(mdo_addr, TypeEntries::type_unknown);
2955 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2956
2957 __ orptr(mdo_addr, TypeEntries::type_unknown);
2958 }
2959 }
2960 }
2961 __ bind(next);
2962 }
2963
2964 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2965 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2966 }
2967
2968
2969 void LIR_Assembler::align_backward_branch_target() {
2970 __ align(BytesPerWord);
2971 }
2972
2973
2974 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2975 if (left->is_single_cpu()) {
2976 __ negl(left->as_register());
2977 move_regs(left->as_register(), dest->as_register());
2978
2979 } else if (left->is_double_cpu()) {
2980 Register lo = left->as_register_lo();
2981 Register dst = dest->as_register_lo();
2982 __ movptr(dst, lo);
2983 __ negptr(dst);
3129 }
3130
3131 void LIR_Assembler::membar_loadstore() {
3132 // no-op
3133 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3134 }
3135
3136 void LIR_Assembler::membar_storeload() {
3137 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3138 }
3139
3140 void LIR_Assembler::on_spin_wait() {
3141 __ pause ();
3142 }
3143
3144 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3145 assert(result_reg->is_register(), "check");
3146 __ mov(result_reg->as_register(), r15_thread);
3147 }
3148
3149
3150 void LIR_Assembler::peephole(LIR_List*) {
3151 // do nothing for now
3152 }
3153
3154 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3155 assert(data == dest, "xchg/xadd uses only 2 operands");
3156
3157 if (data->type() == T_INT) {
3158 if (code == lir_xadd) {
3159 __ lock();
3160 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3161 } else {
3162 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3163 }
3164 } else if (data->is_oop()) {
3165 assert (code == lir_xchg, "xadd for oops");
3166 Register obj = data->as_register();
3167 if (UseCompressedOops) {
3168 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/threadIdentifier.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #include "vmreg_x86.inline.hpp"
51
52
53 // These masks are used to provide 128-bit aligned bitmasks to the XMM
54 // instructions, to allow sign-masking or sign-bit flipping. They allow
55 // fast versions of NegF/NegD and AbsF/AbsD.
56
57 // Note: 'double' and 'long long' have 32-bits alignment on x86.
58 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
59 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
60 // of 128-bits operands for SSE instructions.
61 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
62 // Store the value to a 128-bits operand.
410 // Perform needed unlocking
411 MonitorExitStub* stub = nullptr;
412 if (method()->is_synchronized()) {
413 monitor_address(0, FrameMap::rax_opr);
414 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
415 __ unlock_object(rdi, rsi, rax, *stub->entry());
416 __ bind(*stub->continuation());
417 }
418
419 if (compilation()->env()->dtrace_method_probes()) {
420 __ mov(rdi, r15_thread);
421 __ mov_metadata(rsi, method()->constant_encoding());
422 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
423 }
424
425 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
426 __ mov(rax, rbx); // Restore the exception
427 }
428
429 // remove the activation and dispatch to the unwind handler
430 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
431 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
432
433 // Emit the slow path assembly
434 if (stub != nullptr) {
435 stub->emit_code(this);
436 }
437
438 return offset;
439 }
440
441
442 int LIR_Assembler::emit_deopt_handler() {
443 // generate code for exception handler
444 address handler_base = __ start_a_stub(deopt_handler_size());
445 if (handler_base == nullptr) {
446 // not enough space left for the handler
447 bailout("deopt handler overflow");
448 return -1;
449 }
450
455
456 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457
458 int entry_offset = __ offset();
459
460 __ jmp(start);
461
462 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
463 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
464 "out of bounds read in post-call NOP check");
465 __ end_a_stub();
466
467 return entry_offset;
468 }
469
470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
471 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
472 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
473 assert(result->fpu() == 0, "result must already be on TOS");
474 }
475 if (InlineTypeReturnedAsFields) {
476 #ifndef _LP64
477 Unimplemented();
478 #endif
479 // Check if we are returning a non-null inline type and load its fields into registers
480 ciType* return_type = compilation()->method()->return_type();
481 if (return_type->is_inlinetype()) {
482 ciInlineKlass* vk = return_type->as_inline_klass();
483 if (vk->can_be_returned_as_fields()) {
484 address unpack_handler = vk->unpack_handler();
485 assert(unpack_handler != nullptr, "must be");
486 __ call(RuntimeAddress(unpack_handler));
487 }
488 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
489 Label skip;
490 Label not_null;
491 __ testptr(rax, rax);
492 __ jcc(Assembler::notZero, not_null);
493 // Returned value is null, zero all return registers because they may belong to oop fields
494 __ xorq(j_rarg1, j_rarg1);
495 __ xorq(j_rarg2, j_rarg2);
496 __ xorq(j_rarg3, j_rarg3);
497 __ xorq(j_rarg4, j_rarg4);
498 __ xorq(j_rarg5, j_rarg5);
499 __ jmp(skip);
500 __ bind(not_null);
501
502 // Check if we are returning a non-null inline type and load its fields into registers
503 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
504
505 // Load fields from a buffered value with an inline class specific handler
506 __ load_klass(rdi, rax, rscratch1);
507 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
508 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
509 // Unpack handler can be null if inline type is not scalarizable in returns
510 __ testptr(rdi, rdi);
511 __ jcc(Assembler::zero, skip);
512 __ call(rdi);
513
514 __ bind(skip);
515 }
516 // At this point, rax points to the value object (for interpreter or C1 caller).
517 // The fields of the object are copied into registers (for C2 caller).
518 }
519
520 // Pop the stack before the safepoint code
521 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
522
523 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
524 __ reserved_stack_check();
525 }
526
527 // Note: we do not need to round double result; float result has the right precision
528 // the poll sets the condition code, but no data registers
529
530 code_stub->set_safepoint_offset(__ offset());
531 __ relocate(relocInfo::poll_return_type);
532 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
533 __ ret(0);
534 }
535
536
537 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
538 return (__ store_inline_type_fields_to_buf(vk, false));
539 }
540
541 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
542 guarantee(info != nullptr, "Shouldn't be null");
543 int offset = __ offset();
544 const Register poll_addr = rscratch1;
545 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
546 add_debug_info_for_branch(info);
547 __ relocate(relocInfo::poll_type);
548 address pre_pc = __ pc();
549 __ testl(rax, Address(poll_addr, 0));
550 address post_pc = __ pc();
551 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
552 return offset;
553 }
554
555
556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
557 if (from_reg != to_reg) __ mov(to_reg, from_reg);
558 }
559
560 void LIR_Assembler::swap_reg(Register a, Register b) {
1281 // init_state needs acquire, but x86 is TSO, and so we are already good.
1282 __ cmpb(Address(op->klass()->as_register(),
1283 InstanceKlass::init_state_offset()),
1284 InstanceKlass::fully_initialized);
1285 __ jcc(Assembler::notEqual, *op->stub()->entry());
1286 }
1287 __ allocate_object(op->obj()->as_register(),
1288 op->tmp1()->as_register(),
1289 op->tmp2()->as_register(),
1290 op->header_size(),
1291 op->object_size(),
1292 op->klass()->as_register(),
1293 *op->stub()->entry());
1294 __ bind(*op->stub()->continuation());
1295 }
1296
1297 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1298 Register len = op->len()->as_register();
1299 __ movslq(len, len);
1300
1301 if (UseSlowPath || op->always_slow_path() ||
1302 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1303 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1304 __ jmp(*op->stub()->entry());
1305 } else {
1306 Register tmp1 = op->tmp1()->as_register();
1307 Register tmp2 = op->tmp2()->as_register();
1308 Register tmp3 = op->tmp3()->as_register();
1309 if (len == tmp1) {
1310 tmp1 = tmp3;
1311 } else if (len == tmp2) {
1312 tmp2 = tmp3;
1313 } else if (len == tmp3) {
1314 // everything is ok
1315 } else {
1316 __ mov(tmp3, len);
1317 }
1318 __ allocate_array(op->obj()->as_register(),
1319 len,
1320 tmp1,
1321 tmp2,
1356 int bci = op->profiled_bci();
1357 md = method->method_data_or_null();
1358 assert(md != nullptr, "Sanity");
1359 data = md->bci_to_data(bci);
1360 assert(data != nullptr, "need data for type check");
1361 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1362 }
1363 Label* success_target = success;
1364 Label* failure_target = failure;
1365
1366 if (obj == k_RInfo) {
1367 k_RInfo = dst;
1368 } else if (obj == klass_RInfo) {
1369 klass_RInfo = dst;
1370 }
1371 Rtmp1 = op->tmp3()->as_register();
1372 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1373
1374 assert_different_registers(obj, k_RInfo, klass_RInfo);
1375
1376 if (op->need_null_check()) {
1377 __ testptr(obj, obj);
1378 if (op->should_profile()) {
1379 Label not_null;
1380 Register mdo = klass_RInfo;
1381 __ mov_metadata(mdo, md->constant_encoding());
1382 __ jccb(Assembler::notEqual, not_null);
1383 // Object is null; update MDO and exit
1384 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1385 int header_bits = BitData::null_seen_byte_constant();
1386 __ orb(data_addr, header_bits);
1387 __ jmp(*obj_is_null);
1388 __ bind(not_null);
1389
1390 Register recv = k_RInfo;
1391 __ load_klass(recv, obj, tmp_load_klass);
1392 type_profile_helper(mdo, md, data, recv);
1393 } else {
1394 __ jcc(Assembler::equal, *obj_is_null);
1395 }
1396 }
1397
1398 if (!k->is_loaded()) {
1399 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1400 } else {
1401 __ mov_metadata(k_RInfo, k->constant_encoding());
1402 }
1403 __ verify_oop(obj);
1404
1405 if (op->fast_check()) {
1406 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1407 // get object class
1408 // not a safepoint as obj null check happens earlier
1409 __ load_klass(Rtmp1, obj, tmp_load_klass);
1410 __ cmpptr(k_RInfo, Rtmp1);
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1417 if (k->is_loaded()) {
1418 // See if we get an immediate positive hit
1419 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1420 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1421 __ jcc(Assembler::notEqual, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 } else {
1424 // See if we get an immediate positive hit
1425 __ jcc(Assembler::equal, *success_target);
1426 // check for self
1427 if (k->is_loaded() && k->is_obj_array_klass()) {
1428 // For a direct pointer comparison, we need the refined array klass pointer
1429 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1430 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1431 __ cmpptr(klass_RInfo, tmp_load_klass);
1432 } else {
1433 __ cmpptr(klass_RInfo, k_RInfo);
1434 }
1435 __ jcc(Assembler::equal, *success_target);
1436
1437 __ push_ppx(klass_RInfo);
1438 __ push_ppx(k_RInfo);
1439 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1440 __ pop_ppx(klass_RInfo);
1441 __ pop_ppx(klass_RInfo);
1442 // result is a boolean
1443 __ testl(klass_RInfo, klass_RInfo);
1444 __ jcc(Assembler::equal, *failure_target);
1445 // successful cast, fall through to profile or jump
1446 }
1447 } else {
1448 // perform the fast part of the checking logic
1449 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1450 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1451 __ push_ppx(klass_RInfo);
1452 __ push_ppx(k_RInfo);
1453 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1454 __ pop_ppx(klass_RInfo);
1544 __ mov(dst, obj);
1545 }
1546 } else
1547 if (code == lir_instanceof) {
1548 Register obj = op->object()->as_register();
1549 Register dst = op->result_opr()->as_register();
1550 Label success, failure, done;
1551 emit_typecheck_helper(op, &success, &failure, &failure);
1552 __ bind(failure);
1553 __ xorptr(dst, dst);
1554 __ jmpb(done);
1555 __ bind(success);
1556 __ movptr(dst, 1);
1557 __ bind(done);
1558 } else {
1559 ShouldNotReachHere();
1560 }
1561
1562 }
1563
1564 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1565 // We are loading/storing from/to an array that *may* be a flat array (the
1566 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1567 // If this array is a flat array, take the slow path.
1568 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1569 }
1570
1571 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1572 // We are storing into an array that *may* be null-free (the declared type is
1573 // Object[], abstract[], interface[] or VT.ref[]).
1574 Label test_mark_word;
1575 Register tmp = op->tmp()->as_register();
1576 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1577 __ testl(tmp, markWord::unlocked_value);
1578 __ jccb(Assembler::notZero, test_mark_word);
1579 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1580 __ bind(test_mark_word);
1581 __ testl(tmp, markWord::null_free_array_bit_in_place);
1582 }
1583
1584 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1585 Label L_oops_equal;
1586 Label L_oops_not_equal;
1587 Label L_end;
1588
1589 Register left = op->left()->as_register();
1590 Register right = op->right()->as_register();
1591
1592 __ cmpptr(left, right);
1593 __ jcc(Assembler::equal, L_oops_equal);
1594
1595 // (1) Null check -- if one of the operands is null, the other must not be null (because
1596 // the two references are not equal), so they are not substitutable,
1597 __ testptr(left, left);
1598 __ jcc(Assembler::zero, L_oops_not_equal);
1599 __ testptr(right, right);
1600 __ jcc(Assembler::zero, L_oops_not_equal);
1601
1602 ciKlass* left_klass = op->left_klass();
1603 ciKlass* right_klass = op->right_klass();
1604
1605 // (2) Inline type check -- if either of the operands is not an inline type,
1606 // they are not substitutable. We do this only if we are not sure that the
1607 // operands are inline type
1608 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1609 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1610 Register tmp = op->tmp1()->as_register();
1611 __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1612 __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1613 __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1614 __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1615 __ jcc(Assembler::notEqual, L_oops_not_equal);
1616 }
1617
1618 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1619 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1620 // No need to load klass -- the operands are statically known to be the same inline klass.
1621 __ jmp(*op->stub()->entry());
1622 } else {
1623 Register tmp1 = op->tmp1()->as_register();
1624 Register tmp2 = op->tmp2()->as_register();
1625 if (left == right) { // same operand, so clearly the same klasses, let's save the check
1626 __ jmp (*op->stub()->entry()); // -> do slow check
1627 } else {
1628 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1629 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1630 }
1631 // fall through to L_oops_not_equal
1632 }
1633
1634 __ bind(L_oops_not_equal);
1635 move(op->not_equal_result(), op->result_opr());
1636 __ jmp(L_end);
1637
1638 __ bind(L_oops_equal);
1639 move(op->equal_result(), op->result_opr());
1640 __ jmp(L_end);
1641
1642 // We've returned from the stub. RAX contains 0x0 IFF the two
1643 // operands are not substitutable. (Don't compare against 0x1 in case the
1644 // C compiler is naughty)
1645 __ bind(*op->stub()->continuation());
1646 __ cmpl(rax, 0);
1647 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1648 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1649 // fall-through
1650 __ bind(L_end);
1651 }
1652
1653 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1654 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1655 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1656 Register newval = op->new_value()->as_register();
1657 Register cmpval = op->cmp_value()->as_register();
1658 assert(cmpval == rax, "wrong register");
1659 assert(newval != noreg, "new val must be register");
1660 assert(cmpval != newval, "cmp and new values must be in different registers");
1661 assert(cmpval != addr, "cmp and addr must be in different registers");
1662 assert(newval != addr, "new value and addr must be in different registers");
1663
1664 if (op->code() == lir_cas_obj) {
1665 if (UseCompressedOops) {
1666 __ encode_heap_oop(cmpval);
1667 __ mov(rscratch1, newval);
1668 __ encode_heap_oop(rscratch1);
1669 __ lock();
1670 // cmpval (rax) is implicitly used by this instruction
1671 __ cmpxchgl(rscratch1, Address(addr, 0));
1677 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1678 __ lock();
1679 __ cmpxchgl(newval, Address(addr, 0));
1680 }
1681 } else if (op->code() == lir_cas_long) {
1682 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1683 Register newval = op->new_value()->as_register_lo();
1684 Register cmpval = op->cmp_value()->as_register_lo();
1685 assert(cmpval == rax, "wrong register");
1686 assert(newval != noreg, "new val must be register");
1687 assert(cmpval != newval, "cmp and new values must be in different registers");
1688 assert(cmpval != addr, "cmp and addr must be in different registers");
1689 assert(newval != addr, "new value and addr must be in different registers");
1690 __ lock();
1691 __ cmpxchgq(newval, Address(addr, 0));
1692 } else {
1693 Unimplemented();
1694 }
1695 }
1696
1697 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1698 assert(dst->is_cpu_register(), "must be");
1699 assert(dst->type() == src->type(), "must be");
1700
1701 if (src->is_cpu_register()) {
1702 reg2reg(src, dst);
1703 } else if (src->is_stack()) {
1704 stack2reg(src, dst, dst->type());
1705 } else if (src->is_constant()) {
1706 const2reg(src, dst, lir_patch_none, nullptr);
1707 } else {
1708 ShouldNotReachHere();
1709 }
1710 }
1711
1712 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1713 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1714 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1715
1716 Assembler::Condition acond, ncond;
1717 switch (condition) {
1718 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1719 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1720 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1721 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1722 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1723 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1724 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1725 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1726 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1727 ShouldNotReachHere();
1728 }
1729
1730 if (opr1->is_cpu_register()) {
1731 reg2reg(opr1, result);
2302 int offset = __ offset();
2303 switch (code) {
2304 case lir_static_call:
2305 case lir_optvirtual_call:
2306 case lir_dynamic_call:
2307 offset += NativeCall::displacement_offset;
2308 break;
2309 case lir_icvirtual_call:
2310 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2311 break;
2312 default: ShouldNotReachHere();
2313 }
2314 __ align(BytesPerWord, offset);
2315 }
2316
2317
2318 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2319 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2320 "must be aligned");
2321 __ call(AddressLiteral(op->addr(), rtype));
2322 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2323 __ post_call_nop();
2324 }
2325
2326
2327 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2328 __ ic_call(op->addr());
2329 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2330 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2331 "must be aligned");
2332 __ post_call_nop();
2333 }
2334
2335
2336 void LIR_Assembler::emit_static_call_stub() {
2337 address call_pc = __ pc();
2338 address stub = __ start_a_stub(call_stub_size());
2339 if (stub == nullptr) {
2340 bailout("static call stub overflow");
2341 return;
2342 }
2343
2344 int start = __ offset();
2345
2346 // make sure that the displacement word of the call ends up word aligned
2347 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2348 __ relocate(static_stub_Relocation::spec(call_pc));
2349 __ mov_metadata(rbx, (Metadata*)nullptr);
2476 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2477 }
2478
2479
2480 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2481 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2482 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2483 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2484 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2485 }
2486
2487
2488 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2489 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2490 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2491 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2492 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2493 }
2494
2495
2496 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2497 if (null_check) {
2498 __ testptr(obj, obj);
2499 __ jcc(Assembler::zero, *slow_path->entry());
2500 }
2501 if (is_dest) {
2502 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2503 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2504 } else {
2505 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2506 }
2507 }
2508
2509
2510 // This code replaces a call to arraycopy; no exception may
2511 // be thrown in this code, they must be thrown in the System.arraycopy
2512 // activation frame; we could save some checks if this would not be the case
2513 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2514 ciArrayKlass* default_type = op->expected_type();
2515 Register src = op->src()->as_register();
2516 Register dst = op->dst()->as_register();
2517 Register src_pos = op->src_pos()->as_register();
2518 Register dst_pos = op->dst_pos()->as_register();
2519 Register length = op->length()->as_register();
2520 Register tmp = op->tmp()->as_register();
2521 Register tmp_load_klass = rscratch1;
2522 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2523
2524 CodeStub* stub = op->stub();
2525 int flags = op->flags();
2526 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2527 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2528
2529 if (flags & LIR_OpArrayCopy::always_slow_path) {
2530 __ jmp(*stub->entry());
2531 __ bind(*stub->continuation());
2532 return;
2533 }
2534
2535 // if we don't know anything, just go through the generic arraycopy
2536 if (default_type == nullptr) {
2537 // save outgoing arguments on stack in case call to System.arraycopy is needed
2538 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2539 // for interpreter calling conventions. Now we have to do it in new style conventions.
2540 // For the moment until C1 gets the new register allocator I just force all the
2541 // args to the right place (except the register args) and then on the back side
2542 // reload the register args properly if we go slow path. Yuck
2543
2544 // These are proper for the calling convention
2545 store_parameter(length, 2);
2546 store_parameter(dst_pos, 1);
2547 store_parameter(dst, 0);
2548
2549 // these are just temporary placements until we need to reload
2550 store_parameter(src_pos, 3);
2551 store_parameter(src, 4);
2552
2553 address copyfunc_addr = StubRoutines::generic_arraycopy();
2554 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2591 __ mov(tmp, rax);
2592 __ xorl(tmp, -1);
2593
2594 // Reload values from the stack so they are where the stub
2595 // expects them.
2596 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2597 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2598 __ movptr (length, Address(rsp, 2*BytesPerWord));
2599 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2600 __ movptr (src, Address(rsp, 4*BytesPerWord));
2601
2602 __ subl(length, tmp);
2603 __ addl(src_pos, tmp);
2604 __ addl(dst_pos, tmp);
2605 __ jmp(*stub->entry());
2606
2607 __ bind(*stub->continuation());
2608 return;
2609 }
2610
2611 // Handle inline type arrays
2612 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2613 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2614 }
2615 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2616 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2617 }
2618
2619 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2620
2621 int elem_size = type2aelembytes(basic_type);
2622 Address::ScaleFactor scale;
2623
2624 switch (elem_size) {
2625 case 1 :
2626 scale = Address::times_1;
2627 break;
2628 case 2 :
2629 scale = Address::times_2;
2630 break;
2631 case 4 :
2632 scale = Address::times_4;
2633 break;
2634 case 8 :
2635 scale = Address::times_8;
2636 break;
2637 default:
2638 scale = Address::no_scale;
3136 // first time here. Set profile type.
3137 __ movptr(mdo_addr, tmp);
3138 #ifdef ASSERT
3139 __ andptr(tmp, TypeEntries::type_klass_mask);
3140 __ verify_klass_ptr(tmp);
3141 #endif
3142 } else {
3143 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3144 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3145
3146 __ testptr(mdo_addr, TypeEntries::type_unknown);
3147 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3148
3149 __ orptr(mdo_addr, TypeEntries::type_unknown);
3150 }
3151 }
3152 }
3153 __ bind(next);
3154 }
3155
3156 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3157 Register obj = op->obj()->as_register();
3158 Register tmp = op->tmp()->as_pointer_register();
3159 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3160 bool not_null = op->not_null();
3161 int flag = op->flag();
3162
3163 Label not_inline_type;
3164 if (!not_null) {
3165 __ testptr(obj, obj);
3166 __ jccb(Assembler::zero, not_inline_type);
3167 }
3168
3169 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3170
3171 __ orb(mdo_addr, flag);
3172
3173 __ bind(not_inline_type);
3174 }
3175
3176
3177 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3178 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3179 }
3180
3181
3182 void LIR_Assembler::align_backward_branch_target() {
3183 __ align(BytesPerWord);
3184 }
3185
3186
3187 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3188 if (left->is_single_cpu()) {
3189 __ negl(left->as_register());
3190 move_regs(left->as_register(), dest->as_register());
3191
3192 } else if (left->is_double_cpu()) {
3193 Register lo = left->as_register_lo();
3194 Register dst = dest->as_register_lo();
3195 __ movptr(dst, lo);
3196 __ negptr(dst);
3342 }
3343
3344 void LIR_Assembler::membar_loadstore() {
3345 // no-op
3346 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3347 }
3348
3349 void LIR_Assembler::membar_storeload() {
3350 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3351 }
3352
3353 void LIR_Assembler::on_spin_wait() {
3354 __ pause ();
3355 }
3356
3357 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3358 assert(result_reg->is_register(), "check");
3359 __ mov(result_reg->as_register(), r15_thread);
3360 }
3361
3362 void LIR_Assembler::check_orig_pc() {
3363 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3364 }
3365
3366 void LIR_Assembler::peephole(LIR_List*) {
3367 // do nothing for now
3368 }
3369
3370 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3371 assert(data == dest, "xchg/xadd uses only 2 operands");
3372
3373 if (data->type() == T_INT) {
3374 if (code == lir_xadd) {
3375 __ lock();
3376 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3377 } else {
3378 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3379 }
3380 } else if (data->is_oop()) {
3381 assert (code == lir_xchg, "xadd for oops");
3382 Register obj = data->as_register();
3383 if (UseCompressedOops) {
3384 __ encode_heap_oop(obj);
|