14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "code/aotCodeCache.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "runtime/threadIdentifier.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
407 // Perform needed unlocking
408 MonitorExitStub* stub = nullptr;
409 if (method()->is_synchronized()) {
410 monitor_address(0, FrameMap::rax_opr);
411 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
412 __ unlock_object(rdi, rsi, rax, *stub->entry());
413 __ bind(*stub->continuation());
414 }
415
416 if (compilation()->env()->dtrace_method_probes()) {
417 __ mov(rdi, r15_thread);
418 __ mov_metadata(rsi, method()->constant_encoding());
419 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
420 }
421
422 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
423 __ mov(rax, rbx); // Restore the exception
424 }
425
426 // remove the activation and dispatch to the unwind handler
427 __ remove_frame(initial_frame_size_in_bytes());
428 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
429
430 // Emit the slow path assembly
431 if (stub != nullptr) {
432 stub->emit_code(this);
433 }
434
435 return offset;
436 }
437
438
439 int LIR_Assembler::emit_deopt_handler() {
440 // generate code for exception handler
441 address handler_base = __ start_a_stub(deopt_handler_size());
442 if (handler_base == nullptr) {
443 // not enough space left for the handler
444 bailout("deopt handler overflow");
445 return -1;
446 }
447
452
453 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
454
455 int entry_offset = __ offset();
456
457 __ jmp(start);
458
459 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
460 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
461 "out of bounds read in post-call NOP check");
462 __ end_a_stub();
463
464 return entry_offset;
465 }
466
467 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
468 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
469 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
470 assert(result->fpu() == 0, "result must already be on TOS");
471 }
472
473 // Pop the stack before the safepoint code
474 __ remove_frame(initial_frame_size_in_bytes());
475
476 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
477 __ reserved_stack_check();
478 }
479
480 // Note: we do not need to round double result; float result has the right precision
481 // the poll sets the condition code, but no data registers
482
483 code_stub->set_safepoint_offset(__ offset());
484 __ relocate(relocInfo::poll_return_type);
485 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
486 __ ret(0);
487 }
488
489
490 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
491 guarantee(info != nullptr, "Shouldn't be null");
492 int offset = __ offset();
493 const Register poll_addr = rscratch1;
494 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
495 add_debug_info_for_branch(info);
496 __ relocate(relocInfo::poll_type);
497 address pre_pc = __ pc();
498 __ testl(rax, Address(poll_addr, 0));
499 address post_pc = __ pc();
500 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
501 return offset;
502 }
503
504
505 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
506 if (from_reg != to_reg) __ mov(to_reg, from_reg);
507 }
508
509 void LIR_Assembler::swap_reg(Register a, Register b) {
1230 // init_state needs acquire, but x86 is TSO, and so we are already good.
1231 __ cmpb(Address(op->klass()->as_register(),
1232 InstanceKlass::init_state_offset()),
1233 InstanceKlass::fully_initialized);
1234 __ jcc(Assembler::notEqual, *op->stub()->entry());
1235 }
1236 __ allocate_object(op->obj()->as_register(),
1237 op->tmp1()->as_register(),
1238 op->tmp2()->as_register(),
1239 op->header_size(),
1240 op->object_size(),
1241 op->klass()->as_register(),
1242 *op->stub()->entry());
1243 __ bind(*op->stub()->continuation());
1244 }
1245
1246 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1247 Register len = op->len()->as_register();
1248 __ movslq(len, len);
1249
1250 if (UseSlowPath ||
1251 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1252 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1253 __ jmp(*op->stub()->entry());
1254 } else {
1255 Register tmp1 = op->tmp1()->as_register();
1256 Register tmp2 = op->tmp2()->as_register();
1257 Register tmp3 = op->tmp3()->as_register();
1258 if (len == tmp1) {
1259 tmp1 = tmp3;
1260 } else if (len == tmp2) {
1261 tmp2 = tmp3;
1262 } else if (len == tmp3) {
1263 // everything is ok
1264 } else {
1265 __ mov(tmp3, len);
1266 }
1267 __ allocate_array(op->obj()->as_register(),
1268 len,
1269 tmp1,
1270 tmp2,
1305 int bci = op->profiled_bci();
1306 md = method->method_data_or_null();
1307 assert(md != nullptr, "Sanity");
1308 data = md->bci_to_data(bci);
1309 assert(data != nullptr, "need data for type check");
1310 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1311 }
1312 Label* success_target = success;
1313 Label* failure_target = failure;
1314
1315 if (obj == k_RInfo) {
1316 k_RInfo = dst;
1317 } else if (obj == klass_RInfo) {
1318 klass_RInfo = dst;
1319 }
1320 Rtmp1 = op->tmp3()->as_register();
1321 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1322
1323 assert_different_registers(obj, k_RInfo, klass_RInfo);
1324
1325 __ testptr(obj, obj);
1326 if (op->should_profile()) {
1327 Label not_null;
1328 Register mdo = klass_RInfo;
1329 __ mov_metadata(mdo, md->constant_encoding());
1330 __ jccb(Assembler::notEqual, not_null);
1331 // Object is null; update MDO and exit
1332 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1333 int header_bits = BitData::null_seen_byte_constant();
1334 __ orb(data_addr, header_bits);
1335 __ jmp(*obj_is_null);
1336 __ bind(not_null);
1337
1338 Register recv = k_RInfo;
1339 __ load_klass(recv, obj, tmp_load_klass);
1340 type_profile_helper(mdo, md, data, recv);
1341 } else {
1342 __ jcc(Assembler::equal, *obj_is_null);
1343 }
1344
1345 if (!k->is_loaded()) {
1346 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1347 } else {
1348 __ mov_metadata(k_RInfo, k->constant_encoding());
1349 }
1350 __ verify_oop(obj);
1351
1352 if (op->fast_check()) {
1353 // get object class
1354 // not a safepoint as obj null check happens earlier
1355 __ load_klass(Rtmp1, obj, tmp_load_klass);
1356 __ cmpptr(k_RInfo, Rtmp1);
1357 __ jcc(Assembler::notEqual, *failure_target);
1358 // successful cast, fall through to profile or jump
1359 } else {
1360 // get object class
1361 // not a safepoint as obj null check happens earlier
1362 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1363 if (k->is_loaded()) {
1364 // See if we get an immediate positive hit
1365 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1366 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1367 __ jcc(Assembler::notEqual, *failure_target);
1368 // successful cast, fall through to profile or jump
1369 } else {
1370 // See if we get an immediate positive hit
1371 __ jcc(Assembler::equal, *success_target);
1372 // check for self
1373 __ cmpptr(klass_RInfo, k_RInfo);
1374 __ jcc(Assembler::equal, *success_target);
1375
1376 __ push_ppx(klass_RInfo);
1377 __ push_ppx(k_RInfo);
1378 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1379 __ pop_ppx(klass_RInfo);
1380 __ pop_ppx(klass_RInfo);
1381 // result is a boolean
1382 __ testl(klass_RInfo, klass_RInfo);
1383 __ jcc(Assembler::equal, *failure_target);
1384 // successful cast, fall through to profile or jump
1385 }
1386 } else {
1387 // perform the fast part of the checking logic
1388 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1389 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1390 __ push_ppx(klass_RInfo);
1391 __ push_ppx(k_RInfo);
1392 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1393 __ pop_ppx(klass_RInfo);
1483 __ mov(dst, obj);
1484 }
1485 } else
1486 if (code == lir_instanceof) {
1487 Register obj = op->object()->as_register();
1488 Register dst = op->result_opr()->as_register();
1489 Label success, failure, done;
1490 emit_typecheck_helper(op, &success, &failure, &failure);
1491 __ bind(failure);
1492 __ xorptr(dst, dst);
1493 __ jmpb(done);
1494 __ bind(success);
1495 __ movptr(dst, 1);
1496 __ bind(done);
1497 } else {
1498 ShouldNotReachHere();
1499 }
1500
1501 }
1502
1503
1504 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1505 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1506 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1507 Register newval = op->new_value()->as_register();
1508 Register cmpval = op->cmp_value()->as_register();
1509 assert(cmpval == rax, "wrong register");
1510 assert(newval != noreg, "new val must be register");
1511 assert(cmpval != newval, "cmp and new values must be in different registers");
1512 assert(cmpval != addr, "cmp and addr must be in different registers");
1513 assert(newval != addr, "new value and addr must be in different registers");
1514
1515 if (op->code() == lir_cas_obj) {
1516 if (UseCompressedOops) {
1517 __ encode_heap_oop(cmpval);
1518 __ mov(rscratch1, newval);
1519 __ encode_heap_oop(rscratch1);
1520 __ lock();
1521 // cmpval (rax) is implicitly used by this instruction
1522 __ cmpxchgl(rscratch1, Address(addr, 0));
1528 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1529 __ lock();
1530 __ cmpxchgl(newval, Address(addr, 0));
1531 }
1532 } else if (op->code() == lir_cas_long) {
1533 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1534 Register newval = op->new_value()->as_register_lo();
1535 Register cmpval = op->cmp_value()->as_register_lo();
1536 assert(cmpval == rax, "wrong register");
1537 assert(newval != noreg, "new val must be register");
1538 assert(cmpval != newval, "cmp and new values must be in different registers");
1539 assert(cmpval != addr, "cmp and addr must be in different registers");
1540 assert(newval != addr, "new value and addr must be in different registers");
1541 __ lock();
1542 __ cmpxchgq(newval, Address(addr, 0));
1543 } else {
1544 Unimplemented();
1545 }
1546 }
1547
1548 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1549 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1550 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1551
1552 Assembler::Condition acond, ncond;
1553 switch (condition) {
1554 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1555 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1556 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1557 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1558 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1559 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1560 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1561 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1562 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1563 ShouldNotReachHere();
1564 }
1565
1566 if (opr1->is_cpu_register()) {
1567 reg2reg(opr1, result);
2138 int offset = __ offset();
2139 switch (code) {
2140 case lir_static_call:
2141 case lir_optvirtual_call:
2142 case lir_dynamic_call:
2143 offset += NativeCall::displacement_offset;
2144 break;
2145 case lir_icvirtual_call:
2146 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2147 break;
2148 default: ShouldNotReachHere();
2149 }
2150 __ align(BytesPerWord, offset);
2151 }
2152
2153
2154 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2155 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2156 "must be aligned");
2157 __ call(AddressLiteral(op->addr(), rtype));
2158 add_call_info(code_offset(), op->info());
2159 __ post_call_nop();
2160 }
2161
2162
2163 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2164 __ ic_call(op->addr());
2165 add_call_info(code_offset(), op->info());
2166 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2167 "must be aligned");
2168 __ post_call_nop();
2169 }
2170
2171
2172 void LIR_Assembler::emit_static_call_stub() {
2173 address call_pc = __ pc();
2174 address stub = __ start_a_stub(call_stub_size());
2175 if (stub == nullptr) {
2176 bailout("static call stub overflow");
2177 return;
2178 }
2179
2180 int start = __ offset();
2181
2182 // make sure that the displacement word of the call ends up word aligned
2183 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2184 __ relocate(static_stub_Relocation::spec(call_pc));
2185 __ mov_metadata(rbx, (Metadata*)nullptr);
2312 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2313 }
2314
2315
2316 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2317 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2318 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2319 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2320 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2321 }
2322
2323
2324 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2325 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2326 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2327 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2328 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2329 }
2330
2331
2332 // This code replaces a call to arraycopy; no exception may
2333 // be thrown in this code, they must be thrown in the System.arraycopy
2334 // activation frame; we could save some checks if this would not be the case
2335 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2336 ciArrayKlass* default_type = op->expected_type();
2337 Register src = op->src()->as_register();
2338 Register dst = op->dst()->as_register();
2339 Register src_pos = op->src_pos()->as_register();
2340 Register dst_pos = op->dst_pos()->as_register();
2341 Register length = op->length()->as_register();
2342 Register tmp = op->tmp()->as_register();
2343 Register tmp_load_klass = rscratch1;
2344 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2345
2346 CodeStub* stub = op->stub();
2347 int flags = op->flags();
2348 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2349 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2350
2351 // if we don't know anything, just go through the generic arraycopy
2352 if (default_type == nullptr) {
2353 // save outgoing arguments on stack in case call to System.arraycopy is needed
2354 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2355 // for interpreter calling conventions. Now we have to do it in new style conventions.
2356 // For the moment until C1 gets the new register allocator I just force all the
2357 // args to the right place (except the register args) and then on the back side
2358 // reload the register args properly if we go slow path. Yuck
2359
2360 // These are proper for the calling convention
2361 store_parameter(length, 2);
2362 store_parameter(dst_pos, 1);
2363 store_parameter(dst, 0);
2364
2365 // these are just temporary placements until we need to reload
2366 store_parameter(src_pos, 3);
2367 store_parameter(src, 4);
2368
2369 address copyfunc_addr = StubRoutines::generic_arraycopy();
2370 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2407 __ mov(tmp, rax);
2408 __ xorl(tmp, -1);
2409
2410 // Reload values from the stack so they are where the stub
2411 // expects them.
2412 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2413 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2414 __ movptr (length, Address(rsp, 2*BytesPerWord));
2415 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2416 __ movptr (src, Address(rsp, 4*BytesPerWord));
2417
2418 __ subl(length, tmp);
2419 __ addl(src_pos, tmp);
2420 __ addl(dst_pos, tmp);
2421 __ jmp(*stub->entry());
2422
2423 __ bind(*stub->continuation());
2424 return;
2425 }
2426
2427 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2428
2429 int elem_size = type2aelembytes(basic_type);
2430 Address::ScaleFactor scale;
2431
2432 switch (elem_size) {
2433 case 1 :
2434 scale = Address::times_1;
2435 break;
2436 case 2 :
2437 scale = Address::times_2;
2438 break;
2439 case 4 :
2440 scale = Address::times_4;
2441 break;
2442 case 8 :
2443 scale = Address::times_8;
2444 break;
2445 default:
2446 scale = Address::no_scale;
2944 // first time here. Set profile type.
2945 __ movptr(mdo_addr, tmp);
2946 #ifdef ASSERT
2947 __ andptr(tmp, TypeEntries::type_klass_mask);
2948 __ verify_klass_ptr(tmp);
2949 #endif
2950 } else {
2951 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2952 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2953
2954 __ testptr(mdo_addr, TypeEntries::type_unknown);
2955 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2956
2957 __ orptr(mdo_addr, TypeEntries::type_unknown);
2958 }
2959 }
2960 }
2961 __ bind(next);
2962 }
2963
2964 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2965 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2966 }
2967
2968
2969 void LIR_Assembler::align_backward_branch_target() {
2970 __ align(BytesPerWord);
2971 }
2972
2973
2974 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2975 if (left->is_single_cpu()) {
2976 __ negl(left->as_register());
2977 move_regs(left->as_register(), dest->as_register());
2978
2979 } else if (left->is_double_cpu()) {
2980 Register lo = left->as_register_lo();
2981 Register dst = dest->as_register_lo();
2982 __ movptr(dst, lo);
2983 __ negptr(dst);
3129 }
3130
3131 void LIR_Assembler::membar_loadstore() {
3132 // no-op
3133 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3134 }
3135
3136 void LIR_Assembler::membar_storeload() {
3137 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3138 }
3139
3140 void LIR_Assembler::on_spin_wait() {
3141 __ pause ();
3142 }
3143
3144 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3145 assert(result_reg->is_register(), "check");
3146 __ mov(result_reg->as_register(), r15_thread);
3147 }
3148
3149
3150 void LIR_Assembler::peephole(LIR_List*) {
3151 // do nothing for now
3152 }
3153
3154 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3155 assert(data == dest, "xchg/xadd uses only 2 operands");
3156
3157 if (data->type() == T_INT) {
3158 if (code == lir_xadd) {
3159 __ lock();
3160 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3161 } else {
3162 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3163 }
3164 } else if (data->is_oop()) {
3165 assert (code == lir_xchg, "xadd for oops");
3166 Register obj = data->as_register();
3167 if (UseCompressedOops) {
3168 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/threadIdentifier.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #include "vmreg_x86.inline.hpp"
51
52
53 // These masks are used to provide 128-bit aligned bitmasks to the XMM
54 // instructions, to allow sign-masking or sign-bit flipping. They allow
55 // fast versions of NegF/NegD and AbsF/AbsD.
56
57 // Note: 'double' and 'long long' have 32-bits alignment on x86.
58 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
59 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
60 // of 128-bits operands for SSE instructions.
61 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
62 // Store the value to a 128-bits operand.
410 // Perform needed unlocking
411 MonitorExitStub* stub = nullptr;
412 if (method()->is_synchronized()) {
413 monitor_address(0, FrameMap::rax_opr);
414 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
415 __ unlock_object(rdi, rsi, rax, *stub->entry());
416 __ bind(*stub->continuation());
417 }
418
419 if (compilation()->env()->dtrace_method_probes()) {
420 __ mov(rdi, r15_thread);
421 __ mov_metadata(rsi, method()->constant_encoding());
422 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
423 }
424
425 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
426 __ mov(rax, rbx); // Restore the exception
427 }
428
429 // remove the activation and dispatch to the unwind handler
430 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
431 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
432
433 // Emit the slow path assembly
434 if (stub != nullptr) {
435 stub->emit_code(this);
436 }
437
438 return offset;
439 }
440
441
442 int LIR_Assembler::emit_deopt_handler() {
443 // generate code for exception handler
444 address handler_base = __ start_a_stub(deopt_handler_size());
445 if (handler_base == nullptr) {
446 // not enough space left for the handler
447 bailout("deopt handler overflow");
448 return -1;
449 }
450
455
456 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457
458 int entry_offset = __ offset();
459
460 __ jmp(start);
461
462 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
463 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
464 "out of bounds read in post-call NOP check");
465 __ end_a_stub();
466
467 return entry_offset;
468 }
469
470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
471 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
472 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
473 assert(result->fpu() == 0, "result must already be on TOS");
474 }
475 if (InlineTypeReturnedAsFields) {
476 #ifndef _LP64
477 Unimplemented();
478 #endif
479 // Check if we are returning a non-null inline type and load its fields into registers
480 ciType* return_type = compilation()->method()->return_type();
481 if (return_type->is_inlinetype()) {
482 ciInlineKlass* vk = return_type->as_inline_klass();
483 if (vk->can_be_returned_as_fields()) {
484 address unpack_handler = vk->unpack_handler();
485 assert(unpack_handler != nullptr, "must be");
486 __ call(RuntimeAddress(unpack_handler));
487 }
488 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
489 Label skip;
490 Label not_null;
491 __ testptr(rax, rax);
492 __ jcc(Assembler::notZero, not_null);
493 // Returned value is null, zero all return registers because they may belong to oop fields
494 __ xorq(j_rarg1, j_rarg1);
495 __ xorq(j_rarg2, j_rarg2);
496 __ xorq(j_rarg3, j_rarg3);
497 __ xorq(j_rarg4, j_rarg4);
498 __ xorq(j_rarg5, j_rarg5);
499 __ jmp(skip);
500 __ bind(not_null);
501
502 // Check if we are returning a non-null inline type and load its fields into registers
503 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
504
505 // Load fields from a buffered value with an inline class specific handler
506 __ load_klass(rdi, rax, rscratch1);
507 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
508 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
509 // Unpack handler can be null if inline type is not scalarizable in returns
510 __ testptr(rdi, rdi);
511 __ jcc(Assembler::zero, skip);
512 __ call(rdi);
513
514 __ bind(skip);
515 }
516 // At this point, rax points to the value object (for interpreter or C1 caller).
517 // The fields of the object are copied into registers (for C2 caller).
518 }
519
520 // Pop the stack before the safepoint code
521 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
522
523 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
524 __ reserved_stack_check();
525 }
526
527 // Note: we do not need to round double result; float result has the right precision
528 // the poll sets the condition code, but no data registers
529
530 code_stub->set_safepoint_offset(__ offset());
531 __ relocate(relocInfo::poll_return_type);
532 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
533 __ ret(0);
534 }
535
536
537 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
538 return (__ store_inline_type_fields_to_buf(vk, false));
539 }
540
541 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
542 guarantee(info != nullptr, "Shouldn't be null");
543 int offset = __ offset();
544 const Register poll_addr = rscratch1;
545 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
546 add_debug_info_for_branch(info);
547 __ relocate(relocInfo::poll_type);
548 address pre_pc = __ pc();
549 __ testl(rax, Address(poll_addr, 0));
550 address post_pc = __ pc();
551 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
552 return offset;
553 }
554
555
556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
557 if (from_reg != to_reg) __ mov(to_reg, from_reg);
558 }
559
560 void LIR_Assembler::swap_reg(Register a, Register b) {
1281 // init_state needs acquire, but x86 is TSO, and so we are already good.
1282 __ cmpb(Address(op->klass()->as_register(),
1283 InstanceKlass::init_state_offset()),
1284 InstanceKlass::fully_initialized);
1285 __ jcc(Assembler::notEqual, *op->stub()->entry());
1286 }
1287 __ allocate_object(op->obj()->as_register(),
1288 op->tmp1()->as_register(),
1289 op->tmp2()->as_register(),
1290 op->header_size(),
1291 op->object_size(),
1292 op->klass()->as_register(),
1293 *op->stub()->entry());
1294 __ bind(*op->stub()->continuation());
1295 }
1296
1297 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1298 Register len = op->len()->as_register();
1299 __ movslq(len, len);
1300
1301 if (UseSlowPath || op->always_slow_path() ||
1302 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1303 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1304 __ jmp(*op->stub()->entry());
1305 } else {
1306 Register tmp1 = op->tmp1()->as_register();
1307 Register tmp2 = op->tmp2()->as_register();
1308 Register tmp3 = op->tmp3()->as_register();
1309 if (len == tmp1) {
1310 tmp1 = tmp3;
1311 } else if (len == tmp2) {
1312 tmp2 = tmp3;
1313 } else if (len == tmp3) {
1314 // everything is ok
1315 } else {
1316 __ mov(tmp3, len);
1317 }
1318 __ allocate_array(op->obj()->as_register(),
1319 len,
1320 tmp1,
1321 tmp2,
1356 int bci = op->profiled_bci();
1357 md = method->method_data_or_null();
1358 assert(md != nullptr, "Sanity");
1359 data = md->bci_to_data(bci);
1360 assert(data != nullptr, "need data for type check");
1361 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1362 }
1363 Label* success_target = success;
1364 Label* failure_target = failure;
1365
1366 if (obj == k_RInfo) {
1367 k_RInfo = dst;
1368 } else if (obj == klass_RInfo) {
1369 klass_RInfo = dst;
1370 }
1371 Rtmp1 = op->tmp3()->as_register();
1372 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1373
1374 assert_different_registers(obj, k_RInfo, klass_RInfo);
1375
1376 if (op->need_null_check()) {
1377 __ testptr(obj, obj);
1378 if (op->should_profile()) {
1379 Label not_null;
1380 Register mdo = klass_RInfo;
1381 __ mov_metadata(mdo, md->constant_encoding());
1382 __ jccb(Assembler::notEqual, not_null);
1383 // Object is null; update MDO and exit
1384 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1385 int header_bits = BitData::null_seen_byte_constant();
1386 __ orb(data_addr, header_bits);
1387 __ jmp(*obj_is_null);
1388 __ bind(not_null);
1389
1390 Register recv = k_RInfo;
1391 __ load_klass(recv, obj, tmp_load_klass);
1392 type_profile_helper(mdo, md, data, recv);
1393 } else {
1394 __ jcc(Assembler::equal, *obj_is_null);
1395 }
1396 }
1397
1398 if (!k->is_loaded()) {
1399 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1400 } else {
1401 __ mov_metadata(k_RInfo, k->constant_encoding());
1402 }
1403 __ verify_oop(obj);
1404
1405 if (op->fast_check()) {
1406 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1407 // get object class
1408 // not a safepoint as obj null check happens earlier
1409 __ load_klass(Rtmp1, obj, tmp_load_klass);
1410 __ cmpptr(k_RInfo, Rtmp1);
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1417 if (k->is_loaded()) {
1418 // See if we get an immediate positive hit
1419 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1420 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1421 __ jcc(Assembler::notEqual, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 } else {
1424 // See if we get an immediate positive hit
1425 __ jcc(Assembler::equal, *success_target);
1426 // check for self
1427 if (k->is_loaded() && k->is_obj_array_klass()) {
1428 // For a direct pointer comparison, we need the refined array klass pointer
1429 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1430 if (!k_refined->is_loaded()) {
1431 bailout("encountered unloaded_ciobjarrayklass due to out of memory error");
1432 return;
1433 }
1434 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1435 __ cmpptr(klass_RInfo, tmp_load_klass);
1436 } else {
1437 __ cmpptr(klass_RInfo, k_RInfo);
1438 }
1439 __ jcc(Assembler::equal, *success_target);
1440
1441 __ push_ppx(klass_RInfo);
1442 __ push_ppx(k_RInfo);
1443 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1444 __ pop_ppx(klass_RInfo);
1445 __ pop_ppx(klass_RInfo);
1446 // result is a boolean
1447 __ testl(klass_RInfo, klass_RInfo);
1448 __ jcc(Assembler::equal, *failure_target);
1449 // successful cast, fall through to profile or jump
1450 }
1451 } else {
1452 // perform the fast part of the checking logic
1453 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1454 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1455 __ push_ppx(klass_RInfo);
1456 __ push_ppx(k_RInfo);
1457 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1458 __ pop_ppx(klass_RInfo);
1548 __ mov(dst, obj);
1549 }
1550 } else
1551 if (code == lir_instanceof) {
1552 Register obj = op->object()->as_register();
1553 Register dst = op->result_opr()->as_register();
1554 Label success, failure, done;
1555 emit_typecheck_helper(op, &success, &failure, &failure);
1556 __ bind(failure);
1557 __ xorptr(dst, dst);
1558 __ jmpb(done);
1559 __ bind(success);
1560 __ movptr(dst, 1);
1561 __ bind(done);
1562 } else {
1563 ShouldNotReachHere();
1564 }
1565
1566 }
1567
1568 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1569 // We are loading/storing from/to an array that *may* be a flat array (the
1570 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1571 // If this array is a flat array, take the slow path.
1572 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1573 }
1574
1575 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1576 // We are storing into an array that *may* be null-free (the declared type is
1577 // Object[], abstract[], interface[] or VT.ref[]).
1578 Label test_mark_word;
1579 Register tmp = op->tmp()->as_register();
1580 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1581 __ testl(tmp, markWord::unlocked_value);
1582 __ jccb(Assembler::notZero, test_mark_word);
1583 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1584 __ bind(test_mark_word);
1585 __ testl(tmp, markWord::null_free_array_bit_in_place);
1586 }
1587
1588 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1589 Label L_oops_equal;
1590 Label L_oops_not_equal;
1591 Label L_end;
1592
1593 Register left = op->left()->as_register();
1594 Register right = op->right()->as_register();
1595
1596 __ cmpptr(left, right);
1597 __ jcc(Assembler::equal, L_oops_equal);
1598
1599 // (1) Null check -- if one of the operands is null, the other must not be null (because
1600 // the two references are not equal), so they are not substitutable,
1601 __ testptr(left, left);
1602 __ jcc(Assembler::zero, L_oops_not_equal);
1603 __ testptr(right, right);
1604 __ jcc(Assembler::zero, L_oops_not_equal);
1605
1606 ciKlass* left_klass = op->left_klass();
1607 ciKlass* right_klass = op->right_klass();
1608
1609 // (2) Inline type check -- if either of the operands is not an inline type,
1610 // they are not substitutable. We do this only if we are not sure that the
1611 // operands are inline type
1612 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1613 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1614 Register tmp = op->tmp1()->as_register();
1615 __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1616 __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1617 __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1618 __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1619 __ jcc(Assembler::notEqual, L_oops_not_equal);
1620 }
1621
1622 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1623 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1624 // No need to load klass -- the operands are statically known to be the same inline klass.
1625 __ jmp(*op->stub()->entry());
1626 } else {
1627 Register tmp1 = op->tmp1()->as_register();
1628 Register tmp2 = op->tmp2()->as_register();
1629 if (left == right) { // same operand, so clearly the same klasses, let's save the check
1630 __ jmp (*op->stub()->entry()); // -> do slow check
1631 } else {
1632 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1633 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1634 }
1635 // fall through to L_oops_not_equal
1636 }
1637
1638 __ bind(L_oops_not_equal);
1639 move(op->not_equal_result(), op->result_opr());
1640 __ jmp(L_end);
1641
1642 __ bind(L_oops_equal);
1643 move(op->equal_result(), op->result_opr());
1644 __ jmp(L_end);
1645
1646 // We've returned from the stub. RAX contains 0x0 IFF the two
1647 // operands are not substitutable. (Don't compare against 0x1 in case the
1648 // C compiler is naughty)
1649 __ bind(*op->stub()->continuation());
1650 __ cmpl(rax, 0);
1651 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1652 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1653 // fall-through
1654 __ bind(L_end);
1655 }
1656
1657 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1658 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1659 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1660 Register newval = op->new_value()->as_register();
1661 Register cmpval = op->cmp_value()->as_register();
1662 assert(cmpval == rax, "wrong register");
1663 assert(newval != noreg, "new val must be register");
1664 assert(cmpval != newval, "cmp and new values must be in different registers");
1665 assert(cmpval != addr, "cmp and addr must be in different registers");
1666 assert(newval != addr, "new value and addr must be in different registers");
1667
1668 if (op->code() == lir_cas_obj) {
1669 if (UseCompressedOops) {
1670 __ encode_heap_oop(cmpval);
1671 __ mov(rscratch1, newval);
1672 __ encode_heap_oop(rscratch1);
1673 __ lock();
1674 // cmpval (rax) is implicitly used by this instruction
1675 __ cmpxchgl(rscratch1, Address(addr, 0));
1681 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1682 __ lock();
1683 __ cmpxchgl(newval, Address(addr, 0));
1684 }
1685 } else if (op->code() == lir_cas_long) {
1686 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1687 Register newval = op->new_value()->as_register_lo();
1688 Register cmpval = op->cmp_value()->as_register_lo();
1689 assert(cmpval == rax, "wrong register");
1690 assert(newval != noreg, "new val must be register");
1691 assert(cmpval != newval, "cmp and new values must be in different registers");
1692 assert(cmpval != addr, "cmp and addr must be in different registers");
1693 assert(newval != addr, "new value and addr must be in different registers");
1694 __ lock();
1695 __ cmpxchgq(newval, Address(addr, 0));
1696 } else {
1697 Unimplemented();
1698 }
1699 }
1700
1701 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1702 assert(dst->is_cpu_register(), "must be");
1703 assert(dst->type() == src->type(), "must be");
1704
1705 if (src->is_cpu_register()) {
1706 reg2reg(src, dst);
1707 } else if (src->is_stack()) {
1708 stack2reg(src, dst, dst->type());
1709 } else if (src->is_constant()) {
1710 const2reg(src, dst, lir_patch_none, nullptr);
1711 } else {
1712 ShouldNotReachHere();
1713 }
1714 }
1715
1716 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1717 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1718 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1719
1720 Assembler::Condition acond, ncond;
1721 switch (condition) {
1722 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1723 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1724 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1725 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1726 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1727 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1728 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1729 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1730 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1731 ShouldNotReachHere();
1732 }
1733
1734 if (opr1->is_cpu_register()) {
1735 reg2reg(opr1, result);
2306 int offset = __ offset();
2307 switch (code) {
2308 case lir_static_call:
2309 case lir_optvirtual_call:
2310 case lir_dynamic_call:
2311 offset += NativeCall::displacement_offset;
2312 break;
2313 case lir_icvirtual_call:
2314 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2315 break;
2316 default: ShouldNotReachHere();
2317 }
2318 __ align(BytesPerWord, offset);
2319 }
2320
2321
2322 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2323 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2324 "must be aligned");
2325 __ call(AddressLiteral(op->addr(), rtype));
2326 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2327 __ post_call_nop();
2328 }
2329
2330
2331 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2332 __ ic_call(op->addr());
2333 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2334 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2335 "must be aligned");
2336 __ post_call_nop();
2337 }
2338
2339
2340 void LIR_Assembler::emit_static_call_stub() {
2341 address call_pc = __ pc();
2342 address stub = __ start_a_stub(call_stub_size());
2343 if (stub == nullptr) {
2344 bailout("static call stub overflow");
2345 return;
2346 }
2347
2348 int start = __ offset();
2349
2350 // make sure that the displacement word of the call ends up word aligned
2351 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2352 __ relocate(static_stub_Relocation::spec(call_pc));
2353 __ mov_metadata(rbx, (Metadata*)nullptr);
2480 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2481 }
2482
2483
2484 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2485 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2486 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2487 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2488 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2489 }
2490
2491
2492 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2493 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2494 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2495 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2496 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2497 }
2498
2499
2500 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2501 if (null_check) {
2502 __ testptr(obj, obj);
2503 __ jcc(Assembler::zero, *slow_path->entry());
2504 }
2505 if (is_dest) {
2506 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2507 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2508 } else {
2509 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2510 }
2511 }
2512
2513
2514 // This code replaces a call to arraycopy; no exception may
2515 // be thrown in this code, they must be thrown in the System.arraycopy
2516 // activation frame; we could save some checks if this would not be the case
2517 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2518 ciArrayKlass* default_type = op->expected_type();
2519 Register src = op->src()->as_register();
2520 Register dst = op->dst()->as_register();
2521 Register src_pos = op->src_pos()->as_register();
2522 Register dst_pos = op->dst_pos()->as_register();
2523 Register length = op->length()->as_register();
2524 Register tmp = op->tmp()->as_register();
2525 Register tmp_load_klass = rscratch1;
2526 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2527
2528 CodeStub* stub = op->stub();
2529 int flags = op->flags();
2530 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2531 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2532
2533 if (flags & LIR_OpArrayCopy::always_slow_path) {
2534 __ jmp(*stub->entry());
2535 __ bind(*stub->continuation());
2536 return;
2537 }
2538
2539 // if we don't know anything, just go through the generic arraycopy
2540 if (default_type == nullptr) {
2541 // save outgoing arguments on stack in case call to System.arraycopy is needed
2542 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2543 // for interpreter calling conventions. Now we have to do it in new style conventions.
2544 // For the moment until C1 gets the new register allocator I just force all the
2545 // args to the right place (except the register args) and then on the back side
2546 // reload the register args properly if we go slow path. Yuck
2547
2548 // These are proper for the calling convention
2549 store_parameter(length, 2);
2550 store_parameter(dst_pos, 1);
2551 store_parameter(dst, 0);
2552
2553 // these are just temporary placements until we need to reload
2554 store_parameter(src_pos, 3);
2555 store_parameter(src, 4);
2556
2557 address copyfunc_addr = StubRoutines::generic_arraycopy();
2558 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2595 __ mov(tmp, rax);
2596 __ xorl(tmp, -1);
2597
2598 // Reload values from the stack so they are where the stub
2599 // expects them.
2600 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2601 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2602 __ movptr (length, Address(rsp, 2*BytesPerWord));
2603 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2604 __ movptr (src, Address(rsp, 4*BytesPerWord));
2605
2606 __ subl(length, tmp);
2607 __ addl(src_pos, tmp);
2608 __ addl(dst_pos, tmp);
2609 __ jmp(*stub->entry());
2610
2611 __ bind(*stub->continuation());
2612 return;
2613 }
2614
2615 // Handle inline type arrays
2616 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2617 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2618 }
2619 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2620 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2621 }
2622
2623 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2624
2625 int elem_size = type2aelembytes(basic_type);
2626 Address::ScaleFactor scale;
2627
2628 switch (elem_size) {
2629 case 1 :
2630 scale = Address::times_1;
2631 break;
2632 case 2 :
2633 scale = Address::times_2;
2634 break;
2635 case 4 :
2636 scale = Address::times_4;
2637 break;
2638 case 8 :
2639 scale = Address::times_8;
2640 break;
2641 default:
2642 scale = Address::no_scale;
3140 // first time here. Set profile type.
3141 __ movptr(mdo_addr, tmp);
3142 #ifdef ASSERT
3143 __ andptr(tmp, TypeEntries::type_klass_mask);
3144 __ verify_klass_ptr(tmp);
3145 #endif
3146 } else {
3147 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3148 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3149
3150 __ testptr(mdo_addr, TypeEntries::type_unknown);
3151 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3152
3153 __ orptr(mdo_addr, TypeEntries::type_unknown);
3154 }
3155 }
3156 }
3157 __ bind(next);
3158 }
3159
3160 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3161 Register obj = op->obj()->as_register();
3162 Register tmp = op->tmp()->as_pointer_register();
3163 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3164 bool not_null = op->not_null();
3165 int flag = op->flag();
3166
3167 Label not_inline_type;
3168 if (!not_null) {
3169 __ testptr(obj, obj);
3170 __ jccb(Assembler::zero, not_inline_type);
3171 }
3172
3173 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3174
3175 __ orb(mdo_addr, flag);
3176
3177 __ bind(not_inline_type);
3178 }
3179
3180
3181 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3182 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3183 }
3184
3185
3186 void LIR_Assembler::align_backward_branch_target() {
3187 __ align(BytesPerWord);
3188 }
3189
3190
3191 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3192 if (left->is_single_cpu()) {
3193 __ negl(left->as_register());
3194 move_regs(left->as_register(), dest->as_register());
3195
3196 } else if (left->is_double_cpu()) {
3197 Register lo = left->as_register_lo();
3198 Register dst = dest->as_register_lo();
3199 __ movptr(dst, lo);
3200 __ negptr(dst);
3346 }
3347
3348 void LIR_Assembler::membar_loadstore() {
3349 // no-op
3350 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3351 }
3352
3353 void LIR_Assembler::membar_storeload() {
3354 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3355 }
3356
3357 void LIR_Assembler::on_spin_wait() {
3358 __ pause ();
3359 }
3360
3361 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3362 assert(result_reg->is_register(), "check");
3363 __ mov(result_reg->as_register(), r15_thread);
3364 }
3365
3366 void LIR_Assembler::check_orig_pc() {
3367 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3368 }
3369
3370 void LIR_Assembler::peephole(LIR_List*) {
3371 // do nothing for now
3372 }
3373
3374 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3375 assert(data == dest, "xchg/xadd uses only 2 operands");
3376
3377 if (data->type() == T_INT) {
3378 if (code == lir_xadd) {
3379 __ lock();
3380 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3381 } else {
3382 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3383 }
3384 } else if (data->is_oop()) {
3385 assert (code == lir_xchg, "xadd for oops");
3386 Register obj = data->as_register();
3387 if (UseCompressedOops) {
3388 __ encode_heap_oop(obj);
|