14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "code/aotCodeCache.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_x86.inline.hpp"
47
48
49 // These masks are used to provide 128-bit aligned bitmasks to the XMM
50 // instructions, to allow sign-masking or sign-bit flipping. They allow
51 // fast versions of NegF/NegD and AbsF/AbsD.
52
53 // Note: 'double' and 'long long' have 32-bits alignment on x86.
54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
55 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
56 // of 128-bits operands for SSE instructions.
57 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
58 // Store the value to a 128-bits operand.
59 operand[0] = lo;
412 // Perform needed unlocking
413 MonitorExitStub* stub = nullptr;
414 if (method()->is_synchronized()) {
415 monitor_address(0, FrameMap::rax_opr);
416 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
417 __ unlock_object(rdi, rsi, rax, *stub->entry());
418 __ bind(*stub->continuation());
419 }
420
421 if (compilation()->env()->dtrace_method_probes()) {
422 __ mov(rdi, r15_thread);
423 __ mov_metadata(rsi, method()->constant_encoding());
424 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
425 }
426
427 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
428 __ mov(rax, rbx); // Restore the exception
429 }
430
431 // remove the activation and dispatch to the unwind handler
432 __ remove_frame(initial_frame_size_in_bytes());
433 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
434
435 // Emit the slow path assembly
436 if (stub != nullptr) {
437 stub->emit_code(this);
438 }
439
440 return offset;
441 }
442
443
444 int LIR_Assembler::emit_deopt_handler() {
445 // generate code for exception handler
446 address handler_base = __ start_a_stub(deopt_handler_size());
447 if (handler_base == nullptr) {
448 // not enough space left for the handler
449 bailout("deopt handler overflow");
450 return -1;
451 }
452
457
458 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
459
460 int entry_offset = __ offset();
461
462 __ jmp(start);
463
464 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
465 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
466 "out of bounds read in post-call NOP check");
467 __ end_a_stub();
468
469 return entry_offset;
470 }
471
472 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
473 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
474 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
475 assert(result->fpu() == 0, "result must already be on TOS");
476 }
477
478 // Pop the stack before the safepoint code
479 __ remove_frame(initial_frame_size_in_bytes());
480
481 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
482 __ reserved_stack_check();
483 }
484
485 // Note: we do not need to round double result; float result has the right precision
486 // the poll sets the condition code, but no data registers
487
488 code_stub->set_safepoint_offset(__ offset());
489 __ relocate(relocInfo::poll_return_type);
490 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
491 __ ret(0);
492 }
493
494
495 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
496 guarantee(info != nullptr, "Shouldn't be null");
497 int offset = __ offset();
498 const Register poll_addr = rscratch1;
499 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
500 add_debug_info_for_branch(info);
501 __ relocate(relocInfo::poll_type);
502 address pre_pc = __ pc();
503 __ testl(rax, Address(poll_addr, 0));
504 address post_pc = __ pc();
505 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
506 return offset;
507 }
508
509
510 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
511 if (from_reg != to_reg) __ mov(to_reg, from_reg);
512 }
513
514 void LIR_Assembler::swap_reg(Register a, Register b) {
1222 // init_state needs acquire, but x86 is TSO, and so we are already good.
1223 __ cmpb(Address(op->klass()->as_register(),
1224 InstanceKlass::init_state_offset()),
1225 InstanceKlass::fully_initialized);
1226 __ jcc(Assembler::notEqual, *op->stub()->entry());
1227 }
1228 __ allocate_object(op->obj()->as_register(),
1229 op->tmp1()->as_register(),
1230 op->tmp2()->as_register(),
1231 op->header_size(),
1232 op->object_size(),
1233 op->klass()->as_register(),
1234 *op->stub()->entry());
1235 __ bind(*op->stub()->continuation());
1236 }
1237
1238 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1239 Register len = op->len()->as_register();
1240 __ movslq(len, len);
1241
1242 if (UseSlowPath ||
1243 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1244 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1245 __ jmp(*op->stub()->entry());
1246 } else {
1247 Register tmp1 = op->tmp1()->as_register();
1248 Register tmp2 = op->tmp2()->as_register();
1249 Register tmp3 = op->tmp3()->as_register();
1250 if (len == tmp1) {
1251 tmp1 = tmp3;
1252 } else if (len == tmp2) {
1253 tmp2 = tmp3;
1254 } else if (len == tmp3) {
1255 // everything is ok
1256 } else {
1257 __ mov(tmp3, len);
1258 }
1259 __ allocate_array(op->obj()->as_register(),
1260 len,
1261 tmp1,
1262 tmp2,
1301 assert(data != nullptr, "need data for type check");
1302 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1303 }
1304 Label* success_target = success;
1305 Label* failure_target = failure;
1306
1307 if (obj == k_RInfo) {
1308 k_RInfo = dst;
1309 } else if (obj == klass_RInfo) {
1310 klass_RInfo = dst;
1311 }
1312 if (k->is_loaded() && !UseCompressedClassPointers) {
1313 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1314 } else {
1315 Rtmp1 = op->tmp3()->as_register();
1316 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1317 }
1318
1319 assert_different_registers(obj, k_RInfo, klass_RInfo);
1320
1321 __ testptr(obj, obj);
1322 if (op->should_profile()) {
1323 Label not_null;
1324 Register mdo = klass_RInfo;
1325 __ mov_metadata(mdo, md->constant_encoding());
1326 __ jccb(Assembler::notEqual, not_null);
1327 // Object is null; update MDO and exit
1328 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1329 int header_bits = BitData::null_seen_byte_constant();
1330 __ orb(data_addr, header_bits);
1331 __ jmp(*obj_is_null);
1332 __ bind(not_null);
1333
1334 Register recv = k_RInfo;
1335 __ load_klass(recv, obj, tmp_load_klass);
1336 type_profile_helper(mdo, md, data, recv);
1337 } else {
1338 __ jcc(Assembler::equal, *obj_is_null);
1339 }
1340
1341 if (!k->is_loaded()) {
1342 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1343 } else {
1344 __ mov_metadata(k_RInfo, k->constant_encoding());
1345 }
1346 __ verify_oop(obj);
1347
1348 if (op->fast_check()) {
1349 // get object class
1350 // not a safepoint as obj null check happens earlier
1351 if (UseCompressedClassPointers) {
1352 __ load_klass(Rtmp1, obj, tmp_load_klass);
1353 __ cmpptr(k_RInfo, Rtmp1);
1354 } else {
1355 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1356 }
1357 __ jcc(Assembler::notEqual, *failure_target);
1358 // successful cast, fall through to profile or jump
1359 } else {
1360 // get object class
1361 // not a safepoint as obj null check happens earlier
1362 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1363 if (k->is_loaded()) {
1364 // See if we get an immediate positive hit
1365 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1366 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1367 __ jcc(Assembler::notEqual, *failure_target);
1368 // successful cast, fall through to profile or jump
1369 } else {
1370 // See if we get an immediate positive hit
1371 __ jcc(Assembler::equal, *success_target);
1372 // check for self
1373 __ cmpptr(klass_RInfo, k_RInfo);
1374 __ jcc(Assembler::equal, *success_target);
1375
1376 __ push_ppx(klass_RInfo);
1377 __ push_ppx(k_RInfo);
1378 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1379 __ pop_ppx(klass_RInfo);
1380 __ pop_ppx(klass_RInfo);
1381 // result is a boolean
1382 __ testl(klass_RInfo, klass_RInfo);
1383 __ jcc(Assembler::equal, *failure_target);
1384 // successful cast, fall through to profile or jump
1385 }
1386 } else {
1387 // perform the fast part of the checking logic
1388 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1389 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1390 __ push_ppx(klass_RInfo);
1391 __ push_ppx(k_RInfo);
1392 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1393 __ pop_ppx(klass_RInfo);
1483 __ mov(dst, obj);
1484 }
1485 } else
1486 if (code == lir_instanceof) {
1487 Register obj = op->object()->as_register();
1488 Register dst = op->result_opr()->as_register();
1489 Label success, failure, done;
1490 emit_typecheck_helper(op, &success, &failure, &failure);
1491 __ bind(failure);
1492 __ xorptr(dst, dst);
1493 __ jmpb(done);
1494 __ bind(success);
1495 __ movptr(dst, 1);
1496 __ bind(done);
1497 } else {
1498 ShouldNotReachHere();
1499 }
1500
1501 }
1502
1503
1504 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1505 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1506 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1507 Register newval = op->new_value()->as_register();
1508 Register cmpval = op->cmp_value()->as_register();
1509 assert(cmpval == rax, "wrong register");
1510 assert(newval != noreg, "new val must be register");
1511 assert(cmpval != newval, "cmp and new values must be in different registers");
1512 assert(cmpval != addr, "cmp and addr must be in different registers");
1513 assert(newval != addr, "new value and addr must be in different registers");
1514
1515 if (op->code() == lir_cas_obj) {
1516 if (UseCompressedOops) {
1517 __ encode_heap_oop(cmpval);
1518 __ mov(rscratch1, newval);
1519 __ encode_heap_oop(rscratch1);
1520 __ lock();
1521 // cmpval (rax) is implicitly used by this instruction
1522 __ cmpxchgl(rscratch1, Address(addr, 0));
1528 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1529 __ lock();
1530 __ cmpxchgl(newval, Address(addr, 0));
1531 }
1532 } else if (op->code() == lir_cas_long) {
1533 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1534 Register newval = op->new_value()->as_register_lo();
1535 Register cmpval = op->cmp_value()->as_register_lo();
1536 assert(cmpval == rax, "wrong register");
1537 assert(newval != noreg, "new val must be register");
1538 assert(cmpval != newval, "cmp and new values must be in different registers");
1539 assert(cmpval != addr, "cmp and addr must be in different registers");
1540 assert(newval != addr, "new value and addr must be in different registers");
1541 __ lock();
1542 __ cmpxchgq(newval, Address(addr, 0));
1543 } else {
1544 Unimplemented();
1545 }
1546 }
1547
1548 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1549 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1550 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1551
1552 Assembler::Condition acond, ncond;
1553 switch (condition) {
1554 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1555 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1556 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1557 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1558 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1559 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1560 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1561 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1562 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1563 ShouldNotReachHere();
1564 }
1565
1566 if (opr1->is_cpu_register()) {
1567 reg2reg(opr1, result);
2138 int offset = __ offset();
2139 switch (code) {
2140 case lir_static_call:
2141 case lir_optvirtual_call:
2142 case lir_dynamic_call:
2143 offset += NativeCall::displacement_offset;
2144 break;
2145 case lir_icvirtual_call:
2146 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2147 break;
2148 default: ShouldNotReachHere();
2149 }
2150 __ align(BytesPerWord, offset);
2151 }
2152
2153
2154 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2155 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2156 "must be aligned");
2157 __ call(AddressLiteral(op->addr(), rtype));
2158 add_call_info(code_offset(), op->info());
2159 __ post_call_nop();
2160 }
2161
2162
2163 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2164 __ ic_call(op->addr());
2165 add_call_info(code_offset(), op->info());
2166 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2167 "must be aligned");
2168 __ post_call_nop();
2169 }
2170
2171
2172 void LIR_Assembler::emit_static_call_stub() {
2173 address call_pc = __ pc();
2174 address stub = __ start_a_stub(call_stub_size());
2175 if (stub == nullptr) {
2176 bailout("static call stub overflow");
2177 return;
2178 }
2179
2180 int start = __ offset();
2181
2182 // make sure that the displacement word of the call ends up word aligned
2183 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2184 __ relocate(static_stub_Relocation::spec(call_pc));
2185 __ mov_metadata(rbx, (Metadata*)nullptr);
2312 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2313 }
2314
2315
2316 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2317 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2318 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2319 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2320 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2321 }
2322
2323
2324 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2325 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2326 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2327 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2328 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2329 }
2330
2331
2332 // This code replaces a call to arraycopy; no exception may
2333 // be thrown in this code, they must be thrown in the System.arraycopy
2334 // activation frame; we could save some checks if this would not be the case
2335 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2336 ciArrayKlass* default_type = op->expected_type();
2337 Register src = op->src()->as_register();
2338 Register dst = op->dst()->as_register();
2339 Register src_pos = op->src_pos()->as_register();
2340 Register dst_pos = op->dst_pos()->as_register();
2341 Register length = op->length()->as_register();
2342 Register tmp = op->tmp()->as_register();
2343 Register tmp_load_klass = rscratch1;
2344 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2345
2346 CodeStub* stub = op->stub();
2347 int flags = op->flags();
2348 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2349 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2350
2351 // if we don't know anything, just go through the generic arraycopy
2352 if (default_type == nullptr) {
2353 // save outgoing arguments on stack in case call to System.arraycopy is needed
2354 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2355 // for interpreter calling conventions. Now we have to do it in new style conventions.
2356 // For the moment until C1 gets the new register allocator I just force all the
2357 // args to the right place (except the register args) and then on the back side
2358 // reload the register args properly if we go slow path. Yuck
2359
2360 // These are proper for the calling convention
2361 store_parameter(length, 2);
2362 store_parameter(dst_pos, 1);
2363 store_parameter(dst, 0);
2364
2365 // these are just temporary placements until we need to reload
2366 store_parameter(src_pos, 3);
2367 store_parameter(src, 4);
2368
2369 address copyfunc_addr = StubRoutines::generic_arraycopy();
2370 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2407 __ mov(tmp, rax);
2408 __ xorl(tmp, -1);
2409
2410 // Reload values from the stack so they are where the stub
2411 // expects them.
2412 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2413 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2414 __ movptr (length, Address(rsp, 2*BytesPerWord));
2415 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2416 __ movptr (src, Address(rsp, 4*BytesPerWord));
2417
2418 __ subl(length, tmp);
2419 __ addl(src_pos, tmp);
2420 __ addl(dst_pos, tmp);
2421 __ jmp(*stub->entry());
2422
2423 __ bind(*stub->continuation());
2424 return;
2425 }
2426
2427 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2428
2429 int elem_size = type2aelembytes(basic_type);
2430 Address::ScaleFactor scale;
2431
2432 switch (elem_size) {
2433 case 1 :
2434 scale = Address::times_1;
2435 break;
2436 case 2 :
2437 scale = Address::times_2;
2438 break;
2439 case 4 :
2440 scale = Address::times_4;
2441 break;
2442 case 8 :
2443 scale = Address::times_8;
2444 break;
2445 default:
2446 scale = Address::no_scale;
2946 // first time here. Set profile type.
2947 __ movptr(mdo_addr, tmp);
2948 #ifdef ASSERT
2949 __ andptr(tmp, TypeEntries::type_klass_mask);
2950 __ verify_klass_ptr(tmp);
2951 #endif
2952 } else {
2953 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2954 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2955
2956 __ testptr(mdo_addr, TypeEntries::type_unknown);
2957 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2958
2959 __ orptr(mdo_addr, TypeEntries::type_unknown);
2960 }
2961 }
2962 }
2963 __ bind(next);
2964 }
2965
2966 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2967 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2968 }
2969
2970
2971 void LIR_Assembler::align_backward_branch_target() {
2972 __ align(BytesPerWord);
2973 }
2974
2975
2976 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2977 if (left->is_single_cpu()) {
2978 __ negl(left->as_register());
2979 move_regs(left->as_register(), dest->as_register());
2980
2981 } else if (left->is_double_cpu()) {
2982 Register lo = left->as_register_lo();
2983 Register dst = dest->as_register_lo();
2984 __ movptr(dst, lo);
2985 __ negptr(dst);
3131 }
3132
3133 void LIR_Assembler::membar_loadstore() {
3134 // no-op
3135 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3136 }
3137
3138 void LIR_Assembler::membar_storeload() {
3139 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3140 }
3141
3142 void LIR_Assembler::on_spin_wait() {
3143 __ pause ();
3144 }
3145
3146 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3147 assert(result_reg->is_register(), "check");
3148 __ mov(result_reg->as_register(), r15_thread);
3149 }
3150
3151
3152 void LIR_Assembler::peephole(LIR_List*) {
3153 // do nothing for now
3154 }
3155
3156 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3157 assert(data == dest, "xchg/xadd uses only 2 operands");
3158
3159 if (data->type() == T_INT) {
3160 if (code == lir_xadd) {
3161 __ lock();
3162 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3163 } else {
3164 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3165 }
3166 } else if (data->is_oop()) {
3167 assert (code == lir_xchg, "xadd for oops");
3168 Register obj = data->as_register();
3169 if (UseCompressedOops) {
3170 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "compiler/oopMap.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_x86.hpp"
42 #include "oops/oop.inline.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/safepointMechanism.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "utilities/powerOfTwo.hpp"
49 #include "vmreg_x86.inline.hpp"
50
51
52 // These masks are used to provide 128-bit aligned bitmasks to the XMM
53 // instructions, to allow sign-masking or sign-bit flipping. They allow
54 // fast versions of NegF/NegD and AbsF/AbsD.
55
56 // Note: 'double' and 'long long' have 32-bits alignment on x86.
57 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
58 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
59 // of 128-bits operands for SSE instructions.
60 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
61 // Store the value to a 128-bits operand.
62 operand[0] = lo;
415 // Perform needed unlocking
416 MonitorExitStub* stub = nullptr;
417 if (method()->is_synchronized()) {
418 monitor_address(0, FrameMap::rax_opr);
419 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
420 __ unlock_object(rdi, rsi, rax, *stub->entry());
421 __ bind(*stub->continuation());
422 }
423
424 if (compilation()->env()->dtrace_method_probes()) {
425 __ mov(rdi, r15_thread);
426 __ mov_metadata(rsi, method()->constant_encoding());
427 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
428 }
429
430 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
431 __ mov(rax, rbx); // Restore the exception
432 }
433
434 // remove the activation and dispatch to the unwind handler
435 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
436 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
437
438 // Emit the slow path assembly
439 if (stub != nullptr) {
440 stub->emit_code(this);
441 }
442
443 return offset;
444 }
445
446
447 int LIR_Assembler::emit_deopt_handler() {
448 // generate code for exception handler
449 address handler_base = __ start_a_stub(deopt_handler_size());
450 if (handler_base == nullptr) {
451 // not enough space left for the handler
452 bailout("deopt handler overflow");
453 return -1;
454 }
455
460
461 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
462
463 int entry_offset = __ offset();
464
465 __ jmp(start);
466
467 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
468 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
469 "out of bounds read in post-call NOP check");
470 __ end_a_stub();
471
472 return entry_offset;
473 }
474
475 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
476 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
477 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
478 assert(result->fpu() == 0, "result must already be on TOS");
479 }
480 if (InlineTypeReturnedAsFields) {
481 #ifndef _LP64
482 Unimplemented();
483 #endif
484 // Check if we are returning an non-null inline type and load its fields into registers
485 ciType* return_type = compilation()->method()->return_type();
486 if (return_type->is_inlinetype()) {
487 ciInlineKlass* vk = return_type->as_inline_klass();
488 if (vk->can_be_returned_as_fields()) {
489 address unpack_handler = vk->unpack_handler();
490 assert(unpack_handler != nullptr, "must be");
491 __ call(RuntimeAddress(unpack_handler));
492 }
493 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
494 Label skip;
495 Label not_null;
496 __ testptr(rax, rax);
497 __ jcc(Assembler::notZero, not_null);
498 // Returned value is null, zero all return registers because they may belong to oop fields
499 __ xorq(j_rarg1, j_rarg1);
500 __ xorq(j_rarg2, j_rarg2);
501 __ xorq(j_rarg3, j_rarg3);
502 __ xorq(j_rarg4, j_rarg4);
503 __ xorq(j_rarg5, j_rarg5);
504 __ jmp(skip);
505 __ bind(not_null);
506
507 // Check if we are returning an non-null inline type and load its fields into registers
508 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
509
510 // Load fields from a buffered value with an inline class specific handler
511 __ load_klass(rdi, rax, rscratch1);
512 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
513 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
514 // Unpack handler can be null if inline type is not scalarizable in returns
515 __ testptr(rdi, rdi);
516 __ jcc(Assembler::zero, skip);
517 __ call(rdi);
518
519 __ bind(skip);
520 }
521 // At this point, rax points to the value object (for interpreter or C1 caller).
522 // The fields of the object are copied into registers (for C2 caller).
523 }
524
525 // Pop the stack before the safepoint code
526 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
527
528 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
529 __ reserved_stack_check();
530 }
531
532 // Note: we do not need to round double result; float result has the right precision
533 // the poll sets the condition code, but no data registers
534
535 code_stub->set_safepoint_offset(__ offset());
536 __ relocate(relocInfo::poll_return_type);
537 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
538 __ ret(0);
539 }
540
541
542 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
543 return (__ store_inline_type_fields_to_buf(vk, false));
544 }
545
546 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
547 guarantee(info != nullptr, "Shouldn't be null");
548 int offset = __ offset();
549 const Register poll_addr = rscratch1;
550 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
551 add_debug_info_for_branch(info);
552 __ relocate(relocInfo::poll_type);
553 address pre_pc = __ pc();
554 __ testl(rax, Address(poll_addr, 0));
555 address post_pc = __ pc();
556 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
557 return offset;
558 }
559
560
561 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
562 if (from_reg != to_reg) __ mov(to_reg, from_reg);
563 }
564
565 void LIR_Assembler::swap_reg(Register a, Register b) {
1273 // init_state needs acquire, but x86 is TSO, and so we are already good.
1274 __ cmpb(Address(op->klass()->as_register(),
1275 InstanceKlass::init_state_offset()),
1276 InstanceKlass::fully_initialized);
1277 __ jcc(Assembler::notEqual, *op->stub()->entry());
1278 }
1279 __ allocate_object(op->obj()->as_register(),
1280 op->tmp1()->as_register(),
1281 op->tmp2()->as_register(),
1282 op->header_size(),
1283 op->object_size(),
1284 op->klass()->as_register(),
1285 *op->stub()->entry());
1286 __ bind(*op->stub()->continuation());
1287 }
1288
1289 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1290 Register len = op->len()->as_register();
1291 __ movslq(len, len);
1292
1293 if (UseSlowPath || op->always_slow_path() ||
1294 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1295 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1296 __ jmp(*op->stub()->entry());
1297 } else {
1298 Register tmp1 = op->tmp1()->as_register();
1299 Register tmp2 = op->tmp2()->as_register();
1300 Register tmp3 = op->tmp3()->as_register();
1301 if (len == tmp1) {
1302 tmp1 = tmp3;
1303 } else if (len == tmp2) {
1304 tmp2 = tmp3;
1305 } else if (len == tmp3) {
1306 // everything is ok
1307 } else {
1308 __ mov(tmp3, len);
1309 }
1310 __ allocate_array(op->obj()->as_register(),
1311 len,
1312 tmp1,
1313 tmp2,
1352 assert(data != nullptr, "need data for type check");
1353 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1354 }
1355 Label* success_target = success;
1356 Label* failure_target = failure;
1357
1358 if (obj == k_RInfo) {
1359 k_RInfo = dst;
1360 } else if (obj == klass_RInfo) {
1361 klass_RInfo = dst;
1362 }
1363 if (k->is_loaded() && !UseCompressedClassPointers) {
1364 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1365 } else {
1366 Rtmp1 = op->tmp3()->as_register();
1367 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1368 }
1369
1370 assert_different_registers(obj, k_RInfo, klass_RInfo);
1371
1372 if (op->need_null_check()) {
1373 __ testptr(obj, obj);
1374 if (op->should_profile()) {
1375 Label not_null;
1376 Register mdo = klass_RInfo;
1377 __ mov_metadata(mdo, md->constant_encoding());
1378 __ jccb(Assembler::notEqual, not_null);
1379 // Object is null; update MDO and exit
1380 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1381 int header_bits = BitData::null_seen_byte_constant();
1382 __ orb(data_addr, header_bits);
1383 __ jmp(*obj_is_null);
1384 __ bind(not_null);
1385
1386 Register recv = k_RInfo;
1387 __ load_klass(recv, obj, tmp_load_klass);
1388 type_profile_helper(mdo, md, data, recv);
1389 } else {
1390 __ jcc(Assembler::equal, *obj_is_null);
1391 }
1392 }
1393
1394 if (!k->is_loaded()) {
1395 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1396 } else {
1397 __ mov_metadata(k_RInfo, k->constant_encoding());
1398 }
1399 __ verify_oop(obj);
1400
1401 if (op->fast_check()) {
1402 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1403 // get object class
1404 // not a safepoint as obj null check happens earlier
1405 if (UseCompressedClassPointers) {
1406 __ load_klass(Rtmp1, obj, tmp_load_klass);
1407 __ cmpptr(k_RInfo, Rtmp1);
1408 } else {
1409 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1410 }
1411 __ jcc(Assembler::notEqual, *failure_target);
1412 // successful cast, fall through to profile or jump
1413 } else {
1414 // get object class
1415 // not a safepoint as obj null check happens earlier
1416 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1417 if (k->is_loaded()) {
1418 // See if we get an immediate positive hit
1419 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1420 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1421 __ jcc(Assembler::notEqual, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 } else {
1424 // See if we get an immediate positive hit
1425 __ jcc(Assembler::equal, *success_target);
1426 // check for self
1427 if (k->is_loaded() && k->is_obj_array_klass()) {
1428 // For a direct pointer comparison, we need the refined array klass pointer
1429 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1430 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1431 __ cmpptr(klass_RInfo, tmp_load_klass);
1432 } else {
1433 __ cmpptr(klass_RInfo, k_RInfo);
1434 }
1435 __ jcc(Assembler::equal, *success_target);
1436
1437 __ push_ppx(klass_RInfo);
1438 __ push_ppx(k_RInfo);
1439 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1440 __ pop_ppx(klass_RInfo);
1441 __ pop_ppx(klass_RInfo);
1442 // result is a boolean
1443 __ testl(klass_RInfo, klass_RInfo);
1444 __ jcc(Assembler::equal, *failure_target);
1445 // successful cast, fall through to profile or jump
1446 }
1447 } else {
1448 // perform the fast part of the checking logic
1449 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1450 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1451 __ push_ppx(klass_RInfo);
1452 __ push_ppx(k_RInfo);
1453 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1454 __ pop_ppx(klass_RInfo);
1544 __ mov(dst, obj);
1545 }
1546 } else
1547 if (code == lir_instanceof) {
1548 Register obj = op->object()->as_register();
1549 Register dst = op->result_opr()->as_register();
1550 Label success, failure, done;
1551 emit_typecheck_helper(op, &success, &failure, &failure);
1552 __ bind(failure);
1553 __ xorptr(dst, dst);
1554 __ jmpb(done);
1555 __ bind(success);
1556 __ movptr(dst, 1);
1557 __ bind(done);
1558 } else {
1559 ShouldNotReachHere();
1560 }
1561
1562 }
1563
1564 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1565 // We are loading/storing from/to an array that *may* be a flat array (the
1566 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1567 // If this array is a flat array, take the slow path.
1568 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1569 if (!op->value()->is_illegal()) {
1570 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1571 // The array is not a flat array, but it might be null-free. If we are storing
1572 // a null into a null-free array, take the slow path (which will throw NPE).
1573 Label skip;
1574 __ cmpptr(op->value()->as_register(), NULL_WORD);
1575 __ jcc(Assembler::notEqual, skip);
1576 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1577 __ bind(skip);
1578 }
1579 }
1580
1581 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1582 // We are storing into an array that *may* be null-free (the declared type is
1583 // Object[], abstract[], interface[] or VT.ref[]).
1584 Label test_mark_word;
1585 Register tmp = op->tmp()->as_register();
1586 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1587 __ testl(tmp, markWord::unlocked_value);
1588 __ jccb(Assembler::notZero, test_mark_word);
1589 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1590 __ bind(test_mark_word);
1591 __ testl(tmp, markWord::null_free_array_bit_in_place);
1592 }
1593
1594 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1595 Label L_oops_equal;
1596 Label L_oops_not_equal;
1597 Label L_end;
1598
1599 Register left = op->left()->as_register();
1600 Register right = op->right()->as_register();
1601
1602 __ cmpptr(left, right);
1603 __ jcc(Assembler::equal, L_oops_equal);
1604
1605 // (1) Null check -- if one of the operands is null, the other must not be null (because
1606 // the two references are not equal), so they are not substitutable,
1607 // FIXME: do null check only if the operand is nullable
1608 __ testptr(left, right);
1609 __ jcc(Assembler::zero, L_oops_not_equal);
1610
1611 ciKlass* left_klass = op->left_klass();
1612 ciKlass* right_klass = op->right_klass();
1613
1614 // (2) Inline type check -- if either of the operands is not a inline type,
1615 // they are not substitutable. We do this only if we are not sure that the
1616 // operands are inline type
1617 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1618 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1619 Register tmp1 = op->tmp1()->as_register();
1620 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1621 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1622 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1623 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1624 __ jcc(Assembler::notEqual, L_oops_not_equal);
1625 }
1626
1627 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1628 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1629 // No need to load klass -- the operands are statically known to be the same inline klass.
1630 __ jmp(*op->stub()->entry());
1631 } else {
1632 Register left_klass_op = op->left_klass_op()->as_register();
1633 Register right_klass_op = op->right_klass_op()->as_register();
1634
1635 if (UseCompressedClassPointers) {
1636 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1637 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1638 __ cmpl(left_klass_op, right_klass_op);
1639 } else {
1640 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1641 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1642 __ cmpptr(left_klass_op, right_klass_op);
1643 }
1644
1645 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1646 // fall through to L_oops_not_equal
1647 }
1648
1649 __ bind(L_oops_not_equal);
1650 move(op->not_equal_result(), op->result_opr());
1651 __ jmp(L_end);
1652
1653 __ bind(L_oops_equal);
1654 move(op->equal_result(), op->result_opr());
1655 __ jmp(L_end);
1656
1657 // We've returned from the stub. RAX contains 0x0 IFF the two
1658 // operands are not substitutable. (Don't compare against 0x1 in case the
1659 // C compiler is naughty)
1660 __ bind(*op->stub()->continuation());
1661 __ cmpl(rax, 0);
1662 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1663 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1664 // fall-through
1665 __ bind(L_end);
1666 }
1667
1668 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1669 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1670 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1671 Register newval = op->new_value()->as_register();
1672 Register cmpval = op->cmp_value()->as_register();
1673 assert(cmpval == rax, "wrong register");
1674 assert(newval != noreg, "new val must be register");
1675 assert(cmpval != newval, "cmp and new values must be in different registers");
1676 assert(cmpval != addr, "cmp and addr must be in different registers");
1677 assert(newval != addr, "new value and addr must be in different registers");
1678
1679 if (op->code() == lir_cas_obj) {
1680 if (UseCompressedOops) {
1681 __ encode_heap_oop(cmpval);
1682 __ mov(rscratch1, newval);
1683 __ encode_heap_oop(rscratch1);
1684 __ lock();
1685 // cmpval (rax) is implicitly used by this instruction
1686 __ cmpxchgl(rscratch1, Address(addr, 0));
1692 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1693 __ lock();
1694 __ cmpxchgl(newval, Address(addr, 0));
1695 }
1696 } else if (op->code() == lir_cas_long) {
1697 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1698 Register newval = op->new_value()->as_register_lo();
1699 Register cmpval = op->cmp_value()->as_register_lo();
1700 assert(cmpval == rax, "wrong register");
1701 assert(newval != noreg, "new val must be register");
1702 assert(cmpval != newval, "cmp and new values must be in different registers");
1703 assert(cmpval != addr, "cmp and addr must be in different registers");
1704 assert(newval != addr, "new value and addr must be in different registers");
1705 __ lock();
1706 __ cmpxchgq(newval, Address(addr, 0));
1707 } else {
1708 Unimplemented();
1709 }
1710 }
1711
1712 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1713 assert(dst->is_cpu_register(), "must be");
1714 assert(dst->type() == src->type(), "must be");
1715
1716 if (src->is_cpu_register()) {
1717 reg2reg(src, dst);
1718 } else if (src->is_stack()) {
1719 stack2reg(src, dst, dst->type());
1720 } else if (src->is_constant()) {
1721 const2reg(src, dst, lir_patch_none, nullptr);
1722 } else {
1723 ShouldNotReachHere();
1724 }
1725 }
1726
1727 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1728 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1729 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1730
1731 Assembler::Condition acond, ncond;
1732 switch (condition) {
1733 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1734 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1735 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1736 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1737 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1738 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1739 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1740 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1741 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1742 ShouldNotReachHere();
1743 }
1744
1745 if (opr1->is_cpu_register()) {
1746 reg2reg(opr1, result);
2317 int offset = __ offset();
2318 switch (code) {
2319 case lir_static_call:
2320 case lir_optvirtual_call:
2321 case lir_dynamic_call:
2322 offset += NativeCall::displacement_offset;
2323 break;
2324 case lir_icvirtual_call:
2325 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2326 break;
2327 default: ShouldNotReachHere();
2328 }
2329 __ align(BytesPerWord, offset);
2330 }
2331
2332
2333 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2334 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2335 "must be aligned");
2336 __ call(AddressLiteral(op->addr(), rtype));
2337 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2338 __ post_call_nop();
2339 }
2340
2341
2342 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2343 __ ic_call(op->addr());
2344 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2345 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2346 "must be aligned");
2347 __ post_call_nop();
2348 }
2349
2350
2351 void LIR_Assembler::emit_static_call_stub() {
2352 address call_pc = __ pc();
2353 address stub = __ start_a_stub(call_stub_size());
2354 if (stub == nullptr) {
2355 bailout("static call stub overflow");
2356 return;
2357 }
2358
2359 int start = __ offset();
2360
2361 // make sure that the displacement word of the call ends up word aligned
2362 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2363 __ relocate(static_stub_Relocation::spec(call_pc));
2364 __ mov_metadata(rbx, (Metadata*)nullptr);
2491 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2492 }
2493
2494
2495 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2496 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2497 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2498 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2499 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2500 }
2501
2502
2503 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2504 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2505 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2506 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2507 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2508 }
2509
2510
2511 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2512 if (null_check) {
2513 __ testptr(obj, obj);
2514 __ jcc(Assembler::zero, *slow_path->entry());
2515 }
2516 if (is_dest) {
2517 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2518 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2519 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2520 } else {
2521 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2522 }
2523 }
2524
2525
2526 // This code replaces a call to arraycopy; no exception may
2527 // be thrown in this code, they must be thrown in the System.arraycopy
2528 // activation frame; we could save some checks if this would not be the case
2529 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2530 ciArrayKlass* default_type = op->expected_type();
2531 Register src = op->src()->as_register();
2532 Register dst = op->dst()->as_register();
2533 Register src_pos = op->src_pos()->as_register();
2534 Register dst_pos = op->dst_pos()->as_register();
2535 Register length = op->length()->as_register();
2536 Register tmp = op->tmp()->as_register();
2537 Register tmp_load_klass = rscratch1;
2538 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2539
2540 CodeStub* stub = op->stub();
2541 int flags = op->flags();
2542 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2543 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2544
2545 if (flags & LIR_OpArrayCopy::always_slow_path) {
2546 __ jmp(*stub->entry());
2547 __ bind(*stub->continuation());
2548 return;
2549 }
2550
2551 // if we don't know anything, just go through the generic arraycopy
2552 if (default_type == nullptr) {
2553 // save outgoing arguments on stack in case call to System.arraycopy is needed
2554 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2555 // for interpreter calling conventions. Now we have to do it in new style conventions.
2556 // For the moment until C1 gets the new register allocator I just force all the
2557 // args to the right place (except the register args) and then on the back side
2558 // reload the register args properly if we go slow path. Yuck
2559
2560 // These are proper for the calling convention
2561 store_parameter(length, 2);
2562 store_parameter(dst_pos, 1);
2563 store_parameter(dst, 0);
2564
2565 // these are just temporary placements until we need to reload
2566 store_parameter(src_pos, 3);
2567 store_parameter(src, 4);
2568
2569 address copyfunc_addr = StubRoutines::generic_arraycopy();
2570 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2607 __ mov(tmp, rax);
2608 __ xorl(tmp, -1);
2609
2610 // Reload values from the stack so they are where the stub
2611 // expects them.
2612 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2613 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2614 __ movptr (length, Address(rsp, 2*BytesPerWord));
2615 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2616 __ movptr (src, Address(rsp, 4*BytesPerWord));
2617
2618 __ subl(length, tmp);
2619 __ addl(src_pos, tmp);
2620 __ addl(dst_pos, tmp);
2621 __ jmp(*stub->entry());
2622
2623 __ bind(*stub->continuation());
2624 return;
2625 }
2626
2627 // Handle inline type arrays
2628 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2629 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2630 }
2631 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2632 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2633 }
2634
2635 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2636
2637 int elem_size = type2aelembytes(basic_type);
2638 Address::ScaleFactor scale;
2639
2640 switch (elem_size) {
2641 case 1 :
2642 scale = Address::times_1;
2643 break;
2644 case 2 :
2645 scale = Address::times_2;
2646 break;
2647 case 4 :
2648 scale = Address::times_4;
2649 break;
2650 case 8 :
2651 scale = Address::times_8;
2652 break;
2653 default:
2654 scale = Address::no_scale;
3154 // first time here. Set profile type.
3155 __ movptr(mdo_addr, tmp);
3156 #ifdef ASSERT
3157 __ andptr(tmp, TypeEntries::type_klass_mask);
3158 __ verify_klass_ptr(tmp);
3159 #endif
3160 } else {
3161 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3162 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3163
3164 __ testptr(mdo_addr, TypeEntries::type_unknown);
3165 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3166
3167 __ orptr(mdo_addr, TypeEntries::type_unknown);
3168 }
3169 }
3170 }
3171 __ bind(next);
3172 }
3173
3174 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3175 Register obj = op->obj()->as_register();
3176 Register tmp = op->tmp()->as_pointer_register();
3177 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3178 bool not_null = op->not_null();
3179 int flag = op->flag();
3180
3181 Label not_inline_type;
3182 if (!not_null) {
3183 __ testptr(obj, obj);
3184 __ jccb(Assembler::zero, not_inline_type);
3185 }
3186
3187 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3188
3189 __ orb(mdo_addr, flag);
3190
3191 __ bind(not_inline_type);
3192 }
3193
3194
3195 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3196 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3197 }
3198
3199
3200 void LIR_Assembler::align_backward_branch_target() {
3201 __ align(BytesPerWord);
3202 }
3203
3204
3205 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3206 if (left->is_single_cpu()) {
3207 __ negl(left->as_register());
3208 move_regs(left->as_register(), dest->as_register());
3209
3210 } else if (left->is_double_cpu()) {
3211 Register lo = left->as_register_lo();
3212 Register dst = dest->as_register_lo();
3213 __ movptr(dst, lo);
3214 __ negptr(dst);
3360 }
3361
3362 void LIR_Assembler::membar_loadstore() {
3363 // no-op
3364 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3365 }
3366
3367 void LIR_Assembler::membar_storeload() {
3368 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3369 }
3370
3371 void LIR_Assembler::on_spin_wait() {
3372 __ pause ();
3373 }
3374
3375 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3376 assert(result_reg->is_register(), "check");
3377 __ mov(result_reg->as_register(), r15_thread);
3378 }
3379
3380 void LIR_Assembler::check_orig_pc() {
3381 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3382 }
3383
3384 void LIR_Assembler::peephole(LIR_List*) {
3385 // do nothing for now
3386 }
3387
3388 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3389 assert(data == dest, "xchg/xadd uses only 2 operands");
3390
3391 if (data->type() == T_INT) {
3392 if (code == lir_xadd) {
3393 __ lock();
3394 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3395 } else {
3396 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3397 }
3398 } else if (data->is_oop()) {
3399 assert (code == lir_xchg, "xadd for oops");
3400 Register obj = data->as_register();
3401 if (UseCompressedOops) {
3402 __ encode_heap_oop(obj);
|