14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
411 // Perform needed unlocking
412 MonitorExitStub* stub = nullptr;
413 if (method()->is_synchronized()) {
414 monitor_address(0, FrameMap::rax_opr);
415 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
416 __ unlock_object(rdi, rsi, rax, *stub->entry());
417 __ bind(*stub->continuation());
418 }
419
420 if (compilation()->env()->dtrace_method_probes()) {
421 __ mov(rdi, r15_thread);
422 __ mov_metadata(rsi, method()->constant_encoding());
423 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
424 }
425
426 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
427 __ mov(rax, rbx); // Restore the exception
428 }
429
430 // remove the activation and dispatch to the unwind handler
431 __ remove_frame(initial_frame_size_in_bytes());
432 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
433
434 // Emit the slow path assembly
435 if (stub != nullptr) {
436 stub->emit_code(this);
437 }
438
439 return offset;
440 }
441
442
443 int LIR_Assembler::emit_deopt_handler() {
444 // generate code for exception handler
445 address handler_base = __ start_a_stub(deopt_handler_size());
446 if (handler_base == nullptr) {
447 // not enough space left for the handler
448 bailout("deopt handler overflow");
449 return -1;
450 }
451
452 int offset = code_offset();
453 InternalAddress here(__ pc());
454
455 __ pushptr(here.addr(), rscratch1);
456 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
458 __ end_a_stub();
459
460 return offset;
461 }
462
463 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
464 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
465 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
466 assert(result->fpu() == 0, "result must already be on TOS");
467 }
468
469 // Pop the stack before the safepoint code
470 __ remove_frame(initial_frame_size_in_bytes());
471
472 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
473 __ reserved_stack_check();
474 }
475
476 // Note: we do not need to round double result; float result has the right precision
477 // the poll sets the condition code, but no data registers
478
479 code_stub->set_safepoint_offset(__ offset());
480 __ relocate(relocInfo::poll_return_type);
481 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
482 __ ret(0);
483 }
484
485
486 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
487 guarantee(info != nullptr, "Shouldn't be null");
488 int offset = __ offset();
489 const Register poll_addr = rscratch1;
490 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
491 add_debug_info_for_branch(info);
492 __ relocate(relocInfo::poll_type);
493 address pre_pc = __ pc();
494 __ testl(rax, Address(poll_addr, 0));
495 address post_pc = __ pc();
496 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
497 return offset;
498 }
499
500
501 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
502 if (from_reg != to_reg) __ mov(to_reg, from_reg);
503 }
504
505 void LIR_Assembler::swap_reg(Register a, Register b) {
1204 // init_state needs acquire, but x86 is TSO, and so we are already good.
1205 __ cmpb(Address(op->klass()->as_register(),
1206 InstanceKlass::init_state_offset()),
1207 InstanceKlass::fully_initialized);
1208 __ jcc(Assembler::notEqual, *op->stub()->entry());
1209 }
1210 __ allocate_object(op->obj()->as_register(),
1211 op->tmp1()->as_register(),
1212 op->tmp2()->as_register(),
1213 op->header_size(),
1214 op->object_size(),
1215 op->klass()->as_register(),
1216 *op->stub()->entry());
1217 __ bind(*op->stub()->continuation());
1218 }
1219
1220 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1221 Register len = op->len()->as_register();
1222 __ movslq(len, len);
1223
1224 if (UseSlowPath ||
1225 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1226 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1227 __ jmp(*op->stub()->entry());
1228 } else {
1229 Register tmp1 = op->tmp1()->as_register();
1230 Register tmp2 = op->tmp2()->as_register();
1231 Register tmp3 = op->tmp3()->as_register();
1232 if (len == tmp1) {
1233 tmp1 = tmp3;
1234 } else if (len == tmp2) {
1235 tmp2 = tmp3;
1236 } else if (len == tmp3) {
1237 // everything is ok
1238 } else {
1239 __ mov(tmp3, len);
1240 }
1241 __ allocate_array(op->obj()->as_register(),
1242 len,
1243 tmp1,
1244 tmp2,
1303 assert(data != nullptr, "need data for type check");
1304 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1305 }
1306 Label* success_target = success;
1307 Label* failure_target = failure;
1308
1309 if (obj == k_RInfo) {
1310 k_RInfo = dst;
1311 } else if (obj == klass_RInfo) {
1312 klass_RInfo = dst;
1313 }
1314 if (k->is_loaded() && !UseCompressedClassPointers) {
1315 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1316 } else {
1317 Rtmp1 = op->tmp3()->as_register();
1318 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1319 }
1320
1321 assert_different_registers(obj, k_RInfo, klass_RInfo);
1322
1323 __ testptr(obj, obj);
1324 if (op->should_profile()) {
1325 Label not_null;
1326 Register mdo = klass_RInfo;
1327 __ mov_metadata(mdo, md->constant_encoding());
1328 __ jccb(Assembler::notEqual, not_null);
1329 // Object is null; update MDO and exit
1330 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1331 int header_bits = BitData::null_seen_byte_constant();
1332 __ orb(data_addr, header_bits);
1333 __ jmp(*obj_is_null);
1334 __ bind(not_null);
1335
1336 Label update_done;
1337 Register recv = k_RInfo;
1338 __ load_klass(recv, obj, tmp_load_klass);
1339 type_profile_helper(mdo, md, data, recv, &update_done);
1340
1341 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1342 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1343
1344 __ bind(update_done);
1345 } else {
1346 __ jcc(Assembler::equal, *obj_is_null);
1347 }
1348
1349 if (!k->is_loaded()) {
1350 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1351 } else {
1352 __ mov_metadata(k_RInfo, k->constant_encoding());
1353 }
1354 __ verify_oop(obj);
1355
1356 if (op->fast_check()) {
1357 // get object class
1358 // not a safepoint as obj null check happens earlier
1359 if (UseCompressedClassPointers) {
1360 __ load_klass(Rtmp1, obj, tmp_load_klass);
1361 __ cmpptr(k_RInfo, Rtmp1);
1362 } else {
1363 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1364 }
1365 __ jcc(Assembler::notEqual, *failure_target);
1366 // successful cast, fall through to profile or jump
1367 } else {
1368 // get object class
1369 // not a safepoint as obj null check happens earlier
1370 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1371 if (k->is_loaded()) {
1372 // See if we get an immediate positive hit
1373 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1374 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1375 __ jcc(Assembler::notEqual, *failure_target);
1376 // successful cast, fall through to profile or jump
1377 } else {
1378 // See if we get an immediate positive hit
1379 __ jcc(Assembler::equal, *success_target);
1380 // check for self
1381 __ cmpptr(klass_RInfo, k_RInfo);
1382 __ jcc(Assembler::equal, *success_target);
1383
1384 __ push_ppx(klass_RInfo);
1385 __ push_ppx(k_RInfo);
1386 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1387 __ pop_ppx(klass_RInfo);
1388 __ pop_ppx(klass_RInfo);
1389 // result is a boolean
1390 __ testl(klass_RInfo, klass_RInfo);
1391 __ jcc(Assembler::equal, *failure_target);
1392 // successful cast, fall through to profile or jump
1393 }
1394 } else {
1395 // perform the fast part of the checking logic
1396 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1397 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1398 __ push_ppx(klass_RInfo);
1399 __ push_ppx(k_RInfo);
1400 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1401 __ pop_ppx(klass_RInfo);
1496 __ mov(dst, obj);
1497 }
1498 } else
1499 if (code == lir_instanceof) {
1500 Register obj = op->object()->as_register();
1501 Register dst = op->result_opr()->as_register();
1502 Label success, failure, done;
1503 emit_typecheck_helper(op, &success, &failure, &failure);
1504 __ bind(failure);
1505 __ xorptr(dst, dst);
1506 __ jmpb(done);
1507 __ bind(success);
1508 __ movptr(dst, 1);
1509 __ bind(done);
1510 } else {
1511 ShouldNotReachHere();
1512 }
1513
1514 }
1515
1516
1517 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1518 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1519 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1520 Register newval = op->new_value()->as_register();
1521 Register cmpval = op->cmp_value()->as_register();
1522 assert(cmpval == rax, "wrong register");
1523 assert(newval != noreg, "new val must be register");
1524 assert(cmpval != newval, "cmp and new values must be in different registers");
1525 assert(cmpval != addr, "cmp and addr must be in different registers");
1526 assert(newval != addr, "new value and addr must be in different registers");
1527
1528 if (op->code() == lir_cas_obj) {
1529 if (UseCompressedOops) {
1530 __ encode_heap_oop(cmpval);
1531 __ mov(rscratch1, newval);
1532 __ encode_heap_oop(rscratch1);
1533 __ lock();
1534 // cmpval (rax) is implicitly used by this instruction
1535 __ cmpxchgl(rscratch1, Address(addr, 0));
1541 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1542 __ lock();
1543 __ cmpxchgl(newval, Address(addr, 0));
1544 }
1545 } else if (op->code() == lir_cas_long) {
1546 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1547 Register newval = op->new_value()->as_register_lo();
1548 Register cmpval = op->cmp_value()->as_register_lo();
1549 assert(cmpval == rax, "wrong register");
1550 assert(newval != noreg, "new val must be register");
1551 assert(cmpval != newval, "cmp and new values must be in different registers");
1552 assert(cmpval != addr, "cmp and addr must be in different registers");
1553 assert(newval != addr, "new value and addr must be in different registers");
1554 __ lock();
1555 __ cmpxchgq(newval, Address(addr, 0));
1556 } else {
1557 Unimplemented();
1558 }
1559 }
1560
1561 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1562 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1563 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1564
1565 Assembler::Condition acond, ncond;
1566 switch (condition) {
1567 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1568 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1569 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1570 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1571 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1572 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1573 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1574 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1575 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1576 ShouldNotReachHere();
1577 }
1578
1579 if (opr1->is_cpu_register()) {
1580 reg2reg(opr1, result);
2151 int offset = __ offset();
2152 switch (code) {
2153 case lir_static_call:
2154 case lir_optvirtual_call:
2155 case lir_dynamic_call:
2156 offset += NativeCall::displacement_offset;
2157 break;
2158 case lir_icvirtual_call:
2159 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2160 break;
2161 default: ShouldNotReachHere();
2162 }
2163 __ align(BytesPerWord, offset);
2164 }
2165
2166
2167 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2168 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2169 "must be aligned");
2170 __ call(AddressLiteral(op->addr(), rtype));
2171 add_call_info(code_offset(), op->info());
2172 __ post_call_nop();
2173 }
2174
2175
2176 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2177 __ ic_call(op->addr());
2178 add_call_info(code_offset(), op->info());
2179 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2180 "must be aligned");
2181 __ post_call_nop();
2182 }
2183
2184
2185 void LIR_Assembler::emit_static_call_stub() {
2186 address call_pc = __ pc();
2187 address stub = __ start_a_stub(call_stub_size());
2188 if (stub == nullptr) {
2189 bailout("static call stub overflow");
2190 return;
2191 }
2192
2193 int start = __ offset();
2194
2195 // make sure that the displacement word of the call ends up word aligned
2196 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2197 __ relocate(static_stub_Relocation::spec(call_pc));
2198 __ mov_metadata(rbx, (Metadata*)nullptr);
2325 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2326 }
2327
2328
2329 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2330 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2331 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2332 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2333 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2334 }
2335
2336
2337 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2338 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2339 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2340 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2341 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2342 }
2343
2344
2345 // This code replaces a call to arraycopy; no exception may
2346 // be thrown in this code, they must be thrown in the System.arraycopy
2347 // activation frame; we could save some checks if this would not be the case
2348 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2349 ciArrayKlass* default_type = op->expected_type();
2350 Register src = op->src()->as_register();
2351 Register dst = op->dst()->as_register();
2352 Register src_pos = op->src_pos()->as_register();
2353 Register dst_pos = op->dst_pos()->as_register();
2354 Register length = op->length()->as_register();
2355 Register tmp = op->tmp()->as_register();
2356 Register tmp_load_klass = rscratch1;
2357 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2358
2359 CodeStub* stub = op->stub();
2360 int flags = op->flags();
2361 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2362 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2363
2364 // if we don't know anything, just go through the generic arraycopy
2365 if (default_type == nullptr) {
2366 // save outgoing arguments on stack in case call to System.arraycopy is needed
2367 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2368 // for interpreter calling conventions. Now we have to do it in new style conventions.
2369 // For the moment until C1 gets the new register allocator I just force all the
2370 // args to the right place (except the register args) and then on the back side
2371 // reload the register args properly if we go slow path. Yuck
2372
2373 // These are proper for the calling convention
2374 store_parameter(length, 2);
2375 store_parameter(dst_pos, 1);
2376 store_parameter(dst, 0);
2377
2378 // these are just temporary placements until we need to reload
2379 store_parameter(src_pos, 3);
2380 store_parameter(src, 4);
2381
2382 address copyfunc_addr = StubRoutines::generic_arraycopy();
2383 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2420 __ mov(tmp, rax);
2421 __ xorl(tmp, -1);
2422
2423 // Reload values from the stack so they are where the stub
2424 // expects them.
2425 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2426 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2427 __ movptr (length, Address(rsp, 2*BytesPerWord));
2428 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2429 __ movptr (src, Address(rsp, 4*BytesPerWord));
2430
2431 __ subl(length, tmp);
2432 __ addl(src_pos, tmp);
2433 __ addl(dst_pos, tmp);
2434 __ jmp(*stub->entry());
2435
2436 __ bind(*stub->continuation());
2437 return;
2438 }
2439
2440 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2441
2442 int elem_size = type2aelembytes(basic_type);
2443 Address::ScaleFactor scale;
2444
2445 switch (elem_size) {
2446 case 1 :
2447 scale = Address::times_1;
2448 break;
2449 case 2 :
2450 scale = Address::times_2;
2451 break;
2452 case 4 :
2453 scale = Address::times_4;
2454 break;
2455 case 8 :
2456 scale = Address::times_8;
2457 break;
2458 default:
2459 scale = Address::no_scale;
2646 }
2647
2648 __ jmp(*stub->entry());
2649
2650 __ bind(cont);
2651 __ pop(dst);
2652 __ pop(src);
2653 }
2654 }
2655
2656 #ifdef ASSERT
2657 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2658 // Sanity check the known type with the incoming class. For the
2659 // primitive case the types must match exactly with src.klass and
2660 // dst.klass each exactly matching the default type. For the
2661 // object array case, if no type check is needed then either the
2662 // dst type is exactly the expected type and the src type is a
2663 // subtype which we can't check or src is the same array as dst
2664 // but not necessarily exactly of type default_type.
2665 Label known_ok, halt;
2666 __ mov_metadata(tmp, default_type->constant_encoding());
2667 if (UseCompressedClassPointers) {
2668 __ encode_klass_not_null(tmp, rscratch1);
2669 }
2670
2671 if (basic_type != T_OBJECT) {
2672 __ cmp_klass(tmp, dst, tmp2);
2673 __ jcc(Assembler::notEqual, halt);
2674 __ cmp_klass(tmp, src, tmp2);
2675 __ jcc(Assembler::equal, known_ok);
2676 } else {
2677 __ cmp_klass(tmp, dst, tmp2);
2678 __ jcc(Assembler::equal, known_ok);
2679 __ cmpptr(src, dst);
2680 __ jcc(Assembler::equal, known_ok);
2681 }
2682 __ bind(halt);
2683 __ stop("incorrect type information in arraycopy");
2684 __ bind(known_ok);
2685 }
2982 // first time here. Set profile type.
2983 __ movptr(mdo_addr, tmp);
2984 #ifdef ASSERT
2985 __ andptr(tmp, TypeEntries::type_klass_mask);
2986 __ verify_klass_ptr(tmp);
2987 #endif
2988 } else {
2989 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2990 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2991
2992 __ testptr(mdo_addr, TypeEntries::type_unknown);
2993 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2994
2995 __ orptr(mdo_addr, TypeEntries::type_unknown);
2996 }
2997 }
2998 }
2999 __ bind(next);
3000 }
3001
3002 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3003 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3004 }
3005
3006
3007 void LIR_Assembler::align_backward_branch_target() {
3008 __ align(BytesPerWord);
3009 }
3010
3011
3012 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3013 if (left->is_single_cpu()) {
3014 __ negl(left->as_register());
3015 move_regs(left->as_register(), dest->as_register());
3016
3017 } else if (left->is_double_cpu()) {
3018 Register lo = left->as_register_lo();
3019 Register dst = dest->as_register_lo();
3020 __ movptr(dst, lo);
3021 __ negptr(dst);
3167 }
3168
3169 void LIR_Assembler::membar_loadstore() {
3170 // no-op
3171 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3172 }
3173
3174 void LIR_Assembler::membar_storeload() {
3175 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3176 }
3177
3178 void LIR_Assembler::on_spin_wait() {
3179 __ pause ();
3180 }
3181
3182 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3183 assert(result_reg->is_register(), "check");
3184 __ mov(result_reg->as_register(), r15_thread);
3185 }
3186
3187
3188 void LIR_Assembler::peephole(LIR_List*) {
3189 // do nothing for now
3190 }
3191
3192 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3193 assert(data == dest, "xchg/xadd uses only 2 operands");
3194
3195 if (data->type() == T_INT) {
3196 if (code == lir_xadd) {
3197 __ lock();
3198 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3199 } else {
3200 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3201 }
3202 } else if (data->is_oop()) {
3203 assert (code == lir_xchg, "xadd for oops");
3204 Register obj = data->as_register();
3205 if (UseCompressedOops) {
3206 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
455 int offset = code_offset();
456 InternalAddress here(__ pc());
457
458 __ pushptr(here.addr(), rscratch1);
459 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
460 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
461 __ end_a_stub();
462
463 return offset;
464 }
465
466 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
467 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
468 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
469 assert(result->fpu() == 0, "result must already be on TOS");
470 }
471 if (InlineTypeReturnedAsFields) {
472 #ifndef _LP64
473 Unimplemented();
474 #endif
475 // Check if we are returning an non-null inline type and load its fields into registers
476 ciType* return_type = compilation()->method()->return_type();
477 if (return_type->is_inlinetype()) {
478 ciInlineKlass* vk = return_type->as_inline_klass();
479 if (vk->can_be_returned_as_fields()) {
480 address unpack_handler = vk->unpack_handler();
481 assert(unpack_handler != nullptr, "must be");
482 __ call(RuntimeAddress(unpack_handler));
483 }
484 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
485 Label skip;
486 Label not_null;
487 __ testptr(rax, rax);
488 __ jcc(Assembler::notZero, not_null);
489 // Returned value is null, zero all return registers because they may belong to oop fields
490 __ xorq(j_rarg1, j_rarg1);
491 __ xorq(j_rarg2, j_rarg2);
492 __ xorq(j_rarg3, j_rarg3);
493 __ xorq(j_rarg4, j_rarg4);
494 __ xorq(j_rarg5, j_rarg5);
495 __ jmp(skip);
496 __ bind(not_null);
497
498 // Check if we are returning an non-null inline type and load its fields into registers
499 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
500
501 // Load fields from a buffered value with an inline class specific handler
502 __ load_klass(rdi, rax, rscratch1);
503 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
504 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
505 // Unpack handler can be null if inline type is not scalarizable in returns
506 __ testptr(rdi, rdi);
507 __ jcc(Assembler::zero, skip);
508 __ call(rdi);
509
510 __ bind(skip);
511 }
512 // At this point, rax points to the value object (for interpreter or C1 caller).
513 // The fields of the object are copied into registers (for C2 caller).
514 }
515
516 // Pop the stack before the safepoint code
517 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
518
519 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
520 __ reserved_stack_check();
521 }
522
523 // Note: we do not need to round double result; float result has the right precision
524 // the poll sets the condition code, but no data registers
525
526 code_stub->set_safepoint_offset(__ offset());
527 __ relocate(relocInfo::poll_return_type);
528 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
529 __ ret(0);
530 }
531
532
533 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
534 return (__ store_inline_type_fields_to_buf(vk, false));
535 }
536
537 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
538 guarantee(info != nullptr, "Shouldn't be null");
539 int offset = __ offset();
540 const Register poll_addr = rscratch1;
541 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
542 add_debug_info_for_branch(info);
543 __ relocate(relocInfo::poll_type);
544 address pre_pc = __ pc();
545 __ testl(rax, Address(poll_addr, 0));
546 address post_pc = __ pc();
547 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
548 return offset;
549 }
550
551
552 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
553 if (from_reg != to_reg) __ mov(to_reg, from_reg);
554 }
555
556 void LIR_Assembler::swap_reg(Register a, Register b) {
1255 // init_state needs acquire, but x86 is TSO, and so we are already good.
1256 __ cmpb(Address(op->klass()->as_register(),
1257 InstanceKlass::init_state_offset()),
1258 InstanceKlass::fully_initialized);
1259 __ jcc(Assembler::notEqual, *op->stub()->entry());
1260 }
1261 __ allocate_object(op->obj()->as_register(),
1262 op->tmp1()->as_register(),
1263 op->tmp2()->as_register(),
1264 op->header_size(),
1265 op->object_size(),
1266 op->klass()->as_register(),
1267 *op->stub()->entry());
1268 __ bind(*op->stub()->continuation());
1269 }
1270
1271 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1272 Register len = op->len()->as_register();
1273 __ movslq(len, len);
1274
1275 if (UseSlowPath || op->always_slow_path() ||
1276 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1277 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1278 __ jmp(*op->stub()->entry());
1279 } else {
1280 Register tmp1 = op->tmp1()->as_register();
1281 Register tmp2 = op->tmp2()->as_register();
1282 Register tmp3 = op->tmp3()->as_register();
1283 if (len == tmp1) {
1284 tmp1 = tmp3;
1285 } else if (len == tmp2) {
1286 tmp2 = tmp3;
1287 } else if (len == tmp3) {
1288 // everything is ok
1289 } else {
1290 __ mov(tmp3, len);
1291 }
1292 __ allocate_array(op->obj()->as_register(),
1293 len,
1294 tmp1,
1295 tmp2,
1354 assert(data != nullptr, "need data for type check");
1355 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1356 }
1357 Label* success_target = success;
1358 Label* failure_target = failure;
1359
1360 if (obj == k_RInfo) {
1361 k_RInfo = dst;
1362 } else if (obj == klass_RInfo) {
1363 klass_RInfo = dst;
1364 }
1365 if (k->is_loaded() && !UseCompressedClassPointers) {
1366 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1367 } else {
1368 Rtmp1 = op->tmp3()->as_register();
1369 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1370 }
1371
1372 assert_different_registers(obj, k_RInfo, klass_RInfo);
1373
1374 if (op->need_null_check()) {
1375 __ testptr(obj, obj);
1376 if (op->should_profile()) {
1377 Label not_null;
1378 Register mdo = klass_RInfo;
1379 __ mov_metadata(mdo, md->constant_encoding());
1380 __ jccb(Assembler::notEqual, not_null);
1381 // Object is null; update MDO and exit
1382 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1383 int header_bits = BitData::null_seen_byte_constant();
1384 __ orb(data_addr, header_bits);
1385 __ jmp(*obj_is_null);
1386 __ bind(not_null);
1387
1388 Label update_done;
1389 Register recv = k_RInfo;
1390 __ load_klass(recv, obj, tmp_load_klass);
1391 type_profile_helper(mdo, md, data, recv, &update_done);
1392
1393 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1394 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1395
1396 __ bind(update_done);
1397 } else {
1398 __ jcc(Assembler::equal, *obj_is_null);
1399 }
1400 }
1401
1402 if (!k->is_loaded()) {
1403 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1404 } else {
1405 __ mov_metadata(k_RInfo, k->constant_encoding());
1406 }
1407 __ verify_oop(obj);
1408
1409 if (op->fast_check()) {
1410 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1411 // get object class
1412 // not a safepoint as obj null check happens earlier
1413 if (UseCompressedClassPointers) {
1414 __ load_klass(Rtmp1, obj, tmp_load_klass);
1415 __ cmpptr(k_RInfo, Rtmp1);
1416 } else {
1417 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1418 }
1419 __ jcc(Assembler::notEqual, *failure_target);
1420 // successful cast, fall through to profile or jump
1421 } else {
1422 // get object class
1423 // not a safepoint as obj null check happens earlier
1424 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1425 if (k->is_loaded()) {
1426 // See if we get an immediate positive hit
1427 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1428 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1429 __ jcc(Assembler::notEqual, *failure_target);
1430 // successful cast, fall through to profile or jump
1431 } else {
1432 // See if we get an immediate positive hit
1433 __ jcc(Assembler::equal, *success_target);
1434 // check for self
1435 if (k->is_loaded() && k->is_obj_array_klass()) {
1436 // For a direct pointer comparison, we need the refined array klass pointer
1437 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1438 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1439 __ cmpptr(klass_RInfo, tmp_load_klass);
1440 } else {
1441 __ cmpptr(klass_RInfo, k_RInfo);
1442 }
1443 __ jcc(Assembler::equal, *success_target);
1444
1445 __ push_ppx(klass_RInfo);
1446 __ push_ppx(k_RInfo);
1447 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1448 __ pop_ppx(klass_RInfo);
1449 __ pop_ppx(klass_RInfo);
1450 // result is a boolean
1451 __ testl(klass_RInfo, klass_RInfo);
1452 __ jcc(Assembler::equal, *failure_target);
1453 // successful cast, fall through to profile or jump
1454 }
1455 } else {
1456 // perform the fast part of the checking logic
1457 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1458 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1459 __ push_ppx(klass_RInfo);
1460 __ push_ppx(k_RInfo);
1461 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1462 __ pop_ppx(klass_RInfo);
1557 __ mov(dst, obj);
1558 }
1559 } else
1560 if (code == lir_instanceof) {
1561 Register obj = op->object()->as_register();
1562 Register dst = op->result_opr()->as_register();
1563 Label success, failure, done;
1564 emit_typecheck_helper(op, &success, &failure, &failure);
1565 __ bind(failure);
1566 __ xorptr(dst, dst);
1567 __ jmpb(done);
1568 __ bind(success);
1569 __ movptr(dst, 1);
1570 __ bind(done);
1571 } else {
1572 ShouldNotReachHere();
1573 }
1574
1575 }
1576
1577 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1578 // We are loading/storing from/to an array that *may* be a flat array (the
1579 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1580 // If this array is a flat array, take the slow path.
1581 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1582 if (!op->value()->is_illegal()) {
1583 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1584 // The array is not a flat array, but it might be null-free. If we are storing
1585 // a null into a null-free array, take the slow path (which will throw NPE).
1586 Label skip;
1587 __ cmpptr(op->value()->as_register(), NULL_WORD);
1588 __ jcc(Assembler::notEqual, skip);
1589 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1590 __ bind(skip);
1591 }
1592 }
1593
1594 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1595 // We are storing into an array that *may* be null-free (the declared type is
1596 // Object[], abstract[], interface[] or VT.ref[]).
1597 Label test_mark_word;
1598 Register tmp = op->tmp()->as_register();
1599 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1600 __ testl(tmp, markWord::unlocked_value);
1601 __ jccb(Assembler::notZero, test_mark_word);
1602 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1603 __ bind(test_mark_word);
1604 __ testl(tmp, markWord::null_free_array_bit_in_place);
1605 }
1606
1607 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1608 Label L_oops_equal;
1609 Label L_oops_not_equal;
1610 Label L_end;
1611
1612 Register left = op->left()->as_register();
1613 Register right = op->right()->as_register();
1614
1615 __ cmpptr(left, right);
1616 __ jcc(Assembler::equal, L_oops_equal);
1617
1618 // (1) Null check -- if one of the operands is null, the other must not be null (because
1619 // the two references are not equal), so they are not substitutable,
1620 // FIXME: do null check only if the operand is nullable
1621 __ testptr(left, right);
1622 __ jcc(Assembler::zero, L_oops_not_equal);
1623
1624 ciKlass* left_klass = op->left_klass();
1625 ciKlass* right_klass = op->right_klass();
1626
1627 // (2) Inline type check -- if either of the operands is not a inline type,
1628 // they are not substitutable. We do this only if we are not sure that the
1629 // operands are inline type
1630 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1631 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1632 Register tmp1 = op->tmp1()->as_register();
1633 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1634 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1635 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1636 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1637 __ jcc(Assembler::notEqual, L_oops_not_equal);
1638 }
1639
1640 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1641 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1642 // No need to load klass -- the operands are statically known to be the same inline klass.
1643 __ jmp(*op->stub()->entry());
1644 } else {
1645 Register left_klass_op = op->left_klass_op()->as_register();
1646 Register right_klass_op = op->right_klass_op()->as_register();
1647
1648 if (UseCompressedClassPointers) {
1649 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1650 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1651 __ cmpl(left_klass_op, right_klass_op);
1652 } else {
1653 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1654 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1655 __ cmpptr(left_klass_op, right_klass_op);
1656 }
1657
1658 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1659 // fall through to L_oops_not_equal
1660 }
1661
1662 __ bind(L_oops_not_equal);
1663 move(op->not_equal_result(), op->result_opr());
1664 __ jmp(L_end);
1665
1666 __ bind(L_oops_equal);
1667 move(op->equal_result(), op->result_opr());
1668 __ jmp(L_end);
1669
1670 // We've returned from the stub. RAX contains 0x0 IFF the two
1671 // operands are not substitutable. (Don't compare against 0x1 in case the
1672 // C compiler is naughty)
1673 __ bind(*op->stub()->continuation());
1674 __ cmpl(rax, 0);
1675 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1676 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1677 // fall-through
1678 __ bind(L_end);
1679 }
1680
1681 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1682 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1683 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1684 Register newval = op->new_value()->as_register();
1685 Register cmpval = op->cmp_value()->as_register();
1686 assert(cmpval == rax, "wrong register");
1687 assert(newval != noreg, "new val must be register");
1688 assert(cmpval != newval, "cmp and new values must be in different registers");
1689 assert(cmpval != addr, "cmp and addr must be in different registers");
1690 assert(newval != addr, "new value and addr must be in different registers");
1691
1692 if (op->code() == lir_cas_obj) {
1693 if (UseCompressedOops) {
1694 __ encode_heap_oop(cmpval);
1695 __ mov(rscratch1, newval);
1696 __ encode_heap_oop(rscratch1);
1697 __ lock();
1698 // cmpval (rax) is implicitly used by this instruction
1699 __ cmpxchgl(rscratch1, Address(addr, 0));
1705 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1706 __ lock();
1707 __ cmpxchgl(newval, Address(addr, 0));
1708 }
1709 } else if (op->code() == lir_cas_long) {
1710 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1711 Register newval = op->new_value()->as_register_lo();
1712 Register cmpval = op->cmp_value()->as_register_lo();
1713 assert(cmpval == rax, "wrong register");
1714 assert(newval != noreg, "new val must be register");
1715 assert(cmpval != newval, "cmp and new values must be in different registers");
1716 assert(cmpval != addr, "cmp and addr must be in different registers");
1717 assert(newval != addr, "new value and addr must be in different registers");
1718 __ lock();
1719 __ cmpxchgq(newval, Address(addr, 0));
1720 } else {
1721 Unimplemented();
1722 }
1723 }
1724
1725 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1726 assert(dst->is_cpu_register(), "must be");
1727 assert(dst->type() == src->type(), "must be");
1728
1729 if (src->is_cpu_register()) {
1730 reg2reg(src, dst);
1731 } else if (src->is_stack()) {
1732 stack2reg(src, dst, dst->type());
1733 } else if (src->is_constant()) {
1734 const2reg(src, dst, lir_patch_none, nullptr);
1735 } else {
1736 ShouldNotReachHere();
1737 }
1738 }
1739
1740 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1741 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1742 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1743
1744 Assembler::Condition acond, ncond;
1745 switch (condition) {
1746 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1747 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1748 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1749 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1750 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1751 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1752 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1753 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1754 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1755 ShouldNotReachHere();
1756 }
1757
1758 if (opr1->is_cpu_register()) {
1759 reg2reg(opr1, result);
2330 int offset = __ offset();
2331 switch (code) {
2332 case lir_static_call:
2333 case lir_optvirtual_call:
2334 case lir_dynamic_call:
2335 offset += NativeCall::displacement_offset;
2336 break;
2337 case lir_icvirtual_call:
2338 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2339 break;
2340 default: ShouldNotReachHere();
2341 }
2342 __ align(BytesPerWord, offset);
2343 }
2344
2345
2346 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2347 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2348 "must be aligned");
2349 __ call(AddressLiteral(op->addr(), rtype));
2350 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2351 __ post_call_nop();
2352 }
2353
2354
2355 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2356 __ ic_call(op->addr());
2357 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2358 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2359 "must be aligned");
2360 __ post_call_nop();
2361 }
2362
2363
2364 void LIR_Assembler::emit_static_call_stub() {
2365 address call_pc = __ pc();
2366 address stub = __ start_a_stub(call_stub_size());
2367 if (stub == nullptr) {
2368 bailout("static call stub overflow");
2369 return;
2370 }
2371
2372 int start = __ offset();
2373
2374 // make sure that the displacement word of the call ends up word aligned
2375 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2376 __ relocate(static_stub_Relocation::spec(call_pc));
2377 __ mov_metadata(rbx, (Metadata*)nullptr);
2504 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2505 }
2506
2507
2508 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2509 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2510 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2511 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2512 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2513 }
2514
2515
2516 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2517 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2518 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2519 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2520 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2521 }
2522
2523
2524 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2525 if (null_check) {
2526 __ testptr(obj, obj);
2527 __ jcc(Assembler::zero, *slow_path->entry());
2528 }
2529 if (is_dest) {
2530 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2531 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2532 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2533 } else {
2534 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2535 }
2536 }
2537
2538
2539 // This code replaces a call to arraycopy; no exception may
2540 // be thrown in this code, they must be thrown in the System.arraycopy
2541 // activation frame; we could save some checks if this would not be the case
2542 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2543 ciArrayKlass* default_type = op->expected_type();
2544 Register src = op->src()->as_register();
2545 Register dst = op->dst()->as_register();
2546 Register src_pos = op->src_pos()->as_register();
2547 Register dst_pos = op->dst_pos()->as_register();
2548 Register length = op->length()->as_register();
2549 Register tmp = op->tmp()->as_register();
2550 Register tmp_load_klass = rscratch1;
2551 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2552
2553 CodeStub* stub = op->stub();
2554 int flags = op->flags();
2555 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2556 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2557
2558 if (flags & LIR_OpArrayCopy::always_slow_path) {
2559 __ jmp(*stub->entry());
2560 __ bind(*stub->continuation());
2561 return;
2562 }
2563
2564 // if we don't know anything, just go through the generic arraycopy
2565 if (default_type == nullptr) {
2566 // save outgoing arguments on stack in case call to System.arraycopy is needed
2567 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2568 // for interpreter calling conventions. Now we have to do it in new style conventions.
2569 // For the moment until C1 gets the new register allocator I just force all the
2570 // args to the right place (except the register args) and then on the back side
2571 // reload the register args properly if we go slow path. Yuck
2572
2573 // These are proper for the calling convention
2574 store_parameter(length, 2);
2575 store_parameter(dst_pos, 1);
2576 store_parameter(dst, 0);
2577
2578 // these are just temporary placements until we need to reload
2579 store_parameter(src_pos, 3);
2580 store_parameter(src, 4);
2581
2582 address copyfunc_addr = StubRoutines::generic_arraycopy();
2583 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2620 __ mov(tmp, rax);
2621 __ xorl(tmp, -1);
2622
2623 // Reload values from the stack so they are where the stub
2624 // expects them.
2625 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2626 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2627 __ movptr (length, Address(rsp, 2*BytesPerWord));
2628 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2629 __ movptr (src, Address(rsp, 4*BytesPerWord));
2630
2631 __ subl(length, tmp);
2632 __ addl(src_pos, tmp);
2633 __ addl(dst_pos, tmp);
2634 __ jmp(*stub->entry());
2635
2636 __ bind(*stub->continuation());
2637 return;
2638 }
2639
2640 // Handle inline type arrays
2641 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2642 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2643 }
2644 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2645 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2646 }
2647
2648 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2649
2650 int elem_size = type2aelembytes(basic_type);
2651 Address::ScaleFactor scale;
2652
2653 switch (elem_size) {
2654 case 1 :
2655 scale = Address::times_1;
2656 break;
2657 case 2 :
2658 scale = Address::times_2;
2659 break;
2660 case 4 :
2661 scale = Address::times_4;
2662 break;
2663 case 8 :
2664 scale = Address::times_8;
2665 break;
2666 default:
2667 scale = Address::no_scale;
2854 }
2855
2856 __ jmp(*stub->entry());
2857
2858 __ bind(cont);
2859 __ pop(dst);
2860 __ pop(src);
2861 }
2862 }
2863
2864 #ifdef ASSERT
2865 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2866 // Sanity check the known type with the incoming class. For the
2867 // primitive case the types must match exactly with src.klass and
2868 // dst.klass each exactly matching the default type. For the
2869 // object array case, if no type check is needed then either the
2870 // dst type is exactly the expected type and the src type is a
2871 // subtype which we can't check or src is the same array as dst
2872 // but not necessarily exactly of type default_type.
2873 Label known_ok, halt;
2874
2875 __ mov_metadata(tmp, default_type->constant_encoding());
2876 if (UseCompressedClassPointers) {
2877 __ encode_klass_not_null(tmp, rscratch1);
2878 }
2879
2880 if (basic_type != T_OBJECT) {
2881 __ cmp_klass(tmp, dst, tmp2);
2882 __ jcc(Assembler::notEqual, halt);
2883 __ cmp_klass(tmp, src, tmp2);
2884 __ jcc(Assembler::equal, known_ok);
2885 } else {
2886 __ cmp_klass(tmp, dst, tmp2);
2887 __ jcc(Assembler::equal, known_ok);
2888 __ cmpptr(src, dst);
2889 __ jcc(Assembler::equal, known_ok);
2890 }
2891 __ bind(halt);
2892 __ stop("incorrect type information in arraycopy");
2893 __ bind(known_ok);
2894 }
3191 // first time here. Set profile type.
3192 __ movptr(mdo_addr, tmp);
3193 #ifdef ASSERT
3194 __ andptr(tmp, TypeEntries::type_klass_mask);
3195 __ verify_klass_ptr(tmp);
3196 #endif
3197 } else {
3198 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3199 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3200
3201 __ testptr(mdo_addr, TypeEntries::type_unknown);
3202 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3203
3204 __ orptr(mdo_addr, TypeEntries::type_unknown);
3205 }
3206 }
3207 }
3208 __ bind(next);
3209 }
3210
3211 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3212 Register obj = op->obj()->as_register();
3213 Register tmp = op->tmp()->as_pointer_register();
3214 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3215 bool not_null = op->not_null();
3216 int flag = op->flag();
3217
3218 Label not_inline_type;
3219 if (!not_null) {
3220 __ testptr(obj, obj);
3221 __ jccb(Assembler::zero, not_inline_type);
3222 }
3223
3224 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3225
3226 __ orb(mdo_addr, flag);
3227
3228 __ bind(not_inline_type);
3229 }
3230
3231
3232 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3233 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3234 }
3235
3236
3237 void LIR_Assembler::align_backward_branch_target() {
3238 __ align(BytesPerWord);
3239 }
3240
3241
3242 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3243 if (left->is_single_cpu()) {
3244 __ negl(left->as_register());
3245 move_regs(left->as_register(), dest->as_register());
3246
3247 } else if (left->is_double_cpu()) {
3248 Register lo = left->as_register_lo();
3249 Register dst = dest->as_register_lo();
3250 __ movptr(dst, lo);
3251 __ negptr(dst);
3397 }
3398
3399 void LIR_Assembler::membar_loadstore() {
3400 // no-op
3401 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3402 }
3403
3404 void LIR_Assembler::membar_storeload() {
3405 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3406 }
3407
3408 void LIR_Assembler::on_spin_wait() {
3409 __ pause ();
3410 }
3411
3412 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3413 assert(result_reg->is_register(), "check");
3414 __ mov(result_reg->as_register(), r15_thread);
3415 }
3416
3417 void LIR_Assembler::check_orig_pc() {
3418 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3419 }
3420
3421 void LIR_Assembler::peephole(LIR_List*) {
3422 // do nothing for now
3423 }
3424
3425 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3426 assert(data == dest, "xchg/xadd uses only 2 operands");
3427
3428 if (data->type() == T_INT) {
3429 if (code == lir_xadd) {
3430 __ lock();
3431 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3432 } else {
3433 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3434 }
3435 } else if (data->is_oop()) {
3436 assert (code == lir_xchg, "xadd for oops");
3437 Register obj = data->as_register();
3438 if (UseCompressedOops) {
3439 __ encode_heap_oop(obj);
|