14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
411 // Perform needed unlocking
412 MonitorExitStub* stub = nullptr;
413 if (method()->is_synchronized()) {
414 monitor_address(0, FrameMap::rax_opr);
415 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
416 __ unlock_object(rdi, rsi, rax, *stub->entry());
417 __ bind(*stub->continuation());
418 }
419
420 if (compilation()->env()->dtrace_method_probes()) {
421 __ mov(rdi, r15_thread);
422 __ mov_metadata(rsi, method()->constant_encoding());
423 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
424 }
425
426 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
427 __ mov(rax, rbx); // Restore the exception
428 }
429
430 // remove the activation and dispatch to the unwind handler
431 __ remove_frame(initial_frame_size_in_bytes());
432 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
433
434 // Emit the slow path assembly
435 if (stub != nullptr) {
436 stub->emit_code(this);
437 }
438
439 return offset;
440 }
441
442
443 int LIR_Assembler::emit_deopt_handler() {
444 // generate code for exception handler
445 address handler_base = __ start_a_stub(deopt_handler_size());
446 if (handler_base == nullptr) {
447 // not enough space left for the handler
448 bailout("deopt handler overflow");
449 return -1;
450 }
451
456
457 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
458
459 int entry_offset = __ offset();
460
461 __ jmp(start);
462
463 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
464 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
465 "out of bounds read in post-call NOP check");
466 __ end_a_stub();
467
468 return entry_offset;
469 }
470
471 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
472 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
473 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
474 assert(result->fpu() == 0, "result must already be on TOS");
475 }
476
477 // Pop the stack before the safepoint code
478 __ remove_frame(initial_frame_size_in_bytes());
479
480 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
481 __ reserved_stack_check();
482 }
483
484 // Note: we do not need to round double result; float result has the right precision
485 // the poll sets the condition code, but no data registers
486
487 code_stub->set_safepoint_offset(__ offset());
488 __ relocate(relocInfo::poll_return_type);
489 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
490 __ ret(0);
491 }
492
493
494 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
495 guarantee(info != nullptr, "Shouldn't be null");
496 int offset = __ offset();
497 const Register poll_addr = rscratch1;
498 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
499 add_debug_info_for_branch(info);
500 __ relocate(relocInfo::poll_type);
501 address pre_pc = __ pc();
502 __ testl(rax, Address(poll_addr, 0));
503 address post_pc = __ pc();
504 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
505 return offset;
506 }
507
508
509 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
510 if (from_reg != to_reg) __ mov(to_reg, from_reg);
511 }
512
513 void LIR_Assembler::swap_reg(Register a, Register b) {
1212 // init_state needs acquire, but x86 is TSO, and so we are already good.
1213 __ cmpb(Address(op->klass()->as_register(),
1214 InstanceKlass::init_state_offset()),
1215 InstanceKlass::fully_initialized);
1216 __ jcc(Assembler::notEqual, *op->stub()->entry());
1217 }
1218 __ allocate_object(op->obj()->as_register(),
1219 op->tmp1()->as_register(),
1220 op->tmp2()->as_register(),
1221 op->header_size(),
1222 op->object_size(),
1223 op->klass()->as_register(),
1224 *op->stub()->entry());
1225 __ bind(*op->stub()->continuation());
1226 }
1227
1228 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1229 Register len = op->len()->as_register();
1230 __ movslq(len, len);
1231
1232 if (UseSlowPath ||
1233 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1234 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1235 __ jmp(*op->stub()->entry());
1236 } else {
1237 Register tmp1 = op->tmp1()->as_register();
1238 Register tmp2 = op->tmp2()->as_register();
1239 Register tmp3 = op->tmp3()->as_register();
1240 if (len == tmp1) {
1241 tmp1 = tmp3;
1242 } else if (len == tmp2) {
1243 tmp2 = tmp3;
1244 } else if (len == tmp3) {
1245 // everything is ok
1246 } else {
1247 __ mov(tmp3, len);
1248 }
1249 __ allocate_array(op->obj()->as_register(),
1250 len,
1251 tmp1,
1252 tmp2,
1311 assert(data != nullptr, "need data for type check");
1312 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1313 }
1314 Label* success_target = success;
1315 Label* failure_target = failure;
1316
1317 if (obj == k_RInfo) {
1318 k_RInfo = dst;
1319 } else if (obj == klass_RInfo) {
1320 klass_RInfo = dst;
1321 }
1322 if (k->is_loaded() && !UseCompressedClassPointers) {
1323 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1324 } else {
1325 Rtmp1 = op->tmp3()->as_register();
1326 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1327 }
1328
1329 assert_different_registers(obj, k_RInfo, klass_RInfo);
1330
1331 __ testptr(obj, obj);
1332 if (op->should_profile()) {
1333 Label not_null;
1334 Register mdo = klass_RInfo;
1335 __ mov_metadata(mdo, md->constant_encoding());
1336 __ jccb(Assembler::notEqual, not_null);
1337 // Object is null; update MDO and exit
1338 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1339 int header_bits = BitData::null_seen_byte_constant();
1340 __ orb(data_addr, header_bits);
1341 __ jmp(*obj_is_null);
1342 __ bind(not_null);
1343
1344 Label update_done;
1345 Register recv = k_RInfo;
1346 __ load_klass(recv, obj, tmp_load_klass);
1347 type_profile_helper(mdo, md, data, recv, &update_done);
1348
1349 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1350 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1351
1352 __ bind(update_done);
1353 } else {
1354 __ jcc(Assembler::equal, *obj_is_null);
1355 }
1356
1357 if (!k->is_loaded()) {
1358 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1359 } else {
1360 __ mov_metadata(k_RInfo, k->constant_encoding());
1361 }
1362 __ verify_oop(obj);
1363
1364 if (op->fast_check()) {
1365 // get object class
1366 // not a safepoint as obj null check happens earlier
1367 if (UseCompressedClassPointers) {
1368 __ load_klass(Rtmp1, obj, tmp_load_klass);
1369 __ cmpptr(k_RInfo, Rtmp1);
1370 } else {
1371 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1372 }
1373 __ jcc(Assembler::notEqual, *failure_target);
1374 // successful cast, fall through to profile or jump
1375 } else {
1376 // get object class
1377 // not a safepoint as obj null check happens earlier
1378 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1379 if (k->is_loaded()) {
1380 // See if we get an immediate positive hit
1381 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1382 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1383 __ jcc(Assembler::notEqual, *failure_target);
1384 // successful cast, fall through to profile or jump
1385 } else {
1386 // See if we get an immediate positive hit
1387 __ jcc(Assembler::equal, *success_target);
1388 // check for self
1389 __ cmpptr(klass_RInfo, k_RInfo);
1390 __ jcc(Assembler::equal, *success_target);
1391
1392 __ push_ppx(klass_RInfo);
1393 __ push_ppx(k_RInfo);
1394 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1395 __ pop_ppx(klass_RInfo);
1396 __ pop_ppx(klass_RInfo);
1397 // result is a boolean
1398 __ testl(klass_RInfo, klass_RInfo);
1399 __ jcc(Assembler::equal, *failure_target);
1400 // successful cast, fall through to profile or jump
1401 }
1402 } else {
1403 // perform the fast part of the checking logic
1404 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1405 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1406 __ push_ppx(klass_RInfo);
1407 __ push_ppx(k_RInfo);
1408 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1409 __ pop_ppx(klass_RInfo);
1504 __ mov(dst, obj);
1505 }
1506 } else
1507 if (code == lir_instanceof) {
1508 Register obj = op->object()->as_register();
1509 Register dst = op->result_opr()->as_register();
1510 Label success, failure, done;
1511 emit_typecheck_helper(op, &success, &failure, &failure);
1512 __ bind(failure);
1513 __ xorptr(dst, dst);
1514 __ jmpb(done);
1515 __ bind(success);
1516 __ movptr(dst, 1);
1517 __ bind(done);
1518 } else {
1519 ShouldNotReachHere();
1520 }
1521
1522 }
1523
1524
1525 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1526 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1527 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1528 Register newval = op->new_value()->as_register();
1529 Register cmpval = op->cmp_value()->as_register();
1530 assert(cmpval == rax, "wrong register");
1531 assert(newval != noreg, "new val must be register");
1532 assert(cmpval != newval, "cmp and new values must be in different registers");
1533 assert(cmpval != addr, "cmp and addr must be in different registers");
1534 assert(newval != addr, "new value and addr must be in different registers");
1535
1536 if (op->code() == lir_cas_obj) {
1537 if (UseCompressedOops) {
1538 __ encode_heap_oop(cmpval);
1539 __ mov(rscratch1, newval);
1540 __ encode_heap_oop(rscratch1);
1541 __ lock();
1542 // cmpval (rax) is implicitly used by this instruction
1543 __ cmpxchgl(rscratch1, Address(addr, 0));
1549 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1550 __ lock();
1551 __ cmpxchgl(newval, Address(addr, 0));
1552 }
1553 } else if (op->code() == lir_cas_long) {
1554 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1555 Register newval = op->new_value()->as_register_lo();
1556 Register cmpval = op->cmp_value()->as_register_lo();
1557 assert(cmpval == rax, "wrong register");
1558 assert(newval != noreg, "new val must be register");
1559 assert(cmpval != newval, "cmp and new values must be in different registers");
1560 assert(cmpval != addr, "cmp and addr must be in different registers");
1561 assert(newval != addr, "new value and addr must be in different registers");
1562 __ lock();
1563 __ cmpxchgq(newval, Address(addr, 0));
1564 } else {
1565 Unimplemented();
1566 }
1567 }
1568
1569 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1570 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1571 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1572
1573 Assembler::Condition acond, ncond;
1574 switch (condition) {
1575 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1576 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1577 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1578 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1579 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1580 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1581 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1582 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1583 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1584 ShouldNotReachHere();
1585 }
1586
1587 if (opr1->is_cpu_register()) {
1588 reg2reg(opr1, result);
2159 int offset = __ offset();
2160 switch (code) {
2161 case lir_static_call:
2162 case lir_optvirtual_call:
2163 case lir_dynamic_call:
2164 offset += NativeCall::displacement_offset;
2165 break;
2166 case lir_icvirtual_call:
2167 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2168 break;
2169 default: ShouldNotReachHere();
2170 }
2171 __ align(BytesPerWord, offset);
2172 }
2173
2174
2175 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2176 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2177 "must be aligned");
2178 __ call(AddressLiteral(op->addr(), rtype));
2179 add_call_info(code_offset(), op->info());
2180 __ post_call_nop();
2181 }
2182
2183
2184 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2185 __ ic_call(op->addr());
2186 add_call_info(code_offset(), op->info());
2187 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2188 "must be aligned");
2189 __ post_call_nop();
2190 }
2191
2192
2193 void LIR_Assembler::emit_static_call_stub() {
2194 address call_pc = __ pc();
2195 address stub = __ start_a_stub(call_stub_size());
2196 if (stub == nullptr) {
2197 bailout("static call stub overflow");
2198 return;
2199 }
2200
2201 int start = __ offset();
2202
2203 // make sure that the displacement word of the call ends up word aligned
2204 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2205 __ relocate(static_stub_Relocation::spec(call_pc));
2206 __ mov_metadata(rbx, (Metadata*)nullptr);
2333 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2334 }
2335
2336
2337 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2338 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2339 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2340 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2341 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2342 }
2343
2344
2345 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2346 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2347 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2348 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2349 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2350 }
2351
2352
2353 // This code replaces a call to arraycopy; no exception may
2354 // be thrown in this code, they must be thrown in the System.arraycopy
2355 // activation frame; we could save some checks if this would not be the case
2356 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2357 ciArrayKlass* default_type = op->expected_type();
2358 Register src = op->src()->as_register();
2359 Register dst = op->dst()->as_register();
2360 Register src_pos = op->src_pos()->as_register();
2361 Register dst_pos = op->dst_pos()->as_register();
2362 Register length = op->length()->as_register();
2363 Register tmp = op->tmp()->as_register();
2364 Register tmp_load_klass = rscratch1;
2365 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2366
2367 CodeStub* stub = op->stub();
2368 int flags = op->flags();
2369 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2370 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2371
2372 // if we don't know anything, just go through the generic arraycopy
2373 if (default_type == nullptr) {
2374 // save outgoing arguments on stack in case call to System.arraycopy is needed
2375 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2376 // for interpreter calling conventions. Now we have to do it in new style conventions.
2377 // For the moment until C1 gets the new register allocator I just force all the
2378 // args to the right place (except the register args) and then on the back side
2379 // reload the register args properly if we go slow path. Yuck
2380
2381 // These are proper for the calling convention
2382 store_parameter(length, 2);
2383 store_parameter(dst_pos, 1);
2384 store_parameter(dst, 0);
2385
2386 // these are just temporary placements until we need to reload
2387 store_parameter(src_pos, 3);
2388 store_parameter(src, 4);
2389
2390 address copyfunc_addr = StubRoutines::generic_arraycopy();
2391 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2428 __ mov(tmp, rax);
2429 __ xorl(tmp, -1);
2430
2431 // Reload values from the stack so they are where the stub
2432 // expects them.
2433 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2434 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2435 __ movptr (length, Address(rsp, 2*BytesPerWord));
2436 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2437 __ movptr (src, Address(rsp, 4*BytesPerWord));
2438
2439 __ subl(length, tmp);
2440 __ addl(src_pos, tmp);
2441 __ addl(dst_pos, tmp);
2442 __ jmp(*stub->entry());
2443
2444 __ bind(*stub->continuation());
2445 return;
2446 }
2447
2448 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2449
2450 int elem_size = type2aelembytes(basic_type);
2451 Address::ScaleFactor scale;
2452
2453 switch (elem_size) {
2454 case 1 :
2455 scale = Address::times_1;
2456 break;
2457 case 2 :
2458 scale = Address::times_2;
2459 break;
2460 case 4 :
2461 scale = Address::times_4;
2462 break;
2463 case 8 :
2464 scale = Address::times_8;
2465 break;
2466 default:
2467 scale = Address::no_scale;
2654 }
2655
2656 __ jmp(*stub->entry());
2657
2658 __ bind(cont);
2659 __ pop(dst);
2660 __ pop(src);
2661 }
2662 }
2663
2664 #ifdef ASSERT
2665 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2666 // Sanity check the known type with the incoming class. For the
2667 // primitive case the types must match exactly with src.klass and
2668 // dst.klass each exactly matching the default type. For the
2669 // object array case, if no type check is needed then either the
2670 // dst type is exactly the expected type and the src type is a
2671 // subtype which we can't check or src is the same array as dst
2672 // but not necessarily exactly of type default_type.
2673 Label known_ok, halt;
2674 __ mov_metadata(tmp, default_type->constant_encoding());
2675 if (UseCompressedClassPointers) {
2676 __ encode_klass_not_null(tmp, rscratch1);
2677 }
2678
2679 if (basic_type != T_OBJECT) {
2680 __ cmp_klass(tmp, dst, tmp2);
2681 __ jcc(Assembler::notEqual, halt);
2682 __ cmp_klass(tmp, src, tmp2);
2683 __ jcc(Assembler::equal, known_ok);
2684 } else {
2685 __ cmp_klass(tmp, dst, tmp2);
2686 __ jcc(Assembler::equal, known_ok);
2687 __ cmpptr(src, dst);
2688 __ jcc(Assembler::equal, known_ok);
2689 }
2690 __ bind(halt);
2691 __ stop("incorrect type information in arraycopy");
2692 __ bind(known_ok);
2693 }
2990 // first time here. Set profile type.
2991 __ movptr(mdo_addr, tmp);
2992 #ifdef ASSERT
2993 __ andptr(tmp, TypeEntries::type_klass_mask);
2994 __ verify_klass_ptr(tmp);
2995 #endif
2996 } else {
2997 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2998 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2999
3000 __ testptr(mdo_addr, TypeEntries::type_unknown);
3001 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3002
3003 __ orptr(mdo_addr, TypeEntries::type_unknown);
3004 }
3005 }
3006 }
3007 __ bind(next);
3008 }
3009
3010 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3011 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3012 }
3013
3014
3015 void LIR_Assembler::align_backward_branch_target() {
3016 __ align(BytesPerWord);
3017 }
3018
3019
3020 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3021 if (left->is_single_cpu()) {
3022 __ negl(left->as_register());
3023 move_regs(left->as_register(), dest->as_register());
3024
3025 } else if (left->is_double_cpu()) {
3026 Register lo = left->as_register_lo();
3027 Register dst = dest->as_register_lo();
3028 __ movptr(dst, lo);
3029 __ negptr(dst);
3175 }
3176
3177 void LIR_Assembler::membar_loadstore() {
3178 // no-op
3179 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3180 }
3181
3182 void LIR_Assembler::membar_storeload() {
3183 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3184 }
3185
3186 void LIR_Assembler::on_spin_wait() {
3187 __ pause ();
3188 }
3189
3190 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3191 assert(result_reg->is_register(), "check");
3192 __ mov(result_reg->as_register(), r15_thread);
3193 }
3194
3195
3196 void LIR_Assembler::peephole(LIR_List*) {
3197 // do nothing for now
3198 }
3199
3200 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3201 assert(data == dest, "xchg/xadd uses only 2 operands");
3202
3203 if (data->type() == T_INT) {
3204 if (code == lir_xadd) {
3205 __ lock();
3206 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3207 } else {
3208 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3209 }
3210 } else if (data->is_oop()) {
3211 assert (code == lir_xchg, "xadd for oops");
3212 Register obj = data->as_register();
3213 if (UseCompressedOops) {
3214 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
459
460 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
461
462 int entry_offset = __ offset();
463
464 __ jmp(start);
465
466 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
467 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
468 "out of bounds read in post-call NOP check");
469 __ end_a_stub();
470
471 return entry_offset;
472 }
473
474 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
475 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
476 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
477 assert(result->fpu() == 0, "result must already be on TOS");
478 }
479 if (InlineTypeReturnedAsFields) {
480 #ifndef _LP64
481 Unimplemented();
482 #endif
483 // Check if we are returning an non-null inline type and load its fields into registers
484 ciType* return_type = compilation()->method()->return_type();
485 if (return_type->is_inlinetype()) {
486 ciInlineKlass* vk = return_type->as_inline_klass();
487 if (vk->can_be_returned_as_fields()) {
488 address unpack_handler = vk->unpack_handler();
489 assert(unpack_handler != nullptr, "must be");
490 __ call(RuntimeAddress(unpack_handler));
491 }
492 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
493 Label skip;
494 Label not_null;
495 __ testptr(rax, rax);
496 __ jcc(Assembler::notZero, not_null);
497 // Returned value is null, zero all return registers because they may belong to oop fields
498 __ xorq(j_rarg1, j_rarg1);
499 __ xorq(j_rarg2, j_rarg2);
500 __ xorq(j_rarg3, j_rarg3);
501 __ xorq(j_rarg4, j_rarg4);
502 __ xorq(j_rarg5, j_rarg5);
503 __ jmp(skip);
504 __ bind(not_null);
505
506 // Check if we are returning an non-null inline type and load its fields into registers
507 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
508
509 // Load fields from a buffered value with an inline class specific handler
510 __ load_klass(rdi, rax, rscratch1);
511 __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
512 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
513 // Unpack handler can be null if inline type is not scalarizable in returns
514 __ testptr(rdi, rdi);
515 __ jcc(Assembler::zero, skip);
516 __ call(rdi);
517
518 __ bind(skip);
519 }
520 // At this point, rax points to the value object (for interpreter or C1 caller).
521 // The fields of the object are copied into registers (for C2 caller).
522 }
523
524 // Pop the stack before the safepoint code
525 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
526
527 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
528 __ reserved_stack_check();
529 }
530
531 // Note: we do not need to round double result; float result has the right precision
532 // the poll sets the condition code, but no data registers
533
534 code_stub->set_safepoint_offset(__ offset());
535 __ relocate(relocInfo::poll_return_type);
536 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
537 __ ret(0);
538 }
539
540
541 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
542 return (__ store_inline_type_fields_to_buf(vk, false));
543 }
544
545 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
546 guarantee(info != nullptr, "Shouldn't be null");
547 int offset = __ offset();
548 const Register poll_addr = rscratch1;
549 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
550 add_debug_info_for_branch(info);
551 __ relocate(relocInfo::poll_type);
552 address pre_pc = __ pc();
553 __ testl(rax, Address(poll_addr, 0));
554 address post_pc = __ pc();
555 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
556 return offset;
557 }
558
559
560 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
561 if (from_reg != to_reg) __ mov(to_reg, from_reg);
562 }
563
564 void LIR_Assembler::swap_reg(Register a, Register b) {
1263 // init_state needs acquire, but x86 is TSO, and so we are already good.
1264 __ cmpb(Address(op->klass()->as_register(),
1265 InstanceKlass::init_state_offset()),
1266 InstanceKlass::fully_initialized);
1267 __ jcc(Assembler::notEqual, *op->stub()->entry());
1268 }
1269 __ allocate_object(op->obj()->as_register(),
1270 op->tmp1()->as_register(),
1271 op->tmp2()->as_register(),
1272 op->header_size(),
1273 op->object_size(),
1274 op->klass()->as_register(),
1275 *op->stub()->entry());
1276 __ bind(*op->stub()->continuation());
1277 }
1278
1279 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1280 Register len = op->len()->as_register();
1281 __ movslq(len, len);
1282
1283 if (UseSlowPath || op->always_slow_path() ||
1284 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1285 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1286 __ jmp(*op->stub()->entry());
1287 } else {
1288 Register tmp1 = op->tmp1()->as_register();
1289 Register tmp2 = op->tmp2()->as_register();
1290 Register tmp3 = op->tmp3()->as_register();
1291 if (len == tmp1) {
1292 tmp1 = tmp3;
1293 } else if (len == tmp2) {
1294 tmp2 = tmp3;
1295 } else if (len == tmp3) {
1296 // everything is ok
1297 } else {
1298 __ mov(tmp3, len);
1299 }
1300 __ allocate_array(op->obj()->as_register(),
1301 len,
1302 tmp1,
1303 tmp2,
1362 assert(data != nullptr, "need data for type check");
1363 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1364 }
1365 Label* success_target = success;
1366 Label* failure_target = failure;
1367
1368 if (obj == k_RInfo) {
1369 k_RInfo = dst;
1370 } else if (obj == klass_RInfo) {
1371 klass_RInfo = dst;
1372 }
1373 if (k->is_loaded() && !UseCompressedClassPointers) {
1374 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1375 } else {
1376 Rtmp1 = op->tmp3()->as_register();
1377 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1378 }
1379
1380 assert_different_registers(obj, k_RInfo, klass_RInfo);
1381
1382 if (op->need_null_check()) {
1383 __ testptr(obj, obj);
1384 if (op->should_profile()) {
1385 Label not_null;
1386 Register mdo = klass_RInfo;
1387 __ mov_metadata(mdo, md->constant_encoding());
1388 __ jccb(Assembler::notEqual, not_null);
1389 // Object is null; update MDO and exit
1390 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1391 int header_bits = BitData::null_seen_byte_constant();
1392 __ orb(data_addr, header_bits);
1393 __ jmp(*obj_is_null);
1394 __ bind(not_null);
1395
1396 Label update_done;
1397 Register recv = k_RInfo;
1398 __ load_klass(recv, obj, tmp_load_klass);
1399 type_profile_helper(mdo, md, data, recv, &update_done);
1400
1401 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1402 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1403
1404 __ bind(update_done);
1405 } else {
1406 __ jcc(Assembler::equal, *obj_is_null);
1407 }
1408 }
1409
1410 if (!k->is_loaded()) {
1411 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1412 } else {
1413 __ mov_metadata(k_RInfo, k->constant_encoding());
1414 }
1415 __ verify_oop(obj);
1416
1417 if (op->fast_check()) {
1418 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1419 // get object class
1420 // not a safepoint as obj null check happens earlier
1421 if (UseCompressedClassPointers) {
1422 __ load_klass(Rtmp1, obj, tmp_load_klass);
1423 __ cmpptr(k_RInfo, Rtmp1);
1424 } else {
1425 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1426 }
1427 __ jcc(Assembler::notEqual, *failure_target);
1428 // successful cast, fall through to profile or jump
1429 } else {
1430 // get object class
1431 // not a safepoint as obj null check happens earlier
1432 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1433 if (k->is_loaded()) {
1434 // See if we get an immediate positive hit
1435 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1436 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1437 __ jcc(Assembler::notEqual, *failure_target);
1438 // successful cast, fall through to profile or jump
1439 } else {
1440 // See if we get an immediate positive hit
1441 __ jcc(Assembler::equal, *success_target);
1442 // check for self
1443 if (k->is_loaded() && k->is_obj_array_klass()) {
1444 // For a direct pointer comparison, we need the refined array klass pointer
1445 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1446 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1447 __ cmpptr(klass_RInfo, tmp_load_klass);
1448 } else {
1449 __ cmpptr(klass_RInfo, k_RInfo);
1450 }
1451 __ jcc(Assembler::equal, *success_target);
1452
1453 __ push_ppx(klass_RInfo);
1454 __ push_ppx(k_RInfo);
1455 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1456 __ pop_ppx(klass_RInfo);
1457 __ pop_ppx(klass_RInfo);
1458 // result is a boolean
1459 __ testl(klass_RInfo, klass_RInfo);
1460 __ jcc(Assembler::equal, *failure_target);
1461 // successful cast, fall through to profile or jump
1462 }
1463 } else {
1464 // perform the fast part of the checking logic
1465 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1466 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1467 __ push_ppx(klass_RInfo);
1468 __ push_ppx(k_RInfo);
1469 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1470 __ pop_ppx(klass_RInfo);
1565 __ mov(dst, obj);
1566 }
1567 } else
1568 if (code == lir_instanceof) {
1569 Register obj = op->object()->as_register();
1570 Register dst = op->result_opr()->as_register();
1571 Label success, failure, done;
1572 emit_typecheck_helper(op, &success, &failure, &failure);
1573 __ bind(failure);
1574 __ xorptr(dst, dst);
1575 __ jmpb(done);
1576 __ bind(success);
1577 __ movptr(dst, 1);
1578 __ bind(done);
1579 } else {
1580 ShouldNotReachHere();
1581 }
1582
1583 }
1584
1585 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1586 // We are loading/storing from/to an array that *may* be a flat array (the
1587 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1588 // If this array is a flat array, take the slow path.
1589 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1590 if (!op->value()->is_illegal()) {
1591 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1592 // The array is not a flat array, but it might be null-free. If we are storing
1593 // a null into a null-free array, take the slow path (which will throw NPE).
1594 Label skip;
1595 __ cmpptr(op->value()->as_register(), NULL_WORD);
1596 __ jcc(Assembler::notEqual, skip);
1597 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1598 __ bind(skip);
1599 }
1600 }
1601
1602 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1603 // We are storing into an array that *may* be null-free (the declared type is
1604 // Object[], abstract[], interface[] or VT.ref[]).
1605 Label test_mark_word;
1606 Register tmp = op->tmp()->as_register();
1607 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1608 __ testl(tmp, markWord::unlocked_value);
1609 __ jccb(Assembler::notZero, test_mark_word);
1610 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1611 __ bind(test_mark_word);
1612 __ testl(tmp, markWord::null_free_array_bit_in_place);
1613 }
1614
1615 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1616 Label L_oops_equal;
1617 Label L_oops_not_equal;
1618 Label L_end;
1619
1620 Register left = op->left()->as_register();
1621 Register right = op->right()->as_register();
1622
1623 __ cmpptr(left, right);
1624 __ jcc(Assembler::equal, L_oops_equal);
1625
1626 // (1) Null check -- if one of the operands is null, the other must not be null (because
1627 // the two references are not equal), so they are not substitutable,
1628 // FIXME: do null check only if the operand is nullable
1629 __ testptr(left, right);
1630 __ jcc(Assembler::zero, L_oops_not_equal);
1631
1632 ciKlass* left_klass = op->left_klass();
1633 ciKlass* right_klass = op->right_klass();
1634
1635 // (2) Inline type check -- if either of the operands is not a inline type,
1636 // they are not substitutable. We do this only if we are not sure that the
1637 // operands are inline type
1638 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1639 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1640 Register tmp1 = op->tmp1()->as_register();
1641 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1642 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1643 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1644 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1645 __ jcc(Assembler::notEqual, L_oops_not_equal);
1646 }
1647
1648 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1649 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1650 // No need to load klass -- the operands are statically known to be the same inline klass.
1651 __ jmp(*op->stub()->entry());
1652 } else {
1653 Register left_klass_op = op->left_klass_op()->as_register();
1654 Register right_klass_op = op->right_klass_op()->as_register();
1655
1656 if (UseCompressedClassPointers) {
1657 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1658 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1659 __ cmpl(left_klass_op, right_klass_op);
1660 } else {
1661 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1662 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1663 __ cmpptr(left_klass_op, right_klass_op);
1664 }
1665
1666 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1667 // fall through to L_oops_not_equal
1668 }
1669
1670 __ bind(L_oops_not_equal);
1671 move(op->not_equal_result(), op->result_opr());
1672 __ jmp(L_end);
1673
1674 __ bind(L_oops_equal);
1675 move(op->equal_result(), op->result_opr());
1676 __ jmp(L_end);
1677
1678 // We've returned from the stub. RAX contains 0x0 IFF the two
1679 // operands are not substitutable. (Don't compare against 0x1 in case the
1680 // C compiler is naughty)
1681 __ bind(*op->stub()->continuation());
1682 __ cmpl(rax, 0);
1683 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1684 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1685 // fall-through
1686 __ bind(L_end);
1687 }
1688
1689 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1690 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1691 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1692 Register newval = op->new_value()->as_register();
1693 Register cmpval = op->cmp_value()->as_register();
1694 assert(cmpval == rax, "wrong register");
1695 assert(newval != noreg, "new val must be register");
1696 assert(cmpval != newval, "cmp and new values must be in different registers");
1697 assert(cmpval != addr, "cmp and addr must be in different registers");
1698 assert(newval != addr, "new value and addr must be in different registers");
1699
1700 if (op->code() == lir_cas_obj) {
1701 if (UseCompressedOops) {
1702 __ encode_heap_oop(cmpval);
1703 __ mov(rscratch1, newval);
1704 __ encode_heap_oop(rscratch1);
1705 __ lock();
1706 // cmpval (rax) is implicitly used by this instruction
1707 __ cmpxchgl(rscratch1, Address(addr, 0));
1713 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1714 __ lock();
1715 __ cmpxchgl(newval, Address(addr, 0));
1716 }
1717 } else if (op->code() == lir_cas_long) {
1718 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1719 Register newval = op->new_value()->as_register_lo();
1720 Register cmpval = op->cmp_value()->as_register_lo();
1721 assert(cmpval == rax, "wrong register");
1722 assert(newval != noreg, "new val must be register");
1723 assert(cmpval != newval, "cmp and new values must be in different registers");
1724 assert(cmpval != addr, "cmp and addr must be in different registers");
1725 assert(newval != addr, "new value and addr must be in different registers");
1726 __ lock();
1727 __ cmpxchgq(newval, Address(addr, 0));
1728 } else {
1729 Unimplemented();
1730 }
1731 }
1732
1733 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1734 assert(dst->is_cpu_register(), "must be");
1735 assert(dst->type() == src->type(), "must be");
1736
1737 if (src->is_cpu_register()) {
1738 reg2reg(src, dst);
1739 } else if (src->is_stack()) {
1740 stack2reg(src, dst, dst->type());
1741 } else if (src->is_constant()) {
1742 const2reg(src, dst, lir_patch_none, nullptr);
1743 } else {
1744 ShouldNotReachHere();
1745 }
1746 }
1747
1748 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1749 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1750 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1751
1752 Assembler::Condition acond, ncond;
1753 switch (condition) {
1754 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1755 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1756 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1757 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1758 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1759 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1760 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1761 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1762 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1763 ShouldNotReachHere();
1764 }
1765
1766 if (opr1->is_cpu_register()) {
1767 reg2reg(opr1, result);
2338 int offset = __ offset();
2339 switch (code) {
2340 case lir_static_call:
2341 case lir_optvirtual_call:
2342 case lir_dynamic_call:
2343 offset += NativeCall::displacement_offset;
2344 break;
2345 case lir_icvirtual_call:
2346 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2347 break;
2348 default: ShouldNotReachHere();
2349 }
2350 __ align(BytesPerWord, offset);
2351 }
2352
2353
2354 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2355 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2356 "must be aligned");
2357 __ call(AddressLiteral(op->addr(), rtype));
2358 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2359 __ post_call_nop();
2360 }
2361
2362
2363 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2364 __ ic_call(op->addr());
2365 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2366 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2367 "must be aligned");
2368 __ post_call_nop();
2369 }
2370
2371
2372 void LIR_Assembler::emit_static_call_stub() {
2373 address call_pc = __ pc();
2374 address stub = __ start_a_stub(call_stub_size());
2375 if (stub == nullptr) {
2376 bailout("static call stub overflow");
2377 return;
2378 }
2379
2380 int start = __ offset();
2381
2382 // make sure that the displacement word of the call ends up word aligned
2383 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2384 __ relocate(static_stub_Relocation::spec(call_pc));
2385 __ mov_metadata(rbx, (Metadata*)nullptr);
2512 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2513 }
2514
2515
2516 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2517 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2518 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2519 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2520 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2521 }
2522
2523
2524 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2525 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2526 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2527 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2528 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2529 }
2530
2531
2532 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2533 if (null_check) {
2534 __ testptr(obj, obj);
2535 __ jcc(Assembler::zero, *slow_path->entry());
2536 }
2537 if (is_dest) {
2538 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2539 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2540 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2541 } else {
2542 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2543 }
2544 }
2545
2546
2547 // This code replaces a call to arraycopy; no exception may
2548 // be thrown in this code, they must be thrown in the System.arraycopy
2549 // activation frame; we could save some checks if this would not be the case
2550 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2551 ciArrayKlass* default_type = op->expected_type();
2552 Register src = op->src()->as_register();
2553 Register dst = op->dst()->as_register();
2554 Register src_pos = op->src_pos()->as_register();
2555 Register dst_pos = op->dst_pos()->as_register();
2556 Register length = op->length()->as_register();
2557 Register tmp = op->tmp()->as_register();
2558 Register tmp_load_klass = rscratch1;
2559 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2560
2561 CodeStub* stub = op->stub();
2562 int flags = op->flags();
2563 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2564 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2565
2566 if (flags & LIR_OpArrayCopy::always_slow_path) {
2567 __ jmp(*stub->entry());
2568 __ bind(*stub->continuation());
2569 return;
2570 }
2571
2572 // if we don't know anything, just go through the generic arraycopy
2573 if (default_type == nullptr) {
2574 // save outgoing arguments on stack in case call to System.arraycopy is needed
2575 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2576 // for interpreter calling conventions. Now we have to do it in new style conventions.
2577 // For the moment until C1 gets the new register allocator I just force all the
2578 // args to the right place (except the register args) and then on the back side
2579 // reload the register args properly if we go slow path. Yuck
2580
2581 // These are proper for the calling convention
2582 store_parameter(length, 2);
2583 store_parameter(dst_pos, 1);
2584 store_parameter(dst, 0);
2585
2586 // these are just temporary placements until we need to reload
2587 store_parameter(src_pos, 3);
2588 store_parameter(src, 4);
2589
2590 address copyfunc_addr = StubRoutines::generic_arraycopy();
2591 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2628 __ mov(tmp, rax);
2629 __ xorl(tmp, -1);
2630
2631 // Reload values from the stack so they are where the stub
2632 // expects them.
2633 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2634 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2635 __ movptr (length, Address(rsp, 2*BytesPerWord));
2636 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2637 __ movptr (src, Address(rsp, 4*BytesPerWord));
2638
2639 __ subl(length, tmp);
2640 __ addl(src_pos, tmp);
2641 __ addl(dst_pos, tmp);
2642 __ jmp(*stub->entry());
2643
2644 __ bind(*stub->continuation());
2645 return;
2646 }
2647
2648 // Handle inline type arrays
2649 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2650 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2651 }
2652 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2653 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2654 }
2655
2656 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2657
2658 int elem_size = type2aelembytes(basic_type);
2659 Address::ScaleFactor scale;
2660
2661 switch (elem_size) {
2662 case 1 :
2663 scale = Address::times_1;
2664 break;
2665 case 2 :
2666 scale = Address::times_2;
2667 break;
2668 case 4 :
2669 scale = Address::times_4;
2670 break;
2671 case 8 :
2672 scale = Address::times_8;
2673 break;
2674 default:
2675 scale = Address::no_scale;
2862 }
2863
2864 __ jmp(*stub->entry());
2865
2866 __ bind(cont);
2867 __ pop(dst);
2868 __ pop(src);
2869 }
2870 }
2871
2872 #ifdef ASSERT
2873 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2874 // Sanity check the known type with the incoming class. For the
2875 // primitive case the types must match exactly with src.klass and
2876 // dst.klass each exactly matching the default type. For the
2877 // object array case, if no type check is needed then either the
2878 // dst type is exactly the expected type and the src type is a
2879 // subtype which we can't check or src is the same array as dst
2880 // but not necessarily exactly of type default_type.
2881 Label known_ok, halt;
2882
2883 __ mov_metadata(tmp, default_type->constant_encoding());
2884 if (UseCompressedClassPointers) {
2885 __ encode_klass_not_null(tmp, rscratch1);
2886 }
2887
2888 if (basic_type != T_OBJECT) {
2889 __ cmp_klass(tmp, dst, tmp2);
2890 __ jcc(Assembler::notEqual, halt);
2891 __ cmp_klass(tmp, src, tmp2);
2892 __ jcc(Assembler::equal, known_ok);
2893 } else {
2894 __ cmp_klass(tmp, dst, tmp2);
2895 __ jcc(Assembler::equal, known_ok);
2896 __ cmpptr(src, dst);
2897 __ jcc(Assembler::equal, known_ok);
2898 }
2899 __ bind(halt);
2900 __ stop("incorrect type information in arraycopy");
2901 __ bind(known_ok);
2902 }
3199 // first time here. Set profile type.
3200 __ movptr(mdo_addr, tmp);
3201 #ifdef ASSERT
3202 __ andptr(tmp, TypeEntries::type_klass_mask);
3203 __ verify_klass_ptr(tmp);
3204 #endif
3205 } else {
3206 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3207 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3208
3209 __ testptr(mdo_addr, TypeEntries::type_unknown);
3210 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3211
3212 __ orptr(mdo_addr, TypeEntries::type_unknown);
3213 }
3214 }
3215 }
3216 __ bind(next);
3217 }
3218
3219 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3220 Register obj = op->obj()->as_register();
3221 Register tmp = op->tmp()->as_pointer_register();
3222 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3223 bool not_null = op->not_null();
3224 int flag = op->flag();
3225
3226 Label not_inline_type;
3227 if (!not_null) {
3228 __ testptr(obj, obj);
3229 __ jccb(Assembler::zero, not_inline_type);
3230 }
3231
3232 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3233
3234 __ orb(mdo_addr, flag);
3235
3236 __ bind(not_inline_type);
3237 }
3238
3239
3240 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3241 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3242 }
3243
3244
3245 void LIR_Assembler::align_backward_branch_target() {
3246 __ align(BytesPerWord);
3247 }
3248
3249
3250 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3251 if (left->is_single_cpu()) {
3252 __ negl(left->as_register());
3253 move_regs(left->as_register(), dest->as_register());
3254
3255 } else if (left->is_double_cpu()) {
3256 Register lo = left->as_register_lo();
3257 Register dst = dest->as_register_lo();
3258 __ movptr(dst, lo);
3259 __ negptr(dst);
3405 }
3406
3407 void LIR_Assembler::membar_loadstore() {
3408 // no-op
3409 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3410 }
3411
3412 void LIR_Assembler::membar_storeload() {
3413 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3414 }
3415
3416 void LIR_Assembler::on_spin_wait() {
3417 __ pause ();
3418 }
3419
3420 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3421 assert(result_reg->is_register(), "check");
3422 __ mov(result_reg->as_register(), r15_thread);
3423 }
3424
3425 void LIR_Assembler::check_orig_pc() {
3426 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3427 }
3428
3429 void LIR_Assembler::peephole(LIR_List*) {
3430 // do nothing for now
3431 }
3432
3433 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3434 assert(data == dest, "xchg/xadd uses only 2 operands");
3435
3436 if (data->type() == T_INT) {
3437 if (code == lir_xadd) {
3438 __ lock();
3439 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3440 } else {
3441 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3442 }
3443 } else if (data->is_oop()) {
3444 assert (code == lir_xchg, "xadd for oops");
3445 Register obj = data->as_register();
3446 if (UseCompressedOops) {
3447 __ encode_heap_oop(obj);
|