14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
411 // Perform needed unlocking
412 MonitorExitStub* stub = nullptr;
413 if (method()->is_synchronized()) {
414 monitor_address(0, FrameMap::rax_opr);
415 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
416 __ unlock_object(rdi, rsi, rax, *stub->entry());
417 __ bind(*stub->continuation());
418 }
419
420 if (compilation()->env()->dtrace_method_probes()) {
421 __ mov(rdi, r15_thread);
422 __ mov_metadata(rsi, method()->constant_encoding());
423 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
424 }
425
426 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
427 __ mov(rax, rbx); // Restore the exception
428 }
429
430 // remove the activation and dispatch to the unwind handler
431 __ remove_frame(initial_frame_size_in_bytes());
432 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
433
434 // Emit the slow path assembly
435 if (stub != nullptr) {
436 stub->emit_code(this);
437 }
438
439 return offset;
440 }
441
442
443 int LIR_Assembler::emit_deopt_handler() {
444 // generate code for exception handler
445 address handler_base = __ start_a_stub(deopt_handler_size());
446 if (handler_base == nullptr) {
447 // not enough space left for the handler
448 bailout("deopt handler overflow");
449 return -1;
450 }
451
454 Label start;
455 __ bind(start);
456
457 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
458
459 int entry_offset = __ offset();
460
461 __ jmp(start);
462
463 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
464 __ end_a_stub();
465
466 return entry_offset;
467 }
468
469 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
470 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
471 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
472 assert(result->fpu() == 0, "result must already be on TOS");
473 }
474
475 // Pop the stack before the safepoint code
476 __ remove_frame(initial_frame_size_in_bytes());
477
478 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
479 __ reserved_stack_check();
480 }
481
482 // Note: we do not need to round double result; float result has the right precision
483 // the poll sets the condition code, but no data registers
484
485 code_stub->set_safepoint_offset(__ offset());
486 __ relocate(relocInfo::poll_return_type);
487 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
488 __ ret(0);
489 }
490
491
492 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
493 guarantee(info != nullptr, "Shouldn't be null");
494 int offset = __ offset();
495 const Register poll_addr = rscratch1;
496 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
497 add_debug_info_for_branch(info);
498 __ relocate(relocInfo::poll_type);
499 address pre_pc = __ pc();
500 __ testl(rax, Address(poll_addr, 0));
501 address post_pc = __ pc();
502 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
503 return offset;
504 }
505
506
507 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
508 if (from_reg != to_reg) __ mov(to_reg, from_reg);
509 }
510
511 void LIR_Assembler::swap_reg(Register a, Register b) {
1210 // init_state needs acquire, but x86 is TSO, and so we are already good.
1211 __ cmpb(Address(op->klass()->as_register(),
1212 InstanceKlass::init_state_offset()),
1213 InstanceKlass::fully_initialized);
1214 __ jcc(Assembler::notEqual, *op->stub()->entry());
1215 }
1216 __ allocate_object(op->obj()->as_register(),
1217 op->tmp1()->as_register(),
1218 op->tmp2()->as_register(),
1219 op->header_size(),
1220 op->object_size(),
1221 op->klass()->as_register(),
1222 *op->stub()->entry());
1223 __ bind(*op->stub()->continuation());
1224 }
1225
1226 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1227 Register len = op->len()->as_register();
1228 __ movslq(len, len);
1229
1230 if (UseSlowPath ||
1231 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1232 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1233 __ jmp(*op->stub()->entry());
1234 } else {
1235 Register tmp1 = op->tmp1()->as_register();
1236 Register tmp2 = op->tmp2()->as_register();
1237 Register tmp3 = op->tmp3()->as_register();
1238 if (len == tmp1) {
1239 tmp1 = tmp3;
1240 } else if (len == tmp2) {
1241 tmp2 = tmp3;
1242 } else if (len == tmp3) {
1243 // everything is ok
1244 } else {
1245 __ mov(tmp3, len);
1246 }
1247 __ allocate_array(op->obj()->as_register(),
1248 len,
1249 tmp1,
1250 tmp2,
1309 assert(data != nullptr, "need data for type check");
1310 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1311 }
1312 Label* success_target = success;
1313 Label* failure_target = failure;
1314
1315 if (obj == k_RInfo) {
1316 k_RInfo = dst;
1317 } else if (obj == klass_RInfo) {
1318 klass_RInfo = dst;
1319 }
1320 if (k->is_loaded() && !UseCompressedClassPointers) {
1321 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1322 } else {
1323 Rtmp1 = op->tmp3()->as_register();
1324 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1325 }
1326
1327 assert_different_registers(obj, k_RInfo, klass_RInfo);
1328
1329 __ testptr(obj, obj);
1330 if (op->should_profile()) {
1331 Label not_null;
1332 Register mdo = klass_RInfo;
1333 __ mov_metadata(mdo, md->constant_encoding());
1334 __ jccb(Assembler::notEqual, not_null);
1335 // Object is null; update MDO and exit
1336 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1337 int header_bits = BitData::null_seen_byte_constant();
1338 __ orb(data_addr, header_bits);
1339 __ jmp(*obj_is_null);
1340 __ bind(not_null);
1341
1342 Label update_done;
1343 Register recv = k_RInfo;
1344 __ load_klass(recv, obj, tmp_load_klass);
1345 type_profile_helper(mdo, md, data, recv, &update_done);
1346
1347 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1348 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1349
1350 __ bind(update_done);
1351 } else {
1352 __ jcc(Assembler::equal, *obj_is_null);
1353 }
1354
1355 if (!k->is_loaded()) {
1356 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1357 } else {
1358 __ mov_metadata(k_RInfo, k->constant_encoding());
1359 }
1360 __ verify_oop(obj);
1361
1362 if (op->fast_check()) {
1363 // get object class
1364 // not a safepoint as obj null check happens earlier
1365 if (UseCompressedClassPointers) {
1366 __ load_klass(Rtmp1, obj, tmp_load_klass);
1367 __ cmpptr(k_RInfo, Rtmp1);
1368 } else {
1369 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1370 }
1371 __ jcc(Assembler::notEqual, *failure_target);
1372 // successful cast, fall through to profile or jump
1373 } else {
1374 // get object class
1375 // not a safepoint as obj null check happens earlier
1376 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1377 if (k->is_loaded()) {
1378 // See if we get an immediate positive hit
1379 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1380 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1381 __ jcc(Assembler::notEqual, *failure_target);
1382 // successful cast, fall through to profile or jump
1383 } else {
1384 // See if we get an immediate positive hit
1385 __ jcc(Assembler::equal, *success_target);
1386 // check for self
1387 __ cmpptr(klass_RInfo, k_RInfo);
1388 __ jcc(Assembler::equal, *success_target);
1389
1390 __ push_ppx(klass_RInfo);
1391 __ push_ppx(k_RInfo);
1392 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1393 __ pop_ppx(klass_RInfo);
1394 __ pop_ppx(klass_RInfo);
1395 // result is a boolean
1396 __ testl(klass_RInfo, klass_RInfo);
1397 __ jcc(Assembler::equal, *failure_target);
1398 // successful cast, fall through to profile or jump
1399 }
1400 } else {
1401 // perform the fast part of the checking logic
1402 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1403 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1404 __ push_ppx(klass_RInfo);
1405 __ push_ppx(k_RInfo);
1406 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1407 __ pop_ppx(klass_RInfo);
1502 __ mov(dst, obj);
1503 }
1504 } else
1505 if (code == lir_instanceof) {
1506 Register obj = op->object()->as_register();
1507 Register dst = op->result_opr()->as_register();
1508 Label success, failure, done;
1509 emit_typecheck_helper(op, &success, &failure, &failure);
1510 __ bind(failure);
1511 __ xorptr(dst, dst);
1512 __ jmpb(done);
1513 __ bind(success);
1514 __ movptr(dst, 1);
1515 __ bind(done);
1516 } else {
1517 ShouldNotReachHere();
1518 }
1519
1520 }
1521
1522
1523 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1524 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1525 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1526 Register newval = op->new_value()->as_register();
1527 Register cmpval = op->cmp_value()->as_register();
1528 assert(cmpval == rax, "wrong register");
1529 assert(newval != noreg, "new val must be register");
1530 assert(cmpval != newval, "cmp and new values must be in different registers");
1531 assert(cmpval != addr, "cmp and addr must be in different registers");
1532 assert(newval != addr, "new value and addr must be in different registers");
1533
1534 if (op->code() == lir_cas_obj) {
1535 if (UseCompressedOops) {
1536 __ encode_heap_oop(cmpval);
1537 __ mov(rscratch1, newval);
1538 __ encode_heap_oop(rscratch1);
1539 __ lock();
1540 // cmpval (rax) is implicitly used by this instruction
1541 __ cmpxchgl(rscratch1, Address(addr, 0));
1547 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1548 __ lock();
1549 __ cmpxchgl(newval, Address(addr, 0));
1550 }
1551 } else if (op->code() == lir_cas_long) {
1552 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1553 Register newval = op->new_value()->as_register_lo();
1554 Register cmpval = op->cmp_value()->as_register_lo();
1555 assert(cmpval == rax, "wrong register");
1556 assert(newval != noreg, "new val must be register");
1557 assert(cmpval != newval, "cmp and new values must be in different registers");
1558 assert(cmpval != addr, "cmp and addr must be in different registers");
1559 assert(newval != addr, "new value and addr must be in different registers");
1560 __ lock();
1561 __ cmpxchgq(newval, Address(addr, 0));
1562 } else {
1563 Unimplemented();
1564 }
1565 }
1566
1567 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1568 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1569 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1570
1571 Assembler::Condition acond, ncond;
1572 switch (condition) {
1573 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1574 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1575 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1576 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1577 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1578 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1579 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1580 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1581 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1582 ShouldNotReachHere();
1583 }
1584
1585 if (opr1->is_cpu_register()) {
1586 reg2reg(opr1, result);
2157 int offset = __ offset();
2158 switch (code) {
2159 case lir_static_call:
2160 case lir_optvirtual_call:
2161 case lir_dynamic_call:
2162 offset += NativeCall::displacement_offset;
2163 break;
2164 case lir_icvirtual_call:
2165 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2166 break;
2167 default: ShouldNotReachHere();
2168 }
2169 __ align(BytesPerWord, offset);
2170 }
2171
2172
2173 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2174 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2175 "must be aligned");
2176 __ call(AddressLiteral(op->addr(), rtype));
2177 add_call_info(code_offset(), op->info());
2178 __ post_call_nop();
2179 }
2180
2181
2182 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2183 __ ic_call(op->addr());
2184 add_call_info(code_offset(), op->info());
2185 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2186 "must be aligned");
2187 __ post_call_nop();
2188 }
2189
2190
2191 void LIR_Assembler::emit_static_call_stub() {
2192 address call_pc = __ pc();
2193 address stub = __ start_a_stub(call_stub_size());
2194 if (stub == nullptr) {
2195 bailout("static call stub overflow");
2196 return;
2197 }
2198
2199 int start = __ offset();
2200
2201 // make sure that the displacement word of the call ends up word aligned
2202 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2203 __ relocate(static_stub_Relocation::spec(call_pc));
2204 __ mov_metadata(rbx, (Metadata*)nullptr);
2331 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2332 }
2333
2334
2335 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2336 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2337 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2338 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2339 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2340 }
2341
2342
2343 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2344 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2345 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2346 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2347 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2348 }
2349
2350
2351 // This code replaces a call to arraycopy; no exception may
2352 // be thrown in this code, they must be thrown in the System.arraycopy
2353 // activation frame; we could save some checks if this would not be the case
2354 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2355 ciArrayKlass* default_type = op->expected_type();
2356 Register src = op->src()->as_register();
2357 Register dst = op->dst()->as_register();
2358 Register src_pos = op->src_pos()->as_register();
2359 Register dst_pos = op->dst_pos()->as_register();
2360 Register length = op->length()->as_register();
2361 Register tmp = op->tmp()->as_register();
2362 Register tmp_load_klass = rscratch1;
2363 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2364
2365 CodeStub* stub = op->stub();
2366 int flags = op->flags();
2367 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2368 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2369
2370 // if we don't know anything, just go through the generic arraycopy
2371 if (default_type == nullptr) {
2372 // save outgoing arguments on stack in case call to System.arraycopy is needed
2373 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2374 // for interpreter calling conventions. Now we have to do it in new style conventions.
2375 // For the moment until C1 gets the new register allocator I just force all the
2376 // args to the right place (except the register args) and then on the back side
2377 // reload the register args properly if we go slow path. Yuck
2378
2379 // These are proper for the calling convention
2380 store_parameter(length, 2);
2381 store_parameter(dst_pos, 1);
2382 store_parameter(dst, 0);
2383
2384 // these are just temporary placements until we need to reload
2385 store_parameter(src_pos, 3);
2386 store_parameter(src, 4);
2387
2388 address copyfunc_addr = StubRoutines::generic_arraycopy();
2389 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2426 __ mov(tmp, rax);
2427 __ xorl(tmp, -1);
2428
2429 // Reload values from the stack so they are where the stub
2430 // expects them.
2431 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2432 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2433 __ movptr (length, Address(rsp, 2*BytesPerWord));
2434 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2435 __ movptr (src, Address(rsp, 4*BytesPerWord));
2436
2437 __ subl(length, tmp);
2438 __ addl(src_pos, tmp);
2439 __ addl(dst_pos, tmp);
2440 __ jmp(*stub->entry());
2441
2442 __ bind(*stub->continuation());
2443 return;
2444 }
2445
2446 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2447
2448 int elem_size = type2aelembytes(basic_type);
2449 Address::ScaleFactor scale;
2450
2451 switch (elem_size) {
2452 case 1 :
2453 scale = Address::times_1;
2454 break;
2455 case 2 :
2456 scale = Address::times_2;
2457 break;
2458 case 4 :
2459 scale = Address::times_4;
2460 break;
2461 case 8 :
2462 scale = Address::times_8;
2463 break;
2464 default:
2465 scale = Address::no_scale;
2652 }
2653
2654 __ jmp(*stub->entry());
2655
2656 __ bind(cont);
2657 __ pop(dst);
2658 __ pop(src);
2659 }
2660 }
2661
2662 #ifdef ASSERT
2663 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2664 // Sanity check the known type with the incoming class. For the
2665 // primitive case the types must match exactly with src.klass and
2666 // dst.klass each exactly matching the default type. For the
2667 // object array case, if no type check is needed then either the
2668 // dst type is exactly the expected type and the src type is a
2669 // subtype which we can't check or src is the same array as dst
2670 // but not necessarily exactly of type default_type.
2671 Label known_ok, halt;
2672 __ mov_metadata(tmp, default_type->constant_encoding());
2673 if (UseCompressedClassPointers) {
2674 __ encode_klass_not_null(tmp, rscratch1);
2675 }
2676
2677 if (basic_type != T_OBJECT) {
2678 __ cmp_klass(tmp, dst, tmp2);
2679 __ jcc(Assembler::notEqual, halt);
2680 __ cmp_klass(tmp, src, tmp2);
2681 __ jcc(Assembler::equal, known_ok);
2682 } else {
2683 __ cmp_klass(tmp, dst, tmp2);
2684 __ jcc(Assembler::equal, known_ok);
2685 __ cmpptr(src, dst);
2686 __ jcc(Assembler::equal, known_ok);
2687 }
2688 __ bind(halt);
2689 __ stop("incorrect type information in arraycopy");
2690 __ bind(known_ok);
2691 }
2988 // first time here. Set profile type.
2989 __ movptr(mdo_addr, tmp);
2990 #ifdef ASSERT
2991 __ andptr(tmp, TypeEntries::type_klass_mask);
2992 __ verify_klass_ptr(tmp);
2993 #endif
2994 } else {
2995 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2996 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2997
2998 __ testptr(mdo_addr, TypeEntries::type_unknown);
2999 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3000
3001 __ orptr(mdo_addr, TypeEntries::type_unknown);
3002 }
3003 }
3004 }
3005 __ bind(next);
3006 }
3007
3008 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3009 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3010 }
3011
3012
3013 void LIR_Assembler::align_backward_branch_target() {
3014 __ align(BytesPerWord);
3015 }
3016
3017
3018 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3019 if (left->is_single_cpu()) {
3020 __ negl(left->as_register());
3021 move_regs(left->as_register(), dest->as_register());
3022
3023 } else if (left->is_double_cpu()) {
3024 Register lo = left->as_register_lo();
3025 Register dst = dest->as_register_lo();
3026 __ movptr(dst, lo);
3027 __ negptr(dst);
3173 }
3174
3175 void LIR_Assembler::membar_loadstore() {
3176 // no-op
3177 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3178 }
3179
3180 void LIR_Assembler::membar_storeload() {
3181 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3182 }
3183
3184 void LIR_Assembler::on_spin_wait() {
3185 __ pause ();
3186 }
3187
3188 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3189 assert(result_reg->is_register(), "check");
3190 __ mov(result_reg->as_register(), r15_thread);
3191 }
3192
3193
3194 void LIR_Assembler::peephole(LIR_List*) {
3195 // do nothing for now
3196 }
3197
3198 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3199 assert(data == dest, "xchg/xadd uses only 2 operands");
3200
3201 if (data->type() == T_INT) {
3202 if (code == lir_xadd) {
3203 __ lock();
3204 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3205 } else {
3206 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3207 }
3208 } else if (data->is_oop()) {
3209 assert (code == lir_xchg, "xadd for oops");
3210 Register obj = data->as_register();
3211 if (UseCompressedOops) {
3212 __ encode_heap_oop(obj);
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "ci/ciObjArrayKlass.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
414 // Perform needed unlocking
415 MonitorExitStub* stub = nullptr;
416 if (method()->is_synchronized()) {
417 monitor_address(0, FrameMap::rax_opr);
418 stub = new MonitorExitStub(FrameMap::rax_opr, 0);
419 __ unlock_object(rdi, rsi, rax, *stub->entry());
420 __ bind(*stub->continuation());
421 }
422
423 if (compilation()->env()->dtrace_method_probes()) {
424 __ mov(rdi, r15_thread);
425 __ mov_metadata(rsi, method()->constant_encoding());
426 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
427 }
428
429 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
430 __ mov(rax, rbx); // Restore the exception
431 }
432
433 // remove the activation and dispatch to the unwind handler
434 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
435 __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
436
437 // Emit the slow path assembly
438 if (stub != nullptr) {
439 stub->emit_code(this);
440 }
441
442 return offset;
443 }
444
445
446 int LIR_Assembler::emit_deopt_handler() {
447 // generate code for exception handler
448 address handler_base = __ start_a_stub(deopt_handler_size());
449 if (handler_base == nullptr) {
450 // not enough space left for the handler
451 bailout("deopt handler overflow");
452 return -1;
453 }
454
457 Label start;
458 __ bind(start);
459
460 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
461
462 int entry_offset = __ offset();
463
464 __ jmp(start);
465
466 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
467 __ end_a_stub();
468
469 return entry_offset;
470 }
471
472 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
473 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
474 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
475 assert(result->fpu() == 0, "result must already be on TOS");
476 }
477 if (InlineTypeReturnedAsFields) {
478 #ifndef _LP64
479 Unimplemented();
480 #endif
481 // Check if we are returning an non-null inline type and load its fields into registers
482 ciType* return_type = compilation()->method()->return_type();
483 if (return_type->is_inlinetype()) {
484 ciInlineKlass* vk = return_type->as_inline_klass();
485 if (vk->can_be_returned_as_fields()) {
486 address unpack_handler = vk->unpack_handler();
487 assert(unpack_handler != nullptr, "must be");
488 __ call(RuntimeAddress(unpack_handler));
489 }
490 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
491 Label skip;
492 Label not_null;
493 __ testptr(rax, rax);
494 __ jcc(Assembler::notZero, not_null);
495 // Returned value is null, zero all return registers because they may belong to oop fields
496 __ xorq(j_rarg1, j_rarg1);
497 __ xorq(j_rarg2, j_rarg2);
498 __ xorq(j_rarg3, j_rarg3);
499 __ xorq(j_rarg4, j_rarg4);
500 __ xorq(j_rarg5, j_rarg5);
501 __ jmp(skip);
502 __ bind(not_null);
503
504 // Check if we are returning an non-null inline type and load its fields into registers
505 __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
506
507 // Load fields from a buffered value with an inline class specific handler
508 __ load_klass(rdi, rax, rscratch1);
509 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
510 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
511 // Unpack handler can be null if inline type is not scalarizable in returns
512 __ testptr(rdi, rdi);
513 __ jcc(Assembler::zero, skip);
514 __ call(rdi);
515
516 __ bind(skip);
517 }
518 // At this point, rax points to the value object (for interpreter or C1 caller).
519 // The fields of the object are copied into registers (for C2 caller).
520 }
521
522 // Pop the stack before the safepoint code
523 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
524
525 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
526 __ reserved_stack_check();
527 }
528
529 // Note: we do not need to round double result; float result has the right precision
530 // the poll sets the condition code, but no data registers
531
532 code_stub->set_safepoint_offset(__ offset());
533 __ relocate(relocInfo::poll_return_type);
534 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
535 __ ret(0);
536 }
537
538
539 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
540 return (__ store_inline_type_fields_to_buf(vk, false));
541 }
542
543 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
544 guarantee(info != nullptr, "Shouldn't be null");
545 int offset = __ offset();
546 const Register poll_addr = rscratch1;
547 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
548 add_debug_info_for_branch(info);
549 __ relocate(relocInfo::poll_type);
550 address pre_pc = __ pc();
551 __ testl(rax, Address(poll_addr, 0));
552 address post_pc = __ pc();
553 guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
554 return offset;
555 }
556
557
558 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
559 if (from_reg != to_reg) __ mov(to_reg, from_reg);
560 }
561
562 void LIR_Assembler::swap_reg(Register a, Register b) {
1261 // init_state needs acquire, but x86 is TSO, and so we are already good.
1262 __ cmpb(Address(op->klass()->as_register(),
1263 InstanceKlass::init_state_offset()),
1264 InstanceKlass::fully_initialized);
1265 __ jcc(Assembler::notEqual, *op->stub()->entry());
1266 }
1267 __ allocate_object(op->obj()->as_register(),
1268 op->tmp1()->as_register(),
1269 op->tmp2()->as_register(),
1270 op->header_size(),
1271 op->object_size(),
1272 op->klass()->as_register(),
1273 *op->stub()->entry());
1274 __ bind(*op->stub()->continuation());
1275 }
1276
1277 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1278 Register len = op->len()->as_register();
1279 __ movslq(len, len);
1280
1281 if (UseSlowPath || op->always_slow_path() ||
1282 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1283 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1284 __ jmp(*op->stub()->entry());
1285 } else {
1286 Register tmp1 = op->tmp1()->as_register();
1287 Register tmp2 = op->tmp2()->as_register();
1288 Register tmp3 = op->tmp3()->as_register();
1289 if (len == tmp1) {
1290 tmp1 = tmp3;
1291 } else if (len == tmp2) {
1292 tmp2 = tmp3;
1293 } else if (len == tmp3) {
1294 // everything is ok
1295 } else {
1296 __ mov(tmp3, len);
1297 }
1298 __ allocate_array(op->obj()->as_register(),
1299 len,
1300 tmp1,
1301 tmp2,
1360 assert(data != nullptr, "need data for type check");
1361 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1362 }
1363 Label* success_target = success;
1364 Label* failure_target = failure;
1365
1366 if (obj == k_RInfo) {
1367 k_RInfo = dst;
1368 } else if (obj == klass_RInfo) {
1369 klass_RInfo = dst;
1370 }
1371 if (k->is_loaded() && !UseCompressedClassPointers) {
1372 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1373 } else {
1374 Rtmp1 = op->tmp3()->as_register();
1375 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1376 }
1377
1378 assert_different_registers(obj, k_RInfo, klass_RInfo);
1379
1380 if (op->need_null_check()) {
1381 __ testptr(obj, obj);
1382 if (op->should_profile()) {
1383 Label not_null;
1384 Register mdo = klass_RInfo;
1385 __ mov_metadata(mdo, md->constant_encoding());
1386 __ jccb(Assembler::notEqual, not_null);
1387 // Object is null; update MDO and exit
1388 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1389 int header_bits = BitData::null_seen_byte_constant();
1390 __ orb(data_addr, header_bits);
1391 __ jmp(*obj_is_null);
1392 __ bind(not_null);
1393
1394 Label update_done;
1395 Register recv = k_RInfo;
1396 __ load_klass(recv, obj, tmp_load_klass);
1397 type_profile_helper(mdo, md, data, recv, &update_done);
1398
1399 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1400 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1401
1402 __ bind(update_done);
1403 } else {
1404 __ jcc(Assembler::equal, *obj_is_null);
1405 }
1406 }
1407
1408 if (!k->is_loaded()) {
1409 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1410 } else {
1411 __ mov_metadata(k_RInfo, k->constant_encoding());
1412 }
1413 __ verify_oop(obj);
1414
1415 if (op->fast_check()) {
1416 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1417 // get object class
1418 // not a safepoint as obj null check happens earlier
1419 if (UseCompressedClassPointers) {
1420 __ load_klass(Rtmp1, obj, tmp_load_klass);
1421 __ cmpptr(k_RInfo, Rtmp1);
1422 } else {
1423 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1424 }
1425 __ jcc(Assembler::notEqual, *failure_target);
1426 // successful cast, fall through to profile or jump
1427 } else {
1428 // get object class
1429 // not a safepoint as obj null check happens earlier
1430 __ load_klass(klass_RInfo, obj, tmp_load_klass);
1431 if (k->is_loaded()) {
1432 // See if we get an immediate positive hit
1433 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1434 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1435 __ jcc(Assembler::notEqual, *failure_target);
1436 // successful cast, fall through to profile or jump
1437 } else {
1438 // See if we get an immediate positive hit
1439 __ jcc(Assembler::equal, *success_target);
1440 // check for self
1441 if (k->is_loaded() && k->is_obj_array_klass()) {
1442 // For a direct pointer comparison, we need the refined array klass pointer
1443 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1444 __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1445 __ cmpptr(klass_RInfo, tmp_load_klass);
1446 } else {
1447 __ cmpptr(klass_RInfo, k_RInfo);
1448 }
1449 __ jcc(Assembler::equal, *success_target);
1450
1451 __ push_ppx(klass_RInfo);
1452 __ push_ppx(k_RInfo);
1453 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1454 __ pop_ppx(klass_RInfo);
1455 __ pop_ppx(klass_RInfo);
1456 // result is a boolean
1457 __ testl(klass_RInfo, klass_RInfo);
1458 __ jcc(Assembler::equal, *failure_target);
1459 // successful cast, fall through to profile or jump
1460 }
1461 } else {
1462 // perform the fast part of the checking logic
1463 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1464 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1465 __ push_ppx(klass_RInfo);
1466 __ push_ppx(k_RInfo);
1467 __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1468 __ pop_ppx(klass_RInfo);
1563 __ mov(dst, obj);
1564 }
1565 } else
1566 if (code == lir_instanceof) {
1567 Register obj = op->object()->as_register();
1568 Register dst = op->result_opr()->as_register();
1569 Label success, failure, done;
1570 emit_typecheck_helper(op, &success, &failure, &failure);
1571 __ bind(failure);
1572 __ xorptr(dst, dst);
1573 __ jmpb(done);
1574 __ bind(success);
1575 __ movptr(dst, 1);
1576 __ bind(done);
1577 } else {
1578 ShouldNotReachHere();
1579 }
1580
1581 }
1582
1583 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1584 // We are loading/storing from/to an array that *may* be a flat array (the
1585 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1586 // If this array is a flat array, take the slow path.
1587 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1588 if (!op->value()->is_illegal()) {
1589 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1590 // The array is not a flat array, but it might be null-free. If we are storing
1591 // a null into a null-free array, take the slow path (which will throw NPE).
1592 Label skip;
1593 __ cmpptr(op->value()->as_register(), NULL_WORD);
1594 __ jcc(Assembler::notEqual, skip);
1595 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1596 __ bind(skip);
1597 }
1598 }
1599
1600 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1601 // We are storing into an array that *may* be null-free (the declared type is
1602 // Object[], abstract[], interface[] or VT.ref[]).
1603 Label test_mark_word;
1604 Register tmp = op->tmp()->as_register();
1605 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1606 __ testl(tmp, markWord::unlocked_value);
1607 __ jccb(Assembler::notZero, test_mark_word);
1608 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1609 __ bind(test_mark_word);
1610 __ testl(tmp, markWord::null_free_array_bit_in_place);
1611 }
1612
1613 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1614 Label L_oops_equal;
1615 Label L_oops_not_equal;
1616 Label L_end;
1617
1618 Register left = op->left()->as_register();
1619 Register right = op->right()->as_register();
1620
1621 __ cmpptr(left, right);
1622 __ jcc(Assembler::equal, L_oops_equal);
1623
1624 // (1) Null check -- if one of the operands is null, the other must not be null (because
1625 // the two references are not equal), so they are not substitutable,
1626 // FIXME: do null check only if the operand is nullable
1627 __ testptr(left, right);
1628 __ jcc(Assembler::zero, L_oops_not_equal);
1629
1630 ciKlass* left_klass = op->left_klass();
1631 ciKlass* right_klass = op->right_klass();
1632
1633 // (2) Inline type check -- if either of the operands is not a inline type,
1634 // they are not substitutable. We do this only if we are not sure that the
1635 // operands are inline type
1636 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1637 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1638 Register tmp1 = op->tmp1()->as_register();
1639 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1640 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1641 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1642 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1643 __ jcc(Assembler::notEqual, L_oops_not_equal);
1644 }
1645
1646 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1647 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1648 // No need to load klass -- the operands are statically known to be the same inline klass.
1649 __ jmp(*op->stub()->entry());
1650 } else {
1651 Register left_klass_op = op->left_klass_op()->as_register();
1652 Register right_klass_op = op->right_klass_op()->as_register();
1653
1654 if (UseCompressedClassPointers) {
1655 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1656 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1657 __ cmpl(left_klass_op, right_klass_op);
1658 } else {
1659 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1660 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1661 __ cmpptr(left_klass_op, right_klass_op);
1662 }
1663
1664 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1665 // fall through to L_oops_not_equal
1666 }
1667
1668 __ bind(L_oops_not_equal);
1669 move(op->not_equal_result(), op->result_opr());
1670 __ jmp(L_end);
1671
1672 __ bind(L_oops_equal);
1673 move(op->equal_result(), op->result_opr());
1674 __ jmp(L_end);
1675
1676 // We've returned from the stub. RAX contains 0x0 IFF the two
1677 // operands are not substitutable. (Don't compare against 0x1 in case the
1678 // C compiler is naughty)
1679 __ bind(*op->stub()->continuation());
1680 __ cmpl(rax, 0);
1681 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1682 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1683 // fall-through
1684 __ bind(L_end);
1685 }
1686
1687 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1688 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1689 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1690 Register newval = op->new_value()->as_register();
1691 Register cmpval = op->cmp_value()->as_register();
1692 assert(cmpval == rax, "wrong register");
1693 assert(newval != noreg, "new val must be register");
1694 assert(cmpval != newval, "cmp and new values must be in different registers");
1695 assert(cmpval != addr, "cmp and addr must be in different registers");
1696 assert(newval != addr, "new value and addr must be in different registers");
1697
1698 if (op->code() == lir_cas_obj) {
1699 if (UseCompressedOops) {
1700 __ encode_heap_oop(cmpval);
1701 __ mov(rscratch1, newval);
1702 __ encode_heap_oop(rscratch1);
1703 __ lock();
1704 // cmpval (rax) is implicitly used by this instruction
1705 __ cmpxchgl(rscratch1, Address(addr, 0));
1711 assert(op->code() == lir_cas_int, "lir_cas_int expected");
1712 __ lock();
1713 __ cmpxchgl(newval, Address(addr, 0));
1714 }
1715 } else if (op->code() == lir_cas_long) {
1716 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1717 Register newval = op->new_value()->as_register_lo();
1718 Register cmpval = op->cmp_value()->as_register_lo();
1719 assert(cmpval == rax, "wrong register");
1720 assert(newval != noreg, "new val must be register");
1721 assert(cmpval != newval, "cmp and new values must be in different registers");
1722 assert(cmpval != addr, "cmp and addr must be in different registers");
1723 assert(newval != addr, "new value and addr must be in different registers");
1724 __ lock();
1725 __ cmpxchgq(newval, Address(addr, 0));
1726 } else {
1727 Unimplemented();
1728 }
1729 }
1730
1731 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1732 assert(dst->is_cpu_register(), "must be");
1733 assert(dst->type() == src->type(), "must be");
1734
1735 if (src->is_cpu_register()) {
1736 reg2reg(src, dst);
1737 } else if (src->is_stack()) {
1738 stack2reg(src, dst, dst->type());
1739 } else if (src->is_constant()) {
1740 const2reg(src, dst, lir_patch_none, nullptr);
1741 } else {
1742 ShouldNotReachHere();
1743 }
1744 }
1745
1746 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1747 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1748 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1749
1750 Assembler::Condition acond, ncond;
1751 switch (condition) {
1752 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1753 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1754 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1755 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1756 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1757 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1758 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1759 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1760 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1761 ShouldNotReachHere();
1762 }
1763
1764 if (opr1->is_cpu_register()) {
1765 reg2reg(opr1, result);
2336 int offset = __ offset();
2337 switch (code) {
2338 case lir_static_call:
2339 case lir_optvirtual_call:
2340 case lir_dynamic_call:
2341 offset += NativeCall::displacement_offset;
2342 break;
2343 case lir_icvirtual_call:
2344 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2345 break;
2346 default: ShouldNotReachHere();
2347 }
2348 __ align(BytesPerWord, offset);
2349 }
2350
2351
2352 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2353 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2354 "must be aligned");
2355 __ call(AddressLiteral(op->addr(), rtype));
2356 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2357 __ post_call_nop();
2358 }
2359
2360
2361 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2362 __ ic_call(op->addr());
2363 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2364 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2365 "must be aligned");
2366 __ post_call_nop();
2367 }
2368
2369
2370 void LIR_Assembler::emit_static_call_stub() {
2371 address call_pc = __ pc();
2372 address stub = __ start_a_stub(call_stub_size());
2373 if (stub == nullptr) {
2374 bailout("static call stub overflow");
2375 return;
2376 }
2377
2378 int start = __ offset();
2379
2380 // make sure that the displacement word of the call ends up word aligned
2381 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2382 __ relocate(static_stub_Relocation::spec(call_pc));
2383 __ mov_metadata(rbx, (Metadata*)nullptr);
2510 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2511 }
2512
2513
2514 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2515 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2516 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2517 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2518 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2519 }
2520
2521
2522 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2523 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2524 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2525 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2526 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2527 }
2528
2529
2530 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2531 if (null_check) {
2532 __ testptr(obj, obj);
2533 __ jcc(Assembler::zero, *slow_path->entry());
2534 }
2535 if (is_dest) {
2536 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2537 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2538 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2539 } else {
2540 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2541 }
2542 }
2543
2544
2545 // This code replaces a call to arraycopy; no exception may
2546 // be thrown in this code, they must be thrown in the System.arraycopy
2547 // activation frame; we could save some checks if this would not be the case
2548 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2549 ciArrayKlass* default_type = op->expected_type();
2550 Register src = op->src()->as_register();
2551 Register dst = op->dst()->as_register();
2552 Register src_pos = op->src_pos()->as_register();
2553 Register dst_pos = op->dst_pos()->as_register();
2554 Register length = op->length()->as_register();
2555 Register tmp = op->tmp()->as_register();
2556 Register tmp_load_klass = rscratch1;
2557 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2558
2559 CodeStub* stub = op->stub();
2560 int flags = op->flags();
2561 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2562 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2563
2564 if (flags & LIR_OpArrayCopy::always_slow_path) {
2565 __ jmp(*stub->entry());
2566 __ bind(*stub->continuation());
2567 return;
2568 }
2569
2570 // if we don't know anything, just go through the generic arraycopy
2571 if (default_type == nullptr) {
2572 // save outgoing arguments on stack in case call to System.arraycopy is needed
2573 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2574 // for interpreter calling conventions. Now we have to do it in new style conventions.
2575 // For the moment until C1 gets the new register allocator I just force all the
2576 // args to the right place (except the register args) and then on the back side
2577 // reload the register args properly if we go slow path. Yuck
2578
2579 // These are proper for the calling convention
2580 store_parameter(length, 2);
2581 store_parameter(dst_pos, 1);
2582 store_parameter(dst, 0);
2583
2584 // these are just temporary placements until we need to reload
2585 store_parameter(src_pos, 3);
2586 store_parameter(src, 4);
2587
2588 address copyfunc_addr = StubRoutines::generic_arraycopy();
2589 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2626 __ mov(tmp, rax);
2627 __ xorl(tmp, -1);
2628
2629 // Reload values from the stack so they are where the stub
2630 // expects them.
2631 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2632 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2633 __ movptr (length, Address(rsp, 2*BytesPerWord));
2634 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2635 __ movptr (src, Address(rsp, 4*BytesPerWord));
2636
2637 __ subl(length, tmp);
2638 __ addl(src_pos, tmp);
2639 __ addl(dst_pos, tmp);
2640 __ jmp(*stub->entry());
2641
2642 __ bind(*stub->continuation());
2643 return;
2644 }
2645
2646 // Handle inline type arrays
2647 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2648 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2649 }
2650 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2651 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2652 }
2653
2654 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2655
2656 int elem_size = type2aelembytes(basic_type);
2657 Address::ScaleFactor scale;
2658
2659 switch (elem_size) {
2660 case 1 :
2661 scale = Address::times_1;
2662 break;
2663 case 2 :
2664 scale = Address::times_2;
2665 break;
2666 case 4 :
2667 scale = Address::times_4;
2668 break;
2669 case 8 :
2670 scale = Address::times_8;
2671 break;
2672 default:
2673 scale = Address::no_scale;
2860 }
2861
2862 __ jmp(*stub->entry());
2863
2864 __ bind(cont);
2865 __ pop(dst);
2866 __ pop(src);
2867 }
2868 }
2869
2870 #ifdef ASSERT
2871 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2872 // Sanity check the known type with the incoming class. For the
2873 // primitive case the types must match exactly with src.klass and
2874 // dst.klass each exactly matching the default type. For the
2875 // object array case, if no type check is needed then either the
2876 // dst type is exactly the expected type and the src type is a
2877 // subtype which we can't check or src is the same array as dst
2878 // but not necessarily exactly of type default_type.
2879 Label known_ok, halt;
2880
2881 __ mov_metadata(tmp, default_type->constant_encoding());
2882 if (UseCompressedClassPointers) {
2883 __ encode_klass_not_null(tmp, rscratch1);
2884 }
2885
2886 if (basic_type != T_OBJECT) {
2887 __ cmp_klass(tmp, dst, tmp2);
2888 __ jcc(Assembler::notEqual, halt);
2889 __ cmp_klass(tmp, src, tmp2);
2890 __ jcc(Assembler::equal, known_ok);
2891 } else {
2892 __ cmp_klass(tmp, dst, tmp2);
2893 __ jcc(Assembler::equal, known_ok);
2894 __ cmpptr(src, dst);
2895 __ jcc(Assembler::equal, known_ok);
2896 }
2897 __ bind(halt);
2898 __ stop("incorrect type information in arraycopy");
2899 __ bind(known_ok);
2900 }
3197 // first time here. Set profile type.
3198 __ movptr(mdo_addr, tmp);
3199 #ifdef ASSERT
3200 __ andptr(tmp, TypeEntries::type_klass_mask);
3201 __ verify_klass_ptr(tmp);
3202 #endif
3203 } else {
3204 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3205 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3206
3207 __ testptr(mdo_addr, TypeEntries::type_unknown);
3208 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3209
3210 __ orptr(mdo_addr, TypeEntries::type_unknown);
3211 }
3212 }
3213 }
3214 __ bind(next);
3215 }
3216
3217 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3218 Register obj = op->obj()->as_register();
3219 Register tmp = op->tmp()->as_pointer_register();
3220 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3221 bool not_null = op->not_null();
3222 int flag = op->flag();
3223
3224 Label not_inline_type;
3225 if (!not_null) {
3226 __ testptr(obj, obj);
3227 __ jccb(Assembler::zero, not_inline_type);
3228 }
3229
3230 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3231
3232 __ orb(mdo_addr, flag);
3233
3234 __ bind(not_inline_type);
3235 }
3236
3237
3238 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3239 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3240 }
3241
3242
3243 void LIR_Assembler::align_backward_branch_target() {
3244 __ align(BytesPerWord);
3245 }
3246
3247
3248 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3249 if (left->is_single_cpu()) {
3250 __ negl(left->as_register());
3251 move_regs(left->as_register(), dest->as_register());
3252
3253 } else if (left->is_double_cpu()) {
3254 Register lo = left->as_register_lo();
3255 Register dst = dest->as_register_lo();
3256 __ movptr(dst, lo);
3257 __ negptr(dst);
3403 }
3404
3405 void LIR_Assembler::membar_loadstore() {
3406 // no-op
3407 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3408 }
3409
3410 void LIR_Assembler::membar_storeload() {
3411 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3412 }
3413
3414 void LIR_Assembler::on_spin_wait() {
3415 __ pause ();
3416 }
3417
3418 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3419 assert(result_reg->is_register(), "check");
3420 __ mov(result_reg->as_register(), r15_thread);
3421 }
3422
3423 void LIR_Assembler::check_orig_pc() {
3424 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3425 }
3426
3427 void LIR_Assembler::peephole(LIR_List*) {
3428 // do nothing for now
3429 }
3430
3431 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3432 assert(data == dest, "xchg/xadd uses only 2 operands");
3433
3434 if (data->type() == T_INT) {
3435 if (code == lir_xadd) {
3436 __ lock();
3437 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3438 } else {
3439 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3440 }
3441 } else if (data->is_oop()) {
3442 assert (code == lir_xchg, "xadd for oops");
3443 Register obj = data->as_register();
3444 if (UseCompressedOops) {
3445 __ encode_heap_oop(obj);
|