< prev index next >

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

Print this page

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"

  34 #include "ci/ciInstance.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/collectedHeap.hpp"
  37 #include "gc/shared/gc_globals.hpp"
  38 #include "nativeInst_x86.hpp"

  39 #include "oops/objArrayKlass.hpp"
  40 #include "runtime/frame.inline.hpp"
  41 #include "runtime/safepointMechanism.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "runtime/stubRoutines.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_x86.inline.hpp"
  46 
  47 
  48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  49 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  50 // fast versions of NegF/NegD and AbsF/AbsD.
  51 
  52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  54   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  55   // of 128-bits operands for SSE instructions.
  56   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  57   // Store the value to a 128-bits operand.
  58   operand[0] = lo;

 411   // Perform needed unlocking
 412   MonitorExitStub* stub = nullptr;
 413   if (method()->is_synchronized()) {
 414     monitor_address(0, FrameMap::rax_opr);
 415     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 416     __ unlock_object(rdi, rsi, rax, *stub->entry());
 417     __ bind(*stub->continuation());
 418   }
 419 
 420   if (compilation()->env()->dtrace_method_probes()) {
 421     __ mov(rdi, r15_thread);
 422     __ mov_metadata(rsi, method()->constant_encoding());
 423     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 424   }
 425 
 426   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 427     __ mov(rax, rbx);  // Restore the exception
 428   }
 429 
 430   // remove the activation and dispatch to the unwind handler
 431   __ remove_frame(initial_frame_size_in_bytes());
 432   __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
 433 
 434   // Emit the slow path assembly
 435   if (stub != nullptr) {
 436     stub->emit_code(this);
 437   }
 438 
 439   return offset;
 440 }
 441 
 442 
 443 int LIR_Assembler::emit_deopt_handler() {
 444   // generate code for exception handler
 445   address handler_base = __ start_a_stub(deopt_handler_size());
 446   if (handler_base == nullptr) {
 447     // not enough space left for the handler
 448     bailout("deopt handler overflow");
 449     return -1;
 450   }
 451 
 452   int offset = code_offset();
 453   InternalAddress here(__ pc());
 454 
 455   __ pushptr(here.addr(), rscratch1);
 456   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 457   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 458   __ end_a_stub();
 459 
 460   return offset;
 461 }
 462 
 463 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 464   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 465   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 466     assert(result->fpu() == 0, "result must already be on TOS");
 467   }












































 468 
 469   // Pop the stack before the safepoint code
 470   __ remove_frame(initial_frame_size_in_bytes());
 471 
 472   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 473     __ reserved_stack_check();
 474   }
 475 
 476   // Note: we do not need to round double result; float result has the right precision
 477   // the poll sets the condition code, but no data registers
 478 
 479   code_stub->set_safepoint_offset(__ offset());
 480   __ relocate(relocInfo::poll_return_type);
 481   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 482   __ ret(0);
 483 }
 484 
 485 




 486 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 487   guarantee(info != nullptr, "Shouldn't be null");
 488   int offset = __ offset();
 489   const Register poll_addr = rscratch1;
 490   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 491   add_debug_info_for_branch(info);
 492   __ relocate(relocInfo::poll_type);
 493   address pre_pc = __ pc();
 494   __ testl(rax, Address(poll_addr, 0));
 495   address post_pc = __ pc();
 496   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 497   return offset;
 498 }
 499 
 500 
 501 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 502   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 503 }
 504 
 505 void LIR_Assembler::swap_reg(Register a, Register b) {

1204     // init_state needs acquire, but x86 is TSO, and so we are already good.
1205     __ cmpb(Address(op->klass()->as_register(),
1206                     InstanceKlass::init_state_offset()),
1207                     InstanceKlass::fully_initialized);
1208     __ jcc(Assembler::notEqual, *op->stub()->entry());
1209   }
1210   __ allocate_object(op->obj()->as_register(),
1211                      op->tmp1()->as_register(),
1212                      op->tmp2()->as_register(),
1213                      op->header_size(),
1214                      op->object_size(),
1215                      op->klass()->as_register(),
1216                      *op->stub()->entry());
1217   __ bind(*op->stub()->continuation());
1218 }
1219 
1220 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1221   Register len =  op->len()->as_register();
1222   __ movslq(len, len);
1223 
1224   if (UseSlowPath ||
1225       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1226       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1227     __ jmp(*op->stub()->entry());
1228   } else {
1229     Register tmp1 = op->tmp1()->as_register();
1230     Register tmp2 = op->tmp2()->as_register();
1231     Register tmp3 = op->tmp3()->as_register();
1232     if (len == tmp1) {
1233       tmp1 = tmp3;
1234     } else if (len == tmp2) {
1235       tmp2 = tmp3;
1236     } else if (len == tmp3) {
1237       // everything is ok
1238     } else {
1239       __ mov(tmp3, len);
1240     }
1241     __ allocate_array(op->obj()->as_register(),
1242                       len,
1243                       tmp1,
1244                       tmp2,

1303     assert(data != nullptr,                "need data for type check");
1304     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1305   }
1306   Label* success_target = success;
1307   Label* failure_target = failure;
1308 
1309   if (obj == k_RInfo) {
1310     k_RInfo = dst;
1311   } else if (obj == klass_RInfo) {
1312     klass_RInfo = dst;
1313   }
1314   if (k->is_loaded() && !UseCompressedClassPointers) {
1315     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1316   } else {
1317     Rtmp1 = op->tmp3()->as_register();
1318     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1319   }
1320 
1321   assert_different_registers(obj, k_RInfo, klass_RInfo);
1322 
1323   __ testptr(obj, obj);
1324   if (op->should_profile()) {
1325     Label not_null;
1326     Register mdo  = klass_RInfo;
1327     __ mov_metadata(mdo, md->constant_encoding());
1328     __ jccb(Assembler::notEqual, not_null);
1329     // Object is null; update MDO and exit
1330     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1331     int header_bits = BitData::null_seen_byte_constant();
1332     __ orb(data_addr, header_bits);
1333     __ jmp(*obj_is_null);
1334     __ bind(not_null);
1335 
1336     Label update_done;
1337     Register recv = k_RInfo;
1338     __ load_klass(recv, obj, tmp_load_klass);
1339     type_profile_helper(mdo, md, data, recv, &update_done);
1340 
1341     Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1342     __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1343 
1344     __ bind(update_done);
1345   } else {
1346     __ jcc(Assembler::equal, *obj_is_null);


1347   }
1348 
1349   if (!k->is_loaded()) {
1350     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1351   } else {
1352     __ mov_metadata(k_RInfo, k->constant_encoding());
1353   }
1354   __ verify_oop(obj);
1355 
1356   if (op->fast_check()) {

1357     // get object class
1358     // not a safepoint as obj null check happens earlier
1359     if (UseCompressedClassPointers) {
1360       __ load_klass(Rtmp1, obj, tmp_load_klass);
1361       __ cmpptr(k_RInfo, Rtmp1);
1362     } else {
1363       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1364     }
1365     __ jcc(Assembler::notEqual, *failure_target);
1366     // successful cast, fall through to profile or jump
1367   } else {
1368     // get object class
1369     // not a safepoint as obj null check happens earlier
1370     __ load_klass(klass_RInfo, obj, tmp_load_klass);
1371     if (k->is_loaded()) {
1372       // See if we get an immediate positive hit
1373       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1374       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1375         __ jcc(Assembler::notEqual, *failure_target);
1376         // successful cast, fall through to profile or jump

1496         __ mov(dst, obj);
1497       }
1498     } else
1499       if (code == lir_instanceof) {
1500         Register obj = op->object()->as_register();
1501         Register dst = op->result_opr()->as_register();
1502         Label success, failure, done;
1503         emit_typecheck_helper(op, &success, &failure, &failure);
1504         __ bind(failure);
1505         __ xorptr(dst, dst);
1506         __ jmpb(done);
1507         __ bind(success);
1508         __ movptr(dst, 1);
1509         __ bind(done);
1510       } else {
1511         ShouldNotReachHere();
1512       }
1513 
1514 }
1515 







































































































1516 
1517 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1518   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1519     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1520     Register newval = op->new_value()->as_register();
1521     Register cmpval = op->cmp_value()->as_register();
1522     assert(cmpval == rax, "wrong register");
1523     assert(newval != noreg, "new val must be register");
1524     assert(cmpval != newval, "cmp and new values must be in different registers");
1525     assert(cmpval != addr, "cmp and addr must be in different registers");
1526     assert(newval != addr, "new value and addr must be in different registers");
1527 
1528     if (op->code() == lir_cas_obj) {
1529       if (UseCompressedOops) {
1530         __ encode_heap_oop(cmpval);
1531         __ mov(rscratch1, newval);
1532         __ encode_heap_oop(rscratch1);
1533         __ lock();
1534         // cmpval (rax) is implicitly used by this instruction
1535         __ cmpxchgl(rscratch1, Address(addr, 0));

1541       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1542       __ lock();
1543       __ cmpxchgl(newval, Address(addr, 0));
1544     }
1545   } else if (op->code() == lir_cas_long) {
1546     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1547     Register newval = op->new_value()->as_register_lo();
1548     Register cmpval = op->cmp_value()->as_register_lo();
1549     assert(cmpval == rax, "wrong register");
1550     assert(newval != noreg, "new val must be register");
1551     assert(cmpval != newval, "cmp and new values must be in different registers");
1552     assert(cmpval != addr, "cmp and addr must be in different registers");
1553     assert(newval != addr, "new value and addr must be in different registers");
1554     __ lock();
1555     __ cmpxchgq(newval, Address(addr, 0));
1556   } else {
1557     Unimplemented();
1558   }
1559 }
1560 















1561 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1562                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1563   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1564 
1565   Assembler::Condition acond, ncond;
1566   switch (condition) {
1567     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1568     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1569     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1570     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1571     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1572     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1573     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1574     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1575     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1576                                 ShouldNotReachHere();
1577   }
1578 
1579   if (opr1->is_cpu_register()) {
1580     reg2reg(opr1, result);

2151   int offset = __ offset();
2152   switch (code) {
2153   case lir_static_call:
2154   case lir_optvirtual_call:
2155   case lir_dynamic_call:
2156     offset += NativeCall::displacement_offset;
2157     break;
2158   case lir_icvirtual_call:
2159     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2160     break;
2161   default: ShouldNotReachHere();
2162   }
2163   __ align(BytesPerWord, offset);
2164 }
2165 
2166 
2167 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2168   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2169          "must be aligned");
2170   __ call(AddressLiteral(op->addr(), rtype));
2171   add_call_info(code_offset(), op->info());
2172   __ post_call_nop();
2173 }
2174 
2175 
2176 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2177   __ ic_call(op->addr());
2178   add_call_info(code_offset(), op->info());
2179   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2180          "must be aligned");
2181   __ post_call_nop();
2182 }
2183 
2184 
2185 void LIR_Assembler::emit_static_call_stub() {
2186   address call_pc = __ pc();
2187   address stub = __ start_a_stub(call_stub_size());
2188   if (stub == nullptr) {
2189     bailout("static call stub overflow");
2190     return;
2191   }
2192 
2193   int start = __ offset();
2194 
2195   // make sure that the displacement word of the call ends up word aligned
2196   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2197   __ relocate(static_stub_Relocation::spec(call_pc));
2198   __ mov_metadata(rbx, (Metadata*)nullptr);

2325   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2326 }
2327 
2328 
2329 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2330   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2331   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2332   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2333   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2334 }
2335 
2336 
2337 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2338   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2339   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2340   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2341   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2342 }
2343 
2344 















2345 // This code replaces a call to arraycopy; no exception may
2346 // be thrown in this code, they must be thrown in the System.arraycopy
2347 // activation frame; we could save some checks if this would not be the case
2348 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2349   ciArrayKlass* default_type = op->expected_type();
2350   Register src = op->src()->as_register();
2351   Register dst = op->dst()->as_register();
2352   Register src_pos = op->src_pos()->as_register();
2353   Register dst_pos = op->dst_pos()->as_register();
2354   Register length  = op->length()->as_register();
2355   Register tmp = op->tmp()->as_register();
2356   Register tmp_load_klass = rscratch1;
2357   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2358 
2359   CodeStub* stub = op->stub();
2360   int flags = op->flags();
2361   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2362   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2363 






2364   // if we don't know anything, just go through the generic arraycopy
2365   if (default_type == nullptr) {
2366     // save outgoing arguments on stack in case call to System.arraycopy is needed
2367     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2368     // for interpreter calling conventions. Now we have to do it in new style conventions.
2369     // For the moment until C1 gets the new register allocator I just force all the
2370     // args to the right place (except the register args) and then on the back side
2371     // reload the register args properly if we go slow path. Yuck
2372 
2373     // These are proper for the calling convention
2374     store_parameter(length, 2);
2375     store_parameter(dst_pos, 1);
2376     store_parameter(dst, 0);
2377 
2378     // these are just temporary placements until we need to reload
2379     store_parameter(src_pos, 3);
2380     store_parameter(src, 4);
2381 
2382     address copyfunc_addr = StubRoutines::generic_arraycopy();
2383     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");

2420     __ mov(tmp, rax);
2421     __ xorl(tmp, -1);
2422 
2423     // Reload values from the stack so they are where the stub
2424     // expects them.
2425     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2426     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2427     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2428     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2429     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2430 
2431     __ subl(length, tmp);
2432     __ addl(src_pos, tmp);
2433     __ addl(dst_pos, tmp);
2434     __ jmp(*stub->entry());
2435 
2436     __ bind(*stub->continuation());
2437     return;
2438   }
2439 








2440   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2441 
2442   int elem_size = type2aelembytes(basic_type);
2443   Address::ScaleFactor scale;
2444 
2445   switch (elem_size) {
2446     case 1 :
2447       scale = Address::times_1;
2448       break;
2449     case 2 :
2450       scale = Address::times_2;
2451       break;
2452     case 4 :
2453       scale = Address::times_4;
2454       break;
2455     case 8 :
2456       scale = Address::times_8;
2457       break;
2458     default:
2459       scale = Address::no_scale;

2646       }
2647 
2648       __ jmp(*stub->entry());
2649 
2650       __ bind(cont);
2651       __ pop(dst);
2652       __ pop(src);
2653     }
2654   }
2655 
2656 #ifdef ASSERT
2657   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2658     // Sanity check the known type with the incoming class.  For the
2659     // primitive case the types must match exactly with src.klass and
2660     // dst.klass each exactly matching the default type.  For the
2661     // object array case, if no type check is needed then either the
2662     // dst type is exactly the expected type and the src type is a
2663     // subtype which we can't check or src is the same array as dst
2664     // but not necessarily exactly of type default_type.
2665     Label known_ok, halt;

2666     __ mov_metadata(tmp, default_type->constant_encoding());
2667     if (UseCompressedClassPointers) {
2668       __ encode_klass_not_null(tmp, rscratch1);
2669     }
2670 
2671     if (basic_type != T_OBJECT) {
2672       __ cmp_klass(tmp, dst, tmp2);
2673       __ jcc(Assembler::notEqual, halt);
2674       __ cmp_klass(tmp, src, tmp2);
2675       __ jcc(Assembler::equal, known_ok);
2676     } else {
2677       __ cmp_klass(tmp, dst, tmp2);
2678       __ jcc(Assembler::equal, known_ok);
2679       __ cmpptr(src, dst);
2680       __ jcc(Assembler::equal, known_ok);
2681     }
2682     __ bind(halt);
2683     __ stop("incorrect type information in arraycopy");
2684     __ bind(known_ok);
2685   }

2984         // first time here. Set profile type.
2985         __ movptr(mdo_addr, tmp);
2986 #ifdef ASSERT
2987         __ andptr(tmp, TypeEntries::type_klass_mask);
2988         __ verify_klass_ptr(tmp);
2989 #endif
2990       } else {
2991         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2992                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2993 
2994         __ testptr(mdo_addr, TypeEntries::type_unknown);
2995         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2996 
2997         __ orptr(mdo_addr, TypeEntries::type_unknown);
2998       }
2999     }
3000   }
3001   __ bind(next);
3002 }
3003 




















3004 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3005   Unimplemented();
3006 }
3007 
3008 
3009 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3010   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3011 }
3012 
3013 
3014 void LIR_Assembler::align_backward_branch_target() {
3015   __ align(BytesPerWord);
3016 }
3017 
3018 
3019 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3020   if (left->is_single_cpu()) {
3021     __ negl(left->as_register());
3022     move_regs(left->as_register(), dest->as_register());
3023 

3174 }
3175 
3176 void LIR_Assembler::membar_loadstore() {
3177   // no-op
3178   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3179 }
3180 
3181 void LIR_Assembler::membar_storeload() {
3182   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3183 }
3184 
3185 void LIR_Assembler::on_spin_wait() {
3186   __ pause ();
3187 }
3188 
3189 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3190   assert(result_reg->is_register(), "check");
3191   __ mov(result_reg->as_register(), r15_thread);
3192 }
3193 



3194 
3195 void LIR_Assembler::peephole(LIR_List*) {
3196   // do nothing for now
3197 }
3198 
3199 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3200   assert(data == dest, "xchg/xadd uses only 2 operands");
3201 
3202   if (data->type() == T_INT) {
3203     if (code == lir_xadd) {
3204       __ lock();
3205       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3206     } else {
3207       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3208     }
3209   } else if (data->is_oop()) {
3210     assert (code == lir_xchg, "xadd for oops");
3211     Register obj = data->as_register();
3212     if (UseCompressedOops) {
3213       __ encode_heap_oop(obj);

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInlineKlass.hpp"
  35 #include "ci/ciInstance.hpp"
  36 #include "compiler/oopMap.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/gc_globals.hpp"
  39 #include "nativeInst_x86.hpp"
  40 #include "oops/oop.inline.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "utilities/powerOfTwo.hpp"
  47 #include "vmreg_x86.inline.hpp"
  48 
  49 
  50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  51 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  52 // fast versions of NegF/NegD and AbsF/AbsD.
  53 
  54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  56   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  57   // of 128-bits operands for SSE instructions.
  58   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  59   // Store the value to a 128-bits operand.
  60   operand[0] = lo;

 413   // Perform needed unlocking
 414   MonitorExitStub* stub = nullptr;
 415   if (method()->is_synchronized()) {
 416     monitor_address(0, FrameMap::rax_opr);
 417     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 418     __ unlock_object(rdi, rsi, rax, *stub->entry());
 419     __ bind(*stub->continuation());
 420   }
 421 
 422   if (compilation()->env()->dtrace_method_probes()) {
 423     __ mov(rdi, r15_thread);
 424     __ mov_metadata(rsi, method()->constant_encoding());
 425     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 426   }
 427 
 428   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 429     __ mov(rax, rbx);  // Restore the exception
 430   }
 431 
 432   // remove the activation and dispatch to the unwind handler
 433   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 434   __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
 435 
 436   // Emit the slow path assembly
 437   if (stub != nullptr) {
 438     stub->emit_code(this);
 439   }
 440 
 441   return offset;
 442 }
 443 
 444 
 445 int LIR_Assembler::emit_deopt_handler() {
 446   // generate code for exception handler
 447   address handler_base = __ start_a_stub(deopt_handler_size());
 448   if (handler_base == nullptr) {
 449     // not enough space left for the handler
 450     bailout("deopt handler overflow");
 451     return -1;
 452   }
 453 
 454   int offset = code_offset();
 455   InternalAddress here(__ pc());
 456 
 457   __ pushptr(here.addr(), rscratch1);
 458   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 459   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 460   __ end_a_stub();
 461 
 462   return offset;
 463 }
 464 
 465 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 466   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 467   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 468     assert(result->fpu() == 0, "result must already be on TOS");
 469   }
 470   if (InlineTypeReturnedAsFields) {
 471   #ifndef _LP64
 472      Unimplemented();
 473   #endif
 474     // Check if we are returning an non-null inline type and load its fields into registers
 475     ciType* return_type = compilation()->method()->return_type();
 476     if (return_type->is_inlinetype()) {
 477       ciInlineKlass* vk = return_type->as_inline_klass();
 478       if (vk->can_be_returned_as_fields()) {
 479         address unpack_handler = vk->unpack_handler();
 480         assert(unpack_handler != nullptr, "must be");
 481         __ call(RuntimeAddress(unpack_handler));
 482       }
 483     } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
 484       Label skip;
 485       Label not_null;
 486       __ testptr(rax, rax);
 487       __ jcc(Assembler::notZero, not_null);
 488       // Returned value is null, zero all return registers because they may belong to oop fields
 489       __ xorq(j_rarg1, j_rarg1);
 490       __ xorq(j_rarg2, j_rarg2);
 491       __ xorq(j_rarg3, j_rarg3);
 492       __ xorq(j_rarg4, j_rarg4);
 493       __ xorq(j_rarg5, j_rarg5);
 494       __ jmp(skip);
 495       __ bind(not_null);
 496 
 497       // Check if we are returning an non-null inline type and load its fields into registers
 498       __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
 499 
 500       // Load fields from a buffered value with an inline class specific handler
 501       __ load_klass(rdi, rax, rscratch1);
 502       __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 503       __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
 504       // Unpack handler can be null if inline type is not scalarizable in returns
 505       __ testptr(rdi, rdi);
 506       __ jcc(Assembler::zero, skip);
 507       __ call(rdi);
 508 
 509       __ bind(skip);
 510     }
 511     // At this point, rax points to the value object (for interpreter or C1 caller).
 512     // The fields of the object are copied into registers (for C2 caller).
 513   }
 514 
 515   // Pop the stack before the safepoint code
 516   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 517 
 518   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 519     __ reserved_stack_check();
 520   }
 521 
 522   // Note: we do not need to round double result; float result has the right precision
 523   // the poll sets the condition code, but no data registers
 524 
 525   code_stub->set_safepoint_offset(__ offset());
 526   __ relocate(relocInfo::poll_return_type);
 527   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 528   __ ret(0);
 529 }
 530 
 531 
 532 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 533   return (__ store_inline_type_fields_to_buf(vk, false));
 534 }
 535 
 536 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 537   guarantee(info != nullptr, "Shouldn't be null");
 538   int offset = __ offset();
 539   const Register poll_addr = rscratch1;
 540   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 541   add_debug_info_for_branch(info);
 542   __ relocate(relocInfo::poll_type);
 543   address pre_pc = __ pc();
 544   __ testl(rax, Address(poll_addr, 0));
 545   address post_pc = __ pc();
 546   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 547   return offset;
 548 }
 549 
 550 
 551 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 552   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 553 }
 554 
 555 void LIR_Assembler::swap_reg(Register a, Register b) {

1254     // init_state needs acquire, but x86 is TSO, and so we are already good.
1255     __ cmpb(Address(op->klass()->as_register(),
1256                     InstanceKlass::init_state_offset()),
1257                     InstanceKlass::fully_initialized);
1258     __ jcc(Assembler::notEqual, *op->stub()->entry());
1259   }
1260   __ allocate_object(op->obj()->as_register(),
1261                      op->tmp1()->as_register(),
1262                      op->tmp2()->as_register(),
1263                      op->header_size(),
1264                      op->object_size(),
1265                      op->klass()->as_register(),
1266                      *op->stub()->entry());
1267   __ bind(*op->stub()->continuation());
1268 }
1269 
1270 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1271   Register len =  op->len()->as_register();
1272   __ movslq(len, len);
1273 
1274   if (UseSlowPath || op->always_slow_path() ||
1275       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1276       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1277     __ jmp(*op->stub()->entry());
1278   } else {
1279     Register tmp1 = op->tmp1()->as_register();
1280     Register tmp2 = op->tmp2()->as_register();
1281     Register tmp3 = op->tmp3()->as_register();
1282     if (len == tmp1) {
1283       tmp1 = tmp3;
1284     } else if (len == tmp2) {
1285       tmp2 = tmp3;
1286     } else if (len == tmp3) {
1287       // everything is ok
1288     } else {
1289       __ mov(tmp3, len);
1290     }
1291     __ allocate_array(op->obj()->as_register(),
1292                       len,
1293                       tmp1,
1294                       tmp2,

1353     assert(data != nullptr,                "need data for type check");
1354     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1355   }
1356   Label* success_target = success;
1357   Label* failure_target = failure;
1358 
1359   if (obj == k_RInfo) {
1360     k_RInfo = dst;
1361   } else if (obj == klass_RInfo) {
1362     klass_RInfo = dst;
1363   }
1364   if (k->is_loaded() && !UseCompressedClassPointers) {
1365     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1366   } else {
1367     Rtmp1 = op->tmp3()->as_register();
1368     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1369   }
1370 
1371   assert_different_registers(obj, k_RInfo, klass_RInfo);
1372 
1373   if (op->need_null_check()) {
1374     __ testptr(obj, obj);
1375     if (op->should_profile()) {
1376       Label not_null;
1377       Register mdo  = klass_RInfo;
1378       __ mov_metadata(mdo, md->constant_encoding());
1379       __ jccb(Assembler::notEqual, not_null);
1380       // Object is null; update MDO and exit
1381       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1382       int header_bits = BitData::null_seen_byte_constant();
1383       __ orb(data_addr, header_bits);
1384       __ jmp(*obj_is_null);
1385       __ bind(not_null);
1386 
1387       Label update_done;
1388       Register recv = k_RInfo;
1389       __ load_klass(recv, obj, tmp_load_klass);
1390       type_profile_helper(mdo, md, data, recv, &update_done);
1391 
1392       Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1393       __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1394 
1395       __ bind(update_done);
1396     } else {
1397       __ jcc(Assembler::equal, *obj_is_null);
1398     }
1399   }
1400 
1401   if (!k->is_loaded()) {
1402     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1403   } else {
1404     __ mov_metadata(k_RInfo, k->constant_encoding());
1405   }
1406   __ verify_oop(obj);
1407 
1408   if (op->fast_check()) {
1409     // TODO 8366668 Is this correct? I don't think so. Probably we now always go to the slow path here. Same on AArch64.
1410     // get object class
1411     // not a safepoint as obj null check happens earlier
1412     if (UseCompressedClassPointers) {
1413       __ load_klass(Rtmp1, obj, tmp_load_klass);
1414       __ cmpptr(k_RInfo, Rtmp1);
1415     } else {
1416       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1417     }
1418     __ jcc(Assembler::notEqual, *failure_target);
1419     // successful cast, fall through to profile or jump
1420   } else {
1421     // get object class
1422     // not a safepoint as obj null check happens earlier
1423     __ load_klass(klass_RInfo, obj, tmp_load_klass);
1424     if (k->is_loaded()) {
1425       // See if we get an immediate positive hit
1426       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1427       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1428         __ jcc(Assembler::notEqual, *failure_target);
1429         // successful cast, fall through to profile or jump

1549         __ mov(dst, obj);
1550       }
1551     } else
1552       if (code == lir_instanceof) {
1553         Register obj = op->object()->as_register();
1554         Register dst = op->result_opr()->as_register();
1555         Label success, failure, done;
1556         emit_typecheck_helper(op, &success, &failure, &failure);
1557         __ bind(failure);
1558         __ xorptr(dst, dst);
1559         __ jmpb(done);
1560         __ bind(success);
1561         __ movptr(dst, 1);
1562         __ bind(done);
1563       } else {
1564         ShouldNotReachHere();
1565       }
1566 
1567 }
1568 
1569 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1570   // We are loading/storing from/to an array that *may* be a flat array (the
1571   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1572   // If this array is a flat array, take the slow path.
1573   __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1574   if (!op->value()->is_illegal()) {
1575     // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1576     // The array is not a flat array, but it might be null-free. If we are storing
1577     // a null into a null-free array, take the slow path (which will throw NPE).
1578     Label skip;
1579     __ cmpptr(op->value()->as_register(), NULL_WORD);
1580     __ jcc(Assembler::notEqual, skip);
1581     __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1582     __ bind(skip);
1583   }
1584 }
1585 
1586 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1587   // We are storing into an array that *may* be null-free (the declared type is
1588   // Object[], abstract[], interface[] or VT.ref[]).
1589   Label test_mark_word;
1590   Register tmp = op->tmp()->as_register();
1591   __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1592   __ testl(tmp, markWord::unlocked_value);
1593   __ jccb(Assembler::notZero, test_mark_word);
1594   __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1595   __ bind(test_mark_word);
1596   __ testl(tmp, markWord::null_free_array_bit_in_place);
1597 }
1598 
1599 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1600   Label L_oops_equal;
1601   Label L_oops_not_equal;
1602   Label L_end;
1603 
1604   Register left  = op->left()->as_register();
1605   Register right = op->right()->as_register();
1606 
1607   __ cmpptr(left, right);
1608   __ jcc(Assembler::equal, L_oops_equal);
1609 
1610   // (1) Null check -- if one of the operands is null, the other must not be null (because
1611   //     the two references are not equal), so they are not substitutable,
1612   //     FIXME: do null check only if the operand is nullable
1613   __ testptr(left, right);
1614   __ jcc(Assembler::zero, L_oops_not_equal);
1615 
1616   ciKlass* left_klass = op->left_klass();
1617   ciKlass* right_klass = op->right_klass();
1618 
1619   // (2) Inline type check -- if either of the operands is not a inline type,
1620   //     they are not substitutable. We do this only if we are not sure that the
1621   //     operands are inline type
1622   if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1623       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1624     Register tmp1  = op->tmp1()->as_register();
1625     __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1626     __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1627     __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1628     __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1629     __ jcc(Assembler::notEqual, L_oops_not_equal);
1630   }
1631 
1632   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1633   if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1634     // No need to load klass -- the operands are statically known to be the same inline klass.
1635     __ jmp(*op->stub()->entry());
1636   } else {
1637     Register left_klass_op = op->left_klass_op()->as_register();
1638     Register right_klass_op = op->right_klass_op()->as_register();
1639 
1640     if (UseCompressedClassPointers) {
1641       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1642       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1643       __ cmpl(left_klass_op, right_klass_op);
1644     } else {
1645       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1646       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1647       __ cmpptr(left_klass_op, right_klass_op);
1648     }
1649 
1650     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1651     // fall through to L_oops_not_equal
1652   }
1653 
1654   __ bind(L_oops_not_equal);
1655   move(op->not_equal_result(), op->result_opr());
1656   __ jmp(L_end);
1657 
1658   __ bind(L_oops_equal);
1659   move(op->equal_result(), op->result_opr());
1660   __ jmp(L_end);
1661 
1662   // We've returned from the stub. RAX contains 0x0 IFF the two
1663   // operands are not substitutable. (Don't compare against 0x1 in case the
1664   // C compiler is naughty)
1665   __ bind(*op->stub()->continuation());
1666   __ cmpl(rax, 0);
1667   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1668   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1669   // fall-through
1670   __ bind(L_end);
1671 }
1672 
1673 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1674   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1675     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1676     Register newval = op->new_value()->as_register();
1677     Register cmpval = op->cmp_value()->as_register();
1678     assert(cmpval == rax, "wrong register");
1679     assert(newval != noreg, "new val must be register");
1680     assert(cmpval != newval, "cmp and new values must be in different registers");
1681     assert(cmpval != addr, "cmp and addr must be in different registers");
1682     assert(newval != addr, "new value and addr must be in different registers");
1683 
1684     if (op->code() == lir_cas_obj) {
1685       if (UseCompressedOops) {
1686         __ encode_heap_oop(cmpval);
1687         __ mov(rscratch1, newval);
1688         __ encode_heap_oop(rscratch1);
1689         __ lock();
1690         // cmpval (rax) is implicitly used by this instruction
1691         __ cmpxchgl(rscratch1, Address(addr, 0));

1697       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1698       __ lock();
1699       __ cmpxchgl(newval, Address(addr, 0));
1700     }
1701   } else if (op->code() == lir_cas_long) {
1702     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1703     Register newval = op->new_value()->as_register_lo();
1704     Register cmpval = op->cmp_value()->as_register_lo();
1705     assert(cmpval == rax, "wrong register");
1706     assert(newval != noreg, "new val must be register");
1707     assert(cmpval != newval, "cmp and new values must be in different registers");
1708     assert(cmpval != addr, "cmp and addr must be in different registers");
1709     assert(newval != addr, "new value and addr must be in different registers");
1710     __ lock();
1711     __ cmpxchgq(newval, Address(addr, 0));
1712   } else {
1713     Unimplemented();
1714   }
1715 }
1716 
1717 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1718   assert(dst->is_cpu_register(), "must be");
1719   assert(dst->type() == src->type(), "must be");
1720 
1721   if (src->is_cpu_register()) {
1722     reg2reg(src, dst);
1723   } else if (src->is_stack()) {
1724     stack2reg(src, dst, dst->type());
1725   } else if (src->is_constant()) {
1726     const2reg(src, dst, lir_patch_none, nullptr);
1727   } else {
1728     ShouldNotReachHere();
1729   }
1730 }
1731 
1732 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1733                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1734   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1735 
1736   Assembler::Condition acond, ncond;
1737   switch (condition) {
1738     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1739     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1740     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1741     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1742     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1743     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1744     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1745     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1746     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1747                                 ShouldNotReachHere();
1748   }
1749 
1750   if (opr1->is_cpu_register()) {
1751     reg2reg(opr1, result);

2322   int offset = __ offset();
2323   switch (code) {
2324   case lir_static_call:
2325   case lir_optvirtual_call:
2326   case lir_dynamic_call:
2327     offset += NativeCall::displacement_offset;
2328     break;
2329   case lir_icvirtual_call:
2330     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2331     break;
2332   default: ShouldNotReachHere();
2333   }
2334   __ align(BytesPerWord, offset);
2335 }
2336 
2337 
2338 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2339   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2340          "must be aligned");
2341   __ call(AddressLiteral(op->addr(), rtype));
2342   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2343   __ post_call_nop();
2344 }
2345 
2346 
2347 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2348   __ ic_call(op->addr());
2349   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2350   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2351          "must be aligned");
2352   __ post_call_nop();
2353 }
2354 
2355 
2356 void LIR_Assembler::emit_static_call_stub() {
2357   address call_pc = __ pc();
2358   address stub = __ start_a_stub(call_stub_size());
2359   if (stub == nullptr) {
2360     bailout("static call stub overflow");
2361     return;
2362   }
2363 
2364   int start = __ offset();
2365 
2366   // make sure that the displacement word of the call ends up word aligned
2367   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2368   __ relocate(static_stub_Relocation::spec(call_pc));
2369   __ mov_metadata(rbx, (Metadata*)nullptr);

2496   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2497 }
2498 
2499 
2500 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2501   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2502   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2503   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2504   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2505 }
2506 
2507 
2508 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2509   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2510   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2511   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2512   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2513 }
2514 
2515 
2516 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2517   if (null_check) {
2518     __ testptr(obj, obj);
2519     __ jcc(Assembler::zero, *slow_path->entry());
2520   }
2521   if (is_dest) {
2522     __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2523     // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2524     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2525   } else {
2526     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2527   }
2528 }
2529 
2530 
2531 // This code replaces a call to arraycopy; no exception may
2532 // be thrown in this code, they must be thrown in the System.arraycopy
2533 // activation frame; we could save some checks if this would not be the case
2534 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2535   ciArrayKlass* default_type = op->expected_type();
2536   Register src = op->src()->as_register();
2537   Register dst = op->dst()->as_register();
2538   Register src_pos = op->src_pos()->as_register();
2539   Register dst_pos = op->dst_pos()->as_register();
2540   Register length  = op->length()->as_register();
2541   Register tmp = op->tmp()->as_register();
2542   Register tmp_load_klass = rscratch1;
2543   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2544 
2545   CodeStub* stub = op->stub();
2546   int flags = op->flags();
2547   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2548   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2549 
2550   if (flags & LIR_OpArrayCopy::always_slow_path) {
2551     __ jmp(*stub->entry());
2552     __ bind(*stub->continuation());
2553     return;
2554   }
2555 
2556   // if we don't know anything, just go through the generic arraycopy
2557   if (default_type == nullptr) {
2558     // save outgoing arguments on stack in case call to System.arraycopy is needed
2559     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2560     // for interpreter calling conventions. Now we have to do it in new style conventions.
2561     // For the moment until C1 gets the new register allocator I just force all the
2562     // args to the right place (except the register args) and then on the back side
2563     // reload the register args properly if we go slow path. Yuck
2564 
2565     // These are proper for the calling convention
2566     store_parameter(length, 2);
2567     store_parameter(dst_pos, 1);
2568     store_parameter(dst, 0);
2569 
2570     // these are just temporary placements until we need to reload
2571     store_parameter(src_pos, 3);
2572     store_parameter(src, 4);
2573 
2574     address copyfunc_addr = StubRoutines::generic_arraycopy();
2575     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");

2612     __ mov(tmp, rax);
2613     __ xorl(tmp, -1);
2614 
2615     // Reload values from the stack so they are where the stub
2616     // expects them.
2617     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2618     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2619     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2620     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2621     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2622 
2623     __ subl(length, tmp);
2624     __ addl(src_pos, tmp);
2625     __ addl(dst_pos, tmp);
2626     __ jmp(*stub->entry());
2627 
2628     __ bind(*stub->continuation());
2629     return;
2630   }
2631 
2632   // Handle inline type arrays
2633   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2634     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2635   }
2636   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2637     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2638   }
2639 
2640   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2641 
2642   int elem_size = type2aelembytes(basic_type);
2643   Address::ScaleFactor scale;
2644 
2645   switch (elem_size) {
2646     case 1 :
2647       scale = Address::times_1;
2648       break;
2649     case 2 :
2650       scale = Address::times_2;
2651       break;
2652     case 4 :
2653       scale = Address::times_4;
2654       break;
2655     case 8 :
2656       scale = Address::times_8;
2657       break;
2658     default:
2659       scale = Address::no_scale;

2846       }
2847 
2848       __ jmp(*stub->entry());
2849 
2850       __ bind(cont);
2851       __ pop(dst);
2852       __ pop(src);
2853     }
2854   }
2855 
2856 #ifdef ASSERT
2857   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2858     // Sanity check the known type with the incoming class.  For the
2859     // primitive case the types must match exactly with src.klass and
2860     // dst.klass each exactly matching the default type.  For the
2861     // object array case, if no type check is needed then either the
2862     // dst type is exactly the expected type and the src type is a
2863     // subtype which we can't check or src is the same array as dst
2864     // but not necessarily exactly of type default_type.
2865     Label known_ok, halt;
2866 
2867     __ mov_metadata(tmp, default_type->constant_encoding());
2868     if (UseCompressedClassPointers) {
2869       __ encode_klass_not_null(tmp, rscratch1);
2870     }
2871 
2872     if (basic_type != T_OBJECT) {
2873       __ cmp_klass(tmp, dst, tmp2);
2874       __ jcc(Assembler::notEqual, halt);
2875       __ cmp_klass(tmp, src, tmp2);
2876       __ jcc(Assembler::equal, known_ok);
2877     } else {
2878       __ cmp_klass(tmp, dst, tmp2);
2879       __ jcc(Assembler::equal, known_ok);
2880       __ cmpptr(src, dst);
2881       __ jcc(Assembler::equal, known_ok);
2882     }
2883     __ bind(halt);
2884     __ stop("incorrect type information in arraycopy");
2885     __ bind(known_ok);
2886   }

3185         // first time here. Set profile type.
3186         __ movptr(mdo_addr, tmp);
3187 #ifdef ASSERT
3188         __ andptr(tmp, TypeEntries::type_klass_mask);
3189         __ verify_klass_ptr(tmp);
3190 #endif
3191       } else {
3192         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3193                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3194 
3195         __ testptr(mdo_addr, TypeEntries::type_unknown);
3196         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3197 
3198         __ orptr(mdo_addr, TypeEntries::type_unknown);
3199       }
3200     }
3201   }
3202   __ bind(next);
3203 }
3204 
3205 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3206   Register obj = op->obj()->as_register();
3207   Register tmp = op->tmp()->as_pointer_register();
3208   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3209   bool not_null = op->not_null();
3210   int flag = op->flag();
3211 
3212   Label not_inline_type;
3213   if (!not_null) {
3214     __ testptr(obj, obj);
3215     __ jccb(Assembler::zero, not_inline_type);
3216   }
3217 
3218   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3219 
3220   __ orb(mdo_addr, flag);
3221 
3222   __ bind(not_inline_type);
3223 }
3224 
3225 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3226   Unimplemented();
3227 }
3228 
3229 
3230 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3231   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3232 }
3233 
3234 
3235 void LIR_Assembler::align_backward_branch_target() {
3236   __ align(BytesPerWord);
3237 }
3238 
3239 
3240 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3241   if (left->is_single_cpu()) {
3242     __ negl(left->as_register());
3243     move_regs(left->as_register(), dest->as_register());
3244 

3395 }
3396 
3397 void LIR_Assembler::membar_loadstore() {
3398   // no-op
3399   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3400 }
3401 
3402 void LIR_Assembler::membar_storeload() {
3403   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3404 }
3405 
3406 void LIR_Assembler::on_spin_wait() {
3407   __ pause ();
3408 }
3409 
3410 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3411   assert(result_reg->is_register(), "check");
3412   __ mov(result_reg->as_register(), r15_thread);
3413 }
3414 
3415 void LIR_Assembler::check_orig_pc() {
3416   __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3417 }
3418 
3419 void LIR_Assembler::peephole(LIR_List*) {
3420   // do nothing for now
3421 }
3422 
3423 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3424   assert(data == dest, "xchg/xadd uses only 2 operands");
3425 
3426   if (data->type() == T_INT) {
3427     if (code == lir_xadd) {
3428       __ lock();
3429       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3430     } else {
3431       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3432     }
3433   } else if (data->is_oop()) {
3434     assert (code == lir_xchg, "xadd for oops");
3435     Register obj = data->as_register();
3436     if (UseCompressedOops) {
3437       __ encode_heap_oop(obj);
< prev index next >