< prev index next >

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

Print this page

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"

  34 #include "ci/ciInstance.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/collectedHeap.hpp"
  37 #include "gc/shared/gc_globals.hpp"
  38 #include "nativeInst_x86.hpp"

  39 #include "oops/objArrayKlass.hpp"
  40 #include "runtime/frame.inline.hpp"
  41 #include "runtime/safepointMechanism.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "runtime/stubRoutines.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_x86.inline.hpp"
  46 
  47 
  48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  49 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  50 // fast versions of NegF/NegD and AbsF/AbsD.
  51 
  52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  54   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  55   // of 128-bits operands for SSE instructions.
  56   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  57   // Store the value to a 128-bits operand.
  58   operand[0] = lo;

 415     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 416     if (LockingMode == LM_MONITOR) {
 417       __ jmp(*stub->entry());
 418     } else {
 419       __ unlock_object(rdi, rsi, rax, *stub->entry());
 420     }
 421     __ bind(*stub->continuation());
 422   }
 423 
 424   if (compilation()->env()->dtrace_method_probes()) {
 425     __ mov(rdi, r15_thread);
 426     __ mov_metadata(rsi, method()->constant_encoding());
 427     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 428   }
 429 
 430   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 431     __ mov(rax, rbx);  // Restore the exception
 432   }
 433 
 434   // remove the activation and dispatch to the unwind handler
 435   __ remove_frame(initial_frame_size_in_bytes());
 436   __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
 437 
 438   // Emit the slow path assembly
 439   if (stub != nullptr) {
 440     stub->emit_code(this);
 441   }
 442 
 443   return offset;
 444 }
 445 
 446 
 447 int LIR_Assembler::emit_deopt_handler() {
 448   // generate code for exception handler
 449   address handler_base = __ start_a_stub(deopt_handler_size());
 450   if (handler_base == nullptr) {
 451     // not enough space left for the handler
 452     bailout("deopt handler overflow");
 453     return -1;
 454   }
 455 
 456   int offset = code_offset();
 457   InternalAddress here(__ pc());
 458 
 459   __ pushptr(here.addr(), rscratch1);
 460   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 461   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 462   __ end_a_stub();
 463 
 464   return offset;
 465 }
 466 
 467 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 468   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 469   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 470     assert(result->fpu() == 0, "result must already be on TOS");
 471   }































 472 
 473   // Pop the stack before the safepoint code
 474   __ remove_frame(initial_frame_size_in_bytes());
 475 
 476   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 477     __ reserved_stack_check();
 478   }
 479 
 480   // Note: we do not need to round double result; float result has the right precision
 481   // the poll sets the condition code, but no data registers
 482 
 483   code_stub->set_safepoint_offset(__ offset());
 484   __ relocate(relocInfo::poll_return_type);
 485   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 486   __ ret(0);
 487 }
 488 
 489 




 490 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 491   guarantee(info != nullptr, "Shouldn't be null");
 492   int offset = __ offset();
 493   const Register poll_addr = rscratch1;
 494   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 495   add_debug_info_for_branch(info);
 496   __ relocate(relocInfo::poll_type);
 497   address pre_pc = __ pc();
 498   __ testl(rax, Address(poll_addr, 0));
 499   address post_pc = __ pc();
 500   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 501   return offset;
 502 }
 503 
 504 
 505 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 506   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 507 }
 508 
 509 void LIR_Assembler::swap_reg(Register a, Register b) {

1208     // init_state needs acquire, but x86 is TSO, and so we are already good.
1209     __ cmpb(Address(op->klass()->as_register(),
1210                     InstanceKlass::init_state_offset()),
1211                     InstanceKlass::fully_initialized);
1212     __ jcc(Assembler::notEqual, *op->stub()->entry());
1213   }
1214   __ allocate_object(op->obj()->as_register(),
1215                      op->tmp1()->as_register(),
1216                      op->tmp2()->as_register(),
1217                      op->header_size(),
1218                      op->object_size(),
1219                      op->klass()->as_register(),
1220                      *op->stub()->entry());
1221   __ bind(*op->stub()->continuation());
1222 }
1223 
1224 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1225   Register len =  op->len()->as_register();
1226   __ movslq(len, len);
1227 
1228   if (UseSlowPath ||
1229       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1230       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1231     __ jmp(*op->stub()->entry());
1232   } else {
1233     Register tmp1 = op->tmp1()->as_register();
1234     Register tmp2 = op->tmp2()->as_register();
1235     Register tmp3 = op->tmp3()->as_register();
1236     if (len == tmp1) {
1237       tmp1 = tmp3;
1238     } else if (len == tmp2) {
1239       tmp2 = tmp3;
1240     } else if (len == tmp3) {
1241       // everything is ok
1242     } else {
1243       __ mov(tmp3, len);
1244     }
1245     __ allocate_array(op->obj()->as_register(),
1246                       len,
1247                       tmp1,
1248                       tmp2,

1307     assert(data != nullptr,                "need data for type check");
1308     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1309   }
1310   Label* success_target = success;
1311   Label* failure_target = failure;
1312 
1313   if (obj == k_RInfo) {
1314     k_RInfo = dst;
1315   } else if (obj == klass_RInfo) {
1316     klass_RInfo = dst;
1317   }
1318   if (k->is_loaded() && !UseCompressedClassPointers) {
1319     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1320   } else {
1321     Rtmp1 = op->tmp3()->as_register();
1322     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1323   }
1324 
1325   assert_different_registers(obj, k_RInfo, klass_RInfo);
1326 
1327   __ testptr(obj, obj);
1328   if (op->should_profile()) {
1329     Label not_null;
1330     Register mdo  = klass_RInfo;
1331     __ mov_metadata(mdo, md->constant_encoding());
1332     __ jccb(Assembler::notEqual, not_null);
1333     // Object is null; update MDO and exit
1334     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1335     int header_bits = BitData::null_seen_byte_constant();
1336     __ orb(data_addr, header_bits);
1337     __ jmp(*obj_is_null);
1338     __ bind(not_null);
1339 
1340     Label update_done;
1341     Register recv = k_RInfo;
1342     __ load_klass(recv, obj, tmp_load_klass);
1343     type_profile_helper(mdo, md, data, recv, &update_done);
1344 
1345     Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1346     __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1347 
1348     __ bind(update_done);
1349   } else {
1350     __ jcc(Assembler::equal, *obj_is_null);


1351   }
1352 
1353   if (!k->is_loaded()) {
1354     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1355   } else {
1356     __ mov_metadata(k_RInfo, k->constant_encoding());
1357   }
1358   __ verify_oop(obj);
1359 
1360   if (op->fast_check()) {
1361     // get object class
1362     // not a safepoint as obj null check happens earlier
1363     if (UseCompressedClassPointers) {
1364       __ load_klass(Rtmp1, obj, tmp_load_klass);
1365       __ cmpptr(k_RInfo, Rtmp1);
1366     } else {
1367       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1368     }
1369     __ jcc(Assembler::notEqual, *failure_target);
1370     // successful cast, fall through to profile or jump

1500         __ mov(dst, obj);
1501       }
1502     } else
1503       if (code == lir_instanceof) {
1504         Register obj = op->object()->as_register();
1505         Register dst = op->result_opr()->as_register();
1506         Label success, failure, done;
1507         emit_typecheck_helper(op, &success, &failure, &failure);
1508         __ bind(failure);
1509         __ xorptr(dst, dst);
1510         __ jmpb(done);
1511         __ bind(success);
1512         __ movptr(dst, 1);
1513         __ bind(done);
1514       } else {
1515         ShouldNotReachHere();
1516       }
1517 
1518 }
1519 







































































































1520 
1521 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1522   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1523     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1524     Register newval = op->new_value()->as_register();
1525     Register cmpval = op->cmp_value()->as_register();
1526     assert(cmpval == rax, "wrong register");
1527     assert(newval != noreg, "new val must be register");
1528     assert(cmpval != newval, "cmp and new values must be in different registers");
1529     assert(cmpval != addr, "cmp and addr must be in different registers");
1530     assert(newval != addr, "new value and addr must be in different registers");
1531 
1532     if (op->code() == lir_cas_obj) {
1533       if (UseCompressedOops) {
1534         __ encode_heap_oop(cmpval);
1535         __ mov(rscratch1, newval);
1536         __ encode_heap_oop(rscratch1);
1537         __ lock();
1538         // cmpval (rax) is implicitly used by this instruction
1539         __ cmpxchgl(rscratch1, Address(addr, 0));

1545       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1546       __ lock();
1547       __ cmpxchgl(newval, Address(addr, 0));
1548     }
1549   } else if (op->code() == lir_cas_long) {
1550     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1551     Register newval = op->new_value()->as_register_lo();
1552     Register cmpval = op->cmp_value()->as_register_lo();
1553     assert(cmpval == rax, "wrong register");
1554     assert(newval != noreg, "new val must be register");
1555     assert(cmpval != newval, "cmp and new values must be in different registers");
1556     assert(cmpval != addr, "cmp and addr must be in different registers");
1557     assert(newval != addr, "new value and addr must be in different registers");
1558     __ lock();
1559     __ cmpxchgq(newval, Address(addr, 0));
1560   } else {
1561     Unimplemented();
1562   }
1563 }
1564 















1565 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1566                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1567   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1568 
1569   Assembler::Condition acond, ncond;
1570   switch (condition) {
1571     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1572     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1573     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1574     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1575     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1576     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1577     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1578     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1579     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1580                                 ShouldNotReachHere();
1581   }
1582 
1583   if (opr1->is_cpu_register()) {
1584     reg2reg(opr1, result);

2155   int offset = __ offset();
2156   switch (code) {
2157   case lir_static_call:
2158   case lir_optvirtual_call:
2159   case lir_dynamic_call:
2160     offset += NativeCall::displacement_offset;
2161     break;
2162   case lir_icvirtual_call:
2163     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2164     break;
2165   default: ShouldNotReachHere();
2166   }
2167   __ align(BytesPerWord, offset);
2168 }
2169 
2170 
2171 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2172   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2173          "must be aligned");
2174   __ call(AddressLiteral(op->addr(), rtype));
2175   add_call_info(code_offset(), op->info());
2176   __ post_call_nop();
2177 }
2178 
2179 
2180 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2181   __ ic_call(op->addr());
2182   add_call_info(code_offset(), op->info());
2183   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2184          "must be aligned");
2185   __ post_call_nop();
2186 }
2187 
2188 
2189 void LIR_Assembler::emit_static_call_stub() {
2190   address call_pc = __ pc();
2191   address stub = __ start_a_stub(call_stub_size());
2192   if (stub == nullptr) {
2193     bailout("static call stub overflow");
2194     return;
2195   }
2196 
2197   int start = __ offset();
2198 
2199   // make sure that the displacement word of the call ends up word aligned
2200   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2201   __ relocate(static_stub_Relocation::spec(call_pc));
2202   __ mov_metadata(rbx, (Metadata*)nullptr);

2329   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2330 }
2331 
2332 
2333 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2334   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2335   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2336   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2337   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2338 }
2339 
2340 
2341 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2342   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2343   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2344   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2345   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2346 }
2347 
2348 















2349 // This code replaces a call to arraycopy; no exception may
2350 // be thrown in this code, they must be thrown in the System.arraycopy
2351 // activation frame; we could save some checks if this would not be the case
2352 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2353   ciArrayKlass* default_type = op->expected_type();
2354   Register src = op->src()->as_register();
2355   Register dst = op->dst()->as_register();
2356   Register src_pos = op->src_pos()->as_register();
2357   Register dst_pos = op->dst_pos()->as_register();
2358   Register length  = op->length()->as_register();
2359   Register tmp = op->tmp()->as_register();
2360   Register tmp_load_klass = rscratch1;
2361   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2362 
2363   CodeStub* stub = op->stub();
2364   int flags = op->flags();
2365   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2366   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2367 






2368   // if we don't know anything, just go through the generic arraycopy
2369   if (default_type == nullptr) {
2370     // save outgoing arguments on stack in case call to System.arraycopy is needed
2371     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2372     // for interpreter calling conventions. Now we have to do it in new style conventions.
2373     // For the moment until C1 gets the new register allocator I just force all the
2374     // args to the right place (except the register args) and then on the back side
2375     // reload the register args properly if we go slow path. Yuck
2376 
2377     // These are proper for the calling convention
2378     store_parameter(length, 2);
2379     store_parameter(dst_pos, 1);
2380     store_parameter(dst, 0);
2381 
2382     // these are just temporary placements until we need to reload
2383     store_parameter(src_pos, 3);
2384     store_parameter(src, 4);
2385 
2386     address copyfunc_addr = StubRoutines::generic_arraycopy();
2387     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");

2424     __ mov(tmp, rax);
2425     __ xorl(tmp, -1);
2426 
2427     // Reload values from the stack so they are where the stub
2428     // expects them.
2429     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2430     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2431     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2432     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2433     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2434 
2435     __ subl(length, tmp);
2436     __ addl(src_pos, tmp);
2437     __ addl(dst_pos, tmp);
2438     __ jmp(*stub->entry());
2439 
2440     __ bind(*stub->continuation());
2441     return;
2442   }
2443 








2444   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2445 
2446   int elem_size = type2aelembytes(basic_type);
2447   Address::ScaleFactor scale;
2448 
2449   switch (elem_size) {
2450     case 1 :
2451       scale = Address::times_1;
2452       break;
2453     case 2 :
2454       scale = Address::times_2;
2455       break;
2456     case 4 :
2457       scale = Address::times_4;
2458       break;
2459     case 8 :
2460       scale = Address::times_8;
2461       break;
2462     default:
2463       scale = Address::no_scale;

2994         // first time here. Set profile type.
2995         __ movptr(mdo_addr, tmp);
2996 #ifdef ASSERT
2997         __ andptr(tmp, TypeEntries::type_klass_mask);
2998         __ verify_klass_ptr(tmp);
2999 #endif
3000       } else {
3001         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3002                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3003 
3004         __ testptr(mdo_addr, TypeEntries::type_unknown);
3005         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3006 
3007         __ orptr(mdo_addr, TypeEntries::type_unknown);
3008       }
3009     }
3010   }
3011   __ bind(next);
3012 }
3013 




















3014 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3015   Unimplemented();
3016 }
3017 
3018 
3019 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3020   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3021 }
3022 
3023 
3024 void LIR_Assembler::align_backward_branch_target() {
3025   __ align(BytesPerWord);
3026 }
3027 
3028 
3029 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3030   if (left->is_single_cpu()) {
3031     __ negl(left->as_register());
3032     move_regs(left->as_register(), dest->as_register());
3033 

3184 }
3185 
3186 void LIR_Assembler::membar_loadstore() {
3187   // no-op
3188   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3189 }
3190 
3191 void LIR_Assembler::membar_storeload() {
3192   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3193 }
3194 
3195 void LIR_Assembler::on_spin_wait() {
3196   __ pause ();
3197 }
3198 
3199 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3200   assert(result_reg->is_register(), "check");
3201   __ mov(result_reg->as_register(), r15_thread);
3202 }
3203 



3204 
3205 void LIR_Assembler::peephole(LIR_List*) {
3206   // do nothing for now
3207 }
3208 
3209 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3210   assert(data == dest, "xchg/xadd uses only 2 operands");
3211 
3212   if (data->type() == T_INT) {
3213     if (code == lir_xadd) {
3214       __ lock();
3215       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3216     } else {
3217       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3218     }
3219   } else if (data->is_oop()) {
3220     assert (code == lir_xchg, "xadd for oops");
3221     Register obj = data->as_register();
3222     if (UseCompressedOops) {
3223       __ encode_heap_oop(obj);

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInlineKlass.hpp"
  35 #include "ci/ciInstance.hpp"
  36 #include "compiler/oopMap.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/gc_globals.hpp"
  39 #include "nativeInst_x86.hpp"
  40 #include "oops/oop.inline.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "utilities/powerOfTwo.hpp"
  47 #include "vmreg_x86.inline.hpp"
  48 
  49 
  50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  51 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  52 // fast versions of NegF/NegD and AbsF/AbsD.
  53 
  54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  56   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  57   // of 128-bits operands for SSE instructions.
  58   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  59   // Store the value to a 128-bits operand.
  60   operand[0] = lo;

 417     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 418     if (LockingMode == LM_MONITOR) {
 419       __ jmp(*stub->entry());
 420     } else {
 421       __ unlock_object(rdi, rsi, rax, *stub->entry());
 422     }
 423     __ bind(*stub->continuation());
 424   }
 425 
 426   if (compilation()->env()->dtrace_method_probes()) {
 427     __ mov(rdi, r15_thread);
 428     __ mov_metadata(rsi, method()->constant_encoding());
 429     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 430   }
 431 
 432   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 433     __ mov(rax, rbx);  // Restore the exception
 434   }
 435 
 436   // remove the activation and dispatch to the unwind handler
 437   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 438   __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
 439 
 440   // Emit the slow path assembly
 441   if (stub != nullptr) {
 442     stub->emit_code(this);
 443   }
 444 
 445   return offset;
 446 }
 447 
 448 
 449 int LIR_Assembler::emit_deopt_handler() {
 450   // generate code for exception handler
 451   address handler_base = __ start_a_stub(deopt_handler_size());
 452   if (handler_base == nullptr) {
 453     // not enough space left for the handler
 454     bailout("deopt handler overflow");
 455     return -1;
 456   }
 457 
 458   int offset = code_offset();
 459   InternalAddress here(__ pc());
 460 
 461   __ pushptr(here.addr(), rscratch1);
 462   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 463   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 464   __ end_a_stub();
 465 
 466   return offset;
 467 }
 468 
 469 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 470   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 471   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 472     assert(result->fpu() == 0, "result must already be on TOS");
 473   }
 474   if (InlineTypeReturnedAsFields) {
 475   #ifndef _LP64
 476      Unimplemented();
 477   #endif
 478     // Check if we are returning an non-null inline type and load its fields into registers
 479     ciType* return_type = compilation()->method()->return_type();
 480     if (return_type->is_inlinetype()) {
 481       ciInlineKlass* vk = return_type->as_inline_klass();
 482       if (vk->can_be_returned_as_fields()) {
 483         address unpack_handler = vk->unpack_handler();
 484         assert(unpack_handler != nullptr, "must be");
 485         __ call(RuntimeAddress(unpack_handler));
 486       }
 487     } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
 488       Label skip;
 489       __ test_oop_is_not_inline_type(rax, rscratch1, skip);
 490 
 491       // Load fields from a buffered value with an inline class specific handler
 492       __ load_klass(rdi, rax, rscratch1);
 493       __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 494       __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
 495       // Unpack handler can be null if inline type is not scalarizable in returns
 496       __ testptr(rdi, rdi);
 497       __ jcc(Assembler::zero, skip);
 498       __ call(rdi);
 499 
 500       __ bind(skip);
 501     }
 502     // At this point, rax points to the value object (for interpreter or C1 caller).
 503     // The fields of the object are copied into registers (for C2 caller).
 504   }
 505 
 506   // Pop the stack before the safepoint code
 507   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 508 
 509   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 510     __ reserved_stack_check();
 511   }
 512 
 513   // Note: we do not need to round double result; float result has the right precision
 514   // the poll sets the condition code, but no data registers
 515 
 516   code_stub->set_safepoint_offset(__ offset());
 517   __ relocate(relocInfo::poll_return_type);
 518   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 519   __ ret(0);
 520 }
 521 
 522 
 523 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 524   return (__ store_inline_type_fields_to_buf(vk, false));
 525 }
 526 
 527 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 528   guarantee(info != nullptr, "Shouldn't be null");
 529   int offset = __ offset();
 530   const Register poll_addr = rscratch1;
 531   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 532   add_debug_info_for_branch(info);
 533   __ relocate(relocInfo::poll_type);
 534   address pre_pc = __ pc();
 535   __ testl(rax, Address(poll_addr, 0));
 536   address post_pc = __ pc();
 537   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 538   return offset;
 539 }
 540 
 541 
 542 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 543   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 544 }
 545 
 546 void LIR_Assembler::swap_reg(Register a, Register b) {

1245     // init_state needs acquire, but x86 is TSO, and so we are already good.
1246     __ cmpb(Address(op->klass()->as_register(),
1247                     InstanceKlass::init_state_offset()),
1248                     InstanceKlass::fully_initialized);
1249     __ jcc(Assembler::notEqual, *op->stub()->entry());
1250   }
1251   __ allocate_object(op->obj()->as_register(),
1252                      op->tmp1()->as_register(),
1253                      op->tmp2()->as_register(),
1254                      op->header_size(),
1255                      op->object_size(),
1256                      op->klass()->as_register(),
1257                      *op->stub()->entry());
1258   __ bind(*op->stub()->continuation());
1259 }
1260 
1261 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1262   Register len =  op->len()->as_register();
1263   __ movslq(len, len);
1264 
1265   if (UseSlowPath || op->is_null_free() ||
1266       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1267       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1268     __ jmp(*op->stub()->entry());
1269   } else {
1270     Register tmp1 = op->tmp1()->as_register();
1271     Register tmp2 = op->tmp2()->as_register();
1272     Register tmp3 = op->tmp3()->as_register();
1273     if (len == tmp1) {
1274       tmp1 = tmp3;
1275     } else if (len == tmp2) {
1276       tmp2 = tmp3;
1277     } else if (len == tmp3) {
1278       // everything is ok
1279     } else {
1280       __ mov(tmp3, len);
1281     }
1282     __ allocate_array(op->obj()->as_register(),
1283                       len,
1284                       tmp1,
1285                       tmp2,

1344     assert(data != nullptr,                "need data for type check");
1345     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1346   }
1347   Label* success_target = success;
1348   Label* failure_target = failure;
1349 
1350   if (obj == k_RInfo) {
1351     k_RInfo = dst;
1352   } else if (obj == klass_RInfo) {
1353     klass_RInfo = dst;
1354   }
1355   if (k->is_loaded() && !UseCompressedClassPointers) {
1356     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1357   } else {
1358     Rtmp1 = op->tmp3()->as_register();
1359     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1360   }
1361 
1362   assert_different_registers(obj, k_RInfo, klass_RInfo);
1363 
1364   if (op->need_null_check()) {
1365     __ testptr(obj, obj);
1366     if (op->should_profile()) {
1367       Label not_null;
1368       Register mdo  = klass_RInfo;
1369       __ mov_metadata(mdo, md->constant_encoding());
1370       __ jccb(Assembler::notEqual, not_null);
1371       // Object is null; update MDO and exit
1372       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1373       int header_bits = BitData::null_seen_byte_constant();
1374       __ orb(data_addr, header_bits);
1375       __ jmp(*obj_is_null);
1376       __ bind(not_null);
1377 
1378       Label update_done;
1379       Register recv = k_RInfo;
1380       __ load_klass(recv, obj, tmp_load_klass);
1381       type_profile_helper(mdo, md, data, recv, &update_done);
1382 
1383       Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1384       __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1385 
1386       __ bind(update_done);
1387     } else {
1388       __ jcc(Assembler::equal, *obj_is_null);
1389     }
1390   }
1391 
1392   if (!k->is_loaded()) {
1393     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1394   } else {
1395     __ mov_metadata(k_RInfo, k->constant_encoding());
1396   }
1397   __ verify_oop(obj);
1398 
1399   if (op->fast_check()) {
1400     // get object class
1401     // not a safepoint as obj null check happens earlier
1402     if (UseCompressedClassPointers) {
1403       __ load_klass(Rtmp1, obj, tmp_load_klass);
1404       __ cmpptr(k_RInfo, Rtmp1);
1405     } else {
1406       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1407     }
1408     __ jcc(Assembler::notEqual, *failure_target);
1409     // successful cast, fall through to profile or jump

1539         __ mov(dst, obj);
1540       }
1541     } else
1542       if (code == lir_instanceof) {
1543         Register obj = op->object()->as_register();
1544         Register dst = op->result_opr()->as_register();
1545         Label success, failure, done;
1546         emit_typecheck_helper(op, &success, &failure, &failure);
1547         __ bind(failure);
1548         __ xorptr(dst, dst);
1549         __ jmpb(done);
1550         __ bind(success);
1551         __ movptr(dst, 1);
1552         __ bind(done);
1553       } else {
1554         ShouldNotReachHere();
1555       }
1556 
1557 }
1558 
1559 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1560   // We are loading/storing from/to an array that *may* be a flat array (the
1561   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1562   // If this array is a flat array, take the slow path.
1563   __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1564   if (!op->value()->is_illegal()) {
1565     // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1566     // The array is not a flat array, but it might be null-free. If we are storing
1567     // a null into a null-free array, take the slow path (which will throw NPE).
1568     Label skip;
1569     __ cmpptr(op->value()->as_register(), NULL_WORD);
1570     __ jcc(Assembler::notEqual, skip);
1571     __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1572     __ bind(skip);
1573   }
1574 }
1575 
1576 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1577   // We are storing into an array that *may* be null-free (the declared type is
1578   // Object[], abstract[], interface[] or VT.ref[]).
1579   Label test_mark_word;
1580   Register tmp = op->tmp()->as_register();
1581   __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1582   __ testl(tmp, markWord::unlocked_value);
1583   __ jccb(Assembler::notZero, test_mark_word);
1584   __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1585   __ bind(test_mark_word);
1586   __ testl(tmp, markWord::null_free_array_bit_in_place);
1587 }
1588 
1589 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1590   Label L_oops_equal;
1591   Label L_oops_not_equal;
1592   Label L_end;
1593 
1594   Register left  = op->left()->as_register();
1595   Register right = op->right()->as_register();
1596 
1597   __ cmpptr(left, right);
1598   __ jcc(Assembler::equal, L_oops_equal);
1599 
1600   // (1) Null check -- if one of the operands is null, the other must not be null (because
1601   //     the two references are not equal), so they are not substitutable,
1602   //     FIXME: do null check only if the operand is nullable
1603   __ testptr(left, right);
1604   __ jcc(Assembler::zero, L_oops_not_equal);
1605 
1606   ciKlass* left_klass = op->left_klass();
1607   ciKlass* right_klass = op->right_klass();
1608 
1609   // (2) Inline type check -- if either of the operands is not a inline type,
1610   //     they are not substitutable. We do this only if we are not sure that the
1611   //     operands are inline type
1612   if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1613       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1614     Register tmp1  = op->tmp1()->as_register();
1615     __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1616     __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1617     __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1618     __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1619     __ jcc(Assembler::notEqual, L_oops_not_equal);
1620   }
1621 
1622   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1623   if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1624     // No need to load klass -- the operands are statically known to be the same inline klass.
1625     __ jmp(*op->stub()->entry());
1626   } else {
1627     Register left_klass_op = op->left_klass_op()->as_register();
1628     Register right_klass_op = op->right_klass_op()->as_register();
1629 
1630     if (UseCompressedClassPointers) {
1631       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1632       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1633       __ cmpl(left_klass_op, right_klass_op);
1634     } else {
1635       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1636       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1637       __ cmpptr(left_klass_op, right_klass_op);
1638     }
1639 
1640     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1641     // fall through to L_oops_not_equal
1642   }
1643 
1644   __ bind(L_oops_not_equal);
1645   move(op->not_equal_result(), op->result_opr());
1646   __ jmp(L_end);
1647 
1648   __ bind(L_oops_equal);
1649   move(op->equal_result(), op->result_opr());
1650   __ jmp(L_end);
1651 
1652   // We've returned from the stub. RAX contains 0x0 IFF the two
1653   // operands are not substitutable. (Don't compare against 0x1 in case the
1654   // C compiler is naughty)
1655   __ bind(*op->stub()->continuation());
1656   __ cmpl(rax, 0);
1657   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1658   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1659   // fall-through
1660   __ bind(L_end);
1661 }
1662 
1663 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1664   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1665     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1666     Register newval = op->new_value()->as_register();
1667     Register cmpval = op->cmp_value()->as_register();
1668     assert(cmpval == rax, "wrong register");
1669     assert(newval != noreg, "new val must be register");
1670     assert(cmpval != newval, "cmp and new values must be in different registers");
1671     assert(cmpval != addr, "cmp and addr must be in different registers");
1672     assert(newval != addr, "new value and addr must be in different registers");
1673 
1674     if (op->code() == lir_cas_obj) {
1675       if (UseCompressedOops) {
1676         __ encode_heap_oop(cmpval);
1677         __ mov(rscratch1, newval);
1678         __ encode_heap_oop(rscratch1);
1679         __ lock();
1680         // cmpval (rax) is implicitly used by this instruction
1681         __ cmpxchgl(rscratch1, Address(addr, 0));

1687       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1688       __ lock();
1689       __ cmpxchgl(newval, Address(addr, 0));
1690     }
1691   } else if (op->code() == lir_cas_long) {
1692     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1693     Register newval = op->new_value()->as_register_lo();
1694     Register cmpval = op->cmp_value()->as_register_lo();
1695     assert(cmpval == rax, "wrong register");
1696     assert(newval != noreg, "new val must be register");
1697     assert(cmpval != newval, "cmp and new values must be in different registers");
1698     assert(cmpval != addr, "cmp and addr must be in different registers");
1699     assert(newval != addr, "new value and addr must be in different registers");
1700     __ lock();
1701     __ cmpxchgq(newval, Address(addr, 0));
1702   } else {
1703     Unimplemented();
1704   }
1705 }
1706 
1707 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1708   assert(dst->is_cpu_register(), "must be");
1709   assert(dst->type() == src->type(), "must be");
1710 
1711   if (src->is_cpu_register()) {
1712     reg2reg(src, dst);
1713   } else if (src->is_stack()) {
1714     stack2reg(src, dst, dst->type());
1715   } else if (src->is_constant()) {
1716     const2reg(src, dst, lir_patch_none, nullptr);
1717   } else {
1718     ShouldNotReachHere();
1719   }
1720 }
1721 
1722 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1723                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1724   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1725 
1726   Assembler::Condition acond, ncond;
1727   switch (condition) {
1728     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1729     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1730     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1731     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1732     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1733     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1734     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1735     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1736     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1737                                 ShouldNotReachHere();
1738   }
1739 
1740   if (opr1->is_cpu_register()) {
1741     reg2reg(opr1, result);

2312   int offset = __ offset();
2313   switch (code) {
2314   case lir_static_call:
2315   case lir_optvirtual_call:
2316   case lir_dynamic_call:
2317     offset += NativeCall::displacement_offset;
2318     break;
2319   case lir_icvirtual_call:
2320     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2321     break;
2322   default: ShouldNotReachHere();
2323   }
2324   __ align(BytesPerWord, offset);
2325 }
2326 
2327 
2328 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2329   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2330          "must be aligned");
2331   __ call(AddressLiteral(op->addr(), rtype));
2332   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2333   __ post_call_nop();
2334 }
2335 
2336 
2337 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2338   __ ic_call(op->addr());
2339   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2340   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2341          "must be aligned");
2342   __ post_call_nop();
2343 }
2344 
2345 
2346 void LIR_Assembler::emit_static_call_stub() {
2347   address call_pc = __ pc();
2348   address stub = __ start_a_stub(call_stub_size());
2349   if (stub == nullptr) {
2350     bailout("static call stub overflow");
2351     return;
2352   }
2353 
2354   int start = __ offset();
2355 
2356   // make sure that the displacement word of the call ends up word aligned
2357   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2358   __ relocate(static_stub_Relocation::spec(call_pc));
2359   __ mov_metadata(rbx, (Metadata*)nullptr);

2486   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2487 }
2488 
2489 
2490 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2491   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2492   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2493   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2494   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2495 }
2496 
2497 
2498 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2499   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2500   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2501   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2502   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2503 }
2504 
2505 
2506 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2507   if (null_check) {
2508     __ testptr(obj, obj);
2509     __ jcc(Assembler::zero, *slow_path->entry());
2510   }
2511   if (is_dest) {
2512     __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2513     // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2514     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2515   } else {
2516     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2517   }
2518 }
2519 
2520 
2521 // This code replaces a call to arraycopy; no exception may
2522 // be thrown in this code, they must be thrown in the System.arraycopy
2523 // activation frame; we could save some checks if this would not be the case
2524 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2525   ciArrayKlass* default_type = op->expected_type();
2526   Register src = op->src()->as_register();
2527   Register dst = op->dst()->as_register();
2528   Register src_pos = op->src_pos()->as_register();
2529   Register dst_pos = op->dst_pos()->as_register();
2530   Register length  = op->length()->as_register();
2531   Register tmp = op->tmp()->as_register();
2532   Register tmp_load_klass = rscratch1;
2533   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2534 
2535   CodeStub* stub = op->stub();
2536   int flags = op->flags();
2537   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2538   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2539 
2540   if (flags & LIR_OpArrayCopy::always_slow_path) {
2541     __ jmp(*stub->entry());
2542     __ bind(*stub->continuation());
2543     return;
2544   }
2545 
2546   // if we don't know anything, just go through the generic arraycopy
2547   if (default_type == nullptr) {
2548     // save outgoing arguments on stack in case call to System.arraycopy is needed
2549     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2550     // for interpreter calling conventions. Now we have to do it in new style conventions.
2551     // For the moment until C1 gets the new register allocator I just force all the
2552     // args to the right place (except the register args) and then on the back side
2553     // reload the register args properly if we go slow path. Yuck
2554 
2555     // These are proper for the calling convention
2556     store_parameter(length, 2);
2557     store_parameter(dst_pos, 1);
2558     store_parameter(dst, 0);
2559 
2560     // these are just temporary placements until we need to reload
2561     store_parameter(src_pos, 3);
2562     store_parameter(src, 4);
2563 
2564     address copyfunc_addr = StubRoutines::generic_arraycopy();
2565     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");

2602     __ mov(tmp, rax);
2603     __ xorl(tmp, -1);
2604 
2605     // Reload values from the stack so they are where the stub
2606     // expects them.
2607     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2608     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2609     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2610     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2611     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2612 
2613     __ subl(length, tmp);
2614     __ addl(src_pos, tmp);
2615     __ addl(dst_pos, tmp);
2616     __ jmp(*stub->entry());
2617 
2618     __ bind(*stub->continuation());
2619     return;
2620   }
2621 
2622   // Handle inline type arrays
2623   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2624     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2625   }
2626   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2627     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2628   }
2629 
2630   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2631 
2632   int elem_size = type2aelembytes(basic_type);
2633   Address::ScaleFactor scale;
2634 
2635   switch (elem_size) {
2636     case 1 :
2637       scale = Address::times_1;
2638       break;
2639     case 2 :
2640       scale = Address::times_2;
2641       break;
2642     case 4 :
2643       scale = Address::times_4;
2644       break;
2645     case 8 :
2646       scale = Address::times_8;
2647       break;
2648     default:
2649       scale = Address::no_scale;

3180         // first time here. Set profile type.
3181         __ movptr(mdo_addr, tmp);
3182 #ifdef ASSERT
3183         __ andptr(tmp, TypeEntries::type_klass_mask);
3184         __ verify_klass_ptr(tmp);
3185 #endif
3186       } else {
3187         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3188                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3189 
3190         __ testptr(mdo_addr, TypeEntries::type_unknown);
3191         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3192 
3193         __ orptr(mdo_addr, TypeEntries::type_unknown);
3194       }
3195     }
3196   }
3197   __ bind(next);
3198 }
3199 
3200 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3201   Register obj = op->obj()->as_register();
3202   Register tmp = op->tmp()->as_pointer_register();
3203   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3204   bool not_null = op->not_null();
3205   int flag = op->flag();
3206 
3207   Label not_inline_type;
3208   if (!not_null) {
3209     __ testptr(obj, obj);
3210     __ jccb(Assembler::zero, not_inline_type);
3211   }
3212 
3213   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3214 
3215   __ orb(mdo_addr, flag);
3216 
3217   __ bind(not_inline_type);
3218 }
3219 
3220 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3221   Unimplemented();
3222 }
3223 
3224 
3225 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3226   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3227 }
3228 
3229 
3230 void LIR_Assembler::align_backward_branch_target() {
3231   __ align(BytesPerWord);
3232 }
3233 
3234 
3235 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3236   if (left->is_single_cpu()) {
3237     __ negl(left->as_register());
3238     move_regs(left->as_register(), dest->as_register());
3239 

3390 }
3391 
3392 void LIR_Assembler::membar_loadstore() {
3393   // no-op
3394   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3395 }
3396 
3397 void LIR_Assembler::membar_storeload() {
3398   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3399 }
3400 
3401 void LIR_Assembler::on_spin_wait() {
3402   __ pause ();
3403 }
3404 
3405 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3406   assert(result_reg->is_register(), "check");
3407   __ mov(result_reg->as_register(), r15_thread);
3408 }
3409 
3410 void LIR_Assembler::check_orig_pc() {
3411   __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3412 }
3413 
3414 void LIR_Assembler::peephole(LIR_List*) {
3415   // do nothing for now
3416 }
3417 
3418 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3419   assert(data == dest, "xchg/xadd uses only 2 operands");
3420 
3421   if (data->type() == T_INT) {
3422     if (code == lir_xadd) {
3423       __ lock();
3424       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3425     } else {
3426       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3427     }
3428   } else if (data->is_oop()) {
3429     assert (code == lir_xchg, "xadd for oops");
3430     Register obj = data->as_register();
3431     if (UseCompressedOops) {
3432       __ encode_heap_oop(obj);
< prev index next >