< prev index next >

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

Print this page

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"

  34 #include "ci/ciInstance.hpp"

  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/collectedHeap.hpp"
  37 #include "gc/shared/gc_globals.hpp"
  38 #include "nativeInst_x86.hpp"

  39 #include "oops/objArrayKlass.hpp"
  40 #include "runtime/frame.inline.hpp"
  41 #include "runtime/safepointMechanism.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "runtime/stubRoutines.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_x86.inline.hpp"
  46 
  47 
  48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  49 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  50 // fast versions of NegF/NegD and AbsF/AbsD.
  51 
  52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  54   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  55   // of 128-bits operands for SSE instructions.
  56   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  57   // Store the value to a 128-bits operand.
  58   operand[0] = lo;

 411   // Perform needed unlocking
 412   MonitorExitStub* stub = nullptr;
 413   if (method()->is_synchronized()) {
 414     monitor_address(0, FrameMap::rax_opr);
 415     stub = new MonitorExitStub(FrameMap::rax_opr, 0);
 416     __ unlock_object(rdi, rsi, rax, *stub->entry());
 417     __ bind(*stub->continuation());
 418   }
 419 
 420   if (compilation()->env()->dtrace_method_probes()) {
 421     __ mov(rdi, r15_thread);
 422     __ mov_metadata(rsi, method()->constant_encoding());
 423     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 424   }
 425 
 426   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 427     __ mov(rax, rbx);  // Restore the exception
 428   }
 429 
 430   // remove the activation and dispatch to the unwind handler
 431   __ remove_frame(initial_frame_size_in_bytes());
 432   __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
 433 
 434   // Emit the slow path assembly
 435   if (stub != nullptr) {
 436     stub->emit_code(this);
 437   }
 438 
 439   return offset;
 440 }
 441 
 442 
 443 int LIR_Assembler::emit_deopt_handler() {
 444   // generate code for exception handler
 445   address handler_base = __ start_a_stub(deopt_handler_size());
 446   if (handler_base == nullptr) {
 447     // not enough space left for the handler
 448     bailout("deopt handler overflow");
 449     return -1;
 450   }
 451 
 452   int offset = code_offset();
 453   InternalAddress here(__ pc());
 454 
 455   __ pushptr(here.addr(), rscratch1);
 456   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 457   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 458   __ end_a_stub();
 459 
 460   return offset;
 461 }
 462 
 463 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 464   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 465   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 466     assert(result->fpu() == 0, "result must already be on TOS");
 467   }












































 468 
 469   // Pop the stack before the safepoint code
 470   __ remove_frame(initial_frame_size_in_bytes());
 471 
 472   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 473     __ reserved_stack_check();
 474   }
 475 
 476   // Note: we do not need to round double result; float result has the right precision
 477   // the poll sets the condition code, but no data registers
 478 
 479   code_stub->set_safepoint_offset(__ offset());
 480   __ relocate(relocInfo::poll_return_type);
 481   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 482   __ ret(0);
 483 }
 484 
 485 




 486 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 487   guarantee(info != nullptr, "Shouldn't be null");
 488   int offset = __ offset();
 489   const Register poll_addr = rscratch1;
 490   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 491   add_debug_info_for_branch(info);
 492   __ relocate(relocInfo::poll_type);
 493   address pre_pc = __ pc();
 494   __ testl(rax, Address(poll_addr, 0));
 495   address post_pc = __ pc();
 496   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 497   return offset;
 498 }
 499 
 500 
 501 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 502   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 503 }
 504 
 505 void LIR_Assembler::swap_reg(Register a, Register b) {

1204     // init_state needs acquire, but x86 is TSO, and so we are already good.
1205     __ cmpb(Address(op->klass()->as_register(),
1206                     InstanceKlass::init_state_offset()),
1207                     InstanceKlass::fully_initialized);
1208     __ jcc(Assembler::notEqual, *op->stub()->entry());
1209   }
1210   __ allocate_object(op->obj()->as_register(),
1211                      op->tmp1()->as_register(),
1212                      op->tmp2()->as_register(),
1213                      op->header_size(),
1214                      op->object_size(),
1215                      op->klass()->as_register(),
1216                      *op->stub()->entry());
1217   __ bind(*op->stub()->continuation());
1218 }
1219 
1220 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1221   Register len =  op->len()->as_register();
1222   __ movslq(len, len);
1223 
1224   if (UseSlowPath ||
1225       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1226       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1227     __ jmp(*op->stub()->entry());
1228   } else {
1229     Register tmp1 = op->tmp1()->as_register();
1230     Register tmp2 = op->tmp2()->as_register();
1231     Register tmp3 = op->tmp3()->as_register();
1232     if (len == tmp1) {
1233       tmp1 = tmp3;
1234     } else if (len == tmp2) {
1235       tmp2 = tmp3;
1236     } else if (len == tmp3) {
1237       // everything is ok
1238     } else {
1239       __ mov(tmp3, len);
1240     }
1241     __ allocate_array(op->obj()->as_register(),
1242                       len,
1243                       tmp1,
1244                       tmp2,

1303     assert(data != nullptr,                "need data for type check");
1304     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1305   }
1306   Label* success_target = success;
1307   Label* failure_target = failure;
1308 
1309   if (obj == k_RInfo) {
1310     k_RInfo = dst;
1311   } else if (obj == klass_RInfo) {
1312     klass_RInfo = dst;
1313   }
1314   if (k->is_loaded() && !UseCompressedClassPointers) {
1315     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1316   } else {
1317     Rtmp1 = op->tmp3()->as_register();
1318     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1319   }
1320 
1321   assert_different_registers(obj, k_RInfo, klass_RInfo);
1322 
1323   __ testptr(obj, obj);
1324   if (op->should_profile()) {
1325     Label not_null;
1326     Register mdo  = klass_RInfo;
1327     __ mov_metadata(mdo, md->constant_encoding());
1328     __ jccb(Assembler::notEqual, not_null);
1329     // Object is null; update MDO and exit
1330     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1331     int header_bits = BitData::null_seen_byte_constant();
1332     __ orb(data_addr, header_bits);
1333     __ jmp(*obj_is_null);
1334     __ bind(not_null);
1335 
1336     Label update_done;
1337     Register recv = k_RInfo;
1338     __ load_klass(recv, obj, tmp_load_klass);
1339     type_profile_helper(mdo, md, data, recv, &update_done);
1340 
1341     Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1342     __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1343 
1344     __ bind(update_done);
1345   } else {
1346     __ jcc(Assembler::equal, *obj_is_null);


1347   }
1348 
1349   if (!k->is_loaded()) {
1350     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1351   } else {
1352     __ mov_metadata(k_RInfo, k->constant_encoding());
1353   }
1354   __ verify_oop(obj);
1355 
1356   if (op->fast_check()) {

1357     // get object class
1358     // not a safepoint as obj null check happens earlier
1359     if (UseCompressedClassPointers) {
1360       __ load_klass(Rtmp1, obj, tmp_load_klass);
1361       __ cmpptr(k_RInfo, Rtmp1);
1362     } else {
1363       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1364     }
1365     __ jcc(Assembler::notEqual, *failure_target);
1366     // successful cast, fall through to profile or jump
1367   } else {
1368     // get object class
1369     // not a safepoint as obj null check happens earlier
1370     __ load_klass(klass_RInfo, obj, tmp_load_klass);
1371     if (k->is_loaded()) {
1372       // See if we get an immediate positive hit
1373       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1374       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1375         __ jcc(Assembler::notEqual, *failure_target);
1376         // successful cast, fall through to profile or jump
1377       } else {
1378         // See if we get an immediate positive hit
1379         __ jcc(Assembler::equal, *success_target);
1380         // check for self
1381         __ cmpptr(klass_RInfo, k_RInfo);







1382         __ jcc(Assembler::equal, *success_target);
1383 
1384         __ push_ppx(klass_RInfo);
1385         __ push_ppx(k_RInfo);
1386         __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1387         __ pop_ppx(klass_RInfo);
1388         __ pop_ppx(klass_RInfo);
1389         // result is a boolean
1390         __ testl(klass_RInfo, klass_RInfo);
1391         __ jcc(Assembler::equal, *failure_target);
1392         // successful cast, fall through to profile or jump
1393       }
1394     } else {
1395       // perform the fast part of the checking logic
1396       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1397       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1398       __ push_ppx(klass_RInfo);
1399       __ push_ppx(k_RInfo);
1400       __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1401       __ pop_ppx(klass_RInfo);

1496         __ mov(dst, obj);
1497       }
1498     } else
1499       if (code == lir_instanceof) {
1500         Register obj = op->object()->as_register();
1501         Register dst = op->result_opr()->as_register();
1502         Label success, failure, done;
1503         emit_typecheck_helper(op, &success, &failure, &failure);
1504         __ bind(failure);
1505         __ xorptr(dst, dst);
1506         __ jmpb(done);
1507         __ bind(success);
1508         __ movptr(dst, 1);
1509         __ bind(done);
1510       } else {
1511         ShouldNotReachHere();
1512       }
1513 
1514 }
1515 







































































































1516 
1517 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1518   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1519     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1520     Register newval = op->new_value()->as_register();
1521     Register cmpval = op->cmp_value()->as_register();
1522     assert(cmpval == rax, "wrong register");
1523     assert(newval != noreg, "new val must be register");
1524     assert(cmpval != newval, "cmp and new values must be in different registers");
1525     assert(cmpval != addr, "cmp and addr must be in different registers");
1526     assert(newval != addr, "new value and addr must be in different registers");
1527 
1528     if (op->code() == lir_cas_obj) {
1529       if (UseCompressedOops) {
1530         __ encode_heap_oop(cmpval);
1531         __ mov(rscratch1, newval);
1532         __ encode_heap_oop(rscratch1);
1533         __ lock();
1534         // cmpval (rax) is implicitly used by this instruction
1535         __ cmpxchgl(rscratch1, Address(addr, 0));

1541       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1542       __ lock();
1543       __ cmpxchgl(newval, Address(addr, 0));
1544     }
1545   } else if (op->code() == lir_cas_long) {
1546     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1547     Register newval = op->new_value()->as_register_lo();
1548     Register cmpval = op->cmp_value()->as_register_lo();
1549     assert(cmpval == rax, "wrong register");
1550     assert(newval != noreg, "new val must be register");
1551     assert(cmpval != newval, "cmp and new values must be in different registers");
1552     assert(cmpval != addr, "cmp and addr must be in different registers");
1553     assert(newval != addr, "new value and addr must be in different registers");
1554     __ lock();
1555     __ cmpxchgq(newval, Address(addr, 0));
1556   } else {
1557     Unimplemented();
1558   }
1559 }
1560 















1561 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1562                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1563   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1564 
1565   Assembler::Condition acond, ncond;
1566   switch (condition) {
1567     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1568     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1569     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1570     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1571     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1572     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1573     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1574     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1575     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1576                                 ShouldNotReachHere();
1577   }
1578 
1579   if (opr1->is_cpu_register()) {
1580     reg2reg(opr1, result);

2151   int offset = __ offset();
2152   switch (code) {
2153   case lir_static_call:
2154   case lir_optvirtual_call:
2155   case lir_dynamic_call:
2156     offset += NativeCall::displacement_offset;
2157     break;
2158   case lir_icvirtual_call:
2159     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2160     break;
2161   default: ShouldNotReachHere();
2162   }
2163   __ align(BytesPerWord, offset);
2164 }
2165 
2166 
2167 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2168   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2169          "must be aligned");
2170   __ call(AddressLiteral(op->addr(), rtype));
2171   add_call_info(code_offset(), op->info());
2172   __ post_call_nop();
2173 }
2174 
2175 
2176 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2177   __ ic_call(op->addr());
2178   add_call_info(code_offset(), op->info());
2179   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2180          "must be aligned");
2181   __ post_call_nop();
2182 }
2183 
2184 
2185 void LIR_Assembler::emit_static_call_stub() {
2186   address call_pc = __ pc();
2187   address stub = __ start_a_stub(call_stub_size());
2188   if (stub == nullptr) {
2189     bailout("static call stub overflow");
2190     return;
2191   }
2192 
2193   int start = __ offset();
2194 
2195   // make sure that the displacement word of the call ends up word aligned
2196   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2197   __ relocate(static_stub_Relocation::spec(call_pc));
2198   __ mov_metadata(rbx, (Metadata*)nullptr);

2325   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2326 }
2327 
2328 
2329 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2330   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2331   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2332   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2333   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2334 }
2335 
2336 
2337 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2338   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2339   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2340   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2341   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2342 }
2343 
2344 















2345 // This code replaces a call to arraycopy; no exception may
2346 // be thrown in this code, they must be thrown in the System.arraycopy
2347 // activation frame; we could save some checks if this would not be the case
2348 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2349   ciArrayKlass* default_type = op->expected_type();
2350   Register src = op->src()->as_register();
2351   Register dst = op->dst()->as_register();
2352   Register src_pos = op->src_pos()->as_register();
2353   Register dst_pos = op->dst_pos()->as_register();
2354   Register length  = op->length()->as_register();
2355   Register tmp = op->tmp()->as_register();
2356   Register tmp_load_klass = rscratch1;
2357   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2358 
2359   CodeStub* stub = op->stub();
2360   int flags = op->flags();
2361   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2362   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2363 






2364   // if we don't know anything, just go through the generic arraycopy
2365   if (default_type == nullptr) {
2366     // save outgoing arguments on stack in case call to System.arraycopy is needed
2367     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2368     // for interpreter calling conventions. Now we have to do it in new style conventions.
2369     // For the moment until C1 gets the new register allocator I just force all the
2370     // args to the right place (except the register args) and then on the back side
2371     // reload the register args properly if we go slow path. Yuck
2372 
2373     // These are proper for the calling convention
2374     store_parameter(length, 2);
2375     store_parameter(dst_pos, 1);
2376     store_parameter(dst, 0);
2377 
2378     // these are just temporary placements until we need to reload
2379     store_parameter(src_pos, 3);
2380     store_parameter(src, 4);
2381 
2382     address copyfunc_addr = StubRoutines::generic_arraycopy();
2383     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");

2420     __ mov(tmp, rax);
2421     __ xorl(tmp, -1);
2422 
2423     // Reload values from the stack so they are where the stub
2424     // expects them.
2425     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2426     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2427     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2428     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2429     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2430 
2431     __ subl(length, tmp);
2432     __ addl(src_pos, tmp);
2433     __ addl(dst_pos, tmp);
2434     __ jmp(*stub->entry());
2435 
2436     __ bind(*stub->continuation());
2437     return;
2438   }
2439 








2440   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2441 
2442   int elem_size = type2aelembytes(basic_type);
2443   Address::ScaleFactor scale;
2444 
2445   switch (elem_size) {
2446     case 1 :
2447       scale = Address::times_1;
2448       break;
2449     case 2 :
2450       scale = Address::times_2;
2451       break;
2452     case 4 :
2453       scale = Address::times_4;
2454       break;
2455     case 8 :
2456       scale = Address::times_8;
2457       break;
2458     default:
2459       scale = Address::no_scale;

2646       }
2647 
2648       __ jmp(*stub->entry());
2649 
2650       __ bind(cont);
2651       __ pop(dst);
2652       __ pop(src);
2653     }
2654   }
2655 
2656 #ifdef ASSERT
2657   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2658     // Sanity check the known type with the incoming class.  For the
2659     // primitive case the types must match exactly with src.klass and
2660     // dst.klass each exactly matching the default type.  For the
2661     // object array case, if no type check is needed then either the
2662     // dst type is exactly the expected type and the src type is a
2663     // subtype which we can't check or src is the same array as dst
2664     // but not necessarily exactly of type default_type.
2665     Label known_ok, halt;

2666     __ mov_metadata(tmp, default_type->constant_encoding());
2667     if (UseCompressedClassPointers) {
2668       __ encode_klass_not_null(tmp, rscratch1);
2669     }
2670 
2671     if (basic_type != T_OBJECT) {
2672       __ cmp_klass(tmp, dst, tmp2);
2673       __ jcc(Assembler::notEqual, halt);
2674       __ cmp_klass(tmp, src, tmp2);
2675       __ jcc(Assembler::equal, known_ok);
2676     } else {
2677       __ cmp_klass(tmp, dst, tmp2);
2678       __ jcc(Assembler::equal, known_ok);
2679       __ cmpptr(src, dst);
2680       __ jcc(Assembler::equal, known_ok);
2681     }
2682     __ bind(halt);
2683     __ stop("incorrect type information in arraycopy");
2684     __ bind(known_ok);
2685   }

2982         // first time here. Set profile type.
2983         __ movptr(mdo_addr, tmp);
2984 #ifdef ASSERT
2985         __ andptr(tmp, TypeEntries::type_klass_mask);
2986         __ verify_klass_ptr(tmp);
2987 #endif
2988       } else {
2989         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2990                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2991 
2992         __ testptr(mdo_addr, TypeEntries::type_unknown);
2993         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2994 
2995         __ orptr(mdo_addr, TypeEntries::type_unknown);
2996       }
2997     }
2998   }
2999   __ bind(next);
3000 }
3001 





















3002 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3003   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3004 }
3005 
3006 
3007 void LIR_Assembler::align_backward_branch_target() {
3008   __ align(BytesPerWord);
3009 }
3010 
3011 
3012 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3013   if (left->is_single_cpu()) {
3014     __ negl(left->as_register());
3015     move_regs(left->as_register(), dest->as_register());
3016 
3017   } else if (left->is_double_cpu()) {
3018     Register lo = left->as_register_lo();
3019     Register dst = dest->as_register_lo();
3020     __ movptr(dst, lo);
3021     __ negptr(dst);

3167 }
3168 
3169 void LIR_Assembler::membar_loadstore() {
3170   // no-op
3171   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3172 }
3173 
3174 void LIR_Assembler::membar_storeload() {
3175   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3176 }
3177 
3178 void LIR_Assembler::on_spin_wait() {
3179   __ pause ();
3180 }
3181 
3182 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3183   assert(result_reg->is_register(), "check");
3184   __ mov(result_reg->as_register(), r15_thread);
3185 }
3186 



3187 
3188 void LIR_Assembler::peephole(LIR_List*) {
3189   // do nothing for now
3190 }
3191 
3192 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3193   assert(data == dest, "xchg/xadd uses only 2 operands");
3194 
3195   if (data->type() == T_INT) {
3196     if (code == lir_xadd) {
3197       __ lock();
3198       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3199     } else {
3200       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3201     }
3202   } else if (data->is_oop()) {
3203     assert (code == lir_xchg, "xadd for oops");
3204     Register obj = data->as_register();
3205     if (UseCompressedOops) {
3206       __ encode_heap_oop(obj);

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInlineKlass.hpp"
  35 #include "ci/ciInstance.hpp"
  36 #include "ci/ciObjArrayKlass.hpp"
  37 #include "compiler/oopMap.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "gc/shared/gc_globals.hpp"
  40 #include "nativeInst_x86.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/safepointMechanism.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 #include "vmreg_x86.inline.hpp"
  49 
  50 
  51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  52 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  53 // fast versions of NegF/NegD and AbsF/AbsD.
  54 
  55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  57   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  58   // of 128-bits operands for SSE instructions.
  59   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  60   // Store the value to a 128-bits operand.
  61   operand[0] = lo;

 414   // Perform needed unlocking
 415   MonitorExitStub* stub = nullptr;
 416   if (method()->is_synchronized()) {
 417     monitor_address(0, FrameMap::rax_opr);
 418     stub = new MonitorExitStub(FrameMap::rax_opr, 0);
 419     __ unlock_object(rdi, rsi, rax, *stub->entry());
 420     __ bind(*stub->continuation());
 421   }
 422 
 423   if (compilation()->env()->dtrace_method_probes()) {
 424     __ mov(rdi, r15_thread);
 425     __ mov_metadata(rsi, method()->constant_encoding());
 426     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 427   }
 428 
 429   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 430     __ mov(rax, rbx);  // Restore the exception
 431   }
 432 
 433   // remove the activation and dispatch to the unwind handler
 434   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 435   __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
 436 
 437   // Emit the slow path assembly
 438   if (stub != nullptr) {
 439     stub->emit_code(this);
 440   }
 441 
 442   return offset;
 443 }
 444 
 445 
 446 int LIR_Assembler::emit_deopt_handler() {
 447   // generate code for exception handler
 448   address handler_base = __ start_a_stub(deopt_handler_size());
 449   if (handler_base == nullptr) {
 450     // not enough space left for the handler
 451     bailout("deopt handler overflow");
 452     return -1;
 453   }
 454 
 455   int offset = code_offset();
 456   InternalAddress here(__ pc());
 457 
 458   __ pushptr(here.addr(), rscratch1);
 459   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 460   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 461   __ end_a_stub();
 462 
 463   return offset;
 464 }
 465 
 466 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 467   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 468   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 469     assert(result->fpu() == 0, "result must already be on TOS");
 470   }
 471   if (InlineTypeReturnedAsFields) {
 472   #ifndef _LP64
 473      Unimplemented();
 474   #endif
 475     // Check if we are returning an non-null inline type and load its fields into registers
 476     ciType* return_type = compilation()->method()->return_type();
 477     if (return_type->is_inlinetype()) {
 478       ciInlineKlass* vk = return_type->as_inline_klass();
 479       if (vk->can_be_returned_as_fields()) {
 480         address unpack_handler = vk->unpack_handler();
 481         assert(unpack_handler != nullptr, "must be");
 482         __ call(RuntimeAddress(unpack_handler));
 483       }
 484     } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
 485       Label skip;
 486       Label not_null;
 487       __ testptr(rax, rax);
 488       __ jcc(Assembler::notZero, not_null);
 489       // Returned value is null, zero all return registers because they may belong to oop fields
 490       __ xorq(j_rarg1, j_rarg1);
 491       __ xorq(j_rarg2, j_rarg2);
 492       __ xorq(j_rarg3, j_rarg3);
 493       __ xorq(j_rarg4, j_rarg4);
 494       __ xorq(j_rarg5, j_rarg5);
 495       __ jmp(skip);
 496       __ bind(not_null);
 497 
 498       // Check if we are returning an non-null inline type and load its fields into registers
 499       __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
 500 
 501       // Load fields from a buffered value with an inline class specific handler
 502       __ load_klass(rdi, rax, rscratch1);
 503       __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 504       __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
 505       // Unpack handler can be null if inline type is not scalarizable in returns
 506       __ testptr(rdi, rdi);
 507       __ jcc(Assembler::zero, skip);
 508       __ call(rdi);
 509 
 510       __ bind(skip);
 511     }
 512     // At this point, rax points to the value object (for interpreter or C1 caller).
 513     // The fields of the object are copied into registers (for C2 caller).
 514   }
 515 
 516   // Pop the stack before the safepoint code
 517   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 518 
 519   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 520     __ reserved_stack_check();
 521   }
 522 
 523   // Note: we do not need to round double result; float result has the right precision
 524   // the poll sets the condition code, but no data registers
 525 
 526   code_stub->set_safepoint_offset(__ offset());
 527   __ relocate(relocInfo::poll_return_type);
 528   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 529   __ ret(0);
 530 }
 531 
 532 
 533 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 534   return (__ store_inline_type_fields_to_buf(vk, false));
 535 }
 536 
 537 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 538   guarantee(info != nullptr, "Shouldn't be null");
 539   int offset = __ offset();
 540   const Register poll_addr = rscratch1;
 541   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 542   add_debug_info_for_branch(info);
 543   __ relocate(relocInfo::poll_type);
 544   address pre_pc = __ pc();
 545   __ testl(rax, Address(poll_addr, 0));
 546   address post_pc = __ pc();
 547   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 548   return offset;
 549 }
 550 
 551 
 552 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 553   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 554 }
 555 
 556 void LIR_Assembler::swap_reg(Register a, Register b) {

1255     // init_state needs acquire, but x86 is TSO, and so we are already good.
1256     __ cmpb(Address(op->klass()->as_register(),
1257                     InstanceKlass::init_state_offset()),
1258                     InstanceKlass::fully_initialized);
1259     __ jcc(Assembler::notEqual, *op->stub()->entry());
1260   }
1261   __ allocate_object(op->obj()->as_register(),
1262                      op->tmp1()->as_register(),
1263                      op->tmp2()->as_register(),
1264                      op->header_size(),
1265                      op->object_size(),
1266                      op->klass()->as_register(),
1267                      *op->stub()->entry());
1268   __ bind(*op->stub()->continuation());
1269 }
1270 
1271 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1272   Register len =  op->len()->as_register();
1273   __ movslq(len, len);
1274 
1275   if (UseSlowPath || op->always_slow_path() ||
1276       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1277       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1278     __ jmp(*op->stub()->entry());
1279   } else {
1280     Register tmp1 = op->tmp1()->as_register();
1281     Register tmp2 = op->tmp2()->as_register();
1282     Register tmp3 = op->tmp3()->as_register();
1283     if (len == tmp1) {
1284       tmp1 = tmp3;
1285     } else if (len == tmp2) {
1286       tmp2 = tmp3;
1287     } else if (len == tmp3) {
1288       // everything is ok
1289     } else {
1290       __ mov(tmp3, len);
1291     }
1292     __ allocate_array(op->obj()->as_register(),
1293                       len,
1294                       tmp1,
1295                       tmp2,

1354     assert(data != nullptr,                "need data for type check");
1355     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1356   }
1357   Label* success_target = success;
1358   Label* failure_target = failure;
1359 
1360   if (obj == k_RInfo) {
1361     k_RInfo = dst;
1362   } else if (obj == klass_RInfo) {
1363     klass_RInfo = dst;
1364   }
1365   if (k->is_loaded() && !UseCompressedClassPointers) {
1366     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1367   } else {
1368     Rtmp1 = op->tmp3()->as_register();
1369     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1370   }
1371 
1372   assert_different_registers(obj, k_RInfo, klass_RInfo);
1373 
1374   if (op->need_null_check()) {
1375     __ testptr(obj, obj);
1376     if (op->should_profile()) {
1377       Label not_null;
1378       Register mdo  = klass_RInfo;
1379       __ mov_metadata(mdo, md->constant_encoding());
1380       __ jccb(Assembler::notEqual, not_null);
1381       // Object is null; update MDO and exit
1382       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1383       int header_bits = BitData::null_seen_byte_constant();
1384       __ orb(data_addr, header_bits);
1385       __ jmp(*obj_is_null);
1386       __ bind(not_null);
1387 
1388       Label update_done;
1389       Register recv = k_RInfo;
1390       __ load_klass(recv, obj, tmp_load_klass);
1391       type_profile_helper(mdo, md, data, recv, &update_done);
1392 
1393       Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1394       __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1395 
1396       __ bind(update_done);
1397     } else {
1398       __ jcc(Assembler::equal, *obj_is_null);
1399     }
1400   }
1401 
1402   if (!k->is_loaded()) {
1403     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1404   } else {
1405     __ mov_metadata(k_RInfo, k->constant_encoding());
1406   }
1407   __ verify_oop(obj);
1408 
1409   if (op->fast_check()) {
1410     assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1411     // get object class
1412     // not a safepoint as obj null check happens earlier
1413     if (UseCompressedClassPointers) {
1414       __ load_klass(Rtmp1, obj, tmp_load_klass);
1415       __ cmpptr(k_RInfo, Rtmp1);
1416     } else {
1417       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1418     }
1419     __ jcc(Assembler::notEqual, *failure_target);
1420     // successful cast, fall through to profile or jump
1421   } else {
1422     // get object class
1423     // not a safepoint as obj null check happens earlier
1424     __ load_klass(klass_RInfo, obj, tmp_load_klass);
1425     if (k->is_loaded()) {
1426       // See if we get an immediate positive hit
1427       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1428       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1429         __ jcc(Assembler::notEqual, *failure_target);
1430         // successful cast, fall through to profile or jump
1431       } else {
1432         // See if we get an immediate positive hit
1433         __ jcc(Assembler::equal, *success_target);
1434         // check for self
1435         if (k->is_loaded() && k->is_obj_array_klass()) {
1436           // For a direct pointer comparison, we need the refined array klass pointer
1437           ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1438           __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1439           __ cmpptr(klass_RInfo, tmp_load_klass);
1440         } else {
1441           __ cmpptr(klass_RInfo, k_RInfo);
1442         }
1443         __ jcc(Assembler::equal, *success_target);
1444 
1445         __ push_ppx(klass_RInfo);
1446         __ push_ppx(k_RInfo);
1447         __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1448         __ pop_ppx(klass_RInfo);
1449         __ pop_ppx(klass_RInfo);
1450         // result is a boolean
1451         __ testl(klass_RInfo, klass_RInfo);
1452         __ jcc(Assembler::equal, *failure_target);
1453         // successful cast, fall through to profile or jump
1454       }
1455     } else {
1456       // perform the fast part of the checking logic
1457       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1458       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1459       __ push_ppx(klass_RInfo);
1460       __ push_ppx(k_RInfo);
1461       __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1462       __ pop_ppx(klass_RInfo);

1557         __ mov(dst, obj);
1558       }
1559     } else
1560       if (code == lir_instanceof) {
1561         Register obj = op->object()->as_register();
1562         Register dst = op->result_opr()->as_register();
1563         Label success, failure, done;
1564         emit_typecheck_helper(op, &success, &failure, &failure);
1565         __ bind(failure);
1566         __ xorptr(dst, dst);
1567         __ jmpb(done);
1568         __ bind(success);
1569         __ movptr(dst, 1);
1570         __ bind(done);
1571       } else {
1572         ShouldNotReachHere();
1573       }
1574 
1575 }
1576 
1577 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1578   // We are loading/storing from/to an array that *may* be a flat array (the
1579   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1580   // If this array is a flat array, take the slow path.
1581   __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1582   if (!op->value()->is_illegal()) {
1583     // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1584     // The array is not a flat array, but it might be null-free. If we are storing
1585     // a null into a null-free array, take the slow path (which will throw NPE).
1586     Label skip;
1587     __ cmpptr(op->value()->as_register(), NULL_WORD);
1588     __ jcc(Assembler::notEqual, skip);
1589     __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1590     __ bind(skip);
1591   }
1592 }
1593 
1594 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1595   // We are storing into an array that *may* be null-free (the declared type is
1596   // Object[], abstract[], interface[] or VT.ref[]).
1597   Label test_mark_word;
1598   Register tmp = op->tmp()->as_register();
1599   __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1600   __ testl(tmp, markWord::unlocked_value);
1601   __ jccb(Assembler::notZero, test_mark_word);
1602   __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1603   __ bind(test_mark_word);
1604   __ testl(tmp, markWord::null_free_array_bit_in_place);
1605 }
1606 
1607 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1608   Label L_oops_equal;
1609   Label L_oops_not_equal;
1610   Label L_end;
1611 
1612   Register left  = op->left()->as_register();
1613   Register right = op->right()->as_register();
1614 
1615   __ cmpptr(left, right);
1616   __ jcc(Assembler::equal, L_oops_equal);
1617 
1618   // (1) Null check -- if one of the operands is null, the other must not be null (because
1619   //     the two references are not equal), so they are not substitutable,
1620   //     FIXME: do null check only if the operand is nullable
1621   __ testptr(left, right);
1622   __ jcc(Assembler::zero, L_oops_not_equal);
1623 
1624   ciKlass* left_klass = op->left_klass();
1625   ciKlass* right_klass = op->right_klass();
1626 
1627   // (2) Inline type check -- if either of the operands is not a inline type,
1628   //     they are not substitutable. We do this only if we are not sure that the
1629   //     operands are inline type
1630   if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1631       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1632     Register tmp1  = op->tmp1()->as_register();
1633     __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1634     __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1635     __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1636     __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1637     __ jcc(Assembler::notEqual, L_oops_not_equal);
1638   }
1639 
1640   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1641   if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1642     // No need to load klass -- the operands are statically known to be the same inline klass.
1643     __ jmp(*op->stub()->entry());
1644   } else {
1645     Register left_klass_op = op->left_klass_op()->as_register();
1646     Register right_klass_op = op->right_klass_op()->as_register();
1647 
1648     if (UseCompressedClassPointers) {
1649       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1650       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1651       __ cmpl(left_klass_op, right_klass_op);
1652     } else {
1653       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1654       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1655       __ cmpptr(left_klass_op, right_klass_op);
1656     }
1657 
1658     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1659     // fall through to L_oops_not_equal
1660   }
1661 
1662   __ bind(L_oops_not_equal);
1663   move(op->not_equal_result(), op->result_opr());
1664   __ jmp(L_end);
1665 
1666   __ bind(L_oops_equal);
1667   move(op->equal_result(), op->result_opr());
1668   __ jmp(L_end);
1669 
1670   // We've returned from the stub. RAX contains 0x0 IFF the two
1671   // operands are not substitutable. (Don't compare against 0x1 in case the
1672   // C compiler is naughty)
1673   __ bind(*op->stub()->continuation());
1674   __ cmpl(rax, 0);
1675   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1676   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1677   // fall-through
1678   __ bind(L_end);
1679 }
1680 
1681 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1682   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1683     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1684     Register newval = op->new_value()->as_register();
1685     Register cmpval = op->cmp_value()->as_register();
1686     assert(cmpval == rax, "wrong register");
1687     assert(newval != noreg, "new val must be register");
1688     assert(cmpval != newval, "cmp and new values must be in different registers");
1689     assert(cmpval != addr, "cmp and addr must be in different registers");
1690     assert(newval != addr, "new value and addr must be in different registers");
1691 
1692     if (op->code() == lir_cas_obj) {
1693       if (UseCompressedOops) {
1694         __ encode_heap_oop(cmpval);
1695         __ mov(rscratch1, newval);
1696         __ encode_heap_oop(rscratch1);
1697         __ lock();
1698         // cmpval (rax) is implicitly used by this instruction
1699         __ cmpxchgl(rscratch1, Address(addr, 0));

1705       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1706       __ lock();
1707       __ cmpxchgl(newval, Address(addr, 0));
1708     }
1709   } else if (op->code() == lir_cas_long) {
1710     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1711     Register newval = op->new_value()->as_register_lo();
1712     Register cmpval = op->cmp_value()->as_register_lo();
1713     assert(cmpval == rax, "wrong register");
1714     assert(newval != noreg, "new val must be register");
1715     assert(cmpval != newval, "cmp and new values must be in different registers");
1716     assert(cmpval != addr, "cmp and addr must be in different registers");
1717     assert(newval != addr, "new value and addr must be in different registers");
1718     __ lock();
1719     __ cmpxchgq(newval, Address(addr, 0));
1720   } else {
1721     Unimplemented();
1722   }
1723 }
1724 
1725 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1726   assert(dst->is_cpu_register(), "must be");
1727   assert(dst->type() == src->type(), "must be");
1728 
1729   if (src->is_cpu_register()) {
1730     reg2reg(src, dst);
1731   } else if (src->is_stack()) {
1732     stack2reg(src, dst, dst->type());
1733   } else if (src->is_constant()) {
1734     const2reg(src, dst, lir_patch_none, nullptr);
1735   } else {
1736     ShouldNotReachHere();
1737   }
1738 }
1739 
1740 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1741                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1742   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1743 
1744   Assembler::Condition acond, ncond;
1745   switch (condition) {
1746     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1747     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1748     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1749     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1750     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1751     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1752     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1753     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1754     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1755                                 ShouldNotReachHere();
1756   }
1757 
1758   if (opr1->is_cpu_register()) {
1759     reg2reg(opr1, result);

2330   int offset = __ offset();
2331   switch (code) {
2332   case lir_static_call:
2333   case lir_optvirtual_call:
2334   case lir_dynamic_call:
2335     offset += NativeCall::displacement_offset;
2336     break;
2337   case lir_icvirtual_call:
2338     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2339     break;
2340   default: ShouldNotReachHere();
2341   }
2342   __ align(BytesPerWord, offset);
2343 }
2344 
2345 
2346 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2347   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2348          "must be aligned");
2349   __ call(AddressLiteral(op->addr(), rtype));
2350   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2351   __ post_call_nop();
2352 }
2353 
2354 
2355 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2356   __ ic_call(op->addr());
2357   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2358   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2359          "must be aligned");
2360   __ post_call_nop();
2361 }
2362 
2363 
2364 void LIR_Assembler::emit_static_call_stub() {
2365   address call_pc = __ pc();
2366   address stub = __ start_a_stub(call_stub_size());
2367   if (stub == nullptr) {
2368     bailout("static call stub overflow");
2369     return;
2370   }
2371 
2372   int start = __ offset();
2373 
2374   // make sure that the displacement word of the call ends up word aligned
2375   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2376   __ relocate(static_stub_Relocation::spec(call_pc));
2377   __ mov_metadata(rbx, (Metadata*)nullptr);

2504   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2505 }
2506 
2507 
2508 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2509   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2510   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2511   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2512   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2513 }
2514 
2515 
2516 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2517   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2518   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2519   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2520   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2521 }
2522 
2523 
2524 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2525   if (null_check) {
2526     __ testptr(obj, obj);
2527     __ jcc(Assembler::zero, *slow_path->entry());
2528   }
2529   if (is_dest) {
2530     __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2531     // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2532     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2533   } else {
2534     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2535   }
2536 }
2537 
2538 
2539 // This code replaces a call to arraycopy; no exception may
2540 // be thrown in this code, they must be thrown in the System.arraycopy
2541 // activation frame; we could save some checks if this would not be the case
2542 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2543   ciArrayKlass* default_type = op->expected_type();
2544   Register src = op->src()->as_register();
2545   Register dst = op->dst()->as_register();
2546   Register src_pos = op->src_pos()->as_register();
2547   Register dst_pos = op->dst_pos()->as_register();
2548   Register length  = op->length()->as_register();
2549   Register tmp = op->tmp()->as_register();
2550   Register tmp_load_klass = rscratch1;
2551   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2552 
2553   CodeStub* stub = op->stub();
2554   int flags = op->flags();
2555   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2556   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2557 
2558   if (flags & LIR_OpArrayCopy::always_slow_path) {
2559     __ jmp(*stub->entry());
2560     __ bind(*stub->continuation());
2561     return;
2562   }
2563 
2564   // if we don't know anything, just go through the generic arraycopy
2565   if (default_type == nullptr) {
2566     // save outgoing arguments on stack in case call to System.arraycopy is needed
2567     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2568     // for interpreter calling conventions. Now we have to do it in new style conventions.
2569     // For the moment until C1 gets the new register allocator I just force all the
2570     // args to the right place (except the register args) and then on the back side
2571     // reload the register args properly if we go slow path. Yuck
2572 
2573     // These are proper for the calling convention
2574     store_parameter(length, 2);
2575     store_parameter(dst_pos, 1);
2576     store_parameter(dst, 0);
2577 
2578     // these are just temporary placements until we need to reload
2579     store_parameter(src_pos, 3);
2580     store_parameter(src, 4);
2581 
2582     address copyfunc_addr = StubRoutines::generic_arraycopy();
2583     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");

2620     __ mov(tmp, rax);
2621     __ xorl(tmp, -1);
2622 
2623     // Reload values from the stack so they are where the stub
2624     // expects them.
2625     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2626     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2627     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2628     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2629     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2630 
2631     __ subl(length, tmp);
2632     __ addl(src_pos, tmp);
2633     __ addl(dst_pos, tmp);
2634     __ jmp(*stub->entry());
2635 
2636     __ bind(*stub->continuation());
2637     return;
2638   }
2639 
2640   // Handle inline type arrays
2641   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2642     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2643   }
2644   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2645     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2646   }
2647 
2648   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2649 
2650   int elem_size = type2aelembytes(basic_type);
2651   Address::ScaleFactor scale;
2652 
2653   switch (elem_size) {
2654     case 1 :
2655       scale = Address::times_1;
2656       break;
2657     case 2 :
2658       scale = Address::times_2;
2659       break;
2660     case 4 :
2661       scale = Address::times_4;
2662       break;
2663     case 8 :
2664       scale = Address::times_8;
2665       break;
2666     default:
2667       scale = Address::no_scale;

2854       }
2855 
2856       __ jmp(*stub->entry());
2857 
2858       __ bind(cont);
2859       __ pop(dst);
2860       __ pop(src);
2861     }
2862   }
2863 
2864 #ifdef ASSERT
2865   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2866     // Sanity check the known type with the incoming class.  For the
2867     // primitive case the types must match exactly with src.klass and
2868     // dst.klass each exactly matching the default type.  For the
2869     // object array case, if no type check is needed then either the
2870     // dst type is exactly the expected type and the src type is a
2871     // subtype which we can't check or src is the same array as dst
2872     // but not necessarily exactly of type default_type.
2873     Label known_ok, halt;
2874 
2875     __ mov_metadata(tmp, default_type->constant_encoding());
2876     if (UseCompressedClassPointers) {
2877       __ encode_klass_not_null(tmp, rscratch1);
2878     }
2879 
2880     if (basic_type != T_OBJECT) {
2881       __ cmp_klass(tmp, dst, tmp2);
2882       __ jcc(Assembler::notEqual, halt);
2883       __ cmp_klass(tmp, src, tmp2);
2884       __ jcc(Assembler::equal, known_ok);
2885     } else {
2886       __ cmp_klass(tmp, dst, tmp2);
2887       __ jcc(Assembler::equal, known_ok);
2888       __ cmpptr(src, dst);
2889       __ jcc(Assembler::equal, known_ok);
2890     }
2891     __ bind(halt);
2892     __ stop("incorrect type information in arraycopy");
2893     __ bind(known_ok);
2894   }

3191         // first time here. Set profile type.
3192         __ movptr(mdo_addr, tmp);
3193 #ifdef ASSERT
3194         __ andptr(tmp, TypeEntries::type_klass_mask);
3195         __ verify_klass_ptr(tmp);
3196 #endif
3197       } else {
3198         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3199                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3200 
3201         __ testptr(mdo_addr, TypeEntries::type_unknown);
3202         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3203 
3204         __ orptr(mdo_addr, TypeEntries::type_unknown);
3205       }
3206     }
3207   }
3208   __ bind(next);
3209 }
3210 
3211 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3212   Register obj = op->obj()->as_register();
3213   Register tmp = op->tmp()->as_pointer_register();
3214   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3215   bool not_null = op->not_null();
3216   int flag = op->flag();
3217 
3218   Label not_inline_type;
3219   if (!not_null) {
3220     __ testptr(obj, obj);
3221     __ jccb(Assembler::zero, not_inline_type);
3222   }
3223 
3224   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3225 
3226   __ orb(mdo_addr, flag);
3227 
3228   __ bind(not_inline_type);
3229 }
3230 
3231 
3232 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3233   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3234 }
3235 
3236 
3237 void LIR_Assembler::align_backward_branch_target() {
3238   __ align(BytesPerWord);
3239 }
3240 
3241 
3242 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3243   if (left->is_single_cpu()) {
3244     __ negl(left->as_register());
3245     move_regs(left->as_register(), dest->as_register());
3246 
3247   } else if (left->is_double_cpu()) {
3248     Register lo = left->as_register_lo();
3249     Register dst = dest->as_register_lo();
3250     __ movptr(dst, lo);
3251     __ negptr(dst);

3397 }
3398 
3399 void LIR_Assembler::membar_loadstore() {
3400   // no-op
3401   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3402 }
3403 
3404 void LIR_Assembler::membar_storeload() {
3405   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3406 }
3407 
3408 void LIR_Assembler::on_spin_wait() {
3409   __ pause ();
3410 }
3411 
3412 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3413   assert(result_reg->is_register(), "check");
3414   __ mov(result_reg->as_register(), r15_thread);
3415 }
3416 
3417 void LIR_Assembler::check_orig_pc() {
3418   __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3419 }
3420 
3421 void LIR_Assembler::peephole(LIR_List*) {
3422   // do nothing for now
3423 }
3424 
3425 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3426   assert(data == dest, "xchg/xadd uses only 2 operands");
3427 
3428   if (data->type() == T_INT) {
3429     if (code == lir_xadd) {
3430       __ lock();
3431       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3432     } else {
3433       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3434     }
3435   } else if (data->is_oop()) {
3436     assert (code == lir_xchg, "xadd for oops");
3437     Register obj = data->as_register();
3438     if (UseCompressedOops) {
3439       __ encode_heap_oop(obj);
< prev index next >