15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "code/compiledIC.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_aarch64.inline.hpp"
46
47
48 #ifndef PRODUCT
49 #define COMMENT(x) do { __ block_comment(x); } while (0)
50 #else
51 #define COMMENT(x)
52 #endif
53
54 NEEDS_CLEANUP // remove this definitions ?
55 const Register SYNC_header = r0; // synchronization header
56 const Register SHIFT_count = r0; // where count for shift operations must be
57
58 #define __ _masm->
59
60
409 MonitorExitStub* stub = nullptr;
410 if (method()->is_synchronized()) {
411 monitor_address(0, FrameMap::r0_opr);
412 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
413 __ unlock_object(r5, r4, r0, r6, *stub->entry());
414 __ bind(*stub->continuation());
415 }
416
417 if (compilation()->env()->dtrace_method_probes()) {
418 __ mov(c_rarg0, rthread);
419 __ mov_metadata(c_rarg1, method()->constant_encoding());
420 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
421 }
422
423 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
424 __ mov(r0, r19); // Restore the exception
425 }
426
427 // remove the activation and dispatch to the unwind handler
428 __ block_comment("remove_frame and dispatch to the unwind handler");
429 __ remove_frame(initial_frame_size_in_bytes());
430 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
431
432 // Emit the slow path assembly
433 if (stub != nullptr) {
434 stub->emit_code(this);
435 }
436
437 return offset;
438 }
439
440
441 int LIR_Assembler::emit_deopt_handler() {
442 // generate code for exception handler
443 address handler_base = __ start_a_stub(deopt_handler_size());
444 if (handler_base == nullptr) {
445 // not enough space left for the handler
446 bailout("deopt handler overflow");
447 return -1;
448 }
449
453 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
454 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
455 __ end_a_stub();
456
457 return offset;
458 }
459
460 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
461 _masm->code_section()->relocate(adr, relocInfo::poll_type);
462 int pc_offset = code_offset();
463 flush_debug_info(pc_offset);
464 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
465 if (info->exception_handlers() != nullptr) {
466 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
467 }
468 }
469
470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
471 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
472
473 // Pop the stack before the safepoint code
474 __ remove_frame(initial_frame_size_in_bytes());
475
476 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
477 __ reserved_stack_check();
478 }
479
480 code_stub->set_safepoint_offset(__ offset());
481 __ relocate(relocInfo::poll_return_type);
482 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
483 __ ret(lr);
484 }
485
486 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
487 guarantee(info != nullptr, "Shouldn't be null");
488 __ get_polling_page(rscratch1, relocInfo::poll_type);
489 add_debug_info_for_branch(info); // This isn't just debug info:
490 // it's the oop map
491 __ read_polling_page(rscratch1, relocInfo::poll_type);
492 return __ offset();
493 }
494
495
496 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
497 if (from_reg == r31_sp)
498 from_reg = sp;
499 if (to_reg == r31_sp)
500 to_reg = sp;
501 __ mov(to_reg, from_reg);
502 }
503
504 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
505
512 switch (c->type()) {
513 case T_INT: {
514 assert(patch_code == lir_patch_none, "no patching handled here");
515 __ movw(dest->as_register(), c->as_jint());
516 break;
517 }
518
519 case T_ADDRESS: {
520 assert(patch_code == lir_patch_none, "no patching handled here");
521 __ mov(dest->as_register(), c->as_jint());
522 break;
523 }
524
525 case T_LONG: {
526 assert(patch_code == lir_patch_none, "no patching handled here");
527 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
528 break;
529 }
530
531 case T_OBJECT: {
532 if (patch_code == lir_patch_none) {
533 jobject2reg(c->as_jobject(), dest->as_register());
534 } else {
535 jobject2reg_with_patching(dest->as_register(), info);
536 }
537 break;
538 }
539
540 case T_METADATA: {
541 if (patch_code != lir_patch_none) {
542 klass2reg_with_patching(dest->as_register(), info);
543 } else {
544 __ mov_metadata(dest->as_register(), c->as_metadata());
545 }
546 break;
547 }
548
549 case T_FLOAT: {
550 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
551 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
552 } else {
553 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
554 __ ldrs(dest->as_float_reg(), Address(rscratch1));
555 }
625 LIR_Const* c = src->as_constant_ptr();
626 LIR_Address* to_addr = dest->as_address_ptr();
627
628 void (Assembler::* insn)(Register Rt, const Address &adr);
629
630 switch (type) {
631 case T_ADDRESS:
632 assert(c->as_jint() == 0, "should be");
633 insn = &Assembler::str;
634 break;
635 case T_LONG:
636 assert(c->as_jlong() == 0, "should be");
637 insn = &Assembler::str;
638 break;
639 case T_INT:
640 assert(c->as_jint() == 0, "should be");
641 insn = &Assembler::strw;
642 break;
643 case T_OBJECT:
644 case T_ARRAY:
645 assert(c->as_jobject() == nullptr, "should be");
646 if (UseCompressedOops && !wide) {
647 insn = &Assembler::strw;
648 } else {
649 insn = &Assembler::str;
650 }
651 break;
652 case T_CHAR:
653 case T_SHORT:
654 assert(c->as_jint() == 0, "should be");
655 insn = &Assembler::strh;
656 break;
657 case T_BOOLEAN:
658 case T_BYTE:
659 assert(c->as_jint() == 0, "should be");
660 insn = &Assembler::strb;
661 break;
662 default:
663 ShouldNotReachHere();
664 insn = &Assembler::str; // unreachable
972 case T_CHAR:
973 __ ldrh(dest->as_register(), as_Address(from_addr));
974 break;
975 case T_SHORT:
976 __ ldrsh(dest->as_register(), as_Address(from_addr));
977 break;
978
979 default:
980 ShouldNotReachHere();
981 }
982
983 if (is_reference_type(type)) {
984 if (UseCompressedOops && !wide) {
985 __ decode_heap_oop(dest->as_register());
986 }
987
988 __ verify_oop(dest->as_register());
989 }
990 }
991
992
993 int LIR_Assembler::array_element_size(BasicType type) const {
994 int elem_size = type2aelembytes(type);
995 return exact_log2(elem_size);
996 }
997
998
999 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1000 switch (op->code()) {
1001 case lir_idiv:
1002 case lir_irem:
1003 arithmetic_idiv(op->code(),
1004 op->in_opr1(),
1005 op->in_opr2(),
1006 op->in_opr3(),
1007 op->result_opr(),
1008 op->info());
1009 break;
1010 case lir_fmad:
1011 __ fmaddd(op->result_opr()->as_double_reg(),
1163 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1164 __ ldarb(rscratch1, rscratch1);
1165 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1166 add_debug_info_for_null_check_here(op->stub()->info());
1167 __ br(Assembler::NE, *op->stub()->entry());
1168 }
1169 __ allocate_object(op->obj()->as_register(),
1170 op->tmp1()->as_register(),
1171 op->tmp2()->as_register(),
1172 op->header_size(),
1173 op->object_size(),
1174 op->klass()->as_register(),
1175 *op->stub()->entry());
1176 __ bind(*op->stub()->continuation());
1177 }
1178
1179 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1180 Register len = op->len()->as_register();
1181 __ uxtw(len, len);
1182
1183 if (UseSlowPath ||
1184 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1185 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1186 __ b(*op->stub()->entry());
1187 } else {
1188 Register tmp1 = op->tmp1()->as_register();
1189 Register tmp2 = op->tmp2()->as_register();
1190 Register tmp3 = op->tmp3()->as_register();
1191 if (len == tmp1) {
1192 tmp1 = tmp3;
1193 } else if (len == tmp2) {
1194 tmp2 = tmp3;
1195 } else if (len == tmp3) {
1196 // everything is ok
1197 } else {
1198 __ mov(tmp3, len);
1199 }
1200 __ allocate_array(op->obj()->as_register(),
1201 len,
1202 tmp1,
1203 tmp2,
1275 assert(data != nullptr, "need data for type check");
1276 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1277 }
1278 Label* success_target = success;
1279 Label* failure_target = failure;
1280
1281 if (obj == k_RInfo) {
1282 k_RInfo = dst;
1283 } else if (obj == klass_RInfo) {
1284 klass_RInfo = dst;
1285 }
1286 if (k->is_loaded() && !UseCompressedClassPointers) {
1287 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1288 } else {
1289 Rtmp1 = op->tmp3()->as_register();
1290 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1291 }
1292
1293 assert_different_registers(obj, k_RInfo, klass_RInfo);
1294
1295 if (should_profile) {
1296 Register mdo = klass_RInfo;
1297 __ mov_metadata(mdo, md->constant_encoding());
1298 Label not_null;
1299 __ cbnz(obj, not_null);
1300 // Object is null; update MDO and exit
1301 Address data_addr
1302 = __ form_address(rscratch2, mdo,
1303 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1304 0);
1305 __ ldrb(rscratch1, data_addr);
1306 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1307 __ strb(rscratch1, data_addr);
1308 __ b(*obj_is_null);
1309 __ bind(not_null);
1310
1311 Label update_done;
1312 Register recv = k_RInfo;
1313 __ load_klass(recv, obj);
1314 type_profile_helper(mdo, md, data, recv, &update_done);
1315 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1316 __ addptr(counter_addr, DataLayout::counter_increment);
1317
1318 __ bind(update_done);
1319 } else {
1320 __ cbz(obj, *obj_is_null);
1321 }
1322
1323 if (!k->is_loaded()) {
1324 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1325 } else {
1326 __ mov_metadata(k_RInfo, k->constant_encoding());
1327 }
1328 __ verify_oop(obj);
1329
1330 if (op->fast_check()) {
1331 // get object class
1332 // not a safepoint as obj null check happens earlier
1333 __ load_klass(rscratch1, obj);
1334 __ cmp( rscratch1, k_RInfo);
1335
1336 __ br(Assembler::NE, *failure_target);
1337 // successful cast, fall through to profile or jump
1338 } else {
1339 // get object class
1340 // not a safepoint as obj null check happens earlier
1341 __ load_klass(klass_RInfo, obj);
1342 if (k->is_loaded()) {
1343 // See if we get an immediate positive hit
1344 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1345 __ cmp(k_RInfo, rscratch1);
1346 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1347 __ br(Assembler::NE, *failure_target);
1348 // successful cast, fall through to profile or jump
1349 } else {
1350 // See if we get an immediate positive hit
1351 __ br(Assembler::EQ, *success_target);
1352 // check for self
1353 __ cmp(klass_RInfo, k_RInfo);
1354 __ br(Assembler::EQ, *success_target);
1355
1356 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1357 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1358 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1359 // result is a boolean
1360 __ cbzw(klass_RInfo, *failure_target);
1361 // successful cast, fall through to profile or jump
1362 }
1363 } else {
1364 // perform the fast part of the checking logic
1365 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1366 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1367 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1368 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1369 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1370 // result is a boolean
1371 __ cbz(k_RInfo, *failure_target);
1372 // successful cast, fall through to profile or jump
1373 }
1458 __ bind(success);
1459 if (dst != obj) {
1460 __ mov(dst, obj);
1461 }
1462 } else if (code == lir_instanceof) {
1463 Register obj = op->object()->as_register();
1464 Register dst = op->result_opr()->as_register();
1465 Label success, failure, done;
1466 emit_typecheck_helper(op, &success, &failure, &failure);
1467 __ bind(failure);
1468 __ mov(dst, zr);
1469 __ b(done);
1470 __ bind(success);
1471 __ mov(dst, 1);
1472 __ bind(done);
1473 } else {
1474 ShouldNotReachHere();
1475 }
1476 }
1477
1478 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1479 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1480 __ cset(rscratch1, Assembler::NE);
1481 __ membar(__ AnyAny);
1482 }
1483
1484 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1485 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1486 __ cset(rscratch1, Assembler::NE);
1487 __ membar(__ AnyAny);
1488 }
1489
1490
1491 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1492 Register addr;
1493 if (op->addr()->is_register()) {
1494 addr = as_reg(op->addr());
1495 } else {
1496 assert(op->addr()->is_address(), "what else?");
1497 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1971 __ cmp(left->as_register_lo(), right->as_register_lo());
1972 __ mov(dst->as_register(), (uint64_t)-1L);
1973 __ br(Assembler::LT, done);
1974 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1975 __ bind(done);
1976 } else {
1977 ShouldNotReachHere();
1978 }
1979 }
1980
1981
1982 void LIR_Assembler::align_call(LIR_Code code) { }
1983
1984
1985 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1986 address call = __ trampoline_call(Address(op->addr(), rtype));
1987 if (call == nullptr) {
1988 bailout("trampoline stub overflow");
1989 return;
1990 }
1991 add_call_info(code_offset(), op->info());
1992 __ post_call_nop();
1993 }
1994
1995
1996 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1997 address call = __ ic_call(op->addr());
1998 if (call == nullptr) {
1999 bailout("trampoline stub overflow");
2000 return;
2001 }
2002 add_call_info(code_offset(), op->info());
2003 __ post_call_nop();
2004 }
2005
2006 void LIR_Assembler::emit_static_call_stub() {
2007 address call_pc = __ pc();
2008 address stub = __ start_a_stub(call_stub_size());
2009 if (stub == nullptr) {
2010 bailout("static call stub overflow");
2011 return;
2012 }
2013
2014 int start = __ offset();
2015
2016 __ relocate(static_stub_Relocation::spec(call_pc));
2017 __ emit_static_call_stub();
2018
2019 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2020 <= call_stub_size(), "stub too big");
2021 __ end_a_stub();
2022 }
2145
2146
2147 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2148 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2149 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2150 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2151 __ mov (rscratch1, c);
2152 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2153 }
2154
2155
2156 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2157 ShouldNotReachHere();
2158 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2159 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2160 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2161 __ lea(rscratch1, __ constant_oop_address(o));
2162 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2163 }
2164
2165
2166 // This code replaces a call to arraycopy; no exception may
2167 // be thrown in this code, they must be thrown in the System.arraycopy
2168 // activation frame; we could save some checks if this would not be the case
2169 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2170 ciArrayKlass* default_type = op->expected_type();
2171 Register src = op->src()->as_register();
2172 Register dst = op->dst()->as_register();
2173 Register src_pos = op->src_pos()->as_register();
2174 Register dst_pos = op->dst_pos()->as_register();
2175 Register length = op->length()->as_register();
2176 Register tmp = op->tmp()->as_register();
2177
2178 CodeStub* stub = op->stub();
2179 int flags = op->flags();
2180 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2181 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2182
2183 // if we don't know anything, just go through the generic arraycopy
2184 if (default_type == nullptr // || basic_type == T_OBJECT
2185 ) {
2186 Label done;
2187 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2188
2189 // Save the arguments in case the generic arraycopy fails and we
2190 // have to fall back to the JNI stub
2191 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2192 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2193 __ str(src, Address(sp, 4*BytesPerWord));
2194
2195 address copyfunc_addr = StubRoutines::generic_arraycopy();
2196 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2197
2198 // The arguments are in java calling convention so we shift them
2199 // to C convention
2200 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2201 __ mov(c_rarg0, j_rarg0);
2202 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2216 __ cbz(r0, *stub->continuation());
2217
2218 // Reload values from the stack so they are where the stub
2219 // expects them.
2220 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2221 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2222 __ ldr(src, Address(sp, 4*BytesPerWord));
2223
2224 // r0 is -1^K where K == partial copied count
2225 __ eonw(rscratch1, r0, zr);
2226 // adjust length down and src/end pos up by partial copied count
2227 __ subw(length, length, rscratch1);
2228 __ addw(src_pos, src_pos, rscratch1);
2229 __ addw(dst_pos, dst_pos, rscratch1);
2230 __ b(*stub->entry());
2231
2232 __ bind(*stub->continuation());
2233 return;
2234 }
2235
2236 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2237
2238 int elem_size = type2aelembytes(basic_type);
2239 int scale = exact_log2(elem_size);
2240
2241 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2242 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2243
2244 // test for null
2245 if (flags & LIR_OpArrayCopy::src_null_check) {
2246 __ cbz(src, *stub->entry());
2247 }
2248 if (flags & LIR_OpArrayCopy::dst_null_check) {
2249 __ cbz(dst, *stub->entry());
2250 }
2251
2252 // If the compiler was not able to prove that exact type of the source or the destination
2253 // of the arraycopy is an array type, check at runtime if the source or the destination is
2254 // an instance type.
2255 if (flags & LIR_OpArrayCopy::type_check) {
2757 __ verify_klass_ptr(tmp);
2758 #endif
2759 } else {
2760 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2761 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2762
2763 __ ldr(tmp, mdo_addr);
2764 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2765
2766 __ orr(tmp, tmp, TypeEntries::type_unknown);
2767 __ str(tmp, mdo_addr);
2768 // FIXME: Write barrier needed here?
2769 }
2770 }
2771
2772 __ bind(next);
2773 }
2774 COMMENT("} emit_profile_type");
2775 }
2776
2777
2778 void LIR_Assembler::align_backward_branch_target() {
2779 }
2780
2781
2782 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2783 // tmp must be unused
2784 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2785
2786 if (left->is_single_cpu()) {
2787 assert(dest->is_single_cpu(), "expect single result reg");
2788 __ negw(dest->as_register(), left->as_register());
2789 } else if (left->is_double_cpu()) {
2790 assert(dest->is_double_cpu(), "expect double result reg");
2791 __ neg(dest->as_register_lo(), left->as_register_lo());
2792 } else if (left->is_single_fpu()) {
2793 assert(dest->is_single_fpu(), "expect single float result reg");
2794 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2795 } else {
2796 assert(left->is_double_fpu(), "expect double float operand reg");
2896 void LIR_Assembler::membar_loadload() {
2897 __ membar(Assembler::LoadLoad);
2898 }
2899
2900 void LIR_Assembler::membar_storestore() {
2901 __ membar(MacroAssembler::StoreStore);
2902 }
2903
2904 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2905
2906 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2907
2908 void LIR_Assembler::on_spin_wait() {
2909 __ spin_wait();
2910 }
2911
2912 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2913 __ mov(result_reg->as_register(), rthread);
2914 }
2915
2916
2917 void LIR_Assembler::peephole(LIR_List *lir) {
2918 #if 0
2919 if (tableswitch_count >= max_tableswitches)
2920 return;
2921
2922 /*
2923 This finite-state automaton recognizes sequences of compare-and-
2924 branch instructions. We will turn them into a tableswitch. You
2925 could argue that C1 really shouldn't be doing this sort of
2926 optimization, but without it the code is really horrible.
2927 */
2928
2929 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2930 int first_key, last_key = -2147483648;
2931 int next_key = 0;
2932 int start_insn = -1;
2933 int last_insn = -1;
2934 Register reg = noreg;
2935 LIR_Opr reg_opr;
|
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInlineKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "ci/ciObjArrayKlass.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_aarch64.inline.hpp"
49
50
51 #ifndef PRODUCT
52 #define COMMENT(x) do { __ block_comment(x); } while (0)
53 #else
54 #define COMMENT(x)
55 #endif
56
57 NEEDS_CLEANUP // remove this definitions ?
58 const Register SYNC_header = r0; // synchronization header
59 const Register SHIFT_count = r0; // where count for shift operations must be
60
61 #define __ _masm->
62
63
412 MonitorExitStub* stub = nullptr;
413 if (method()->is_synchronized()) {
414 monitor_address(0, FrameMap::r0_opr);
415 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
416 __ unlock_object(r5, r4, r0, r6, *stub->entry());
417 __ bind(*stub->continuation());
418 }
419
420 if (compilation()->env()->dtrace_method_probes()) {
421 __ mov(c_rarg0, rthread);
422 __ mov_metadata(c_rarg1, method()->constant_encoding());
423 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
424 }
425
426 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
427 __ mov(r0, r19); // Restore the exception
428 }
429
430 // remove the activation and dispatch to the unwind handler
431 __ block_comment("remove_frame and dispatch to the unwind handler");
432 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
433 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
434
435 // Emit the slow path assembly
436 if (stub != nullptr) {
437 stub->emit_code(this);
438 }
439
440 return offset;
441 }
442
443
444 int LIR_Assembler::emit_deopt_handler() {
445 // generate code for exception handler
446 address handler_base = __ start_a_stub(deopt_handler_size());
447 if (handler_base == nullptr) {
448 // not enough space left for the handler
449 bailout("deopt handler overflow");
450 return -1;
451 }
452
456 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
458 __ end_a_stub();
459
460 return offset;
461 }
462
463 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
464 _masm->code_section()->relocate(adr, relocInfo::poll_type);
465 int pc_offset = code_offset();
466 flush_debug_info(pc_offset);
467 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
468 if (info->exception_handlers() != nullptr) {
469 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
470 }
471 }
472
473 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
474 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
475
476 if (InlineTypeReturnedAsFields) {
477 // Check if we are returning an non-null inline type and load its fields into registers
478 ciType* return_type = compilation()->method()->return_type();
479 if (return_type->is_inlinetype()) {
480 ciInlineKlass* vk = return_type->as_inline_klass();
481 if (vk->can_be_returned_as_fields()) {
482 address unpack_handler = vk->unpack_handler();
483 assert(unpack_handler != nullptr, "must be");
484 __ far_call(RuntimeAddress(unpack_handler));
485 }
486 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
487 Label skip;
488 Label not_null;
489 __ cbnz(r0, not_null);
490 // Returned value is null, zero all return registers because they may belong to oop fields
491 __ mov(j_rarg1, zr);
492 __ mov(j_rarg2, zr);
493 __ mov(j_rarg3, zr);
494 __ mov(j_rarg4, zr);
495 __ mov(j_rarg5, zr);
496 __ mov(j_rarg6, zr);
497 __ mov(j_rarg7, zr);
498 __ b(skip);
499 __ bind(not_null);
500
501 // Check if we are returning an non-null inline type and load its fields into registers
502 __ test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
503
504 // Load fields from a buffered value with an inline class specific handler
505 __ load_klass(rscratch1 /*dst*/, r0 /*src*/);
506 __ ldr(rscratch1, Address(rscratch1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
507 __ ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
508 // Unpack handler can be null if inline type is not scalarizable in returns
509 __ cbz(rscratch1, skip);
510 __ blr(rscratch1);
511
512 __ bind(skip);
513 }
514 // At this point, r0 points to the value object (for interpreter or C1 caller).
515 // The fields of the object are copied into registers (for C2 caller).
516 }
517
518 // Pop the stack before the safepoint code
519 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
520
521 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
522 __ reserved_stack_check();
523 }
524
525 code_stub->set_safepoint_offset(__ offset());
526 __ relocate(relocInfo::poll_return_type);
527 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
528 __ ret(lr);
529 }
530
531 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
532 return (__ store_inline_type_fields_to_buf(vk, false));
533 }
534
535 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
536 guarantee(info != nullptr, "Shouldn't be null");
537 __ get_polling_page(rscratch1, relocInfo::poll_type);
538 add_debug_info_for_branch(info); // This isn't just debug info:
539 // it's the oop map
540 __ read_polling_page(rscratch1, relocInfo::poll_type);
541 return __ offset();
542 }
543
544
545 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
546 if (from_reg == r31_sp)
547 from_reg = sp;
548 if (to_reg == r31_sp)
549 to_reg = sp;
550 __ mov(to_reg, from_reg);
551 }
552
553 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
554
561 switch (c->type()) {
562 case T_INT: {
563 assert(patch_code == lir_patch_none, "no patching handled here");
564 __ movw(dest->as_register(), c->as_jint());
565 break;
566 }
567
568 case T_ADDRESS: {
569 assert(patch_code == lir_patch_none, "no patching handled here");
570 __ mov(dest->as_register(), c->as_jint());
571 break;
572 }
573
574 case T_LONG: {
575 assert(patch_code == lir_patch_none, "no patching handled here");
576 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
577 break;
578 }
579
580 case T_OBJECT: {
581 if (patch_code != lir_patch_none) {
582 jobject2reg_with_patching(dest->as_register(), info);
583 } else {
584 jobject2reg(c->as_jobject(), dest->as_register());
585 }
586 break;
587 }
588
589 case T_METADATA: {
590 if (patch_code != lir_patch_none) {
591 klass2reg_with_patching(dest->as_register(), info);
592 } else {
593 __ mov_metadata(dest->as_register(), c->as_metadata());
594 }
595 break;
596 }
597
598 case T_FLOAT: {
599 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
600 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
601 } else {
602 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
603 __ ldrs(dest->as_float_reg(), Address(rscratch1));
604 }
674 LIR_Const* c = src->as_constant_ptr();
675 LIR_Address* to_addr = dest->as_address_ptr();
676
677 void (Assembler::* insn)(Register Rt, const Address &adr);
678
679 switch (type) {
680 case T_ADDRESS:
681 assert(c->as_jint() == 0, "should be");
682 insn = &Assembler::str;
683 break;
684 case T_LONG:
685 assert(c->as_jlong() == 0, "should be");
686 insn = &Assembler::str;
687 break;
688 case T_INT:
689 assert(c->as_jint() == 0, "should be");
690 insn = &Assembler::strw;
691 break;
692 case T_OBJECT:
693 case T_ARRAY:
694 // Non-null case is not handled on aarch64 but handled on x86
695 // FIXME: do we need to add it here?
696 assert(c->as_jobject() == nullptr, "should be");
697 if (UseCompressedOops && !wide) {
698 insn = &Assembler::strw;
699 } else {
700 insn = &Assembler::str;
701 }
702 break;
703 case T_CHAR:
704 case T_SHORT:
705 assert(c->as_jint() == 0, "should be");
706 insn = &Assembler::strh;
707 break;
708 case T_BOOLEAN:
709 case T_BYTE:
710 assert(c->as_jint() == 0, "should be");
711 insn = &Assembler::strb;
712 break;
713 default:
714 ShouldNotReachHere();
715 insn = &Assembler::str; // unreachable
1023 case T_CHAR:
1024 __ ldrh(dest->as_register(), as_Address(from_addr));
1025 break;
1026 case T_SHORT:
1027 __ ldrsh(dest->as_register(), as_Address(from_addr));
1028 break;
1029
1030 default:
1031 ShouldNotReachHere();
1032 }
1033
1034 if (is_reference_type(type)) {
1035 if (UseCompressedOops && !wide) {
1036 __ decode_heap_oop(dest->as_register());
1037 }
1038
1039 __ verify_oop(dest->as_register());
1040 }
1041 }
1042
1043 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1044 assert(dst->is_cpu_register(), "must be");
1045 assert(dst->type() == src->type(), "must be");
1046
1047 if (src->is_cpu_register()) {
1048 reg2reg(src, dst);
1049 } else if (src->is_stack()) {
1050 stack2reg(src, dst, dst->type());
1051 } else if (src->is_constant()) {
1052 const2reg(src, dst, lir_patch_none, nullptr);
1053 } else {
1054 ShouldNotReachHere();
1055 }
1056 }
1057
1058 int LIR_Assembler::array_element_size(BasicType type) const {
1059 int elem_size = type2aelembytes(type);
1060 return exact_log2(elem_size);
1061 }
1062
1063
1064 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1065 switch (op->code()) {
1066 case lir_idiv:
1067 case lir_irem:
1068 arithmetic_idiv(op->code(),
1069 op->in_opr1(),
1070 op->in_opr2(),
1071 op->in_opr3(),
1072 op->result_opr(),
1073 op->info());
1074 break;
1075 case lir_fmad:
1076 __ fmaddd(op->result_opr()->as_double_reg(),
1228 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1229 __ ldarb(rscratch1, rscratch1);
1230 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1231 add_debug_info_for_null_check_here(op->stub()->info());
1232 __ br(Assembler::NE, *op->stub()->entry());
1233 }
1234 __ allocate_object(op->obj()->as_register(),
1235 op->tmp1()->as_register(),
1236 op->tmp2()->as_register(),
1237 op->header_size(),
1238 op->object_size(),
1239 op->klass()->as_register(),
1240 *op->stub()->entry());
1241 __ bind(*op->stub()->continuation());
1242 }
1243
1244 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1245 Register len = op->len()->as_register();
1246 __ uxtw(len, len);
1247
1248 if (UseSlowPath || op->always_slow_path() ||
1249 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1250 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1251 __ b(*op->stub()->entry());
1252 } else {
1253 Register tmp1 = op->tmp1()->as_register();
1254 Register tmp2 = op->tmp2()->as_register();
1255 Register tmp3 = op->tmp3()->as_register();
1256 if (len == tmp1) {
1257 tmp1 = tmp3;
1258 } else if (len == tmp2) {
1259 tmp2 = tmp3;
1260 } else if (len == tmp3) {
1261 // everything is ok
1262 } else {
1263 __ mov(tmp3, len);
1264 }
1265 __ allocate_array(op->obj()->as_register(),
1266 len,
1267 tmp1,
1268 tmp2,
1340 assert(data != nullptr, "need data for type check");
1341 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1342 }
1343 Label* success_target = success;
1344 Label* failure_target = failure;
1345
1346 if (obj == k_RInfo) {
1347 k_RInfo = dst;
1348 } else if (obj == klass_RInfo) {
1349 klass_RInfo = dst;
1350 }
1351 if (k->is_loaded() && !UseCompressedClassPointers) {
1352 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1353 } else {
1354 Rtmp1 = op->tmp3()->as_register();
1355 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1356 }
1357
1358 assert_different_registers(obj, k_RInfo, klass_RInfo);
1359
1360 if (op->need_null_check()) {
1361 if (should_profile) {
1362 Register mdo = klass_RInfo;
1363 __ mov_metadata(mdo, md->constant_encoding());
1364 Label not_null;
1365 __ cbnz(obj, not_null);
1366 // Object is null; update MDO and exit
1367 Address data_addr
1368 = __ form_address(rscratch2, mdo,
1369 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1370 0);
1371 __ ldrb(rscratch1, data_addr);
1372 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1373 __ strb(rscratch1, data_addr);
1374 __ b(*obj_is_null);
1375 __ bind(not_null);
1376
1377 Label update_done;
1378 Register recv = k_RInfo;
1379 __ load_klass(recv, obj);
1380 type_profile_helper(mdo, md, data, recv, &update_done);
1381 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1382 __ addptr(counter_addr, DataLayout::counter_increment);
1383
1384 __ bind(update_done);
1385 } else {
1386 __ cbz(obj, *obj_is_null);
1387 }
1388 }
1389
1390 if (!k->is_loaded()) {
1391 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1392 } else {
1393 __ mov_metadata(k_RInfo, k->constant_encoding());
1394 }
1395 __ verify_oop(obj);
1396
1397 if (op->fast_check()) {
1398 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1399 // get object class
1400 // not a safepoint as obj null check happens earlier
1401 __ load_klass(rscratch1, obj);
1402 __ cmp( rscratch1, k_RInfo);
1403
1404 __ br(Assembler::NE, *failure_target);
1405 // successful cast, fall through to profile or jump
1406 } else {
1407 // get object class
1408 // not a safepoint as obj null check happens earlier
1409 __ load_klass(klass_RInfo, obj);
1410 if (k->is_loaded()) {
1411 // See if we get an immediate positive hit
1412 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1413 __ cmp(k_RInfo, rscratch1);
1414 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1415 __ br(Assembler::NE, *failure_target);
1416 // successful cast, fall through to profile or jump
1417 } else {
1418 // See if we get an immediate positive hit
1419 __ br(Assembler::EQ, *success_target);
1420 // check for self
1421 if (k->is_loaded() && k->is_obj_array_klass()) {
1422 // For a direct pointer comparison, we need the refined array klass pointer
1423 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1424 __ mov_metadata(rscratch1, k_refined->constant_encoding());
1425 __ cmp(klass_RInfo, rscratch1);
1426 } else {
1427 __ cmp(klass_RInfo, k_RInfo);
1428 }
1429 __ br(Assembler::EQ, *success_target);
1430
1431 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1432 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1433 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1434 // result is a boolean
1435 __ cbzw(klass_RInfo, *failure_target);
1436 // successful cast, fall through to profile or jump
1437 }
1438 } else {
1439 // perform the fast part of the checking logic
1440 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1441 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1442 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1443 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1444 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1445 // result is a boolean
1446 __ cbz(k_RInfo, *failure_target);
1447 // successful cast, fall through to profile or jump
1448 }
1533 __ bind(success);
1534 if (dst != obj) {
1535 __ mov(dst, obj);
1536 }
1537 } else if (code == lir_instanceof) {
1538 Register obj = op->object()->as_register();
1539 Register dst = op->result_opr()->as_register();
1540 Label success, failure, done;
1541 emit_typecheck_helper(op, &success, &failure, &failure);
1542 __ bind(failure);
1543 __ mov(dst, zr);
1544 __ b(done);
1545 __ bind(success);
1546 __ mov(dst, 1);
1547 __ bind(done);
1548 } else {
1549 ShouldNotReachHere();
1550 }
1551 }
1552
1553 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1554 // We are loading/storing from/to an array that *may* be a flat array (the
1555 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1556 // If this array is a flat array, take the slow path.
1557 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1558 if (!op->value()->is_illegal()) {
1559 // The array is not a flat array, but it might be null-free. If we are storing
1560 // a null into a null-free array, take the slow path (which will throw NPE).
1561 Label skip;
1562 __ cbnz(op->value()->as_register(), skip);
1563 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1564 __ bind(skip);
1565 }
1566 }
1567
1568 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1569 // We are storing into an array that *may* be null-free (the declared type is
1570 // Object[], abstract[], interface[] or VT.ref[]).
1571 Label test_mark_word;
1572 Register tmp = op->tmp()->as_register();
1573 __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1574 __ tst(tmp, markWord::unlocked_value);
1575 __ br(Assembler::NE, test_mark_word);
1576 __ load_prototype_header(tmp, op->array()->as_register());
1577 __ bind(test_mark_word);
1578 __ tst(tmp, markWord::null_free_array_bit_in_place);
1579 }
1580
1581 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1582 Label L_oops_equal;
1583 Label L_oops_not_equal;
1584 Label L_end;
1585
1586 Register left = op->left()->as_register();
1587 Register right = op->right()->as_register();
1588
1589 __ cmp(left, right);
1590 __ br(Assembler::EQ, L_oops_equal);
1591
1592 // (1) Null check -- if one of the operands is null, the other must not be null (because
1593 // the two references are not equal), so they are not substitutable,
1594 // FIXME: do null check only if the operand is nullable
1595 {
1596 __ cbz(left, L_oops_not_equal);
1597 __ cbz(right, L_oops_not_equal);
1598 }
1599
1600 ciKlass* left_klass = op->left_klass();
1601 ciKlass* right_klass = op->right_klass();
1602
1603 // (2) Inline type check -- if either of the operands is not a inline type,
1604 // they are not substitutable. We do this only if we are not sure that the
1605 // operands are inline type
1606 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1607 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1608 Register tmp1 = op->tmp1()->as_register();
1609 __ mov(tmp1, markWord::inline_type_pattern);
1610 __ ldr(rscratch1, Address(left, oopDesc::mark_offset_in_bytes()));
1611 __ andr(tmp1, tmp1, rscratch1);
1612 __ ldr(rscratch1, Address(right, oopDesc::mark_offset_in_bytes()));
1613 __ andr(tmp1, tmp1, rscratch1);
1614 __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1615 __ br(Assembler::NE, L_oops_not_equal);
1616 }
1617
1618 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1619 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1620 // No need to load klass -- the operands are statically known to be the same inline klass.
1621 __ b(*op->stub()->entry());
1622 } else {
1623 Register left_klass_op = op->left_klass_op()->as_register();
1624 Register right_klass_op = op->right_klass_op()->as_register();
1625
1626 if (UseCompressedClassPointers) {
1627 __ ldrw(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1628 __ ldrw(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1629 __ cmpw(left_klass_op, right_klass_op);
1630 } else {
1631 __ ldr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1632 __ ldr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1633 __ cmp(left_klass_op, right_klass_op);
1634 }
1635
1636 __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1637 // fall through to L_oops_not_equal
1638 }
1639
1640 __ bind(L_oops_not_equal);
1641 move(op->not_equal_result(), op->result_opr());
1642 __ b(L_end);
1643
1644 __ bind(L_oops_equal);
1645 move(op->equal_result(), op->result_opr());
1646 __ b(L_end);
1647
1648 // We've returned from the stub. R0 contains 0x0 IFF the two
1649 // operands are not substitutable. (Don't compare against 0x1 in case the
1650 // C compiler is naughty)
1651 __ bind(*op->stub()->continuation());
1652 __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1653 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1654 // fall-through
1655 __ bind(L_end);
1656 }
1657
1658
1659 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1660 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1661 __ cset(rscratch1, Assembler::NE);
1662 __ membar(__ AnyAny);
1663 }
1664
1665 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1666 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1667 __ cset(rscratch1, Assembler::NE);
1668 __ membar(__ AnyAny);
1669 }
1670
1671
1672 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1673 Register addr;
1674 if (op->addr()->is_register()) {
1675 addr = as_reg(op->addr());
1676 } else {
1677 assert(op->addr()->is_address(), "what else?");
1678 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
2152 __ cmp(left->as_register_lo(), right->as_register_lo());
2153 __ mov(dst->as_register(), (uint64_t)-1L);
2154 __ br(Assembler::LT, done);
2155 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2156 __ bind(done);
2157 } else {
2158 ShouldNotReachHere();
2159 }
2160 }
2161
2162
2163 void LIR_Assembler::align_call(LIR_Code code) { }
2164
2165
2166 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2167 address call = __ trampoline_call(Address(op->addr(), rtype));
2168 if (call == nullptr) {
2169 bailout("trampoline stub overflow");
2170 return;
2171 }
2172 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2173 __ post_call_nop();
2174 }
2175
2176
2177 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2178 address call = __ ic_call(op->addr());
2179 if (call == nullptr) {
2180 bailout("trampoline stub overflow");
2181 return;
2182 }
2183 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2184 __ post_call_nop();
2185 }
2186
2187 void LIR_Assembler::emit_static_call_stub() {
2188 address call_pc = __ pc();
2189 address stub = __ start_a_stub(call_stub_size());
2190 if (stub == nullptr) {
2191 bailout("static call stub overflow");
2192 return;
2193 }
2194
2195 int start = __ offset();
2196
2197 __ relocate(static_stub_Relocation::spec(call_pc));
2198 __ emit_static_call_stub();
2199
2200 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2201 <= call_stub_size(), "stub too big");
2202 __ end_a_stub();
2203 }
2326
2327
2328 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2329 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2330 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2331 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2332 __ mov (rscratch1, c);
2333 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2334 }
2335
2336
2337 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2338 ShouldNotReachHere();
2339 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2340 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2341 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2342 __ lea(rscratch1, __ constant_oop_address(o));
2343 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2344 }
2345
2346 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2347 if (null_check) {
2348 __ cbz(obj, *slow_path->entry());
2349 }
2350 if (is_dest) {
2351 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2352 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2353 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2354 } else {
2355 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2356 }
2357 }
2358
2359 // This code replaces a call to arraycopy; no exception may
2360 // be thrown in this code, they must be thrown in the System.arraycopy
2361 // activation frame; we could save some checks if this would not be the case
2362 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2363 ciArrayKlass* default_type = op->expected_type();
2364 Register src = op->src()->as_register();
2365 Register dst = op->dst()->as_register();
2366 Register src_pos = op->src_pos()->as_register();
2367 Register dst_pos = op->dst_pos()->as_register();
2368 Register length = op->length()->as_register();
2369 Register tmp = op->tmp()->as_register();
2370
2371 CodeStub* stub = op->stub();
2372 int flags = op->flags();
2373 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2374 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2375
2376 if (flags & LIR_OpArrayCopy::always_slow_path) {
2377 __ b(*stub->entry());
2378 __ bind(*stub->continuation());
2379 return;
2380 }
2381
2382 // if we don't know anything, just go through the generic arraycopy
2383 if (default_type == nullptr // || basic_type == T_OBJECT
2384 ) {
2385 Label done;
2386 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2387
2388 // Save the arguments in case the generic arraycopy fails and we
2389 // have to fall back to the JNI stub
2390 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2391 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2392 __ str(src, Address(sp, 4*BytesPerWord));
2393
2394 address copyfunc_addr = StubRoutines::generic_arraycopy();
2395 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2396
2397 // The arguments are in java calling convention so we shift them
2398 // to C convention
2399 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2400 __ mov(c_rarg0, j_rarg0);
2401 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2415 __ cbz(r0, *stub->continuation());
2416
2417 // Reload values from the stack so they are where the stub
2418 // expects them.
2419 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2420 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2421 __ ldr(src, Address(sp, 4*BytesPerWord));
2422
2423 // r0 is -1^K where K == partial copied count
2424 __ eonw(rscratch1, r0, zr);
2425 // adjust length down and src/end pos up by partial copied count
2426 __ subw(length, length, rscratch1);
2427 __ addw(src_pos, src_pos, rscratch1);
2428 __ addw(dst_pos, dst_pos, rscratch1);
2429 __ b(*stub->entry());
2430
2431 __ bind(*stub->continuation());
2432 return;
2433 }
2434
2435 // Handle inline type arrays
2436 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2437 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2438 }
2439 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2440 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2441 }
2442
2443 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2444
2445 int elem_size = type2aelembytes(basic_type);
2446 int scale = exact_log2(elem_size);
2447
2448 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2449 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2450
2451 // test for null
2452 if (flags & LIR_OpArrayCopy::src_null_check) {
2453 __ cbz(src, *stub->entry());
2454 }
2455 if (flags & LIR_OpArrayCopy::dst_null_check) {
2456 __ cbz(dst, *stub->entry());
2457 }
2458
2459 // If the compiler was not able to prove that exact type of the source or the destination
2460 // of the arraycopy is an array type, check at runtime if the source or the destination is
2461 // an instance type.
2462 if (flags & LIR_OpArrayCopy::type_check) {
2964 __ verify_klass_ptr(tmp);
2965 #endif
2966 } else {
2967 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2968 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2969
2970 __ ldr(tmp, mdo_addr);
2971 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2972
2973 __ orr(tmp, tmp, TypeEntries::type_unknown);
2974 __ str(tmp, mdo_addr);
2975 // FIXME: Write barrier needed here?
2976 }
2977 }
2978
2979 __ bind(next);
2980 }
2981 COMMENT("} emit_profile_type");
2982 }
2983
2984 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
2985 Register obj = op->obj()->as_register();
2986 Register tmp = op->tmp()->as_pointer_register();
2987 bool not_null = op->not_null();
2988 int flag = op->flag();
2989
2990 Label not_inline_type;
2991 if (!not_null) {
2992 __ cbz(obj, not_inline_type);
2993 }
2994
2995 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
2996
2997 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), rscratch2);
2998 __ ldrb(rscratch1, mdo_addr);
2999 __ orr(rscratch1, rscratch1, flag);
3000 __ strb(rscratch1, mdo_addr);
3001
3002 __ bind(not_inline_type);
3003 }
3004
3005 void LIR_Assembler::align_backward_branch_target() {
3006 }
3007
3008
3009 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3010 // tmp must be unused
3011 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3012
3013 if (left->is_single_cpu()) {
3014 assert(dest->is_single_cpu(), "expect single result reg");
3015 __ negw(dest->as_register(), left->as_register());
3016 } else if (left->is_double_cpu()) {
3017 assert(dest->is_double_cpu(), "expect double result reg");
3018 __ neg(dest->as_register_lo(), left->as_register_lo());
3019 } else if (left->is_single_fpu()) {
3020 assert(dest->is_single_fpu(), "expect single float result reg");
3021 __ fnegs(dest->as_float_reg(), left->as_float_reg());
3022 } else {
3023 assert(left->is_double_fpu(), "expect double float operand reg");
3123 void LIR_Assembler::membar_loadload() {
3124 __ membar(Assembler::LoadLoad);
3125 }
3126
3127 void LIR_Assembler::membar_storestore() {
3128 __ membar(MacroAssembler::StoreStore);
3129 }
3130
3131 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3132
3133 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3134
3135 void LIR_Assembler::on_spin_wait() {
3136 __ spin_wait();
3137 }
3138
3139 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3140 __ mov(result_reg->as_register(), rthread);
3141 }
3142
3143 void LIR_Assembler::check_orig_pc() {
3144 __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3145 __ cmp(rscratch2, (u1)NULL_WORD);
3146 }
3147
3148 void LIR_Assembler::peephole(LIR_List *lir) {
3149 #if 0
3150 if (tableswitch_count >= max_tableswitches)
3151 return;
3152
3153 /*
3154 This finite-state automaton recognizes sequences of compare-and-
3155 branch instructions. We will turn them into a tableswitch. You
3156 could argue that C1 really shouldn't be doing this sort of
3157 optimization, but without it the code is really horrible.
3158 */
3159
3160 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3161 int first_key, last_key = -2147483648;
3162 int next_key = 0;
3163 int start_insn = -1;
3164 int last_insn = -1;
3165 Register reg = noreg;
3166 LIR_Opr reg_opr;
|