15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "code/compiledIC.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_aarch64.inline.hpp"
46
47
48 #ifndef PRODUCT
49 #define COMMENT(x) do { __ block_comment(x); } while (0)
50 #else
51 #define COMMENT(x)
52 #endif
53
54 NEEDS_CLEANUP // remove this definitions ?
55 const Register SYNC_header = r0; // synchronization header
56 const Register SHIFT_count = r0; // where count for shift operations must be
57
58 #define __ _masm->
59
60
409 MonitorExitStub* stub = nullptr;
410 if (method()->is_synchronized()) {
411 monitor_address(0, FrameMap::r0_opr);
412 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
413 __ unlock_object(r5, r4, r0, r6, *stub->entry());
414 __ bind(*stub->continuation());
415 }
416
417 if (compilation()->env()->dtrace_method_probes()) {
418 __ mov(c_rarg0, rthread);
419 __ mov_metadata(c_rarg1, method()->constant_encoding());
420 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
421 }
422
423 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
424 __ mov(r0, r19); // Restore the exception
425 }
426
427 // remove the activation and dispatch to the unwind handler
428 __ block_comment("remove_frame and dispatch to the unwind handler");
429 __ remove_frame(initial_frame_size_in_bytes());
430 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
431
432 // Emit the slow path assembly
433 if (stub != nullptr) {
434 stub->emit_code(this);
435 }
436
437 return offset;
438 }
439
440
441 int LIR_Assembler::emit_deopt_handler() {
442 // generate code for exception handler
443 address handler_base = __ start_a_stub(deopt_handler_size());
444 if (handler_base == nullptr) {
445 // not enough space left for the handler
446 bailout("deopt handler overflow");
447 return -1;
448 }
449
461 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
462 "out of bounds read in post-call NOP check");
463 __ end_a_stub();
464
465 return entry_offset;
466 }
467
468 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
469 _masm->code_section()->relocate(adr, relocInfo::poll_type);
470 int pc_offset = code_offset();
471 flush_debug_info(pc_offset);
472 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
473 if (info->exception_handlers() != nullptr) {
474 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
475 }
476 }
477
478 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
479 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
480
481 // Pop the stack before the safepoint code
482 __ remove_frame(initial_frame_size_in_bytes());
483
484 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
485 __ reserved_stack_check();
486 }
487
488 code_stub->set_safepoint_offset(__ offset());
489 __ relocate(relocInfo::poll_return_type);
490 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
491 __ ret(lr);
492 }
493
494 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
495 guarantee(info != nullptr, "Shouldn't be null");
496 __ get_polling_page(rscratch1, relocInfo::poll_type);
497 add_debug_info_for_branch(info); // This isn't just debug info:
498 // it's the oop map
499 __ read_polling_page(rscratch1, relocInfo::poll_type);
500 return __ offset();
501 }
502
503
504 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
505 if (from_reg == r31_sp)
506 from_reg = sp;
507 if (to_reg == r31_sp)
508 to_reg = sp;
509 __ mov(to_reg, from_reg);
510 }
511
512 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
513
520 switch (c->type()) {
521 case T_INT: {
522 assert(patch_code == lir_patch_none, "no patching handled here");
523 __ movw(dest->as_register(), c->as_jint());
524 break;
525 }
526
527 case T_ADDRESS: {
528 assert(patch_code == lir_patch_none, "no patching handled here");
529 __ mov(dest->as_register(), c->as_jint());
530 break;
531 }
532
533 case T_LONG: {
534 assert(patch_code == lir_patch_none, "no patching handled here");
535 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
536 break;
537 }
538
539 case T_OBJECT: {
540 if (patch_code == lir_patch_none) {
541 jobject2reg(c->as_jobject(), dest->as_register());
542 } else {
543 jobject2reg_with_patching(dest->as_register(), info);
544 }
545 break;
546 }
547
548 case T_METADATA: {
549 if (patch_code != lir_patch_none) {
550 klass2reg_with_patching(dest->as_register(), info);
551 } else {
552 __ mov_metadata(dest->as_register(), c->as_metadata());
553 }
554 break;
555 }
556
557 case T_FLOAT: {
558 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
559 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
560 } else {
561 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
562 __ ldrs(dest->as_float_reg(), Address(rscratch1));
563 }
633 LIR_Const* c = src->as_constant_ptr();
634 LIR_Address* to_addr = dest->as_address_ptr();
635
636 void (Assembler::* insn)(Register Rt, const Address &adr);
637
638 switch (type) {
639 case T_ADDRESS:
640 assert(c->as_jint() == 0, "should be");
641 insn = &Assembler::str;
642 break;
643 case T_LONG:
644 assert(c->as_jlong() == 0, "should be");
645 insn = &Assembler::str;
646 break;
647 case T_INT:
648 assert(c->as_jint() == 0, "should be");
649 insn = &Assembler::strw;
650 break;
651 case T_OBJECT:
652 case T_ARRAY:
653 assert(c->as_jobject() == nullptr, "should be");
654 if (UseCompressedOops && !wide) {
655 insn = &Assembler::strw;
656 } else {
657 insn = &Assembler::str;
658 }
659 break;
660 case T_CHAR:
661 case T_SHORT:
662 assert(c->as_jint() == 0, "should be");
663 insn = &Assembler::strh;
664 break;
665 case T_BOOLEAN:
666 case T_BYTE:
667 assert(c->as_jint() == 0, "should be");
668 insn = &Assembler::strb;
669 break;
670 default:
671 ShouldNotReachHere();
672 insn = &Assembler::str; // unreachable
980 case T_CHAR:
981 __ ldrh(dest->as_register(), as_Address(from_addr));
982 break;
983 case T_SHORT:
984 __ ldrsh(dest->as_register(), as_Address(from_addr));
985 break;
986
987 default:
988 ShouldNotReachHere();
989 }
990
991 if (is_reference_type(type)) {
992 if (UseCompressedOops && !wide) {
993 __ decode_heap_oop(dest->as_register());
994 }
995
996 __ verify_oop(dest->as_register());
997 }
998 }
999
1000
1001 int LIR_Assembler::array_element_size(BasicType type) const {
1002 int elem_size = type2aelembytes(type);
1003 return exact_log2(elem_size);
1004 }
1005
1006
1007 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1008 switch (op->code()) {
1009 case lir_idiv:
1010 case lir_irem:
1011 arithmetic_idiv(op->code(),
1012 op->in_opr1(),
1013 op->in_opr2(),
1014 op->in_opr3(),
1015 op->result_opr(),
1016 op->info());
1017 break;
1018 case lir_fmad:
1019 __ fmaddd(op->result_opr()->as_double_reg(),
1171 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1172 __ ldarb(rscratch1, rscratch1);
1173 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1174 add_debug_info_for_null_check_here(op->stub()->info());
1175 __ br(Assembler::NE, *op->stub()->entry());
1176 }
1177 __ allocate_object(op->obj()->as_register(),
1178 op->tmp1()->as_register(),
1179 op->tmp2()->as_register(),
1180 op->header_size(),
1181 op->object_size(),
1182 op->klass()->as_register(),
1183 *op->stub()->entry());
1184 __ bind(*op->stub()->continuation());
1185 }
1186
1187 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1188 Register len = op->len()->as_register();
1189 __ uxtw(len, len);
1190
1191 if (UseSlowPath ||
1192 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1193 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1194 __ b(*op->stub()->entry());
1195 } else {
1196 Register tmp1 = op->tmp1()->as_register();
1197 Register tmp2 = op->tmp2()->as_register();
1198 Register tmp3 = op->tmp3()->as_register();
1199 if (len == tmp1) {
1200 tmp1 = tmp3;
1201 } else if (len == tmp2) {
1202 tmp2 = tmp3;
1203 } else if (len == tmp3) {
1204 // everything is ok
1205 } else {
1206 __ mov(tmp3, len);
1207 }
1208 __ allocate_array(op->obj()->as_register(),
1209 len,
1210 tmp1,
1211 tmp2,
1283 assert(data != nullptr, "need data for type check");
1284 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1285 }
1286 Label* success_target = success;
1287 Label* failure_target = failure;
1288
1289 if (obj == k_RInfo) {
1290 k_RInfo = dst;
1291 } else if (obj == klass_RInfo) {
1292 klass_RInfo = dst;
1293 }
1294 if (k->is_loaded() && !UseCompressedClassPointers) {
1295 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1296 } else {
1297 Rtmp1 = op->tmp3()->as_register();
1298 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1299 }
1300
1301 assert_different_registers(obj, k_RInfo, klass_RInfo);
1302
1303 if (should_profile) {
1304 Register mdo = klass_RInfo;
1305 __ mov_metadata(mdo, md->constant_encoding());
1306 Label not_null;
1307 __ cbnz(obj, not_null);
1308 // Object is null; update MDO and exit
1309 Address data_addr
1310 = __ form_address(rscratch2, mdo,
1311 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1312 0);
1313 __ ldrb(rscratch1, data_addr);
1314 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1315 __ strb(rscratch1, data_addr);
1316 __ b(*obj_is_null);
1317 __ bind(not_null);
1318
1319 Label update_done;
1320 Register recv = k_RInfo;
1321 __ load_klass(recv, obj);
1322 type_profile_helper(mdo, md, data, recv, &update_done);
1323 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1324 __ addptr(counter_addr, DataLayout::counter_increment);
1325
1326 __ bind(update_done);
1327 } else {
1328 __ cbz(obj, *obj_is_null);
1329 }
1330
1331 if (!k->is_loaded()) {
1332 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1333 } else {
1334 __ mov_metadata(k_RInfo, k->constant_encoding());
1335 }
1336 __ verify_oop(obj);
1337
1338 if (op->fast_check()) {
1339 // get object class
1340 // not a safepoint as obj null check happens earlier
1341 __ load_klass(rscratch1, obj);
1342 __ cmp( rscratch1, k_RInfo);
1343
1344 __ br(Assembler::NE, *failure_target);
1345 // successful cast, fall through to profile or jump
1346 } else {
1347 // get object class
1348 // not a safepoint as obj null check happens earlier
1349 __ load_klass(klass_RInfo, obj);
1350 if (k->is_loaded()) {
1351 // See if we get an immediate positive hit
1352 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1353 __ cmp(k_RInfo, rscratch1);
1354 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1355 __ br(Assembler::NE, *failure_target);
1356 // successful cast, fall through to profile or jump
1357 } else {
1358 // See if we get an immediate positive hit
1359 __ br(Assembler::EQ, *success_target);
1360 // check for self
1361 __ cmp(klass_RInfo, k_RInfo);
1362 __ br(Assembler::EQ, *success_target);
1363
1364 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1365 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1366 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1367 // result is a boolean
1368 __ cbzw(klass_RInfo, *failure_target);
1369 // successful cast, fall through to profile or jump
1370 }
1371 } else {
1372 // perform the fast part of the checking logic
1373 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1374 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1375 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1376 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1377 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1378 // result is a boolean
1379 __ cbz(k_RInfo, *failure_target);
1380 // successful cast, fall through to profile or jump
1381 }
1466 __ bind(success);
1467 if (dst != obj) {
1468 __ mov(dst, obj);
1469 }
1470 } else if (code == lir_instanceof) {
1471 Register obj = op->object()->as_register();
1472 Register dst = op->result_opr()->as_register();
1473 Label success, failure, done;
1474 emit_typecheck_helper(op, &success, &failure, &failure);
1475 __ bind(failure);
1476 __ mov(dst, zr);
1477 __ b(done);
1478 __ bind(success);
1479 __ mov(dst, 1);
1480 __ bind(done);
1481 } else {
1482 ShouldNotReachHere();
1483 }
1484 }
1485
1486 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1487 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1488 __ cset(rscratch1, Assembler::NE);
1489 __ membar(__ AnyAny);
1490 }
1491
1492 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1493 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1494 __ cset(rscratch1, Assembler::NE);
1495 __ membar(__ AnyAny);
1496 }
1497
1498
1499 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1500 Register addr;
1501 if (op->addr()->is_register()) {
1502 addr = as_reg(op->addr());
1503 } else {
1504 assert(op->addr()->is_address(), "what else?");
1505 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1979 __ cmp(left->as_register_lo(), right->as_register_lo());
1980 __ mov(dst->as_register(), (uint64_t)-1L);
1981 __ br(Assembler::LT, done);
1982 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1983 __ bind(done);
1984 } else {
1985 ShouldNotReachHere();
1986 }
1987 }
1988
1989
1990 void LIR_Assembler::align_call(LIR_Code code) { }
1991
1992
1993 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1994 address call = __ trampoline_call(Address(op->addr(), rtype));
1995 if (call == nullptr) {
1996 bailout("trampoline stub overflow");
1997 return;
1998 }
1999 add_call_info(code_offset(), op->info());
2000 __ post_call_nop();
2001 }
2002
2003
2004 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2005 address call = __ ic_call(op->addr());
2006 if (call == nullptr) {
2007 bailout("trampoline stub overflow");
2008 return;
2009 }
2010 add_call_info(code_offset(), op->info());
2011 __ post_call_nop();
2012 }
2013
2014 void LIR_Assembler::emit_static_call_stub() {
2015 address call_pc = __ pc();
2016 address stub = __ start_a_stub(call_stub_size());
2017 if (stub == nullptr) {
2018 bailout("static call stub overflow");
2019 return;
2020 }
2021
2022 int start = __ offset();
2023
2024 __ relocate(static_stub_Relocation::spec(call_pc));
2025 __ emit_static_call_stub();
2026
2027 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2028 <= call_stub_size(), "stub too big");
2029 __ end_a_stub();
2030 }
2153
2154
2155 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2156 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2157 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2158 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2159 __ mov (rscratch1, c);
2160 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2161 }
2162
2163
2164 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2165 ShouldNotReachHere();
2166 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2167 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2168 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2169 __ lea(rscratch1, __ constant_oop_address(o));
2170 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2171 }
2172
2173
2174 // This code replaces a call to arraycopy; no exception may
2175 // be thrown in this code, they must be thrown in the System.arraycopy
2176 // activation frame; we could save some checks if this would not be the case
2177 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2178 ciArrayKlass* default_type = op->expected_type();
2179 Register src = op->src()->as_register();
2180 Register dst = op->dst()->as_register();
2181 Register src_pos = op->src_pos()->as_register();
2182 Register dst_pos = op->dst_pos()->as_register();
2183 Register length = op->length()->as_register();
2184 Register tmp = op->tmp()->as_register();
2185
2186 CodeStub* stub = op->stub();
2187 int flags = op->flags();
2188 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2189 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2190
2191 // if we don't know anything, just go through the generic arraycopy
2192 if (default_type == nullptr // || basic_type == T_OBJECT
2193 ) {
2194 Label done;
2195 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2196
2197 // Save the arguments in case the generic arraycopy fails and we
2198 // have to fall back to the JNI stub
2199 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2200 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2201 __ str(src, Address(sp, 4*BytesPerWord));
2202
2203 address copyfunc_addr = StubRoutines::generic_arraycopy();
2204 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2205
2206 // The arguments are in java calling convention so we shift them
2207 // to C convention
2208 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2209 __ mov(c_rarg0, j_rarg0);
2210 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2224 __ cbz(r0, *stub->continuation());
2225
2226 // Reload values from the stack so they are where the stub
2227 // expects them.
2228 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2229 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2230 __ ldr(src, Address(sp, 4*BytesPerWord));
2231
2232 // r0 is -1^K where K == partial copied count
2233 __ eonw(rscratch1, r0, zr);
2234 // adjust length down and src/end pos up by partial copied count
2235 __ subw(length, length, rscratch1);
2236 __ addw(src_pos, src_pos, rscratch1);
2237 __ addw(dst_pos, dst_pos, rscratch1);
2238 __ b(*stub->entry());
2239
2240 __ bind(*stub->continuation());
2241 return;
2242 }
2243
2244 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2245
2246 int elem_size = type2aelembytes(basic_type);
2247 int scale = exact_log2(elem_size);
2248
2249 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2250 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2251
2252 // test for null
2253 if (flags & LIR_OpArrayCopy::src_null_check) {
2254 __ cbz(src, *stub->entry());
2255 }
2256 if (flags & LIR_OpArrayCopy::dst_null_check) {
2257 __ cbz(dst, *stub->entry());
2258 }
2259
2260 // If the compiler was not able to prove that exact type of the source or the destination
2261 // of the arraycopy is an array type, check at runtime if the source or the destination is
2262 // an instance type.
2263 if (flags & LIR_OpArrayCopy::type_check) {
2765 __ verify_klass_ptr(tmp);
2766 #endif
2767 } else {
2768 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2769 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2770
2771 __ ldr(tmp, mdo_addr);
2772 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2773
2774 __ orr(tmp, tmp, TypeEntries::type_unknown);
2775 __ str(tmp, mdo_addr);
2776 // FIXME: Write barrier needed here?
2777 }
2778 }
2779
2780 __ bind(next);
2781 }
2782 COMMENT("} emit_profile_type");
2783 }
2784
2785
2786 void LIR_Assembler::align_backward_branch_target() {
2787 }
2788
2789
2790 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2791 // tmp must be unused
2792 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2793
2794 if (left->is_single_cpu()) {
2795 assert(dest->is_single_cpu(), "expect single result reg");
2796 __ negw(dest->as_register(), left->as_register());
2797 } else if (left->is_double_cpu()) {
2798 assert(dest->is_double_cpu(), "expect double result reg");
2799 __ neg(dest->as_register_lo(), left->as_register_lo());
2800 } else if (left->is_single_fpu()) {
2801 assert(dest->is_single_fpu(), "expect single float result reg");
2802 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2803 } else {
2804 assert(left->is_double_fpu(), "expect double float operand reg");
2904 void LIR_Assembler::membar_loadload() {
2905 __ membar(Assembler::LoadLoad);
2906 }
2907
2908 void LIR_Assembler::membar_storestore() {
2909 __ membar(MacroAssembler::StoreStore);
2910 }
2911
2912 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2913
2914 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2915
2916 void LIR_Assembler::on_spin_wait() {
2917 __ spin_wait();
2918 }
2919
2920 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2921 __ mov(result_reg->as_register(), rthread);
2922 }
2923
2924
2925 void LIR_Assembler::peephole(LIR_List *lir) {
2926 #if 0
2927 if (tableswitch_count >= max_tableswitches)
2928 return;
2929
2930 /*
2931 This finite-state automaton recognizes sequences of compare-and-
2932 branch instructions. We will turn them into a tableswitch. You
2933 could argue that C1 really shouldn't be doing this sort of
2934 optimization, but without it the code is really horrible.
2935 */
2936
2937 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2938 int first_key, last_key = -2147483648;
2939 int next_key = 0;
2940 int start_insn = -1;
2941 int last_insn = -1;
2942 Register reg = noreg;
2943 LIR_Opr reg_opr;
|
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInlineKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "ci/ciObjArrayKlass.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_aarch64.inline.hpp"
49
50
51 #ifndef PRODUCT
52 #define COMMENT(x) do { __ block_comment(x); } while (0)
53 #else
54 #define COMMENT(x)
55 #endif
56
57 NEEDS_CLEANUP // remove this definitions ?
58 const Register SYNC_header = r0; // synchronization header
59 const Register SHIFT_count = r0; // where count for shift operations must be
60
61 #define __ _masm->
62
63
412 MonitorExitStub* stub = nullptr;
413 if (method()->is_synchronized()) {
414 monitor_address(0, FrameMap::r0_opr);
415 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
416 __ unlock_object(r5, r4, r0, r6, *stub->entry());
417 __ bind(*stub->continuation());
418 }
419
420 if (compilation()->env()->dtrace_method_probes()) {
421 __ mov(c_rarg0, rthread);
422 __ mov_metadata(c_rarg1, method()->constant_encoding());
423 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
424 }
425
426 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
427 __ mov(r0, r19); // Restore the exception
428 }
429
430 // remove the activation and dispatch to the unwind handler
431 __ block_comment("remove_frame and dispatch to the unwind handler");
432 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
433 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
434
435 // Emit the slow path assembly
436 if (stub != nullptr) {
437 stub->emit_code(this);
438 }
439
440 return offset;
441 }
442
443
444 int LIR_Assembler::emit_deopt_handler() {
445 // generate code for exception handler
446 address handler_base = __ start_a_stub(deopt_handler_size());
447 if (handler_base == nullptr) {
448 // not enough space left for the handler
449 bailout("deopt handler overflow");
450 return -1;
451 }
452
464 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
465 "out of bounds read in post-call NOP check");
466 __ end_a_stub();
467
468 return entry_offset;
469 }
470
471 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
472 _masm->code_section()->relocate(adr, relocInfo::poll_type);
473 int pc_offset = code_offset();
474 flush_debug_info(pc_offset);
475 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
476 if (info->exception_handlers() != nullptr) {
477 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
478 }
479 }
480
481 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
482 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
483
484 if (InlineTypeReturnedAsFields) {
485 // Check if we are returning an non-null inline type and load its fields into registers
486 ciType* return_type = compilation()->method()->return_type();
487 if (return_type->is_inlinetype()) {
488 ciInlineKlass* vk = return_type->as_inline_klass();
489 if (vk->can_be_returned_as_fields()) {
490 address unpack_handler = vk->unpack_handler();
491 assert(unpack_handler != nullptr, "must be");
492 __ far_call(RuntimeAddress(unpack_handler));
493 }
494 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
495 Label skip;
496 Label not_null;
497 __ cbnz(r0, not_null);
498 // Returned value is null, zero all return registers because they may belong to oop fields
499 __ mov(j_rarg1, zr);
500 __ mov(j_rarg2, zr);
501 __ mov(j_rarg3, zr);
502 __ mov(j_rarg4, zr);
503 __ mov(j_rarg5, zr);
504 __ mov(j_rarg6, zr);
505 __ mov(j_rarg7, zr);
506 __ b(skip);
507 __ bind(not_null);
508
509 // Check if we are returning an non-null inline type and load its fields into registers
510 __ test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
511
512 // Load fields from a buffered value with an inline class specific handler
513 __ load_klass(rscratch1 /*dst*/, r0 /*src*/);
514 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
515 __ ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
516 // Unpack handler can be null if inline type is not scalarizable in returns
517 __ cbz(rscratch1, skip);
518 __ blr(rscratch1);
519
520 __ bind(skip);
521 }
522 // At this point, r0 points to the value object (for interpreter or C1 caller).
523 // The fields of the object are copied into registers (for C2 caller).
524 }
525
526 // Pop the stack before the safepoint code
527 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
528
529 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
530 __ reserved_stack_check();
531 }
532
533 code_stub->set_safepoint_offset(__ offset());
534 __ relocate(relocInfo::poll_return_type);
535 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
536 __ ret(lr);
537 }
538
539 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
540 return (__ store_inline_type_fields_to_buf(vk, false));
541 }
542
543 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
544 guarantee(info != nullptr, "Shouldn't be null");
545 __ get_polling_page(rscratch1, relocInfo::poll_type);
546 add_debug_info_for_branch(info); // This isn't just debug info:
547 // it's the oop map
548 __ read_polling_page(rscratch1, relocInfo::poll_type);
549 return __ offset();
550 }
551
552
553 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
554 if (from_reg == r31_sp)
555 from_reg = sp;
556 if (to_reg == r31_sp)
557 to_reg = sp;
558 __ mov(to_reg, from_reg);
559 }
560
561 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
562
569 switch (c->type()) {
570 case T_INT: {
571 assert(patch_code == lir_patch_none, "no patching handled here");
572 __ movw(dest->as_register(), c->as_jint());
573 break;
574 }
575
576 case T_ADDRESS: {
577 assert(patch_code == lir_patch_none, "no patching handled here");
578 __ mov(dest->as_register(), c->as_jint());
579 break;
580 }
581
582 case T_LONG: {
583 assert(patch_code == lir_patch_none, "no patching handled here");
584 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
585 break;
586 }
587
588 case T_OBJECT: {
589 if (patch_code != lir_patch_none) {
590 jobject2reg_with_patching(dest->as_register(), info);
591 } else {
592 jobject2reg(c->as_jobject(), dest->as_register());
593 }
594 break;
595 }
596
597 case T_METADATA: {
598 if (patch_code != lir_patch_none) {
599 klass2reg_with_patching(dest->as_register(), info);
600 } else {
601 __ mov_metadata(dest->as_register(), c->as_metadata());
602 }
603 break;
604 }
605
606 case T_FLOAT: {
607 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
608 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
609 } else {
610 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
611 __ ldrs(dest->as_float_reg(), Address(rscratch1));
612 }
682 LIR_Const* c = src->as_constant_ptr();
683 LIR_Address* to_addr = dest->as_address_ptr();
684
685 void (Assembler::* insn)(Register Rt, const Address &adr);
686
687 switch (type) {
688 case T_ADDRESS:
689 assert(c->as_jint() == 0, "should be");
690 insn = &Assembler::str;
691 break;
692 case T_LONG:
693 assert(c->as_jlong() == 0, "should be");
694 insn = &Assembler::str;
695 break;
696 case T_INT:
697 assert(c->as_jint() == 0, "should be");
698 insn = &Assembler::strw;
699 break;
700 case T_OBJECT:
701 case T_ARRAY:
702 // Non-null case is not handled on aarch64 but handled on x86
703 // FIXME: do we need to add it here?
704 assert(c->as_jobject() == nullptr, "should be");
705 if (UseCompressedOops && !wide) {
706 insn = &Assembler::strw;
707 } else {
708 insn = &Assembler::str;
709 }
710 break;
711 case T_CHAR:
712 case T_SHORT:
713 assert(c->as_jint() == 0, "should be");
714 insn = &Assembler::strh;
715 break;
716 case T_BOOLEAN:
717 case T_BYTE:
718 assert(c->as_jint() == 0, "should be");
719 insn = &Assembler::strb;
720 break;
721 default:
722 ShouldNotReachHere();
723 insn = &Assembler::str; // unreachable
1031 case T_CHAR:
1032 __ ldrh(dest->as_register(), as_Address(from_addr));
1033 break;
1034 case T_SHORT:
1035 __ ldrsh(dest->as_register(), as_Address(from_addr));
1036 break;
1037
1038 default:
1039 ShouldNotReachHere();
1040 }
1041
1042 if (is_reference_type(type)) {
1043 if (UseCompressedOops && !wide) {
1044 __ decode_heap_oop(dest->as_register());
1045 }
1046
1047 __ verify_oop(dest->as_register());
1048 }
1049 }
1050
1051 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1052 assert(dst->is_cpu_register(), "must be");
1053 assert(dst->type() == src->type(), "must be");
1054
1055 if (src->is_cpu_register()) {
1056 reg2reg(src, dst);
1057 } else if (src->is_stack()) {
1058 stack2reg(src, dst, dst->type());
1059 } else if (src->is_constant()) {
1060 const2reg(src, dst, lir_patch_none, nullptr);
1061 } else {
1062 ShouldNotReachHere();
1063 }
1064 }
1065
1066 int LIR_Assembler::array_element_size(BasicType type) const {
1067 int elem_size = type2aelembytes(type);
1068 return exact_log2(elem_size);
1069 }
1070
1071
1072 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1073 switch (op->code()) {
1074 case lir_idiv:
1075 case lir_irem:
1076 arithmetic_idiv(op->code(),
1077 op->in_opr1(),
1078 op->in_opr2(),
1079 op->in_opr3(),
1080 op->result_opr(),
1081 op->info());
1082 break;
1083 case lir_fmad:
1084 __ fmaddd(op->result_opr()->as_double_reg(),
1236 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1237 __ ldarb(rscratch1, rscratch1);
1238 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1239 add_debug_info_for_null_check_here(op->stub()->info());
1240 __ br(Assembler::NE, *op->stub()->entry());
1241 }
1242 __ allocate_object(op->obj()->as_register(),
1243 op->tmp1()->as_register(),
1244 op->tmp2()->as_register(),
1245 op->header_size(),
1246 op->object_size(),
1247 op->klass()->as_register(),
1248 *op->stub()->entry());
1249 __ bind(*op->stub()->continuation());
1250 }
1251
1252 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1253 Register len = op->len()->as_register();
1254 __ uxtw(len, len);
1255
1256 if (UseSlowPath || op->always_slow_path() ||
1257 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1258 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1259 __ b(*op->stub()->entry());
1260 } else {
1261 Register tmp1 = op->tmp1()->as_register();
1262 Register tmp2 = op->tmp2()->as_register();
1263 Register tmp3 = op->tmp3()->as_register();
1264 if (len == tmp1) {
1265 tmp1 = tmp3;
1266 } else if (len == tmp2) {
1267 tmp2 = tmp3;
1268 } else if (len == tmp3) {
1269 // everything is ok
1270 } else {
1271 __ mov(tmp3, len);
1272 }
1273 __ allocate_array(op->obj()->as_register(),
1274 len,
1275 tmp1,
1276 tmp2,
1348 assert(data != nullptr, "need data for type check");
1349 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1350 }
1351 Label* success_target = success;
1352 Label* failure_target = failure;
1353
1354 if (obj == k_RInfo) {
1355 k_RInfo = dst;
1356 } else if (obj == klass_RInfo) {
1357 klass_RInfo = dst;
1358 }
1359 if (k->is_loaded() && !UseCompressedClassPointers) {
1360 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1361 } else {
1362 Rtmp1 = op->tmp3()->as_register();
1363 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1364 }
1365
1366 assert_different_registers(obj, k_RInfo, klass_RInfo);
1367
1368 if (op->need_null_check()) {
1369 if (should_profile) {
1370 Register mdo = klass_RInfo;
1371 __ mov_metadata(mdo, md->constant_encoding());
1372 Label not_null;
1373 __ cbnz(obj, not_null);
1374 // Object is null; update MDO and exit
1375 Address data_addr
1376 = __ form_address(rscratch2, mdo,
1377 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1378 0);
1379 __ ldrb(rscratch1, data_addr);
1380 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1381 __ strb(rscratch1, data_addr);
1382 __ b(*obj_is_null);
1383 __ bind(not_null);
1384
1385 Label update_done;
1386 Register recv = k_RInfo;
1387 __ load_klass(recv, obj);
1388 type_profile_helper(mdo, md, data, recv, &update_done);
1389 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1390 __ addptr(counter_addr, DataLayout::counter_increment);
1391
1392 __ bind(update_done);
1393 } else {
1394 __ cbz(obj, *obj_is_null);
1395 }
1396 }
1397
1398 if (!k->is_loaded()) {
1399 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1400 } else {
1401 __ mov_metadata(k_RInfo, k->constant_encoding());
1402 }
1403 __ verify_oop(obj);
1404
1405 if (op->fast_check()) {
1406 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1407 // get object class
1408 // not a safepoint as obj null check happens earlier
1409 __ load_klass(rscratch1, obj);
1410 __ cmp( rscratch1, k_RInfo);
1411
1412 __ br(Assembler::NE, *failure_target);
1413 // successful cast, fall through to profile or jump
1414 } else {
1415 // get object class
1416 // not a safepoint as obj null check happens earlier
1417 __ load_klass(klass_RInfo, obj);
1418 if (k->is_loaded()) {
1419 // See if we get an immediate positive hit
1420 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1421 __ cmp(k_RInfo, rscratch1);
1422 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1423 __ br(Assembler::NE, *failure_target);
1424 // successful cast, fall through to profile or jump
1425 } else {
1426 // See if we get an immediate positive hit
1427 __ br(Assembler::EQ, *success_target);
1428 // check for self
1429 if (k->is_loaded() && k->is_obj_array_klass()) {
1430 // For a direct pointer comparison, we need the refined array klass pointer
1431 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1432 __ mov_metadata(rscratch1, k_refined->constant_encoding());
1433 __ cmp(klass_RInfo, rscratch1);
1434 } else {
1435 __ cmp(klass_RInfo, k_RInfo);
1436 }
1437 __ br(Assembler::EQ, *success_target);
1438
1439 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1440 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1441 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1442 // result is a boolean
1443 __ cbzw(klass_RInfo, *failure_target);
1444 // successful cast, fall through to profile or jump
1445 }
1446 } else {
1447 // perform the fast part of the checking logic
1448 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1449 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1450 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1451 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1452 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1453 // result is a boolean
1454 __ cbz(k_RInfo, *failure_target);
1455 // successful cast, fall through to profile or jump
1456 }
1541 __ bind(success);
1542 if (dst != obj) {
1543 __ mov(dst, obj);
1544 }
1545 } else if (code == lir_instanceof) {
1546 Register obj = op->object()->as_register();
1547 Register dst = op->result_opr()->as_register();
1548 Label success, failure, done;
1549 emit_typecheck_helper(op, &success, &failure, &failure);
1550 __ bind(failure);
1551 __ mov(dst, zr);
1552 __ b(done);
1553 __ bind(success);
1554 __ mov(dst, 1);
1555 __ bind(done);
1556 } else {
1557 ShouldNotReachHere();
1558 }
1559 }
1560
1561 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1562 // We are loading/storing from/to an array that *may* be a flat array (the
1563 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1564 // If this array is a flat array, take the slow path.
1565 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1566 if (!op->value()->is_illegal()) {
1567 // The array is not a flat array, but it might be null-free. If we are storing
1568 // a null into a null-free array, take the slow path (which will throw NPE).
1569 Label skip;
1570 __ cbnz(op->value()->as_register(), skip);
1571 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1572 __ bind(skip);
1573 }
1574 }
1575
1576 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1577 // We are storing into an array that *may* be null-free (the declared type is
1578 // Object[], abstract[], interface[] or VT.ref[]).
1579 Label test_mark_word;
1580 Register tmp = op->tmp()->as_register();
1581 __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1582 __ tst(tmp, markWord::unlocked_value);
1583 __ br(Assembler::NE, test_mark_word);
1584 __ load_prototype_header(tmp, op->array()->as_register());
1585 __ bind(test_mark_word);
1586 __ tst(tmp, markWord::null_free_array_bit_in_place);
1587 }
1588
1589 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1590 Label L_oops_equal;
1591 Label L_oops_not_equal;
1592 Label L_end;
1593
1594 Register left = op->left()->as_register();
1595 Register right = op->right()->as_register();
1596
1597 __ cmp(left, right);
1598 __ br(Assembler::EQ, L_oops_equal);
1599
1600 // (1) Null check -- if one of the operands is null, the other must not be null (because
1601 // the two references are not equal), so they are not substitutable,
1602 // FIXME: do null check only if the operand is nullable
1603 {
1604 __ cbz(left, L_oops_not_equal);
1605 __ cbz(right, L_oops_not_equal);
1606 }
1607
1608 ciKlass* left_klass = op->left_klass();
1609 ciKlass* right_klass = op->right_klass();
1610
1611 // (2) Inline type check -- if either of the operands is not a inline type,
1612 // they are not substitutable. We do this only if we are not sure that the
1613 // operands are inline type
1614 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1615 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1616 Register tmp1 = op->tmp1()->as_register();
1617 __ mov(tmp1, markWord::inline_type_pattern);
1618 __ ldr(rscratch1, Address(left, oopDesc::mark_offset_in_bytes()));
1619 __ andr(tmp1, tmp1, rscratch1);
1620 __ ldr(rscratch1, Address(right, oopDesc::mark_offset_in_bytes()));
1621 __ andr(tmp1, tmp1, rscratch1);
1622 __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1623 __ br(Assembler::NE, L_oops_not_equal);
1624 }
1625
1626 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1627 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1628 // No need to load klass -- the operands are statically known to be the same inline klass.
1629 __ b(*op->stub()->entry());
1630 } else {
1631 Register left_klass_op = op->left_klass_op()->as_register();
1632 Register right_klass_op = op->right_klass_op()->as_register();
1633
1634 if (UseCompressedClassPointers) {
1635 __ ldrw(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1636 __ ldrw(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1637 __ cmpw(left_klass_op, right_klass_op);
1638 } else {
1639 __ ldr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1640 __ ldr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1641 __ cmp(left_klass_op, right_klass_op);
1642 }
1643
1644 __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1645 // fall through to L_oops_not_equal
1646 }
1647
1648 __ bind(L_oops_not_equal);
1649 move(op->not_equal_result(), op->result_opr());
1650 __ b(L_end);
1651
1652 __ bind(L_oops_equal);
1653 move(op->equal_result(), op->result_opr());
1654 __ b(L_end);
1655
1656 // We've returned from the stub. R0 contains 0x0 IFF the two
1657 // operands are not substitutable. (Don't compare against 0x1 in case the
1658 // C compiler is naughty)
1659 __ bind(*op->stub()->continuation());
1660 __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1661 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1662 // fall-through
1663 __ bind(L_end);
1664 }
1665
1666
1667 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1668 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1669 __ cset(rscratch1, Assembler::NE);
1670 __ membar(__ AnyAny);
1671 }
1672
1673 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1674 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1675 __ cset(rscratch1, Assembler::NE);
1676 __ membar(__ AnyAny);
1677 }
1678
1679
1680 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1681 Register addr;
1682 if (op->addr()->is_register()) {
1683 addr = as_reg(op->addr());
1684 } else {
1685 assert(op->addr()->is_address(), "what else?");
1686 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
2160 __ cmp(left->as_register_lo(), right->as_register_lo());
2161 __ mov(dst->as_register(), (uint64_t)-1L);
2162 __ br(Assembler::LT, done);
2163 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2164 __ bind(done);
2165 } else {
2166 ShouldNotReachHere();
2167 }
2168 }
2169
2170
2171 void LIR_Assembler::align_call(LIR_Code code) { }
2172
2173
2174 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2175 address call = __ trampoline_call(Address(op->addr(), rtype));
2176 if (call == nullptr) {
2177 bailout("trampoline stub overflow");
2178 return;
2179 }
2180 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2181 __ post_call_nop();
2182 }
2183
2184
2185 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2186 address call = __ ic_call(op->addr());
2187 if (call == nullptr) {
2188 bailout("trampoline stub overflow");
2189 return;
2190 }
2191 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2192 __ post_call_nop();
2193 }
2194
2195 void LIR_Assembler::emit_static_call_stub() {
2196 address call_pc = __ pc();
2197 address stub = __ start_a_stub(call_stub_size());
2198 if (stub == nullptr) {
2199 bailout("static call stub overflow");
2200 return;
2201 }
2202
2203 int start = __ offset();
2204
2205 __ relocate(static_stub_Relocation::spec(call_pc));
2206 __ emit_static_call_stub();
2207
2208 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2209 <= call_stub_size(), "stub too big");
2210 __ end_a_stub();
2211 }
2334
2335
2336 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2337 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2338 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2339 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2340 __ mov (rscratch1, c);
2341 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2342 }
2343
2344
2345 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2346 ShouldNotReachHere();
2347 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2348 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2349 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2350 __ lea(rscratch1, __ constant_oop_address(o));
2351 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2352 }
2353
2354 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2355 if (null_check) {
2356 __ cbz(obj, *slow_path->entry());
2357 }
2358 if (is_dest) {
2359 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2360 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2361 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2362 } else {
2363 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2364 }
2365 }
2366
2367 // This code replaces a call to arraycopy; no exception may
2368 // be thrown in this code, they must be thrown in the System.arraycopy
2369 // activation frame; we could save some checks if this would not be the case
2370 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2371 ciArrayKlass* default_type = op->expected_type();
2372 Register src = op->src()->as_register();
2373 Register dst = op->dst()->as_register();
2374 Register src_pos = op->src_pos()->as_register();
2375 Register dst_pos = op->dst_pos()->as_register();
2376 Register length = op->length()->as_register();
2377 Register tmp = op->tmp()->as_register();
2378
2379 CodeStub* stub = op->stub();
2380 int flags = op->flags();
2381 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2382 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2383
2384 if (flags & LIR_OpArrayCopy::always_slow_path) {
2385 __ b(*stub->entry());
2386 __ bind(*stub->continuation());
2387 return;
2388 }
2389
2390 // if we don't know anything, just go through the generic arraycopy
2391 if (default_type == nullptr // || basic_type == T_OBJECT
2392 ) {
2393 Label done;
2394 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2395
2396 // Save the arguments in case the generic arraycopy fails and we
2397 // have to fall back to the JNI stub
2398 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2399 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2400 __ str(src, Address(sp, 4*BytesPerWord));
2401
2402 address copyfunc_addr = StubRoutines::generic_arraycopy();
2403 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2404
2405 // The arguments are in java calling convention so we shift them
2406 // to C convention
2407 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2408 __ mov(c_rarg0, j_rarg0);
2409 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2423 __ cbz(r0, *stub->continuation());
2424
2425 // Reload values from the stack so they are where the stub
2426 // expects them.
2427 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2428 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2429 __ ldr(src, Address(sp, 4*BytesPerWord));
2430
2431 // r0 is -1^K where K == partial copied count
2432 __ eonw(rscratch1, r0, zr);
2433 // adjust length down and src/end pos up by partial copied count
2434 __ subw(length, length, rscratch1);
2435 __ addw(src_pos, src_pos, rscratch1);
2436 __ addw(dst_pos, dst_pos, rscratch1);
2437 __ b(*stub->entry());
2438
2439 __ bind(*stub->continuation());
2440 return;
2441 }
2442
2443 // Handle inline type arrays
2444 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2445 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2446 }
2447 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2448 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2449 }
2450
2451 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2452
2453 int elem_size = type2aelembytes(basic_type);
2454 int scale = exact_log2(elem_size);
2455
2456 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2457 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2458
2459 // test for null
2460 if (flags & LIR_OpArrayCopy::src_null_check) {
2461 __ cbz(src, *stub->entry());
2462 }
2463 if (flags & LIR_OpArrayCopy::dst_null_check) {
2464 __ cbz(dst, *stub->entry());
2465 }
2466
2467 // If the compiler was not able to prove that exact type of the source or the destination
2468 // of the arraycopy is an array type, check at runtime if the source or the destination is
2469 // an instance type.
2470 if (flags & LIR_OpArrayCopy::type_check) {
2972 __ verify_klass_ptr(tmp);
2973 #endif
2974 } else {
2975 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2976 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2977
2978 __ ldr(tmp, mdo_addr);
2979 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2980
2981 __ orr(tmp, tmp, TypeEntries::type_unknown);
2982 __ str(tmp, mdo_addr);
2983 // FIXME: Write barrier needed here?
2984 }
2985 }
2986
2987 __ bind(next);
2988 }
2989 COMMENT("} emit_profile_type");
2990 }
2991
2992 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
2993 Register obj = op->obj()->as_register();
2994 Register tmp = op->tmp()->as_pointer_register();
2995 bool not_null = op->not_null();
2996 int flag = op->flag();
2997
2998 Label not_inline_type;
2999 if (!not_null) {
3000 __ cbz(obj, not_inline_type);
3001 }
3002
3003 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3004
3005 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), rscratch2);
3006 __ ldrb(rscratch1, mdo_addr);
3007 __ orr(rscratch1, rscratch1, flag);
3008 __ strb(rscratch1, mdo_addr);
3009
3010 __ bind(not_inline_type);
3011 }
3012
3013 void LIR_Assembler::align_backward_branch_target() {
3014 }
3015
3016
3017 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3018 // tmp must be unused
3019 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3020
3021 if (left->is_single_cpu()) {
3022 assert(dest->is_single_cpu(), "expect single result reg");
3023 __ negw(dest->as_register(), left->as_register());
3024 } else if (left->is_double_cpu()) {
3025 assert(dest->is_double_cpu(), "expect double result reg");
3026 __ neg(dest->as_register_lo(), left->as_register_lo());
3027 } else if (left->is_single_fpu()) {
3028 assert(dest->is_single_fpu(), "expect single float result reg");
3029 __ fnegs(dest->as_float_reg(), left->as_float_reg());
3030 } else {
3031 assert(left->is_double_fpu(), "expect double float operand reg");
3131 void LIR_Assembler::membar_loadload() {
3132 __ membar(Assembler::LoadLoad);
3133 }
3134
3135 void LIR_Assembler::membar_storestore() {
3136 __ membar(MacroAssembler::StoreStore);
3137 }
3138
3139 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3140
3141 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3142
3143 void LIR_Assembler::on_spin_wait() {
3144 __ spin_wait();
3145 }
3146
3147 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3148 __ mov(result_reg->as_register(), rthread);
3149 }
3150
3151 void LIR_Assembler::check_orig_pc() {
3152 __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3153 __ cmp(rscratch2, (u1)NULL_WORD);
3154 }
3155
3156 void LIR_Assembler::peephole(LIR_List *lir) {
3157 #if 0
3158 if (tableswitch_count >= max_tableswitches)
3159 return;
3160
3161 /*
3162 This finite-state automaton recognizes sequences of compare-and-
3163 branch instructions. We will turn them into a tableswitch. You
3164 could argue that C1 really shouldn't be doing this sort of
3165 optimization, but without it the code is really horrible.
3166 */
3167
3168 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3169 int first_key, last_key = -2147483648;
3170 int next_key = 0;
3171 int start_insn = -1;
3172 int last_insn = -1;
3173 Register reg = noreg;
3174 LIR_Opr reg_opr;
|