16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/threadIdentifier.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_aarch64.inline.hpp"
49
50
51 #ifndef PRODUCT
52 #define COMMENT(x) do { __ block_comment(x); } while (0)
53 #else
54 #define COMMENT(x)
55 #endif
56
57 NEEDS_CLEANUP // remove this definitions ?
58 const Register SYNC_header = r0; // synchronization header
59 const Register SHIFT_count = r0; // where count for shift operations must be
60
61 #define __ _masm->
62
396 MonitorExitStub* stub = nullptr;
397 if (method()->is_synchronized()) {
398 monitor_address(0, FrameMap::r0_opr);
399 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
400 __ unlock_object(r5, r4, r0, r6, *stub->entry());
401 __ bind(*stub->continuation());
402 }
403
404 if (compilation()->env()->dtrace_method_probes()) {
405 __ mov(c_rarg0, rthread);
406 __ mov_metadata(c_rarg1, method()->constant_encoding());
407 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
408 }
409
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(r0, r19); // Restore the exception
412 }
413
414 // remove the activation and dispatch to the unwind handler
415 __ block_comment("remove_frame and dispatch to the unwind handler");
416 __ remove_frame(initial_frame_size_in_bytes());
417 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
418
419 // Emit the slow path assembly
420 if (stub != nullptr) {
421 stub->emit_code(this);
422 }
423
424 return offset;
425 }
426
427
428 int LIR_Assembler::emit_deopt_handler() {
429 // generate code for exception handler
430 address handler_base = __ start_a_stub(deopt_handler_size());
431 if (handler_base == nullptr) {
432 // not enough space left for the handler
433 bailout("deopt handler overflow");
434 return -1;
435 }
436
448 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
449 "out of bounds read in post-call NOP check");
450 __ end_a_stub();
451
452 return entry_offset;
453 }
454
455 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
456 _masm->code_section()->relocate(adr, relocInfo::poll_type);
457 int pc_offset = code_offset();
458 flush_debug_info(pc_offset);
459 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
460 if (info->exception_handlers() != nullptr) {
461 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
462 }
463 }
464
465 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
466 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
467
468 // Pop the stack before the safepoint code
469 __ remove_frame(initial_frame_size_in_bytes());
470
471 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
472 __ reserved_stack_check();
473 }
474
475 code_stub->set_safepoint_offset(__ offset());
476 __ relocate(relocInfo::poll_return_type);
477 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
478 __ ret(lr);
479 }
480
481 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
482 guarantee(info != nullptr, "Shouldn't be null");
483 __ get_polling_page(rscratch1, relocInfo::poll_type);
484 add_debug_info_for_branch(info); // This isn't just debug info:
485 // it's the oop map
486 __ read_polling_page(rscratch1, relocInfo::poll_type);
487 return __ offset();
488 }
489
490
491 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
492 if (from_reg == r31_sp)
493 from_reg = sp;
494 if (to_reg == r31_sp)
495 to_reg = sp;
496 __ mov(to_reg, from_reg);
497 }
498
499 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
500
520 case T_LONG: {
521 assert(patch_code == lir_patch_none, "no patching handled here");
522 #if INCLUDE_CDS
523 if (AOTCodeCache::is_on_for_dump()) {
524 address b = c->as_pointer();
525 if (b == (address)ThreadIdentifier::unsafe_offset()) {
526 __ lea(dest->as_register_lo(), ExternalAddress(b));
527 break;
528 }
529 if (AOTRuntimeConstants::contains(b)) {
530 __ load_aotrc_address(dest->as_register_lo(), b);
531 break;
532 }
533 }
534 #endif
535 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
536 break;
537 }
538
539 case T_OBJECT: {
540 if (patch_code == lir_patch_none) {
541 jobject2reg(c->as_jobject(), dest->as_register());
542 } else {
543 jobject2reg_with_patching(dest->as_register(), info);
544 }
545 break;
546 }
547
548 case T_METADATA: {
549 if (patch_code != lir_patch_none) {
550 klass2reg_with_patching(dest->as_register(), info);
551 } else {
552 __ mov_metadata(dest->as_register(), c->as_metadata());
553 }
554 break;
555 }
556
557 case T_FLOAT: {
558 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
559 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
560 } else {
561 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
562 __ ldrs(dest->as_float_reg(), Address(rscratch1));
563 }
996 case T_BYTE:
997 __ ldrsb(dest->as_register(), as_Address(from_addr));
998 break;
999 case T_BOOLEAN: {
1000 __ ldrb(dest->as_register(), as_Address(from_addr));
1001 break;
1002 }
1003
1004 case T_CHAR:
1005 __ ldrh(dest->as_register(), as_Address(from_addr));
1006 break;
1007 case T_SHORT:
1008 __ ldrsh(dest->as_register(), as_Address(from_addr));
1009 break;
1010
1011 default:
1012 ShouldNotReachHere();
1013 }
1014 }
1015
1016 void LIR_Assembler::load_volatile(LIR_Address *from_addr, LIR_Opr dest,
1017 BasicType type, CodeEmitInfo* info) {
1018 __ lea(rscratch1, as_Address(from_addr));
1019
1020 Register dest_reg = rscratch2;
1021 if (!is_floating_point_type(type)) {
1022 dest_reg = (dest->is_single_cpu()
1023 ? dest->as_register() : dest->as_register_lo());
1024 }
1025
1026 if (info != nullptr) {
1027 add_debug_info_for_null_check_here(info);
1028 }
1029
1030 // Uses LDAR to ensure memory ordering.
1031 __ load_store_volatile(dest_reg, type, rscratch1, /*is_load*/true);
1032
1033 switch (type) {
1034 // LDAR is unsigned so need to sign-extend for byte and short
1035 case T_BYTE:
1223 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1224 __ ldarb(rscratch1, rscratch1);
1225 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1226 add_debug_info_for_null_check_here(op->stub()->info());
1227 __ br(Assembler::NE, *op->stub()->entry());
1228 }
1229 __ allocate_object(op->obj()->as_register(),
1230 op->tmp1()->as_register(),
1231 op->tmp2()->as_register(),
1232 op->header_size(),
1233 op->object_size(),
1234 op->klass()->as_register(),
1235 *op->stub()->entry());
1236 __ bind(*op->stub()->continuation());
1237 }
1238
1239 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1240 Register len = op->len()->as_register();
1241 __ uxtw(len, len);
1242
1243 if (UseSlowPath ||
1244 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1245 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1246 __ b(*op->stub()->entry());
1247 } else {
1248 Register tmp1 = op->tmp1()->as_register();
1249 Register tmp2 = op->tmp2()->as_register();
1250 Register tmp3 = op->tmp3()->as_register();
1251 if (len == tmp1) {
1252 tmp1 = tmp3;
1253 } else if (len == tmp2) {
1254 tmp2 = tmp3;
1255 } else if (len == tmp3) {
1256 // everything is ok
1257 } else {
1258 __ mov(tmp3, len);
1259 }
1260 __ allocate_array(op->obj()->as_register(),
1261 len,
1262 tmp1,
1263 tmp2,
1300 md = method->method_data_or_null();
1301 assert(md != nullptr, "Sanity");
1302 data = md->bci_to_data(bci);
1303 assert(data != nullptr, "need data for type check");
1304 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1305 }
1306 Label* success_target = success;
1307 Label* failure_target = failure;
1308
1309 if (obj == k_RInfo) {
1310 k_RInfo = dst;
1311 } else if (obj == klass_RInfo) {
1312 klass_RInfo = dst;
1313 }
1314
1315 Rtmp1 = op->tmp3()->as_register();
1316 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1317
1318 assert_different_registers(obj, k_RInfo, klass_RInfo);
1319
1320 if (should_profile) {
1321 Register mdo = klass_RInfo;
1322 __ mov_metadata(mdo, md->constant_encoding());
1323 Label not_null;
1324 __ cbnz(obj, not_null);
1325 // Object is null; update MDO and exit
1326 Address data_addr
1327 = __ form_address(rscratch2, mdo,
1328 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1329 0);
1330 __ ldrb(rscratch1, data_addr);
1331 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1332 __ strb(rscratch1, data_addr);
1333 __ b(*obj_is_null);
1334 __ bind(not_null);
1335
1336 Register recv = k_RInfo;
1337 __ load_klass(recv, obj);
1338 type_profile_helper(mdo, md, data, recv);
1339 } else {
1340 __ cbz(obj, *obj_is_null);
1341 }
1342
1343 if (!k->is_loaded()) {
1344 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1345 } else {
1346 __ mov_metadata(k_RInfo, k->constant_encoding());
1347 }
1348 __ verify_oop(obj);
1349
1350 if (op->fast_check()) {
1351 // get object class
1352 // not a safepoint as obj null check happens earlier
1353 __ load_klass(rscratch1, obj);
1354 __ cmp( rscratch1, k_RInfo);
1355
1356 __ br(Assembler::NE, *failure_target);
1357 // successful cast, fall through to profile or jump
1358 } else {
1359 // get object class
1360 // not a safepoint as obj null check happens earlier
1361 __ load_klass(klass_RInfo, obj);
1362 if (k->is_loaded()) {
1363 // See if we get an immediate positive hit
1364 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1365 __ cmp(k_RInfo, rscratch1);
1366 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1367 __ br(Assembler::NE, *failure_target);
1368 // successful cast, fall through to profile or jump
1369 } else {
1370 // See if we get an immediate positive hit
1371 __ br(Assembler::EQ, *success_target);
1372 // check for self
1373 __ cmp(klass_RInfo, k_RInfo);
1374 __ br(Assembler::EQ, *success_target);
1375
1376 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1377 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1378 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1379 // result is a boolean
1380 __ cbzw(klass_RInfo, *failure_target);
1381 // successful cast, fall through to profile or jump
1382 }
1383 } else {
1384 // perform the fast part of the checking logic
1385 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1386 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1387 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1388 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1389 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1390 // result is a boolean
1391 __ cbz(k_RInfo, *failure_target);
1392 // successful cast, fall through to profile or jump
1393 }
1474 __ bind(success);
1475 if (dst != obj) {
1476 __ mov(dst, obj);
1477 }
1478 } else if (code == lir_instanceof) {
1479 Register obj = op->object()->as_register();
1480 Register dst = op->result_opr()->as_register();
1481 Label success, failure, done;
1482 emit_typecheck_helper(op, &success, &failure, &failure);
1483 __ bind(failure);
1484 __ mov(dst, zr);
1485 __ b(done);
1486 __ bind(success);
1487 __ mov(dst, 1);
1488 __ bind(done);
1489 } else {
1490 ShouldNotReachHere();
1491 }
1492 }
1493
1494 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1495 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1496 __ cset(rscratch1, Assembler::NE);
1497 }
1498
1499 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1500 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1501 __ cset(rscratch1, Assembler::NE);
1502 }
1503
1504
1505 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1506 Register addr;
1507 if (op->addr()->is_register()) {
1508 addr = as_reg(op->addr());
1509 } else {
1510 assert(op->addr()->is_address(), "what else?");
1511 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1512 assert(addr_ptr->disp() == 0, "need 0 disp");
1513 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1985 __ cmp(left->as_register_lo(), right->as_register_lo());
1986 __ mov(dst->as_register(), (uint64_t)-1L);
1987 __ br(Assembler::LT, done);
1988 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1989 __ bind(done);
1990 } else {
1991 ShouldNotReachHere();
1992 }
1993 }
1994
1995
1996 void LIR_Assembler::align_call(LIR_Code code) { }
1997
1998
1999 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2000 address call = __ trampoline_call(Address(op->addr(), rtype));
2001 if (call == nullptr) {
2002 bailout("trampoline stub overflow");
2003 return;
2004 }
2005 add_call_info(code_offset(), op->info());
2006 __ post_call_nop();
2007 }
2008
2009
2010 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2011 address call = __ ic_call(op->addr());
2012 if (call == nullptr) {
2013 bailout("trampoline stub overflow");
2014 return;
2015 }
2016 add_call_info(code_offset(), op->info());
2017 __ post_call_nop();
2018 }
2019
2020 void LIR_Assembler::emit_static_call_stub() {
2021 address call_pc = __ pc();
2022 address stub = __ start_a_stub(call_stub_size());
2023 if (stub == nullptr) {
2024 bailout("static call stub overflow");
2025 return;
2026 }
2027
2028 int start = __ offset();
2029
2030 __ relocate(static_stub_Relocation::spec(call_pc));
2031 __ emit_static_call_stub();
2032
2033 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2034 <= call_stub_size(), "stub too big");
2035 __ end_a_stub();
2036 }
2159
2160
2161 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2162 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2163 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2164 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2165 __ mov (rscratch1, c);
2166 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2167 }
2168
2169
2170 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2171 ShouldNotReachHere();
2172 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2173 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2174 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2175 __ lea(rscratch1, __ constant_oop_address(o));
2176 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2177 }
2178
2179
2180 // This code replaces a call to arraycopy; no exception may
2181 // be thrown in this code, they must be thrown in the System.arraycopy
2182 // activation frame; we could save some checks if this would not be the case
2183 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2184 ciArrayKlass* default_type = op->expected_type();
2185 Register src = op->src()->as_register();
2186 Register dst = op->dst()->as_register();
2187 Register src_pos = op->src_pos()->as_register();
2188 Register dst_pos = op->dst_pos()->as_register();
2189 Register length = op->length()->as_register();
2190 Register tmp = op->tmp()->as_register();
2191
2192 CodeStub* stub = op->stub();
2193 int flags = op->flags();
2194 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2195 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2196
2197 // if we don't know anything, just go through the generic arraycopy
2198 if (default_type == nullptr // || basic_type == T_OBJECT
2199 ) {
2200 Label done;
2201 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2202
2203 // Save the arguments in case the generic arraycopy fails and we
2204 // have to fall back to the JNI stub
2205 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2206 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2207 __ str(src, Address(sp, 4*BytesPerWord));
2208
2209 address copyfunc_addr = StubRoutines::generic_arraycopy();
2210 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2211
2212 // The arguments are in java calling convention so we shift them
2213 // to C convention
2214 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2215 __ mov(c_rarg0, j_rarg0);
2216 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2230 __ cbz(r0, *stub->continuation());
2231
2232 // Reload values from the stack so they are where the stub
2233 // expects them.
2234 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2235 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2236 __ ldr(src, Address(sp, 4*BytesPerWord));
2237
2238 // r0 is -1^K where K == partial copied count
2239 __ eonw(rscratch1, r0, zr);
2240 // adjust length down and src/end pos up by partial copied count
2241 __ subw(length, length, rscratch1);
2242 __ addw(src_pos, src_pos, rscratch1);
2243 __ addw(dst_pos, dst_pos, rscratch1);
2244 __ b(*stub->entry());
2245
2246 __ bind(*stub->continuation());
2247 return;
2248 }
2249
2250 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2251
2252 int elem_size = type2aelembytes(basic_type);
2253 int scale = exact_log2(elem_size);
2254
2255 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2256 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2257
2258 // test for null
2259 if (flags & LIR_OpArrayCopy::src_null_check) {
2260 __ cbz(src, *stub->entry());
2261 }
2262 if (flags & LIR_OpArrayCopy::dst_null_check) {
2263 __ cbz(dst, *stub->entry());
2264 }
2265
2266 // If the compiler was not able to prove that exact type of the source or the destination
2267 // of the arraycopy is an array type, check at runtime if the source or the destination is
2268 // an instance type.
2269 if (flags & LIR_OpArrayCopy::type_check) {
2744 __ verify_klass_ptr(tmp);
2745 #endif
2746 } else {
2747 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2748 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2749
2750 __ ldr(tmp, mdo_addr);
2751 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2752
2753 __ orr(tmp, tmp, TypeEntries::type_unknown);
2754 __ str(tmp, mdo_addr);
2755 // FIXME: Write barrier needed here?
2756 }
2757 }
2758
2759 __ bind(next);
2760 }
2761 COMMENT("} emit_profile_type");
2762 }
2763
2764
2765 void LIR_Assembler::align_backward_branch_target() {
2766 }
2767
2768
2769 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2770 // tmp must be unused
2771 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2772
2773 if (left->is_single_cpu()) {
2774 assert(dest->is_single_cpu(), "expect single result reg");
2775 __ negw(dest->as_register(), left->as_register());
2776 } else if (left->is_double_cpu()) {
2777 assert(dest->is_double_cpu(), "expect double result reg");
2778 __ neg(dest->as_register_lo(), left->as_register_lo());
2779 } else if (left->is_single_fpu()) {
2780 assert(dest->is_single_fpu(), "expect single float result reg");
2781 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2782 } else {
2783 assert(left->is_double_fpu(), "expect double float operand reg");
2885 void LIR_Assembler::membar_loadload() {
2886 __ membar(Assembler::LoadLoad);
2887 }
2888
2889 void LIR_Assembler::membar_storestore() {
2890 __ membar(MacroAssembler::StoreStore);
2891 }
2892
2893 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2894
2895 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2896
2897 void LIR_Assembler::on_spin_wait() {
2898 __ spin_wait();
2899 }
2900
2901 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2902 __ mov(result_reg->as_register(), rthread);
2903 }
2904
2905
2906 void LIR_Assembler::peephole(LIR_List *lir) {
2907 #if 0
2908 if (tableswitch_count >= max_tableswitches)
2909 return;
2910
2911 /*
2912 This finite-state automaton recognizes sequences of compare-and-
2913 branch instructions. We will turn them into a tableswitch. You
2914 could argue that C1 really shouldn't be doing this sort of
2915 optimization, but without it the code is really horrible.
2916 */
2917
2918 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2919 int first_key, last_key = -2147483648;
2920 int next_key = 0;
2921 int start_insn = -1;
2922 int last_insn = -1;
2923 Register reg = noreg;
2924 LIR_Opr reg_opr;
|
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInlineKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "ci/ciObjArrayKlass.hpp"
39 #include "code/aotCodeCache.hpp"
40 #include "code/compiledIC.hpp"
41 #include "gc/shared/collectedHeap.hpp"
42 #include "gc/shared/gc_globals.hpp"
43 #include "nativeInst_aarch64.hpp"
44 #include "oops/objArrayKlass.hpp"
45 #include "oops/oop.inline.hpp"
46 #include "runtime/frame.inline.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/threadIdentifier.hpp"
50 #include "utilities/powerOfTwo.hpp"
51 #include "vmreg_aarch64.inline.hpp"
52
53
54 #ifndef PRODUCT
55 #define COMMENT(x) do { __ block_comment(x); } while (0)
56 #else
57 #define COMMENT(x)
58 #endif
59
60 NEEDS_CLEANUP // remove this definitions ?
61 const Register SYNC_header = r0; // synchronization header
62 const Register SHIFT_count = r0; // where count for shift operations must be
63
64 #define __ _masm->
65
399 MonitorExitStub* stub = nullptr;
400 if (method()->is_synchronized()) {
401 monitor_address(0, FrameMap::r0_opr);
402 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
403 __ unlock_object(r5, r4, r0, r6, *stub->entry());
404 __ bind(*stub->continuation());
405 }
406
407 if (compilation()->env()->dtrace_method_probes()) {
408 __ mov(c_rarg0, rthread);
409 __ mov_metadata(c_rarg1, method()->constant_encoding());
410 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
411 }
412
413 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
414 __ mov(r0, r19); // Restore the exception
415 }
416
417 // remove the activation and dispatch to the unwind handler
418 __ block_comment("remove_frame and dispatch to the unwind handler");
419 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
420 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
421
422 // Emit the slow path assembly
423 if (stub != nullptr) {
424 stub->emit_code(this);
425 }
426
427 return offset;
428 }
429
430
431 int LIR_Assembler::emit_deopt_handler() {
432 // generate code for exception handler
433 address handler_base = __ start_a_stub(deopt_handler_size());
434 if (handler_base == nullptr) {
435 // not enough space left for the handler
436 bailout("deopt handler overflow");
437 return -1;
438 }
439
451 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
452 "out of bounds read in post-call NOP check");
453 __ end_a_stub();
454
455 return entry_offset;
456 }
457
458 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
459 _masm->code_section()->relocate(adr, relocInfo::poll_type);
460 int pc_offset = code_offset();
461 flush_debug_info(pc_offset);
462 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
463 if (info->exception_handlers() != nullptr) {
464 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
465 }
466 }
467
468 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
469 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
470
471 if (InlineTypeReturnedAsFields) {
472 // Check if we are returning a non-null inline type and load its fields into registers
473 ciType* return_type = compilation()->method()->return_type();
474 if (return_type->is_inlinetype()) {
475 ciInlineKlass* vk = return_type->as_inline_klass();
476 if (vk->can_be_returned_as_fields()) {
477 address unpack_handler = vk->unpack_handler();
478 assert(unpack_handler != nullptr, "must be");
479 __ far_call(RuntimeAddress(unpack_handler));
480 }
481 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
482 Label skip;
483 Label not_null;
484 __ cbnz(r0, not_null);
485 // Returned value is null, zero all return registers because they may belong to oop fields
486 __ mov(j_rarg1, zr);
487 __ mov(j_rarg2, zr);
488 __ mov(j_rarg3, zr);
489 __ mov(j_rarg4, zr);
490 __ mov(j_rarg5, zr);
491 __ mov(j_rarg6, zr);
492 __ mov(j_rarg7, zr);
493 __ b(skip);
494 __ bind(not_null);
495
496 // Check if we are returning a non-null inline type and load its fields into registers
497 __ test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
498
499 // Load fields from a buffered value with an inline class specific handler
500 __ load_klass(rscratch1 /*dst*/, r0 /*src*/);
501 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
502 __ ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
503 // Unpack handler can be null if inline type is not scalarizable in returns
504 __ cbz(rscratch1, skip);
505 __ blr(rscratch1);
506
507 __ bind(skip);
508 }
509 // At this point, r0 points to the value object (for interpreter or C1 caller).
510 // The fields of the object are copied into registers (for C2 caller).
511 }
512
513 // Pop the stack before the safepoint code
514 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
515
516 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
517 __ reserved_stack_check();
518 }
519
520 code_stub->set_safepoint_offset(__ offset());
521 __ relocate(relocInfo::poll_return_type);
522 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
523 __ ret(lr);
524 }
525
526 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
527 return (__ store_inline_type_fields_to_buf(vk, false));
528 }
529
530 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
531 guarantee(info != nullptr, "Shouldn't be null");
532 __ get_polling_page(rscratch1, relocInfo::poll_type);
533 add_debug_info_for_branch(info); // This isn't just debug info:
534 // it's the oop map
535 __ read_polling_page(rscratch1, relocInfo::poll_type);
536 return __ offset();
537 }
538
539
540 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
541 if (from_reg == r31_sp)
542 from_reg = sp;
543 if (to_reg == r31_sp)
544 to_reg = sp;
545 __ mov(to_reg, from_reg);
546 }
547
548 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
549
569 case T_LONG: {
570 assert(patch_code == lir_patch_none, "no patching handled here");
571 #if INCLUDE_CDS
572 if (AOTCodeCache::is_on_for_dump()) {
573 address b = c->as_pointer();
574 if (b == (address)ThreadIdentifier::unsafe_offset()) {
575 __ lea(dest->as_register_lo(), ExternalAddress(b));
576 break;
577 }
578 if (AOTRuntimeConstants::contains(b)) {
579 __ load_aotrc_address(dest->as_register_lo(), b);
580 break;
581 }
582 }
583 #endif
584 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
585 break;
586 }
587
588 case T_OBJECT: {
589 if (patch_code != lir_patch_none) {
590 jobject2reg_with_patching(dest->as_register(), info);
591 } else {
592 jobject2reg(c->as_jobject(), dest->as_register());
593 }
594 break;
595 }
596
597 case T_METADATA: {
598 if (patch_code != lir_patch_none) {
599 klass2reg_with_patching(dest->as_register(), info);
600 } else {
601 __ mov_metadata(dest->as_register(), c->as_metadata());
602 }
603 break;
604 }
605
606 case T_FLOAT: {
607 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
608 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
609 } else {
610 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
611 __ ldrs(dest->as_float_reg(), Address(rscratch1));
612 }
1045 case T_BYTE:
1046 __ ldrsb(dest->as_register(), as_Address(from_addr));
1047 break;
1048 case T_BOOLEAN: {
1049 __ ldrb(dest->as_register(), as_Address(from_addr));
1050 break;
1051 }
1052
1053 case T_CHAR:
1054 __ ldrh(dest->as_register(), as_Address(from_addr));
1055 break;
1056 case T_SHORT:
1057 __ ldrsh(dest->as_register(), as_Address(from_addr));
1058 break;
1059
1060 default:
1061 ShouldNotReachHere();
1062 }
1063 }
1064
1065 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1066 assert(dst->is_cpu_register(), "must be");
1067 assert(dst->type() == src->type(), "must be");
1068
1069 if (src->is_cpu_register()) {
1070 reg2reg(src, dst);
1071 } else if (src->is_stack()) {
1072 stack2reg(src, dst, dst->type());
1073 } else if (src->is_constant()) {
1074 const2reg(src, dst, lir_patch_none, nullptr);
1075 } else {
1076 ShouldNotReachHere();
1077 }
1078 }
1079
1080 void LIR_Assembler::load_volatile(LIR_Address *from_addr, LIR_Opr dest,
1081 BasicType type, CodeEmitInfo* info) {
1082 __ lea(rscratch1, as_Address(from_addr));
1083
1084 Register dest_reg = rscratch2;
1085 if (!is_floating_point_type(type)) {
1086 dest_reg = (dest->is_single_cpu()
1087 ? dest->as_register() : dest->as_register_lo());
1088 }
1089
1090 if (info != nullptr) {
1091 add_debug_info_for_null_check_here(info);
1092 }
1093
1094 // Uses LDAR to ensure memory ordering.
1095 __ load_store_volatile(dest_reg, type, rscratch1, /*is_load*/true);
1096
1097 switch (type) {
1098 // LDAR is unsigned so need to sign-extend for byte and short
1099 case T_BYTE:
1287 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1288 __ ldarb(rscratch1, rscratch1);
1289 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1290 add_debug_info_for_null_check_here(op->stub()->info());
1291 __ br(Assembler::NE, *op->stub()->entry());
1292 }
1293 __ allocate_object(op->obj()->as_register(),
1294 op->tmp1()->as_register(),
1295 op->tmp2()->as_register(),
1296 op->header_size(),
1297 op->object_size(),
1298 op->klass()->as_register(),
1299 *op->stub()->entry());
1300 __ bind(*op->stub()->continuation());
1301 }
1302
1303 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1304 Register len = op->len()->as_register();
1305 __ uxtw(len, len);
1306
1307 if (UseSlowPath || op->always_slow_path() ||
1308 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1309 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1310 __ b(*op->stub()->entry());
1311 } else {
1312 Register tmp1 = op->tmp1()->as_register();
1313 Register tmp2 = op->tmp2()->as_register();
1314 Register tmp3 = op->tmp3()->as_register();
1315 if (len == tmp1) {
1316 tmp1 = tmp3;
1317 } else if (len == tmp2) {
1318 tmp2 = tmp3;
1319 } else if (len == tmp3) {
1320 // everything is ok
1321 } else {
1322 __ mov(tmp3, len);
1323 }
1324 __ allocate_array(op->obj()->as_register(),
1325 len,
1326 tmp1,
1327 tmp2,
1364 md = method->method_data_or_null();
1365 assert(md != nullptr, "Sanity");
1366 data = md->bci_to_data(bci);
1367 assert(data != nullptr, "need data for type check");
1368 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1369 }
1370 Label* success_target = success;
1371 Label* failure_target = failure;
1372
1373 if (obj == k_RInfo) {
1374 k_RInfo = dst;
1375 } else if (obj == klass_RInfo) {
1376 klass_RInfo = dst;
1377 }
1378
1379 Rtmp1 = op->tmp3()->as_register();
1380 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1381
1382 assert_different_registers(obj, k_RInfo, klass_RInfo);
1383
1384 if (op->need_null_check()) {
1385 if (should_profile) {
1386 Register mdo = klass_RInfo;
1387 __ mov_metadata(mdo, md->constant_encoding());
1388 Label not_null;
1389 __ cbnz(obj, not_null);
1390 // Object is null; update MDO and exit
1391 Address data_addr
1392 = __ form_address(rscratch2, mdo,
1393 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1394 0);
1395 __ ldrb(rscratch1, data_addr);
1396 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1397 __ strb(rscratch1, data_addr);
1398 __ b(*obj_is_null);
1399 __ bind(not_null);
1400
1401 Register recv = k_RInfo;
1402 __ load_klass(recv, obj);
1403 type_profile_helper(mdo, md, data, recv);
1404 } else {
1405 __ cbz(obj, *obj_is_null);
1406 }
1407 }
1408
1409 if (!k->is_loaded()) {
1410 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1411 } else {
1412 __ mov_metadata(k_RInfo, k->constant_encoding());
1413 }
1414 __ verify_oop(obj);
1415
1416 if (op->fast_check()) {
1417 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1418 // get object class
1419 // not a safepoint as obj null check happens earlier
1420 __ load_klass(rscratch1, obj);
1421 __ cmp( rscratch1, k_RInfo);
1422
1423 __ br(Assembler::NE, *failure_target);
1424 // successful cast, fall through to profile or jump
1425 } else {
1426 // get object class
1427 // not a safepoint as obj null check happens earlier
1428 __ load_klass(klass_RInfo, obj);
1429 if (k->is_loaded()) {
1430 // See if we get an immediate positive hit
1431 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1432 __ cmp(k_RInfo, rscratch1);
1433 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1434 __ br(Assembler::NE, *failure_target);
1435 // successful cast, fall through to profile or jump
1436 } else {
1437 // See if we get an immediate positive hit
1438 __ br(Assembler::EQ, *success_target);
1439 // check for self
1440 if (k->is_loaded() && k->is_obj_array_klass()) {
1441 // For a direct pointer comparison, we need the refined array klass pointer
1442 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1443 __ mov_metadata(rscratch1, k_refined->constant_encoding());
1444 __ cmp(klass_RInfo, rscratch1);
1445 } else {
1446 __ cmp(klass_RInfo, k_RInfo);
1447 }
1448 __ br(Assembler::EQ, *success_target);
1449
1450 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1451 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1452 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1453 // result is a boolean
1454 __ cbzw(klass_RInfo, *failure_target);
1455 // successful cast, fall through to profile or jump
1456 }
1457 } else {
1458 // perform the fast part of the checking logic
1459 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1460 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1461 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1462 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1463 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1464 // result is a boolean
1465 __ cbz(k_RInfo, *failure_target);
1466 // successful cast, fall through to profile or jump
1467 }
1548 __ bind(success);
1549 if (dst != obj) {
1550 __ mov(dst, obj);
1551 }
1552 } else if (code == lir_instanceof) {
1553 Register obj = op->object()->as_register();
1554 Register dst = op->result_opr()->as_register();
1555 Label success, failure, done;
1556 emit_typecheck_helper(op, &success, &failure, &failure);
1557 __ bind(failure);
1558 __ mov(dst, zr);
1559 __ b(done);
1560 __ bind(success);
1561 __ mov(dst, 1);
1562 __ bind(done);
1563 } else {
1564 ShouldNotReachHere();
1565 }
1566 }
1567
1568 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1569 // We are loading/storing from/to an array that *may* be a flat array (the
1570 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1571 // If this array is a flat array, take the slow path.
1572 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1573 }
1574
1575 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1576 // We are storing into an array that *may* be null-free (the declared type is
1577 // Object[], abstract[], interface[] or VT.ref[]).
1578 Label test_mark_word;
1579 Register tmp = op->tmp()->as_register();
1580 __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1581 __ tst(tmp, markWord::unlocked_value);
1582 __ br(Assembler::NE, test_mark_word);
1583 __ load_prototype_header(tmp, op->array()->as_register());
1584 __ bind(test_mark_word);
1585 __ tst(tmp, markWord::null_free_array_bit_in_place);
1586 }
1587
1588 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1589 Label L_oops_equal;
1590 Label L_oops_not_equal;
1591 Label L_end;
1592
1593 Register left = op->left()->as_register();
1594 Register right = op->right()->as_register();
1595
1596 __ cmp(left, right);
1597 __ br(Assembler::EQ, L_oops_equal);
1598
1599 // (1) Null check -- if one of the operands is null, the other must not be null (because
1600 // the two references are not equal), so they are not substitutable,
1601 __ cbz(left, L_oops_not_equal);
1602 __ cbz(right, L_oops_not_equal);
1603
1604 ciKlass* left_klass = op->left_klass();
1605 ciKlass* right_klass = op->right_klass();
1606
1607 // (2) Inline type check -- if either of the operands is not an inline type,
1608 // they are not substitutable. We do this only if we are not sure that the
1609 // operands are inline type
1610 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1611 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1612 Register tmp1 = op->tmp1()->as_register();
1613 Register tmp2 = op->tmp2()->as_register();
1614 __ mov(tmp1, markWord::inline_type_pattern);
1615 __ ldr(tmp2, Address(left, oopDesc::mark_offset_in_bytes()));
1616 __ andr(tmp1, tmp1, tmp2);
1617 __ ldr(tmp2, Address(right, oopDesc::mark_offset_in_bytes()));
1618 __ andr(tmp1, tmp1, tmp2);
1619 __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1620 __ br(Assembler::NE, L_oops_not_equal);
1621 }
1622
1623 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1624 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1625 // No need to load klass -- the operands are statically known to be the same inline klass.
1626 __ b(*op->stub()->entry());
1627 } else {
1628 Register tmp1 = op->tmp1()->as_register();
1629 Register tmp2 = op->tmp2()->as_register();
1630 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1631 __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1632 // fall through to L_oops_not_equal
1633 }
1634
1635 __ bind(L_oops_not_equal);
1636 move(op->not_equal_result(), op->result_opr());
1637 __ b(L_end);
1638
1639 __ bind(L_oops_equal);
1640 move(op->equal_result(), op->result_opr());
1641 __ b(L_end);
1642
1643 // We've returned from the stub. R0 contains 0x0 IFF the two
1644 // operands are not substitutable. (Don't compare against 0x1 in case the
1645 // C compiler is naughty)
1646 __ bind(*op->stub()->continuation());
1647 __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1648 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1649 // fall-through
1650 __ bind(L_end);
1651 }
1652
1653
1654 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1655 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1656 __ cset(rscratch1, Assembler::NE);
1657 }
1658
1659 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1660 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1661 __ cset(rscratch1, Assembler::NE);
1662 }
1663
1664
1665 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1666 Register addr;
1667 if (op->addr()->is_register()) {
1668 addr = as_reg(op->addr());
1669 } else {
1670 assert(op->addr()->is_address(), "what else?");
1671 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1672 assert(addr_ptr->disp() == 0, "need 0 disp");
1673 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
2145 __ cmp(left->as_register_lo(), right->as_register_lo());
2146 __ mov(dst->as_register(), (uint64_t)-1L);
2147 __ br(Assembler::LT, done);
2148 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2149 __ bind(done);
2150 } else {
2151 ShouldNotReachHere();
2152 }
2153 }
2154
2155
2156 void LIR_Assembler::align_call(LIR_Code code) { }
2157
2158
2159 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2160 address call = __ trampoline_call(Address(op->addr(), rtype));
2161 if (call == nullptr) {
2162 bailout("trampoline stub overflow");
2163 return;
2164 }
2165 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2166 __ post_call_nop();
2167 }
2168
2169
2170 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2171 address call = __ ic_call(op->addr());
2172 if (call == nullptr) {
2173 bailout("trampoline stub overflow");
2174 return;
2175 }
2176 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2177 __ post_call_nop();
2178 }
2179
2180 void LIR_Assembler::emit_static_call_stub() {
2181 address call_pc = __ pc();
2182 address stub = __ start_a_stub(call_stub_size());
2183 if (stub == nullptr) {
2184 bailout("static call stub overflow");
2185 return;
2186 }
2187
2188 int start = __ offset();
2189
2190 __ relocate(static_stub_Relocation::spec(call_pc));
2191 __ emit_static_call_stub();
2192
2193 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2194 <= call_stub_size(), "stub too big");
2195 __ end_a_stub();
2196 }
2319
2320
2321 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2322 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2323 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2324 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2325 __ mov (rscratch1, c);
2326 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2327 }
2328
2329
2330 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2331 ShouldNotReachHere();
2332 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2333 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2334 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2335 __ lea(rscratch1, __ constant_oop_address(o));
2336 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2337 }
2338
2339 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2340 if (null_check) {
2341 __ cbz(obj, *slow_path->entry());
2342 }
2343 if (is_dest) {
2344 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2345 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2346 } else {
2347 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2348 }
2349 }
2350
2351 // This code replaces a call to arraycopy; no exception may
2352 // be thrown in this code, they must be thrown in the System.arraycopy
2353 // activation frame; we could save some checks if this would not be the case
2354 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2355 ciArrayKlass* default_type = op->expected_type();
2356 Register src = op->src()->as_register();
2357 Register dst = op->dst()->as_register();
2358 Register src_pos = op->src_pos()->as_register();
2359 Register dst_pos = op->dst_pos()->as_register();
2360 Register length = op->length()->as_register();
2361 Register tmp = op->tmp()->as_register();
2362
2363 CodeStub* stub = op->stub();
2364 int flags = op->flags();
2365 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2366 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2367
2368 if (flags & LIR_OpArrayCopy::always_slow_path) {
2369 __ b(*stub->entry());
2370 __ bind(*stub->continuation());
2371 return;
2372 }
2373
2374 // if we don't know anything, just go through the generic arraycopy
2375 if (default_type == nullptr // || basic_type == T_OBJECT
2376 ) {
2377 Label done;
2378 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2379
2380 // Save the arguments in case the generic arraycopy fails and we
2381 // have to fall back to the JNI stub
2382 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2383 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2384 __ str(src, Address(sp, 4*BytesPerWord));
2385
2386 address copyfunc_addr = StubRoutines::generic_arraycopy();
2387 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2388
2389 // The arguments are in java calling convention so we shift them
2390 // to C convention
2391 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2392 __ mov(c_rarg0, j_rarg0);
2393 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2407 __ cbz(r0, *stub->continuation());
2408
2409 // Reload values from the stack so they are where the stub
2410 // expects them.
2411 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2412 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2413 __ ldr(src, Address(sp, 4*BytesPerWord));
2414
2415 // r0 is -1^K where K == partial copied count
2416 __ eonw(rscratch1, r0, zr);
2417 // adjust length down and src/end pos up by partial copied count
2418 __ subw(length, length, rscratch1);
2419 __ addw(src_pos, src_pos, rscratch1);
2420 __ addw(dst_pos, dst_pos, rscratch1);
2421 __ b(*stub->entry());
2422
2423 __ bind(*stub->continuation());
2424 return;
2425 }
2426
2427 // Handle inline type arrays
2428 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2429 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2430 }
2431 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2432 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2433 }
2434
2435 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2436
2437 int elem_size = type2aelembytes(basic_type);
2438 int scale = exact_log2(elem_size);
2439
2440 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2441 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2442
2443 // test for null
2444 if (flags & LIR_OpArrayCopy::src_null_check) {
2445 __ cbz(src, *stub->entry());
2446 }
2447 if (flags & LIR_OpArrayCopy::dst_null_check) {
2448 __ cbz(dst, *stub->entry());
2449 }
2450
2451 // If the compiler was not able to prove that exact type of the source or the destination
2452 // of the arraycopy is an array type, check at runtime if the source or the destination is
2453 // an instance type.
2454 if (flags & LIR_OpArrayCopy::type_check) {
2929 __ verify_klass_ptr(tmp);
2930 #endif
2931 } else {
2932 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2933 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2934
2935 __ ldr(tmp, mdo_addr);
2936 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2937
2938 __ orr(tmp, tmp, TypeEntries::type_unknown);
2939 __ str(tmp, mdo_addr);
2940 // FIXME: Write barrier needed here?
2941 }
2942 }
2943
2944 __ bind(next);
2945 }
2946 COMMENT("} emit_profile_type");
2947 }
2948
2949 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
2950 Register obj = op->obj()->as_register();
2951 Register tmp = op->tmp()->as_pointer_register();
2952 bool not_null = op->not_null();
2953 int flag = op->flag();
2954
2955 Label not_inline_type;
2956 if (!not_null) {
2957 __ cbz(obj, not_inline_type);
2958 }
2959
2960 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
2961
2962 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), rscratch2);
2963 __ ldrb(rscratch1, mdo_addr);
2964 __ orr(rscratch1, rscratch1, flag);
2965 __ strb(rscratch1, mdo_addr);
2966
2967 __ bind(not_inline_type);
2968 }
2969
2970 void LIR_Assembler::align_backward_branch_target() {
2971 }
2972
2973
2974 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2975 // tmp must be unused
2976 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2977
2978 if (left->is_single_cpu()) {
2979 assert(dest->is_single_cpu(), "expect single result reg");
2980 __ negw(dest->as_register(), left->as_register());
2981 } else if (left->is_double_cpu()) {
2982 assert(dest->is_double_cpu(), "expect double result reg");
2983 __ neg(dest->as_register_lo(), left->as_register_lo());
2984 } else if (left->is_single_fpu()) {
2985 assert(dest->is_single_fpu(), "expect single float result reg");
2986 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2987 } else {
2988 assert(left->is_double_fpu(), "expect double float operand reg");
3090 void LIR_Assembler::membar_loadload() {
3091 __ membar(Assembler::LoadLoad);
3092 }
3093
3094 void LIR_Assembler::membar_storestore() {
3095 __ membar(MacroAssembler::StoreStore);
3096 }
3097
3098 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3099
3100 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3101
3102 void LIR_Assembler::on_spin_wait() {
3103 __ spin_wait();
3104 }
3105
3106 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3107 __ mov(result_reg->as_register(), rthread);
3108 }
3109
3110 void LIR_Assembler::check_orig_pc() {
3111 __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3112 __ cmp(rscratch2, (u1)NULL_WORD);
3113 }
3114
3115 void LIR_Assembler::peephole(LIR_List *lir) {
3116 #if 0
3117 if (tableswitch_count >= max_tableswitches)
3118 return;
3119
3120 /*
3121 This finite-state automaton recognizes sequences of compare-and-
3122 branch instructions. We will turn them into a tableswitch. You
3123 could argue that C1 really shouldn't be doing this sort of
3124 optimization, but without it the code is really horrible.
3125 */
3126
3127 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3128 int first_key, last_key = -2147483648;
3129 int next_key = 0;
3130 int start_insn = -1;
3131 int last_insn = -1;
3132 Register reg = noreg;
3133 LIR_Opr reg_opr;
|