16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/threadIdentifier.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_aarch64.inline.hpp"
49
50
51 #ifndef PRODUCT
52 #define COMMENT(x) do { __ block_comment(x); } while (0)
53 #else
54 #define COMMENT(x)
55 #endif
56
57 NEEDS_CLEANUP // remove this definitions ?
58 const Register SYNC_header = r0; // synchronization header
59 const Register SHIFT_count = r0; // where count for shift operations must be
60
61 #define __ _masm->
62
396 MonitorExitStub* stub = nullptr;
397 if (method()->is_synchronized()) {
398 monitor_address(0, FrameMap::r0_opr);
399 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
400 __ unlock_object(r5, r4, r0, r6, *stub->entry());
401 __ bind(*stub->continuation());
402 }
403
404 if (compilation()->env()->dtrace_method_probes()) {
405 __ mov(c_rarg0, rthread);
406 __ mov_metadata(c_rarg1, method()->constant_encoding());
407 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
408 }
409
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(r0, r19); // Restore the exception
412 }
413
414 // remove the activation and dispatch to the unwind handler
415 __ block_comment("remove_frame and dispatch to the unwind handler");
416 __ remove_frame(initial_frame_size_in_bytes());
417 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
418
419 // Emit the slow path assembly
420 if (stub != nullptr) {
421 stub->emit_code(this);
422 }
423
424 return offset;
425 }
426
427
428 int LIR_Assembler::emit_deopt_handler() {
429 // generate code for exception handler
430 address handler_base = __ start_a_stub(deopt_handler_size());
431 if (handler_base == nullptr) {
432 // not enough space left for the handler
433 bailout("deopt handler overflow");
434 return -1;
435 }
436
448 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
449 "out of bounds read in post-call NOP check");
450 __ end_a_stub();
451
452 return entry_offset;
453 }
454
455 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
456 _masm->code_section()->relocate(adr, relocInfo::poll_type);
457 int pc_offset = code_offset();
458 flush_debug_info(pc_offset);
459 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
460 if (info->exception_handlers() != nullptr) {
461 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
462 }
463 }
464
465 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
466 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
467
468 // Pop the stack before the safepoint code
469 __ remove_frame(initial_frame_size_in_bytes());
470
471 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
472 __ reserved_stack_check();
473 }
474
475 code_stub->set_safepoint_offset(__ offset());
476 __ relocate(relocInfo::poll_return_type);
477 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
478 __ ret(lr);
479 }
480
481 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
482 guarantee(info != nullptr, "Shouldn't be null");
483 __ get_polling_page(rscratch1, relocInfo::poll_type);
484 add_debug_info_for_branch(info); // This isn't just debug info:
485 // it's the oop map
486 __ read_polling_page(rscratch1, relocInfo::poll_type);
487 return __ offset();
488 }
489
490
491 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
492 if (from_reg == r31_sp)
493 from_reg = sp;
494 if (to_reg == r31_sp)
495 to_reg = sp;
496 __ mov(to_reg, from_reg);
497 }
498
499 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
500
520 case T_LONG: {
521 assert(patch_code == lir_patch_none, "no patching handled here");
522 #if INCLUDE_CDS
523 if (AOTCodeCache::is_on_for_dump()) {
524 address b = c->as_pointer();
525 if (b == (address)ThreadIdentifier::unsafe_offset()) {
526 __ lea(dest->as_register_lo(), ExternalAddress(b));
527 break;
528 }
529 if (AOTRuntimeConstants::contains(b)) {
530 __ load_aotrc_address(dest->as_register_lo(), b);
531 break;
532 }
533 }
534 #endif
535 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
536 break;
537 }
538
539 case T_OBJECT: {
540 if (patch_code == lir_patch_none) {
541 jobject2reg(c->as_jobject(), dest->as_register());
542 } else {
543 jobject2reg_with_patching(dest->as_register(), info);
544 }
545 break;
546 }
547
548 case T_METADATA: {
549 if (patch_code != lir_patch_none) {
550 klass2reg_with_patching(dest->as_register(), info);
551 } else {
552 __ mov_metadata(dest->as_register(), c->as_metadata());
553 }
554 break;
555 }
556
557 case T_FLOAT: {
558 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
559 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
560 } else {
561 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
562 __ ldrs(dest->as_float_reg(), Address(rscratch1));
563 }
996 case T_BYTE:
997 __ ldrsb(dest->as_register(), as_Address(from_addr));
998 break;
999 case T_BOOLEAN: {
1000 __ ldrb(dest->as_register(), as_Address(from_addr));
1001 break;
1002 }
1003
1004 case T_CHAR:
1005 __ ldrh(dest->as_register(), as_Address(from_addr));
1006 break;
1007 case T_SHORT:
1008 __ ldrsh(dest->as_register(), as_Address(from_addr));
1009 break;
1010
1011 default:
1012 ShouldNotReachHere();
1013 }
1014 }
1015
1016 void LIR_Assembler::load_volatile(LIR_Address *from_addr, LIR_Opr dest,
1017 BasicType type, CodeEmitInfo* info) {
1018 __ lea(rscratch1, as_Address(from_addr));
1019
1020 Register dest_reg = rscratch2;
1021 if (!is_floating_point_type(type)) {
1022 dest_reg = (dest->is_single_cpu()
1023 ? dest->as_register() : dest->as_register_lo());
1024 }
1025
1026 if (info != nullptr) {
1027 add_debug_info_for_null_check_here(info);
1028 }
1029
1030 // Uses LDAR to ensure memory ordering.
1031 __ load_store_volatile(dest_reg, type, rscratch1, /*is_load*/true);
1032
1033 switch (type) {
1034 // LDAR is unsigned so need to sign-extend for byte and short
1035 case T_BYTE:
1223 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1224 __ ldarb(rscratch1, rscratch1);
1225 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1226 add_debug_info_for_null_check_here(op->stub()->info());
1227 __ br(Assembler::NE, *op->stub()->entry());
1228 }
1229 __ allocate_object(op->obj()->as_register(),
1230 op->tmp1()->as_register(),
1231 op->tmp2()->as_register(),
1232 op->header_size(),
1233 op->object_size(),
1234 op->klass()->as_register(),
1235 *op->stub()->entry());
1236 __ bind(*op->stub()->continuation());
1237 }
1238
1239 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1240 Register len = op->len()->as_register();
1241 __ uxtw(len, len);
1242
1243 if (UseSlowPath ||
1244 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1245 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1246 __ b(*op->stub()->entry());
1247 } else {
1248 Register tmp1 = op->tmp1()->as_register();
1249 Register tmp2 = op->tmp2()->as_register();
1250 Register tmp3 = op->tmp3()->as_register();
1251 if (len == tmp1) {
1252 tmp1 = tmp3;
1253 } else if (len == tmp2) {
1254 tmp2 = tmp3;
1255 } else if (len == tmp3) {
1256 // everything is ok
1257 } else {
1258 __ mov(tmp3, len);
1259 }
1260 __ allocate_array(op->obj()->as_register(),
1261 len,
1262 tmp1,
1263 tmp2,
1300 md = method->method_data_or_null();
1301 assert(md != nullptr, "Sanity");
1302 data = md->bci_to_data(bci);
1303 assert(data != nullptr, "need data for type check");
1304 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1305 }
1306 Label* success_target = success;
1307 Label* failure_target = failure;
1308
1309 if (obj == k_RInfo) {
1310 k_RInfo = dst;
1311 } else if (obj == klass_RInfo) {
1312 klass_RInfo = dst;
1313 }
1314
1315 Rtmp1 = op->tmp3()->as_register();
1316 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1317
1318 assert_different_registers(obj, k_RInfo, klass_RInfo);
1319
1320 if (should_profile) {
1321 Register mdo = klass_RInfo;
1322 __ mov_metadata(mdo, md->constant_encoding());
1323 Label not_null;
1324 __ cbnz(obj, not_null);
1325 // Object is null; update MDO and exit
1326 Address data_addr
1327 = __ form_address(rscratch2, mdo,
1328 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1329 0);
1330 __ ldrb(rscratch1, data_addr);
1331 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1332 __ strb(rscratch1, data_addr);
1333 __ b(*obj_is_null);
1334 __ bind(not_null);
1335
1336 Register recv = k_RInfo;
1337 __ load_klass(recv, obj);
1338 type_profile_helper(mdo, md, data, recv);
1339 } else {
1340 __ cbz(obj, *obj_is_null);
1341 }
1342
1343 if (!k->is_loaded()) {
1344 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1345 } else {
1346 __ mov_metadata(k_RInfo, k->constant_encoding());
1347 }
1348 __ verify_oop(obj);
1349
1350 if (op->fast_check()) {
1351 // get object class
1352 // not a safepoint as obj null check happens earlier
1353 __ load_klass(rscratch1, obj);
1354 __ cmp( rscratch1, k_RInfo);
1355
1356 __ br(Assembler::NE, *failure_target);
1357 // successful cast, fall through to profile or jump
1358 } else {
1359 // get object class
1360 // not a safepoint as obj null check happens earlier
1361 __ load_klass(klass_RInfo, obj);
1362 if (k->is_loaded()) {
1363 // See if we get an immediate positive hit
1364 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1365 __ cmp(k_RInfo, rscratch1);
1366 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1367 __ br(Assembler::NE, *failure_target);
1368 // successful cast, fall through to profile or jump
1369 } else {
1370 // See if we get an immediate positive hit
1371 __ br(Assembler::EQ, *success_target);
1372 // check for self
1373 __ cmp(klass_RInfo, k_RInfo);
1374 __ br(Assembler::EQ, *success_target);
1375
1376 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1377 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1378 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1379 // result is a boolean
1380 __ cbzw(klass_RInfo, *failure_target);
1381 // successful cast, fall through to profile or jump
1382 }
1383 } else {
1384 // perform the fast part of the checking logic
1385 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1386 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1387 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1388 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1389 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1390 // result is a boolean
1391 __ cbz(k_RInfo, *failure_target);
1392 // successful cast, fall through to profile or jump
1393 }
1474 __ bind(success);
1475 if (dst != obj) {
1476 __ mov(dst, obj);
1477 }
1478 } else if (code == lir_instanceof) {
1479 Register obj = op->object()->as_register();
1480 Register dst = op->result_opr()->as_register();
1481 Label success, failure, done;
1482 emit_typecheck_helper(op, &success, &failure, &failure);
1483 __ bind(failure);
1484 __ mov(dst, zr);
1485 __ b(done);
1486 __ bind(success);
1487 __ mov(dst, 1);
1488 __ bind(done);
1489 } else {
1490 ShouldNotReachHere();
1491 }
1492 }
1493
1494 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1495 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1496 __ cset(rscratch1, Assembler::NE);
1497 __ membar(__ AnyAny);
1498 }
1499
1500 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1501 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1502 __ cset(rscratch1, Assembler::NE);
1503 __ membar(__ AnyAny);
1504 }
1505
1506
1507 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1508 Register addr;
1509 if (op->addr()->is_register()) {
1510 addr = as_reg(op->addr());
1511 } else {
1512 assert(op->addr()->is_address(), "what else?");
1513 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1987 __ cmp(left->as_register_lo(), right->as_register_lo());
1988 __ mov(dst->as_register(), (uint64_t)-1L);
1989 __ br(Assembler::LT, done);
1990 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1991 __ bind(done);
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 }
1996
1997
1998 void LIR_Assembler::align_call(LIR_Code code) { }
1999
2000
2001 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2002 address call = __ trampoline_call(Address(op->addr(), rtype));
2003 if (call == nullptr) {
2004 bailout("trampoline stub overflow");
2005 return;
2006 }
2007 add_call_info(code_offset(), op->info());
2008 __ post_call_nop();
2009 }
2010
2011
2012 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2013 address call = __ ic_call(op->addr());
2014 if (call == nullptr) {
2015 bailout("trampoline stub overflow");
2016 return;
2017 }
2018 add_call_info(code_offset(), op->info());
2019 __ post_call_nop();
2020 }
2021
2022 void LIR_Assembler::emit_static_call_stub() {
2023 address call_pc = __ pc();
2024 address stub = __ start_a_stub(call_stub_size());
2025 if (stub == nullptr) {
2026 bailout("static call stub overflow");
2027 return;
2028 }
2029
2030 int start = __ offset();
2031
2032 __ relocate(static_stub_Relocation::spec(call_pc));
2033 __ emit_static_call_stub();
2034
2035 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2036 <= call_stub_size(), "stub too big");
2037 __ end_a_stub();
2038 }
2161
2162
2163 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2164 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2165 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2166 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2167 __ mov (rscratch1, c);
2168 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2169 }
2170
2171
2172 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2173 ShouldNotReachHere();
2174 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2175 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2176 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2177 __ lea(rscratch1, __ constant_oop_address(o));
2178 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2179 }
2180
2181
2182 // This code replaces a call to arraycopy; no exception may
2183 // be thrown in this code, they must be thrown in the System.arraycopy
2184 // activation frame; we could save some checks if this would not be the case
2185 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2186 ciArrayKlass* default_type = op->expected_type();
2187 Register src = op->src()->as_register();
2188 Register dst = op->dst()->as_register();
2189 Register src_pos = op->src_pos()->as_register();
2190 Register dst_pos = op->dst_pos()->as_register();
2191 Register length = op->length()->as_register();
2192 Register tmp = op->tmp()->as_register();
2193
2194 CodeStub* stub = op->stub();
2195 int flags = op->flags();
2196 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2197 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2198
2199 // if we don't know anything, just go through the generic arraycopy
2200 if (default_type == nullptr // || basic_type == T_OBJECT
2201 ) {
2202 Label done;
2203 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2204
2205 // Save the arguments in case the generic arraycopy fails and we
2206 // have to fall back to the JNI stub
2207 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2208 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2209 __ str(src, Address(sp, 4*BytesPerWord));
2210
2211 address copyfunc_addr = StubRoutines::generic_arraycopy();
2212 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2213
2214 // The arguments are in java calling convention so we shift them
2215 // to C convention
2216 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2217 __ mov(c_rarg0, j_rarg0);
2218 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2232 __ cbz(r0, *stub->continuation());
2233
2234 // Reload values from the stack so they are where the stub
2235 // expects them.
2236 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2237 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2238 __ ldr(src, Address(sp, 4*BytesPerWord));
2239
2240 // r0 is -1^K where K == partial copied count
2241 __ eonw(rscratch1, r0, zr);
2242 // adjust length down and src/end pos up by partial copied count
2243 __ subw(length, length, rscratch1);
2244 __ addw(src_pos, src_pos, rscratch1);
2245 __ addw(dst_pos, dst_pos, rscratch1);
2246 __ b(*stub->entry());
2247
2248 __ bind(*stub->continuation());
2249 return;
2250 }
2251
2252 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2253
2254 int elem_size = type2aelembytes(basic_type);
2255 int scale = exact_log2(elem_size);
2256
2257 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2258 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2259
2260 // test for null
2261 if (flags & LIR_OpArrayCopy::src_null_check) {
2262 __ cbz(src, *stub->entry());
2263 }
2264 if (flags & LIR_OpArrayCopy::dst_null_check) {
2265 __ cbz(dst, *stub->entry());
2266 }
2267
2268 // If the compiler was not able to prove that exact type of the source or the destination
2269 // of the arraycopy is an array type, check at runtime if the source or the destination is
2270 // an instance type.
2271 if (flags & LIR_OpArrayCopy::type_check) {
2746 __ verify_klass_ptr(tmp);
2747 #endif
2748 } else {
2749 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2750 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2751
2752 __ ldr(tmp, mdo_addr);
2753 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2754
2755 __ orr(tmp, tmp, TypeEntries::type_unknown);
2756 __ str(tmp, mdo_addr);
2757 // FIXME: Write barrier needed here?
2758 }
2759 }
2760
2761 __ bind(next);
2762 }
2763 COMMENT("} emit_profile_type");
2764 }
2765
2766
2767 void LIR_Assembler::align_backward_branch_target() {
2768 }
2769
2770
2771 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2772 // tmp must be unused
2773 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2774
2775 if (left->is_single_cpu()) {
2776 assert(dest->is_single_cpu(), "expect single result reg");
2777 __ negw(dest->as_register(), left->as_register());
2778 } else if (left->is_double_cpu()) {
2779 assert(dest->is_double_cpu(), "expect double result reg");
2780 __ neg(dest->as_register_lo(), left->as_register_lo());
2781 } else if (left->is_single_fpu()) {
2782 assert(dest->is_single_fpu(), "expect single float result reg");
2783 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2784 } else {
2785 assert(left->is_double_fpu(), "expect double float operand reg");
2887 void LIR_Assembler::membar_loadload() {
2888 __ membar(Assembler::LoadLoad);
2889 }
2890
2891 void LIR_Assembler::membar_storestore() {
2892 __ membar(MacroAssembler::StoreStore);
2893 }
2894
2895 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2896
2897 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2898
2899 void LIR_Assembler::on_spin_wait() {
2900 __ spin_wait();
2901 }
2902
2903 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2904 __ mov(result_reg->as_register(), rthread);
2905 }
2906
2907
2908 void LIR_Assembler::peephole(LIR_List *lir) {
2909 #if 0
2910 if (tableswitch_count >= max_tableswitches)
2911 return;
2912
2913 /*
2914 This finite-state automaton recognizes sequences of compare-and-
2915 branch instructions. We will turn them into a tableswitch. You
2916 could argue that C1 really shouldn't be doing this sort of
2917 optimization, but without it the code is really horrible.
2918 */
2919
2920 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2921 int first_key, last_key = -2147483648;
2922 int next_key = 0;
2923 int start_insn = -1;
2924 int last_insn = -1;
2925 Register reg = noreg;
2926 LIR_Opr reg_opr;
|
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInlineKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "ci/ciObjArrayKlass.hpp"
39 #include "code/aotCodeCache.hpp"
40 #include "code/compiledIC.hpp"
41 #include "gc/shared/collectedHeap.hpp"
42 #include "gc/shared/gc_globals.hpp"
43 #include "nativeInst_aarch64.hpp"
44 #include "oops/objArrayKlass.hpp"
45 #include "oops/oop.inline.hpp"
46 #include "runtime/frame.inline.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/threadIdentifier.hpp"
50 #include "utilities/powerOfTwo.hpp"
51 #include "vmreg_aarch64.inline.hpp"
52
53
54 #ifndef PRODUCT
55 #define COMMENT(x) do { __ block_comment(x); } while (0)
56 #else
57 #define COMMENT(x)
58 #endif
59
60 NEEDS_CLEANUP // remove this definitions ?
61 const Register SYNC_header = r0; // synchronization header
62 const Register SHIFT_count = r0; // where count for shift operations must be
63
64 #define __ _masm->
65
399 MonitorExitStub* stub = nullptr;
400 if (method()->is_synchronized()) {
401 monitor_address(0, FrameMap::r0_opr);
402 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
403 __ unlock_object(r5, r4, r0, r6, *stub->entry());
404 __ bind(*stub->continuation());
405 }
406
407 if (compilation()->env()->dtrace_method_probes()) {
408 __ mov(c_rarg0, rthread);
409 __ mov_metadata(c_rarg1, method()->constant_encoding());
410 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
411 }
412
413 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
414 __ mov(r0, r19); // Restore the exception
415 }
416
417 // remove the activation and dispatch to the unwind handler
418 __ block_comment("remove_frame and dispatch to the unwind handler");
419 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
420 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
421
422 // Emit the slow path assembly
423 if (stub != nullptr) {
424 stub->emit_code(this);
425 }
426
427 return offset;
428 }
429
430
431 int LIR_Assembler::emit_deopt_handler() {
432 // generate code for exception handler
433 address handler_base = __ start_a_stub(deopt_handler_size());
434 if (handler_base == nullptr) {
435 // not enough space left for the handler
436 bailout("deopt handler overflow");
437 return -1;
438 }
439
451 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
452 "out of bounds read in post-call NOP check");
453 __ end_a_stub();
454
455 return entry_offset;
456 }
457
458 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
459 _masm->code_section()->relocate(adr, relocInfo::poll_type);
460 int pc_offset = code_offset();
461 flush_debug_info(pc_offset);
462 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
463 if (info->exception_handlers() != nullptr) {
464 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
465 }
466 }
467
468 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
469 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
470
471 if (InlineTypeReturnedAsFields) {
472 // Check if we are returning an non-null inline type and load its fields into registers
473 ciType* return_type = compilation()->method()->return_type();
474 if (return_type->is_inlinetype()) {
475 ciInlineKlass* vk = return_type->as_inline_klass();
476 if (vk->can_be_returned_as_fields()) {
477 address unpack_handler = vk->unpack_handler();
478 assert(unpack_handler != nullptr, "must be");
479 __ far_call(RuntimeAddress(unpack_handler));
480 }
481 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
482 Label skip;
483 Label not_null;
484 __ cbnz(r0, not_null);
485 // Returned value is null, zero all return registers because they may belong to oop fields
486 __ mov(j_rarg1, zr);
487 __ mov(j_rarg2, zr);
488 __ mov(j_rarg3, zr);
489 __ mov(j_rarg4, zr);
490 __ mov(j_rarg5, zr);
491 __ mov(j_rarg6, zr);
492 __ mov(j_rarg7, zr);
493 __ b(skip);
494 __ bind(not_null);
495
496 // Check if we are returning an non-null inline type and load its fields into registers
497 __ test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
498
499 // Load fields from a buffered value with an inline class specific handler
500 __ load_klass(rscratch1 /*dst*/, r0 /*src*/);
501 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
502 __ ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
503 // Unpack handler can be null if inline type is not scalarizable in returns
504 __ cbz(rscratch1, skip);
505 __ blr(rscratch1);
506
507 __ bind(skip);
508 }
509 // At this point, r0 points to the value object (for interpreter or C1 caller).
510 // The fields of the object are copied into registers (for C2 caller).
511 }
512
513 // Pop the stack before the safepoint code
514 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
515
516 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
517 __ reserved_stack_check();
518 }
519
520 code_stub->set_safepoint_offset(__ offset());
521 __ relocate(relocInfo::poll_return_type);
522 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
523 __ ret(lr);
524 }
525
526 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
527 return (__ store_inline_type_fields_to_buf(vk, false));
528 }
529
530 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
531 guarantee(info != nullptr, "Shouldn't be null");
532 __ get_polling_page(rscratch1, relocInfo::poll_type);
533 add_debug_info_for_branch(info); // This isn't just debug info:
534 // it's the oop map
535 __ read_polling_page(rscratch1, relocInfo::poll_type);
536 return __ offset();
537 }
538
539
540 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
541 if (from_reg == r31_sp)
542 from_reg = sp;
543 if (to_reg == r31_sp)
544 to_reg = sp;
545 __ mov(to_reg, from_reg);
546 }
547
548 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
549
569 case T_LONG: {
570 assert(patch_code == lir_patch_none, "no patching handled here");
571 #if INCLUDE_CDS
572 if (AOTCodeCache::is_on_for_dump()) {
573 address b = c->as_pointer();
574 if (b == (address)ThreadIdentifier::unsafe_offset()) {
575 __ lea(dest->as_register_lo(), ExternalAddress(b));
576 break;
577 }
578 if (AOTRuntimeConstants::contains(b)) {
579 __ load_aotrc_address(dest->as_register_lo(), b);
580 break;
581 }
582 }
583 #endif
584 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
585 break;
586 }
587
588 case T_OBJECT: {
589 if (patch_code != lir_patch_none) {
590 jobject2reg_with_patching(dest->as_register(), info);
591 } else {
592 jobject2reg(c->as_jobject(), dest->as_register());
593 }
594 break;
595 }
596
597 case T_METADATA: {
598 if (patch_code != lir_patch_none) {
599 klass2reg_with_patching(dest->as_register(), info);
600 } else {
601 __ mov_metadata(dest->as_register(), c->as_metadata());
602 }
603 break;
604 }
605
606 case T_FLOAT: {
607 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
608 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
609 } else {
610 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
611 __ ldrs(dest->as_float_reg(), Address(rscratch1));
612 }
1045 case T_BYTE:
1046 __ ldrsb(dest->as_register(), as_Address(from_addr));
1047 break;
1048 case T_BOOLEAN: {
1049 __ ldrb(dest->as_register(), as_Address(from_addr));
1050 break;
1051 }
1052
1053 case T_CHAR:
1054 __ ldrh(dest->as_register(), as_Address(from_addr));
1055 break;
1056 case T_SHORT:
1057 __ ldrsh(dest->as_register(), as_Address(from_addr));
1058 break;
1059
1060 default:
1061 ShouldNotReachHere();
1062 }
1063 }
1064
1065 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1066 assert(dst->is_cpu_register(), "must be");
1067 assert(dst->type() == src->type(), "must be");
1068
1069 if (src->is_cpu_register()) {
1070 reg2reg(src, dst);
1071 } else if (src->is_stack()) {
1072 stack2reg(src, dst, dst->type());
1073 } else if (src->is_constant()) {
1074 const2reg(src, dst, lir_patch_none, nullptr);
1075 } else {
1076 ShouldNotReachHere();
1077 }
1078 }
1079
1080 void LIR_Assembler::load_volatile(LIR_Address *from_addr, LIR_Opr dest,
1081 BasicType type, CodeEmitInfo* info) {
1082 __ lea(rscratch1, as_Address(from_addr));
1083
1084 Register dest_reg = rscratch2;
1085 if (!is_floating_point_type(type)) {
1086 dest_reg = (dest->is_single_cpu()
1087 ? dest->as_register() : dest->as_register_lo());
1088 }
1089
1090 if (info != nullptr) {
1091 add_debug_info_for_null_check_here(info);
1092 }
1093
1094 // Uses LDAR to ensure memory ordering.
1095 __ load_store_volatile(dest_reg, type, rscratch1, /*is_load*/true);
1096
1097 switch (type) {
1098 // LDAR is unsigned so need to sign-extend for byte and short
1099 case T_BYTE:
1287 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1288 __ ldarb(rscratch1, rscratch1);
1289 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1290 add_debug_info_for_null_check_here(op->stub()->info());
1291 __ br(Assembler::NE, *op->stub()->entry());
1292 }
1293 __ allocate_object(op->obj()->as_register(),
1294 op->tmp1()->as_register(),
1295 op->tmp2()->as_register(),
1296 op->header_size(),
1297 op->object_size(),
1298 op->klass()->as_register(),
1299 *op->stub()->entry());
1300 __ bind(*op->stub()->continuation());
1301 }
1302
1303 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1304 Register len = op->len()->as_register();
1305 __ uxtw(len, len);
1306
1307 if (UseSlowPath || op->always_slow_path() ||
1308 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1309 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1310 __ b(*op->stub()->entry());
1311 } else {
1312 Register tmp1 = op->tmp1()->as_register();
1313 Register tmp2 = op->tmp2()->as_register();
1314 Register tmp3 = op->tmp3()->as_register();
1315 if (len == tmp1) {
1316 tmp1 = tmp3;
1317 } else if (len == tmp2) {
1318 tmp2 = tmp3;
1319 } else if (len == tmp3) {
1320 // everything is ok
1321 } else {
1322 __ mov(tmp3, len);
1323 }
1324 __ allocate_array(op->obj()->as_register(),
1325 len,
1326 tmp1,
1327 tmp2,
1364 md = method->method_data_or_null();
1365 assert(md != nullptr, "Sanity");
1366 data = md->bci_to_data(bci);
1367 assert(data != nullptr, "need data for type check");
1368 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1369 }
1370 Label* success_target = success;
1371 Label* failure_target = failure;
1372
1373 if (obj == k_RInfo) {
1374 k_RInfo = dst;
1375 } else if (obj == klass_RInfo) {
1376 klass_RInfo = dst;
1377 }
1378
1379 Rtmp1 = op->tmp3()->as_register();
1380 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1381
1382 assert_different_registers(obj, k_RInfo, klass_RInfo);
1383
1384 if (op->need_null_check()) {
1385 if (should_profile) {
1386 Register mdo = klass_RInfo;
1387 __ mov_metadata(mdo, md->constant_encoding());
1388 Label not_null;
1389 __ cbnz(obj, not_null);
1390 // Object is null; update MDO and exit
1391 Address data_addr
1392 = __ form_address(rscratch2, mdo,
1393 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1394 0);
1395 __ ldrb(rscratch1, data_addr);
1396 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1397 __ strb(rscratch1, data_addr);
1398 __ b(*obj_is_null);
1399 __ bind(not_null);
1400
1401 Register recv = k_RInfo;
1402 __ load_klass(recv, obj);
1403 type_profile_helper(mdo, md, data, recv);
1404 } else {
1405 __ cbz(obj, *obj_is_null);
1406 }
1407 }
1408
1409 if (!k->is_loaded()) {
1410 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1411 } else {
1412 __ mov_metadata(k_RInfo, k->constant_encoding());
1413 }
1414 __ verify_oop(obj);
1415
1416 if (op->fast_check()) {
1417 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1418 // get object class
1419 // not a safepoint as obj null check happens earlier
1420 __ load_klass(rscratch1, obj);
1421 __ cmp( rscratch1, k_RInfo);
1422
1423 __ br(Assembler::NE, *failure_target);
1424 // successful cast, fall through to profile or jump
1425 } else {
1426 // get object class
1427 // not a safepoint as obj null check happens earlier
1428 __ load_klass(klass_RInfo, obj);
1429 if (k->is_loaded()) {
1430 // See if we get an immediate positive hit
1431 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1432 __ cmp(k_RInfo, rscratch1);
1433 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1434 __ br(Assembler::NE, *failure_target);
1435 // successful cast, fall through to profile or jump
1436 } else {
1437 // See if we get an immediate positive hit
1438 __ br(Assembler::EQ, *success_target);
1439 // check for self
1440 if (k->is_loaded() && k->is_obj_array_klass()) {
1441 // For a direct pointer comparison, we need the refined array klass pointer
1442 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1443 __ mov_metadata(rscratch1, k_refined->constant_encoding());
1444 __ cmp(klass_RInfo, rscratch1);
1445 } else {
1446 __ cmp(klass_RInfo, k_RInfo);
1447 }
1448 __ br(Assembler::EQ, *success_target);
1449
1450 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1451 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1452 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1453 // result is a boolean
1454 __ cbzw(klass_RInfo, *failure_target);
1455 // successful cast, fall through to profile or jump
1456 }
1457 } else {
1458 // perform the fast part of the checking logic
1459 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1460 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1461 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1462 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1463 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1464 // result is a boolean
1465 __ cbz(k_RInfo, *failure_target);
1466 // successful cast, fall through to profile or jump
1467 }
1548 __ bind(success);
1549 if (dst != obj) {
1550 __ mov(dst, obj);
1551 }
1552 } else if (code == lir_instanceof) {
1553 Register obj = op->object()->as_register();
1554 Register dst = op->result_opr()->as_register();
1555 Label success, failure, done;
1556 emit_typecheck_helper(op, &success, &failure, &failure);
1557 __ bind(failure);
1558 __ mov(dst, zr);
1559 __ b(done);
1560 __ bind(success);
1561 __ mov(dst, 1);
1562 __ bind(done);
1563 } else {
1564 ShouldNotReachHere();
1565 }
1566 }
1567
1568 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1569 // We are loading/storing from/to an array that *may* be a flat array (the
1570 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1571 // If this array is a flat array, take the slow path.
1572 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1573 }
1574
1575 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1576 // We are storing into an array that *may* be null-free (the declared type is
1577 // Object[], abstract[], interface[] or VT.ref[]).
1578 Label test_mark_word;
1579 Register tmp = op->tmp()->as_register();
1580 __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1581 __ tst(tmp, markWord::unlocked_value);
1582 __ br(Assembler::NE, test_mark_word);
1583 __ load_prototype_header(tmp, op->array()->as_register());
1584 __ bind(test_mark_word);
1585 __ tst(tmp, markWord::null_free_array_bit_in_place);
1586 }
1587
1588 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1589 Label L_oops_equal;
1590 Label L_oops_not_equal;
1591 Label L_end;
1592
1593 Register left = op->left()->as_register();
1594 Register right = op->right()->as_register();
1595
1596 __ cmp(left, right);
1597 __ br(Assembler::EQ, L_oops_equal);
1598
1599 // (1) Null check -- if one of the operands is null, the other must not be null (because
1600 // the two references are not equal), so they are not substitutable,
1601 __ cbz(left, L_oops_not_equal);
1602 __ cbz(right, L_oops_not_equal);
1603
1604 ciKlass* left_klass = op->left_klass();
1605 ciKlass* right_klass = op->right_klass();
1606
1607 // (2) Inline type check -- if either of the operands is not a inline type,
1608 // they are not substitutable. We do this only if we are not sure that the
1609 // operands are inline type
1610 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1611 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1612 Register tmp1 = op->tmp1()->as_register();
1613 Register tmp2 = op->tmp2()->as_register();
1614 __ mov(tmp1, markWord::inline_type_pattern);
1615 __ ldr(tmp2, Address(left, oopDesc::mark_offset_in_bytes()));
1616 __ andr(tmp1, tmp1, tmp2);
1617 __ ldr(tmp2, Address(right, oopDesc::mark_offset_in_bytes()));
1618 __ andr(tmp1, tmp1, tmp2);
1619 __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1620 __ br(Assembler::NE, L_oops_not_equal);
1621 }
1622
1623 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1624 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1625 // No need to load klass -- the operands are statically known to be the same inline klass.
1626 __ b(*op->stub()->entry());
1627 } else {
1628 Register tmp1 = op->tmp1()->as_register();
1629 Register tmp2 = op->tmp2()->as_register();
1630 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1631 __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1632 // fall through to L_oops_not_equal
1633 }
1634
1635 __ bind(L_oops_not_equal);
1636 move(op->not_equal_result(), op->result_opr());
1637 __ b(L_end);
1638
1639 __ bind(L_oops_equal);
1640 move(op->equal_result(), op->result_opr());
1641 __ b(L_end);
1642
1643 // We've returned from the stub. R0 contains 0x0 IFF the two
1644 // operands are not substitutable. (Don't compare against 0x1 in case the
1645 // C compiler is naughty)
1646 __ bind(*op->stub()->continuation());
1647 __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1648 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1649 // fall-through
1650 __ bind(L_end);
1651 }
1652
1653
1654 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1655 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1656 __ cset(rscratch1, Assembler::NE);
1657 __ membar(__ AnyAny);
1658 }
1659
1660 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1661 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1662 __ cset(rscratch1, Assembler::NE);
1663 __ membar(__ AnyAny);
1664 }
1665
1666
1667 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1668 Register addr;
1669 if (op->addr()->is_register()) {
1670 addr = as_reg(op->addr());
1671 } else {
1672 assert(op->addr()->is_address(), "what else?");
1673 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
2147 __ cmp(left->as_register_lo(), right->as_register_lo());
2148 __ mov(dst->as_register(), (uint64_t)-1L);
2149 __ br(Assembler::LT, done);
2150 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2151 __ bind(done);
2152 } else {
2153 ShouldNotReachHere();
2154 }
2155 }
2156
2157
2158 void LIR_Assembler::align_call(LIR_Code code) { }
2159
2160
2161 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2162 address call = __ trampoline_call(Address(op->addr(), rtype));
2163 if (call == nullptr) {
2164 bailout("trampoline stub overflow");
2165 return;
2166 }
2167 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2168 __ post_call_nop();
2169 }
2170
2171
2172 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2173 address call = __ ic_call(op->addr());
2174 if (call == nullptr) {
2175 bailout("trampoline stub overflow");
2176 return;
2177 }
2178 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2179 __ post_call_nop();
2180 }
2181
2182 void LIR_Assembler::emit_static_call_stub() {
2183 address call_pc = __ pc();
2184 address stub = __ start_a_stub(call_stub_size());
2185 if (stub == nullptr) {
2186 bailout("static call stub overflow");
2187 return;
2188 }
2189
2190 int start = __ offset();
2191
2192 __ relocate(static_stub_Relocation::spec(call_pc));
2193 __ emit_static_call_stub();
2194
2195 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2196 <= call_stub_size(), "stub too big");
2197 __ end_a_stub();
2198 }
2321
2322
2323 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2324 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2325 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2326 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2327 __ mov (rscratch1, c);
2328 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2329 }
2330
2331
2332 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2333 ShouldNotReachHere();
2334 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2335 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2336 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2337 __ lea(rscratch1, __ constant_oop_address(o));
2338 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2339 }
2340
2341 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2342 if (null_check) {
2343 __ cbz(obj, *slow_path->entry());
2344 }
2345 if (is_dest) {
2346 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2347 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2348 } else {
2349 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2350 }
2351 }
2352
2353 // This code replaces a call to arraycopy; no exception may
2354 // be thrown in this code, they must be thrown in the System.arraycopy
2355 // activation frame; we could save some checks if this would not be the case
2356 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2357 ciArrayKlass* default_type = op->expected_type();
2358 Register src = op->src()->as_register();
2359 Register dst = op->dst()->as_register();
2360 Register src_pos = op->src_pos()->as_register();
2361 Register dst_pos = op->dst_pos()->as_register();
2362 Register length = op->length()->as_register();
2363 Register tmp = op->tmp()->as_register();
2364
2365 CodeStub* stub = op->stub();
2366 int flags = op->flags();
2367 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2368 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2369
2370 if (flags & LIR_OpArrayCopy::always_slow_path) {
2371 __ b(*stub->entry());
2372 __ bind(*stub->continuation());
2373 return;
2374 }
2375
2376 // if we don't know anything, just go through the generic arraycopy
2377 if (default_type == nullptr // || basic_type == T_OBJECT
2378 ) {
2379 Label done;
2380 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2381
2382 // Save the arguments in case the generic arraycopy fails and we
2383 // have to fall back to the JNI stub
2384 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2385 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2386 __ str(src, Address(sp, 4*BytesPerWord));
2387
2388 address copyfunc_addr = StubRoutines::generic_arraycopy();
2389 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2390
2391 // The arguments are in java calling convention so we shift them
2392 // to C convention
2393 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2394 __ mov(c_rarg0, j_rarg0);
2395 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2409 __ cbz(r0, *stub->continuation());
2410
2411 // Reload values from the stack so they are where the stub
2412 // expects them.
2413 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2414 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2415 __ ldr(src, Address(sp, 4*BytesPerWord));
2416
2417 // r0 is -1^K where K == partial copied count
2418 __ eonw(rscratch1, r0, zr);
2419 // adjust length down and src/end pos up by partial copied count
2420 __ subw(length, length, rscratch1);
2421 __ addw(src_pos, src_pos, rscratch1);
2422 __ addw(dst_pos, dst_pos, rscratch1);
2423 __ b(*stub->entry());
2424
2425 __ bind(*stub->continuation());
2426 return;
2427 }
2428
2429 // Handle inline type arrays
2430 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2431 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2432 }
2433 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2434 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2435 }
2436
2437 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2438
2439 int elem_size = type2aelembytes(basic_type);
2440 int scale = exact_log2(elem_size);
2441
2442 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2443 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2444
2445 // test for null
2446 if (flags & LIR_OpArrayCopy::src_null_check) {
2447 __ cbz(src, *stub->entry());
2448 }
2449 if (flags & LIR_OpArrayCopy::dst_null_check) {
2450 __ cbz(dst, *stub->entry());
2451 }
2452
2453 // If the compiler was not able to prove that exact type of the source or the destination
2454 // of the arraycopy is an array type, check at runtime if the source or the destination is
2455 // an instance type.
2456 if (flags & LIR_OpArrayCopy::type_check) {
2931 __ verify_klass_ptr(tmp);
2932 #endif
2933 } else {
2934 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2935 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2936
2937 __ ldr(tmp, mdo_addr);
2938 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2939
2940 __ orr(tmp, tmp, TypeEntries::type_unknown);
2941 __ str(tmp, mdo_addr);
2942 // FIXME: Write barrier needed here?
2943 }
2944 }
2945
2946 __ bind(next);
2947 }
2948 COMMENT("} emit_profile_type");
2949 }
2950
2951 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
2952 Register obj = op->obj()->as_register();
2953 Register tmp = op->tmp()->as_pointer_register();
2954 bool not_null = op->not_null();
2955 int flag = op->flag();
2956
2957 Label not_inline_type;
2958 if (!not_null) {
2959 __ cbz(obj, not_inline_type);
2960 }
2961
2962 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
2963
2964 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), rscratch2);
2965 __ ldrb(rscratch1, mdo_addr);
2966 __ orr(rscratch1, rscratch1, flag);
2967 __ strb(rscratch1, mdo_addr);
2968
2969 __ bind(not_inline_type);
2970 }
2971
2972 void LIR_Assembler::align_backward_branch_target() {
2973 }
2974
2975
2976 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2977 // tmp must be unused
2978 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2979
2980 if (left->is_single_cpu()) {
2981 assert(dest->is_single_cpu(), "expect single result reg");
2982 __ negw(dest->as_register(), left->as_register());
2983 } else if (left->is_double_cpu()) {
2984 assert(dest->is_double_cpu(), "expect double result reg");
2985 __ neg(dest->as_register_lo(), left->as_register_lo());
2986 } else if (left->is_single_fpu()) {
2987 assert(dest->is_single_fpu(), "expect single float result reg");
2988 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2989 } else {
2990 assert(left->is_double_fpu(), "expect double float operand reg");
3092 void LIR_Assembler::membar_loadload() {
3093 __ membar(Assembler::LoadLoad);
3094 }
3095
3096 void LIR_Assembler::membar_storestore() {
3097 __ membar(MacroAssembler::StoreStore);
3098 }
3099
3100 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3101
3102 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3103
3104 void LIR_Assembler::on_spin_wait() {
3105 __ spin_wait();
3106 }
3107
3108 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3109 __ mov(result_reg->as_register(), rthread);
3110 }
3111
3112 void LIR_Assembler::check_orig_pc() {
3113 __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3114 __ cmp(rscratch2, (u1)NULL_WORD);
3115 }
3116
3117 void LIR_Assembler::peephole(LIR_List *lir) {
3118 #if 0
3119 if (tableswitch_count >= max_tableswitches)
3120 return;
3121
3122 /*
3123 This finite-state automaton recognizes sequences of compare-and-
3124 branch instructions. We will turn them into a tableswitch. You
3125 could argue that C1 really shouldn't be doing this sort of
3126 optimization, but without it the code is really horrible.
3127 */
3128
3129 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3130 int first_key, last_key = -2147483648;
3131 int next_key = 0;
3132 int start_insn = -1;
3133 int last_insn = -1;
3134 Register reg = noreg;
3135 LIR_Opr reg_opr;
|