16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "code/aotCodeCache.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "runtime/threadIdentifier.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_aarch64.inline.hpp"
49
50
51 #ifndef PRODUCT
52 #define COMMENT(x) do { __ block_comment(x); } while (0)
53 #else
54 #define COMMENT(x)
55 #endif
56
57 NEEDS_CLEANUP // remove this definitions ?
58 const Register SYNC_header = r0; // synchronization header
59 const Register SHIFT_count = r0; // where count for shift operations must be
60
61 #define __ _masm->
62
396 MonitorExitStub* stub = nullptr;
397 if (method()->is_synchronized()) {
398 monitor_address(0, FrameMap::r0_opr);
399 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
400 __ unlock_object(r5, r4, r0, r6, *stub->entry());
401 __ bind(*stub->continuation());
402 }
403
404 if (compilation()->env()->dtrace_method_probes()) {
405 __ mov(c_rarg0, rthread);
406 __ mov_metadata(c_rarg1, method()->constant_encoding());
407 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
408 }
409
410 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
411 __ mov(r0, r19); // Restore the exception
412 }
413
414 // remove the activation and dispatch to the unwind handler
415 __ block_comment("remove_frame and dispatch to the unwind handler");
416 __ remove_frame(initial_frame_size_in_bytes());
417 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
418
419 // Emit the slow path assembly
420 if (stub != nullptr) {
421 stub->emit_code(this);
422 }
423
424 return offset;
425 }
426
427
428 int LIR_Assembler::emit_deopt_handler() {
429 // generate code for exception handler
430 address handler_base = __ start_a_stub(deopt_handler_size());
431 if (handler_base == nullptr) {
432 // not enough space left for the handler
433 bailout("deopt handler overflow");
434 return -1;
435 }
436
438
439 Label start;
440 __ bind(start);
441
442 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
443
444 int entry_offset = __ offset();
445 __ b(start);
446
447 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
448 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
449 "out of bounds read in post-call NOP check");
450 __ end_a_stub();
451
452 return entry_offset;
453 }
454
455 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
456 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
457
458 // Pop the stack before the safepoint code
459 __ remove_frame(initial_frame_size_in_bytes());
460
461 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
462 __ reserved_stack_check();
463 }
464
465 code_stub->set_safepoint_offset(__ offset());
466 __ relocate(relocInfo::poll_return_type);
467 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
468 __ ret(lr);
469 }
470
471 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
472 guarantee(info != nullptr, "Shouldn't be null");
473 __ get_polling_page(rscratch1, relocInfo::poll_type);
474 add_debug_info_for_branch(info); // This isn't just debug info:
475 // it's the oop map
476 __ read_polling_page(rscratch1, relocInfo::poll_type);
477 return __ offset();
478 }
479
480
481 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
482 if (from_reg == r31_sp)
483 from_reg = sp;
484 if (to_reg == r31_sp)
485 to_reg = sp;
486 __ mov(to_reg, from_reg);
487 }
488
489 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
490
510 case T_LONG: {
511 assert(patch_code == lir_patch_none, "no patching handled here");
512 #if INCLUDE_CDS
513 if (AOTCodeCache::is_on_for_dump()) {
514 address b = c->as_pointer();
515 if (b == (address)ThreadIdentifier::unsafe_offset()) {
516 __ lea(dest->as_register_lo(), ExternalAddress(b));
517 break;
518 }
519 if (AOTRuntimeConstants::contains(b)) {
520 __ load_aotrc_address(dest->as_register_lo(), b);
521 break;
522 }
523 }
524 #endif
525 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
526 break;
527 }
528
529 case T_OBJECT: {
530 if (patch_code == lir_patch_none) {
531 jobject2reg(c->as_jobject(), dest->as_register());
532 } else {
533 jobject2reg_with_patching(dest->as_register(), info);
534 }
535 break;
536 }
537
538 case T_METADATA: {
539 if (patch_code != lir_patch_none) {
540 klass2reg_with_patching(dest->as_register(), info);
541 } else {
542 __ mov_metadata(dest->as_register(), c->as_metadata());
543 }
544 break;
545 }
546
547 case T_FLOAT: {
548 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
549 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
550 } else {
551 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
552 __ ldrs(dest->as_float_reg(), Address(rscratch1));
553 }
986 case T_BYTE:
987 __ ldrsb(dest->as_register(), as_Address(from_addr));
988 break;
989 case T_BOOLEAN: {
990 __ ldrb(dest->as_register(), as_Address(from_addr));
991 break;
992 }
993
994 case T_CHAR:
995 __ ldrh(dest->as_register(), as_Address(from_addr));
996 break;
997 case T_SHORT:
998 __ ldrsh(dest->as_register(), as_Address(from_addr));
999 break;
1000
1001 default:
1002 ShouldNotReachHere();
1003 }
1004 }
1005
1006 void LIR_Assembler::load_volatile(LIR_Address *from_addr, LIR_Opr dest,
1007 BasicType type, CodeEmitInfo* info) {
1008 __ lea(rscratch1, as_Address(from_addr));
1009
1010 Register dest_reg = rscratch2;
1011 if (!is_floating_point_type(type)) {
1012 dest_reg = (dest->is_single_cpu()
1013 ? dest->as_register() : dest->as_register_lo());
1014 }
1015
1016 if (info != nullptr) {
1017 add_debug_info_for_null_check_here(info);
1018 }
1019
1020 // Uses LDAR to ensure memory ordering.
1021 __ load_store_volatile(dest_reg, type, rscratch1, /*is_load*/true);
1022
1023 switch (type) {
1024 // LDAR is unsigned so need to sign-extend for byte and short
1025 case T_BYTE:
1213 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1214 __ ldarb(rscratch1, rscratch1);
1215 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1216 add_debug_info_for_null_check_here(op->stub()->info());
1217 __ br(Assembler::NE, *op->stub()->entry());
1218 }
1219 __ allocate_object(op->obj()->as_register(),
1220 op->tmp1()->as_register(),
1221 op->tmp2()->as_register(),
1222 op->header_size(),
1223 op->object_size(),
1224 op->klass()->as_register(),
1225 *op->stub()->entry());
1226 __ bind(*op->stub()->continuation());
1227 }
1228
1229 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1230 Register len = op->len()->as_register();
1231 __ uxtw(len, len);
1232
1233 if (UseSlowPath ||
1234 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1235 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1236 __ b(*op->stub()->entry());
1237 } else {
1238 Register tmp1 = op->tmp1()->as_register();
1239 Register tmp2 = op->tmp2()->as_register();
1240 Register tmp3 = op->tmp3()->as_register();
1241 if (len == tmp1) {
1242 tmp1 = tmp3;
1243 } else if (len == tmp2) {
1244 tmp2 = tmp3;
1245 } else if (len == tmp3) {
1246 // everything is ok
1247 } else {
1248 __ mov(tmp3, len);
1249 }
1250 __ allocate_array(op->obj()->as_register(),
1251 len,
1252 tmp1,
1253 tmp2,
1290 md = method->method_data_or_null();
1291 assert(md != nullptr, "Sanity");
1292 data = md->bci_to_data(bci);
1293 assert(data != nullptr, "need data for type check");
1294 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1295 }
1296 Label* success_target = success;
1297 Label* failure_target = failure;
1298
1299 if (obj == k_RInfo) {
1300 k_RInfo = dst;
1301 } else if (obj == klass_RInfo) {
1302 klass_RInfo = dst;
1303 }
1304
1305 Rtmp1 = op->tmp3()->as_register();
1306 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1307
1308 assert_different_registers(obj, k_RInfo, klass_RInfo);
1309
1310 if (should_profile) {
1311 Register mdo = klass_RInfo;
1312 __ mov_metadata(mdo, md->constant_encoding());
1313 Label not_null;
1314 __ cbnz(obj, not_null);
1315 // Object is null; update MDO and exit
1316 Address data_addr
1317 = __ form_address(rscratch2, mdo,
1318 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1319 0);
1320 __ ldrb(rscratch1, data_addr);
1321 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1322 __ strb(rscratch1, data_addr);
1323 __ b(*obj_is_null);
1324 __ bind(not_null);
1325
1326 Register recv = k_RInfo;
1327 __ load_klass(recv, obj, rscratch1);
1328 type_profile_helper(mdo, md, data, recv);
1329 } else {
1330 __ cbz(obj, *obj_is_null);
1331 }
1332
1333 if (!k->is_loaded()) {
1334 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1335 } else {
1336 __ mov_metadata(k_RInfo, k->constant_encoding());
1337 }
1338 __ verify_oop(obj);
1339
1340 if (op->fast_check()) {
1341 // get object class
1342 // not a safepoint as obj null check happens earlier
1343 __ load_klass(rscratch2, obj, rscratch1);
1344 __ cmp( rscratch2, k_RInfo);
1345
1346 __ br(Assembler::NE, *failure_target);
1347 // successful cast, fall through to profile or jump
1348 } else {
1349 // get object class
1350 // not a safepoint as obj null check happens earlier
1351 __ load_klass(klass_RInfo, obj, rscratch1);
1352 if (k->is_loaded()) {
1353 // See if we get an immediate positive hit
1354 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1355 __ cmp(k_RInfo, rscratch1);
1356 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1357 __ br(Assembler::NE, *failure_target);
1358 // successful cast, fall through to profile or jump
1359 } else {
1360 // See if we get an immediate positive hit
1361 __ br(Assembler::EQ, *success_target);
1362 // check for self
1363 __ cmp(klass_RInfo, k_RInfo);
1364 __ br(Assembler::EQ, *success_target);
1365
1366 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1367 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1368 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1369 // result is a boolean
1370 __ cbzw(klass_RInfo, *failure_target);
1371 // successful cast, fall through to profile or jump
1372 }
1373 } else {
1374 // perform the fast part of the checking logic
1375 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1376 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1377 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1378 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1379 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1380 // result is a boolean
1381 __ cbz(k_RInfo, *failure_target);
1382 // successful cast, fall through to profile or jump
1383 }
1464 __ bind(success);
1465 if (dst != obj) {
1466 __ mov(dst, obj);
1467 }
1468 } else if (code == lir_instanceof) {
1469 Register obj = op->object()->as_register();
1470 Register dst = op->result_opr()->as_register();
1471 Label success, failure, done;
1472 emit_typecheck_helper(op, &success, &failure, &failure);
1473 __ bind(failure);
1474 __ mov(dst, zr);
1475 __ b(done);
1476 __ bind(success);
1477 __ mov(dst, 1);
1478 __ bind(done);
1479 } else {
1480 ShouldNotReachHere();
1481 }
1482 }
1483
1484 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1485 __ cmpxchg(addr, cmpval, newval, Assembler::word, memory_order_seq_cst, rscratch1);
1486 __ cset(rscratch1, Assembler::NE);
1487 }
1488
1489 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1490 __ cmpxchg(addr, cmpval, newval, Assembler::xword, memory_order_seq_cst, rscratch1);
1491 __ cset(rscratch1, Assembler::NE);
1492 }
1493
1494
1495 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1496 Register addr;
1497 if (op->addr()->is_register()) {
1498 addr = as_reg(op->addr());
1499 } else {
1500 assert(op->addr()->is_address(), "what else?");
1501 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1502 assert(addr_ptr->disp() == 0, "need 0 disp");
1503 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1975 __ cmp(left->as_register_lo(), right->as_register_lo());
1976 __ mov(dst->as_register(), (uint64_t)-1L);
1977 __ br(Assembler::LT, done);
1978 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1979 __ bind(done);
1980 } else {
1981 ShouldNotReachHere();
1982 }
1983 }
1984
1985
1986 void LIR_Assembler::align_call(LIR_Code code) { }
1987
1988
1989 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1990 address call = __ trampoline_call(Address(op->addr(), rtype));
1991 if (call == nullptr) {
1992 bailout("trampoline stub overflow");
1993 return;
1994 }
1995 add_call_info(code_offset(), op->info());
1996 __ post_call_nop();
1997 }
1998
1999
2000 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2001 address call = __ ic_call(op->addr());
2002 if (call == nullptr) {
2003 bailout("trampoline stub overflow");
2004 return;
2005 }
2006 add_call_info(code_offset(), op->info());
2007 __ post_call_nop();
2008 }
2009
2010 void LIR_Assembler::emit_static_call_stub() {
2011 address call_pc = __ pc();
2012 address stub = __ start_a_stub(call_stub_size());
2013 if (stub == nullptr) {
2014 bailout("static call stub overflow");
2015 return;
2016 }
2017
2018 int start = __ offset();
2019
2020 __ relocate(static_stub_Relocation::spec(call_pc));
2021 __ emit_static_call_stub();
2022
2023 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2024 <= call_stub_size(), "stub too big");
2025 __ end_a_stub();
2026 }
2149
2150
2151 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2152 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2153 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2154 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2155 __ mov (rscratch1, c);
2156 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2157 }
2158
2159
2160 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2161 ShouldNotReachHere();
2162 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2163 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2164 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2165 __ lea(rscratch1, __ constant_oop_address(o));
2166 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2167 }
2168
2169
2170 // This code replaces a call to arraycopy; no exception may
2171 // be thrown in this code, they must be thrown in the System.arraycopy
2172 // activation frame; we could save some checks if this would not be the case
2173 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2174 ciArrayKlass* default_type = op->expected_type();
2175 Register src = op->src()->as_register();
2176 Register dst = op->dst()->as_register();
2177 Register src_pos = op->src_pos()->as_register();
2178 Register dst_pos = op->dst_pos()->as_register();
2179 Register length = op->length()->as_register();
2180 Register tmp = op->tmp()->as_register();
2181
2182 CodeStub* stub = op->stub();
2183 int flags = op->flags();
2184 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2185 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2186
2187 // if we don't know anything, just go through the generic arraycopy
2188 if (default_type == nullptr // || basic_type == T_OBJECT
2189 ) {
2190 Label done;
2191 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2192
2193 // Save the arguments in case the generic arraycopy fails and we
2194 // have to fall back to the JNI stub
2195 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2196 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2197 __ str(src, Address(sp, 4*BytesPerWord));
2198
2199 address copyfunc_addr = StubRoutines::generic_arraycopy();
2200 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2201
2202 // The arguments are in java calling convention so we shift them
2203 // to C convention
2204 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2205 __ mov(c_rarg0, j_rarg0);
2206 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2220 __ cbz(r0, *stub->continuation());
2221
2222 // Reload values from the stack so they are where the stub
2223 // expects them.
2224 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2225 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2226 __ ldr(src, Address(sp, 4*BytesPerWord));
2227
2228 // r0 is -1^K where K == partial copied count
2229 __ eonw(rscratch1, r0, zr);
2230 // adjust length down and src/end pos up by partial copied count
2231 __ subw(length, length, rscratch1);
2232 __ addw(src_pos, src_pos, rscratch1);
2233 __ addw(dst_pos, dst_pos, rscratch1);
2234 __ b(*stub->entry());
2235
2236 __ bind(*stub->continuation());
2237 return;
2238 }
2239
2240 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2241
2242 int elem_size = type2aelembytes(basic_type);
2243 int scale = exact_log2(elem_size);
2244
2245 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2246 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2247
2248 // test for null
2249 if (flags & LIR_OpArrayCopy::src_null_check) {
2250 __ cbz(src, *stub->entry());
2251 }
2252 if (flags & LIR_OpArrayCopy::dst_null_check) {
2253 __ cbz(dst, *stub->entry());
2254 }
2255
2256 // If the compiler was not able to prove that exact type of the source or the destination
2257 // of the arraycopy is an array type, check at runtime if the source or the destination is
2258 // an instance type.
2259 if (flags & LIR_OpArrayCopy::type_check) {
2734 __ verify_klass_ptr(tmp);
2735 #endif
2736 } else {
2737 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2738 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2739
2740 __ ldr(tmp, mdo_addr);
2741 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2742
2743 __ orr(tmp, tmp, TypeEntries::type_unknown);
2744 __ str(tmp, mdo_addr);
2745 // FIXME: Write barrier needed here?
2746 }
2747 }
2748
2749 __ bind(next);
2750 }
2751 COMMENT("} emit_profile_type");
2752 }
2753
2754
2755 void LIR_Assembler::align_backward_branch_target() {
2756 }
2757
2758
2759 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2760 // tmp must be unused
2761 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2762
2763 if (left->is_single_cpu()) {
2764 assert(dest->is_single_cpu(), "expect single result reg");
2765 __ negw(dest->as_register(), left->as_register());
2766 } else if (left->is_double_cpu()) {
2767 assert(dest->is_double_cpu(), "expect double result reg");
2768 __ neg(dest->as_register_lo(), left->as_register_lo());
2769 } else if (left->is_single_fpu()) {
2770 assert(dest->is_single_fpu(), "expect single float result reg");
2771 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2772 } else {
2773 assert(left->is_double_fpu(), "expect double float operand reg");
2875 void LIR_Assembler::membar_loadload() {
2876 __ membar(Assembler::LoadLoad);
2877 }
2878
2879 void LIR_Assembler::membar_storestore() {
2880 __ membar(MacroAssembler::StoreStore);
2881 }
2882
2883 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2884
2885 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2886
2887 void LIR_Assembler::on_spin_wait() {
2888 __ spin_wait();
2889 }
2890
2891 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2892 __ mov(result_reg->as_register(), rthread);
2893 }
2894
2895
2896 void LIR_Assembler::peephole(LIR_List *lir) {
2897 #if 0
2898 if (tableswitch_count >= max_tableswitches)
2899 return;
2900
2901 /*
2902 This finite-state automaton recognizes sequences of compare-and-
2903 branch instructions. We will turn them into a tableswitch. You
2904 could argue that C1 really shouldn't be doing this sort of
2905 optimization, but without it the code is really horrible.
2906 */
2907
2908 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2909 int first_key, last_key = -2147483648;
2910 int next_key = 0;
2911 int start_insn = -1;
2912 int last_insn = -1;
2913 Register reg = noreg;
2914 LIR_Opr reg_opr;
|
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInlineKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "ci/ciObjArrayKlass.hpp"
39 #include "code/aotCodeCache.hpp"
40 #include "code/compiledIC.hpp"
41 #include "gc/shared/collectedHeap.hpp"
42 #include "gc/shared/gc_globals.hpp"
43 #include "nativeInst_aarch64.hpp"
44 #include "oops/objArrayKlass.hpp"
45 #include "oops/oop.inline.hpp"
46 #include "runtime/frame.inline.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/threadIdentifier.hpp"
50 #include "utilities/powerOfTwo.hpp"
51 #include "vmreg_aarch64.inline.hpp"
52
53
54 #ifndef PRODUCT
55 #define COMMENT(x) do { __ block_comment(x); } while (0)
56 #else
57 #define COMMENT(x)
58 #endif
59
60 NEEDS_CLEANUP // remove this definitions ?
61 const Register SYNC_header = r0; // synchronization header
62 const Register SHIFT_count = r0; // where count for shift operations must be
63
64 #define __ _masm->
65
399 MonitorExitStub* stub = nullptr;
400 if (method()->is_synchronized()) {
401 monitor_address(0, FrameMap::r0_opr);
402 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
403 __ unlock_object(r5, r4, r0, r6, *stub->entry());
404 __ bind(*stub->continuation());
405 }
406
407 if (compilation()->env()->dtrace_method_probes()) {
408 __ mov(c_rarg0, rthread);
409 __ mov_metadata(c_rarg1, method()->constant_encoding());
410 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
411 }
412
413 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
414 __ mov(r0, r19); // Restore the exception
415 }
416
417 // remove the activation and dispatch to the unwind handler
418 __ block_comment("remove_frame and dispatch to the unwind handler");
419 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
420 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
421
422 // Emit the slow path assembly
423 if (stub != nullptr) {
424 stub->emit_code(this);
425 }
426
427 return offset;
428 }
429
430
431 int LIR_Assembler::emit_deopt_handler() {
432 // generate code for exception handler
433 address handler_base = __ start_a_stub(deopt_handler_size());
434 if (handler_base == nullptr) {
435 // not enough space left for the handler
436 bailout("deopt handler overflow");
437 return -1;
438 }
439
441
442 Label start;
443 __ bind(start);
444
445 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
446
447 int entry_offset = __ offset();
448 __ b(start);
449
450 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
451 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
452 "out of bounds read in post-call NOP check");
453 __ end_a_stub();
454
455 return entry_offset;
456 }
457
458 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
459 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
460
461 if (InlineTypeReturnedAsFields) {
462 // Check if we are returning a non-null inline type and load its fields into registers
463 ciType* return_type = compilation()->method()->return_type();
464 if (return_type->is_inlinetype()) {
465 ciInlineKlass* vk = return_type->as_inline_klass();
466 if (vk->can_be_returned_as_fields()) {
467 address unpack_handler = vk->unpack_handler();
468 assert(unpack_handler != nullptr, "must be");
469 __ far_call(RuntimeAddress(unpack_handler));
470 }
471 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
472 Label skip;
473 Label not_null;
474 __ cbnz(r0, not_null);
475 // Returned value is null, zero all return registers because they may belong to oop fields
476 __ mov(j_rarg1, zr);
477 __ mov(j_rarg2, zr);
478 __ mov(j_rarg3, zr);
479 __ mov(j_rarg4, zr);
480 __ mov(j_rarg5, zr);
481 __ mov(j_rarg6, zr);
482 __ mov(j_rarg7, zr);
483 __ b(skip);
484 __ bind(not_null);
485
486 // Check if we are returning a non-null inline type and load its fields into registers
487 __ test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
488
489 // Load fields from a buffered value with an inline class specific handler
490 __ load_klass(rscratch1 /*dst*/, r0 /*src*/, rscratch2 /*tmp*/);
491 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
492 __ ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
493 // Unpack handler can be null if inline type is not scalarizable in returns
494 __ cbz(rscratch1, skip);
495 __ blr(rscratch1);
496
497 __ bind(skip);
498 }
499 // At this point, r0 points to the value object (for interpreter or C1 caller).
500 // The fields of the object are copied into registers (for C2 caller).
501 }
502
503 // Pop the stack before the safepoint code
504 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
505
506 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
507 __ reserved_stack_check();
508 }
509
510 code_stub->set_safepoint_offset(__ offset());
511 __ relocate(relocInfo::poll_return_type);
512 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
513 __ ret(lr);
514 }
515
516 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
517 return (__ store_inline_type_fields_to_buf(vk, false));
518 }
519
520 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
521 guarantee(info != nullptr, "Shouldn't be null");
522 __ get_polling_page(rscratch1, relocInfo::poll_type);
523 add_debug_info_for_branch(info); // This isn't just debug info:
524 // it's the oop map
525 __ read_polling_page(rscratch1, relocInfo::poll_type);
526 return __ offset();
527 }
528
529
530 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
531 if (from_reg == r31_sp)
532 from_reg = sp;
533 if (to_reg == r31_sp)
534 to_reg = sp;
535 __ mov(to_reg, from_reg);
536 }
537
538 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
539
559 case T_LONG: {
560 assert(patch_code == lir_patch_none, "no patching handled here");
561 #if INCLUDE_CDS
562 if (AOTCodeCache::is_on_for_dump()) {
563 address b = c->as_pointer();
564 if (b == (address)ThreadIdentifier::unsafe_offset()) {
565 __ lea(dest->as_register_lo(), ExternalAddress(b));
566 break;
567 }
568 if (AOTRuntimeConstants::contains(b)) {
569 __ load_aotrc_address(dest->as_register_lo(), b);
570 break;
571 }
572 }
573 #endif
574 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
575 break;
576 }
577
578 case T_OBJECT: {
579 if (patch_code != lir_patch_none) {
580 jobject2reg_with_patching(dest->as_register(), info);
581 } else {
582 jobject2reg(c->as_jobject(), dest->as_register());
583 }
584 break;
585 }
586
587 case T_METADATA: {
588 if (patch_code != lir_patch_none) {
589 klass2reg_with_patching(dest->as_register(), info);
590 } else {
591 __ mov_metadata(dest->as_register(), c->as_metadata());
592 }
593 break;
594 }
595
596 case T_FLOAT: {
597 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
598 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
599 } else {
600 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
601 __ ldrs(dest->as_float_reg(), Address(rscratch1));
602 }
1035 case T_BYTE:
1036 __ ldrsb(dest->as_register(), as_Address(from_addr));
1037 break;
1038 case T_BOOLEAN: {
1039 __ ldrb(dest->as_register(), as_Address(from_addr));
1040 break;
1041 }
1042
1043 case T_CHAR:
1044 __ ldrh(dest->as_register(), as_Address(from_addr));
1045 break;
1046 case T_SHORT:
1047 __ ldrsh(dest->as_register(), as_Address(from_addr));
1048 break;
1049
1050 default:
1051 ShouldNotReachHere();
1052 }
1053 }
1054
1055 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1056 assert(dst->is_cpu_register(), "must be");
1057 assert(dst->type() == src->type(), "must be");
1058
1059 if (src->is_cpu_register()) {
1060 reg2reg(src, dst);
1061 } else if (src->is_stack()) {
1062 stack2reg(src, dst, dst->type());
1063 } else if (src->is_constant()) {
1064 const2reg(src, dst, lir_patch_none, nullptr);
1065 } else {
1066 ShouldNotReachHere();
1067 }
1068 }
1069
1070 void LIR_Assembler::load_volatile(LIR_Address *from_addr, LIR_Opr dest,
1071 BasicType type, CodeEmitInfo* info) {
1072 __ lea(rscratch1, as_Address(from_addr));
1073
1074 Register dest_reg = rscratch2;
1075 if (!is_floating_point_type(type)) {
1076 dest_reg = (dest->is_single_cpu()
1077 ? dest->as_register() : dest->as_register_lo());
1078 }
1079
1080 if (info != nullptr) {
1081 add_debug_info_for_null_check_here(info);
1082 }
1083
1084 // Uses LDAR to ensure memory ordering.
1085 __ load_store_volatile(dest_reg, type, rscratch1, /*is_load*/true);
1086
1087 switch (type) {
1088 // LDAR is unsigned so need to sign-extend for byte and short
1089 case T_BYTE:
1277 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1278 __ ldarb(rscratch1, rscratch1);
1279 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1280 add_debug_info_for_null_check_here(op->stub()->info());
1281 __ br(Assembler::NE, *op->stub()->entry());
1282 }
1283 __ allocate_object(op->obj()->as_register(),
1284 op->tmp1()->as_register(),
1285 op->tmp2()->as_register(),
1286 op->header_size(),
1287 op->object_size(),
1288 op->klass()->as_register(),
1289 *op->stub()->entry());
1290 __ bind(*op->stub()->continuation());
1291 }
1292
1293 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1294 Register len = op->len()->as_register();
1295 __ uxtw(len, len);
1296
1297 if (UseSlowPath || op->always_slow_path() ||
1298 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1299 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1300 __ b(*op->stub()->entry());
1301 } else {
1302 Register tmp1 = op->tmp1()->as_register();
1303 Register tmp2 = op->tmp2()->as_register();
1304 Register tmp3 = op->tmp3()->as_register();
1305 if (len == tmp1) {
1306 tmp1 = tmp3;
1307 } else if (len == tmp2) {
1308 tmp2 = tmp3;
1309 } else if (len == tmp3) {
1310 // everything is ok
1311 } else {
1312 __ mov(tmp3, len);
1313 }
1314 __ allocate_array(op->obj()->as_register(),
1315 len,
1316 tmp1,
1317 tmp2,
1354 md = method->method_data_or_null();
1355 assert(md != nullptr, "Sanity");
1356 data = md->bci_to_data(bci);
1357 assert(data != nullptr, "need data for type check");
1358 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1359 }
1360 Label* success_target = success;
1361 Label* failure_target = failure;
1362
1363 if (obj == k_RInfo) {
1364 k_RInfo = dst;
1365 } else if (obj == klass_RInfo) {
1366 klass_RInfo = dst;
1367 }
1368
1369 Rtmp1 = op->tmp3()->as_register();
1370 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1371
1372 assert_different_registers(obj, k_RInfo, klass_RInfo);
1373
1374 if (op->need_null_check()) {
1375 if (should_profile) {
1376 Register mdo = klass_RInfo;
1377 __ mov_metadata(mdo, md->constant_encoding());
1378 Label not_null;
1379 __ cbnz(obj, not_null);
1380 // Object is null; update MDO and exit
1381 Address data_addr
1382 = __ form_address(rscratch2, mdo,
1383 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1384 0);
1385 __ ldrb(rscratch1, data_addr);
1386 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1387 __ strb(rscratch1, data_addr);
1388 __ b(*obj_is_null);
1389 __ bind(not_null);
1390
1391 Register recv = k_RInfo;
1392 __ load_klass(recv, obj, rscratch1);
1393 type_profile_helper(mdo, md, data, recv);
1394 } else {
1395 __ cbz(obj, *obj_is_null);
1396 }
1397 }
1398
1399 if (!k->is_loaded()) {
1400 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1401 } else {
1402 __ mov_metadata(k_RInfo, k->constant_encoding());
1403 }
1404 __ verify_oop(obj);
1405
1406 if (op->fast_check()) {
1407 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1408 // get object class
1409 // not a safepoint as obj null check happens earlier
1410 __ load_klass(rscratch2, obj, rscratch1);
1411 __ cmp( rscratch2, k_RInfo);
1412
1413 __ br(Assembler::NE, *failure_target);
1414 // successful cast, fall through to profile or jump
1415 } else {
1416 // get object class
1417 // not a safepoint as obj null check happens earlier
1418 __ load_klass(klass_RInfo, obj, rscratch1);
1419 if (k->is_loaded()) {
1420 // See if we get an immediate positive hit
1421 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1422 __ cmp(k_RInfo, rscratch1);
1423 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1424 __ br(Assembler::NE, *failure_target);
1425 // successful cast, fall through to profile or jump
1426 } else {
1427 // See if we get an immediate positive hit
1428 __ br(Assembler::EQ, *success_target);
1429 // check for self
1430 if (k->is_loaded() && k->is_obj_array_klass()) {
1431 // For a direct pointer comparison, we need the refined array klass pointer
1432 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1433 if (!k_refined->is_loaded()) {
1434 bailout("encountered unloaded_ciobjarrayklass due to out of memory error");
1435 return;
1436 }
1437 __ mov_metadata(rscratch1, k_refined->constant_encoding());
1438 __ cmp(klass_RInfo, rscratch1);
1439 } else {
1440 __ cmp(klass_RInfo, k_RInfo);
1441 }
1442 __ br(Assembler::EQ, *success_target);
1443
1444 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1445 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1446 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1447 // result is a boolean
1448 __ cbzw(klass_RInfo, *failure_target);
1449 // successful cast, fall through to profile or jump
1450 }
1451 } else {
1452 // perform the fast part of the checking logic
1453 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1454 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1455 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1456 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1457 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1458 // result is a boolean
1459 __ cbz(k_RInfo, *failure_target);
1460 // successful cast, fall through to profile or jump
1461 }
1542 __ bind(success);
1543 if (dst != obj) {
1544 __ mov(dst, obj);
1545 }
1546 } else if (code == lir_instanceof) {
1547 Register obj = op->object()->as_register();
1548 Register dst = op->result_opr()->as_register();
1549 Label success, failure, done;
1550 emit_typecheck_helper(op, &success, &failure, &failure);
1551 __ bind(failure);
1552 __ mov(dst, zr);
1553 __ b(done);
1554 __ bind(success);
1555 __ mov(dst, 1);
1556 __ bind(done);
1557 } else {
1558 ShouldNotReachHere();
1559 }
1560 }
1561
1562 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1563 // We are loading/storing from/to an array that *may* be a flat array (the
1564 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1565 // If this array is a flat array, take the slow path.
1566 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1567 }
1568
1569 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1570 // We are storing into an array that *may* be null-free (the declared type is
1571 // Object[], abstract[], interface[] or VT.ref[]).
1572 Label test_mark_word;
1573 Register tmp = op->tmp()->as_register();
1574 __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1575 __ tst(tmp, markWord::unlocked_value);
1576 __ br(Assembler::NE, test_mark_word);
1577 __ load_prototype_header(tmp, op->array()->as_register());
1578 __ bind(test_mark_word);
1579 __ tst(tmp, markWord::null_free_array_bit_in_place);
1580 }
1581
1582 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1583 Label L_oops_equal;
1584 Label L_oops_not_equal;
1585 Label L_end;
1586
1587 Register left = op->left()->as_register();
1588 Register right = op->right()->as_register();
1589
1590 __ cmp(left, right);
1591 __ br(Assembler::EQ, L_oops_equal);
1592
1593 // (1) Null check -- if one of the operands is null, the other must not be null (because
1594 // the two references are not equal), so they are not substitutable,
1595 __ cbz(left, L_oops_not_equal);
1596 __ cbz(right, L_oops_not_equal);
1597
1598 ciKlass* left_klass = op->left_klass();
1599 ciKlass* right_klass = op->right_klass();
1600
1601 // (2) Inline type check -- if either of the operands is not an inline type,
1602 // they are not substitutable. We do this only if we are not sure that the
1603 // operands are inline type
1604 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1605 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1606 Register tmp1 = op->tmp1()->as_register();
1607 Register tmp2 = op->tmp2()->as_register();
1608 __ mov(tmp1, markWord::inline_type_pattern);
1609 __ ldr(tmp2, Address(left, oopDesc::mark_offset_in_bytes()));
1610 __ andr(tmp1, tmp1, tmp2);
1611 __ ldr(tmp2, Address(right, oopDesc::mark_offset_in_bytes()));
1612 __ andr(tmp1, tmp1, tmp2);
1613 __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1614 __ br(Assembler::NE, L_oops_not_equal);
1615 }
1616
1617 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1618 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1619 // No need to load klass -- the operands are statically known to be the same inline klass.
1620 __ b(*op->stub()->entry());
1621 } else {
1622 Register tmp1 = op->tmp1()->as_register();
1623 Register tmp2 = op->tmp2()->as_register();
1624 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1625 __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1626 // fall through to L_oops_not_equal
1627 }
1628
1629 __ bind(L_oops_not_equal);
1630 move(op->not_equal_result(), op->result_opr());
1631 __ b(L_end);
1632
1633 // We've returned from the stub. R0 contains 0x0 IFF the two
1634 // operands are not substitutable. (Don't compare against 0x1 in case the
1635 // C compiler is naughty)
1636 __ bind(*op->stub()->continuation());
1637 __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1638
1639 __ bind(L_oops_equal);
1640 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1641 // fall-through
1642 __ bind(L_end);
1643 }
1644
1645
1646 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1647 __ cmpxchg(addr, cmpval, newval, Assembler::word, memory_order_seq_cst, rscratch1);
1648 __ cset(rscratch1, Assembler::NE);
1649 }
1650
1651 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1652 __ cmpxchg(addr, cmpval, newval, Assembler::xword, memory_order_seq_cst, rscratch1);
1653 __ cset(rscratch1, Assembler::NE);
1654 }
1655
1656
1657 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1658 Register addr;
1659 if (op->addr()->is_register()) {
1660 addr = as_reg(op->addr());
1661 } else {
1662 assert(op->addr()->is_address(), "what else?");
1663 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1664 assert(addr_ptr->disp() == 0, "need 0 disp");
1665 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
2137 __ cmp(left->as_register_lo(), right->as_register_lo());
2138 __ mov(dst->as_register(), (uint64_t)-1L);
2139 __ br(Assembler::LT, done);
2140 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2141 __ bind(done);
2142 } else {
2143 ShouldNotReachHere();
2144 }
2145 }
2146
2147
2148 void LIR_Assembler::align_call(LIR_Code code) { }
2149
2150
2151 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2152 address call = __ trampoline_call(Address(op->addr(), rtype));
2153 if (call == nullptr) {
2154 bailout("trampoline stub overflow");
2155 return;
2156 }
2157 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2158 __ post_call_nop();
2159 }
2160
2161
2162 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2163 address call = __ ic_call(op->addr());
2164 if (call == nullptr) {
2165 bailout("trampoline stub overflow");
2166 return;
2167 }
2168 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2169 __ post_call_nop();
2170 }
2171
2172 void LIR_Assembler::emit_static_call_stub() {
2173 address call_pc = __ pc();
2174 address stub = __ start_a_stub(call_stub_size());
2175 if (stub == nullptr) {
2176 bailout("static call stub overflow");
2177 return;
2178 }
2179
2180 int start = __ offset();
2181
2182 __ relocate(static_stub_Relocation::spec(call_pc));
2183 __ emit_static_call_stub();
2184
2185 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2186 <= call_stub_size(), "stub too big");
2187 __ end_a_stub();
2188 }
2311
2312
2313 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2314 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2315 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2316 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2317 __ mov (rscratch1, c);
2318 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2319 }
2320
2321
2322 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2323 ShouldNotReachHere();
2324 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2325 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2326 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2327 __ lea(rscratch1, __ constant_oop_address(o));
2328 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2329 }
2330
2331 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2332 if (null_check) {
2333 __ cbz(obj, *slow_path->entry());
2334 }
2335 if (is_dest) {
2336 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2337 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2338 } else {
2339 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2340 }
2341 }
2342
2343 // This code replaces a call to arraycopy; no exception may
2344 // be thrown in this code, they must be thrown in the System.arraycopy
2345 // activation frame; we could save some checks if this would not be the case
2346 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2347 ciArrayKlass* default_type = op->expected_type();
2348 Register src = op->src()->as_register();
2349 Register dst = op->dst()->as_register();
2350 Register src_pos = op->src_pos()->as_register();
2351 Register dst_pos = op->dst_pos()->as_register();
2352 Register length = op->length()->as_register();
2353 Register tmp = op->tmp()->as_register();
2354
2355 CodeStub* stub = op->stub();
2356 int flags = op->flags();
2357 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2358 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2359
2360 if (flags & LIR_OpArrayCopy::always_slow_path) {
2361 __ b(*stub->entry());
2362 __ bind(*stub->continuation());
2363 return;
2364 }
2365
2366 // if we don't know anything, just go through the generic arraycopy
2367 if (default_type == nullptr // || basic_type == T_OBJECT
2368 ) {
2369 Label done;
2370 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2371
2372 // Save the arguments in case the generic arraycopy fails and we
2373 // have to fall back to the JNI stub
2374 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2375 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2376 __ str(src, Address(sp, 4*BytesPerWord));
2377
2378 address copyfunc_addr = StubRoutines::generic_arraycopy();
2379 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2380
2381 // The arguments are in java calling convention so we shift them
2382 // to C convention
2383 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2384 __ mov(c_rarg0, j_rarg0);
2385 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2399 __ cbz(r0, *stub->continuation());
2400
2401 // Reload values from the stack so they are where the stub
2402 // expects them.
2403 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2404 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2405 __ ldr(src, Address(sp, 4*BytesPerWord));
2406
2407 // r0 is -1^K where K == partial copied count
2408 __ eonw(rscratch1, r0, zr);
2409 // adjust length down and src/end pos up by partial copied count
2410 __ subw(length, length, rscratch1);
2411 __ addw(src_pos, src_pos, rscratch1);
2412 __ addw(dst_pos, dst_pos, rscratch1);
2413 __ b(*stub->entry());
2414
2415 __ bind(*stub->continuation());
2416 return;
2417 }
2418
2419 // Handle inline type arrays
2420 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2421 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2422 }
2423 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2424 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2425 }
2426
2427 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2428
2429 int elem_size = type2aelembytes(basic_type);
2430 int scale = exact_log2(elem_size);
2431
2432 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2433 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2434
2435 // test for null
2436 if (flags & LIR_OpArrayCopy::src_null_check) {
2437 __ cbz(src, *stub->entry());
2438 }
2439 if (flags & LIR_OpArrayCopy::dst_null_check) {
2440 __ cbz(dst, *stub->entry());
2441 }
2442
2443 // If the compiler was not able to prove that exact type of the source or the destination
2444 // of the arraycopy is an array type, check at runtime if the source or the destination is
2445 // an instance type.
2446 if (flags & LIR_OpArrayCopy::type_check) {
2921 __ verify_klass_ptr(tmp);
2922 #endif
2923 } else {
2924 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2925 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2926
2927 __ ldr(tmp, mdo_addr);
2928 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2929
2930 __ orr(tmp, tmp, TypeEntries::type_unknown);
2931 __ str(tmp, mdo_addr);
2932 // FIXME: Write barrier needed here?
2933 }
2934 }
2935
2936 __ bind(next);
2937 }
2938 COMMENT("} emit_profile_type");
2939 }
2940
2941 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
2942 Register obj = op->obj()->as_register();
2943 Register tmp = op->tmp()->as_pointer_register();
2944 bool not_null = op->not_null();
2945 int flag = op->flag();
2946
2947 Label not_inline_type;
2948 if (!not_null) {
2949 __ cbz(obj, not_inline_type);
2950 }
2951
2952 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
2953
2954 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), rscratch2);
2955 __ ldrb(rscratch1, mdo_addr);
2956 __ orr(rscratch1, rscratch1, flag);
2957 __ strb(rscratch1, mdo_addr);
2958
2959 __ bind(not_inline_type);
2960 }
2961
2962 void LIR_Assembler::align_backward_branch_target() {
2963 }
2964
2965
2966 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2967 // tmp must be unused
2968 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2969
2970 if (left->is_single_cpu()) {
2971 assert(dest->is_single_cpu(), "expect single result reg");
2972 __ negw(dest->as_register(), left->as_register());
2973 } else if (left->is_double_cpu()) {
2974 assert(dest->is_double_cpu(), "expect double result reg");
2975 __ neg(dest->as_register_lo(), left->as_register_lo());
2976 } else if (left->is_single_fpu()) {
2977 assert(dest->is_single_fpu(), "expect single float result reg");
2978 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2979 } else {
2980 assert(left->is_double_fpu(), "expect double float operand reg");
3082 void LIR_Assembler::membar_loadload() {
3083 __ membar(Assembler::LoadLoad);
3084 }
3085
3086 void LIR_Assembler::membar_storestore() {
3087 __ membar(MacroAssembler::StoreStore);
3088 }
3089
3090 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3091
3092 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3093
3094 void LIR_Assembler::on_spin_wait() {
3095 __ spin_wait();
3096 }
3097
3098 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3099 __ mov(result_reg->as_register(), rthread);
3100 }
3101
3102 void LIR_Assembler::check_orig_pc() {
3103 __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3104 __ cmp(rscratch2, (u1)NULL_WORD);
3105 }
3106
3107 void LIR_Assembler::peephole(LIR_List *lir) {
3108 #if 0
3109 if (tableswitch_count >= max_tableswitches)
3110 return;
3111
3112 /*
3113 This finite-state automaton recognizes sequences of compare-and-
3114 branch instructions. We will turn them into a tableswitch. You
3115 could argue that C1 really shouldn't be doing this sort of
3116 optimization, but without it the code is really horrible.
3117 */
3118
3119 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3120 int first_key, last_key = -2147483648;
3121 int next_key = 0;
3122 int start_insn = -1;
3123 int last_insn = -1;
3124 Register reg = noreg;
3125 LIR_Opr reg_opr;
|