16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/assembler.hpp"
28 #include "asm/macroAssembler.inline.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "code/compiledIC.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "nativeInst_riscv.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/powerOfTwo.hpp"
44 #include "vmreg_riscv.inline.hpp"
45
46 #ifndef PRODUCT
47 #define COMMENT(x) do { __ block_comment(x); } while (0)
48 #else
49 #define COMMENT(x)
50 #endif
51
52 NEEDS_CLEANUP // remove this definitions ?
53 const Register SYNC_header = x10; // synchronization header
54 const Register SHIFT_count = x10; // where count for shift operations must be
55
56 #define __ _masm->
57
58 static void select_different_registers(Register preserve,
59 Register extra,
60 Register &tmp1,
324 MonitorExitStub* stub = nullptr;
325 if (method()->is_synchronized()) {
326 monitor_address(0, FrameMap::r10_opr);
327 stub = new MonitorExitStub(FrameMap::r10_opr, 0);
328 __ unlock_object(x15, x14, x10, x16, *stub->entry());
329 __ bind(*stub->continuation());
330 }
331
332 if (compilation()->env()->dtrace_method_probes()) {
333 __ mv(c_rarg0, xthread);
334 __ mov_metadata(c_rarg1, method()->constant_encoding());
335 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
336 }
337
338 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
339 __ mv(x10, x9); // Restore the exception
340 }
341
342 // remove the activation and dispatch to the unwind handler
343 __ block_comment("remove_frame and dispatch to the unwind handler");
344 __ remove_frame(initial_frame_size_in_bytes());
345 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
346
347 // Emit the slow path assembly
348 if (stub != nullptr) {
349 stub->emit_code(this);
350 }
351
352 return offset;
353 }
354
355 int LIR_Assembler::emit_deopt_handler() {
356 // generate code for exception handler
357 address handler_base = __ start_a_stub(deopt_handler_size());
358 if (handler_base == nullptr) {
359 // not enough space left for the handler
360 bailout("deopt handler overflow");
361 return -1;
362 }
363
364 int offset = code_offset();
365
366 Label start;
367 __ bind(start);
368
369 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
370
371 int entry_offset = __ offset();
372 __ j(start);
373
374 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
375 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
376 "out of bounds read in post-call NOP check");
377 __ end_a_stub();
378
379 return entry_offset;
380 }
381
382 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
383 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
384
385 // Pop the stack before the safepoint code
386 __ remove_frame(initial_frame_size_in_bytes());
387
388 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
389 __ reserved_stack_check();
390 }
391
392 code_stub->set_safepoint_offset(__ offset());
393 __ relocate(relocInfo::poll_return_type);
394 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
395 __ ret();
396 }
397
398 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
399 guarantee(info != nullptr, "Shouldn't be null");
400 __ get_polling_page(t0, relocInfo::poll_type);
401 add_debug_info_for_branch(info); // This isn't just debug info:
402 // it's the oop map
403 __ read_polling_page(t0, 0, relocInfo::poll_type);
404 return __ offset();
405 }
406
407 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
408 __ mv(to_reg, from_reg);
409 }
410
411 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
412
413 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
414 assert(src->is_constant(), "should not call otherwise");
415 assert(dest->is_register(), "should not call otherwise");
416 LIR_Const* c = src->as_constant_ptr();
417 address const_addr = nullptr;
419 jdouble dconst;
420
421 switch (c->type()) {
422 case T_INT:
423 assert(patch_code == lir_patch_none, "no patching handled here");
424 __ mv(dest->as_register(), c->as_jint());
425 break;
426
427 case T_ADDRESS:
428 assert(patch_code == lir_patch_none, "no patching handled here");
429 __ mv(dest->as_register(), c->as_jint());
430 break;
431
432 case T_LONG:
433 assert(patch_code == lir_patch_none, "no patching handled here");
434 __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
435 break;
436
437 case T_OBJECT:
438 case T_ARRAY:
439 if (patch_code == lir_patch_none) {
440 jobject2reg(c->as_jobject(), dest->as_register());
441 } else {
442 jobject2reg_with_patching(dest->as_register(), info);
443 }
444 break;
445
446 case T_METADATA:
447 if (patch_code != lir_patch_none) {
448 klass2reg_with_patching(dest->as_register(), info);
449 } else {
450 __ mov_metadata(dest->as_register(), c->as_metadata());
451 }
452 break;
453
454 case T_FLOAT:
455 fconst = c->as_jfloat();
456 if (MacroAssembler::can_fp_imm_load(fconst)) {
457 __ fli_s(dest->as_float_reg(), fconst);
458 } else {
459 const_addr = float_constant(fconst);
460 assert(const_addr != nullptr, "must create float constant in the constant table");
461 __ flw(dest->as_float_reg(), InternalAddress(const_addr));
462 }
826 break;
827 case T_CHAR:
828 __ lhu(dest->as_register(), as_Address(from_addr));
829 break;
830 case T_SHORT:
831 __ lh(dest->as_register(), as_Address(from_addr));
832 break;
833 default:
834 ShouldNotReachHere();
835 }
836
837 if (is_reference_type(type)) {
838 if (UseCompressedOops && !wide) {
839 __ decode_heap_oop(dest->as_register());
840 }
841
842 __ verify_oop(dest->as_register());
843 }
844 }
845
846 void LIR_Assembler::emit_op3(LIR_Op3* op) {
847 switch (op->code()) {
848 case lir_idiv: // fall through
849 case lir_irem:
850 arithmetic_idiv(op->code(),
851 op->in_opr1(),
852 op->in_opr2(),
853 op->in_opr3(),
854 op->result_opr(),
855 op->info());
856 break;
857 case lir_fmad:
858 __ fmadd_d(op->result_opr()->as_double_reg(),
859 op->in_opr1()->as_double_reg(),
860 op->in_opr2()->as_double_reg(),
861 op->in_opr3()->as_double_reg());
862 break;
863 case lir_fmaf:
864 __ fmadd_s(op->result_opr()->as_float_reg(),
865 op->in_opr1()->as_float_reg(),
980 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
981 __ mv(t1, (u1)InstanceKlass::fully_initialized);
982 add_debug_info_for_null_check_here(op->stub()->info());
983 __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
984 }
985
986 __ allocate_object(op->obj()->as_register(),
987 op->tmp1()->as_register(),
988 op->tmp2()->as_register(),
989 op->header_size(),
990 op->object_size(),
991 op->klass()->as_register(),
992 *op->stub()->entry());
993
994 __ bind(*op->stub()->continuation());
995 }
996
997 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
998 Register len = op->len()->as_register();
999
1000 if (UseSlowPath ||
1001 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1002 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1003 __ j(*op->stub()->entry());
1004 } else {
1005 Register tmp1 = op->tmp1()->as_register();
1006 Register tmp2 = op->tmp2()->as_register();
1007 Register tmp3 = op->tmp3()->as_register();
1008 if (len == tmp1) {
1009 tmp1 = tmp3;
1010 } else if (len == tmp2) {
1011 tmp2 = tmp3;
1012 } else if (len == tmp3) {
1013 // everything is ok
1014 } else {
1015 __ mv(tmp3, len);
1016 }
1017 __ allocate_array(op->obj()->as_register(),
1018 len,
1019 tmp1,
1020 tmp2,
1043 assert(*data != nullptr, "need data for type check");
1044 assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1045 }
1046
1047 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1048 Register k_RInfo, Register klass_RInfo,
1049 Label *failure_target, Label *success_target) {
1050 // get object class
1051 // not a safepoint as obj null check happens earlier
1052 __ load_klass(klass_RInfo, obj);
1053 if (k->is_loaded()) {
1054 // See if we get an immediate positive hit
1055 __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1056 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1057 __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1058 // successful cast, fall through to profile or jump
1059 } else {
1060 // See if we get an immediate positive hit
1061 __ beq(k_RInfo, t0, *success_target);
1062 // check for self
1063 __ beq(klass_RInfo, k_RInfo, *success_target);
1064
1065 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo
1066 __ sd(k_RInfo, Address(sp, 0)); // sub klass
1067 __ sd(klass_RInfo, Address(sp, wordSize)); // super klass
1068 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1069 // load result to k_RInfo
1070 __ ld(k_RInfo, Address(sp, 0));
1071 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1072 // result is a boolean
1073 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1074 // successful cast, fall through to profile or jump
1075 }
1076 } else {
1077 // perform the fast part of the checking logic
1078 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1079 // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1080 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo
1081 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
1082 __ sd(k_RInfo, Address(sp, 0)); // super klass
1083 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1084 // load result to k_RInfo
1129 ciMethodData* md = nullptr;
1130 ciProfileData* data = nullptr;
1131
1132 const bool should_profile = op->should_profile();
1133 if (should_profile) {
1134 data_check(op, &md, &data);
1135 }
1136 Label* success_target = success;
1137 Label* failure_target = failure;
1138
1139 if (obj == k_RInfo) {
1140 k_RInfo = dst;
1141 } else if (obj == klass_RInfo) {
1142 klass_RInfo = dst;
1143 }
1144 Rtmp1 = op->tmp3()->as_register();
1145 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1146
1147 assert_different_registers(obj, k_RInfo, klass_RInfo);
1148
1149 if (should_profile) {
1150 profile_object(md, data, obj, k_RInfo, klass_RInfo, obj_is_null);
1151 } else {
1152 __ beqz(obj, *obj_is_null);
1153 }
1154
1155 typecheck_loaded(op, k, k_RInfo);
1156 __ verify_oop(obj);
1157
1158 if (op->fast_check()) {
1159 // get object class
1160 // not a safepoint as obj null check happens earlier
1161 __ load_klass(t0, obj, t1);
1162 __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1163 // successful cast, fall through to profile or jump
1164 } else {
1165 typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1166 }
1167
1168 __ j(*success);
1169 }
1170
1171 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1172 const bool should_profile = op->should_profile();
1173
1174 LIR_Code code = op->code();
1175 if (code == lir_store_check) {
1176 typecheck_lir_store(op, should_profile);
1177 } else if (code == lir_checkcast) {
1178 Register obj = op->object()->as_register();
1182 __ bind(success);
1183 if (dst != obj) {
1184 __ mv(dst, obj);
1185 }
1186 } else if (code == lir_instanceof) {
1187 Register obj = op->object()->as_register();
1188 Register dst = op->result_opr()->as_register();
1189 Label success, failure, done;
1190 emit_typecheck_helper(op, &success, &failure, &failure);
1191 __ bind(failure);
1192 __ mv(dst, zr);
1193 __ j(done);
1194 __ bind(success);
1195 __ mv(dst, 1);
1196 __ bind(done);
1197 } else {
1198 ShouldNotReachHere();
1199 }
1200 }
1201
1202 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1203 Register addr;
1204 if (op->addr()->is_register()) {
1205 addr = as_reg(op->addr());
1206 } else {
1207 assert(op->addr()->is_address(), "what else?");
1208 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1209 assert(addr_ptr->disp() == 0, "need 0 disp");
1210 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1211 addr = as_reg(addr_ptr->base());
1212 }
1213 Register newval = as_reg(op->new_value());
1214 Register cmpval = as_reg(op->cmp_value());
1215
1216 if (op->code() == lir_cas_obj) {
1217 if (UseCompressedOops) {
1218 Register tmp1 = op->tmp1()->as_register();
1219 assert(op->tmp1()->is_valid(), "must be");
1220 Register tmp2 = op->tmp2()->as_register();
1221 assert(op->tmp2()->is_valid(), "must be");
1303 __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1304 } else {
1305 ShouldNotReachHere();
1306 }
1307 }
1308
1309 void LIR_Assembler::align_call(LIR_Code code) {
1310 // With RVC a call instruction may get 2-byte aligned.
1311 // The address of the call instruction needs to be 4-byte aligned to
1312 // ensure that it does not span a cache line so that it can be patched.
1313 __ align(NativeInstruction::instruction_size);
1314 }
1315
1316 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1317 Assembler::IncompressibleScope scope(_masm);
1318 address call = __ reloc_call(Address(op->addr(), rtype));
1319 if (call == nullptr) {
1320 bailout("reloc call address stub overflow");
1321 return;
1322 }
1323 add_call_info(code_offset(), op->info());
1324 __ post_call_nop();
1325 }
1326
1327 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1328 Assembler::IncompressibleScope scope(_masm);
1329 address call = __ ic_call(op->addr());
1330 if (call == nullptr) {
1331 bailout("reloc call address stub overflow");
1332 return;
1333 }
1334 add_call_info(code_offset(), op->info());
1335 __ post_call_nop();
1336 }
1337
1338 void LIR_Assembler::emit_static_call_stub() {
1339 address call_pc = __ pc();
1340 MacroAssembler::assert_alignment(call_pc);
1341 address stub = __ start_a_stub(call_stub_size());
1342 if (stub == nullptr) {
1343 bailout("static call stub overflow");
1344 return;
1345 }
1346
1347 int start = __ offset();
1348
1349 __ relocate(static_stub_Relocation::spec(call_pc));
1350 __ emit_static_call_stub();
1351
1352 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
1353 <= call_stub_size(), "stub too big");
1354 __ end_a_stub();
|
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/assembler.hpp"
28 #include "asm/macroAssembler.inline.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInlineKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "ci/ciObjArrayKlass.hpp"
39 #include "code/compiledIC.hpp"
40 #include "gc/shared/collectedHeap.hpp"
41 #include "nativeInst_riscv.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_riscv.inline.hpp"
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register SYNC_header = x10; // synchronization header
57 const Register SHIFT_count = x10; // where count for shift operations must be
58
59 #define __ _masm->
60
61 static void select_different_registers(Register preserve,
62 Register extra,
63 Register &tmp1,
327 MonitorExitStub* stub = nullptr;
328 if (method()->is_synchronized()) {
329 monitor_address(0, FrameMap::r10_opr);
330 stub = new MonitorExitStub(FrameMap::r10_opr, 0);
331 __ unlock_object(x15, x14, x10, x16, *stub->entry());
332 __ bind(*stub->continuation());
333 }
334
335 if (compilation()->env()->dtrace_method_probes()) {
336 __ mv(c_rarg0, xthread);
337 __ mov_metadata(c_rarg1, method()->constant_encoding());
338 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
339 }
340
341 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
342 __ mv(x10, x9); // Restore the exception
343 }
344
345 // remove the activation and dispatch to the unwind handler
346 __ block_comment("remove_frame and dispatch to the unwind handler");
347 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
348 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
349
350 // Emit the slow path assembly
351 if (stub != nullptr) {
352 stub->emit_code(this);
353 }
354
355 return offset;
356 }
357
358 int LIR_Assembler::emit_deopt_handler() {
359 // generate code for exception handler
360 address handler_base = __ start_a_stub(deopt_handler_size());
361 if (handler_base == nullptr) {
362 // not enough space left for the handler
363 bailout("deopt handler overflow");
364 return -1;
365 }
366
367 int offset = code_offset();
368
369 Label start;
370 __ bind(start);
371
372 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
373
374 int entry_offset = __ offset();
375 __ j(start);
376
377 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
378 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
379 "out of bounds read in post-call NOP check");
380 __ end_a_stub();
381
382 return entry_offset;
383 }
384
385 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
386 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
387
388 assert(!InlineTypeReturnedAsFields, "unimplemented");
389
390 // Pop the stack before the safepoint code
391 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
392
393 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
394 __ reserved_stack_check();
395 }
396
397 code_stub->set_safepoint_offset(__ offset());
398 __ relocate(relocInfo::poll_return_type);
399 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
400 __ ret();
401 }
402
403 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
404 Unimplemented();
405 return 0;
406 }
407
408 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
409 guarantee(info != nullptr, "Shouldn't be null");
410 __ get_polling_page(t0, relocInfo::poll_type);
411 add_debug_info_for_branch(info); // This isn't just debug info:
412 // it's the oop map
413 __ read_polling_page(t0, 0, relocInfo::poll_type);
414 return __ offset();
415 }
416
417 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
418 __ mv(to_reg, from_reg);
419 }
420
421 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
422
423 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
424 assert(src->is_constant(), "should not call otherwise");
425 assert(dest->is_register(), "should not call otherwise");
426 LIR_Const* c = src->as_constant_ptr();
427 address const_addr = nullptr;
429 jdouble dconst;
430
431 switch (c->type()) {
432 case T_INT:
433 assert(patch_code == lir_patch_none, "no patching handled here");
434 __ mv(dest->as_register(), c->as_jint());
435 break;
436
437 case T_ADDRESS:
438 assert(patch_code == lir_patch_none, "no patching handled here");
439 __ mv(dest->as_register(), c->as_jint());
440 break;
441
442 case T_LONG:
443 assert(patch_code == lir_patch_none, "no patching handled here");
444 __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
445 break;
446
447 case T_OBJECT:
448 case T_ARRAY:
449 if (patch_code != lir_patch_none) {
450 jobject2reg_with_patching(dest->as_register(), info);
451 } else {
452 jobject2reg(c->as_jobject(), dest->as_register());
453 }
454 break;
455
456 case T_METADATA:
457 if (patch_code != lir_patch_none) {
458 klass2reg_with_patching(dest->as_register(), info);
459 } else {
460 __ mov_metadata(dest->as_register(), c->as_metadata());
461 }
462 break;
463
464 case T_FLOAT:
465 fconst = c->as_jfloat();
466 if (MacroAssembler::can_fp_imm_load(fconst)) {
467 __ fli_s(dest->as_float_reg(), fconst);
468 } else {
469 const_addr = float_constant(fconst);
470 assert(const_addr != nullptr, "must create float constant in the constant table");
471 __ flw(dest->as_float_reg(), InternalAddress(const_addr));
472 }
836 break;
837 case T_CHAR:
838 __ lhu(dest->as_register(), as_Address(from_addr));
839 break;
840 case T_SHORT:
841 __ lh(dest->as_register(), as_Address(from_addr));
842 break;
843 default:
844 ShouldNotReachHere();
845 }
846
847 if (is_reference_type(type)) {
848 if (UseCompressedOops && !wide) {
849 __ decode_heap_oop(dest->as_register());
850 }
851
852 __ verify_oop(dest->as_register());
853 }
854 }
855
856 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
857 assert(dst->is_cpu_register(), "must be");
858 assert(dst->type() == src->type(), "must be");
859
860 if (src->is_cpu_register()) {
861 reg2reg(src, dst);
862 } else if (src->is_stack()) {
863 stack2reg(src, dst, dst->type());
864 } else if (src->is_constant()) {
865 const2reg(src, dst, lir_patch_none, nullptr);
866 } else {
867 ShouldNotReachHere();
868 }
869 }
870
871 void LIR_Assembler::emit_op3(LIR_Op3* op) {
872 switch (op->code()) {
873 case lir_idiv: // fall through
874 case lir_irem:
875 arithmetic_idiv(op->code(),
876 op->in_opr1(),
877 op->in_opr2(),
878 op->in_opr3(),
879 op->result_opr(),
880 op->info());
881 break;
882 case lir_fmad:
883 __ fmadd_d(op->result_opr()->as_double_reg(),
884 op->in_opr1()->as_double_reg(),
885 op->in_opr2()->as_double_reg(),
886 op->in_opr3()->as_double_reg());
887 break;
888 case lir_fmaf:
889 __ fmadd_s(op->result_opr()->as_float_reg(),
890 op->in_opr1()->as_float_reg(),
1005 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1006 __ mv(t1, (u1)InstanceKlass::fully_initialized);
1007 add_debug_info_for_null_check_here(op->stub()->info());
1008 __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
1009 }
1010
1011 __ allocate_object(op->obj()->as_register(),
1012 op->tmp1()->as_register(),
1013 op->tmp2()->as_register(),
1014 op->header_size(),
1015 op->object_size(),
1016 op->klass()->as_register(),
1017 *op->stub()->entry());
1018
1019 __ bind(*op->stub()->continuation());
1020 }
1021
1022 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1023 Register len = op->len()->as_register();
1024
1025 if (UseSlowPath || op->always_slow_path() ||
1026 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1027 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1028 __ j(*op->stub()->entry());
1029 } else {
1030 Register tmp1 = op->tmp1()->as_register();
1031 Register tmp2 = op->tmp2()->as_register();
1032 Register tmp3 = op->tmp3()->as_register();
1033 if (len == tmp1) {
1034 tmp1 = tmp3;
1035 } else if (len == tmp2) {
1036 tmp2 = tmp3;
1037 } else if (len == tmp3) {
1038 // everything is ok
1039 } else {
1040 __ mv(tmp3, len);
1041 }
1042 __ allocate_array(op->obj()->as_register(),
1043 len,
1044 tmp1,
1045 tmp2,
1068 assert(*data != nullptr, "need data for type check");
1069 assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1070 }
1071
1072 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1073 Register k_RInfo, Register klass_RInfo,
1074 Label *failure_target, Label *success_target) {
1075 // get object class
1076 // not a safepoint as obj null check happens earlier
1077 __ load_klass(klass_RInfo, obj);
1078 if (k->is_loaded()) {
1079 // See if we get an immediate positive hit
1080 __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1081 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1082 __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1083 // successful cast, fall through to profile or jump
1084 } else {
1085 // See if we get an immediate positive hit
1086 __ beq(k_RInfo, t0, *success_target);
1087 // check for self
1088 if (k->is_loaded() && k->is_obj_array_klass()) {
1089 // For a direct pointer comparison, we need the refined array klass pointer
1090 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1091 if (!k_refined->is_loaded()) {
1092 bailout("encountered unloaded_ciobjarrayklass due to out of memory error");
1093 return;
1094 }
1095 __ mov_metadata(t0, k_refined->constant_encoding());
1096 __ beq(klass_RInfo, t0, *success_target);
1097 } else {
1098 __ beq(klass_RInfo, k_RInfo, *success_target);
1099 }
1100 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo
1101 __ sd(k_RInfo, Address(sp, 0)); // sub klass
1102 __ sd(klass_RInfo, Address(sp, wordSize)); // super klass
1103 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1104 // load result to k_RInfo
1105 __ ld(k_RInfo, Address(sp, 0));
1106 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1107 // result is a boolean
1108 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1109 // successful cast, fall through to profile or jump
1110 }
1111 } else {
1112 // perform the fast part of the checking logic
1113 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1114 // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1115 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo
1116 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
1117 __ sd(k_RInfo, Address(sp, 0)); // super klass
1118 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1119 // load result to k_RInfo
1164 ciMethodData* md = nullptr;
1165 ciProfileData* data = nullptr;
1166
1167 const bool should_profile = op->should_profile();
1168 if (should_profile) {
1169 data_check(op, &md, &data);
1170 }
1171 Label* success_target = success;
1172 Label* failure_target = failure;
1173
1174 if (obj == k_RInfo) {
1175 k_RInfo = dst;
1176 } else if (obj == klass_RInfo) {
1177 klass_RInfo = dst;
1178 }
1179 Rtmp1 = op->tmp3()->as_register();
1180 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1181
1182 assert_different_registers(obj, k_RInfo, klass_RInfo);
1183
1184 if (op->need_null_check()) {
1185 if (should_profile) {
1186 profile_object(md, data, obj, k_RInfo, klass_RInfo, obj_is_null);
1187 } else {
1188 __ beqz(obj, *obj_is_null);
1189 }
1190 }
1191
1192 typecheck_loaded(op, k, k_RInfo);
1193 __ verify_oop(obj);
1194
1195 if (op->fast_check()) {
1196 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1197 // get object class
1198 // not a safepoint as obj null check happens earlier
1199 __ load_klass(t0, obj, t1);
1200 __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1201 // successful cast, fall through to profile or jump
1202 } else {
1203 typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1204 }
1205
1206 __ j(*success);
1207 }
1208
1209 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1210 const bool should_profile = op->should_profile();
1211
1212 LIR_Code code = op->code();
1213 if (code == lir_store_check) {
1214 typecheck_lir_store(op, should_profile);
1215 } else if (code == lir_checkcast) {
1216 Register obj = op->object()->as_register();
1220 __ bind(success);
1221 if (dst != obj) {
1222 __ mv(dst, obj);
1223 }
1224 } else if (code == lir_instanceof) {
1225 Register obj = op->object()->as_register();
1226 Register dst = op->result_opr()->as_register();
1227 Label success, failure, done;
1228 emit_typecheck_helper(op, &success, &failure, &failure);
1229 __ bind(failure);
1230 __ mv(dst, zr);
1231 __ j(done);
1232 __ bind(success);
1233 __ mv(dst, 1);
1234 __ bind(done);
1235 } else {
1236 ShouldNotReachHere();
1237 }
1238 }
1239
1240 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1241 // We are loading/storing from/to an array that *may* be a flat array (the
1242 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1243 // If this array is a flat array, take the slow path.
1244 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1245 }
1246
1247 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1248 // We are storing into an array that *may* be null-free (the declared type is
1249 // Object[], abstract[], interface[] or VT.ref[]).
1250 Label test_mark_word;
1251 Register tmp = op->tmp()->as_register();
1252 __ ld(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1253 __ test_bit(t0, tmp, exact_log2(markWord::unlocked_value));
1254 __ bnez(t0, test_mark_word);
1255 __ load_prototype_header(tmp, op->array()->as_register());
1256 __ bind(test_mark_word);
1257 __ test_bit(tmp, tmp, exact_log2(markWord::null_free_array_bit_in_place));
1258 }
1259
1260 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1261 Label L_oops_equal;
1262 Label L_oops_not_equal;
1263 Label L_end;
1264
1265 Register left = op->left()->as_register();
1266 Register right = op->right()->as_register();
1267
1268 __ beq(left, right, L_oops_equal);
1269
1270 // (1) Null check -- if one of the operands is null, the other must not be null (because
1271 // the two references are not equal), so they are not substitutable,
1272 __ beqz(left, L_oops_not_equal);
1273 __ beqz(right, L_oops_not_equal);
1274
1275 ciKlass* left_klass = op->left_klass();
1276 ciKlass* right_klass = op->right_klass();
1277
1278 // (2) Inline type check -- if either of the operands is not a inline type,
1279 // they are not substitutable. We do this only if we are not sure that the
1280 // operands are inline type
1281 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1282 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1283 Register tmp1 = op->tmp1()->as_register();
1284 Register tmp2 = op->tmp2()->as_register();
1285 __ mv(tmp1, markWord::inline_type_pattern);
1286 __ ld(tmp2, Address(left, oopDesc::mark_offset_in_bytes()));
1287 __ andr(tmp1, tmp1, tmp2);
1288 __ ld(tmp2, Address(right, oopDesc::mark_offset_in_bytes()));
1289 __ andr(tmp1, tmp1, tmp2);
1290 __ mv(tmp2, (u1)markWord::inline_type_pattern);
1291 __ bne(tmp1, tmp2, L_oops_not_equal);
1292 }
1293
1294 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1295 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1296 // No need to load klass -- the operands are statically known to be the same inline klass.
1297 __ j(*op->stub()->entry());
1298 } else {
1299 Register left_klass_op = op->tmp1()->as_register();
1300 Register right_klass_op = op->tmp2()->as_register();
1301 if (UseCompactObjectHeaders) {
1302 __ load_narrow_klass_compact(left_klass_op, left);
1303 __ load_narrow_klass_compact(right_klass_op, right);
1304 } else {
1305 __ lwu(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1306 __ lwu(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1307 }
1308 __ beq(left_klass_op, right_klass_op, *op->stub()->entry(), /* is_far */ true); // same klass -> do slow check
1309 // fall through to L_oops_not_equal
1310 }
1311
1312 __ bind(L_oops_not_equal);
1313 move(op->not_equal_result(), op->result_opr());
1314 __ j(L_end);
1315
1316 // We've returned from the stub. X10 contains 0x0 IFF the two
1317 // operands are not substitutable. (Don't compare against 0x1 in case the
1318 // C compiler is naughty)
1319 __ bind(*op->stub()->continuation());
1320 __ beqz(x10, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1321
1322 __ bind(L_oops_equal);
1323 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1324 // fall-through
1325 __ bind(L_end);
1326 }
1327
1328 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
1329 Register obj = op->obj()->as_register();
1330 Register tmp = op->tmp()->as_pointer_register();
1331 bool not_null = op->not_null();
1332 int flag = op->flag();
1333
1334 assert_different_registers(tmp, t0, t1);
1335
1336 Label not_inline_type;
1337 if (!not_null) {
1338 __ beqz(obj, not_inline_type);
1339 }
1340
1341 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
1342
1343 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), t1);
1344 __ lbu(tmp, mdo_addr);
1345 __ mv(t0, flag);
1346 __ orr(tmp, tmp, t0);
1347 __ sb(tmp, mdo_addr);
1348
1349 __ bind(not_inline_type);
1350 }
1351
1352 void LIR_Assembler::check_orig_pc() {
1353 Unimplemented();
1354 }
1355
1356 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1357 Register addr;
1358 if (op->addr()->is_register()) {
1359 addr = as_reg(op->addr());
1360 } else {
1361 assert(op->addr()->is_address(), "what else?");
1362 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1363 assert(addr_ptr->disp() == 0, "need 0 disp");
1364 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1365 addr = as_reg(addr_ptr->base());
1366 }
1367 Register newval = as_reg(op->new_value());
1368 Register cmpval = as_reg(op->cmp_value());
1369
1370 if (op->code() == lir_cas_obj) {
1371 if (UseCompressedOops) {
1372 Register tmp1 = op->tmp1()->as_register();
1373 assert(op->tmp1()->is_valid(), "must be");
1374 Register tmp2 = op->tmp2()->as_register();
1375 assert(op->tmp2()->is_valid(), "must be");
1457 __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1458 } else {
1459 ShouldNotReachHere();
1460 }
1461 }
1462
1463 void LIR_Assembler::align_call(LIR_Code code) {
1464 // With RVC a call instruction may get 2-byte aligned.
1465 // The address of the call instruction needs to be 4-byte aligned to
1466 // ensure that it does not span a cache line so that it can be patched.
1467 __ align(NativeInstruction::instruction_size);
1468 }
1469
1470 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1471 Assembler::IncompressibleScope scope(_masm);
1472 address call = __ reloc_call(Address(op->addr(), rtype));
1473 if (call == nullptr) {
1474 bailout("reloc call address stub overflow");
1475 return;
1476 }
1477 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
1478 __ post_call_nop();
1479 }
1480
1481 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1482 Assembler::IncompressibleScope scope(_masm);
1483 address call = __ ic_call(op->addr());
1484 if (call == nullptr) {
1485 bailout("reloc call address stub overflow");
1486 return;
1487 }
1488 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
1489 __ post_call_nop();
1490 }
1491
1492 void LIR_Assembler::emit_static_call_stub() {
1493 address call_pc = __ pc();
1494 MacroAssembler::assert_alignment(call_pc);
1495 address stub = __ start_a_stub(call_stub_size());
1496 if (stub == nullptr) {
1497 bailout("static call stub overflow");
1498 return;
1499 }
1500
1501 int start = __ offset();
1502
1503 __ relocate(static_stub_Relocation::spec(call_pc));
1504 __ emit_static_call_stub();
1505
1506 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
1507 <= call_stub_size(), "stub too big");
1508 __ end_a_stub();
|