1 /*
2 * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/assembler.hpp"
28 #include "asm/macroAssembler.inline.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInlineKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "ci/ciObjArrayKlass.hpp"
39 #include "code/compiledIC.hpp"
40 #include "gc/shared/collectedHeap.hpp"
41 #include "nativeInst_riscv.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_riscv.inline.hpp"
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register SYNC_header = x10; // synchronization header
57 const Register SHIFT_count = x10; // where count for shift operations must be
58
59 #define __ _masm->
60
61 static void select_different_registers(Register preserve,
62 Register extra,
63 Register &tmp1,
64 Register &tmp2,
65 Register &tmp3) {
66 if (tmp1 == preserve) {
67 assert_different_registers(tmp1, tmp2, tmp3, extra);
68 tmp1 = extra;
69 } else if (tmp2 == preserve) {
70 assert_different_registers(tmp1, tmp2, tmp3, extra);
71 tmp2 = extra;
72 } else if (tmp3 == preserve) {
73 assert_different_registers(tmp1, tmp2, tmp3, extra);
74 tmp3 = extra;
75 }
76 assert_different_registers(preserve, tmp1, tmp2, tmp3);
77 }
78
79 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
80
81 void LIR_Assembler::clinit_barrier(ciMethod* method) {
82 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
83 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
84
85 Label L_skip_barrier;
86
87 __ mov_metadata(t1, method->holder()->constant_encoding());
88 __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */);
89 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
90 __ bind(L_skip_barrier);
91 }
92
93 LIR_Opr LIR_Assembler::receiverOpr() {
94 return FrameMap::receiver_opr;
95 }
96
97 LIR_Opr LIR_Assembler::osrBufferPointer() {
98 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
99 }
100
101 void LIR_Assembler::breakpoint() { Unimplemented(); }
102
103 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
104
105 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
106
107 static jlong as_long(LIR_Opr data) {
108 jlong result;
109 switch (data->type()) {
110 case T_INT:
111 result = (data->as_jint());
112 break;
113 case T_LONG:
114 result = (data->as_jlong());
115 break;
116 default:
117 ShouldNotReachHere();
118 result = 0; // unreachable
119 }
120 return result;
121 }
122
123 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
124 if (addr->base()->is_illegal()) {
125 assert(addr->index()->is_illegal(), "must be illegal too");
126 __ movptr(tmp, (address)addr->disp());
127 return Address(tmp, 0);
128 }
129
130 Register base = addr->base()->as_pointer_register();
131 LIR_Opr index_opr = addr->index();
132
133 if (index_opr->is_illegal()) {
134 return Address(base, addr->disp());
135 }
136
137 int scale = addr->scale();
138 if (index_opr->is_cpu_register()) {
139 Register index;
140 if (index_opr->is_single_cpu()) {
141 index = index_opr->as_register();
142 } else {
143 index = index_opr->as_register_lo();
144 }
145 if (scale != 0) {
146 __ shadd(tmp, index, base, tmp, scale);
147 } else {
148 __ add(tmp, base, index);
149 }
150 return Address(tmp, addr->disp());
151 } else if (index_opr->is_constant()) {
152 intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp();
153 return Address(base, addr_offset);
154 }
155
156 Unimplemented();
157 return Address();
158 }
159
160 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
161 ShouldNotReachHere();
162 return Address();
163 }
164
165 Address LIR_Assembler::as_Address(LIR_Address* addr) {
166 return as_Address(addr, t0);
167 }
168
169 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
170 return as_Address(addr);
171 }
172
173 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
174 // not encodable as a base + (immediate) offset, generate an explicit address
175 // calculation to hold the address in t0.
176 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) {
177 precond(size == 4 || size == 8);
178 Address addr = frame_map()->address_for_slot(index, adjust);
179 precond(addr.getMode() == Address::base_plus_offset);
180 precond(addr.base() == sp);
181 precond(addr.offset() > 0);
182 uint mask = size - 1;
183 assert((addr.offset() & mask) == 0, "scaled offsets only");
184
185 return addr;
186 }
187
188 void LIR_Assembler::osr_entry() {
189 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
190 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
191 guarantee(osr_entry != nullptr, "null osr_entry!");
192 ValueStack* entry_state = osr_entry->state();
193 int number_of_locks = entry_state->locks_size();
194
195 // we jump here if osr happens with the interpreter
196 // state set up to continue at the beginning of the
197 // loop that triggered osr - in particular, we have
198 // the following registers setup:
199 //
200 // x12: osr buffer
201 //
202
203 //build frame
204 ciMethod* m = compilation()->method();
205 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
206
207 // OSR buffer is
208 //
209 // locals[nlocals-1..0]
210 // monitors[0..number_of_locks]
211 //
212 // locals is a direct copy of the interpreter frame so in the osr buffer
213 // so first slot in the local array is the last local from the interpreter
214 // and last slot is local[0] (receiver) from the interpreter
215 //
216 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
217 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
218 // in the interpreter frame (the method lock if a sync method)
219
220 // Initialize monitors in the compiled activation.
221 // x12: pointer to osr buffer
222 // All other registers are dead at this point and the locals will be
223 // copied into place by code emitted in the IR.
224
225 Register OSR_buf = osrBufferPointer()->as_pointer_register();
226 {
227 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
228 int monitor_offset = BytesPerWord * method()->max_locals() +
229 (2 * BytesPerWord) * (number_of_locks - 1);
230 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
231 // the OSR buffer using 2 word entries: first the lock and then
232 // the oop.
233 for (int i = 0; i < number_of_locks; i++) {
234 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
235 #ifdef ASSERT
236 // verify the interpreter's monitor has a non-null object
237 {
238 Label L;
239 __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
240 __ bnez(t0, L);
241 __ stop("locked object is null");
242 __ bind(L);
243 }
244 #endif // ASSERT
245 __ ld(x9, Address(OSR_buf, slot_offset + 0));
246 __ sd(x9, frame_map()->address_for_monitor_lock(i));
247 __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
248 __ sd(x9, frame_map()->address_for_monitor_object(i));
249 }
250 }
251 }
252
253 // inline cache check; done before the frame is built.
254 int LIR_Assembler::check_icache() {
255 return __ ic_check(CodeEntryAlignment);
256 }
257
258 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
259 if (o == nullptr) {
260 __ mv(reg, zr);
261 } else {
262 __ movoop(reg, o);
263 }
264 }
265
266 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
267 deoptimize_trap(info);
268 }
269
270 // This specifies the rsp decrement needed to build the frame
271 int LIR_Assembler::initial_frame_size_in_bytes() const {
272 // if rounding, must let FrameMap know!
273
274 return in_bytes(frame_map()->framesize_in_bytes());
275 }
276
277 int LIR_Assembler::emit_exception_handler() {
278 // generate code for exception handler
279 address handler_base = __ start_a_stub(exception_handler_size());
280 if (handler_base == nullptr) {
281 // not enough space left for the handler
282 bailout("exception handler overflow");
283 return -1;
284 }
285
286 int offset = code_offset();
287
288 // the exception oop and pc are in x10, and x13
289 // no other registers need to be preserved, so invalidate them
290 __ invalidate_registers(false, true, true, false, true, true);
291
292 // check that there is really an exception
293 __ verify_not_null_oop(x10);
294
295 // search an exception handler (x10: exception oop, x13: throwing pc)
296 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
297 __ should_not_reach_here();
298 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
299 __ end_a_stub();
300
301 return offset;
302 }
303
304 // Emit the code to remove the frame from the stack in the exception
305 // unwind path.
306 int LIR_Assembler::emit_unwind_handler() {
307 #ifndef PRODUCT
308 if (CommentedAssembly) {
309 _masm->block_comment("Unwind handler");
310 }
311 #endif // PRODUCT
312
313 int offset = code_offset();
314
315 // Fetch the exception from TLS and clear out exception related thread state
316 __ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
317 __ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
318 __ sd(zr, Address(xthread, JavaThread::exception_pc_offset()));
319
320 __ bind(_unwind_handler_entry);
321 __ verify_not_null_oop(x10);
322 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
323 __ mv(x9, x10); // Preserve the exception
324 }
325
326 // Perform needed unlocking
327 MonitorExitStub* stub = nullptr;
328 if (method()->is_synchronized()) {
329 monitor_address(0, FrameMap::r10_opr);
330 stub = new MonitorExitStub(FrameMap::r10_opr, 0);
331 __ unlock_object(x15, x14, x10, x16, *stub->entry());
332 __ bind(*stub->continuation());
333 }
334
335 if (compilation()->env()->dtrace_method_probes()) {
336 __ mv(c_rarg0, xthread);
337 __ mov_metadata(c_rarg1, method()->constant_encoding());
338 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
339 }
340
341 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
342 __ mv(x10, x9); // Restore the exception
343 }
344
345 // remove the activation and dispatch to the unwind handler
346 __ block_comment("remove_frame and dispatch to the unwind handler");
347 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
348 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
349
350 // Emit the slow path assembly
351 if (stub != nullptr) {
352 stub->emit_code(this);
353 }
354
355 return offset;
356 }
357
358 int LIR_Assembler::emit_deopt_handler() {
359 // generate code for exception handler
360 address handler_base = __ start_a_stub(deopt_handler_size());
361 if (handler_base == nullptr) {
362 // not enough space left for the handler
363 bailout("deopt handler overflow");
364 return -1;
365 }
366
367 int offset = code_offset();
368
369 Label start;
370 __ bind(start);
371
372 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
373
374 int entry_offset = __ offset();
375 __ j(start);
376
377 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
378 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
379 "out of bounds read in post-call NOP check");
380 __ end_a_stub();
381
382 return entry_offset;
383 }
384
385 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
386 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
387
388 assert(!InlineTypeReturnedAsFields, "unimplemented");
389
390 // Pop the stack before the safepoint code
391 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
392
393 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
394 __ reserved_stack_check();
395 }
396
397 code_stub->set_safepoint_offset(__ offset());
398 __ relocate(relocInfo::poll_return_type);
399 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
400 __ ret();
401 }
402
403 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
404 Unimplemented();
405 return 0;
406 }
407
408 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
409 guarantee(info != nullptr, "Shouldn't be null");
410 __ get_polling_page(t0, relocInfo::poll_type);
411 add_debug_info_for_branch(info); // This isn't just debug info:
412 // it's the oop map
413 __ read_polling_page(t0, 0, relocInfo::poll_type);
414 return __ offset();
415 }
416
417 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
418 __ mv(to_reg, from_reg);
419 }
420
421 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
422
423 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
424 assert(src->is_constant(), "should not call otherwise");
425 assert(dest->is_register(), "should not call otherwise");
426 LIR_Const* c = src->as_constant_ptr();
427 address const_addr = nullptr;
428 jfloat fconst;
429 jdouble dconst;
430
431 switch (c->type()) {
432 case T_INT:
433 assert(patch_code == lir_patch_none, "no patching handled here");
434 __ mv(dest->as_register(), c->as_jint());
435 break;
436
437 case T_ADDRESS:
438 assert(patch_code == lir_patch_none, "no patching handled here");
439 __ mv(dest->as_register(), c->as_jint());
440 break;
441
442 case T_LONG:
443 assert(patch_code == lir_patch_none, "no patching handled here");
444 __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
445 break;
446
447 case T_OBJECT:
448 case T_ARRAY:
449 if (patch_code != lir_patch_none) {
450 jobject2reg_with_patching(dest->as_register(), info);
451 } else {
452 jobject2reg(c->as_jobject(), dest->as_register());
453 }
454 break;
455
456 case T_METADATA:
457 if (patch_code != lir_patch_none) {
458 klass2reg_with_patching(dest->as_register(), info);
459 } else {
460 __ mov_metadata(dest->as_register(), c->as_metadata());
461 }
462 break;
463
464 case T_FLOAT:
465 fconst = c->as_jfloat();
466 if (MacroAssembler::can_fp_imm_load(fconst)) {
467 __ fli_s(dest->as_float_reg(), fconst);
468 } else {
469 const_addr = float_constant(fconst);
470 assert(const_addr != nullptr, "must create float constant in the constant table");
471 __ flw(dest->as_float_reg(), InternalAddress(const_addr));
472 }
473 break;
474
475 case T_DOUBLE:
476 dconst = c->as_jdouble();
477 if (MacroAssembler::can_dp_imm_load(dconst)) {
478 __ fli_d(dest->as_double_reg(), dconst);
479 } else {
480 const_addr = double_constant(c->as_jdouble());
481 assert(const_addr != nullptr, "must create double constant in the constant table");
482 __ fld(dest->as_double_reg(), InternalAddress(const_addr));
483 }
484 break;
485
486 default:
487 ShouldNotReachHere();
488 }
489 }
490
491 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
492 assert(src->is_constant(), "should not call otherwise");
493 assert(dest->is_stack(), "should not call otherwise");
494 LIR_Const* c = src->as_constant_ptr();
495 switch (c->type()) {
496 case T_OBJECT:
497 if (c->as_jobject() == nullptr) {
498 __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
499 } else {
500 const2reg(src, FrameMap::t1_opr, lir_patch_none, nullptr);
501 reg2stack(FrameMap::t1_opr, dest, c->type());
502 }
503 break;
504 case T_ADDRESS: // fall through
505 const2reg(src, FrameMap::t1_opr, lir_patch_none, nullptr);
506 reg2stack(FrameMap::t1_opr, dest, c->type());
507 case T_INT: // fall through
508 case T_FLOAT:
509 if (c->as_jint_bits() == 0) {
510 __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
511 } else {
512 __ mv(t1, c->as_jint_bits());
513 __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix()));
514 }
515 break;
516 case T_LONG: // fall through
517 case T_DOUBLE:
518 if (c->as_jlong_bits() == 0) {
519 __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
520 lo_word_offset_in_bytes));
521 } else {
522 __ mv(t1, (intptr_t)c->as_jlong_bits());
523 __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(),
524 lo_word_offset_in_bytes));
525 }
526 break;
527 default:
528 ShouldNotReachHere();
529 }
530 }
531
532 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
533 assert(src->is_constant(), "should not call otherwise");
534 assert(dest->is_address(), "should not call otherwise");
535 LIR_Const* c = src->as_constant_ptr();
536 LIR_Address* to_addr = dest->as_address_ptr();
537 void (MacroAssembler::* insn)(Register Rt, const Address &adr, Register temp);
538 switch (type) {
539 case T_ADDRESS:
540 assert(c->as_jint() == 0, "should be");
541 insn = &MacroAssembler::sd; break;
542 case T_LONG:
543 assert(c->as_jlong() == 0, "should be");
544 insn = &MacroAssembler::sd; break;
545 case T_DOUBLE:
546 assert(c->as_jdouble() == 0.0, "should be");
547 insn = &MacroAssembler::sd; break;
548 case T_INT:
549 assert(c->as_jint() == 0, "should be");
550 insn = &MacroAssembler::sw; break;
551 case T_FLOAT:
552 assert(c->as_jfloat() == 0.0f, "should be");
553 insn = &MacroAssembler::sw; break;
554 case T_OBJECT: // fall through
555 case T_ARRAY:
556 assert(c->as_jobject() == nullptr, "should be");
557 if (UseCompressedOops && !wide) {
558 insn = &MacroAssembler::sw;
559 } else {
560 insn = &MacroAssembler::sd;
561 }
562 break;
563 case T_CHAR: // fall through
564 case T_SHORT:
565 assert(c->as_jint() == 0, "should be");
566 insn = &MacroAssembler::sh;
567 break;
568 case T_BOOLEAN: // fall through
569 case T_BYTE:
570 assert(c->as_jint() == 0, "should be");
571 insn = &MacroAssembler::sb; break;
572 default:
573 ShouldNotReachHere();
574 insn = &MacroAssembler::sd; // unreachable
575 }
576 if (info != nullptr) {
577 add_debug_info_for_null_check_here(info);
578 }
579 (_masm->*insn)(zr, as_Address(to_addr), t0);
580 }
581
582 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
583 assert(src->is_register(), "should not call otherwise");
584 assert(dest->is_register(), "should not call otherwise");
585
586 // move between cpu-registers
587 if (dest->is_single_cpu()) {
588 if (src->type() == T_LONG) {
589 // Can do LONG -> OBJECT
590 move_regs(src->as_register_lo(), dest->as_register());
591 return;
592 }
593 assert(src->is_single_cpu(), "must match");
594 if (src->type() == T_OBJECT) {
595 __ verify_oop(src->as_register());
596 }
597 move_regs(src->as_register(), dest->as_register());
598 } else if (dest->is_double_cpu()) {
599 if (is_reference_type(src->type())) {
600 __ verify_oop(src->as_register());
601 move_regs(src->as_register(), dest->as_register_lo());
602 return;
603 }
604 assert(src->is_double_cpu(), "must match");
605 Register f_lo = src->as_register_lo();
606 Register f_hi = src->as_register_hi();
607 Register t_lo = dest->as_register_lo();
608 Register t_hi = dest->as_register_hi();
609 assert(f_hi == f_lo, "must be same");
610 assert(t_hi == t_lo, "must be same");
611 move_regs(f_lo, t_lo);
612 } else if (dest->is_single_fpu()) {
613 assert(src->is_single_fpu(), "expect single fpu");
614 __ fmv_s(dest->as_float_reg(), src->as_float_reg());
615 } else if (dest->is_double_fpu()) {
616 assert(src->is_double_fpu(), "expect double fpu");
617 __ fmv_d(dest->as_double_reg(), src->as_double_reg());
618 } else {
619 ShouldNotReachHere();
620 }
621 }
622
623 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
624 precond(src->is_register() && dest->is_stack());
625
626 uint const c_sz32 = sizeof(uint32_t);
627 uint const c_sz64 = sizeof(uint64_t);
628
629 assert(src->is_register(), "should not call otherwise");
630 assert(dest->is_stack(), "should not call otherwise");
631 if (src->is_single_cpu()) {
632 int index = dest->single_stack_ix();
633 if (is_reference_type(type)) {
634 __ sd(src->as_register(), stack_slot_address(index, c_sz64));
635 __ verify_oop(src->as_register());
636 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
637 __ sd(src->as_register(), stack_slot_address(index, c_sz64));
638 } else {
639 __ sw(src->as_register(), stack_slot_address(index, c_sz32));
640 }
641 } else if (src->is_double_cpu()) {
642 int index = dest->double_stack_ix();
643 Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
644 __ sd(src->as_register_lo(), dest_addr_LO);
645 } else if (src->is_single_fpu()) {
646 int index = dest->single_stack_ix();
647 __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32));
648 } else if (src->is_double_fpu()) {
649 int index = dest->double_stack_ix();
650 __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64));
651 } else {
652 ShouldNotReachHere();
653 }
654 }
655
656 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
657 LIR_Address* to_addr = dest->as_address_ptr();
658 // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src
659 Register compressed_src = t1;
660
661 if (patch_code != lir_patch_none) {
662 deoptimize_trap(info);
663 return;
664 }
665
666 if (is_reference_type(type)) {
667 __ verify_oop(src->as_register());
668
669 if (UseCompressedOops && !wide) {
670 __ encode_heap_oop(compressed_src, src->as_register());
671 } else {
672 compressed_src = src->as_register();
673 }
674 }
675
676 int null_check_here = code_offset();
677
678 switch (type) {
679 case T_FLOAT:
680 __ fsw(src->as_float_reg(), as_Address(to_addr));
681 break;
682
683 case T_DOUBLE:
684 __ fsd(src->as_double_reg(), as_Address(to_addr));
685 break;
686
687 case T_ARRAY: // fall through
688 case T_OBJECT:
689 if (UseCompressedOops && !wide) {
690 __ sw(compressed_src, as_Address(to_addr));
691 } else {
692 __ sd(compressed_src, as_Address(to_addr));
693 }
694 break;
695 case T_METADATA:
696 // We get here to store a method pointer to the stack to pass to
697 // a dtrace runtime call. This can't work on 64 bit with
698 // compressed klass ptrs: T_METADATA can be compressed klass
699 // ptr or a 64 bit method pointer.
700 ShouldNotReachHere();
701 __ sd(src->as_register(), as_Address(to_addr));
702 break;
703 case T_ADDRESS:
704 __ sd(src->as_register(), as_Address(to_addr));
705 break;
706 case T_INT:
707 __ sw(src->as_register(), as_Address(to_addr));
708 break;
709 case T_LONG:
710 __ sd(src->as_register_lo(), as_Address(to_addr));
711 break;
712 case T_BYTE: // fall through
713 case T_BOOLEAN:
714 __ sb(src->as_register(), as_Address(to_addr));
715 break;
716 case T_CHAR: // fall through
717 case T_SHORT:
718 __ sh(src->as_register(), as_Address(to_addr));
719 break;
720 default:
721 ShouldNotReachHere();
722 }
723
724 if (info != nullptr) {
725 add_debug_info_for_null_check(null_check_here, info);
726 }
727 }
728
729 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
730 precond(src->is_stack() && dest->is_register());
731
732 uint const c_sz32 = sizeof(uint32_t);
733 uint const c_sz64 = sizeof(uint64_t);
734
735 if (dest->is_single_cpu()) {
736 int index = src->single_stack_ix();
737 if (type == T_INT) {
738 __ lw(dest->as_register(), stack_slot_address(index, c_sz32));
739 } else if (is_reference_type(type)) {
740 __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
741 __ verify_oop(dest->as_register());
742 } else if (type == T_METADATA || type == T_ADDRESS) {
743 __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
744 } else {
745 __ lwu(dest->as_register(), stack_slot_address(index, c_sz32));
746 }
747 } else if (dest->is_double_cpu()) {
748 int index = src->double_stack_ix();
749 Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
750 __ ld(dest->as_register_lo(), src_addr_LO);
751 } else if (dest->is_single_fpu()) {
752 int index = src->single_stack_ix();
753 __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32));
754 } else if (dest->is_double_fpu()) {
755 int index = src->double_stack_ix();
756 __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64));
757 } else {
758 ShouldNotReachHere();
759 }
760 }
761
762 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
763 deoptimize_trap(info);
764 }
765
766 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
767 LIR_Opr temp;
768 if (type == T_LONG || type == T_DOUBLE) {
769 temp = FrameMap::t1_long_opr;
770 } else {
771 temp = FrameMap::t1_opr;
772 }
773
774 stack2reg(src, temp, src->type());
775 reg2stack(temp, dest, dest->type());
776 }
777
778 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
779 assert(src->is_address(), "should not call otherwise");
780 assert(dest->is_register(), "should not call otherwise");
781
782 LIR_Address* addr = src->as_address_ptr();
783 LIR_Address* from_addr = src->as_address_ptr();
784
785 if (addr->base()->type() == T_OBJECT) {
786 __ verify_oop(addr->base()->as_pointer_register());
787 }
788
789 if (patch_code != lir_patch_none) {
790 deoptimize_trap(info);
791 return;
792 }
793
794 if (info != nullptr) {
795 add_debug_info_for_null_check_here(info);
796 }
797
798 int null_check_here = code_offset();
799 switch (type) {
800 case T_FLOAT:
801 __ flw(dest->as_float_reg(), as_Address(from_addr));
802 break;
803 case T_DOUBLE:
804 __ fld(dest->as_double_reg(), as_Address(from_addr));
805 break;
806 case T_ARRAY: // fall through
807 case T_OBJECT:
808 if (UseCompressedOops && !wide) {
809 __ lwu(dest->as_register(), as_Address(from_addr));
810 } else {
811 __ ld(dest->as_register(), as_Address(from_addr));
812 }
813 break;
814 case T_METADATA:
815 // We get here to store a method pointer to the stack to pass to
816 // a dtrace runtime call. This can't work on 64 bit with
817 // compressed klass ptrs: T_METADATA can be a compressed klass
818 // ptr or a 64 bit method pointer.
819 ShouldNotReachHere();
820 __ ld(dest->as_register(), as_Address(from_addr));
821 break;
822 case T_ADDRESS:
823 __ ld(dest->as_register(), as_Address(from_addr));
824 break;
825 case T_INT:
826 __ lw(dest->as_register(), as_Address(from_addr));
827 break;
828 case T_LONG:
829 __ ld(dest->as_register_lo(), as_Address_lo(from_addr));
830 break;
831 case T_BYTE:
832 __ lb(dest->as_register(), as_Address(from_addr));
833 break;
834 case T_BOOLEAN:
835 __ lbu(dest->as_register(), as_Address(from_addr));
836 break;
837 case T_CHAR:
838 __ lhu(dest->as_register(), as_Address(from_addr));
839 break;
840 case T_SHORT:
841 __ lh(dest->as_register(), as_Address(from_addr));
842 break;
843 default:
844 ShouldNotReachHere();
845 }
846
847 if (is_reference_type(type)) {
848 if (UseCompressedOops && !wide) {
849 __ decode_heap_oop(dest->as_register());
850 }
851
852 __ verify_oop(dest->as_register());
853 }
854 }
855
856 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
857 assert(dst->is_cpu_register(), "must be");
858 assert(dst->type() == src->type(), "must be");
859
860 if (src->is_cpu_register()) {
861 reg2reg(src, dst);
862 } else if (src->is_stack()) {
863 stack2reg(src, dst, dst->type());
864 } else if (src->is_constant()) {
865 const2reg(src, dst, lir_patch_none, nullptr);
866 } else {
867 ShouldNotReachHere();
868 }
869 }
870
871 void LIR_Assembler::emit_op3(LIR_Op3* op) {
872 switch (op->code()) {
873 case lir_idiv: // fall through
874 case lir_irem:
875 arithmetic_idiv(op->code(),
876 op->in_opr1(),
877 op->in_opr2(),
878 op->in_opr3(),
879 op->result_opr(),
880 op->info());
881 break;
882 case lir_fmad:
883 __ fmadd_d(op->result_opr()->as_double_reg(),
884 op->in_opr1()->as_double_reg(),
885 op->in_opr2()->as_double_reg(),
886 op->in_opr3()->as_double_reg());
887 break;
888 case lir_fmaf:
889 __ fmadd_s(op->result_opr()->as_float_reg(),
890 op->in_opr1()->as_float_reg(),
891 op->in_opr2()->as_float_reg(),
892 op->in_opr3()->as_float_reg());
893 break;
894 default:
895 ShouldNotReachHere();
896 }
897 }
898
899 // Consider using cmov (Zicond)
900 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
901 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
902 Label label;
903
904 emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false,
905 /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true);
906
907 Label done;
908 move_op(opr2, result, type, lir_patch_none, nullptr,
909 false); // wide
910 __ j(done);
911 __ bind(label);
912 move_op(opr1, result, type, lir_patch_none, nullptr,
913 false); // wide
914 __ bind(done);
915 }
916
917 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
918 LIR_Condition condition = op->cond();
919 if (condition == lir_cond_always) {
920 if (op->info() != nullptr) {
921 add_debug_info_for_branch(op->info());
922 }
923 } else {
924 assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands");
925 }
926 bool is_unordered = (op->ublock() == op->block());
927 emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered);
928 }
929
930 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label,
931 bool is_far, bool is_unordered) {
932
933 if (cmp_flag == lir_cond_always) {
934 __ j(label);
935 return;
936 }
937
938 if (cmp1->is_cpu_register()) {
939 Register reg1 = as_reg(cmp1);
940 if (cmp2->is_cpu_register()) {
941 Register reg2 = as_reg(cmp2);
942 __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far);
943 } else if (cmp2->is_constant()) {
944 const2reg_helper(cmp2);
945 __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far);
946 } else {
947 ShouldNotReachHere();
948 }
949 } else if (cmp1->is_single_fpu()) {
950 assert(cmp2->is_single_fpu(), "expect single float register");
951 __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered);
952 } else if (cmp1->is_double_fpu()) {
953 assert(cmp2->is_double_fpu(), "expect double float register");
954 __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask,
955 cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered);
956 } else {
957 ShouldNotReachHere();
958 }
959 }
960
961 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
962 LIR_Opr src = op->in_opr();
963 LIR_Opr dest = op->result_opr();
964
965 switch (op->bytecode()) {
966 case Bytecodes::_i2f:
967 __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break;
968 case Bytecodes::_i2d:
969 __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break;
970 case Bytecodes::_l2d:
971 __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break;
972 case Bytecodes::_l2f:
973 __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break;
974 case Bytecodes::_f2d:
975 __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break;
976 case Bytecodes::_d2f:
977 __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break;
978 case Bytecodes::_i2c:
979 __ zext(dest->as_register(), src->as_register(), 16); break;
980 case Bytecodes::_i2l:
981 __ sext(dest->as_register_lo(), src->as_register(), 32); break;
982 case Bytecodes::_i2s:
983 __ sext(dest->as_register(), src->as_register(), 16); break;
984 case Bytecodes::_i2b:
985 __ sext(dest->as_register(), src->as_register(), 8); break;
986 case Bytecodes::_l2i:
987 __ sext(dest->as_register(), src->as_register_lo(), 32); break;
988 case Bytecodes::_d2l:
989 __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break;
990 case Bytecodes::_f2i:
991 __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break;
992 case Bytecodes::_f2l:
993 __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break;
994 case Bytecodes::_d2i:
995 __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break;
996 default:
997 ShouldNotReachHere();
998 }
999 }
1000
1001 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1002 if (op->init_check()) {
1003 __ lbu(t0, Address(op->klass()->as_register(),
1004 InstanceKlass::init_state_offset()));
1005 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1006 __ mv(t1, (u1)InstanceKlass::fully_initialized);
1007 add_debug_info_for_null_check_here(op->stub()->info());
1008 __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
1009 }
1010
1011 __ allocate_object(op->obj()->as_register(),
1012 op->tmp1()->as_register(),
1013 op->tmp2()->as_register(),
1014 op->header_size(),
1015 op->object_size(),
1016 op->klass()->as_register(),
1017 *op->stub()->entry());
1018
1019 __ bind(*op->stub()->continuation());
1020 }
1021
1022 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1023 Register len = op->len()->as_register();
1024
1025 if (UseSlowPath || op->always_slow_path() ||
1026 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1027 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1028 __ j(*op->stub()->entry());
1029 } else {
1030 Register tmp1 = op->tmp1()->as_register();
1031 Register tmp2 = op->tmp2()->as_register();
1032 Register tmp3 = op->tmp3()->as_register();
1033 if (len == tmp1) {
1034 tmp1 = tmp3;
1035 } else if (len == tmp2) {
1036 tmp2 = tmp3;
1037 } else if (len == tmp3) {
1038 // everything is ok
1039 } else {
1040 __ mv(tmp3, len);
1041 }
1042 __ allocate_array(op->obj()->as_register(),
1043 len,
1044 tmp1,
1045 tmp2,
1046 arrayOopDesc::base_offset_in_bytes(op->type()),
1047 array_element_size(op->type()),
1048 op->klass()->as_register(),
1049 *op->stub()->entry(),
1050 op->zero_array());
1051 }
1052 __ bind(*op->stub()->continuation());
1053 }
1054
1055 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md,
1056 ciProfileData *data, Register recv) {
1057 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1058 __ profile_receiver_type(recv, mdo, mdp_offset);
1059 }
1060
1061 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
1062 ciMethod* method = op->profiled_method();
1063 assert(method != nullptr, "Should have method");
1064 int bci = op->profiled_bci();
1065 *md = method->method_data_or_null();
1066 guarantee(*md != nullptr, "Sanity");
1067 *data = ((*md)->bci_to_data(bci));
1068 assert(*data != nullptr, "need data for type check");
1069 assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1070 }
1071
1072 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1073 Register k_RInfo, Register klass_RInfo,
1074 Label *failure_target, Label *success_target) {
1075 // get object class
1076 // not a safepoint as obj null check happens earlier
1077 __ load_klass(klass_RInfo, obj);
1078 if (k->is_loaded()) {
1079 // See if we get an immediate positive hit
1080 __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1081 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1082 __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1083 // successful cast, fall through to profile or jump
1084 } else {
1085 // See if we get an immediate positive hit
1086 __ beq(k_RInfo, t0, *success_target);
1087 // check for self
1088 if (k->is_loaded() && k->is_obj_array_klass()) {
1089 // For a direct pointer comparison, we need the refined array klass pointer
1090 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1091 if (!k_refined->is_loaded()) {
1092 bailout("encountered unloaded_ciobjarrayklass due to out of memory error");
1093 return;
1094 }
1095 __ mov_metadata(t0, k_refined->constant_encoding());
1096 __ beq(klass_RInfo, t0, *success_target);
1097 } else {
1098 __ beq(klass_RInfo, k_RInfo, *success_target);
1099 }
1100 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo
1101 __ sd(k_RInfo, Address(sp, 0)); // sub klass
1102 __ sd(klass_RInfo, Address(sp, wordSize)); // super klass
1103 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1104 // load result to k_RInfo
1105 __ ld(k_RInfo, Address(sp, 0));
1106 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1107 // result is a boolean
1108 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1109 // successful cast, fall through to profile or jump
1110 }
1111 } else {
1112 // perform the fast part of the checking logic
1113 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1114 // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1115 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo
1116 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
1117 __ sd(k_RInfo, Address(sp, 0)); // super klass
1118 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1119 // load result to k_RInfo
1120 __ ld(k_RInfo, Address(sp, 0));
1121 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1122 // result is a boolean
1123 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1124 // successful cast, fall thriugh to profile or jump
1125 }
1126 }
1127
1128 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj,
1129 Register k_RInfo, Register klass_RInfo, Label* obj_is_null) {
1130 Register mdo = klass_RInfo;
1131 __ mov_metadata(mdo, md->constant_encoding());
1132 Label not_null;
1133 __ bnez(obj, not_null);
1134 // Object is null, update MDO and exit
1135 Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1136 __ lbu(t0, data_addr);
1137 __ ori(t0, t0, BitData::null_seen_byte_constant());
1138 __ sb(t0, data_addr);
1139 __ j(*obj_is_null);
1140 __ bind(not_null);
1141
1142 Register recv = k_RInfo;
1143 __ load_klass(recv, obj);
1144 type_profile_helper(mdo, md, data, recv);
1145 }
1146
1147 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
1148 if (!k->is_loaded()) {
1149 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1150 } else {
1151 __ mov_metadata(k_RInfo, k->constant_encoding());
1152 }
1153 }
1154
1155 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1156 Register obj = op->object()->as_register();
1157 Register k_RInfo = op->tmp1()->as_register();
1158 Register klass_RInfo = op->tmp2()->as_register();
1159 Register dst = op->result_opr()->as_register();
1160 ciKlass* k = op->klass();
1161 Register Rtmp1 = noreg;
1162
1163 // check if it needs to be profiled
1164 ciMethodData* md = nullptr;
1165 ciProfileData* data = nullptr;
1166
1167 const bool should_profile = op->should_profile();
1168 if (should_profile) {
1169 data_check(op, &md, &data);
1170 }
1171 Label* success_target = success;
1172 Label* failure_target = failure;
1173
1174 if (obj == k_RInfo) {
1175 k_RInfo = dst;
1176 } else if (obj == klass_RInfo) {
1177 klass_RInfo = dst;
1178 }
1179 Rtmp1 = op->tmp3()->as_register();
1180 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1181
1182 assert_different_registers(obj, k_RInfo, klass_RInfo);
1183
1184 if (op->need_null_check()) {
1185 if (should_profile) {
1186 profile_object(md, data, obj, k_RInfo, klass_RInfo, obj_is_null);
1187 } else {
1188 __ beqz(obj, *obj_is_null);
1189 }
1190 }
1191
1192 typecheck_loaded(op, k, k_RInfo);
1193 __ verify_oop(obj);
1194
1195 if (op->fast_check()) {
1196 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1197 // get object class
1198 // not a safepoint as obj null check happens earlier
1199 __ load_klass(t0, obj, t1);
1200 __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1201 // successful cast, fall through to profile or jump
1202 } else {
1203 typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1204 }
1205
1206 __ j(*success);
1207 }
1208
1209 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1210 const bool should_profile = op->should_profile();
1211
1212 LIR_Code code = op->code();
1213 if (code == lir_store_check) {
1214 typecheck_lir_store(op, should_profile);
1215 } else if (code == lir_checkcast) {
1216 Register obj = op->object()->as_register();
1217 Register dst = op->result_opr()->as_register();
1218 Label success;
1219 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1220 __ bind(success);
1221 if (dst != obj) {
1222 __ mv(dst, obj);
1223 }
1224 } else if (code == lir_instanceof) {
1225 Register obj = op->object()->as_register();
1226 Register dst = op->result_opr()->as_register();
1227 Label success, failure, done;
1228 emit_typecheck_helper(op, &success, &failure, &failure);
1229 __ bind(failure);
1230 __ mv(dst, zr);
1231 __ j(done);
1232 __ bind(success);
1233 __ mv(dst, 1);
1234 __ bind(done);
1235 } else {
1236 ShouldNotReachHere();
1237 }
1238 }
1239
1240 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1241 // We are loading/storing from/to an array that *may* be a flat array (the
1242 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1243 // If this array is a flat array, take the slow path.
1244 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1245 }
1246
1247 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1248 // We are storing into an array that *may* be null-free (the declared type is
1249 // Object[], abstract[], interface[] or VT.ref[]).
1250 Label test_mark_word;
1251 Register tmp = op->tmp()->as_register();
1252 __ ld(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1253 __ test_bit(t0, tmp, exact_log2(markWord::unlocked_value));
1254 __ bnez(t0, test_mark_word);
1255 __ load_prototype_header(tmp, op->array()->as_register());
1256 __ bind(test_mark_word);
1257 __ test_bit(tmp, tmp, exact_log2(markWord::null_free_array_bit_in_place));
1258 }
1259
1260 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1261 Label L_oops_equal;
1262 Label L_oops_not_equal;
1263 Label L_end;
1264
1265 Register left = op->left()->as_register();
1266 Register right = op->right()->as_register();
1267
1268 __ beq(left, right, L_oops_equal);
1269
1270 // (1) Null check -- if one of the operands is null, the other must not be null (because
1271 // the two references are not equal), so they are not substitutable,
1272 __ beqz(left, L_oops_not_equal);
1273 __ beqz(right, L_oops_not_equal);
1274
1275 ciKlass* left_klass = op->left_klass();
1276 ciKlass* right_klass = op->right_klass();
1277
1278 // (2) Inline type check -- if either of the operands is not a inline type,
1279 // they are not substitutable. We do this only if we are not sure that the
1280 // operands are inline type
1281 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1282 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1283 Register tmp1 = op->tmp1()->as_register();
1284 Register tmp2 = op->tmp2()->as_register();
1285 __ mv(tmp1, markWord::inline_type_pattern);
1286 __ ld(tmp2, Address(left, oopDesc::mark_offset_in_bytes()));
1287 __ andr(tmp1, tmp1, tmp2);
1288 __ ld(tmp2, Address(right, oopDesc::mark_offset_in_bytes()));
1289 __ andr(tmp1, tmp1, tmp2);
1290 __ mv(tmp2, (u1)markWord::inline_type_pattern);
1291 __ bne(tmp1, tmp2, L_oops_not_equal);
1292 }
1293
1294 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1295 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1296 // No need to load klass -- the operands are statically known to be the same inline klass.
1297 __ j(*op->stub()->entry());
1298 } else {
1299 Register left_klass_op = op->tmp1()->as_register();
1300 Register right_klass_op = op->tmp2()->as_register();
1301 if (UseCompactObjectHeaders) {
1302 __ load_narrow_klass_compact(left_klass_op, left);
1303 __ load_narrow_klass_compact(right_klass_op, right);
1304 } else {
1305 __ lwu(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1306 __ lwu(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1307 }
1308 __ beq(left_klass_op, right_klass_op, *op->stub()->entry(), /* is_far */ true); // same klass -> do slow check
1309 // fall through to L_oops_not_equal
1310 }
1311
1312 __ bind(L_oops_not_equal);
1313 move(op->not_equal_result(), op->result_opr());
1314 __ j(L_end);
1315
1316 // We've returned from the stub. X10 contains 0x0 IFF the two
1317 // operands are not substitutable. (Don't compare against 0x1 in case the
1318 // C compiler is naughty)
1319 __ bind(*op->stub()->continuation());
1320 __ beqz(x10, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1321
1322 __ bind(L_oops_equal);
1323 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1324 // fall-through
1325 __ bind(L_end);
1326 }
1327
1328 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
1329 Register obj = op->obj()->as_register();
1330 Register tmp = op->tmp()->as_pointer_register();
1331 bool not_null = op->not_null();
1332 int flag = op->flag();
1333
1334 assert_different_registers(tmp, t0, t1);
1335
1336 Label not_inline_type;
1337 if (!not_null) {
1338 __ beqz(obj, not_inline_type);
1339 }
1340
1341 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
1342
1343 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), t1);
1344 __ lbu(tmp, mdo_addr);
1345 __ mv(t0, flag);
1346 __ orr(tmp, tmp, t0);
1347 __ sb(tmp, mdo_addr);
1348
1349 __ bind(not_inline_type);
1350 }
1351
1352 void LIR_Assembler::check_orig_pc() {
1353 Unimplemented();
1354 }
1355
1356 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1357 Register addr;
1358 if (op->addr()->is_register()) {
1359 addr = as_reg(op->addr());
1360 } else {
1361 assert(op->addr()->is_address(), "what else?");
1362 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1363 assert(addr_ptr->disp() == 0, "need 0 disp");
1364 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1365 addr = as_reg(addr_ptr->base());
1366 }
1367 Register newval = as_reg(op->new_value());
1368 Register cmpval = as_reg(op->cmp_value());
1369
1370 if (op->code() == lir_cas_obj) {
1371 if (UseCompressedOops) {
1372 Register tmp1 = op->tmp1()->as_register();
1373 assert(op->tmp1()->is_valid(), "must be");
1374 Register tmp2 = op->tmp2()->as_register();
1375 assert(op->tmp2()->is_valid(), "must be");
1376
1377 __ encode_heap_oop(tmp1, cmpval);
1378 cmpval = tmp1;
1379 __ encode_heap_oop(tmp2, newval);
1380 newval = tmp2;
1381 caswu(addr, newval, cmpval);
1382 } else {
1383 casl(addr, newval, cmpval);
1384 }
1385 } else if (op->code() == lir_cas_int) {
1386 casw(addr, newval, cmpval);
1387 } else {
1388 casl(addr, newval, cmpval);
1389 }
1390
1391 if (op->result_opr()->is_valid()) {
1392 assert(op->result_opr()->is_register(), "need a register");
1393 __ mv(as_reg(op->result_opr()), t0); // cas result in t0, and 0 for success
1394 }
1395 }
1396
1397 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1398 switch (code) {
1399 case lir_abs: __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break;
1400 case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break;
1401 default: ShouldNotReachHere();
1402 }
1403 }
1404
1405 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1406 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1407 Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1408 if (dst->is_single_cpu()) {
1409 Register Rdst = dst->as_register();
1410 if (right->is_constant()) {
1411 int right_const = right->as_jint();
1412 if (Assembler::is_simm12(right_const)) {
1413 logic_op_imm(Rdst, Rleft, right_const, code);
1414 __ sext(Rdst, Rdst, 32);
1415 } else {
1416 __ mv(t0, right_const);
1417 logic_op_reg32(Rdst, Rleft, t0, code);
1418 }
1419 } else {
1420 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1421 logic_op_reg32(Rdst, Rleft, Rright, code);
1422 }
1423 } else {
1424 Register Rdst = dst->as_register_lo();
1425 if (right->is_constant()) {
1426 long right_const = right->as_jlong();
1427 if (Assembler::is_simm12(right_const)) {
1428 logic_op_imm(Rdst, Rleft, right_const, code);
1429 } else {
1430 __ mv(t0, right_const);
1431 logic_op_reg(Rdst, Rleft, t0, code);
1432 }
1433 } else {
1434 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1435 logic_op_reg(Rdst, Rleft, Rright, code);
1436 }
1437 }
1438 }
1439
1440 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) {
1441 ShouldNotCallThis();
1442 }
1443
1444 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1445 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1446 bool is_unordered_less = (code == lir_ucmp_fd2i);
1447 if (left->is_single_fpu()) {
1448 __ float_cmp(true, is_unordered_less ? -1 : 1,
1449 left->as_float_reg(), right->as_float_reg(), dst->as_register());
1450 } else if (left->is_double_fpu()) {
1451 __ float_cmp(false, is_unordered_less ? -1 : 1,
1452 left->as_double_reg(), right->as_double_reg(), dst->as_register());
1453 } else {
1454 ShouldNotReachHere();
1455 }
1456 } else if (code == lir_cmp_l2i) {
1457 __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1458 } else {
1459 ShouldNotReachHere();
1460 }
1461 }
1462
1463 void LIR_Assembler::align_call(LIR_Code code) {
1464 // With RVC a call instruction may get 2-byte aligned.
1465 // The address of the call instruction needs to be 4-byte aligned to
1466 // ensure that it does not span a cache line so that it can be patched.
1467 __ align(NativeInstruction::instruction_size);
1468 }
1469
1470 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1471 Assembler::IncompressibleScope scope(_masm);
1472 address call = __ reloc_call(Address(op->addr(), rtype));
1473 if (call == nullptr) {
1474 bailout("reloc call address stub overflow");
1475 return;
1476 }
1477 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
1478 __ post_call_nop();
1479 }
1480
1481 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1482 Assembler::IncompressibleScope scope(_masm);
1483 address call = __ ic_call(op->addr());
1484 if (call == nullptr) {
1485 bailout("reloc call address stub overflow");
1486 return;
1487 }
1488 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
1489 __ post_call_nop();
1490 }
1491
1492 void LIR_Assembler::emit_static_call_stub() {
1493 address call_pc = __ pc();
1494 MacroAssembler::assert_alignment(call_pc);
1495 address stub = __ start_a_stub(call_stub_size());
1496 if (stub == nullptr) {
1497 bailout("static call stub overflow");
1498 return;
1499 }
1500
1501 int start = __ offset();
1502
1503 __ relocate(static_stub_Relocation::spec(call_pc));
1504 __ emit_static_call_stub();
1505
1506 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
1507 <= call_stub_size(), "stub too big");
1508 __ end_a_stub();
1509 }
1510
1511 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1512 assert(exceptionOop->as_register() == x10, "must match");
1513 assert(exceptionPC->as_register() == x13, "must match");
1514
1515 // exception object is not added to oop map by LinearScan
1516 // (LinearScan assumes that no oops are in fixed registers)
1517 info->add_register_oop(exceptionOop);
1518 StubId unwind_id;
1519
1520 // get current pc information
1521 // pc is only needed if the method has an exception handler, the unwind code does not need it.
1522 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
1523 // As no instructions have been generated yet for this LIR node it's
1524 // possible that an oop map already exists for the current offset.
1525 // In that case insert an dummy NOP here to ensure all oop map PCs
1526 // are unique. See JDK-8237483.
1527 __ nop();
1528 }
1529 int pc_for_athrow_offset = __ offset();
1530 InternalAddress pc_for_athrow(__ pc());
1531 __ la(exceptionPC->as_register(), pc_for_athrow);
1532 add_call_info(pc_for_athrow_offset, info); // for exception handler
1533
1534 __ verify_not_null_oop(x10);
1535 // search an exception handler (x10: exception oop, x13: throwing pc)
1536 if (compilation()->has_fpu_code()) {
1537 unwind_id = StubId::c1_handle_exception_id;
1538 } else {
1539 unwind_id = StubId::c1_handle_exception_nofpu_id;
1540 }
1541 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
1542 __ nop();
1543 }
1544
1545 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1546 assert(exceptionOop->as_register() == x10, "must match");
1547 __ j(_unwind_handler_entry);
1548 }
1549
1550 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
1551 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1552 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1553 Register count_reg = count->as_register();
1554 if (dest->is_single_cpu()) {
1555 assert (dest->type() == T_INT, "unexpected result type");
1556 assert (left->type() == T_INT, "unexpected left type");
1557 __ andi(t0, count_reg, 31); // should not shift more than 31 bits
1558 switch (code) {
1559 case lir_shl: __ sllw(dest_reg, left_reg, t0); break;
1560 case lir_shr: __ sraw(dest_reg, left_reg, t0); break;
1561 case lir_ushr: __ srlw(dest_reg, left_reg, t0); break;
1562 default: ShouldNotReachHere();
1563 }
1564 } else if (dest->is_double_cpu()) {
1565 __ andi(t0, count_reg, 63); // should not shift more than 63 bits
1566 switch (code) {
1567 case lir_shl: __ sll(dest_reg, left_reg, t0); break;
1568 case lir_shr: __ sra(dest_reg, left_reg, t0); break;
1569 case lir_ushr: __ srl(dest_reg, left_reg, t0); break;
1570 default: ShouldNotReachHere();
1571 }
1572 } else {
1573 ShouldNotReachHere();
1574 }
1575 }
1576
1577 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
1578 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1579 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1580 if (dest->is_single_cpu()) {
1581 assert (dest->type() == T_INT, "unexpected result type");
1582 assert (left->type() == T_INT, "unexpected left type");
1583 count &= 0x1f;
1584 if (count != 0) {
1585 switch (code) {
1586 case lir_shl: __ slliw(dest_reg, left_reg, count); break;
1587 case lir_shr: __ sraiw(dest_reg, left_reg, count); break;
1588 case lir_ushr: __ srliw(dest_reg, left_reg, count); break;
1589 default: ShouldNotReachHere();
1590 }
1591 } else {
1592 move_regs(left_reg, dest_reg);
1593 }
1594 } else if (dest->is_double_cpu()) {
1595 count &= 0x3f;
1596 if (count != 0) {
1597 switch (code) {
1598 case lir_shl: __ slli(dest_reg, left_reg, count); break;
1599 case lir_shr: __ srai(dest_reg, left_reg, count); break;
1600 case lir_ushr: __ srli(dest_reg, left_reg, count); break;
1601 default: ShouldNotReachHere();
1602 }
1603 } else {
1604 move_regs(left->as_register_lo(), dest->as_register_lo());
1605 }
1606 } else {
1607 ShouldNotReachHere();
1608 }
1609 }
1610
1611 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
1612 Register obj = op->obj_opr()->as_register(); // may not be an oop
1613 Register hdr = op->hdr_opr()->as_register();
1614 Register lock = op->lock_opr()->as_register();
1615 Register temp = op->scratch_opr()->as_register();
1616 if (op->code() == lir_lock) {
1617 // add debug info for NullPointerException only if one is possible
1618 int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
1619 if (op->info() != nullptr) {
1620 add_debug_info_for_null_check(null_check_offset, op->info());
1621 }
1622 } else if (op->code() == lir_unlock) {
1623 __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
1624 } else {
1625 Unimplemented();
1626 }
1627 __ bind(*op->stub()->continuation());
1628 }
1629
1630 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
1631 Register obj = op->obj()->as_pointer_register();
1632 Register result = op->result_opr()->as_pointer_register();
1633
1634 CodeEmitInfo* info = op->info();
1635 if (info != nullptr) {
1636 add_debug_info_for_null_check_here(info);
1637 }
1638
1639 __ load_klass(result, obj);
1640 }
1641
1642 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
1643 ciMethod* method = op->profiled_method();
1644 int bci = op->profiled_bci();
1645
1646 // Update counter for all call types
1647 ciMethodData* md = method->method_data_or_null();
1648 guarantee(md != nullptr, "Sanity");
1649 ciProfileData* data = md->bci_to_data(bci);
1650 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
1651 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
1652 Register mdo = op->mdo()->as_register();
1653 __ mov_metadata(mdo, md->constant_encoding());
1654 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1655 // Perform additional virtual call profiling for invokevirtual and
1656 // invokeinterface bytecodes
1657 if (op->should_profile_receiver_type()) {
1658 assert(op->recv()->is_single_cpu(), "recv must be allocated");
1659 Register recv = op->recv()->as_register();
1660 assert_different_registers(mdo, recv);
1661 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
1662 ciKlass* known_klass = op->known_holder();
1663 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
1664 // We know the type that will be seen at this call site; we can
1665 // statically update the MethodData* rather than needing to do
1666 // dynamic tests on the receiver type
1667 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
1668 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
1669 ciKlass* receiver = vc_data->receiver(i);
1670 if (known_klass->equals(receiver)) {
1671 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1672 __ increment(data_addr, DataLayout::counter_increment);
1673 return;
1674 }
1675 }
1676 // Receiver type is not found in profile data.
1677 // Fall back to runtime helper to handle the rest at runtime.
1678 __ mov_metadata(recv, known_klass->constant_encoding());
1679 } else {
1680 __ load_klass(recv, recv);
1681 }
1682 type_profile_helper(mdo, md, data, recv);
1683 } else {
1684 // Static call
1685 __ increment(counter_addr, DataLayout::counter_increment);
1686 }
1687 }
1688
1689 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
1690 __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
1691 }
1692
1693 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
1694 assert(op->crc()->is_single_cpu(), "crc must be register");
1695 assert(op->val()->is_single_cpu(), "byte value must be register");
1696 assert(op->result_opr()->is_single_cpu(), "result must be register");
1697 Register crc = op->crc()->as_register();
1698 Register val = op->val()->as_register();
1699 Register res = op->result_opr()->as_register();
1700
1701 assert_different_registers(val, crc, res);
1702 __ la(res, ExternalAddress(StubRoutines::crc_table_addr()));
1703
1704 __ notr(crc, crc); // ~crc
1705 __ zext(crc, crc, 32);
1706 __ update_byte_crc32(crc, val, res);
1707 __ notr(res, crc); // ~crc
1708 }
1709
1710 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
1711 Register tmp, Label &next, Label &none,
1712 Address mdo_addr) {
1713 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
1714 if (exact_klass != nullptr) {
1715 __ mov_metadata(tmp, exact_klass->constant_encoding());
1716 } else {
1717 __ load_klass(tmp, tmp);
1718 }
1719
1720 __ ld(t1, mdo_addr);
1721 __ xorr(tmp, tmp, t1);
1722 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1723 // klass seen before, nothing to do. The unknown bit may have been
1724 // set already but no need to check.
1725 __ beqz(t0, next);
1726
1727 // already unknown. Nothing to do anymore.
1728 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown));
1729 __ bnez(t0, next);
1730
1731 if (TypeEntries::is_type_none(current_klass)) {
1732 __ beqz(t1, none);
1733 __ mv(t0, (u1)TypeEntries::null_seen);
1734 __ beq(t0, t1, none);
1735 // There is a chance that the checks above
1736 // fail if another thread has just set the
1737 // profiling to this obj's klass
1738 __ membar(MacroAssembler::LoadLoad);
1739 __ xorr(tmp, tmp, t1); // get back original value before XOR
1740 __ ld(t1, mdo_addr);
1741 __ xorr(tmp, tmp, t1);
1742 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1743 __ beqz(t0, next);
1744 }
1745 } else {
1746 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
1747 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
1748
1749 __ ld(tmp, mdo_addr);
1750 // already unknown. Nothing to do anymore.
1751 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown));
1752 __ bnez(t0, next);
1753 }
1754
1755 // different than before. Cannot keep accurate profile.
1756 __ ld(t1, mdo_addr);
1757 __ ori(t1, t1, TypeEntries::type_unknown);
1758 __ sd(t1, mdo_addr);
1759
1760 if (TypeEntries::is_type_none(current_klass)) {
1761 __ j(next);
1762
1763 __ bind(none);
1764 // first time here. Set profile type.
1765 __ sd(tmp, mdo_addr);
1766 #ifdef ASSERT
1767 __ andi(tmp, tmp, TypeEntries::type_mask);
1768 __ verify_klass_ptr(tmp);
1769 #endif
1770 }
1771 }
1772
1773 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp,
1774 Address mdo_addr, Label &next) {
1775 // There's a single possible klass at this profile point
1776 assert(exact_klass != nullptr, "should be");
1777 if (TypeEntries::is_type_none(current_klass)) {
1778 __ mov_metadata(tmp, exact_klass->constant_encoding());
1779 __ ld(t1, mdo_addr);
1780 __ xorr(tmp, tmp, t1);
1781 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1782 __ beqz(t0, next);
1783 #ifdef ASSERT
1784 {
1785 Label ok;
1786 __ ld(t0, mdo_addr);
1787 __ beqz(t0, ok);
1788 __ mv(t1, (u1)TypeEntries::null_seen);
1789 __ beq(t0, t1, ok);
1790 // may have been set by another thread
1791 __ membar(MacroAssembler::LoadLoad);
1792 __ mov_metadata(t0, exact_klass->constant_encoding());
1793 __ ld(t1, mdo_addr);
1794 __ xorr(t1, t0, t1);
1795 __ andi(t1, t1, TypeEntries::type_mask);
1796 __ beqz(t1, ok);
1797
1798 __ stop("unexpected profiling mismatch");
1799 __ bind(ok);
1800 }
1801 #endif
1802 // first time here. Set profile type.
1803 __ sd(tmp, mdo_addr);
1804 #ifdef ASSERT
1805 __ andi(tmp, tmp, TypeEntries::type_mask);
1806 __ verify_klass_ptr(tmp);
1807 #endif
1808 } else {
1809 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
1810 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
1811
1812 __ ld(tmp, mdo_addr);
1813 // already unknown. Nothing to do anymore.
1814 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown));
1815 __ bnez(t0, next);
1816
1817 __ ori(tmp, tmp, TypeEntries::type_unknown);
1818 __ sd(tmp, mdo_addr);
1819 }
1820 }
1821
1822 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass,
1823 Address mdo_addr, bool do_update, Label &next) {
1824 __ bnez(tmp, update);
1825 if (!TypeEntries::was_null_seen(current_klass)) {
1826 __ ld(t1, mdo_addr);
1827 __ ori(t1, t1, TypeEntries::null_seen);
1828 __ sd(t1, mdo_addr);
1829 }
1830 if (do_update) {
1831 __ j(next);
1832 }
1833 }
1834
1835 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
1836 COMMENT("emit_profile_type {");
1837 Register obj = op->obj()->as_register();
1838 Register tmp = op->tmp()->as_pointer_register();
1839 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
1840 ciKlass* exact_klass = op->exact_klass();
1841 intptr_t current_klass = op->current_klass();
1842 bool not_null = op->not_null();
1843 bool no_conflict = op->no_conflict();
1844
1845 Label update, next, none;
1846
1847 bool do_null = !not_null;
1848 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
1849 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
1850
1851 assert(do_null || do_update, "why are we here?");
1852 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
1853 assert_different_registers(tmp, t0, t1, mdo_addr.base());
1854
1855 __ verify_oop(obj);
1856
1857 if (tmp != obj) {
1858 __ mv(tmp, obj);
1859 }
1860 if (do_null) {
1861 check_null(tmp, update, current_klass, mdo_addr, do_update, next);
1862 #ifdef ASSERT
1863 } else {
1864 __ bnez(tmp, update);
1865 __ stop("unexpected null obj");
1866 #endif
1867 }
1868
1869 __ bind(update);
1870
1871 if (do_update) {
1872 #ifdef ASSERT
1873 if (exact_klass != nullptr) {
1874 check_exact_klass(tmp, exact_klass);
1875 }
1876 #endif
1877 if (!no_conflict) {
1878 check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr);
1879 } else {
1880 check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next);
1881 }
1882
1883 __ bind(next);
1884 }
1885 COMMENT("} emit_profile_type");
1886 }
1887
1888 void LIR_Assembler::align_backward_branch_target() { }
1889
1890 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
1891 // tmp must be unused
1892 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
1893
1894 if (left->is_single_cpu()) {
1895 assert(dest->is_single_cpu(), "expect single result reg");
1896 __ negw(dest->as_register(), left->as_register());
1897 } else if (left->is_double_cpu()) {
1898 assert(dest->is_double_cpu(), "expect double result reg");
1899 __ neg(dest->as_register_lo(), left->as_register_lo());
1900 } else if (left->is_single_fpu()) {
1901 assert(dest->is_single_fpu(), "expect single float result reg");
1902 __ fneg_s(dest->as_float_reg(), left->as_float_reg());
1903 } else {
1904 assert(left->is_double_fpu(), "expect double float operand reg");
1905 assert(dest->is_double_fpu(), "expect double float result reg");
1906 __ fneg_d(dest->as_double_reg(), left->as_double_reg());
1907 }
1908 }
1909
1910
1911 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1912 if (patch_code != lir_patch_none) {
1913 deoptimize_trap(info);
1914 return;
1915 }
1916
1917 LIR_Address* adr = addr->as_address_ptr();
1918 Register dst = dest->as_pointer_register();
1919
1920 assert_different_registers(dst, t0);
1921 if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) {
1922 int scale = adr->scale();
1923 intptr_t offset = adr->disp();
1924 LIR_Opr index_op = adr->index();
1925 if (index_op->is_constant()) {
1926 offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale;
1927 }
1928
1929 if (!Assembler::is_simm12(offset)) {
1930 __ la(t0, as_Address(adr));
1931 __ mv(dst, t0);
1932 return;
1933 }
1934 }
1935
1936 __ la(dst, as_Address(adr));
1937 }
1938
1939
1940 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
1941 assert(!tmp->is_valid(), "don't need temporary");
1942
1943 Assembler::IncompressibleScope scope(_masm);
1944 // Post call nops must be natural aligned due to cmodx rules.
1945 align_call(lir_rtcall);
1946
1947 __ rt_call(dest);
1948
1949 if (info != nullptr) {
1950 add_call_info_here(info);
1951 }
1952 __ post_call_nop();
1953 }
1954
1955 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
1956 if (dest->is_address() || src->is_address()) {
1957 move_op(src, dest, type, lir_patch_none, info, /* wide */ false);
1958 } else {
1959 ShouldNotReachHere();
1960 }
1961 }
1962
1963 #ifdef ASSERT
1964 // emit run-time assertion
1965 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
1966 assert(op->code() == lir_assert, "must be");
1967
1968 Label ok;
1969 if (op->in_opr1()->is_valid()) {
1970 assert(op->in_opr2()->is_valid(), "both operands must be valid");
1971 bool is_unordered = false;
1972 LIR_Condition cond = op->condition();
1973 emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false,
1974 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true);
1975 } else {
1976 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
1977 assert(op->condition() == lir_cond_always, "no other conditions allowed");
1978 }
1979
1980 if (op->halt()) {
1981 const char* str = __ code_string(op->msg());
1982 __ stop(str);
1983 } else {
1984 breakpoint();
1985 }
1986 __ bind(ok);
1987 }
1988 #endif
1989
1990 #ifndef PRODUCT
1991 #define COMMENT(x) do { __ block_comment(x); } while (0)
1992 #else
1993 #define COMMENT(x)
1994 #endif
1995
1996 void LIR_Assembler::membar() {
1997 COMMENT("membar");
1998 __ membar(MacroAssembler::AnyAny);
1999 }
2000
2001 void LIR_Assembler::membar_acquire() {
2002 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2003 }
2004
2005 void LIR_Assembler::membar_release() {
2006 __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
2007 }
2008
2009 void LIR_Assembler::membar_loadload() {
2010 __ membar(MacroAssembler::LoadLoad);
2011 }
2012
2013 void LIR_Assembler::membar_storestore() {
2014 __ membar(MacroAssembler::StoreStore);
2015 }
2016
2017 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2018
2019 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2020
2021 void LIR_Assembler::on_spin_wait() {
2022 __ pause();
2023 }
2024
2025 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2026 __ mv(result_reg->as_register(), xthread);
2027 }
2028
2029 void LIR_Assembler::peephole(LIR_List *lir) {}
2030
2031 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
2032 Address addr = as_Address(src->as_address_ptr());
2033 BasicType type = src->type();
2034 bool is_oop = is_reference_type(type);
2035
2036 get_op(type);
2037
2038 switch (code) {
2039 case lir_xadd:
2040 {
2041 RegisterOrConstant inc;
2042 Register tmp = as_reg(tmp_op);
2043 Register dst = as_reg(dest);
2044 if (data->is_constant()) {
2045 inc = RegisterOrConstant(as_long(data));
2046 assert_different_registers(dst, addr.base(), tmp);
2047 assert_different_registers(tmp, t0);
2048 } else {
2049 inc = RegisterOrConstant(as_reg(data));
2050 assert_different_registers(inc.as_register(), dst, addr.base(), tmp);
2051 }
2052 __ la(tmp, addr);
2053 (_masm->*add)(dst, inc, tmp);
2054 break;
2055 }
2056 case lir_xchg:
2057 {
2058 Register tmp = tmp_op->as_register();
2059 Register obj = as_reg(data);
2060 Register dst = as_reg(dest);
2061 if (is_oop && UseCompressedOops) {
2062 __ encode_heap_oop(t0, obj);
2063 obj = t0;
2064 }
2065 assert_different_registers(obj, addr.base(), tmp);
2066 assert_different_registers(dst, addr.base(), tmp);
2067 __ la(tmp, addr);
2068 (_masm->*xchg)(dst, obj, tmp);
2069 if (is_oop && UseCompressedOops) {
2070 __ decode_heap_oop(dst);
2071 }
2072 }
2073 break;
2074 default:
2075 ShouldNotReachHere();
2076 }
2077 __ membar(MacroAssembler::AnyAny);
2078 }
2079
2080 int LIR_Assembler::array_element_size(BasicType type) const {
2081 int elem_size = type2aelembytes(type);
2082 return exact_log2(elem_size);
2083 }
2084
2085 // helper functions which checks for overflow and sets bailout if it
2086 // occurs. Always returns a valid embeddable pointer but in the
2087 // bailout case the pointer won't be to unique storage.
2088 address LIR_Assembler::float_constant(float f) {
2089 address const_addr = __ float_constant(f);
2090 if (const_addr == nullptr) {
2091 bailout("const section overflow");
2092 return __ code()->consts()->start();
2093 } else {
2094 return const_addr;
2095 }
2096 }
2097
2098 address LIR_Assembler::double_constant(double d) {
2099 address const_addr = __ double_constant(d);
2100 if (const_addr == nullptr) {
2101 bailout("const section overflow");
2102 return __ code()->consts()->start();
2103 } else {
2104 return const_addr;
2105 }
2106 }
2107
2108 address LIR_Assembler::int_constant(jlong n) {
2109 address const_addr = __ long_constant(n);
2110 if (const_addr == nullptr) {
2111 bailout("const section overflow");
2112 return __ code()->consts()->start();
2113 } else {
2114 return const_addr;
2115 }
2116 }
2117
2118 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
2119 __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */,
2120 Assembler::rl /* release */, t0, true /* result as bool */);
2121 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2122 __ membar(MacroAssembler::AnyAny);
2123 }
2124
2125 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) {
2126 __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */,
2127 Assembler::rl /* release */, t0, true /* result as bool */);
2128 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2129 __ membar(MacroAssembler::AnyAny);
2130 }
2131
2132 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
2133 __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */,
2134 Assembler::rl /* release */, t0, true /* result as bool */);
2135 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2136 __ membar(MacroAssembler::AnyAny);
2137 }
2138
2139 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
2140 address target = nullptr;
2141
2142 switch (patching_id(info)) {
2143 case PatchingStub::access_field_id:
2144 target = Runtime1::entry_for(StubId::c1_access_field_patching_id);
2145 break;
2146 case PatchingStub::load_klass_id:
2147 target = Runtime1::entry_for(StubId::c1_load_klass_patching_id);
2148 break;
2149 case PatchingStub::load_mirror_id:
2150 target = Runtime1::entry_for(StubId::c1_load_mirror_patching_id);
2151 break;
2152 case PatchingStub::load_appendix_id:
2153 target = Runtime1::entry_for(StubId::c1_load_appendix_patching_id);
2154 break;
2155 default: ShouldNotReachHere();
2156 }
2157
2158 __ far_call(RuntimeAddress(target));
2159 add_call_info_here(info);
2160 }
2161
2162 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) {
2163 Label ok;
2164 __ load_klass(tmp, tmp);
2165 __ mov_metadata(t0, exact_klass->constant_encoding());
2166 __ beq(tmp, t0, ok);
2167 __ stop("exact klass and actual klass differ");
2168 __ bind(ok);
2169 }
2170
2171 void LIR_Assembler::get_op(BasicType type) {
2172 switch (type) {
2173 case T_INT:
2174 xchg = &MacroAssembler::atomic_xchgalw;
2175 add = &MacroAssembler::atomic_addalw;
2176 break;
2177 case T_LONG:
2178 xchg = &MacroAssembler::atomic_xchgal;
2179 add = &MacroAssembler::atomic_addal;
2180 break;
2181 case T_OBJECT:
2182 case T_ARRAY:
2183 if (UseCompressedOops) {
2184 xchg = &MacroAssembler::atomic_xchgalwu;
2185 add = &MacroAssembler::atomic_addalw;
2186 } else {
2187 xchg = &MacroAssembler::atomic_xchgal;
2188 add = &MacroAssembler::atomic_addal;
2189 }
2190 break;
2191 default:
2192 ShouldNotReachHere();
2193 }
2194 }
2195
2196 // emit_opTypeCheck sub functions
2197 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) {
2198 Register value = op->object()->as_register();
2199 Register array = op->array()->as_register();
2200 Register k_RInfo = op->tmp1()->as_register();
2201 Register klass_RInfo = op->tmp2()->as_register();
2202 Register Rtmp1 = op->tmp3()->as_register();
2203
2204 CodeStub* stub = op->stub();
2205
2206 // check if it needs to be profiled
2207 ciMethodData* md = nullptr;
2208 ciProfileData* data = nullptr;
2209
2210 if (should_profile) {
2211 data_check(op, &md, &data);
2212 }
2213 Label done;
2214 Label* success_target = &done;
2215 Label* failure_target = stub->entry();
2216
2217 if (should_profile) {
2218 profile_object(md, data, value, k_RInfo, klass_RInfo, &done);
2219 } else {
2220 __ beqz(value, done);
2221 }
2222
2223 add_debug_info_for_null_check_here(op->info_for_exception());
2224 __ load_klass(k_RInfo, array);
2225 __ load_klass(klass_RInfo, value);
2226
2227 lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target);
2228
2229 __ bind(done);
2230 }
2231
2232 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1,
2233 Label* success_target, Label* failure_target) {
2234 // get instance klass (it's already uncompressed)
2235 __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2236 // perform the fast part of the checking logic
2237 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
2238 // call out-of-line instance of __ check_klass_subtype_slow_path(...)
2239 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo
2240 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
2241 __ sd(k_RInfo, Address(sp, 0)); // super klass
2242 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2243 // load result to k_RInfo
2244 __ ld(k_RInfo, Address(sp, 0));
2245 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
2246 // result is a boolean
2247 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
2248 }
2249
2250 void LIR_Assembler::const2reg_helper(LIR_Opr src) {
2251 switch (src->as_constant_ptr()->type()) {
2252 case T_INT:
2253 case T_ADDRESS:
2254 case T_OBJECT:
2255 case T_ARRAY:
2256 case T_METADATA:
2257 const2reg(src, FrameMap::t0_opr, lir_patch_none, nullptr);
2258 break;
2259 case T_LONG:
2260 const2reg(src, FrameMap::t0_long_opr, lir_patch_none, nullptr);
2261 break;
2262 case T_FLOAT:
2263 case T_DOUBLE:
2264 default:
2265 ShouldNotReachHere();
2266 }
2267 }
2268
2269 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) {
2270 switch (code) {
2271 case lir_logic_and: __ andrw(dst, left, right); break;
2272 case lir_logic_or: __ orrw (dst, left, right); break;
2273 case lir_logic_xor: __ xorrw(dst, left, right); break;
2274 default: ShouldNotReachHere();
2275 }
2276 }
2277
2278 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) {
2279 switch (code) {
2280 case lir_logic_and: __ andr(dst, left, right); break;
2281 case lir_logic_or: __ orr (dst, left, right); break;
2282 case lir_logic_xor: __ xorr(dst, left, right); break;
2283 default: ShouldNotReachHere();
2284 }
2285 }
2286
2287 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) {
2288 switch (code) {
2289 case lir_logic_and: __ andi(dst, left, right); break;
2290 case lir_logic_or: __ ori (dst, left, right); break;
2291 case lir_logic_xor: __ xori(dst, left, right); break;
2292 default: ShouldNotReachHere();
2293 }
2294 }
2295
2296 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2297 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2298 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2299 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2300 __ sd(r, Address(sp, offset_from_rsp_in_bytes));
2301 }
2302
2303 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2304 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2305 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2306 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2307 __ mv(t0, c);
2308 __ sd(t0, Address(sp, offset_from_rsp_in_bytes));
2309 }
2310
2311 #undef __