1 /*
2 * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "c1/c1_CodeStubs.hpp"
31 #include "c1/c1_Compilation.hpp"
32 #include "c1/c1_LIRAssembler.hpp"
33 #include "c1/c1_MacroAssembler.hpp"
34 #include "c1/c1_Runtime1.hpp"
35 #include "c1/c1_ValueStack.hpp"
36 #include "ci/ciArrayKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "nativeInst_riscv.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_riscv.inline.hpp"
46
47 #ifndef PRODUCT
48 #define COMMENT(x) do { __ block_comment(x); } while (0)
49 #else
50 #define COMMENT(x)
51 #endif
52
53 NEEDS_CLEANUP // remove this definitions ?
54 const Register IC_Klass = t1; // where the IC klass is cached
55 const Register SYNC_header = x10; // synchronization header
56 const Register SHIFT_count = x10; // where count for shift operations must be
57
58 #define __ _masm->
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74 static void select_different_registers(Register preserve,
75 Register extra,
76 Register &tmp1,
77 Register &tmp2,
78 Register &tmp3) {
79 if (tmp1 == preserve) {
80 assert_different_registers(tmp1, tmp2, tmp3, extra);
81 tmp1 = extra;
82 } else if (tmp2 == preserve) {
83 assert_different_registers(tmp1, tmp2, tmp3, extra);
84 tmp2 = extra;
85 } else if (tmp3 == preserve) {
86 assert_different_registers(tmp1, tmp2, tmp3, extra);
87 tmp3 = extra;
88 }
89 assert_different_registers(preserve, tmp1, tmp2, tmp3);
90 }
91
92 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
93
94 void LIR_Assembler::clinit_barrier(ciMethod* method) {
95 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
96 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
97
98 Label L_skip_barrier;
99
100 __ mov_metadata(t1, method->holder()->constant_encoding());
101 __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */);
102 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
103 __ bind(L_skip_barrier);
104 }
105
106 LIR_Opr LIR_Assembler::receiverOpr() {
107 return FrameMap::receiver_opr;
108 }
109
110 LIR_Opr LIR_Assembler::osrBufferPointer() {
111 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
112 }
113
114 void LIR_Assembler::breakpoint() { Unimplemented(); }
115
116 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
117
118 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
119
120 static jlong as_long(LIR_Opr data) {
121 jlong result;
122 switch (data->type()) {
123 case T_INT:
124 result = (data->as_jint());
125 break;
126 case T_LONG:
127 result = (data->as_jlong());
128 break;
129 default:
130 ShouldNotReachHere();
131 result = 0; // unreachable
132 }
133 return result;
134 }
135
136 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
137 if (addr->base()->is_illegal()) {
138 assert(addr->index()->is_illegal(), "must be illegal too");
139 __ movptr(tmp, addr->disp());
140 return Address(tmp, 0);
141 }
142
143 Register base = addr->base()->as_pointer_register();
144 LIR_Opr index_opr = addr->index();
145
146 if (index_opr->is_illegal()) {
147 return Address(base, addr->disp());
148 }
149
150 int scale = addr->scale();
151 if (index_opr->is_cpu_register()) {
152 Register index;
153 if (index_opr->is_single_cpu()) {
154 index = index_opr->as_register();
155 } else {
156 index = index_opr->as_register_lo();
157 }
158 if (scale != 0) {
159 __ shadd(tmp, index, base, tmp, scale);
160 } else {
161 __ add(tmp, base, index);
162 }
163 return Address(tmp, addr->disp());
164 } else if (index_opr->is_constant()) {
165 intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp();
166 return Address(base, addr_offset);
167 }
168
169 Unimplemented();
170 return Address();
171 }
172
173 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
174 ShouldNotReachHere();
175 return Address();
176 }
177
178 Address LIR_Assembler::as_Address(LIR_Address* addr) {
179 return as_Address(addr, t0);
180 }
181
182 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
183 return as_Address(addr);
184 }
185
186 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
187 // not encodable as a base + (immediate) offset, generate an explicit address
188 // calculation to hold the address in t0.
189 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) {
190 precond(size == 4 || size == 8);
191 Address addr = frame_map()->address_for_slot(index, adjust);
192 precond(addr.getMode() == Address::base_plus_offset);
193 precond(addr.base() == sp);
194 precond(addr.offset() > 0);
195 uint mask = size - 1;
196 assert((addr.offset() & mask) == 0, "scaled offsets only");
197
198 return addr;
199 }
200
201 void LIR_Assembler::osr_entry() {
202 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
203 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
204 guarantee(osr_entry != NULL, "NULL osr_entry!");
205 ValueStack* entry_state = osr_entry->state();
206 int number_of_locks = entry_state->locks_size();
207
208 // we jump here if osr happens with the interpreter
209 // state set up to continue at the beginning of the
210 // loop that triggered osr - in particular, we have
211 // the following registers setup:
212 //
213 // x12: osr buffer
214 //
215
216 //build frame
217 ciMethod* m = compilation()->method();
218 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes(), compilation()->max_monitors());
219
220 // OSR buffer is
221 //
222 // locals[nlocals-1..0]
223 // monitors[0..number_of_locks]
224 //
225 // locals is a direct copy of the interpreter frame so in the osr buffer
226 // so first slot in the local array is the last local from the interpreter
227 // and last slot is local[0] (receiver) from the interpreter
228 //
229 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
230 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
231 // in the interpreter frame (the method lock if a sync method)
232
233 // Initialize monitors in the compiled activation.
234 // x12: pointer to osr buffer
235 // All other registers are dead at this point and the locals will be
236 // copied into place by code emitted in the IR.
237
238 Register OSR_buf = osrBufferPointer()->as_pointer_register();
239 {
240 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
241 int monitor_offset = BytesPerWord * method()->max_locals() +
242 (2 * BytesPerWord) * (number_of_locks - 1);
243 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
244 // the OSR buffer using 2 word entries: first the lock and then
245 // the oop.
246 for (int i = 0; i < number_of_locks; i++) {
247 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
248 #ifdef ASSERT
249 // verify the interpreter's monitor has a non-null object
250 {
251 Label L;
252 __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
253 __ bnez(t0, L);
254 __ stop("locked object is NULL");
255 __ bind(L);
256 }
257 #endif // ASSERT
258 __ ld(x9, Address(OSR_buf, slot_offset + 0));
259 __ sd(x9, frame_map()->address_for_monitor_lock(i));
260 __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
261 __ sd(x9, frame_map()->address_for_monitor_object(i));
262 }
263 }
264 }
265
266 // inline cache check; done before the frame is built.
267 int LIR_Assembler::check_icache() {
268 Register receiver = FrameMap::receiver_opr->as_register();
269 Register ic_klass = IC_Klass;
270 int start_offset = __ offset();
271 Label dont;
272 __ inline_cache_check(receiver, ic_klass, dont);
273
274 // if icache check fails, then jump to runtime routine
275 // Note: RECEIVER must still contain the receiver!
276 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
277
278 // We align the verified entry point unless the method body
279 // (including its inline cache check) will fit in a single 64-byte
280 // icache line.
281 if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
282 // force alignment after the cache check.
283 __ align(CodeEntryAlignment);
284 }
285
286 __ bind(dont);
287 return start_offset;
288 }
289
290 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
291 if (o == NULL) {
292 __ mv(reg, zr);
293 } else {
294 __ movoop(reg, o);
295 }
296 }
297
298 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
299 deoptimize_trap(info);
300 }
301
302 // This specifies the rsp decrement needed to build the frame
303 int LIR_Assembler::initial_frame_size_in_bytes() const {
304 // if rounding, must let FrameMap know!
305
306 return in_bytes(frame_map()->framesize_in_bytes());
307 }
308
309 int LIR_Assembler::emit_exception_handler() {
310 // generate code for exception handler
311 address handler_base = __ start_a_stub(exception_handler_size());
312 if (handler_base == NULL) {
313 // not enough space left for the handler
314 bailout("exception handler overflow");
315 return -1;
316 }
317
318 int offset = code_offset();
319
320 // the exception oop and pc are in x10, and x13
321 // no other registers need to be preserved, so invalidate them
322 __ invalidate_registers(false, true, true, false, true, true);
323
324 // check that there is really an exception
325 __ verify_not_null_oop(x10);
326
327 // search an exception handler (x10: exception oop, x13: throwing pc)
328 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
329 __ should_not_reach_here();
330 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
331 __ end_a_stub();
332
333 return offset;
334 }
335
336 // Emit the code to remove the frame from the stack in the exception
337 // unwind path.
338 int LIR_Assembler::emit_unwind_handler() {
339 #ifndef PRODUCT
340 if (CommentedAssembly) {
341 _masm->block_comment("Unwind handler");
342 }
343 #endif // PRODUCT
344
345 int offset = code_offset();
346
347 // Fetch the exception from TLS and clear out exception related thread state
348 __ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
349 __ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
350 __ sd(zr, Address(xthread, JavaThread::exception_pc_offset()));
351
352 __ bind(_unwind_handler_entry);
353 __ verify_not_null_oop(x10);
354 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
355 __ mv(x9, x10); // Preserve the exception
356 }
357
358 // Perform needed unlocking
359 MonitorExitStub* stub = NULL;
360 if (method()->is_synchronized()) {
361 monitor_address(0, FrameMap::r10_opr);
362 stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
363 if (UseHeavyMonitors) {
364 __ j(*stub->entry());
365 } else {
366 __ unlock_object(x15, x14, x10, *stub->entry());
367 }
368 __ bind(*stub->continuation());
369 }
370
371 if (compilation()->env()->dtrace_method_probes()) {
372 __ mv(c_rarg0, xthread);
373 __ mov_metadata(c_rarg1, method()->constant_encoding());
374 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
375 }
376
377 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
378 __ mv(x10, x9); // Restore the exception
379 }
380
381 // remove the activation and dispatch to the unwind handler
382 __ block_comment("remove_frame and dispatch to the unwind handler");
383 __ remove_frame(initial_frame_size_in_bytes());
384 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
385
386 // Emit the slow path assembly
387 if (stub != NULL) {
388 stub->emit_code(this);
389 }
390
391 return offset;
392 }
393
394 int LIR_Assembler::emit_deopt_handler() {
395 // generate code for exception handler
396 address handler_base = __ start_a_stub(deopt_handler_size());
397 if (handler_base == NULL) {
398 // not enough space left for the handler
399 bailout("deopt handler overflow");
400 return -1;
401 }
402
403 int offset = code_offset();
404
405 __ auipc(ra, 0);
406 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
407 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
408 __ end_a_stub();
409
410 return offset;
411 }
412
413 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
414 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
415
416 // Pop the stack before the safepoint code
417 __ remove_frame(initial_frame_size_in_bytes());
418
419 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
420 __ reserved_stack_check();
421 }
422
423 code_stub->set_safepoint_offset(__ offset());
424 __ relocate(relocInfo::poll_return_type);
425 __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
426 __ ret();
427 }
428
429 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
430 guarantee(info != NULL, "Shouldn't be NULL");
431 __ get_polling_page(t0, relocInfo::poll_type);
432 add_debug_info_for_branch(info); // This isn't just debug info:
433 // it's the oop map
434 __ read_polling_page(t0, 0, relocInfo::poll_type);
435 return __ offset();
436 }
437
438 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
439 __ mv(to_reg, from_reg);
440 }
441
442 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
443
444 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
445 assert(src->is_constant(), "should not call otherwise");
446 assert(dest->is_register(), "should not call otherwise");
447 LIR_Const* c = src->as_constant_ptr();
448 address const_addr = NULL;
449
450 switch (c->type()) {
451 case T_INT:
452 assert(patch_code == lir_patch_none, "no patching handled here");
453 __ mv(dest->as_register(), c->as_jint());
454 break;
455
456 case T_ADDRESS:
457 assert(patch_code == lir_patch_none, "no patching handled here");
458 __ mv(dest->as_register(), c->as_jint());
459 break;
460
461 case T_LONG:
462 assert(patch_code == lir_patch_none, "no patching handled here");
463 __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
464 break;
465
466 case T_OBJECT:
467 case T_ARRAY:
468 if (patch_code == lir_patch_none) {
469 jobject2reg(c->as_jobject(), dest->as_register());
470 } else {
471 jobject2reg_with_patching(dest->as_register(), info);
472 }
473 break;
474
475 case T_METADATA:
476 if (patch_code != lir_patch_none) {
477 klass2reg_with_patching(dest->as_register(), info);
478 } else {
479 __ mov_metadata(dest->as_register(), c->as_metadata());
480 }
481 break;
482
483 case T_FLOAT:
484 const_addr = float_constant(c->as_jfloat());
485 assert(const_addr != NULL, "must create float constant in the constant table");
486 __ flw(dest->as_float_reg(), InternalAddress(const_addr));
487 break;
488
489 case T_DOUBLE:
490 const_addr = double_constant(c->as_jdouble());
491 assert(const_addr != NULL, "must create double constant in the constant table");
492 __ fld(dest->as_double_reg(), InternalAddress(const_addr));
493 break;
494
495 default:
496 ShouldNotReachHere();
497 }
498 }
499
500 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
501 assert(src->is_constant(), "should not call otherwise");
502 assert(dest->is_stack(), "should not call otherwise");
503 LIR_Const* c = src->as_constant_ptr();
504 switch (c->type()) {
505 case T_OBJECT:
506 if (c->as_jobject() == NULL) {
507 __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
508 } else {
509 const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
510 reg2stack(FrameMap::t1_opr, dest, c->type(), false);
511 }
512 break;
513 case T_ADDRESS: // fall through
514 const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
515 reg2stack(FrameMap::t1_opr, dest, c->type(), false);
516 case T_INT: // fall through
517 case T_FLOAT:
518 if (c->as_jint_bits() == 0) {
519 __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
520 } else {
521 __ mv(t1, c->as_jint_bits());
522 __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix()));
523 }
524 break;
525 case T_LONG: // fall through
526 case T_DOUBLE:
527 if (c->as_jlong_bits() == 0) {
528 __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
529 lo_word_offset_in_bytes));
530 } else {
531 __ mv(t1, (intptr_t)c->as_jlong_bits());
532 __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(),
533 lo_word_offset_in_bytes));
534 }
535 break;
536 default:
537 ShouldNotReachHere();
538 }
539 }
540
541 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
542 assert(src->is_constant(), "should not call otherwise");
543 assert(dest->is_address(), "should not call otherwise");
544 LIR_Const* c = src->as_constant_ptr();
545 LIR_Address* to_addr = dest->as_address_ptr();
546 void (MacroAssembler::* insn)(Register Rt, const Address &adr, Register temp);
547 switch (type) {
548 case T_ADDRESS:
549 assert(c->as_jint() == 0, "should be");
550 insn = &MacroAssembler::sd; break;
551 case T_LONG:
552 assert(c->as_jlong() == 0, "should be");
553 insn = &MacroAssembler::sd; break;
554 case T_DOUBLE:
555 assert(c->as_jdouble() == 0.0, "should be");
556 insn = &MacroAssembler::sd; break;
557 case T_INT:
558 assert(c->as_jint() == 0, "should be");
559 insn = &MacroAssembler::sw; break;
560 case T_FLOAT:
561 assert(c->as_jfloat() == 0.0f, "should be");
562 insn = &MacroAssembler::sw; break;
563 case T_OBJECT: // fall through
564 case T_ARRAY:
565 assert(c->as_jobject() == 0, "should be");
566 if (UseCompressedOops && !wide) {
567 insn = &MacroAssembler::sw;
568 } else {
569 insn = &MacroAssembler::sd;
570 }
571 break;
572 case T_CHAR: // fall through
573 case T_SHORT:
574 assert(c->as_jint() == 0, "should be");
575 insn = &MacroAssembler::sh;
576 break;
577 case T_BOOLEAN: // fall through
578 case T_BYTE:
579 assert(c->as_jint() == 0, "should be");
580 insn = &MacroAssembler::sb; break;
581 default:
582 ShouldNotReachHere();
583 insn = &MacroAssembler::sd; // unreachable
584 }
585 if (info != NULL) {
586 add_debug_info_for_null_check_here(info);
587 }
588 (_masm->*insn)(zr, as_Address(to_addr), t0);
589 }
590
591 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
592 assert(src->is_register(), "should not call otherwise");
593 assert(dest->is_register(), "should not call otherwise");
594
595 // move between cpu-registers
596 if (dest->is_single_cpu()) {
597 if (src->type() == T_LONG) {
598 // Can do LONG -> OBJECT
599 move_regs(src->as_register_lo(), dest->as_register());
600 return;
601 }
602 assert(src->is_single_cpu(), "must match");
603 if (src->type() == T_OBJECT) {
604 __ verify_oop(src->as_register());
605 }
606 move_regs(src->as_register(), dest->as_register());
607 } else if (dest->is_double_cpu()) {
608 if (is_reference_type(src->type())) {
609 __ verify_oop(src->as_register());
610 move_regs(src->as_register(), dest->as_register_lo());
611 return;
612 }
613 assert(src->is_double_cpu(), "must match");
614 Register f_lo = src->as_register_lo();
615 Register f_hi = src->as_register_hi();
616 Register t_lo = dest->as_register_lo();
617 Register t_hi = dest->as_register_hi();
618 assert(f_hi == f_lo, "must be same");
619 assert(t_hi == t_lo, "must be same");
620 move_regs(f_lo, t_lo);
621 } else if (dest->is_single_fpu()) {
622 assert(src->is_single_fpu(), "expect single fpu");
623 __ fmv_s(dest->as_float_reg(), src->as_float_reg());
624 } else if (dest->is_double_fpu()) {
625 assert(src->is_double_fpu(), "expect double fpu");
626 __ fmv_d(dest->as_double_reg(), src->as_double_reg());
627 } else {
628 ShouldNotReachHere();
629 }
630 }
631
632 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
633 precond(src->is_register() && dest->is_stack());
634
635 uint const c_sz32 = sizeof(uint32_t);
636 uint const c_sz64 = sizeof(uint64_t);
637
638 assert(src->is_register(), "should not call otherwise");
639 assert(dest->is_stack(), "should not call otherwise");
640 if (src->is_single_cpu()) {
641 int index = dest->single_stack_ix();
642 if (is_reference_type(type)) {
643 __ sd(src->as_register(), stack_slot_address(index, c_sz64));
644 __ verify_oop(src->as_register());
645 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
646 __ sd(src->as_register(), stack_slot_address(index, c_sz64));
647 } else {
648 __ sw(src->as_register(), stack_slot_address(index, c_sz32));
649 }
650 } else if (src->is_double_cpu()) {
651 int index = dest->double_stack_ix();
652 Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
653 __ sd(src->as_register_lo(), dest_addr_LO);
654 } else if (src->is_single_fpu()) {
655 int index = dest->single_stack_ix();
656 __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32));
657 } else if (src->is_double_fpu()) {
658 int index = dest->double_stack_ix();
659 __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64));
660 } else {
661 ShouldNotReachHere();
662 }
663 }
664
665 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
666 LIR_Address* to_addr = dest->as_address_ptr();
667 // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src
668 Register compressed_src = t1;
669
670 if (patch_code != lir_patch_none) {
671 deoptimize_trap(info);
672 return;
673 }
674
675 if (is_reference_type(type)) {
676 __ verify_oop(src->as_register());
677
678 if (UseCompressedOops && !wide) {
679 __ encode_heap_oop(compressed_src, src->as_register());
680 } else {
681 compressed_src = src->as_register();
682 }
683 }
684
685 int null_check_here = code_offset();
686
687 switch (type) {
688 case T_FLOAT:
689 __ fsw(src->as_float_reg(), as_Address(to_addr));
690 break;
691
692 case T_DOUBLE:
693 __ fsd(src->as_double_reg(), as_Address(to_addr));
694 break;
695
696 case T_ARRAY: // fall through
697 case T_OBJECT:
698 if (UseCompressedOops && !wide) {
699 __ sw(compressed_src, as_Address(to_addr));
700 } else {
701 __ sd(compressed_src, as_Address(to_addr));
702 }
703 break;
704 case T_METADATA:
705 // We get here to store a method pointer to the stack to pass to
706 // a dtrace runtime call. This can't work on 64 bit with
707 // compressed klass ptrs: T_METADATA can be compressed klass
708 // ptr or a 64 bit method pointer.
709 ShouldNotReachHere();
710 __ sd(src->as_register(), as_Address(to_addr));
711 break;
712 case T_ADDRESS:
713 __ sd(src->as_register(), as_Address(to_addr));
714 break;
715 case T_INT:
716 __ sw(src->as_register(), as_Address(to_addr));
717 break;
718 case T_LONG:
719 __ sd(src->as_register_lo(), as_Address(to_addr));
720 break;
721 case T_BYTE: // fall through
722 case T_BOOLEAN:
723 __ sb(src->as_register(), as_Address(to_addr));
724 break;
725 case T_CHAR: // fall through
726 case T_SHORT:
727 __ sh(src->as_register(), as_Address(to_addr));
728 break;
729 default:
730 ShouldNotReachHere();
731 }
732
733 if (info != NULL) {
734 add_debug_info_for_null_check(null_check_here, info);
735 }
736 }
737
738 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
739 precond(src->is_stack() && dest->is_register());
740
741 uint const c_sz32 = sizeof(uint32_t);
742 uint const c_sz64 = sizeof(uint64_t);
743
744 if (dest->is_single_cpu()) {
745 int index = src->single_stack_ix();
746 if (type == T_INT) {
747 __ lw(dest->as_register(), stack_slot_address(index, c_sz32));
748 } else if (is_reference_type(type)) {
749 __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
750 __ verify_oop(dest->as_register());
751 } else if (type == T_METADATA || type == T_ADDRESS) {
752 __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
753 } else {
754 __ lwu(dest->as_register(), stack_slot_address(index, c_sz32));
755 }
756 } else if (dest->is_double_cpu()) {
757 int index = src->double_stack_ix();
758 Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
759 __ ld(dest->as_register_lo(), src_addr_LO);
760 } else if (dest->is_single_fpu()) {
761 int index = src->single_stack_ix();
762 __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32));
763 } else if (dest->is_double_fpu()) {
764 int index = src->double_stack_ix();
765 __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64));
766 } else {
767 ShouldNotReachHere();
768 }
769 }
770
771 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
772 deoptimize_trap(info);
773 }
774
775 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
776 LIR_Opr temp;
777 if (type == T_LONG || type == T_DOUBLE) {
778 temp = FrameMap::t1_long_opr;
779 } else {
780 temp = FrameMap::t1_opr;
781 }
782
783 stack2reg(src, temp, src->type());
784 reg2stack(temp, dest, dest->type(), false);
785 }
786
787 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
788 assert(src->is_address(), "should not call otherwise");
789 assert(dest->is_register(), "should not call otherwise");
790
791 LIR_Address* addr = src->as_address_ptr();
792 LIR_Address* from_addr = src->as_address_ptr();
793
794 if (addr->base()->type() == T_OBJECT) {
795 __ verify_oop(addr->base()->as_pointer_register());
796 }
797
798 if (patch_code != lir_patch_none) {
799 deoptimize_trap(info);
800 return;
801 }
802
803 if (info != NULL) {
804 add_debug_info_for_null_check_here(info);
805 }
806
807 int null_check_here = code_offset();
808 switch (type) {
809 case T_FLOAT:
810 __ flw(dest->as_float_reg(), as_Address(from_addr));
811 break;
812 case T_DOUBLE:
813 __ fld(dest->as_double_reg(), as_Address(from_addr));
814 break;
815 case T_ARRAY: // fall through
816 case T_OBJECT:
817 if (UseCompressedOops && !wide) {
818 __ lwu(dest->as_register(), as_Address(from_addr));
819 } else {
820 __ ld(dest->as_register(), as_Address(from_addr));
821 }
822 break;
823 case T_METADATA:
824 // We get here to store a method pointer to the stack to pass to
825 // a dtrace runtime call. This can't work on 64 bit with
826 // compressed klass ptrs: T_METADATA can be a compressed klass
827 // ptr or a 64 bit method pointer.
828 ShouldNotReachHere();
829 __ ld(dest->as_register(), as_Address(from_addr));
830 break;
831 case T_ADDRESS:
832 __ ld(dest->as_register(), as_Address(from_addr));
833 break;
834 case T_INT:
835 __ lw(dest->as_register(), as_Address(from_addr));
836 break;
837 case T_LONG:
838 __ ld(dest->as_register_lo(), as_Address_lo(from_addr));
839 break;
840 case T_BYTE:
841 __ lb(dest->as_register(), as_Address(from_addr));
842 break;
843 case T_BOOLEAN:
844 __ lbu(dest->as_register(), as_Address(from_addr));
845 break;
846 case T_CHAR:
847 __ lhu(dest->as_register(), as_Address(from_addr));
848 break;
849 case T_SHORT:
850 __ lh(dest->as_register(), as_Address(from_addr));
851 break;
852 default:
853 ShouldNotReachHere();
854 }
855
856 if (is_reference_type(type)) {
857 if (UseCompressedOops && !wide) {
858 __ decode_heap_oop(dest->as_register());
859 }
860
861 if (!UseZGC) {
862 // Load barrier has not yet been applied, so ZGC can't verify the oop here
863 __ verify_oop(dest->as_register());
864 }
865 }
866 }
867
868 void LIR_Assembler::emit_op3(LIR_Op3* op) {
869 switch (op->code()) {
870 case lir_idiv: // fall through
871 case lir_irem:
872 arithmetic_idiv(op->code(),
873 op->in_opr1(),
874 op->in_opr2(),
875 op->in_opr3(),
876 op->result_opr(),
877 op->info());
878 break;
879 case lir_fmad:
880 __ fmadd_d(op->result_opr()->as_double_reg(),
881 op->in_opr1()->as_double_reg(),
882 op->in_opr2()->as_double_reg(),
883 op->in_opr3()->as_double_reg());
884 break;
885 case lir_fmaf:
886 __ fmadd_s(op->result_opr()->as_float_reg(),
887 op->in_opr1()->as_float_reg(),
888 op->in_opr2()->as_float_reg(),
889 op->in_opr3()->as_float_reg());
890 break;
891 default:
892 ShouldNotReachHere();
893 }
894 }
895
896 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
897 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
898 Label label;
899
900 emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false,
901 /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true);
902
903 Label done;
904 move_op(opr2, result, type, lir_patch_none, NULL,
905 false, // pop_fpu_stack
906 false); // wide
907 __ j(done);
908 __ bind(label);
909 move_op(opr1, result, type, lir_patch_none, NULL,
910 false, // pop_fpu_stack
911 false); // wide
912 __ bind(done);
913 }
914
915 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
916 LIR_Condition condition = op->cond();
917 if (condition == lir_cond_always) {
918 if (op->info() != NULL) {
919 add_debug_info_for_branch(op->info());
920 }
921 } else {
922 assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands");
923 }
924 bool is_unordered = (op->ublock() == op->block());
925 emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered);
926 }
927
928 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label,
929 bool is_far, bool is_unordered) {
930
931 if (cmp_flag == lir_cond_always) {
932 __ j(label);
933 return;
934 }
935
936 if (cmp1->is_cpu_register()) {
937 Register reg1 = as_reg(cmp1);
938 if (cmp2->is_cpu_register()) {
939 Register reg2 = as_reg(cmp2);
940 __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far);
941 } else if (cmp2->is_constant()) {
942 const2reg_helper(cmp2);
943 __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far);
944 } else {
945 ShouldNotReachHere();
946 }
947 } else if (cmp1->is_single_fpu()) {
948 assert(cmp2->is_single_fpu(), "expect single float register");
949 __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered);
950 } else if (cmp1->is_double_fpu()) {
951 assert(cmp2->is_double_fpu(), "expect double float register");
952 __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask,
953 cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered);
954 } else {
955 ShouldNotReachHere();
956 }
957 }
958
959 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
960 LIR_Opr src = op->in_opr();
961 LIR_Opr dest = op->result_opr();
962
963 switch (op->bytecode()) {
964 case Bytecodes::_i2f:
965 __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break;
966 case Bytecodes::_i2d:
967 __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break;
968 case Bytecodes::_l2d:
969 __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break;
970 case Bytecodes::_l2f:
971 __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break;
972 case Bytecodes::_f2d:
973 __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break;
974 case Bytecodes::_d2f:
975 __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break;
976 case Bytecodes::_i2c:
977 __ zero_extend(dest->as_register(), src->as_register(), 16); break;
978 case Bytecodes::_i2l:
979 __ addw(dest->as_register_lo(), src->as_register(), zr); break;
980 case Bytecodes::_i2s:
981 __ sign_extend(dest->as_register(), src->as_register(), 16); break;
982 case Bytecodes::_i2b:
983 __ sign_extend(dest->as_register(), src->as_register(), 8); break;
984 case Bytecodes::_l2i:
985 _masm->block_comment("FIXME: This coulde be no-op");
986 __ addw(dest->as_register(), src->as_register_lo(), zr); break;
987 case Bytecodes::_d2l:
988 __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break;
989 case Bytecodes::_f2i:
990 __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break;
991 case Bytecodes::_f2l:
992 __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break;
993 case Bytecodes::_d2i:
994 __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break;
995 default:
996 ShouldNotReachHere();
997 }
998 }
999
1000 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1001 if (op->init_check()) {
1002 __ lbu(t0, Address(op->klass()->as_register(),
1003 InstanceKlass::init_state_offset()));
1004 __ mv(t1, (u1)InstanceKlass::fully_initialized);
1005 add_debug_info_for_null_check_here(op->stub()->info());
1006 __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
1007 }
1008
1009 __ allocate_object(op->obj()->as_register(),
1010 op->tmp1()->as_register(),
1011 op->tmp2()->as_register(),
1012 op->header_size(),
1013 op->object_size(),
1014 op->klass()->as_register(),
1015 *op->stub()->entry());
1016
1017 __ bind(*op->stub()->continuation());
1018 }
1019
1020 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1021 Register len = op->len()->as_register();
1022
1023 if (UseSlowPath ||
1024 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1025 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1026 __ j(*op->stub()->entry());
1027 } else {
1028 Register tmp1 = op->tmp1()->as_register();
1029 Register tmp2 = op->tmp2()->as_register();
1030 Register tmp3 = op->tmp3()->as_register();
1031 if (len == tmp1) {
1032 tmp1 = tmp3;
1033 } else if (len == tmp2) {
1034 tmp2 = tmp3;
1035 } else if (len == tmp3) {
1036 // everything is ok
1037 } else {
1038 __ mv(tmp3, len);
1039 }
1040 __ allocate_array(op->obj()->as_register(),
1041 len,
1042 tmp1,
1043 tmp2,
1044 arrayOopDesc::header_size(op->type()),
1045 array_element_size(op->type()),
1046 op->klass()->as_register(),
1047 *op->stub()->entry());
1048 }
1049 __ bind(*op->stub()->continuation());
1050 }
1051
1052 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
1053 Register recv, Label* update_done) {
1054 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1055 Label next_test;
1056 // See if the receiver is receiver[n].
1057 __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1058 __ bne(recv, t1, next_test);
1059 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1060 __ increment(data_addr, DataLayout::counter_increment);
1061 __ j(*update_done);
1062 __ bind(next_test);
1063 }
1064
1065 // Didn't find receiver; find next empty slot and fill it in
1066 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1067 Label next_test;
1068 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1069 __ ld(t1, recv_addr);
1070 __ bnez(t1, next_test);
1071 __ sd(recv, recv_addr);
1072 __ mv(t1, DataLayout::counter_increment);
1073 __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1074 __ j(*update_done);
1075 __ bind(next_test);
1076 }
1077 }
1078
1079 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
1080 ciMethod* method = op->profiled_method();
1081 assert(method != NULL, "Should have method");
1082 int bci = op->profiled_bci();
1083 *md = method->method_data_or_null();
1084 guarantee(*md != NULL, "Sanity");
1085 *data = ((*md)->bci_to_data(bci));
1086 assert(*data != NULL, "need data for type check");
1087 assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1088 }
1089
1090 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1091 Register k_RInfo, Register klass_RInfo,
1092 Label *failure_target, Label *success_target) {
1093 // get object class
1094 // not a safepoint as obj null check happens earlier
1095 __ load_klass(klass_RInfo, obj);
1096 if (k->is_loaded()) {
1097 // See if we get an immediate positive hit
1098 __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1099 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1100 __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1101 // successful cast, fall through to profile or jump
1102 } else {
1103 // See if we get an immediate positive hit
1104 __ beq(k_RInfo, t0, *success_target);
1105 // check for self
1106 __ beq(klass_RInfo, k_RInfo, *success_target);
1107
1108 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1109 __ sd(k_RInfo, Address(sp, 0)); // sub klass
1110 __ sd(klass_RInfo, Address(sp, wordSize)); // super klass
1111 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1112 // load result to k_RInfo
1113 __ ld(k_RInfo, Address(sp, 0));
1114 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1115 // result is a boolean
1116 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1117 // successful cast, fall through to profile or jump
1118 }
1119 } else {
1120 // perform the fast part of the checking logic
1121 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1122 // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1123 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1124 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
1125 __ sd(k_RInfo, Address(sp, 0)); // super klass
1126 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1127 // load result to k_RInfo
1128 __ ld(k_RInfo, Address(sp, 0));
1129 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1130 // result is a boolean
1131 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1132 // successful cast, fall thriugh to profile or jump
1133 }
1134 }
1135
1136 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj,
1137 Register klass_RInfo, Label* obj_is_null) {
1138 Label not_null;
1139 __ bnez(obj, not_null);
1140 // Object is null, update MDO and exit
1141 Register mdo = klass_RInfo;
1142 __ mov_metadata(mdo, md->constant_encoding());
1143 Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1144 __ lbu(t0, data_addr);
1145 __ ori(t0, t0, BitData::null_seen_byte_constant());
1146 __ sb(t0, data_addr);
1147 __ j(*obj_is_null);
1148 __ bind(not_null);
1149 }
1150
1151 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
1152 if (!k->is_loaded()) {
1153 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1154 } else {
1155 __ mov_metadata(k_RInfo, k->constant_encoding());
1156 }
1157 }
1158
1159 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1160 Register obj = op->object()->as_register();
1161 Register k_RInfo = op->tmp1()->as_register();
1162 Register klass_RInfo = op->tmp2()->as_register();
1163 Register dst = op->result_opr()->as_register();
1164 ciKlass* k = op->klass();
1165 Register Rtmp1 = noreg;
1166
1167 // check if it needs to be profiled
1168 ciMethodData* md = NULL;
1169 ciProfileData* data = NULL;
1170
1171 const bool should_profile = op->should_profile();
1172 if (should_profile) {
1173 data_check(op, &md, &data);
1174 }
1175 Label profile_cast_success, profile_cast_failure;
1176 Label *success_target = should_profile ? &profile_cast_success : success;
1177 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1178
1179 if (obj == k_RInfo) {
1180 k_RInfo = dst;
1181 } else if (obj == klass_RInfo) {
1182 klass_RInfo = dst;
1183 }
1184 if (k->is_loaded() && !UseCompressedClassPointers) {
1185 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1186 } else {
1187 Rtmp1 = op->tmp3()->as_register();
1188 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1189 }
1190
1191 assert_different_registers(obj, k_RInfo, klass_RInfo);
1192
1193 if (should_profile) {
1194 profile_object(md, data, obj, klass_RInfo, obj_is_null);
1195 } else {
1196 __ beqz(obj, *obj_is_null);
1197 }
1198
1199 typecheck_loaded(op, k, k_RInfo);
1200 __ verify_oop(obj);
1201
1202 if (op->fast_check()) {
1203 // get object class
1204 // not a safepoint as obj null check happens earlier
1205 __ load_klass(t0, obj, t1);
1206 __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1207 // successful cast, fall through to profile or jump
1208 } else {
1209 typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1210 }
1211 if (should_profile) {
1212 type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure);
1213 }
1214 __ j(*success);
1215 }
1216
1217 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1218 const bool should_profile = op->should_profile();
1219
1220 LIR_Code code = op->code();
1221 if (code == lir_store_check) {
1222 typecheck_lir_store(op, should_profile);
1223 } else if (code == lir_checkcast) {
1224 Register obj = op->object()->as_register();
1225 Register dst = op->result_opr()->as_register();
1226 Label success;
1227 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1228 __ bind(success);
1229 if (dst != obj) {
1230 __ mv(dst, obj);
1231 }
1232 } else if (code == lir_instanceof) {
1233 Register obj = op->object()->as_register();
1234 Register dst = op->result_opr()->as_register();
1235 Label success, failure, done;
1236 emit_typecheck_helper(op, &success, &failure, &failure);
1237 __ bind(failure);
1238 __ mv(dst, zr);
1239 __ j(done);
1240 __ bind(success);
1241 __ mv(dst, 1);
1242 __ bind(done);
1243 } else {
1244 ShouldNotReachHere();
1245 }
1246 }
1247
1248 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1249 assert(VM_Version::supports_cx8(), "wrong machine");
1250 Register addr;
1251 if (op->addr()->is_register()) {
1252 addr = as_reg(op->addr());
1253 } else {
1254 assert(op->addr()->is_address(), "what else?");
1255 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1256 assert(addr_ptr->disp() == 0, "need 0 disp");
1257 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1258 addr = as_reg(addr_ptr->base());
1259 }
1260 Register newval = as_reg(op->new_value());
1261 Register cmpval = as_reg(op->cmp_value());
1262
1263 if (op->code() == lir_cas_obj) {
1264 if (UseCompressedOops) {
1265 Register tmp1 = op->tmp1()->as_register();
1266 assert(op->tmp1()->is_valid(), "must be");
1267 __ encode_heap_oop(tmp1, cmpval);
1268 cmpval = tmp1;
1269 __ encode_heap_oop(t1, newval);
1270 newval = t1;
1271 caswu(addr, newval, cmpval);
1272 } else {
1273 casl(addr, newval, cmpval);
1274 }
1275 } else if (op->code() == lir_cas_int) {
1276 casw(addr, newval, cmpval);
1277 } else {
1278 casl(addr, newval, cmpval);
1279 }
1280 }
1281
1282 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1283 switch (code) {
1284 case lir_abs: __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break;
1285 case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break;
1286 default: ShouldNotReachHere();
1287 }
1288 }
1289
1290 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1291 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1292 Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1293 if (dst->is_single_cpu()) {
1294 Register Rdst = dst->as_register();
1295 if (right->is_constant()) {
1296 int right_const = right->as_jint();
1297 if (Assembler::operand_valid_for_add_immediate(right_const)) {
1298 logic_op_imm(Rdst, Rleft, right_const, code);
1299 __ addw(Rdst, Rdst, zr);
1300 } else {
1301 __ mv(t0, right_const);
1302 logic_op_reg32(Rdst, Rleft, t0, code);
1303 }
1304 } else {
1305 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1306 logic_op_reg32(Rdst, Rleft, Rright, code);
1307 }
1308 } else {
1309 Register Rdst = dst->as_register_lo();
1310 if (right->is_constant()) {
1311 long right_const = right->as_jlong();
1312 if (Assembler::operand_valid_for_add_immediate(right_const)) {
1313 logic_op_imm(Rdst, Rleft, right_const, code);
1314 } else {
1315 __ mv(t0, right_const);
1316 logic_op_reg(Rdst, Rleft, t0, code);
1317 }
1318 } else {
1319 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1320 logic_op_reg(Rdst, Rleft, Rright, code);
1321 }
1322 }
1323 }
1324
1325 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) {
1326 ShouldNotCallThis();
1327 }
1328
1329 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1330 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1331 bool is_unordered_less = (code == lir_ucmp_fd2i);
1332 if (left->is_single_fpu()) {
1333 __ float_cmp(true, is_unordered_less ? -1 : 1,
1334 left->as_float_reg(), right->as_float_reg(), dst->as_register());
1335 } else if (left->is_double_fpu()) {
1336 __ float_cmp(false, is_unordered_less ? -1 : 1,
1337 left->as_double_reg(), right->as_double_reg(), dst->as_register());
1338 } else {
1339 ShouldNotReachHere();
1340 }
1341 } else if (code == lir_cmp_l2i) {
1342 __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1343 } else {
1344 ShouldNotReachHere();
1345 }
1346 }
1347
1348 void LIR_Assembler::align_call(LIR_Code code) {
1349 // With RVC a call instruction may get 2-byte aligned.
1350 // The address of the call instruction needs to be 4-byte aligned to
1351 // ensure that it does not span a cache line so that it can be patched.
1352 __ align(NativeInstruction::instruction_size);
1353 }
1354
1355 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1356 address call = __ trampoline_call(Address(op->addr(), rtype));
1357 if (call == NULL) {
1358 bailout("trampoline stub overflow");
1359 return;
1360 }
1361 add_call_info(code_offset(), op->info());
1362 __ post_call_nop();
1363 }
1364
1365 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1366 address call = __ ic_call(op->addr());
1367 if (call == NULL) {
1368 bailout("trampoline stub overflow");
1369 return;
1370 }
1371 add_call_info(code_offset(), op->info());
1372 __ post_call_nop();
1373 }
1374
1375 void LIR_Assembler::emit_static_call_stub() {
1376 address call_pc = __ pc();
1377 MacroAssembler::assert_alignment(call_pc);
1378 address stub = __ start_a_stub(call_stub_size());
1379 if (stub == NULL) {
1380 bailout("static call stub overflow");
1381 return;
1382 }
1383
1384 int start = __ offset();
1385
1386 __ relocate(static_stub_Relocation::spec(call_pc));
1387 __ emit_static_call_stub();
1388
1389 assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
1390 <= call_stub_size(), "stub too big");
1391 __ end_a_stub();
1392 }
1393
1394 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1395 assert(exceptionOop->as_register() == x10, "must match");
1396 assert(exceptionPC->as_register() == x13, "must match");
1397
1398 // exception object is not added to oop map by LinearScan
1399 // (LinearScan assumes that no oops are in fixed registers)
1400 info->add_register_oop(exceptionOop);
1401 Runtime1::StubID unwind_id;
1402
1403 // get current pc information
1404 // pc is only needed if the method has an exception handler, the unwind code does not need it.
1405 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
1406 // As no instructions have been generated yet for this LIR node it's
1407 // possible that an oop map already exists for the current offset.
1408 // In that case insert an dummy NOP here to ensure all oop map PCs
1409 // are unique. See JDK-8237483.
1410 __ nop();
1411 }
1412 int pc_for_athrow_offset = __ offset();
1413 InternalAddress pc_for_athrow(__ pc());
1414 __ relocate(pc_for_athrow.rspec(), [&] {
1415 int32_t offset;
1416 __ la_patchable(exceptionPC->as_register(), pc_for_athrow, offset);
1417 __ addi(exceptionPC->as_register(), exceptionPC->as_register(), offset);
1418 });
1419 add_call_info(pc_for_athrow_offset, info); // for exception handler
1420
1421 __ verify_not_null_oop(x10);
1422 // search an exception handler (x10: exception oop, x13: throwing pc)
1423 if (compilation()->has_fpu_code()) {
1424 unwind_id = Runtime1::handle_exception_id;
1425 } else {
1426 unwind_id = Runtime1::handle_exception_nofpu_id;
1427 }
1428 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
1429 __ nop();
1430 }
1431
1432 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1433 assert(exceptionOop->as_register() == x10, "must match");
1434 __ j(_unwind_handler_entry);
1435 }
1436
1437 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
1438 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1439 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1440 Register count_reg = count->as_register();
1441 if (dest->is_single_cpu()) {
1442 assert (dest->type() == T_INT, "unexpected result type");
1443 assert (left->type() == T_INT, "unexpected left type");
1444 __ andi(t0, count_reg, 31); // should not shift more than 31 bits
1445 switch (code) {
1446 case lir_shl: __ sllw(dest_reg, left_reg, t0); break;
1447 case lir_shr: __ sraw(dest_reg, left_reg, t0); break;
1448 case lir_ushr: __ srlw(dest_reg, left_reg, t0); break;
1449 default: ShouldNotReachHere();
1450 }
1451 } else if (dest->is_double_cpu()) {
1452 __ andi(t0, count_reg, 63); // should not shift more than 63 bits
1453 switch (code) {
1454 case lir_shl: __ sll(dest_reg, left_reg, t0); break;
1455 case lir_shr: __ sra(dest_reg, left_reg, t0); break;
1456 case lir_ushr: __ srl(dest_reg, left_reg, t0); break;
1457 default: ShouldNotReachHere();
1458 }
1459 } else {
1460 ShouldNotReachHere();
1461 }
1462 }
1463
1464 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
1465 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1466 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1467 if (dest->is_single_cpu()) {
1468 assert (dest->type() == T_INT, "unexpected result type");
1469 assert (left->type() == T_INT, "unexpected left type");
1470 count &= 0x1f;
1471 if (count != 0) {
1472 switch (code) {
1473 case lir_shl: __ slliw(dest_reg, left_reg, count); break;
1474 case lir_shr: __ sraiw(dest_reg, left_reg, count); break;
1475 case lir_ushr: __ srliw(dest_reg, left_reg, count); break;
1476 default: ShouldNotReachHere();
1477 }
1478 } else {
1479 move_regs(left_reg, dest_reg);
1480 }
1481 } else if (dest->is_double_cpu()) {
1482 count &= 0x3f;
1483 if (count != 0) {
1484 switch (code) {
1485 case lir_shl: __ slli(dest_reg, left_reg, count); break;
1486 case lir_shr: __ srai(dest_reg, left_reg, count); break;
1487 case lir_ushr: __ srli(dest_reg, left_reg, count); break;
1488 default: ShouldNotReachHere();
1489 }
1490 } else {
1491 move_regs(left->as_register_lo(), dest->as_register_lo());
1492 }
1493 } else {
1494 ShouldNotReachHere();
1495 }
1496 }
1497
1498 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
1499 Register obj = op->obj_opr()->as_register(); // may not be an oop
1500 Register hdr = op->hdr_opr()->as_register();
1501 Register lock = op->lock_opr()->as_register();
1502 if (UseHeavyMonitors) {
1503 if (op->info() != NULL) {
1504 add_debug_info_for_null_check_here(op->info());
1505 __ null_check(obj);
1506 }
1507 __ j(*op->stub()->entry());
1508 } else if (op->code() == lir_lock) {
1509 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1510 // add debug info for NullPointerException only if one is possible
1511 int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
1512 if (op->info() != NULL) {
1513 add_debug_info_for_null_check(null_check_offset, op->info());
1514 }
1515 } else if (op->code() == lir_unlock) {
1516 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1517 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
1518 } else {
1519 Unimplemented();
1520 }
1521 __ bind(*op->stub()->continuation());
1522 }
1523
1524 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
1525 Register obj = op->obj()->as_pointer_register();
1526 Register result = op->result_opr()->as_pointer_register();
1527
1528 CodeEmitInfo* info = op->info();
1529 if (info != NULL) {
1530 add_debug_info_for_null_check_here(info);
1531 }
1532
1533 if (UseCompressedClassPointers) {
1534 __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes()));
1535 __ decode_klass_not_null(result);
1536 } else {
1537 __ ld(result, Address(obj, oopDesc::klass_offset_in_bytes()));
1538 }
1539 }
1540
1541 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
1542 ciMethod* method = op->profiled_method();
1543 int bci = op->profiled_bci();
1544
1545 // Update counter for all call types
1546 ciMethodData* md = method->method_data_or_null();
1547 guarantee(md != NULL, "Sanity");
1548 ciProfileData* data = md->bci_to_data(bci);
1549 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
1550 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
1551 Register mdo = op->mdo()->as_register();
1552 __ mov_metadata(mdo, md->constant_encoding());
1553 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1554 // Perform additional virtual call profiling for invokevirtual and
1555 // invokeinterface bytecodes
1556 if (op->should_profile_receiver_type()) {
1557 assert(op->recv()->is_single_cpu(), "recv must be allocated");
1558 Register recv = op->recv()->as_register();
1559 assert_different_registers(mdo, recv);
1560 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
1561 ciKlass* known_klass = op->known_holder();
1562 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
1563 // We know the type that will be seen at this call site; we can
1564 // statically update the MethodData* rather than needing to do
1565 // dynamic tests on the receiver type
1566 // NOTE: we should probably put a lock around this search to
1567 // avoid collisions by concurrent compilations
1568 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
1569 uint i;
1570 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1571 ciKlass* receiver = vc_data->receiver(i);
1572 if (known_klass->equals(receiver)) {
1573 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1574 __ increment(data_addr, DataLayout::counter_increment);
1575 return;
1576 }
1577 }
1578
1579 // Receiver type not found in profile data; select an empty slot
1580 // Note that this is less efficient than it should be because it
1581 // always does a write to the receiver part of the
1582 // VirtualCallData rather than just the first time
1583 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1584 ciKlass* receiver = vc_data->receiver(i);
1585 if (receiver == NULL) {
1586 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
1587 __ mov_metadata(t1, known_klass->constant_encoding());
1588 __ sd(t1, recv_addr);
1589 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1590 __ increment(data_addr, DataLayout::counter_increment);
1591 return;
1592 }
1593 }
1594 } else {
1595 __ load_klass(recv, recv);
1596 Label update_done;
1597 type_profile_helper(mdo, md, data, recv, &update_done);
1598 // Receiver did not match any saved receiver and there is no empty row for it.
1599 // Increment total counter to indicate polymorphic case.
1600 __ increment(counter_addr, DataLayout::counter_increment);
1601
1602 __ bind(update_done);
1603 }
1604 } else {
1605 // Static call
1606 __ increment(counter_addr, DataLayout::counter_increment);
1607 }
1608 }
1609
1610 void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); }
1611
1612 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
1613 __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
1614 }
1615
1616 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); }
1617
1618 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
1619 Register tmp, Label &next, Label &none,
1620 Address mdo_addr) {
1621 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
1622 if (exact_klass != NULL) {
1623 __ mov_metadata(tmp, exact_klass->constant_encoding());
1624 } else {
1625 __ load_klass(tmp, tmp);
1626 }
1627
1628 __ ld(t1, mdo_addr);
1629 __ xorr(tmp, tmp, t1);
1630 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1631 // klass seen before, nothing to do. The unknown bit may have been
1632 // set already but no need to check.
1633 __ beqz(t0, next);
1634
1635 // already unknown. Nothing to do anymore.
1636 __ andi(t0, tmp, TypeEntries::type_unknown);
1637 __ bnez(t0, next);
1638
1639 if (TypeEntries::is_type_none(current_klass)) {
1640 __ beqz(t1, none);
1641 __ mv(t0, (u1)TypeEntries::null_seen);
1642 __ beq(t0, t1, none);
1643 // There is a chance that the checks above (re-reading profiling
1644 // data from memory) fail if another thread has just set the
1645 // profiling to this obj's klass
1646 __ membar(MacroAssembler::LoadLoad);
1647 __ ld(t1, mdo_addr);
1648 __ xorr(tmp, tmp, t1);
1649 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1650 __ beqz(t0, next);
1651 }
1652 } else {
1653 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1654 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
1655
1656 __ ld(tmp, mdo_addr);
1657 // already unknown. Nothing to do anymore.
1658 __ andi(t0, tmp, TypeEntries::type_unknown);
1659 __ bnez(t0, next);
1660 }
1661
1662 // different than before. Cannot keep accurate profile.
1663 __ ld(t1, mdo_addr);
1664 __ ori(t1, t1, TypeEntries::type_unknown);
1665 __ sd(t1, mdo_addr);
1666
1667 if (TypeEntries::is_type_none(current_klass)) {
1668 __ j(next);
1669
1670 __ bind(none);
1671 // first time here. Set profile type.
1672 __ sd(tmp, mdo_addr);
1673 }
1674 }
1675
1676 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp,
1677 Address mdo_addr, Label &next) {
1678 // There's a single possible klass at this profile point
1679 assert(exact_klass != NULL, "should be");
1680 if (TypeEntries::is_type_none(current_klass)) {
1681 __ mov_metadata(tmp, exact_klass->constant_encoding());
1682 __ ld(t1, mdo_addr);
1683 __ xorr(tmp, tmp, t1);
1684 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1685 __ beqz(t0, next);
1686 #ifdef ASSERT
1687 {
1688 Label ok;
1689 __ ld(t0, mdo_addr);
1690 __ beqz(t0, ok);
1691 __ mv(t1, (u1)TypeEntries::null_seen);
1692 __ beq(t0, t1, ok);
1693 // may have been set by another thread
1694 __ membar(MacroAssembler::LoadLoad);
1695 __ mov_metadata(t0, exact_klass->constant_encoding());
1696 __ ld(t1, mdo_addr);
1697 __ xorr(t1, t0, t1);
1698 __ andi(t1, t1, TypeEntries::type_mask);
1699 __ beqz(t1, ok);
1700
1701 __ stop("unexpected profiling mismatch");
1702 __ bind(ok);
1703 }
1704 #endif
1705 // first time here. Set profile type.
1706 __ sd(tmp, mdo_addr);
1707 } else {
1708 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1709 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
1710
1711 __ ld(tmp, mdo_addr);
1712 // already unknown. Nothing to do anymore.
1713 __ andi(t0, tmp, TypeEntries::type_unknown);
1714 __ bnez(t0, next);
1715
1716 __ ori(tmp, tmp, TypeEntries::type_unknown);
1717 __ sd(tmp, mdo_addr);
1718 }
1719 }
1720
1721 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass,
1722 Address mdo_addr, bool do_update, Label &next) {
1723 __ bnez(tmp, update);
1724 if (!TypeEntries::was_null_seen(current_klass)) {
1725 __ ld(t1, mdo_addr);
1726 __ ori(t1, t1, TypeEntries::null_seen);
1727 __ sd(t1, mdo_addr);
1728 }
1729 if (do_update) {
1730 __ j(next);
1731 }
1732 }
1733
1734 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
1735 COMMENT("emit_profile_type {");
1736 Register obj = op->obj()->as_register();
1737 Register tmp = op->tmp()->as_pointer_register();
1738 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
1739 ciKlass* exact_klass = op->exact_klass();
1740 intptr_t current_klass = op->current_klass();
1741 bool not_null = op->not_null();
1742 bool no_conflict = op->no_conflict();
1743
1744 Label update, next, none;
1745
1746 bool do_null = !not_null;
1747 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
1748 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
1749
1750 assert(do_null || do_update, "why are we here?");
1751 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
1752 assert_different_registers(tmp, t0, t1, mdo_addr.base());
1753
1754 __ verify_oop(obj);
1755
1756 if (tmp != obj) {
1757 __ mv(tmp, obj);
1758 }
1759 if (do_null) {
1760 check_null(tmp, update, current_klass, mdo_addr, do_update, next);
1761 #ifdef ASSERT
1762 } else {
1763 __ bnez(tmp, update);
1764 __ stop("unexpected null obj");
1765 #endif
1766 }
1767
1768 __ bind(update);
1769
1770 if (do_update) {
1771 #ifdef ASSERT
1772 if (exact_klass != NULL) {
1773 check_exact_klass(tmp, exact_klass);
1774 }
1775 #endif
1776 if (!no_conflict) {
1777 check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr);
1778 } else {
1779 check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next);
1780 }
1781
1782 __ bind(next);
1783 }
1784 COMMENT("} emit_profile_type");
1785 }
1786
1787 void LIR_Assembler::align_backward_branch_target() { }
1788
1789 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
1790 // tmp must be unused
1791 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
1792
1793 if (left->is_single_cpu()) {
1794 assert(dest->is_single_cpu(), "expect single result reg");
1795 __ negw(dest->as_register(), left->as_register());
1796 } else if (left->is_double_cpu()) {
1797 assert(dest->is_double_cpu(), "expect double result reg");
1798 __ neg(dest->as_register_lo(), left->as_register_lo());
1799 } else if (left->is_single_fpu()) {
1800 assert(dest->is_single_fpu(), "expect single float result reg");
1801 __ fneg_s(dest->as_float_reg(), left->as_float_reg());
1802 } else {
1803 assert(left->is_double_fpu(), "expect double float operand reg");
1804 assert(dest->is_double_fpu(), "expect double float result reg");
1805 __ fneg_d(dest->as_double_reg(), left->as_double_reg());
1806 }
1807 }
1808
1809
1810 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1811 if (patch_code != lir_patch_none) {
1812 deoptimize_trap(info);
1813 return;
1814 }
1815
1816 LIR_Address* adr = addr->as_address_ptr();
1817 Register dst = dest->as_register_lo();
1818
1819 assert_different_registers(dst, t0);
1820 if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) {
1821 int scale = adr->scale();
1822 intptr_t offset = adr->disp();
1823 LIR_Opr index_op = adr->index();
1824 if (index_op->is_constant()) {
1825 offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale;
1826 }
1827
1828 if (!is_imm_in_range(offset, 12, 0)) {
1829 __ la(t0, as_Address(adr));
1830 __ mv(dst, t0);
1831 return;
1832 }
1833 }
1834
1835 __ la(dst, as_Address(adr));
1836 }
1837
1838
1839 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
1840 assert(!tmp->is_valid(), "don't need temporary");
1841
1842 CodeBlob *cb = CodeCache::find_blob(dest);
1843 if (cb != NULL) {
1844 __ far_call(RuntimeAddress(dest));
1845 } else {
1846 RuntimeAddress target(dest);
1847 __ relocate(target.rspec(), [&] {
1848 int32_t offset;
1849 __ la_patchable(t0, target, offset);
1850 __ jalr(x1, t0, offset);
1851 });
1852 }
1853
1854 if (info != NULL) {
1855 add_call_info_here(info);
1856 }
1857 __ post_call_nop();
1858 }
1859
1860 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
1861 if (dest->is_address() || src->is_address()) {
1862 move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false);
1863 } else {
1864 ShouldNotReachHere();
1865 }
1866 }
1867
1868 #ifdef ASSERT
1869 // emit run-time assertion
1870 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
1871 assert(op->code() == lir_assert, "must be");
1872
1873 Label ok;
1874 if (op->in_opr1()->is_valid()) {
1875 assert(op->in_opr2()->is_valid(), "both operands must be valid");
1876 bool is_unordered = false;
1877 LIR_Condition cond = op->condition();
1878 emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false,
1879 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true);
1880 } else {
1881 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
1882 assert(op->condition() == lir_cond_always, "no other conditions allowed");
1883 }
1884
1885 if (op->halt()) {
1886 const char* str = __ code_string(op->msg());
1887 __ stop(str);
1888 } else {
1889 breakpoint();
1890 }
1891 __ bind(ok);
1892 }
1893 #endif
1894
1895 #ifndef PRODUCT
1896 #define COMMENT(x) do { __ block_comment(x); } while (0)
1897 #else
1898 #define COMMENT(x)
1899 #endif
1900
1901 void LIR_Assembler::membar() {
1902 COMMENT("membar");
1903 __ membar(MacroAssembler::AnyAny);
1904 }
1905
1906 void LIR_Assembler::membar_acquire() {
1907 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1908 }
1909
1910 void LIR_Assembler::membar_release() {
1911 __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1912 }
1913
1914 void LIR_Assembler::membar_loadload() {
1915 __ membar(MacroAssembler::LoadLoad);
1916 }
1917
1918 void LIR_Assembler::membar_storestore() {
1919 __ membar(MacroAssembler::StoreStore);
1920 }
1921
1922 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
1923
1924 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
1925
1926 void LIR_Assembler::on_spin_wait() {
1927 __ pause();
1928 }
1929
1930 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
1931 __ mv(result_reg->as_register(), xthread);
1932 }
1933
1934 void LIR_Assembler::peephole(LIR_List *lir) {}
1935
1936 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
1937 Address addr = as_Address(src->as_address_ptr());
1938 BasicType type = src->type();
1939 bool is_oop = is_reference_type(type);
1940
1941 get_op(type);
1942
1943 switch (code) {
1944 case lir_xadd:
1945 {
1946 RegisterOrConstant inc;
1947 Register tmp = as_reg(tmp_op);
1948 Register dst = as_reg(dest);
1949 if (data->is_constant()) {
1950 inc = RegisterOrConstant(as_long(data));
1951 assert_different_registers(dst, addr.base(), tmp);
1952 assert_different_registers(tmp, t0);
1953 } else {
1954 inc = RegisterOrConstant(as_reg(data));
1955 assert_different_registers(inc.as_register(), dst, addr.base(), tmp);
1956 }
1957 __ la(tmp, addr);
1958 (_masm->*add)(dst, inc, tmp);
1959 break;
1960 }
1961 case lir_xchg:
1962 {
1963 Register tmp = tmp_op->as_register();
1964 Register obj = as_reg(data);
1965 Register dst = as_reg(dest);
1966 if (is_oop && UseCompressedOops) {
1967 __ encode_heap_oop(t0, obj);
1968 obj = t0;
1969 }
1970 assert_different_registers(obj, addr.base(), tmp);
1971 assert_different_registers(dst, addr.base(), tmp);
1972 __ la(tmp, addr);
1973 (_masm->*xchg)(dst, obj, tmp);
1974 if (is_oop && UseCompressedOops) {
1975 __ decode_heap_oop(dst);
1976 }
1977 }
1978 break;
1979 default:
1980 ShouldNotReachHere();
1981 }
1982 __ membar(MacroAssembler::AnyAny);
1983 }
1984
1985 int LIR_Assembler::array_element_size(BasicType type) const {
1986 int elem_size = type2aelembytes(type);
1987 return exact_log2(elem_size);
1988 }
1989
1990 // helper functions which checks for overflow and sets bailout if it
1991 // occurs. Always returns a valid embeddable pointer but in the
1992 // bailout case the pointer won't be to unique storage.
1993 address LIR_Assembler::float_constant(float f) {
1994 address const_addr = __ float_constant(f);
1995 if (const_addr == NULL) {
1996 bailout("const section overflow");
1997 return __ code()->consts()->start();
1998 } else {
1999 return const_addr;
2000 }
2001 }
2002
2003 address LIR_Assembler::double_constant(double d) {
2004 address const_addr = __ double_constant(d);
2005 if (const_addr == NULL) {
2006 bailout("const section overflow");
2007 return __ code()->consts()->start();
2008 } else {
2009 return const_addr;
2010 }
2011 }
2012
2013 address LIR_Assembler::int_constant(jlong n) {
2014 address const_addr = __ long_constant(n);
2015 if (const_addr == NULL) {
2016 bailout("const section overflow");
2017 return __ code()->consts()->start();
2018 } else {
2019 return const_addr;
2020 }
2021 }
2022
2023 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
2024 __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */,
2025 Assembler::rl /* release */, t0, true /* result as bool */);
2026 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2027 __ membar(MacroAssembler::AnyAny);
2028 }
2029
2030 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) {
2031 __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */,
2032 Assembler::rl /* release */, t0, true /* result as bool */);
2033 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2034 __ membar(MacroAssembler::AnyAny);
2035 }
2036
2037 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
2038 __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */,
2039 Assembler::rl /* release */, t0, true /* result as bool */);
2040 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2041 __ membar(MacroAssembler::AnyAny);
2042 }
2043
2044 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
2045 address target = NULL;
2046
2047 switch (patching_id(info)) {
2048 case PatchingStub::access_field_id:
2049 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
2050 break;
2051 case PatchingStub::load_klass_id:
2052 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
2053 break;
2054 case PatchingStub::load_mirror_id:
2055 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
2056 break;
2057 case PatchingStub::load_appendix_id:
2058 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
2059 break;
2060 default: ShouldNotReachHere();
2061 }
2062
2063 __ far_call(RuntimeAddress(target));
2064 add_call_info_here(info);
2065 }
2066
2067 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) {
2068 Label ok;
2069 __ load_klass(tmp, tmp);
2070 __ mov_metadata(t0, exact_klass->constant_encoding());
2071 __ beq(tmp, t0, ok);
2072 __ stop("exact klass and actual klass differ");
2073 __ bind(ok);
2074 }
2075
2076 void LIR_Assembler::get_op(BasicType type) {
2077 switch (type) {
2078 case T_INT:
2079 xchg = &MacroAssembler::atomic_xchgalw;
2080 add = &MacroAssembler::atomic_addalw;
2081 break;
2082 case T_LONG:
2083 xchg = &MacroAssembler::atomic_xchgal;
2084 add = &MacroAssembler::atomic_addal;
2085 break;
2086 case T_OBJECT:
2087 case T_ARRAY:
2088 if (UseCompressedOops) {
2089 xchg = &MacroAssembler::atomic_xchgalwu;
2090 add = &MacroAssembler::atomic_addalw;
2091 } else {
2092 xchg = &MacroAssembler::atomic_xchgal;
2093 add = &MacroAssembler::atomic_addal;
2094 }
2095 break;
2096 default:
2097 ShouldNotReachHere();
2098 }
2099 }
2100
2101 // emit_opTypeCheck sub functions
2102 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) {
2103 Register value = op->object()->as_register();
2104 Register array = op->array()->as_register();
2105 Register k_RInfo = op->tmp1()->as_register();
2106 Register klass_RInfo = op->tmp2()->as_register();
2107 Register Rtmp1 = op->tmp3()->as_register();
2108
2109 CodeStub* stub = op->stub();
2110
2111 // check if it needs to be profiled
2112 ciMethodData* md = NULL;
2113 ciProfileData* data = NULL;
2114
2115 if (should_profile) {
2116 data_check(op, &md, &data);
2117 }
2118 Label profile_cast_success, profile_cast_failure, done;
2119 Label *success_target = should_profile ? &profile_cast_success : &done;
2120 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
2121
2122 if (should_profile) {
2123 profile_object(md, data, value, klass_RInfo, &done);
2124 } else {
2125 __ beqz(value, done);
2126 }
2127
2128 add_debug_info_for_null_check_here(op->info_for_exception());
2129 __ load_klass(k_RInfo, array);
2130 __ load_klass(klass_RInfo, value);
2131
2132 lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target);
2133
2134 // fall through to the success case
2135 if (should_profile) {
2136 Register mdo = klass_RInfo;
2137 Register recv = k_RInfo;
2138 __ bind(profile_cast_success);
2139 __ mov_metadata(mdo, md->constant_encoding());
2140 __ load_klass(recv, value);
2141 type_profile_helper(mdo, md, data, recv, &done);
2142 __ j(done);
2143
2144 __ bind(profile_cast_failure);
2145 __ mov_metadata(mdo, md->constant_encoding());
2146 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2147 __ ld(t1, counter_addr);
2148 __ addi(t1, t1, -DataLayout::counter_increment);
2149 __ sd(t1, counter_addr);
2150 __ j(*stub->entry());
2151 }
2152
2153 __ bind(done);
2154 }
2155
2156 void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo,
2157 ciProfileData* data, Label* success, Label* failure,
2158 Label& profile_cast_success, Label& profile_cast_failure) {
2159 Register mdo = klass_RInfo;
2160 Register recv = k_RInfo;
2161 __ bind(profile_cast_success);
2162 __ mov_metadata(mdo, md->constant_encoding());
2163 __ load_klass(recv, obj);
2164 Label update_done;
2165 type_profile_helper(mdo, md, data, recv, success);
2166 __ j(*success);
2167
2168 __ bind(profile_cast_failure);
2169 __ mov_metadata(mdo, md->constant_encoding());
2170 Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2171 __ ld(t0, counter_addr);
2172 __ addi(t0, t0, -DataLayout::counter_increment);
2173 __ sd(t0, counter_addr);
2174 __ j(*failure);
2175 }
2176
2177 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1,
2178 Label* success_target, Label* failure_target) {
2179 // get instance klass (it's already uncompressed)
2180 __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2181 // perform the fast part of the checking logic
2182 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2183 // call out-of-line instance of __ check_klass_subtype_slow_path(...)
2184 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
2185 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
2186 __ sd(k_RInfo, Address(sp, 0)); // super klass
2187 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2188 // load result to k_RInfo
2189 __ ld(k_RInfo, Address(sp, 0));
2190 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
2191 // result is a boolean
2192 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
2193 }
2194
2195 void LIR_Assembler::const2reg_helper(LIR_Opr src) {
2196 switch (src->as_constant_ptr()->type()) {
2197 case T_INT:
2198 case T_ADDRESS:
2199 case T_OBJECT:
2200 case T_ARRAY:
2201 case T_METADATA:
2202 const2reg(src, FrameMap::t0_opr, lir_patch_none, NULL);
2203 break;
2204 case T_LONG:
2205 const2reg(src, FrameMap::t0_long_opr, lir_patch_none, NULL);
2206 break;
2207 case T_FLOAT:
2208 case T_DOUBLE:
2209 default:
2210 ShouldNotReachHere();
2211 }
2212 }
2213
2214 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) {
2215 switch (code) {
2216 case lir_logic_and: __ andrw(dst, left, right); break;
2217 case lir_logic_or: __ orrw (dst, left, right); break;
2218 case lir_logic_xor: __ xorrw(dst, left, right); break;
2219 default: ShouldNotReachHere();
2220 }
2221 }
2222
2223 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) {
2224 switch (code) {
2225 case lir_logic_and: __ andr(dst, left, right); break;
2226 case lir_logic_or: __ orr (dst, left, right); break;
2227 case lir_logic_xor: __ xorr(dst, left, right); break;
2228 default: ShouldNotReachHere();
2229 }
2230 }
2231
2232 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) {
2233 switch (code) {
2234 case lir_logic_and: __ andi(dst, left, right); break;
2235 case lir_logic_or: __ ori (dst, left, right); break;
2236 case lir_logic_xor: __ xori(dst, left, right); break;
2237 default: ShouldNotReachHere();
2238 }
2239 }
2240
2241 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2242 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2243 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2244 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2245 __ sd(r, Address(sp, offset_from_rsp_in_bytes));
2246 }
2247
2248 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2249 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2250 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2251 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2252 __ mv(t0, c);
2253 __ sd(t0, Address(sp, offset_from_rsp_in_bytes));
2254 }
2255
2256 #undef __