1 /*
2 * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "c1/c1_CodeStubs.hpp"
31 #include "c1/c1_Compilation.hpp"
32 #include "c1/c1_LIRAssembler.hpp"
33 #include "c1/c1_MacroAssembler.hpp"
34 #include "c1/c1_Runtime1.hpp"
35 #include "c1/c1_ValueStack.hpp"
36 #include "ci/ciArrayKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "nativeInst_riscv.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_riscv.inline.hpp"
46
47 #ifndef PRODUCT
48 #define COMMENT(x) do { __ block_comment(x); } while (0)
49 #else
50 #define COMMENT(x)
51 #endif
52
53 NEEDS_CLEANUP // remove this definitions ?
54 const Register IC_Klass = t1; // where the IC klass is cached
55 const Register SYNC_header = x10; // synchronization header
56 const Register SHIFT_count = x10; // where count for shift operations must be
57
58 #define __ _masm->
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74 static void select_different_registers(Register preserve,
75 Register extra,
76 Register &tmp1,
77 Register &tmp2,
78 Register &tmp3) {
79 if (tmp1 == preserve) {
80 assert_different_registers(tmp1, tmp2, tmp3, extra);
81 tmp1 = extra;
82 } else if (tmp2 == preserve) {
83 assert_different_registers(tmp1, tmp2, tmp3, extra);
84 tmp2 = extra;
85 } else if (tmp3 == preserve) {
86 assert_different_registers(tmp1, tmp2, tmp3, extra);
87 tmp3 = extra;
88 }
89 assert_different_registers(preserve, tmp1, tmp2, tmp3);
90 }
91
92 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
93
94 void LIR_Assembler::clinit_barrier(ciMethod* method) {
95 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
96 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
97
98 Label L_skip_barrier;
99
100 __ mov_metadata(t1, method->holder()->constant_encoding());
101 __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */);
102 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
103 __ bind(L_skip_barrier);
104 }
105
106 LIR_Opr LIR_Assembler::receiverOpr() {
107 return FrameMap::receiver_opr;
108 }
109
110 LIR_Opr LIR_Assembler::osrBufferPointer() {
111 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
112 }
113
114 void LIR_Assembler::breakpoint() { Unimplemented(); }
115
116 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
117
118 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
119
120 static jlong as_long(LIR_Opr data) {
121 jlong result;
122 switch (data->type()) {
123 case T_INT:
124 result = (data->as_jint());
125 break;
126 case T_LONG:
127 result = (data->as_jlong());
128 break;
129 default:
130 ShouldNotReachHere();
131 result = 0; // unreachable
132 }
133 return result;
134 }
135
136 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
137 ShouldNotReachHere();
138 return Address();
139 }
140
141 Address LIR_Assembler::as_Address(LIR_Address* addr) {
142 return as_Address(addr, t0);
143 }
144
145 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
146 return as_Address(addr);
147 }
148
149 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
150 // not encodable as a base + (immediate) offset, generate an explicit address
151 // calculation to hold the address in t0.
152 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) {
153 precond(size == 4 || size == 8);
154 Address addr = frame_map()->address_for_slot(index, adjust);
155 precond(addr.getMode() == Address::base_plus_offset);
156 precond(addr.base() == sp);
157 precond(addr.offset() > 0);
158 uint mask = size - 1;
159 assert((addr.offset() & mask) == 0, "scaled offsets only");
160
161 return addr;
162 }
163
164 void LIR_Assembler::osr_entry() {
165 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
166 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
167 guarantee(osr_entry != NULL, "NULL osr_entry!");
168 ValueStack* entry_state = osr_entry->state();
169 int number_of_locks = entry_state->locks_size();
170
171 // we jump here if osr happens with the interpreter
172 // state set up to continue at the beginning of the
173 // loop that triggered osr - in particular, we have
174 // the following registers setup:
175 //
176 // x12: osr buffer
177 //
178
179 //build frame
180 ciMethod* m = compilation()->method();
181 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
182
183 // OSR buffer is
184 //
185 // locals[nlocals-1..0]
186 // monitors[0..number_of_locks]
187 //
188 // locals is a direct copy of the interpreter frame so in the osr buffer
189 // so first slot in the local array is the last local from the interpreter
190 // and last slot is local[0] (receiver) from the interpreter
191 //
192 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
193 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
194 // in the interpreter frame (the method lock if a sync method)
195
196 // Initialize monitors in the compiled activation.
197 // x12: pointer to osr buffer
198 // All other registers are dead at this point and the locals will be
199 // copied into place by code emitted in the IR.
200
201 Register OSR_buf = osrBufferPointer()->as_pointer_register();
202 {
203 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
204 int monitor_offset = BytesPerWord * method()->max_locals() +
205 (2 * BytesPerWord) * (number_of_locks - 1);
206 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
207 // the OSR buffer using 2 word entries: first the lock and then
208 // the oop.
209 for (int i = 0; i < number_of_locks; i++) {
210 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
211 #ifdef ASSERT
212 // verify the interpreter's monitor has a non-null object
213 {
214 Label L;
215 __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
216 __ bnez(t0, L);
217 __ stop("locked object is NULL");
218 __ bind(L);
219 }
220 #endif // ASSERT
221 __ ld(x9, Address(OSR_buf, slot_offset + 0));
222 __ sd(x9, frame_map()->address_for_monitor_lock(i));
223 __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
224 __ sd(x9, frame_map()->address_for_monitor_object(i));
225 }
226 }
227 }
228
229 // inline cache check; done before the frame is built.
230 int LIR_Assembler::check_icache() {
231 Register receiver = FrameMap::receiver_opr->as_register();
232 Register ic_klass = IC_Klass;
233 int start_offset = __ offset();
234 Label dont;
235 __ inline_cache_check(receiver, ic_klass, dont);
236
237 // if icache check fails, then jump to runtime routine
238 // Note: RECEIVER must still contain the receiver!
239 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
240
241 // We align the verified entry point unless the method body
242 // (including its inline cache check) will fit in a single 64-byte
243 // icache line.
244 if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
245 // force alignment after the cache check.
246 __ align(CodeEntryAlignment);
247 }
248
249 __ bind(dont);
250 return start_offset;
251 }
252
253 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
254 if (o == NULL) {
255 __ mv(reg, zr);
256 } else {
257 __ movoop(reg, o, /* immediate */ true);
258 }
259 }
260
261 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
262 deoptimize_trap(info);
263 }
264
265 // This specifies the rsp decrement needed to build the frame
266 int LIR_Assembler::initial_frame_size_in_bytes() const {
267 // if rounding, must let FrameMap know!
268
269 return in_bytes(frame_map()->framesize_in_bytes());
270 }
271
272 int LIR_Assembler::emit_exception_handler() {
273 // if the last instruction is a call (typically to do a throw which
274 // is coming at the end after block reordering) the return address
275 // must still point into the code area in order to avoid assertion
276 // failures when searching for the corresponding bci ==> add a nop
277 // (was bug 5/14/1999 -gri)
278 __ nop();
279
280 // generate code for exception handler
281 address handler_base = __ start_a_stub(exception_handler_size());
282 if (handler_base == NULL) {
283 // not enough space left for the handler
284 bailout("exception handler overflow");
285 return -1;
286 }
287
288 int offset = code_offset();
289
290 // the exception oop and pc are in x10, and x13
291 // no other registers need to be preserved, so invalidate them
292 __ invalidate_registers(false, true, true, false, true, true);
293
294 // check that there is really an exception
295 __ verify_not_null_oop(x10);
296
297 // search an exception handler (x10: exception oop, x13: throwing pc)
298 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
299 __ should_not_reach_here();
300 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
301 __ end_a_stub();
302
303 return offset;
304 }
305
306 // Emit the code to remove the frame from the stack in the exception
307 // unwind path.
308 int LIR_Assembler::emit_unwind_handler() {
309 #ifndef PRODUCT
310 if (CommentedAssembly) {
311 _masm->block_comment("Unwind handler");
312 }
313 #endif // PRODUCT
314
315 int offset = code_offset();
316
317 // Fetch the exception from TLS and clear out exception related thread state
318 __ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
319 __ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
320 __ sd(zr, Address(xthread, JavaThread::exception_pc_offset()));
321
322 __ bind(_unwind_handler_entry);
323 __ verify_not_null_oop(x10);
324 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
325 __ mv(x9, x10); // Perserve the exception
326 }
327
328 // Preform needed unlocking
329 MonitorExitStub* stub = NULL;
330 if (method()->is_synchronized()) {
331 monitor_address(0, FrameMap::r10_opr);
332 stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
333 __ unlock_object(x15, x14, x10, *stub->entry());
334 __ bind(*stub->continuation());
335 }
336
337 if (compilation()->env()->dtrace_method_probes()) {
338 __ mv(c_rarg0, xthread);
339 __ mov_metadata(c_rarg1, method()->constant_encoding());
340 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
341 }
342
343 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
344 __ mv(x10, x9); // Restore the exception
345 }
346
347 // remove the activation and dispatch to the unwind handler
348 __ block_comment("remove_frame and dispatch to the unwind handler");
349 __ remove_frame(initial_frame_size_in_bytes());
350 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
351
352 // Emit the slow path assembly
353 if (stub != NULL) {
354 stub->emit_code(this);
355 }
356
357 return offset;
358 }
359
360 int LIR_Assembler::emit_deopt_handler() {
361 // if the last instruciton is a call (typically to do a throw which
362 // is coming at the end after block reordering) the return address
363 // must still point into the code area in order to avoid assertion
364 // failures when searching for the corresponding bck => add a nop
365 // (was bug 5/14/1999 - gri)
366 __ nop();
367
368 // generate code for exception handler
369 address handler_base = __ start_a_stub(deopt_handler_size());
370 if (handler_base == NULL) {
371 // not enough space left for the handler
372 bailout("deopt handler overflow");
373 return -1;
374 }
375
376 int offset = code_offset();
377
378 __ auipc(ra, 0);
379 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
380 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
381 __ end_a_stub();
382
383 return offset;
384 }
385
386 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
387 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
388
389 // Pop the stack before the safepoint code
390 __ remove_frame(initial_frame_size_in_bytes());
391
392 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
393 __ reserved_stack_check();
394 }
395
396 code_stub->set_safepoint_offset(__ offset());
397 __ relocate(relocInfo::poll_return_type);
398 __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
399 __ ret();
400 }
401
402 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
403 guarantee(info != NULL, "Shouldn't be NULL");
404 __ get_polling_page(t0, relocInfo::poll_type);
405 add_debug_info_for_branch(info); // This isn't just debug info:
406 // it's the oop map
407 __ read_polling_page(t0, 0, relocInfo::poll_type);
408 return __ offset();
409 }
410
411 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
412 __ mv(to_reg, from_reg);
413 }
414
415 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
416
417 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
418 assert(src->is_constant(), "should not call otherwise");
419 assert(dest->is_register(), "should not call otherwise");
420 LIR_Const* c = src->as_constant_ptr();
421 address const_addr = NULL;
422
423 switch (c->type()) {
424 case T_INT:
425 assert(patch_code == lir_patch_none, "no patching handled here");
426 __ mvw(dest->as_register(), c->as_jint());
427 break;
428
429 case T_ADDRESS:
430 assert(patch_code == lir_patch_none, "no patching handled here");
431 __ mv(dest->as_register(), c->as_jint());
432 break;
433
434 case T_LONG:
435 assert(patch_code == lir_patch_none, "no patching handled here");
436 __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
437 break;
438
439 case T_OBJECT:
440 case T_ARRAY:
441 if (patch_code == lir_patch_none) {
442 jobject2reg(c->as_jobject(), dest->as_register());
443 } else {
444 jobject2reg_with_patching(dest->as_register(), info);
445 }
446 break;
447
448 case T_METADATA:
449 if (patch_code != lir_patch_none) {
450 klass2reg_with_patching(dest->as_register(), info);
451 } else {
452 __ mov_metadata(dest->as_register(), c->as_metadata());
453 }
454 break;
455
456 case T_FLOAT:
457 const_addr = float_constant(c->as_jfloat());
458 assert(const_addr != NULL, "must create float constant in the constant table");
459 __ flw(dest->as_float_reg(), InternalAddress(const_addr));
460 break;
461
462 case T_DOUBLE:
463 const_addr = double_constant(c->as_jdouble());
464 assert(const_addr != NULL, "must create double constant in the constant table");
465 __ fld(dest->as_double_reg(), InternalAddress(const_addr));
466 break;
467
468 default:
469 ShouldNotReachHere();
470 }
471 }
472
473 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
474 assert(src->is_constant(), "should not call otherwise");
475 assert(dest->is_stack(), "should not call otherwise");
476 LIR_Const* c = src->as_constant_ptr();
477 switch (c->type()) {
478 case T_OBJECT:
479 if (c->as_jobject() == NULL) {
480 __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
481 } else {
482 const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
483 reg2stack(FrameMap::t1_opr, dest, c->type(), false);
484 }
485 break;
486 case T_ADDRESS: // fall through
487 const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
488 reg2stack(FrameMap::t1_opr, dest, c->type(), false);
489 case T_INT: // fall through
490 case T_FLOAT:
491 if (c->as_jint_bits() == 0) {
492 __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
493 } else {
494 __ mvw(t1, c->as_jint_bits());
495 __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix()));
496 }
497 break;
498 case T_LONG: // fall through
499 case T_DOUBLE:
500 if (c->as_jlong_bits() == 0) {
501 __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
502 lo_word_offset_in_bytes));
503 } else {
504 __ mv(t1, (intptr_t)c->as_jlong_bits());
505 __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(),
506 lo_word_offset_in_bytes));
507 }
508 break;
509 default:
510 ShouldNotReachHere();
511 }
512 }
513
514 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
515 assert(src->is_constant(), "should not call otherwise");
516 assert(dest->is_address(), "should not call otherwise");
517 LIR_Const* c = src->as_constant_ptr();
518 LIR_Address* to_addr = dest->as_address_ptr();
519 void (Assembler::* insn)(Register Rt, const Address &adr, Register temp);
520 switch (type) {
521 case T_ADDRESS:
522 assert(c->as_jint() == 0, "should be");
523 insn = &Assembler::sd; break;
524 case T_LONG:
525 assert(c->as_jlong() == 0, "should be");
526 insn = &Assembler::sd; break;
527 case T_DOUBLE:
528 assert(c->as_jdouble() == 0.0, "should be");
529 insn = &Assembler::sd; break;
530 case T_INT:
531 assert(c->as_jint() == 0, "should be");
532 insn = &Assembler::sw; break;
533 case T_FLOAT:
534 assert(c->as_jfloat() == 0.0f, "should be");
535 insn = &Assembler::sw; break;
536 case T_OBJECT: // fall through
537 case T_ARRAY:
538 assert(c->as_jobject() == 0, "should be");
539 if (UseCompressedOops && !wide) {
540 insn = &Assembler::sw;
541 } else {
542 insn = &Assembler::sd;
543 }
544 break;
545 case T_CHAR: // fall through
546 case T_SHORT:
547 assert(c->as_jint() == 0, "should be");
548 insn = &Assembler::sh;
549 break;
550 case T_BOOLEAN: // fall through
551 case T_BYTE:
552 assert(c->as_jint() == 0, "should be");
553 insn = &Assembler::sb; break;
554 default:
555 ShouldNotReachHere();
556 insn = &Assembler::sd; // unreachable
557 }
558 if (info != NULL) {
559 add_debug_info_for_null_check_here(info);
560 }
561 (_masm->*insn)(zr, as_Address(to_addr), t0);
562 }
563
564 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
565 assert(src->is_register(), "should not call otherwise");
566 assert(dest->is_register(), "should not call otherwise");
567
568 // move between cpu-registers
569 if (dest->is_single_cpu()) {
570 if (src->type() == T_LONG) {
571 // Can do LONG -> OBJECT
572 move_regs(src->as_register_lo(), dest->as_register());
573 return;
574 }
575 assert(src->is_single_cpu(), "must match");
576 if (src->type() == T_OBJECT) {
577 __ verify_oop(src->as_register());
578 }
579 move_regs(src->as_register(), dest->as_register());
580 } else if (dest->is_double_cpu()) {
581 if (is_reference_type(src->type())) {
582 __ verify_oop(src->as_register());
583 move_regs(src->as_register(), dest->as_register_lo());
584 return;
585 }
586 assert(src->is_double_cpu(), "must match");
587 Register f_lo = src->as_register_lo();
588 Register f_hi = src->as_register_hi();
589 Register t_lo = dest->as_register_lo();
590 Register t_hi = dest->as_register_hi();
591 assert(f_hi == f_lo, "must be same");
592 assert(t_hi == t_lo, "must be same");
593 move_regs(f_lo, t_lo);
594 } else if (dest->is_single_fpu()) {
595 assert(src->is_single_fpu(), "expect single fpu");
596 __ fmv_s(dest->as_float_reg(), src->as_float_reg());
597 } else if (dest->is_double_fpu()) {
598 assert(src->is_double_fpu(), "expect double fpu");
599 __ fmv_d(dest->as_double_reg(), src->as_double_reg());
600 } else {
601 ShouldNotReachHere();
602 }
603 }
604
605 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
606 precond(src->is_register() && dest->is_stack());
607
608 uint const c_sz32 = sizeof(uint32_t);
609 uint const c_sz64 = sizeof(uint64_t);
610
611 assert(src->is_register(), "should not call otherwise");
612 assert(dest->is_stack(), "should not call otherwise");
613 if (src->is_single_cpu()) {
614 int index = dest->single_stack_ix();
615 if (is_reference_type(type)) {
616 __ sd(src->as_register(), stack_slot_address(index, c_sz64));
617 __ verify_oop(src->as_register());
618 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
619 __ sd(src->as_register(), stack_slot_address(index, c_sz64));
620 } else {
621 __ sw(src->as_register(), stack_slot_address(index, c_sz32));
622 }
623 } else if (src->is_double_cpu()) {
624 int index = dest->double_stack_ix();
625 Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
626 __ sd(src->as_register_lo(), dest_addr_LO);
627 } else if (src->is_single_fpu()) {
628 int index = dest->single_stack_ix();
629 __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32));
630 } else if (src->is_double_fpu()) {
631 int index = dest->double_stack_ix();
632 __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64));
633 } else {
634 ShouldNotReachHere();
635 }
636 }
637
638 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
639 LIR_Address* to_addr = dest->as_address_ptr();
640 // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src
641 Register compressed_src = t1;
642
643 if (patch_code != lir_patch_none) {
644 deoptimize_trap(info);
645 return;
646 }
647
648 if (is_reference_type(type)) {
649 __ verify_oop(src->as_register());
650
651 if (UseCompressedOops && !wide) {
652 __ encode_heap_oop(compressed_src, src->as_register());
653 } else {
654 compressed_src = src->as_register();
655 }
656 }
657
658 int null_check_here = code_offset();
659
660 switch (type) {
661 case T_FLOAT:
662 __ fsw(src->as_float_reg(), as_Address(to_addr));
663 break;
664
665 case T_DOUBLE:
666 __ fsd(src->as_double_reg(), as_Address(to_addr));
667 break;
668
669 case T_ARRAY: // fall through
670 case T_OBJECT:
671 if (UseCompressedOops && !wide) {
672 __ sw(compressed_src, as_Address(to_addr));
673 } else {
674 __ sd(compressed_src, as_Address(to_addr));
675 }
676 break;
677 case T_METADATA:
678 // We get here to store a method pointer to the stack to pass to
679 // a dtrace runtime call. This can't work on 64 bit with
680 // compressed klass ptrs: T_METADATA can be compressed klass
681 // ptr or a 64 bit method pointer.
682 ShouldNotReachHere();
683 __ sd(src->as_register(), as_Address(to_addr));
684 break;
685 case T_ADDRESS:
686 __ sd(src->as_register(), as_Address(to_addr));
687 break;
688 case T_INT:
689 __ sw(src->as_register(), as_Address(to_addr));
690 break;
691 case T_LONG:
692 __ sd(src->as_register_lo(), as_Address(to_addr));
693 break;
694 case T_BYTE: // fall through
695 case T_BOOLEAN:
696 __ sb(src->as_register(), as_Address(to_addr));
697 break;
698 case T_CHAR: // fall through
699 case T_SHORT:
700 __ sh(src->as_register(), as_Address(to_addr));
701 break;
702 default:
703 ShouldNotReachHere();
704 }
705
706 if (info != NULL) {
707 add_debug_info_for_null_check(null_check_here, info);
708 }
709 }
710
711 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
712 precond(src->is_stack() && dest->is_register());
713
714 uint const c_sz32 = sizeof(uint32_t);
715 uint const c_sz64 = sizeof(uint64_t);
716
717 if (dest->is_single_cpu()) {
718 int index = src->single_stack_ix();
719 if (type == T_INT) {
720 __ lw(dest->as_register(), stack_slot_address(index, c_sz32));
721 } else if (is_reference_type(type)) {
722 __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
723 __ verify_oop(dest->as_register());
724 } else if (type == T_METADATA || type == T_ADDRESS) {
725 __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
726 } else {
727 __ lwu(dest->as_register(), stack_slot_address(index, c_sz32));
728 }
729 } else if (dest->is_double_cpu()) {
730 int index = src->double_stack_ix();
731 Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
732 __ ld(dest->as_register_lo(), src_addr_LO);
733 } else if (dest->is_single_fpu()) {
734 int index = src->single_stack_ix();
735 __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32));
736 } else if (dest->is_double_fpu()) {
737 int index = src->double_stack_ix();
738 __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64));
739 } else {
740 ShouldNotReachHere();
741 }
742 }
743
744 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
745 deoptimize_trap(info);
746 }
747
748 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
749 LIR_Opr temp;
750 if (type == T_LONG || type == T_DOUBLE) {
751 temp = FrameMap::t1_long_opr;
752 } else {
753 temp = FrameMap::t1_opr;
754 }
755
756 stack2reg(src, temp, src->type());
757 reg2stack(temp, dest, dest->type(), false);
758 }
759
760 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
761 assert(src->is_address(), "should not call otherwise");
762 assert(dest->is_register(), "should not call otherwise");
763
764 LIR_Address* addr = src->as_address_ptr();
765 LIR_Address* from_addr = src->as_address_ptr();
766
767 if (addr->base()->type() == T_OBJECT) {
768 __ verify_oop(addr->base()->as_pointer_register());
769 }
770
771 if (patch_code != lir_patch_none) {
772 deoptimize_trap(info);
773 return;
774 }
775
776 if (info != NULL) {
777 add_debug_info_for_null_check_here(info);
778 }
779
780 int null_check_here = code_offset();
781 switch (type) {
782 case T_FLOAT:
783 __ flw(dest->as_float_reg(), as_Address(from_addr));
784 break;
785 case T_DOUBLE:
786 __ fld(dest->as_double_reg(), as_Address(from_addr));
787 break;
788 case T_ARRAY: // fall through
789 case T_OBJECT:
790 if (UseCompressedOops && !wide) {
791 __ lwu(dest->as_register(), as_Address(from_addr));
792 } else {
793 __ ld(dest->as_register(), as_Address(from_addr));
794 }
795 break;
796 case T_METADATA:
797 // We get here to store a method pointer to the stack to pass to
798 // a dtrace runtime call. This can't work on 64 bit with
799 // compressed klass ptrs: T_METADATA can be a compressed klass
800 // ptr or a 64 bit method pointer.
801 ShouldNotReachHere();
802 __ ld(dest->as_register(), as_Address(from_addr));
803 break;
804 case T_ADDRESS:
805 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
806 __ lwu(dest->as_register(), as_Address(from_addr));
807 } else {
808 __ ld(dest->as_register(), as_Address(from_addr));
809 }
810 break;
811 case T_INT:
812 __ lw(dest->as_register(), as_Address(from_addr));
813 break;
814 case T_LONG:
815 __ ld(dest->as_register_lo(), as_Address_lo(from_addr));
816 break;
817 case T_BYTE:
818 __ lb(dest->as_register(), as_Address(from_addr));
819 break;
820 case T_BOOLEAN:
821 __ lbu(dest->as_register(), as_Address(from_addr));
822 break;
823 case T_CHAR:
824 __ lhu(dest->as_register(), as_Address(from_addr));
825 break;
826 case T_SHORT:
827 __ lh(dest->as_register(), as_Address(from_addr));
828 break;
829 default:
830 ShouldNotReachHere();
831 }
832
833 if (is_reference_type(type)) {
834 if (UseCompressedOops && !wide) {
835 __ decode_heap_oop(dest->as_register());
836 }
837
838 if (!UseZGC) {
839 // Load barrier has not yet been applied, so ZGC can't verify the oop here
840 __ verify_oop(dest->as_register());
841 }
842 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
843 if (UseCompressedClassPointers) {
844 __ decode_klass_not_null(dest->as_register());
845 }
846 }
847 }
848
849 void LIR_Assembler::emit_op3(LIR_Op3* op) {
850 switch (op->code()) {
851 case lir_idiv: // fall through
852 case lir_irem:
853 arithmetic_idiv(op->code(),
854 op->in_opr1(),
855 op->in_opr2(),
856 op->in_opr3(),
857 op->result_opr(),
858 op->info());
859 break;
860 case lir_fmad:
861 __ fmadd_d(op->result_opr()->as_double_reg(),
862 op->in_opr1()->as_double_reg(),
863 op->in_opr2()->as_double_reg(),
864 op->in_opr3()->as_double_reg());
865 break;
866 case lir_fmaf:
867 __ fmadd_s(op->result_opr()->as_float_reg(),
868 op->in_opr1()->as_float_reg(),
869 op->in_opr2()->as_float_reg(),
870 op->in_opr3()->as_float_reg());
871 break;
872 default:
873 ShouldNotReachHere();
874 }
875 }
876
877 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr cmp_opr1, LIR_Opr cmp_opr2, LIR_Opr result, BasicType type) {
878 Label label;
879
880 emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false,
881 /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true);
882
883 Label done;
884 move_op(opr2, result, type, lir_patch_none, NULL,
885 false, // pop_fpu_stack
886 false); // wide
887 __ j(done);
888 __ bind(label);
889 move_op(opr1, result, type, lir_patch_none, NULL,
890 false, // pop_fpu_stack
891 false); // wide
892 __ bind(done);
893 }
894
895 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
896 LIR_Condition condition = op->cond();
897 if (condition == lir_cond_always) {
898 if (op->info() != NULL) {
899 add_debug_info_for_branch(op->info());
900 }
901 } else {
902 assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands");
903 }
904 bool is_unordered = (op->ublock() == op->block());
905 emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered);
906 }
907
908 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label,
909 bool is_far, bool is_unordered) {
910
911 if (cmp_flag == lir_cond_always) {
912 __ j(label);
913 return;
914 }
915
916 if (cmp1->is_cpu_register()) {
917 Register reg1 = as_reg(cmp1);
918 if (cmp2->is_cpu_register()) {
919 Register reg2 = as_reg(cmp2);
920 __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far);
921 } else if (cmp2->is_constant()) {
922 const2reg_helper(cmp2);
923 __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far);
924 } else {
925 ShouldNotReachHere();
926 }
927 } else if (cmp1->is_single_fpu()) {
928 assert(cmp2->is_single_fpu(), "expect single float register");
929 __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered);
930 } else if (cmp1->is_double_fpu()) {
931 assert(cmp2->is_double_fpu(), "expect double float register");
932 __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask,
933 cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered);
934 } else {
935 ShouldNotReachHere();
936 }
937 }
938
939 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
940 LIR_Opr src = op->in_opr();
941 LIR_Opr dest = op->result_opr();
942
943 switch (op->bytecode()) {
944 case Bytecodes::_i2f:
945 __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break;
946 case Bytecodes::_i2d:
947 __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break;
948 case Bytecodes::_l2d:
949 __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break;
950 case Bytecodes::_l2f:
951 __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break;
952 case Bytecodes::_f2d:
953 __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break;
954 case Bytecodes::_d2f:
955 __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break;
956 case Bytecodes::_i2c:
957 __ zero_ext(dest->as_register(), src->as_register(), registerSize - 16); break; // 16: char size
958 case Bytecodes::_i2l:
959 __ addw(dest->as_register_lo(), src->as_register(), zr); break;
960 case Bytecodes::_i2s:
961 __ sign_ext(dest->as_register(), src->as_register(), registerSize - 16); break; // 16: short size
962 case Bytecodes::_i2b:
963 __ sign_ext(dest->as_register(), src->as_register(), registerSize - 8); break; // 8: byte size
964 case Bytecodes::_l2i:
965 _masm->block_comment("FIXME: This coulde be no-op");
966 __ addw(dest->as_register(), src->as_register_lo(), zr); break;
967 case Bytecodes::_d2l:
968 __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break;
969 case Bytecodes::_f2i:
970 __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break;
971 case Bytecodes::_f2l:
972 __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break;
973 case Bytecodes::_d2i:
974 __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break;
975 default:
976 ShouldNotReachHere();
977 }
978 }
979
980 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
981 if (op->init_check()) {
982 __ lbu(t0, Address(op->klass()->as_register(),
983 InstanceKlass::init_state_offset()));
984 __ mvw(t1, InstanceKlass::fully_initialized);
985 add_debug_info_for_null_check_here(op->stub()->info());
986 __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
987 }
988
989 __ allocate_object(op->obj()->as_register(),
990 op->tmp1()->as_register(),
991 op->tmp2()->as_register(),
992 op->header_size(),
993 op->object_size(),
994 op->klass()->as_register(),
995 *op->stub()->entry());
996
997 __ bind(*op->stub()->continuation());
998 }
999
1000 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1001 Register len = op->len()->as_register();
1002
1003 if (UseSlowPath ||
1004 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1005 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1006 __ j(*op->stub()->entry());
1007 } else {
1008 Register tmp1 = op->tmp1()->as_register();
1009 Register tmp2 = op->tmp2()->as_register();
1010 Register tmp3 = op->tmp3()->as_register();
1011 if (len == tmp1) {
1012 tmp1 = tmp3;
1013 } else if (len == tmp2) {
1014 tmp2 = tmp3;
1015 } else if (len == tmp3) {
1016 // everything is ok
1017 } else {
1018 __ mv(tmp3, len);
1019 }
1020 __ allocate_array(op->obj()->as_register(),
1021 len,
1022 tmp1,
1023 tmp2,
1024 arrayOopDesc::header_size(op->type()),
1025 array_element_size(op->type()),
1026 op->klass()->as_register(),
1027 *op->stub()->entry());
1028 }
1029 __ bind(*op->stub()->continuation());
1030 }
1031
1032 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
1033 Register recv, Label* update_done) {
1034 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1035 Label next_test;
1036 // See if the receiver is receiver[n].
1037 __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1038 __ bne(recv, t1, next_test);
1039 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1040 __ add_memory_int64(data_addr, DataLayout::counter_increment);
1041 __ j(*update_done);
1042 __ bind(next_test);
1043 }
1044
1045 // Didn't find receiver; find next empty slot and fill it in
1046 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1047 Label next_test;
1048 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1049 __ ld(t1, recv_addr);
1050 __ bnez(t1, next_test);
1051 __ sd(recv, recv_addr);
1052 __ li(t1, DataLayout::counter_increment);
1053 __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1054 __ j(*update_done);
1055 __ bind(next_test);
1056 }
1057 }
1058
1059 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
1060 ciMethod* method = op->profiled_method();
1061 assert(method != NULL, "Should have method");
1062 int bci = op->profiled_bci();
1063 *md = method->method_data_or_null();
1064 guarantee(*md != NULL, "Sanity");
1065 *data = ((*md)->bci_to_data(bci));
1066 assert(*data != NULL, "need data for type check");
1067 assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1068 }
1069
1070 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1071 Register k_RInfo, Register klass_RInfo,
1072 Label *failure_target, Label *success_target) {
1073 // get object class
1074 // not a safepoint as obj null check happens earlier
1075 __ load_klass(klass_RInfo, obj);
1076 if (k->is_loaded()) {
1077 // See if we get an immediate positive hit
1078 __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1079 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1080 __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1081 // successful cast, fall through to profile or jump
1082 } else {
1083 // See if we get an immediate positive hit
1084 __ beq(k_RInfo, t0, *success_target);
1085 // check for self
1086 __ beq(klass_RInfo, k_RInfo, *success_target);
1087
1088 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1089 __ sd(k_RInfo, Address(sp, 0)); // sub klass
1090 __ sd(klass_RInfo, Address(sp, wordSize)); // super klass
1091 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1092 // load result to k_RInfo
1093 __ ld(k_RInfo, Address(sp, 0));
1094 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1095 // result is a boolean
1096 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1097 // successful cast, fall through to profile or jump
1098 }
1099 } else {
1100 // perform the fast part of the checking logic
1101 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1102 // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1103 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1104 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
1105 __ sd(k_RInfo, Address(sp, 0)); // super klass
1106 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1107 // load result to k_RInfo
1108 __ ld(k_RInfo, Address(sp, 0));
1109 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1110 // result is a boolean
1111 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1112 // successful cast, fall thriugh to profile or jump
1113 }
1114 }
1115
1116 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj,
1117 Register klass_RInfo, Label* obj_is_null) {
1118 Label not_null;
1119 __ bnez(obj, not_null);
1120 // Object is null, update MDO and exit
1121 Register mdo = klass_RInfo;
1122 __ mov_metadata(mdo, md->constant_encoding());
1123 Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1124 __ lbu(t0, data_addr);
1125 __ ori(t0, t0, BitData::null_seen_byte_constant());
1126 __ sb(t0, data_addr);
1127 __ j(*obj_is_null);
1128 __ bind(not_null);
1129 }
1130
1131 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
1132 if (!k->is_loaded()) {
1133 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1134 } else {
1135 __ mov_metadata(k_RInfo, k->constant_encoding());
1136 }
1137 }
1138
1139 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1140 Register obj = op->object()->as_register();
1141 Register k_RInfo = op->tmp1()->as_register();
1142 Register klass_RInfo = op->tmp2()->as_register();
1143 Register dst = op->result_opr()->as_register();
1144 ciKlass* k = op->klass();
1145 Register Rtmp1 = noreg;
1146
1147 // check if it needs to be profiled
1148 ciMethodData* md = NULL;
1149 ciProfileData* data = NULL;
1150
1151 const bool should_profile = op->should_profile();
1152 if (should_profile) {
1153 data_check(op, &md, &data);
1154 }
1155 Label profile_cast_success, profile_cast_failure;
1156 Label *success_target = should_profile ? &profile_cast_success : success;
1157 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1158
1159 if (obj == k_RInfo) {
1160 k_RInfo = dst;
1161 } else if (obj == klass_RInfo) {
1162 klass_RInfo = dst;
1163 }
1164 if (k->is_loaded() && !UseCompressedClassPointers) {
1165 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1166 } else {
1167 Rtmp1 = op->tmp3()->as_register();
1168 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1169 }
1170
1171 assert_different_registers(obj, k_RInfo, klass_RInfo);
1172
1173 if (should_profile) {
1174 profile_object(md, data, obj, klass_RInfo, obj_is_null);
1175 } else {
1176 __ beqz(obj, *obj_is_null);
1177 }
1178
1179 typecheck_loaded(op, k, k_RInfo);
1180 __ verify_oop(obj);
1181
1182 if (op->fast_check()) {
1183 // get object class
1184 // not a safepoint as obj null check happens earlier
1185 __ load_klass(t0, obj);
1186 __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1187 // successful cast, fall through to profile or jump
1188 } else {
1189 typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1190 }
1191 if (should_profile) {
1192 type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure);
1193 }
1194 __ j(*success);
1195 }
1196
1197 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1198 const bool should_profile = op->should_profile();
1199
1200 LIR_Code code = op->code();
1201 if (code == lir_store_check) {
1202 typecheck_lir_store(op, should_profile);
1203 } else if (code == lir_checkcast) {
1204 Register obj = op->object()->as_register();
1205 Register dst = op->result_opr()->as_register();
1206 Label success;
1207 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1208 __ bind(success);
1209 if (dst != obj) {
1210 __ mv(dst, obj);
1211 }
1212 } else if (code == lir_instanceof) {
1213 Register obj = op->object()->as_register();
1214 Register dst = op->result_opr()->as_register();
1215 Label success, failure, done;
1216 emit_typecheck_helper(op, &success, &failure, &failure);
1217 __ bind(failure);
1218 __ mv(dst, zr);
1219 __ j(done);
1220 __ bind(success);
1221 __ mv(dst, 1);
1222 __ bind(done);
1223 } else {
1224 ShouldNotReachHere();
1225 }
1226 }
1227
1228 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1229 assert(VM_Version::supports_cx8(), "wrong machine");
1230 Register addr;
1231 if (op->addr()->is_register()) {
1232 addr = as_reg(op->addr());
1233 } else {
1234 assert(op->addr()->is_address(), "what else?");
1235 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1236 assert(addr_ptr->disp() == 0, "need 0 disp");
1237 assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1238 addr = as_reg(addr_ptr->base());
1239 }
1240 Register newval = as_reg(op->new_value());
1241 Register cmpval = as_reg(op->cmp_value());
1242
1243 if (op->code() == lir_cas_obj) {
1244 if (UseCompressedOops) {
1245 Register tmp1 = op->tmp1()->as_register();
1246 assert(op->tmp1()->is_valid(), "must be");
1247 __ encode_heap_oop(tmp1, cmpval);
1248 cmpval = tmp1;
1249 __ encode_heap_oop(t1, newval);
1250 newval = t1;
1251 caswu(addr, newval, cmpval);
1252 } else {
1253 casl(addr, newval, cmpval);
1254 }
1255 } else if (op->code() == lir_cas_int) {
1256 casw(addr, newval, cmpval);
1257 } else {
1258 casl(addr, newval, cmpval);
1259 }
1260 }
1261
1262 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1263 switch (code) {
1264 case lir_abs: __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break;
1265 case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break;
1266 default: ShouldNotReachHere();
1267 }
1268 }
1269
1270 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1271 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1272 Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1273 if (dst->is_single_cpu()) {
1274 Register Rdst = dst->as_register();
1275 if (right->is_constant()) {
1276 int right_const = right->as_jint();
1277 if (Assembler::operand_valid_for_add_immediate(right_const)) {
1278 logic_op_imm(Rdst, Rleft, right_const, code);
1279 __ addw(Rdst, Rdst, zr);
1280 } else {
1281 __ mv(t0, right_const);
1282 logic_op_reg32(Rdst, Rleft, t0, code);
1283 }
1284 } else {
1285 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1286 logic_op_reg32(Rdst, Rleft, Rright, code);
1287 }
1288 } else {
1289 Register Rdst = dst->as_register_lo();
1290 if (right->is_constant()) {
1291 long right_const = right->as_jlong();
1292 if (Assembler::operand_valid_for_add_immediate(right_const)) {
1293 logic_op_imm(Rdst, Rleft, right_const, code);
1294 } else {
1295 __ mv(t0, right_const);
1296 logic_op_reg(Rdst, Rleft, t0, code);
1297 }
1298 } else {
1299 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1300 logic_op_reg(Rdst, Rleft, Rright, code);
1301 }
1302 }
1303 }
1304
1305 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) {
1306 ShouldNotCallThis();
1307 }
1308
1309 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1310 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1311 bool is_unordered_less = (code == lir_ucmp_fd2i);
1312 if (left->is_single_fpu()) {
1313 __ float_cmp(true, is_unordered_less ? -1 : 1,
1314 left->as_float_reg(), right->as_float_reg(), dst->as_register());
1315 } else if (left->is_double_fpu()) {
1316 __ float_cmp(false, is_unordered_less ? -1 : 1,
1317 left->as_double_reg(), right->as_double_reg(), dst->as_register());
1318 } else {
1319 ShouldNotReachHere();
1320 }
1321 } else if (code == lir_cmp_l2i) {
1322 __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1323 } else {
1324 ShouldNotReachHere();
1325 }
1326 }
1327
1328 void LIR_Assembler::align_call(LIR_Code code) { }
1329
1330 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1331 address call = __ trampoline_call(Address(op->addr(), rtype));
1332 if (call == NULL) {
1333 bailout("trampoline stub overflow");
1334 return;
1335 }
1336 add_call_info(code_offset(), op->info());
1337 }
1338
1339 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1340 address call = __ ic_call(op->addr());
1341 if (call == NULL) {
1342 bailout("trampoline stub overflow");
1343 return;
1344 }
1345 add_call_info(code_offset(), op->info());
1346 }
1347
1348 void LIR_Assembler::emit_static_call_stub() {
1349 address call_pc = __ pc();
1350 address stub = __ start_a_stub(call_stub_size());
1351 if (stub == NULL) {
1352 bailout("static call stub overflow");
1353 return;
1354 }
1355
1356 int start = __ offset();
1357
1358 __ relocate(static_stub_Relocation::spec(call_pc));
1359 __ emit_static_call_stub();
1360
1361 assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size() <= call_stub_size(), "stub too big");
1362 __ end_a_stub();
1363 }
1364
1365 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1366 assert(exceptionOop->as_register() == x10, "must match");
1367 assert(exceptionPC->as_register() == x13, "must match");
1368
1369 // exception object is not added to oop map by LinearScan
1370 // (LinearScan assumes that no oops are in fixed registers)
1371 info->add_register_oop(exceptionOop);
1372 Runtime1::StubID unwind_id;
1373
1374 // get current pc information
1375 // pc is only needed if the method has an exception handler, the unwind code does not need it.
1376 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
1377 // As no instructions have been generated yet for this LIR node it's
1378 // possible that an oop map already exists for the current offset.
1379 // In that case insert an dummy NOP here to ensure all oop map PCs
1380 // are unique. See JDK-8237483.
1381 __ nop();
1382 }
1383 int pc_for_athrow_offset = __ offset();
1384 InternalAddress pc_for_athrow(__ pc());
1385 int32_t off = 0;
1386 __ la_patchable(exceptionPC->as_register(), pc_for_athrow, off);
1387 __ addi(exceptionPC->as_register(), exceptionPC->as_register(), off);
1388 add_call_info(pc_for_athrow_offset, info); // for exception handler
1389
1390 __ verify_not_null_oop(x10);
1391 // search an exception handler (x10: exception oop, x13: throwing pc)
1392 if (compilation()->has_fpu_code()) {
1393 unwind_id = Runtime1::handle_exception_id;
1394 } else {
1395 unwind_id = Runtime1::handle_exception_nofpu_id;
1396 }
1397 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
1398 __ nop();
1399 }
1400
1401 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1402 assert(exceptionOop->as_register() == x10, "must match");
1403 __ j(_unwind_handler_entry);
1404 }
1405
1406 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
1407 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1408 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1409 Register count_reg = count->as_register();
1410 if (dest->is_single_cpu()) {
1411 assert (dest->type() == T_INT, "unexpected result type");
1412 assert (left->type() == T_INT, "unexpected left type");
1413 __ andi(t0, count_reg, 31); // should not shift more than 31 bits
1414 switch (code) {
1415 case lir_shl: __ sllw(dest_reg, left_reg, t0); break;
1416 case lir_shr: __ sraw(dest_reg, left_reg, t0); break;
1417 case lir_ushr: __ srlw(dest_reg, left_reg, t0); break;
1418 default: ShouldNotReachHere();
1419 }
1420 } else if (dest->is_double_cpu()) {
1421 __ andi(t0, count_reg, 63); // should not shift more than 63 bits
1422 switch (code) {
1423 case lir_shl: __ sll(dest_reg, left_reg, t0); break;
1424 case lir_shr: __ sra(dest_reg, left_reg, t0); break;
1425 case lir_ushr: __ srl(dest_reg, left_reg, t0); break;
1426 default: ShouldNotReachHere();
1427 }
1428 } else {
1429 ShouldNotReachHere();
1430 }
1431 }
1432
1433 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
1434 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1435 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1436 if (dest->is_single_cpu()) {
1437 assert (dest->type() == T_INT, "unexpected result type");
1438 assert (left->type() == T_INT, "unexpected left type");
1439 count &= 0x1f;
1440 if (count != 0) {
1441 switch (code) {
1442 case lir_shl: __ slliw(dest_reg, left_reg, count); break;
1443 case lir_shr: __ sraiw(dest_reg, left_reg, count); break;
1444 case lir_ushr: __ srliw(dest_reg, left_reg, count); break;
1445 default: ShouldNotReachHere();
1446 }
1447 } else {
1448 move_regs(left_reg, dest_reg);
1449 }
1450 } else if (dest->is_double_cpu()) {
1451 count &= 0x3f;
1452 if (count != 0) {
1453 switch (code) {
1454 case lir_shl: __ slli(dest_reg, left_reg, count); break;
1455 case lir_shr: __ srai(dest_reg, left_reg, count); break;
1456 case lir_ushr: __ srli(dest_reg, left_reg, count); break;
1457 default: ShouldNotReachHere();
1458 }
1459 } else {
1460 move_regs(left->as_register_lo(), dest->as_register_lo());
1461 }
1462 } else {
1463 ShouldNotReachHere();
1464 }
1465 }
1466
1467 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
1468 Register obj = op->obj_opr()->as_register(); // may not be an oop
1469 Register hdr = op->hdr_opr()->as_register();
1470 Register lock = op->lock_opr()->as_register();
1471 if (!UseFastLocking) {
1472 __ j(*op->stub()->entry());
1473 } else if (op->code() == lir_lock) {
1474 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1475 // add debug info for NullPointerException only if one is possible
1476 int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
1477 if (op->info() != NULL) {
1478 add_debug_info_for_null_check(null_check_offset, op->info());
1479 }
1480 } else if (op->code() == lir_unlock) {
1481 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1482 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
1483 } else {
1484 Unimplemented();
1485 }
1486 __ bind(*op->stub()->continuation());
1487 }
1488
1489 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
1490 ciMethod* method = op->profiled_method();
1491 int bci = op->profiled_bci();
1492
1493 // Update counter for all call types
1494 ciMethodData* md = method->method_data_or_null();
1495 guarantee(md != NULL, "Sanity");
1496 ciProfileData* data = md->bci_to_data(bci);
1497 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
1498 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
1499 Register mdo = op->mdo()->as_register();
1500 __ mov_metadata(mdo, md->constant_encoding());
1501 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1502 // Perform additional virtual call profiling for invokevirtual and
1503 // invokeinterface bytecodes
1504 if (op->should_profile_receiver_type()) {
1505 assert(op->recv()->is_single_cpu(), "recv must be allocated");
1506 Register recv = op->recv()->as_register();
1507 assert_different_registers(mdo, recv);
1508 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
1509 ciKlass* known_klass = op->known_holder();
1510 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
1511 // We know the type that will be seen at this call site; we can
1512 // statically update the MethodData* rather than needing to do
1513 // dynamic tests on the receiver type
1514 // NOTE: we should probably put a lock around this search to
1515 // avoid collisions by concurrent compilations
1516 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
1517 uint i;
1518 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1519 ciKlass* receiver = vc_data->receiver(i);
1520 if (known_klass->equals(receiver)) {
1521 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1522 __ add_memory_int64(data_addr, DataLayout::counter_increment);
1523 return;
1524 }
1525 }
1526
1527 // Receiver type not found in profile data; select an empty slot
1528 // Note that this is less efficient than it should be because it
1529 // always does a write to the receiver part of the
1530 // VirtualCallData rather than just the first time
1531 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1532 ciKlass* receiver = vc_data->receiver(i);
1533 if (receiver == NULL) {
1534 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
1535 __ mov_metadata(t1, known_klass->constant_encoding());
1536 __ sd(t1, recv_addr);
1537 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1538 __ add_memory_int64(data_addr, DataLayout::counter_increment);
1539 return;
1540 }
1541 }
1542 } else {
1543 __ load_klass(recv, recv);
1544 Label update_done;
1545 type_profile_helper(mdo, md, data, recv, &update_done);
1546 // Receiver did not match any saved receiver and there is no empty row for it.
1547 // Increment total counter to indicate polymorphic case.
1548 __ add_memory_int64(counter_addr, DataLayout::counter_increment);
1549
1550 __ bind(update_done);
1551 }
1552 } else {
1553 // Static call
1554 __ add_memory_int64(counter_addr, DataLayout::counter_increment);
1555 }
1556 }
1557
1558 void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); }
1559
1560 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
1561 __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
1562 }
1563
1564 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); }
1565
1566 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
1567 Register tmp, Label &next, Label &none,
1568 Address mdo_addr) {
1569 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
1570 if (exact_klass != NULL) {
1571 __ mov_metadata(tmp, exact_klass->constant_encoding());
1572 } else {
1573 __ load_klass(tmp, tmp);
1574 }
1575
1576 __ ld(t1, mdo_addr);
1577 __ xorr(tmp, tmp, t1);
1578 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1579 // klass seen before, nothing to do. The unknown bit may have been
1580 // set already but no need to check.
1581 __ beqz(t0, next);
1582
1583 // already unknown. Nothing to do anymore.
1584 __ andi(t0, tmp, TypeEntries::type_unknown);
1585 __ bnez(t0, next);
1586
1587 if (TypeEntries::is_type_none(current_klass)) {
1588 __ beqz(t1, none);
1589 __ li(t0, (u1)TypeEntries::null_seen);
1590 __ beq(t0, t1, none);
1591 // There is a chance that the checks above (re-reading profiling
1592 // data from memory) fail if another thread has just set the
1593 // profiling to this obj's klass
1594 __ membar(MacroAssembler::LoadLoad);
1595 __ ld(t1, mdo_addr);
1596 __ xorr(tmp, tmp, t1);
1597 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1598 __ beqz(t0, next);
1599 }
1600 } else {
1601 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1602 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
1603
1604 __ ld(tmp, mdo_addr);
1605 // already unknown. Nothing to do anymore.
1606 __ andi(t0, tmp, TypeEntries::type_unknown);
1607 __ bnez(t0, next);
1608 }
1609
1610 // different than before. Cannot keep accurate profile.
1611 __ ld(t1, mdo_addr);
1612 __ ori(t1, t1, TypeEntries::type_unknown);
1613 __ sd(t1, mdo_addr);
1614
1615 if (TypeEntries::is_type_none(current_klass)) {
1616 __ j(next);
1617
1618 __ bind(none);
1619 // first time here. Set profile type.
1620 __ sd(tmp, mdo_addr);
1621 }
1622 }
1623
1624 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp,
1625 Address mdo_addr, Label &next) {
1626 // There's a single possible klass at this profile point
1627 assert(exact_klass != NULL, "should be");
1628 if (TypeEntries::is_type_none(current_klass)) {
1629 __ mov_metadata(tmp, exact_klass->constant_encoding());
1630 __ ld(t1, mdo_addr);
1631 __ xorr(tmp, tmp, t1);
1632 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1633 __ beqz(t0, next);
1634 #ifdef ASSERT
1635 {
1636 Label ok;
1637 __ ld(t0, mdo_addr);
1638 __ beqz(t0, ok);
1639 __ li(t1, (u1)TypeEntries::null_seen);
1640 __ beq(t0, t1, ok);
1641 // may have been set by another thread
1642 __ membar(MacroAssembler::LoadLoad);
1643 __ mov_metadata(t0, exact_klass->constant_encoding());
1644 __ ld(t1, mdo_addr);
1645 __ xorr(t1, t0, t1);
1646 __ andi(t1, t1, TypeEntries::type_mask);
1647 __ beqz(t1, ok);
1648
1649 __ stop("unexpected profiling mismatch");
1650 __ bind(ok);
1651 }
1652 #endif
1653 // first time here. Set profile type.
1654 __ sd(tmp, mdo_addr);
1655 } else {
1656 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1657 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
1658
1659 __ ld(tmp, mdo_addr);
1660 // already unknown. Nothing to do anymore.
1661 __ andi(t0, tmp, TypeEntries::type_unknown);
1662 __ bnez(t0, next);
1663
1664 __ ori(tmp, tmp, TypeEntries::type_unknown);
1665 __ sd(tmp, mdo_addr);
1666 }
1667 }
1668
1669 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass,
1670 Address mdo_addr, bool do_update, Label &next) {
1671 __ bnez(tmp, update);
1672 if (!TypeEntries::was_null_seen(current_klass)) {
1673 __ ld(t1, mdo_addr);
1674 __ ori(t1, t1, TypeEntries::null_seen);
1675 __ sd(t1, mdo_addr);
1676 }
1677 if (do_update) {
1678 __ j(next);
1679 }
1680 }
1681
1682 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
1683 COMMENT("emit_profile_type {");
1684 Register obj = op->obj()->as_register();
1685 Register tmp = op->tmp()->as_pointer_register();
1686 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
1687 ciKlass* exact_klass = op->exact_klass();
1688 intptr_t current_klass = op->current_klass();
1689 bool not_null = op->not_null();
1690 bool no_conflict = op->no_conflict();
1691
1692 Label update, next, none;
1693
1694 bool do_null = !not_null;
1695 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
1696 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
1697
1698 assert(do_null || do_update, "why are we here?");
1699 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
1700 assert_different_registers(tmp, t0, t1, mdo_addr.base());
1701
1702 __ verify_oop(obj);
1703
1704 if (tmp != obj) {
1705 __ mv(tmp, obj);
1706 }
1707 if (do_null) {
1708 check_null(tmp, update, current_klass, mdo_addr, do_update, next);
1709 #ifdef ASSERT
1710 } else {
1711 __ bnez(tmp, update);
1712 __ stop("unexpected null obj");
1713 #endif
1714 }
1715
1716 __ bind(update);
1717
1718 if (do_update) {
1719 #ifdef ASSERT
1720 if (exact_klass != NULL) {
1721 check_exact_klass(tmp, exact_klass);
1722 }
1723 #endif
1724 if (!no_conflict) {
1725 check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr);
1726 } else {
1727 check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next);
1728 }
1729
1730 __ bind(next);
1731 }
1732 COMMENT("} emit_profile_type");
1733 }
1734
1735 void LIR_Assembler::align_backward_branch_target() { }
1736
1737 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
1738 // tmp must be unused
1739 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
1740
1741 if (left->is_single_cpu()) {
1742 assert(dest->is_single_cpu(), "expect single result reg");
1743 __ negw(dest->as_register(), left->as_register());
1744 } else if (left->is_double_cpu()) {
1745 assert(dest->is_double_cpu(), "expect double result reg");
1746 __ neg(dest->as_register_lo(), left->as_register_lo());
1747 } else if (left->is_single_fpu()) {
1748 assert(dest->is_single_fpu(), "expect single float result reg");
1749 __ fneg_s(dest->as_float_reg(), left->as_float_reg());
1750 } else {
1751 assert(left->is_double_fpu(), "expect double float operand reg");
1752 assert(dest->is_double_fpu(), "expect double float result reg");
1753 __ fneg_d(dest->as_double_reg(), left->as_double_reg());
1754 }
1755 }
1756
1757
1758 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1759 if (patch_code != lir_patch_none) {
1760 deoptimize_trap(info);
1761 return;
1762 }
1763
1764 LIR_Address* adr = addr->as_address_ptr();
1765 Register dst = dest->as_register_lo();
1766
1767 assert_different_registers(dst, t0);
1768 if(adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) {
1769
1770 intptr_t offset = adr->disp();
1771 LIR_Opr index_op = adr->index();
1772 int scale = adr->scale();
1773 if(index_op->is_constant()) {
1774 offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale;
1775 }
1776
1777 if(!is_imm_in_range(offset, 12, 0)) {
1778 __ la(t0, as_Address(adr));
1779 __ mv(dst, t0);
1780 return;
1781 }
1782 }
1783 __ la(dst, as_Address(adr));
1784 }
1785
1786
1787 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
1788 assert(!tmp->is_valid(), "don't need temporary");
1789
1790 CodeBlob *cb = CodeCache::find_blob(dest);
1791 if (cb != NULL) {
1792 __ far_call(RuntimeAddress(dest));
1793 } else {
1794 int32_t offset = 0;
1795 __ la_patchable(t0, RuntimeAddress(dest), offset);
1796 __ jalr(x1, t0, offset);
1797 }
1798
1799 if (info != NULL) {
1800 add_call_info_here(info);
1801 }
1802 }
1803
1804 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
1805 if (dest->is_address() || src->is_address()) {
1806 move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false);
1807 } else {
1808 ShouldNotReachHere();
1809 }
1810 }
1811
1812 #ifdef ASSERT
1813 // emit run-time assertion
1814 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
1815 assert(op->code() == lir_assert, "must be");
1816
1817 Label ok;
1818 if (op->in_opr1()->is_valid()) {
1819 assert(op->in_opr2()->is_valid(), "both operands must be valid");
1820 bool is_unordered = false;
1821 LIR_Condition cond = op->condition();
1822 emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false,
1823 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true);
1824 } else {
1825 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
1826 assert(op->condition() == lir_cond_always, "no other conditions allowed");
1827 }
1828
1829 if (op->halt()) {
1830 const char* str = __ code_string(op->msg());
1831 __ stop(str);
1832 } else {
1833 breakpoint();
1834 }
1835 __ bind(ok);
1836 }
1837 #endif
1838
1839 #ifndef PRODUCT
1840 #define COMMENT(x) do { __ block_comment(x); } while (0)
1841 #else
1842 #define COMMENT(x)
1843 #endif
1844
1845 void LIR_Assembler::membar() {
1846 COMMENT("membar");
1847 __ membar(MacroAssembler::AnyAny);
1848 }
1849
1850 void LIR_Assembler::membar_acquire() {
1851 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1852 }
1853
1854 void LIR_Assembler::membar_release() {
1855 __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1856 }
1857
1858 void LIR_Assembler::membar_loadload() {
1859 __ membar(MacroAssembler::LoadLoad);
1860 }
1861
1862 void LIR_Assembler::membar_storestore() {
1863 __ membar(MacroAssembler::StoreStore);
1864 }
1865
1866 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
1867
1868 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
1869
1870 void LIR_Assembler::on_spin_wait() {
1871 Unimplemented();
1872 }
1873
1874 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
1875 __ mv(result_reg->as_register(), xthread);
1876 }
1877
1878 void LIR_Assembler::peephole(LIR_List *lir) {}
1879
1880 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
1881 Address addr = as_Address(src->as_address_ptr());
1882 BasicType type = src->type();
1883 bool is_oop = is_reference_type(type);
1884
1885 get_op(type);
1886
1887 switch (code) {
1888 case lir_xadd:
1889 {
1890 RegisterOrConstant inc;
1891 Register tmp = as_reg(tmp_op);
1892 Register dst = as_reg(dest);
1893 if (data->is_constant()) {
1894 inc = RegisterOrConstant(as_long(data));
1895 assert_different_registers(dst, addr.base(), tmp);
1896 assert_different_registers(tmp, t0);
1897 } else {
1898 inc = RegisterOrConstant(as_reg(data));
1899 assert_different_registers(inc.as_register(), dst, addr.base(), tmp);
1900 }
1901 __ la(tmp, addr);
1902 (_masm->*add)(dst, inc, tmp);
1903 break;
1904 }
1905 case lir_xchg:
1906 {
1907 Register tmp = tmp_op->as_register();
1908 Register obj = as_reg(data);
1909 Register dst = as_reg(dest);
1910 if (is_oop && UseCompressedOops) {
1911 __ encode_heap_oop(t0, obj);
1912 obj = t0;
1913 }
1914 assert_different_registers(obj, addr.base(), tmp, dst);
1915 __ la(tmp, addr);
1916 (_masm->*xchg)(dst, obj, tmp);
1917 if (is_oop && UseCompressedOops) {
1918 __ decode_heap_oop(dst);
1919 }
1920 }
1921 break;
1922 default:
1923 ShouldNotReachHere();
1924 }
1925 __ membar(MacroAssembler::AnyAny);
1926 }
1927
1928 int LIR_Assembler::array_element_size(BasicType type) const {
1929 int elem_size = type2aelembytes(type);
1930 return log2i_exact(elem_size);
1931 }
1932
1933 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
1934 if (addr->base()->is_illegal()) {
1935 assert(addr->index()->is_illegal(), "must be illegal too");
1936 __ movptr(tmp, addr->disp());
1937 return Address(tmp, 0);
1938 }
1939
1940 Register base = addr->base()->as_pointer_register();
1941 LIR_Opr index_op = addr->index();
1942 int scale = addr->scale();
1943
1944 if (index_op->is_illegal()) {
1945 return Address(base, addr->disp());
1946 } else if (index_op->is_cpu_register()) {
1947 Register index;
1948 if (index_op->is_single_cpu()) {
1949 index = index_op->as_register();
1950 } else {
1951 index = index_op->as_register_lo();
1952 }
1953 if (scale != 0) {
1954 __ slli(tmp, index, scale);
1955 __ add(tmp, base, tmp);
1956 } else {
1957 __ add(tmp, base, index);
1958 }
1959 return Address(tmp, addr->disp());
1960 } else if (index_op->is_constant()) {
1961 intptr_t addr_offset = (((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale) + addr->disp();
1962 return Address(base, addr_offset);
1963 }
1964
1965 Unimplemented();
1966 return Address();
1967 }
1968
1969 // helper functions which checks for overflow and sets bailout if it
1970 // occurs. Always returns a valid embeddable pointer but in the
1971 // bailout case the pointer won't be to unique storage.
1972 address LIR_Assembler::float_constant(float f) {
1973 address const_addr = __ float_constant(f);
1974 if (const_addr == NULL) {
1975 bailout("const section overflow");
1976 return __ code()->consts()->start();
1977 } else {
1978 return const_addr;
1979 }
1980 }
1981
1982 address LIR_Assembler::double_constant(double d) {
1983 address const_addr = __ double_constant(d);
1984 if (const_addr == NULL) {
1985 bailout("const section overflow");
1986 return __ code()->consts()->start();
1987 } else {
1988 return const_addr;
1989 }
1990 }
1991
1992 address LIR_Assembler::int_constant(jlong n) {
1993 address const_addr = __ long_constant(n);
1994 if (const_addr == NULL) {
1995 bailout("const section overflow");
1996 return __ code()->consts()->start();
1997 } else {
1998 return const_addr;
1999 }
2000 }
2001
2002 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
2003 __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */,
2004 Assembler::rl /* release */, t0, true /* result as bool */);
2005 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2006 __ membar(MacroAssembler::AnyAny);
2007 }
2008
2009 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) {
2010 __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */,
2011 Assembler::rl /* release */, t0, true /* result as bool */);
2012 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2013 __ membar(MacroAssembler::AnyAny);
2014 }
2015
2016 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
2017 __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */,
2018 Assembler::rl /* release */, t0, true /* result as bool */);
2019 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2020 __ membar(MacroAssembler::AnyAny);
2021 }
2022
2023 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
2024 address target = NULL;
2025
2026 switch (patching_id(info)) {
2027 case PatchingStub::access_field_id:
2028 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
2029 break;
2030 case PatchingStub::load_klass_id:
2031 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
2032 break;
2033 case PatchingStub::load_mirror_id:
2034 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
2035 break;
2036 case PatchingStub::load_appendix_id:
2037 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
2038 break;
2039 default: ShouldNotReachHere();
2040 }
2041
2042 __ far_call(RuntimeAddress(target));
2043 add_call_info_here(info);
2044 }
2045
2046 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) {
2047 Label ok;
2048 __ load_klass(tmp, tmp);
2049 __ mov_metadata(t0, exact_klass->constant_encoding());
2050 __ beq(tmp, t0, ok);
2051 __ stop("exact klass and actual klass differ");
2052 __ bind(ok);
2053 }
2054
2055 void LIR_Assembler::get_op(BasicType type) {
2056 switch (type) {
2057 case T_INT:
2058 xchg = &MacroAssembler::atomic_xchgalw;
2059 add = &MacroAssembler::atomic_addalw;
2060 break;
2061 case T_LONG:
2062 xchg = &MacroAssembler::atomic_xchgal;
2063 add = &MacroAssembler::atomic_addal;
2064 break;
2065 case T_OBJECT:
2066 case T_ARRAY:
2067 if (UseCompressedOops) {
2068 xchg = &MacroAssembler::atomic_xchgalwu;
2069 add = &MacroAssembler::atomic_addalw;
2070 } else {
2071 xchg = &MacroAssembler::atomic_xchgal;
2072 add = &MacroAssembler::atomic_addal;
2073 }
2074 break;
2075 default:
2076 ShouldNotReachHere();
2077 }
2078 }
2079
2080 // emit_opTypeCheck sub functions
2081 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) {
2082 Register value = op->object()->as_register();
2083 Register array = op->array()->as_register();
2084 Register k_RInfo = op->tmp1()->as_register();
2085 Register klass_RInfo = op->tmp2()->as_register();
2086 Register Rtmp1 = op->tmp3()->as_register();
2087
2088 CodeStub* stub = op->stub();
2089
2090 // check if it needs to be profiled
2091 ciMethodData* md = NULL;
2092 ciProfileData* data = NULL;
2093
2094 if (should_profile) {
2095 data_check(op, &md, &data);
2096 }
2097 Label profile_cast_success, profile_cast_failure, done;
2098 Label *success_target = should_profile ? &profile_cast_success : &done;
2099 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
2100
2101 if (should_profile) {
2102 profile_object(md, data, value, klass_RInfo, &done);
2103 } else {
2104 __ beqz(value, done);
2105 }
2106
2107 add_debug_info_for_null_check_here(op->info_for_exception());
2108 __ load_klass(k_RInfo, array);
2109 __ load_klass(klass_RInfo, value);
2110
2111 lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target);
2112
2113 // fall through to the success case
2114 if (should_profile) {
2115 Register mdo = klass_RInfo;
2116 Register recv = k_RInfo;
2117 __ bind(profile_cast_success);
2118 __ mov_metadata(mdo, md->constant_encoding());
2119 __ load_klass(recv, value);
2120 type_profile_helper(mdo, md, data, recv, &done);
2121 __ j(done);
2122
2123 __ bind(profile_cast_failure);
2124 __ mov_metadata(mdo, md->constant_encoding());
2125 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2126 __ ld(t1, counter_addr);
2127 __ addi(t1, t1, -DataLayout::counter_increment);
2128 __ sd(t1, counter_addr);
2129 __ j(*stub->entry());
2130 }
2131
2132 __ bind(done);
2133 }
2134
2135 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
2136 _masm->code_section()->relocate(adr, relocInfo::poll_type);
2137 int pc_offset = code_offset();
2138 flush_debug_info(pc_offset);
2139 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
2140 if (info->exception_handlers() != NULL) {
2141 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
2142 }
2143 }
2144
2145 void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo,
2146 ciProfileData* data, Label* success, Label* failure,
2147 Label& profile_cast_success, Label& profile_cast_failure) {
2148 Register mdo = klass_RInfo;
2149 Register recv = k_RInfo;
2150 __ bind(profile_cast_success);
2151 __ mov_metadata(mdo, md->constant_encoding());
2152 __ load_klass(recv, obj);
2153 Label update_done;
2154 type_profile_helper(mdo, md, data, recv, success);
2155 __ j(*success);
2156
2157 __ bind(profile_cast_failure);
2158 __ mov_metadata(mdo, md->constant_encoding());
2159 Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2160 __ ld(t0, counter_addr);
2161 __ addi(t0, t0, -DataLayout::counter_increment);
2162 __ sd(t0, counter_addr);
2163 __ j(*failure);
2164 }
2165
2166 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1,
2167 Label* success_target, Label* failure_target) {
2168 // get instance klass (it's already uncompressed)
2169 __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2170 // perform the fast part of the checking logic
2171 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2172 // call out-of-line instance of __ check_klass_subtype_slow_path(...)
2173 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
2174 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
2175 __ sd(k_RInfo, Address(sp, 0)); // super klass
2176 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2177 // load result to k_RInfo
2178 __ ld(k_RInfo, Address(sp, 0));
2179 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
2180 // result is a boolean
2181 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
2182 }
2183
2184 void LIR_Assembler::const2reg_helper(LIR_Opr src) {
2185 switch (src->as_constant_ptr()->type()) {
2186 case T_INT:
2187 case T_ADDRESS:
2188 case T_OBJECT:
2189 case T_ARRAY:
2190 case T_METADATA:
2191 const2reg(src, FrameMap::t0_opr, lir_patch_none, NULL);
2192 break;
2193 case T_LONG:
2194 const2reg(src, FrameMap::t0_long_opr, lir_patch_none, NULL);
2195 break;
2196 case T_FLOAT:
2197 case T_DOUBLE:
2198 default:
2199 ShouldNotReachHere();
2200 }
2201 }
2202
2203 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) {
2204 switch (code) {
2205 case lir_logic_and: __ andrw(dst, left, right); break;
2206 case lir_logic_or: __ orrw (dst, left, right); break;
2207 case lir_logic_xor: __ xorrw(dst, left, right); break;
2208 default: ShouldNotReachHere();
2209 }
2210 }
2211
2212 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) {
2213 switch (code) {
2214 case lir_logic_and: __ andr(dst, left, right); break;
2215 case lir_logic_or: __ orr (dst, left, right); break;
2216 case lir_logic_xor: __ xorr(dst, left, right); break;
2217 default: ShouldNotReachHere();
2218 }
2219 }
2220
2221 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) {
2222 switch (code) {
2223 case lir_logic_and: __ andi(dst, left, right); break;
2224 case lir_logic_or: __ ori (dst, left, right); break;
2225 case lir_logic_xor: __ xori(dst, left, right); break;
2226 default: ShouldNotReachHere();
2227 }
2228 }
2229
2230 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2231 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2232 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2233 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2234 __ sd(r, Address(sp, offset_from_rsp_in_bytes));
2235 }
2236
2237 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2238 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2239 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2240 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2241 __ li(t0, c);
2242 __ sd(t0, Address(sp, offset_from_rsp_in_bytes));
2243 }
2244
2245 #undef __