1 /*
2 * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "c1/c1_CodeStubs.hpp"
31 #include "c1/c1_Compilation.hpp"
32 #include "c1/c1_LIRAssembler.hpp"
33 #include "c1/c1_MacroAssembler.hpp"
34 #include "c1/c1_Runtime1.hpp"
35 #include "c1/c1_ValueStack.hpp"
36 #include "ci/ciArrayKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "nativeInst_riscv.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_riscv.inline.hpp"
46
47 #ifndef PRODUCT
48 #define COMMENT(x) do { __ block_comment(x); } while (0)
49 #else
50 #define COMMENT(x)
51 #endif
52
53 NEEDS_CLEANUP // remove this definitions ?
54 const Register IC_Klass = t1; // where the IC klass is cached
55 const Register SYNC_header = x10; // synchronization header
56 const Register SHIFT_count = x10; // where count for shift operations must be
57
58 #define __ _masm->
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74 static void select_different_registers(Register preserve,
75 Register extra,
76 Register &tmp1,
77 Register &tmp2,
78 Register &tmp3) {
79 if (tmp1 == preserve) {
80 assert_different_registers(tmp1, tmp2, tmp3, extra);
81 tmp1 = extra;
82 } else if (tmp2 == preserve) {
83 assert_different_registers(tmp1, tmp2, tmp3, extra);
84 tmp2 = extra;
85 } else if (tmp3 == preserve) {
86 assert_different_registers(tmp1, tmp2, tmp3, extra);
87 tmp3 = extra;
88 }
89 assert_different_registers(preserve, tmp1, tmp2, tmp3);
90 }
91
92 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
93
94 void LIR_Assembler::clinit_barrier(ciMethod* method) {
95 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
96 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
97
98 Label L_skip_barrier;
99
100 __ mov_metadata(t1, method->holder()->constant_encoding());
101 __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */);
102 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
103 __ bind(L_skip_barrier);
104 }
105
106 LIR_Opr LIR_Assembler::receiverOpr() {
107 return FrameMap::receiver_opr;
108 }
109
110 LIR_Opr LIR_Assembler::osrBufferPointer() {
111 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
112 }
113
114 void LIR_Assembler::breakpoint() { Unimplemented(); }
115
116 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
117
118 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
119
120 static jlong as_long(LIR_Opr data) {
121 jlong result;
122 switch (data->type()) {
123 case T_INT:
124 result = (data->as_jint());
125 break;
126 case T_LONG:
127 result = (data->as_jlong());
128 break;
129 default:
130 ShouldNotReachHere();
131 result = 0; // unreachable
132 }
133 return result;
134 }
135
136 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
137 if (addr->base()->is_illegal()) {
138 assert(addr->index()->is_illegal(), "must be illegal too");
139 __ movptr(tmp, addr->disp());
140 return Address(tmp, 0);
141 }
142
143 Register base = addr->base()->as_pointer_register();
144 LIR_Opr index_opr = addr->index();
145
146 if (index_opr->is_illegal()) {
147 return Address(base, addr->disp());
148 }
149
150 int scale = addr->scale();
151 if (index_opr->is_cpu_register()) {
152 Register index;
153 if (index_opr->is_single_cpu()) {
154 index = index_opr->as_register();
155 } else {
156 index = index_opr->as_register_lo();
157 }
158 if (scale != 0) {
159 __ shadd(tmp, index, base, tmp, scale);
160 } else {
161 __ add(tmp, base, index);
162 }
163 return Address(tmp, addr->disp());
164 } else if (index_opr->is_constant()) {
165 intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp();
166 return Address(base, addr_offset);
167 }
168
169 Unimplemented();
170 return Address();
171 }
172
173 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
174 ShouldNotReachHere();
175 return Address();
176 }
177
178 Address LIR_Assembler::as_Address(LIR_Address* addr) {
179 return as_Address(addr, t0);
180 }
181
182 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
183 return as_Address(addr);
184 }
185
186 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
187 // not encodable as a base + (immediate) offset, generate an explicit address
188 // calculation to hold the address in t0.
189 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) {
190 precond(size == 4 || size == 8);
191 Address addr = frame_map()->address_for_slot(index, adjust);
192 precond(addr.getMode() == Address::base_plus_offset);
193 precond(addr.base() == sp);
194 precond(addr.offset() > 0);
195 uint mask = size - 1;
196 assert((addr.offset() & mask) == 0, "scaled offsets only");
197
198 return addr;
199 }
200
201 void LIR_Assembler::osr_entry() {
202 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
203 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
204 guarantee(osr_entry != NULL, "NULL osr_entry!");
205 ValueStack* entry_state = osr_entry->state();
206 int number_of_locks = entry_state->locks_size();
207
208 // we jump here if osr happens with the interpreter
209 // state set up to continue at the beginning of the
210 // loop that triggered osr - in particular, we have
211 // the following registers setup:
212 //
213 // x12: osr buffer
214 //
215
216 //build frame
217 ciMethod* m = compilation()->method();
218 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
219
220 // OSR buffer is
221 //
222 // locals[nlocals-1..0]
223 // monitors[0..number_of_locks]
224 //
225 // locals is a direct copy of the interpreter frame so in the osr buffer
226 // so first slot in the local array is the last local from the interpreter
227 // and last slot is local[0] (receiver) from the interpreter
228 //
229 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
230 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
231 // in the interpreter frame (the method lock if a sync method)
232
233 // Initialize monitors in the compiled activation.
234 // x12: pointer to osr buffer
235 // All other registers are dead at this point and the locals will be
236 // copied into place by code emitted in the IR.
237
238 Register OSR_buf = osrBufferPointer()->as_pointer_register();
239 {
240 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
241 int monitor_offset = BytesPerWord * method()->max_locals() +
242 (2 * BytesPerWord) * (number_of_locks - 1);
243 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
244 // the OSR buffer using 2 word entries: first the lock and then
245 // the oop.
246 for (int i = 0; i < number_of_locks; i++) {
247 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
248 #ifdef ASSERT
249 // verify the interpreter's monitor has a non-null object
250 {
251 Label L;
252 __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
253 __ bnez(t0, L);
254 __ stop("locked object is NULL");
255 __ bind(L);
256 }
257 #endif // ASSERT
258 __ ld(x9, Address(OSR_buf, slot_offset + 0));
259 __ sd(x9, frame_map()->address_for_monitor_lock(i));
260 __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
261 __ sd(x9, frame_map()->address_for_monitor_object(i));
262 }
263 }
264 }
265
266 // inline cache check; done before the frame is built.
267 int LIR_Assembler::check_icache() {
268 Register receiver = FrameMap::receiver_opr->as_register();
269 Register ic_klass = IC_Klass;
270 int start_offset = __ offset();
271 Label dont;
272 __ inline_cache_check(receiver, ic_klass, dont);
273
274 // if icache check fails, then jump to runtime routine
275 // Note: RECEIVER must still contain the receiver!
276 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
277
278 // We align the verified entry point unless the method body
279 // (including its inline cache check) will fit in a single 64-byte
280 // icache line.
281 if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
282 // force alignment after the cache check.
283 __ align(CodeEntryAlignment);
284 }
285
286 __ bind(dont);
287 return start_offset;
288 }
289
290 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
291 if (o == NULL) {
292 __ mv(reg, zr);
293 } else {
294 __ movoop(reg, o, /* immediate */ true);
295 }
296 }
297
298 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
299 deoptimize_trap(info);
300 }
301
302 // This specifies the rsp decrement needed to build the frame
303 int LIR_Assembler::initial_frame_size_in_bytes() const {
304 // if rounding, must let FrameMap know!
305
306 return in_bytes(frame_map()->framesize_in_bytes());
307 }
308
309 int LIR_Assembler::emit_exception_handler() {
310 // if the last instruction is a call (typically to do a throw which
311 // is coming at the end after block reordering) the return address
312 // must still point into the code area in order to avoid assertion
313 // failures when searching for the corresponding bci ==> add a nop
314 // (was bug 5/14/1999 -gri)
315 __ nop();
316
317 // generate code for exception handler
318 address handler_base = __ start_a_stub(exception_handler_size());
319 if (handler_base == NULL) {
320 // not enough space left for the handler
321 bailout("exception handler overflow");
322 return -1;
323 }
324
325 int offset = code_offset();
326
327 // the exception oop and pc are in x10, and x13
328 // no other registers need to be preserved, so invalidate them
329 __ invalidate_registers(false, true, true, false, true, true);
330
331 // check that there is really an exception
332 __ verify_not_null_oop(x10);
333
334 // search an exception handler (x10: exception oop, x13: throwing pc)
335 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
336 __ should_not_reach_here();
337 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
338 __ end_a_stub();
339
340 return offset;
341 }
342
343 // Emit the code to remove the frame from the stack in the exception
344 // unwind path.
345 int LIR_Assembler::emit_unwind_handler() {
346 #ifndef PRODUCT
347 if (CommentedAssembly) {
348 _masm->block_comment("Unwind handler");
349 }
350 #endif // PRODUCT
351
352 int offset = code_offset();
353
354 // Fetch the exception from TLS and clear out exception related thread state
355 __ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
356 __ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
357 __ sd(zr, Address(xthread, JavaThread::exception_pc_offset()));
358
359 __ bind(_unwind_handler_entry);
360 __ verify_not_null_oop(x10);
361 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
362 __ mv(x9, x10); // Perserve the exception
363 }
364
365 // Preform needed unlocking
366 MonitorExitStub* stub = NULL;
367 if (method()->is_synchronized()) {
368 monitor_address(0, FrameMap::r10_opr);
369 stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
370 if (UseHeavyMonitors) {
371 __ j(*stub->entry());
372 } else {
373 __ unlock_object(x15, x14, x10, *stub->entry());
374 }
375 __ bind(*stub->continuation());
376 }
377
378 if (compilation()->env()->dtrace_method_probes()) {
379 __ mv(c_rarg0, xthread);
380 __ mov_metadata(c_rarg1, method()->constant_encoding());
381 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
382 }
383
384 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
385 __ mv(x10, x9); // Restore the exception
386 }
387
388 // remove the activation and dispatch to the unwind handler
389 __ block_comment("remove_frame and dispatch to the unwind handler");
390 __ remove_frame(initial_frame_size_in_bytes());
391 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
392
393 // Emit the slow path assembly
394 if (stub != NULL) {
395 stub->emit_code(this);
396 }
397
398 return offset;
399 }
400
401 int LIR_Assembler::emit_deopt_handler() {
402 // if the last instruciton is a call (typically to do a throw which
403 // is coming at the end after block reordering) the return address
404 // must still point into the code area in order to avoid assertion
405 // failures when searching for the corresponding bck => add a nop
406 // (was bug 5/14/1999 - gri)
407 __ nop();
408
409 // generate code for exception handler
410 address handler_base = __ start_a_stub(deopt_handler_size());
411 if (handler_base == NULL) {
412 // not enough space left for the handler
413 bailout("deopt handler overflow");
414 return -1;
415 }
416
417 int offset = code_offset();
418
419 __ auipc(ra, 0);
420 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
421 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
422 __ end_a_stub();
423
424 return offset;
425 }
426
427 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
428 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
429
430 // Pop the stack before the safepoint code
431 __ remove_frame(initial_frame_size_in_bytes());
432
433 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
434 __ reserved_stack_check();
435 }
436
437 code_stub->set_safepoint_offset(__ offset());
438 __ relocate(relocInfo::poll_return_type);
439 __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
440 __ ret();
441 }
442
443 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
444 guarantee(info != NULL, "Shouldn't be NULL");
445 __ get_polling_page(t0, relocInfo::poll_type);
446 add_debug_info_for_branch(info); // This isn't just debug info:
447 // it's the oop map
448 __ read_polling_page(t0, 0, relocInfo::poll_type);
449 return __ offset();
450 }
451
452 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
453 __ mv(to_reg, from_reg);
454 }
455
456 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
457
458 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
459 assert(src->is_constant(), "should not call otherwise");
460 assert(dest->is_register(), "should not call otherwise");
461 LIR_Const* c = src->as_constant_ptr();
462 address const_addr = NULL;
463
464 switch (c->type()) {
465 case T_INT:
466 assert(patch_code == lir_patch_none, "no patching handled here");
467 __ mvw(dest->as_register(), c->as_jint());
468 break;
469
470 case T_ADDRESS:
471 assert(patch_code == lir_patch_none, "no patching handled here");
472 __ mv(dest->as_register(), c->as_jint());
473 break;
474
475 case T_LONG:
476 assert(patch_code == lir_patch_none, "no patching handled here");
477 __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
478 break;
479
480 case T_OBJECT:
481 case T_ARRAY:
482 if (patch_code == lir_patch_none) {
483 jobject2reg(c->as_jobject(), dest->as_register());
484 } else {
485 jobject2reg_with_patching(dest->as_register(), info);
486 }
487 break;
488
489 case T_METADATA:
490 if (patch_code != lir_patch_none) {
491 klass2reg_with_patching(dest->as_register(), info);
492 } else {
493 __ mov_metadata(dest->as_register(), c->as_metadata());
494 }
495 break;
496
497 case T_FLOAT:
498 const_addr = float_constant(c->as_jfloat());
499 assert(const_addr != NULL, "must create float constant in the constant table");
500 __ flw(dest->as_float_reg(), InternalAddress(const_addr));
501 break;
502
503 case T_DOUBLE:
504 const_addr = double_constant(c->as_jdouble());
505 assert(const_addr != NULL, "must create double constant in the constant table");
506 __ fld(dest->as_double_reg(), InternalAddress(const_addr));
507 break;
508
509 default:
510 ShouldNotReachHere();
511 }
512 }
513
514 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
515 assert(src->is_constant(), "should not call otherwise");
516 assert(dest->is_stack(), "should not call otherwise");
517 LIR_Const* c = src->as_constant_ptr();
518 switch (c->type()) {
519 case T_OBJECT:
520 if (c->as_jobject() == NULL) {
521 __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
522 } else {
523 const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
524 reg2stack(FrameMap::t1_opr, dest, c->type(), false);
525 }
526 break;
527 case T_ADDRESS: // fall through
528 const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
529 reg2stack(FrameMap::t1_opr, dest, c->type(), false);
530 case T_INT: // fall through
531 case T_FLOAT:
532 if (c->as_jint_bits() == 0) {
533 __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
534 } else {
535 __ mvw(t1, c->as_jint_bits());
536 __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix()));
537 }
538 break;
539 case T_LONG: // fall through
540 case T_DOUBLE:
541 if (c->as_jlong_bits() == 0) {
542 __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
543 lo_word_offset_in_bytes));
544 } else {
545 __ mv(t1, (intptr_t)c->as_jlong_bits());
546 __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(),
547 lo_word_offset_in_bytes));
548 }
549 break;
550 default:
551 ShouldNotReachHere();
552 }
553 }
554
555 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
556 assert(src->is_constant(), "should not call otherwise");
557 assert(dest->is_address(), "should not call otherwise");
558 LIR_Const* c = src->as_constant_ptr();
559 LIR_Address* to_addr = dest->as_address_ptr();
560 void (Assembler::* insn)(Register Rt, const Address &adr, Register temp);
561 switch (type) {
562 case T_ADDRESS:
563 assert(c->as_jint() == 0, "should be");
564 insn = &Assembler::sd; break;
565 case T_LONG:
566 assert(c->as_jlong() == 0, "should be");
567 insn = &Assembler::sd; break;
568 case T_DOUBLE:
569 assert(c->as_jdouble() == 0.0, "should be");
570 insn = &Assembler::sd; break;
571 case T_INT:
572 assert(c->as_jint() == 0, "should be");
573 insn = &Assembler::sw; break;
574 case T_FLOAT:
575 assert(c->as_jfloat() == 0.0f, "should be");
576 insn = &Assembler::sw; break;
577 case T_OBJECT: // fall through
578 case T_ARRAY:
579 assert(c->as_jobject() == 0, "should be");
580 if (UseCompressedOops && !wide) {
581 insn = &Assembler::sw;
582 } else {
583 insn = &Assembler::sd;
584 }
585 break;
586 case T_CHAR: // fall through
587 case T_SHORT:
588 assert(c->as_jint() == 0, "should be");
589 insn = &Assembler::sh;
590 break;
591 case T_BOOLEAN: // fall through
592 case T_BYTE:
593 assert(c->as_jint() == 0, "should be");
594 insn = &Assembler::sb; break;
595 default:
596 ShouldNotReachHere();
597 insn = &Assembler::sd; // unreachable
598 }
599 if (info != NULL) {
600 add_debug_info_for_null_check_here(info);
601 }
602 (_masm->*insn)(zr, as_Address(to_addr), t0);
603 }
604
605 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
606 assert(src->is_register(), "should not call otherwise");
607 assert(dest->is_register(), "should not call otherwise");
608
609 // move between cpu-registers
610 if (dest->is_single_cpu()) {
611 if (src->type() == T_LONG) {
612 // Can do LONG -> OBJECT
613 move_regs(src->as_register_lo(), dest->as_register());
614 return;
615 }
616 assert(src->is_single_cpu(), "must match");
617 if (src->type() == T_OBJECT) {
618 __ verify_oop(src->as_register());
619 }
620 move_regs(src->as_register(), dest->as_register());
621 } else if (dest->is_double_cpu()) {
622 if (is_reference_type(src->type())) {
623 __ verify_oop(src->as_register());
624 move_regs(src->as_register(), dest->as_register_lo());
625 return;
626 }
627 assert(src->is_double_cpu(), "must match");
628 Register f_lo = src->as_register_lo();
629 Register f_hi = src->as_register_hi();
630 Register t_lo = dest->as_register_lo();
631 Register t_hi = dest->as_register_hi();
632 assert(f_hi == f_lo, "must be same");
633 assert(t_hi == t_lo, "must be same");
634 move_regs(f_lo, t_lo);
635 } else if (dest->is_single_fpu()) {
636 assert(src->is_single_fpu(), "expect single fpu");
637 __ fmv_s(dest->as_float_reg(), src->as_float_reg());
638 } else if (dest->is_double_fpu()) {
639 assert(src->is_double_fpu(), "expect double fpu");
640 __ fmv_d(dest->as_double_reg(), src->as_double_reg());
641 } else {
642 ShouldNotReachHere();
643 }
644 }
645
646 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
647 precond(src->is_register() && dest->is_stack());
648
649 uint const c_sz32 = sizeof(uint32_t);
650 uint const c_sz64 = sizeof(uint64_t);
651
652 assert(src->is_register(), "should not call otherwise");
653 assert(dest->is_stack(), "should not call otherwise");
654 if (src->is_single_cpu()) {
655 int index = dest->single_stack_ix();
656 if (is_reference_type(type)) {
657 __ sd(src->as_register(), stack_slot_address(index, c_sz64));
658 __ verify_oop(src->as_register());
659 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
660 __ sd(src->as_register(), stack_slot_address(index, c_sz64));
661 } else {
662 __ sw(src->as_register(), stack_slot_address(index, c_sz32));
663 }
664 } else if (src->is_double_cpu()) {
665 int index = dest->double_stack_ix();
666 Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
667 __ sd(src->as_register_lo(), dest_addr_LO);
668 } else if (src->is_single_fpu()) {
669 int index = dest->single_stack_ix();
670 __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32));
671 } else if (src->is_double_fpu()) {
672 int index = dest->double_stack_ix();
673 __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64));
674 } else {
675 ShouldNotReachHere();
676 }
677 }
678
679 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
680 LIR_Address* to_addr = dest->as_address_ptr();
681 // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src
682 Register compressed_src = t1;
683
684 if (patch_code != lir_patch_none) {
685 deoptimize_trap(info);
686 return;
687 }
688
689 if (is_reference_type(type)) {
690 __ verify_oop(src->as_register());
691
692 if (UseCompressedOops && !wide) {
693 __ encode_heap_oop(compressed_src, src->as_register());
694 } else {
695 compressed_src = src->as_register();
696 }
697 }
698
699 int null_check_here = code_offset();
700
701 switch (type) {
702 case T_FLOAT:
703 __ fsw(src->as_float_reg(), as_Address(to_addr));
704 break;
705
706 case T_DOUBLE:
707 __ fsd(src->as_double_reg(), as_Address(to_addr));
708 break;
709
710 case T_ARRAY: // fall through
711 case T_OBJECT:
712 if (UseCompressedOops && !wide) {
713 __ sw(compressed_src, as_Address(to_addr));
714 } else {
715 __ sd(compressed_src, as_Address(to_addr));
716 }
717 break;
718 case T_METADATA:
719 // We get here to store a method pointer to the stack to pass to
720 // a dtrace runtime call. This can't work on 64 bit with
721 // compressed klass ptrs: T_METADATA can be compressed klass
722 // ptr or a 64 bit method pointer.
723 ShouldNotReachHere();
724 __ sd(src->as_register(), as_Address(to_addr));
725 break;
726 case T_ADDRESS:
727 __ sd(src->as_register(), as_Address(to_addr));
728 break;
729 case T_INT:
730 __ sw(src->as_register(), as_Address(to_addr));
731 break;
732 case T_LONG:
733 __ sd(src->as_register_lo(), as_Address(to_addr));
734 break;
735 case T_BYTE: // fall through
736 case T_BOOLEAN:
737 __ sb(src->as_register(), as_Address(to_addr));
738 break;
739 case T_CHAR: // fall through
740 case T_SHORT:
741 __ sh(src->as_register(), as_Address(to_addr));
742 break;
743 default:
744 ShouldNotReachHere();
745 }
746
747 if (info != NULL) {
748 add_debug_info_for_null_check(null_check_here, info);
749 }
750 }
751
752 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
753 precond(src->is_stack() && dest->is_register());
754
755 uint const c_sz32 = sizeof(uint32_t);
756 uint const c_sz64 = sizeof(uint64_t);
757
758 if (dest->is_single_cpu()) {
759 int index = src->single_stack_ix();
760 if (type == T_INT) {
761 __ lw(dest->as_register(), stack_slot_address(index, c_sz32));
762 } else if (is_reference_type(type)) {
763 __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
764 __ verify_oop(dest->as_register());
765 } else if (type == T_METADATA || type == T_ADDRESS) {
766 __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
767 } else {
768 __ lwu(dest->as_register(), stack_slot_address(index, c_sz32));
769 }
770 } else if (dest->is_double_cpu()) {
771 int index = src->double_stack_ix();
772 Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
773 __ ld(dest->as_register_lo(), src_addr_LO);
774 } else if (dest->is_single_fpu()) {
775 int index = src->single_stack_ix();
776 __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32));
777 } else if (dest->is_double_fpu()) {
778 int index = src->double_stack_ix();
779 __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64));
780 } else {
781 ShouldNotReachHere();
782 }
783 }
784
785 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
786 deoptimize_trap(info);
787 }
788
789 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
790 LIR_Opr temp;
791 if (type == T_LONG || type == T_DOUBLE) {
792 temp = FrameMap::t1_long_opr;
793 } else {
794 temp = FrameMap::t1_opr;
795 }
796
797 stack2reg(src, temp, src->type());
798 reg2stack(temp, dest, dest->type(), false);
799 }
800
801 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
802 assert(src->is_address(), "should not call otherwise");
803 assert(dest->is_register(), "should not call otherwise");
804
805 LIR_Address* addr = src->as_address_ptr();
806 LIR_Address* from_addr = src->as_address_ptr();
807
808 if (addr->base()->type() == T_OBJECT) {
809 __ verify_oop(addr->base()->as_pointer_register());
810 }
811
812 if (patch_code != lir_patch_none) {
813 deoptimize_trap(info);
814 return;
815 }
816
817 if (info != NULL) {
818 add_debug_info_for_null_check_here(info);
819 }
820
821 int null_check_here = code_offset();
822 switch (type) {
823 case T_FLOAT:
824 __ flw(dest->as_float_reg(), as_Address(from_addr));
825 break;
826 case T_DOUBLE:
827 __ fld(dest->as_double_reg(), as_Address(from_addr));
828 break;
829 case T_ARRAY: // fall through
830 case T_OBJECT:
831 if (UseCompressedOops && !wide) {
832 __ lwu(dest->as_register(), as_Address(from_addr));
833 } else {
834 __ ld(dest->as_register(), as_Address(from_addr));
835 }
836 break;
837 case T_METADATA:
838 // We get here to store a method pointer to the stack to pass to
839 // a dtrace runtime call. This can't work on 64 bit with
840 // compressed klass ptrs: T_METADATA can be a compressed klass
841 // ptr or a 64 bit method pointer.
842 ShouldNotReachHere();
843 __ ld(dest->as_register(), as_Address(from_addr));
844 break;
845 case T_ADDRESS:
846 __ ld(dest->as_register(), as_Address(from_addr));
847 break;
848 case T_INT:
849 __ lw(dest->as_register(), as_Address(from_addr));
850 break;
851 case T_LONG:
852 __ ld(dest->as_register_lo(), as_Address_lo(from_addr));
853 break;
854 case T_BYTE:
855 __ lb(dest->as_register(), as_Address(from_addr));
856 break;
857 case T_BOOLEAN:
858 __ lbu(dest->as_register(), as_Address(from_addr));
859 break;
860 case T_CHAR:
861 __ lhu(dest->as_register(), as_Address(from_addr));
862 break;
863 case T_SHORT:
864 __ lh(dest->as_register(), as_Address(from_addr));
865 break;
866 default:
867 ShouldNotReachHere();
868 }
869
870 if (is_reference_type(type)) {
871 if (UseCompressedOops && !wide) {
872 __ decode_heap_oop(dest->as_register());
873 }
874
875 if (!UseZGC) {
876 // Load barrier has not yet been applied, so ZGC can't verify the oop here
877 __ verify_oop(dest->as_register());
878 }
879 }
880 }
881
882 void LIR_Assembler::emit_op3(LIR_Op3* op) {
883 switch (op->code()) {
884 case lir_idiv: // fall through
885 case lir_irem:
886 arithmetic_idiv(op->code(),
887 op->in_opr1(),
888 op->in_opr2(),
889 op->in_opr3(),
890 op->result_opr(),
891 op->info());
892 break;
893 case lir_fmad:
894 __ fmadd_d(op->result_opr()->as_double_reg(),
895 op->in_opr1()->as_double_reg(),
896 op->in_opr2()->as_double_reg(),
897 op->in_opr3()->as_double_reg());
898 break;
899 case lir_fmaf:
900 __ fmadd_s(op->result_opr()->as_float_reg(),
901 op->in_opr1()->as_float_reg(),
902 op->in_opr2()->as_float_reg(),
903 op->in_opr3()->as_float_reg());
904 break;
905 default:
906 ShouldNotReachHere();
907 }
908 }
909
910 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
911 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
912 Label label;
913
914 emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false,
915 /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true);
916
917 Label done;
918 move_op(opr2, result, type, lir_patch_none, NULL,
919 false, // pop_fpu_stack
920 false); // wide
921 __ j(done);
922 __ bind(label);
923 move_op(opr1, result, type, lir_patch_none, NULL,
924 false, // pop_fpu_stack
925 false); // wide
926 __ bind(done);
927 }
928
929 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
930 LIR_Condition condition = op->cond();
931 if (condition == lir_cond_always) {
932 if (op->info() != NULL) {
933 add_debug_info_for_branch(op->info());
934 }
935 } else {
936 assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands");
937 }
938 bool is_unordered = (op->ublock() == op->block());
939 emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered);
940 }
941
942 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label,
943 bool is_far, bool is_unordered) {
944
945 if (cmp_flag == lir_cond_always) {
946 __ j(label);
947 return;
948 }
949
950 if (cmp1->is_cpu_register()) {
951 Register reg1 = as_reg(cmp1);
952 if (cmp2->is_cpu_register()) {
953 Register reg2 = as_reg(cmp2);
954 __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far);
955 } else if (cmp2->is_constant()) {
956 const2reg_helper(cmp2);
957 __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far);
958 } else {
959 ShouldNotReachHere();
960 }
961 } else if (cmp1->is_single_fpu()) {
962 assert(cmp2->is_single_fpu(), "expect single float register");
963 __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered);
964 } else if (cmp1->is_double_fpu()) {
965 assert(cmp2->is_double_fpu(), "expect double float register");
966 __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask,
967 cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered);
968 } else {
969 ShouldNotReachHere();
970 }
971 }
972
973 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
974 LIR_Opr src = op->in_opr();
975 LIR_Opr dest = op->result_opr();
976
977 switch (op->bytecode()) {
978 case Bytecodes::_i2f:
979 __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break;
980 case Bytecodes::_i2d:
981 __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break;
982 case Bytecodes::_l2d:
983 __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break;
984 case Bytecodes::_l2f:
985 __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break;
986 case Bytecodes::_f2d:
987 __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break;
988 case Bytecodes::_d2f:
989 __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break;
990 case Bytecodes::_i2c:
991 __ zero_extend(dest->as_register(), src->as_register(), 16); break;
992 case Bytecodes::_i2l:
993 __ addw(dest->as_register_lo(), src->as_register(), zr); break;
994 case Bytecodes::_i2s:
995 __ sign_extend(dest->as_register(), src->as_register(), 16); break;
996 case Bytecodes::_i2b:
997 __ sign_extend(dest->as_register(), src->as_register(), 8); break;
998 case Bytecodes::_l2i:
999 _masm->block_comment("FIXME: This coulde be no-op");
1000 __ addw(dest->as_register(), src->as_register_lo(), zr); break;
1001 case Bytecodes::_d2l:
1002 __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break;
1003 case Bytecodes::_f2i:
1004 __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break;
1005 case Bytecodes::_f2l:
1006 __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break;
1007 case Bytecodes::_d2i:
1008 __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break;
1009 default:
1010 ShouldNotReachHere();
1011 }
1012 }
1013
1014 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1015 if (op->init_check()) {
1016 __ lbu(t0, Address(op->klass()->as_register(),
1017 InstanceKlass::init_state_offset()));
1018 __ mvw(t1, InstanceKlass::fully_initialized);
1019 add_debug_info_for_null_check_here(op->stub()->info());
1020 __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
1021 }
1022
1023 __ allocate_object(op->obj()->as_register(),
1024 op->tmp1()->as_register(),
1025 op->tmp2()->as_register(),
1026 op->header_size(),
1027 op->object_size(),
1028 op->klass()->as_register(),
1029 *op->stub()->entry());
1030
1031 __ bind(*op->stub()->continuation());
1032 }
1033
1034 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1035 Register len = op->len()->as_register();
1036
1037 if (UseSlowPath ||
1038 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1039 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1040 __ j(*op->stub()->entry());
1041 } else {
1042 Register tmp1 = op->tmp1()->as_register();
1043 Register tmp2 = op->tmp2()->as_register();
1044 Register tmp3 = op->tmp3()->as_register();
1045 if (len == tmp1) {
1046 tmp1 = tmp3;
1047 } else if (len == tmp2) {
1048 tmp2 = tmp3;
1049 } else if (len == tmp3) {
1050 // everything is ok
1051 } else {
1052 __ mv(tmp3, len);
1053 }
1054 __ allocate_array(op->obj()->as_register(),
1055 len,
1056 tmp1,
1057 tmp2,
1058 arrayOopDesc::header_size(op->type()),
1059 array_element_size(op->type()),
1060 op->klass()->as_register(),
1061 *op->stub()->entry());
1062 }
1063 __ bind(*op->stub()->continuation());
1064 }
1065
1066 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
1067 Register recv, Label* update_done) {
1068 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1069 Label next_test;
1070 // See if the receiver is receiver[n].
1071 __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1072 __ bne(recv, t1, next_test);
1073 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1074 __ add_memory_int64(data_addr, DataLayout::counter_increment);
1075 __ j(*update_done);
1076 __ bind(next_test);
1077 }
1078
1079 // Didn't find receiver; find next empty slot and fill it in
1080 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1081 Label next_test;
1082 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1083 __ ld(t1, recv_addr);
1084 __ bnez(t1, next_test);
1085 __ sd(recv, recv_addr);
1086 __ li(t1, DataLayout::counter_increment);
1087 __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1088 __ j(*update_done);
1089 __ bind(next_test);
1090 }
1091 }
1092
1093 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
1094 ciMethod* method = op->profiled_method();
1095 assert(method != NULL, "Should have method");
1096 int bci = op->profiled_bci();
1097 *md = method->method_data_or_null();
1098 guarantee(*md != NULL, "Sanity");
1099 *data = ((*md)->bci_to_data(bci));
1100 assert(*data != NULL, "need data for type check");
1101 assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1102 }
1103
1104 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1105 Register k_RInfo, Register klass_RInfo,
1106 Label *failure_target, Label *success_target) {
1107 // get object class
1108 // not a safepoint as obj null check happens earlier
1109 __ load_klass(klass_RInfo, obj);
1110 if (k->is_loaded()) {
1111 // See if we get an immediate positive hit
1112 __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1113 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1114 __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1115 // successful cast, fall through to profile or jump
1116 } else {
1117 // See if we get an immediate positive hit
1118 __ beq(k_RInfo, t0, *success_target);
1119 // check for self
1120 __ beq(klass_RInfo, k_RInfo, *success_target);
1121
1122 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1123 __ sd(k_RInfo, Address(sp, 0)); // sub klass
1124 __ sd(klass_RInfo, Address(sp, wordSize)); // super klass
1125 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1126 // load result to k_RInfo
1127 __ ld(k_RInfo, Address(sp, 0));
1128 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1129 // result is a boolean
1130 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1131 // successful cast, fall through to profile or jump
1132 }
1133 } else {
1134 // perform the fast part of the checking logic
1135 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1136 // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1137 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1138 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
1139 __ sd(k_RInfo, Address(sp, 0)); // super klass
1140 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1141 // load result to k_RInfo
1142 __ ld(k_RInfo, Address(sp, 0));
1143 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1144 // result is a boolean
1145 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1146 // successful cast, fall thriugh to profile or jump
1147 }
1148 }
1149
1150 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj,
1151 Register klass_RInfo, Label* obj_is_null) {
1152 Label not_null;
1153 __ bnez(obj, not_null);
1154 // Object is null, update MDO and exit
1155 Register mdo = klass_RInfo;
1156 __ mov_metadata(mdo, md->constant_encoding());
1157 Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1158 __ lbu(t0, data_addr);
1159 __ ori(t0, t0, BitData::null_seen_byte_constant());
1160 __ sb(t0, data_addr);
1161 __ j(*obj_is_null);
1162 __ bind(not_null);
1163 }
1164
1165 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
1166 if (!k->is_loaded()) {
1167 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1168 } else {
1169 __ mov_metadata(k_RInfo, k->constant_encoding());
1170 }
1171 }
1172
1173 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1174 Register obj = op->object()->as_register();
1175 Register k_RInfo = op->tmp1()->as_register();
1176 Register klass_RInfo = op->tmp2()->as_register();
1177 Register dst = op->result_opr()->as_register();
1178 ciKlass* k = op->klass();
1179 Register Rtmp1 = noreg;
1180
1181 // check if it needs to be profiled
1182 ciMethodData* md = NULL;
1183 ciProfileData* data = NULL;
1184
1185 const bool should_profile = op->should_profile();
1186 if (should_profile) {
1187 data_check(op, &md, &data);
1188 }
1189 Label profile_cast_success, profile_cast_failure;
1190 Label *success_target = should_profile ? &profile_cast_success : success;
1191 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1192
1193 if (obj == k_RInfo) {
1194 k_RInfo = dst;
1195 } else if (obj == klass_RInfo) {
1196 klass_RInfo = dst;
1197 }
1198 if (k->is_loaded() && !UseCompressedClassPointers) {
1199 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1200 } else {
1201 Rtmp1 = op->tmp3()->as_register();
1202 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1203 }
1204
1205 assert_different_registers(obj, k_RInfo, klass_RInfo);
1206
1207 if (should_profile) {
1208 profile_object(md, data, obj, klass_RInfo, obj_is_null);
1209 } else {
1210 __ beqz(obj, *obj_is_null);
1211 }
1212
1213 typecheck_loaded(op, k, k_RInfo);
1214 __ verify_oop(obj);
1215
1216 if (op->fast_check()) {
1217 // get object class
1218 // not a safepoint as obj null check happens earlier
1219 __ load_klass(t0, obj);
1220 __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1221 // successful cast, fall through to profile or jump
1222 } else {
1223 typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1224 }
1225 if (should_profile) {
1226 type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure);
1227 }
1228 __ j(*success);
1229 }
1230
1231 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1232 const bool should_profile = op->should_profile();
1233
1234 LIR_Code code = op->code();
1235 if (code == lir_store_check) {
1236 typecheck_lir_store(op, should_profile);
1237 } else if (code == lir_checkcast) {
1238 Register obj = op->object()->as_register();
1239 Register dst = op->result_opr()->as_register();
1240 Label success;
1241 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1242 __ bind(success);
1243 if (dst != obj) {
1244 __ mv(dst, obj);
1245 }
1246 } else if (code == lir_instanceof) {
1247 Register obj = op->object()->as_register();
1248 Register dst = op->result_opr()->as_register();
1249 Label success, failure, done;
1250 emit_typecheck_helper(op, &success, &failure, &failure);
1251 __ bind(failure);
1252 __ mv(dst, zr);
1253 __ j(done);
1254 __ bind(success);
1255 __ mv(dst, 1);
1256 __ bind(done);
1257 } else {
1258 ShouldNotReachHere();
1259 }
1260 }
1261
1262 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1263 assert(VM_Version::supports_cx8(), "wrong machine");
1264 Register addr;
1265 if (op->addr()->is_register()) {
1266 addr = as_reg(op->addr());
1267 } else {
1268 assert(op->addr()->is_address(), "what else?");
1269 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1270 assert(addr_ptr->disp() == 0, "need 0 disp");
1271 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1272 addr = as_reg(addr_ptr->base());
1273 }
1274 Register newval = as_reg(op->new_value());
1275 Register cmpval = as_reg(op->cmp_value());
1276
1277 if (op->code() == lir_cas_obj) {
1278 if (UseCompressedOops) {
1279 Register tmp1 = op->tmp1()->as_register();
1280 assert(op->tmp1()->is_valid(), "must be");
1281 __ encode_heap_oop(tmp1, cmpval);
1282 cmpval = tmp1;
1283 __ encode_heap_oop(t1, newval);
1284 newval = t1;
1285 caswu(addr, newval, cmpval);
1286 } else {
1287 casl(addr, newval, cmpval);
1288 }
1289 } else if (op->code() == lir_cas_int) {
1290 casw(addr, newval, cmpval);
1291 } else {
1292 casl(addr, newval, cmpval);
1293 }
1294 }
1295
1296 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1297 switch (code) {
1298 case lir_abs: __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break;
1299 case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break;
1300 default: ShouldNotReachHere();
1301 }
1302 }
1303
1304 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1305 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1306 Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1307 if (dst->is_single_cpu()) {
1308 Register Rdst = dst->as_register();
1309 if (right->is_constant()) {
1310 int right_const = right->as_jint();
1311 if (Assembler::operand_valid_for_add_immediate(right_const)) {
1312 logic_op_imm(Rdst, Rleft, right_const, code);
1313 __ addw(Rdst, Rdst, zr);
1314 } else {
1315 __ mv(t0, right_const);
1316 logic_op_reg32(Rdst, Rleft, t0, code);
1317 }
1318 } else {
1319 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1320 logic_op_reg32(Rdst, Rleft, Rright, code);
1321 }
1322 } else {
1323 Register Rdst = dst->as_register_lo();
1324 if (right->is_constant()) {
1325 long right_const = right->as_jlong();
1326 if (Assembler::operand_valid_for_add_immediate(right_const)) {
1327 logic_op_imm(Rdst, Rleft, right_const, code);
1328 } else {
1329 __ mv(t0, right_const);
1330 logic_op_reg(Rdst, Rleft, t0, code);
1331 }
1332 } else {
1333 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1334 logic_op_reg(Rdst, Rleft, Rright, code);
1335 }
1336 }
1337 }
1338
1339 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) {
1340 ShouldNotCallThis();
1341 }
1342
1343 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1344 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1345 bool is_unordered_less = (code == lir_ucmp_fd2i);
1346 if (left->is_single_fpu()) {
1347 __ float_cmp(true, is_unordered_less ? -1 : 1,
1348 left->as_float_reg(), right->as_float_reg(), dst->as_register());
1349 } else if (left->is_double_fpu()) {
1350 __ float_cmp(false, is_unordered_less ? -1 : 1,
1351 left->as_double_reg(), right->as_double_reg(), dst->as_register());
1352 } else {
1353 ShouldNotReachHere();
1354 }
1355 } else if (code == lir_cmp_l2i) {
1356 __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1357 } else {
1358 ShouldNotReachHere();
1359 }
1360 }
1361
1362 void LIR_Assembler::align_call(LIR_Code code) {
1363 // With RVC a call instruction may get 2-byte aligned.
1364 // The address of the call instruction needs to be 4-byte aligned to
1365 // ensure that it does not span a cache line so that it can be patched.
1366 __ align(4);
1367 }
1368
1369 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1370 address call = __ trampoline_call(Address(op->addr(), rtype));
1371 if (call == NULL) {
1372 bailout("trampoline stub overflow");
1373 return;
1374 }
1375 add_call_info(code_offset(), op->info());
1376 }
1377
1378 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1379 address call = __ ic_call(op->addr());
1380 if (call == NULL) {
1381 bailout("trampoline stub overflow");
1382 return;
1383 }
1384 add_call_info(code_offset(), op->info());
1385 }
1386
1387 void LIR_Assembler::emit_static_call_stub() {
1388 address call_pc = __ pc();
1389 assert((__ offset() % 4) == 0, "bad alignment");
1390 address stub = __ start_a_stub(call_stub_size());
1391 if (stub == NULL) {
1392 bailout("static call stub overflow");
1393 return;
1394 }
1395
1396 int start = __ offset();
1397
1398 __ relocate(static_stub_Relocation::spec(call_pc));
1399 __ emit_static_call_stub();
1400
1401 assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
1402 <= call_stub_size(), "stub too big");
1403 __ end_a_stub();
1404 }
1405
1406 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1407 assert(exceptionOop->as_register() == x10, "must match");
1408 assert(exceptionPC->as_register() == x13, "must match");
1409
1410 // exception object is not added to oop map by LinearScan
1411 // (LinearScan assumes that no oops are in fixed registers)
1412 info->add_register_oop(exceptionOop);
1413 Runtime1::StubID unwind_id;
1414
1415 // get current pc information
1416 // pc is only needed if the method has an exception handler, the unwind code does not need it.
1417 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
1418 // As no instructions have been generated yet for this LIR node it's
1419 // possible that an oop map already exists for the current offset.
1420 // In that case insert an dummy NOP here to ensure all oop map PCs
1421 // are unique. See JDK-8237483.
1422 __ nop();
1423 }
1424 int pc_for_athrow_offset = __ offset();
1425 InternalAddress pc_for_athrow(__ pc());
1426 int32_t off = 0;
1427 __ la_patchable(exceptionPC->as_register(), pc_for_athrow, off);
1428 __ addi(exceptionPC->as_register(), exceptionPC->as_register(), off);
1429 add_call_info(pc_for_athrow_offset, info); // for exception handler
1430
1431 __ verify_not_null_oop(x10);
1432 // search an exception handler (x10: exception oop, x13: throwing pc)
1433 if (compilation()->has_fpu_code()) {
1434 unwind_id = Runtime1::handle_exception_id;
1435 } else {
1436 unwind_id = Runtime1::handle_exception_nofpu_id;
1437 }
1438 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
1439 __ nop();
1440 }
1441
1442 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1443 assert(exceptionOop->as_register() == x10, "must match");
1444 __ j(_unwind_handler_entry);
1445 }
1446
1447 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
1448 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1449 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1450 Register count_reg = count->as_register();
1451 if (dest->is_single_cpu()) {
1452 assert (dest->type() == T_INT, "unexpected result type");
1453 assert (left->type() == T_INT, "unexpected left type");
1454 __ andi(t0, count_reg, 31); // should not shift more than 31 bits
1455 switch (code) {
1456 case lir_shl: __ sllw(dest_reg, left_reg, t0); break;
1457 case lir_shr: __ sraw(dest_reg, left_reg, t0); break;
1458 case lir_ushr: __ srlw(dest_reg, left_reg, t0); break;
1459 default: ShouldNotReachHere();
1460 }
1461 } else if (dest->is_double_cpu()) {
1462 __ andi(t0, count_reg, 63); // should not shift more than 63 bits
1463 switch (code) {
1464 case lir_shl: __ sll(dest_reg, left_reg, t0); break;
1465 case lir_shr: __ sra(dest_reg, left_reg, t0); break;
1466 case lir_ushr: __ srl(dest_reg, left_reg, t0); break;
1467 default: ShouldNotReachHere();
1468 }
1469 } else {
1470 ShouldNotReachHere();
1471 }
1472 }
1473
1474 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
1475 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1476 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1477 if (dest->is_single_cpu()) {
1478 assert (dest->type() == T_INT, "unexpected result type");
1479 assert (left->type() == T_INT, "unexpected left type");
1480 count &= 0x1f;
1481 if (count != 0) {
1482 switch (code) {
1483 case lir_shl: __ slliw(dest_reg, left_reg, count); break;
1484 case lir_shr: __ sraiw(dest_reg, left_reg, count); break;
1485 case lir_ushr: __ srliw(dest_reg, left_reg, count); break;
1486 default: ShouldNotReachHere();
1487 }
1488 } else {
1489 move_regs(left_reg, dest_reg);
1490 }
1491 } else if (dest->is_double_cpu()) {
1492 count &= 0x3f;
1493 if (count != 0) {
1494 switch (code) {
1495 case lir_shl: __ slli(dest_reg, left_reg, count); break;
1496 case lir_shr: __ srai(dest_reg, left_reg, count); break;
1497 case lir_ushr: __ srli(dest_reg, left_reg, count); break;
1498 default: ShouldNotReachHere();
1499 }
1500 } else {
1501 move_regs(left->as_register_lo(), dest->as_register_lo());
1502 }
1503 } else {
1504 ShouldNotReachHere();
1505 }
1506 }
1507
1508 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
1509 Register obj = op->obj_opr()->as_register(); // may not be an oop
1510 Register hdr = op->hdr_opr()->as_register();
1511 Register lock = op->lock_opr()->as_register();
1512 if (UseHeavyMonitors) {
1513 __ j(*op->stub()->entry());
1514 } else if (op->code() == lir_lock) {
1515 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1516 // add debug info for NullPointerException only if one is possible
1517 int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
1518 if (op->info() != NULL) {
1519 add_debug_info_for_null_check(null_check_offset, op->info());
1520 }
1521 } else if (op->code() == lir_unlock) {
1522 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1523 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
1524 } else {
1525 Unimplemented();
1526 }
1527 __ bind(*op->stub()->continuation());
1528 }
1529
1530 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
1531 Register obj = op->obj()->as_pointer_register();
1532 Register result = op->result_opr()->as_pointer_register();
1533
1534 CodeEmitInfo* info = op->info();
1535 if (info != NULL) {
1536 add_debug_info_for_null_check_here(info);
1537 }
1538
1539 if (UseCompressedClassPointers) {
1540 __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes()));
1541 __ decode_klass_not_null(result);
1542 } else {
1543 __ ld(result, Address(obj, oopDesc::klass_offset_in_bytes()));
1544 }
1545 }
1546
1547 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
1548 ciMethod* method = op->profiled_method();
1549 int bci = op->profiled_bci();
1550
1551 // Update counter for all call types
1552 ciMethodData* md = method->method_data_or_null();
1553 guarantee(md != NULL, "Sanity");
1554 ciProfileData* data = md->bci_to_data(bci);
1555 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
1556 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
1557 Register mdo = op->mdo()->as_register();
1558 __ mov_metadata(mdo, md->constant_encoding());
1559 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1560 // Perform additional virtual call profiling for invokevirtual and
1561 // invokeinterface bytecodes
1562 if (op->should_profile_receiver_type()) {
1563 assert(op->recv()->is_single_cpu(), "recv must be allocated");
1564 Register recv = op->recv()->as_register();
1565 assert_different_registers(mdo, recv);
1566 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
1567 ciKlass* known_klass = op->known_holder();
1568 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
1569 // We know the type that will be seen at this call site; we can
1570 // statically update the MethodData* rather than needing to do
1571 // dynamic tests on the receiver type
1572 // NOTE: we should probably put a lock around this search to
1573 // avoid collisions by concurrent compilations
1574 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
1575 uint i;
1576 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1577 ciKlass* receiver = vc_data->receiver(i);
1578 if (known_klass->equals(receiver)) {
1579 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1580 __ add_memory_int64(data_addr, DataLayout::counter_increment);
1581 return;
1582 }
1583 }
1584
1585 // Receiver type not found in profile data; select an empty slot
1586 // Note that this is less efficient than it should be because it
1587 // always does a write to the receiver part of the
1588 // VirtualCallData rather than just the first time
1589 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1590 ciKlass* receiver = vc_data->receiver(i);
1591 if (receiver == NULL) {
1592 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
1593 __ mov_metadata(t1, known_klass->constant_encoding());
1594 __ sd(t1, recv_addr);
1595 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1596 __ add_memory_int64(data_addr, DataLayout::counter_increment);
1597 return;
1598 }
1599 }
1600 } else {
1601 __ load_klass(recv, recv);
1602 Label update_done;
1603 type_profile_helper(mdo, md, data, recv, &update_done);
1604 // Receiver did not match any saved receiver and there is no empty row for it.
1605 // Increment total counter to indicate polymorphic case.
1606 __ add_memory_int64(counter_addr, DataLayout::counter_increment);
1607
1608 __ bind(update_done);
1609 }
1610 } else {
1611 // Static call
1612 __ add_memory_int64(counter_addr, DataLayout::counter_increment);
1613 }
1614 }
1615
1616 void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); }
1617
1618 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
1619 __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
1620 }
1621
1622 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); }
1623
1624 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
1625 Register tmp, Label &next, Label &none,
1626 Address mdo_addr) {
1627 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
1628 if (exact_klass != NULL) {
1629 __ mov_metadata(tmp, exact_klass->constant_encoding());
1630 } else {
1631 __ load_klass(tmp, tmp);
1632 }
1633
1634 __ ld(t1, mdo_addr);
1635 __ xorr(tmp, tmp, t1);
1636 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1637 // klass seen before, nothing to do. The unknown bit may have been
1638 // set already but no need to check.
1639 __ beqz(t0, next);
1640
1641 // already unknown. Nothing to do anymore.
1642 __ andi(t0, tmp, TypeEntries::type_unknown);
1643 __ bnez(t0, next);
1644
1645 if (TypeEntries::is_type_none(current_klass)) {
1646 __ beqz(t1, none);
1647 __ li(t0, (u1)TypeEntries::null_seen);
1648 __ beq(t0, t1, none);
1649 // There is a chance that the checks above (re-reading profiling
1650 // data from memory) fail if another thread has just set the
1651 // profiling to this obj's klass
1652 __ membar(MacroAssembler::LoadLoad);
1653 __ ld(t1, mdo_addr);
1654 __ xorr(tmp, tmp, t1);
1655 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1656 __ beqz(t0, next);
1657 }
1658 } else {
1659 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1660 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
1661
1662 __ ld(tmp, mdo_addr);
1663 // already unknown. Nothing to do anymore.
1664 __ andi(t0, tmp, TypeEntries::type_unknown);
1665 __ bnez(t0, next);
1666 }
1667
1668 // different than before. Cannot keep accurate profile.
1669 __ ld(t1, mdo_addr);
1670 __ ori(t1, t1, TypeEntries::type_unknown);
1671 __ sd(t1, mdo_addr);
1672
1673 if (TypeEntries::is_type_none(current_klass)) {
1674 __ j(next);
1675
1676 __ bind(none);
1677 // first time here. Set profile type.
1678 __ sd(tmp, mdo_addr);
1679 }
1680 }
1681
1682 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp,
1683 Address mdo_addr, Label &next) {
1684 // There's a single possible klass at this profile point
1685 assert(exact_klass != NULL, "should be");
1686 if (TypeEntries::is_type_none(current_klass)) {
1687 __ mov_metadata(tmp, exact_klass->constant_encoding());
1688 __ ld(t1, mdo_addr);
1689 __ xorr(tmp, tmp, t1);
1690 __ andi(t0, tmp, TypeEntries::type_klass_mask);
1691 __ beqz(t0, next);
1692 #ifdef ASSERT
1693 {
1694 Label ok;
1695 __ ld(t0, mdo_addr);
1696 __ beqz(t0, ok);
1697 __ li(t1, (u1)TypeEntries::null_seen);
1698 __ beq(t0, t1, ok);
1699 // may have been set by another thread
1700 __ membar(MacroAssembler::LoadLoad);
1701 __ mov_metadata(t0, exact_klass->constant_encoding());
1702 __ ld(t1, mdo_addr);
1703 __ xorr(t1, t0, t1);
1704 __ andi(t1, t1, TypeEntries::type_mask);
1705 __ beqz(t1, ok);
1706
1707 __ stop("unexpected profiling mismatch");
1708 __ bind(ok);
1709 }
1710 #endif
1711 // first time here. Set profile type.
1712 __ sd(tmp, mdo_addr);
1713 } else {
1714 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1715 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
1716
1717 __ ld(tmp, mdo_addr);
1718 // already unknown. Nothing to do anymore.
1719 __ andi(t0, tmp, TypeEntries::type_unknown);
1720 __ bnez(t0, next);
1721
1722 __ ori(tmp, tmp, TypeEntries::type_unknown);
1723 __ sd(tmp, mdo_addr);
1724 }
1725 }
1726
1727 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass,
1728 Address mdo_addr, bool do_update, Label &next) {
1729 __ bnez(tmp, update);
1730 if (!TypeEntries::was_null_seen(current_klass)) {
1731 __ ld(t1, mdo_addr);
1732 __ ori(t1, t1, TypeEntries::null_seen);
1733 __ sd(t1, mdo_addr);
1734 }
1735 if (do_update) {
1736 __ j(next);
1737 }
1738 }
1739
1740 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
1741 COMMENT("emit_profile_type {");
1742 Register obj = op->obj()->as_register();
1743 Register tmp = op->tmp()->as_pointer_register();
1744 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
1745 ciKlass* exact_klass = op->exact_klass();
1746 intptr_t current_klass = op->current_klass();
1747 bool not_null = op->not_null();
1748 bool no_conflict = op->no_conflict();
1749
1750 Label update, next, none;
1751
1752 bool do_null = !not_null;
1753 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
1754 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
1755
1756 assert(do_null || do_update, "why are we here?");
1757 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
1758 assert_different_registers(tmp, t0, t1, mdo_addr.base());
1759
1760 __ verify_oop(obj);
1761
1762 if (tmp != obj) {
1763 __ mv(tmp, obj);
1764 }
1765 if (do_null) {
1766 check_null(tmp, update, current_klass, mdo_addr, do_update, next);
1767 #ifdef ASSERT
1768 } else {
1769 __ bnez(tmp, update);
1770 __ stop("unexpected null obj");
1771 #endif
1772 }
1773
1774 __ bind(update);
1775
1776 if (do_update) {
1777 #ifdef ASSERT
1778 if (exact_klass != NULL) {
1779 check_exact_klass(tmp, exact_klass);
1780 }
1781 #endif
1782 if (!no_conflict) {
1783 check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr);
1784 } else {
1785 check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next);
1786 }
1787
1788 __ bind(next);
1789 }
1790 COMMENT("} emit_profile_type");
1791 }
1792
1793 void LIR_Assembler::align_backward_branch_target() { }
1794
1795 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
1796 // tmp must be unused
1797 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
1798
1799 if (left->is_single_cpu()) {
1800 assert(dest->is_single_cpu(), "expect single result reg");
1801 __ negw(dest->as_register(), left->as_register());
1802 } else if (left->is_double_cpu()) {
1803 assert(dest->is_double_cpu(), "expect double result reg");
1804 __ neg(dest->as_register_lo(), left->as_register_lo());
1805 } else if (left->is_single_fpu()) {
1806 assert(dest->is_single_fpu(), "expect single float result reg");
1807 __ fneg_s(dest->as_float_reg(), left->as_float_reg());
1808 } else {
1809 assert(left->is_double_fpu(), "expect double float operand reg");
1810 assert(dest->is_double_fpu(), "expect double float result reg");
1811 __ fneg_d(dest->as_double_reg(), left->as_double_reg());
1812 }
1813 }
1814
1815
1816 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1817 if (patch_code != lir_patch_none) {
1818 deoptimize_trap(info);
1819 return;
1820 }
1821
1822 LIR_Address* adr = addr->as_address_ptr();
1823 Register dst = dest->as_register_lo();
1824
1825 assert_different_registers(dst, t0);
1826 if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) {
1827 int scale = adr->scale();
1828 intptr_t offset = adr->disp();
1829 LIR_Opr index_op = adr->index();
1830 if (index_op->is_constant()) {
1831 offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale;
1832 }
1833
1834 if (!is_imm_in_range(offset, 12, 0)) {
1835 __ la(t0, as_Address(adr));
1836 __ mv(dst, t0);
1837 return;
1838 }
1839 }
1840
1841 __ la(dst, as_Address(adr));
1842 }
1843
1844
1845 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
1846 assert(!tmp->is_valid(), "don't need temporary");
1847
1848 CodeBlob *cb = CodeCache::find_blob(dest);
1849 if (cb != NULL) {
1850 __ far_call(RuntimeAddress(dest));
1851 } else {
1852 int32_t offset = 0;
1853 __ la_patchable(t0, RuntimeAddress(dest), offset);
1854 __ jalr(x1, t0, offset);
1855 }
1856
1857 if (info != NULL) {
1858 add_call_info_here(info);
1859 }
1860 }
1861
1862 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
1863 if (dest->is_address() || src->is_address()) {
1864 move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false);
1865 } else {
1866 ShouldNotReachHere();
1867 }
1868 }
1869
1870 #ifdef ASSERT
1871 // emit run-time assertion
1872 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
1873 assert(op->code() == lir_assert, "must be");
1874
1875 Label ok;
1876 if (op->in_opr1()->is_valid()) {
1877 assert(op->in_opr2()->is_valid(), "both operands must be valid");
1878 bool is_unordered = false;
1879 LIR_Condition cond = op->condition();
1880 emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false,
1881 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true);
1882 } else {
1883 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
1884 assert(op->condition() == lir_cond_always, "no other conditions allowed");
1885 }
1886
1887 if (op->halt()) {
1888 const char* str = __ code_string(op->msg());
1889 __ stop(str);
1890 } else {
1891 breakpoint();
1892 }
1893 __ bind(ok);
1894 }
1895 #endif
1896
1897 #ifndef PRODUCT
1898 #define COMMENT(x) do { __ block_comment(x); } while (0)
1899 #else
1900 #define COMMENT(x)
1901 #endif
1902
1903 void LIR_Assembler::membar() {
1904 COMMENT("membar");
1905 __ membar(MacroAssembler::AnyAny);
1906 }
1907
1908 void LIR_Assembler::membar_acquire() {
1909 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1910 }
1911
1912 void LIR_Assembler::membar_release() {
1913 __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1914 }
1915
1916 void LIR_Assembler::membar_loadload() {
1917 __ membar(MacroAssembler::LoadLoad);
1918 }
1919
1920 void LIR_Assembler::membar_storestore() {
1921 __ membar(MacroAssembler::StoreStore);
1922 }
1923
1924 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
1925
1926 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
1927
1928 void LIR_Assembler::on_spin_wait() {
1929 Unimplemented();
1930 }
1931
1932 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
1933 __ mv(result_reg->as_register(), xthread);
1934 }
1935
1936 void LIR_Assembler::peephole(LIR_List *lir) {}
1937
1938 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
1939 Address addr = as_Address(src->as_address_ptr());
1940 BasicType type = src->type();
1941 bool is_oop = is_reference_type(type);
1942
1943 get_op(type);
1944
1945 switch (code) {
1946 case lir_xadd:
1947 {
1948 RegisterOrConstant inc;
1949 Register tmp = as_reg(tmp_op);
1950 Register dst = as_reg(dest);
1951 if (data->is_constant()) {
1952 inc = RegisterOrConstant(as_long(data));
1953 assert_different_registers(dst, addr.base(), tmp);
1954 assert_different_registers(tmp, t0);
1955 } else {
1956 inc = RegisterOrConstant(as_reg(data));
1957 assert_different_registers(inc.as_register(), dst, addr.base(), tmp);
1958 }
1959 __ la(tmp, addr);
1960 (_masm->*add)(dst, inc, tmp);
1961 break;
1962 }
1963 case lir_xchg:
1964 {
1965 Register tmp = tmp_op->as_register();
1966 Register obj = as_reg(data);
1967 Register dst = as_reg(dest);
1968 if (is_oop && UseCompressedOops) {
1969 __ encode_heap_oop(t0, obj);
1970 obj = t0;
1971 }
1972 assert_different_registers(obj, addr.base(), tmp, dst);
1973 __ la(tmp, addr);
1974 (_masm->*xchg)(dst, obj, tmp);
1975 if (is_oop && UseCompressedOops) {
1976 __ decode_heap_oop(dst);
1977 }
1978 }
1979 break;
1980 default:
1981 ShouldNotReachHere();
1982 }
1983 __ membar(MacroAssembler::AnyAny);
1984 }
1985
1986 int LIR_Assembler::array_element_size(BasicType type) const {
1987 int elem_size = type2aelembytes(type);
1988 return exact_log2(elem_size);
1989 }
1990
1991 // helper functions which checks for overflow and sets bailout if it
1992 // occurs. Always returns a valid embeddable pointer but in the
1993 // bailout case the pointer won't be to unique storage.
1994 address LIR_Assembler::float_constant(float f) {
1995 address const_addr = __ float_constant(f);
1996 if (const_addr == NULL) {
1997 bailout("const section overflow");
1998 return __ code()->consts()->start();
1999 } else {
2000 return const_addr;
2001 }
2002 }
2003
2004 address LIR_Assembler::double_constant(double d) {
2005 address const_addr = __ double_constant(d);
2006 if (const_addr == NULL) {
2007 bailout("const section overflow");
2008 return __ code()->consts()->start();
2009 } else {
2010 return const_addr;
2011 }
2012 }
2013
2014 address LIR_Assembler::int_constant(jlong n) {
2015 address const_addr = __ long_constant(n);
2016 if (const_addr == NULL) {
2017 bailout("const section overflow");
2018 return __ code()->consts()->start();
2019 } else {
2020 return const_addr;
2021 }
2022 }
2023
2024 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
2025 __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */,
2026 Assembler::rl /* release */, t0, true /* result as bool */);
2027 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2028 __ membar(MacroAssembler::AnyAny);
2029 }
2030
2031 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) {
2032 __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */,
2033 Assembler::rl /* release */, t0, true /* result as bool */);
2034 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2035 __ membar(MacroAssembler::AnyAny);
2036 }
2037
2038 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
2039 __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */,
2040 Assembler::rl /* release */, t0, true /* result as bool */);
2041 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2042 __ membar(MacroAssembler::AnyAny);
2043 }
2044
2045 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
2046 address target = NULL;
2047
2048 switch (patching_id(info)) {
2049 case PatchingStub::access_field_id:
2050 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
2051 break;
2052 case PatchingStub::load_klass_id:
2053 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
2054 break;
2055 case PatchingStub::load_mirror_id:
2056 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
2057 break;
2058 case PatchingStub::load_appendix_id:
2059 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
2060 break;
2061 default: ShouldNotReachHere();
2062 }
2063
2064 __ far_call(RuntimeAddress(target));
2065 add_call_info_here(info);
2066 }
2067
2068 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) {
2069 Label ok;
2070 __ load_klass(tmp, tmp);
2071 __ mov_metadata(t0, exact_klass->constant_encoding());
2072 __ beq(tmp, t0, ok);
2073 __ stop("exact klass and actual klass differ");
2074 __ bind(ok);
2075 }
2076
2077 void LIR_Assembler::get_op(BasicType type) {
2078 switch (type) {
2079 case T_INT:
2080 xchg = &MacroAssembler::atomic_xchgalw;
2081 add = &MacroAssembler::atomic_addalw;
2082 break;
2083 case T_LONG:
2084 xchg = &MacroAssembler::atomic_xchgal;
2085 add = &MacroAssembler::atomic_addal;
2086 break;
2087 case T_OBJECT:
2088 case T_ARRAY:
2089 if (UseCompressedOops) {
2090 xchg = &MacroAssembler::atomic_xchgalwu;
2091 add = &MacroAssembler::atomic_addalw;
2092 } else {
2093 xchg = &MacroAssembler::atomic_xchgal;
2094 add = &MacroAssembler::atomic_addal;
2095 }
2096 break;
2097 default:
2098 ShouldNotReachHere();
2099 }
2100 }
2101
2102 // emit_opTypeCheck sub functions
2103 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) {
2104 Register value = op->object()->as_register();
2105 Register array = op->array()->as_register();
2106 Register k_RInfo = op->tmp1()->as_register();
2107 Register klass_RInfo = op->tmp2()->as_register();
2108 Register Rtmp1 = op->tmp3()->as_register();
2109
2110 CodeStub* stub = op->stub();
2111
2112 // check if it needs to be profiled
2113 ciMethodData* md = NULL;
2114 ciProfileData* data = NULL;
2115
2116 if (should_profile) {
2117 data_check(op, &md, &data);
2118 }
2119 Label profile_cast_success, profile_cast_failure, done;
2120 Label *success_target = should_profile ? &profile_cast_success : &done;
2121 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
2122
2123 if (should_profile) {
2124 profile_object(md, data, value, klass_RInfo, &done);
2125 } else {
2126 __ beqz(value, done);
2127 }
2128
2129 add_debug_info_for_null_check_here(op->info_for_exception());
2130 __ load_klass(k_RInfo, array);
2131 __ load_klass(klass_RInfo, value);
2132
2133 lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target);
2134
2135 // fall through to the success case
2136 if (should_profile) {
2137 Register mdo = klass_RInfo;
2138 Register recv = k_RInfo;
2139 __ bind(profile_cast_success);
2140 __ mov_metadata(mdo, md->constant_encoding());
2141 __ load_klass(recv, value);
2142 type_profile_helper(mdo, md, data, recv, &done);
2143 __ j(done);
2144
2145 __ bind(profile_cast_failure);
2146 __ mov_metadata(mdo, md->constant_encoding());
2147 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2148 __ ld(t1, counter_addr);
2149 __ addi(t1, t1, -DataLayout::counter_increment);
2150 __ sd(t1, counter_addr);
2151 __ j(*stub->entry());
2152 }
2153
2154 __ bind(done);
2155 }
2156
2157 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
2158 _masm->code_section()->relocate(adr, relocInfo::poll_type);
2159 int pc_offset = code_offset();
2160 flush_debug_info(pc_offset);
2161 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
2162 if (info->exception_handlers() != NULL) {
2163 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
2164 }
2165 }
2166
2167 void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo,
2168 ciProfileData* data, Label* success, Label* failure,
2169 Label& profile_cast_success, Label& profile_cast_failure) {
2170 Register mdo = klass_RInfo;
2171 Register recv = k_RInfo;
2172 __ bind(profile_cast_success);
2173 __ mov_metadata(mdo, md->constant_encoding());
2174 __ load_klass(recv, obj);
2175 Label update_done;
2176 type_profile_helper(mdo, md, data, recv, success);
2177 __ j(*success);
2178
2179 __ bind(profile_cast_failure);
2180 __ mov_metadata(mdo, md->constant_encoding());
2181 Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2182 __ ld(t0, counter_addr);
2183 __ addi(t0, t0, -DataLayout::counter_increment);
2184 __ sd(t0, counter_addr);
2185 __ j(*failure);
2186 }
2187
2188 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1,
2189 Label* success_target, Label* failure_target) {
2190 // get instance klass (it's already uncompressed)
2191 __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2192 // perform the fast part of the checking logic
2193 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2194 // call out-of-line instance of __ check_klass_subtype_slow_path(...)
2195 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
2196 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass
2197 __ sd(k_RInfo, Address(sp, 0)); // super klass
2198 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2199 // load result to k_RInfo
2200 __ ld(k_RInfo, Address(sp, 0));
2201 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
2202 // result is a boolean
2203 __ beqz(k_RInfo, *failure_target, /* is_far */ true);
2204 }
2205
2206 void LIR_Assembler::const2reg_helper(LIR_Opr src) {
2207 switch (src->as_constant_ptr()->type()) {
2208 case T_INT:
2209 case T_ADDRESS:
2210 case T_OBJECT:
2211 case T_ARRAY:
2212 case T_METADATA:
2213 const2reg(src, FrameMap::t0_opr, lir_patch_none, NULL);
2214 break;
2215 case T_LONG:
2216 const2reg(src, FrameMap::t0_long_opr, lir_patch_none, NULL);
2217 break;
2218 case T_FLOAT:
2219 case T_DOUBLE:
2220 default:
2221 ShouldNotReachHere();
2222 }
2223 }
2224
2225 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) {
2226 switch (code) {
2227 case lir_logic_and: __ andrw(dst, left, right); break;
2228 case lir_logic_or: __ orrw (dst, left, right); break;
2229 case lir_logic_xor: __ xorrw(dst, left, right); break;
2230 default: ShouldNotReachHere();
2231 }
2232 }
2233
2234 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) {
2235 switch (code) {
2236 case lir_logic_and: __ andr(dst, left, right); break;
2237 case lir_logic_or: __ orr (dst, left, right); break;
2238 case lir_logic_xor: __ xorr(dst, left, right); break;
2239 default: ShouldNotReachHere();
2240 }
2241 }
2242
2243 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) {
2244 switch (code) {
2245 case lir_logic_and: __ andi(dst, left, right); break;
2246 case lir_logic_or: __ ori (dst, left, right); break;
2247 case lir_logic_xor: __ xori(dst, left, right); break;
2248 default: ShouldNotReachHere();
2249 }
2250 }
2251
2252 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2253 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2254 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2255 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2256 __ sd(r, Address(sp, offset_from_rsp_in_bytes));
2257 }
2258
2259 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2260 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2261 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2262 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2263 __ li(t0, c);
2264 __ sd(t0, Address(sp, offset_from_rsp_in_bytes));
2265 }
2266
2267 #undef __