1 /*
2 * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInlineKlass.hpp"
37 #include "ci/ciInstance.hpp"
38 #include "code/compiledIC.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gc_globals.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/frame.inline.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_aarch64.inline.hpp"
49
50
51 #ifndef PRODUCT
52 #define COMMENT(x) do { __ block_comment(x); } while (0)
53 #else
54 #define COMMENT(x)
55 #endif
56
57 NEEDS_CLEANUP // remove this definitions ?
58 const Register IC_Klass = rscratch2; // where the IC klass is cached
59 const Register SYNC_header = r0; // synchronization header
60 const Register SHIFT_count = r0; // where count for shift operations must be
61
62 #define __ _masm->
63
64
65 static void select_different_registers(Register preserve,
66 Register extra,
67 Register &tmp1,
68 Register &tmp2) {
69 if (tmp1 == preserve) {
70 assert_different_registers(tmp1, tmp2, extra);
71 tmp1 = extra;
72 } else if (tmp2 == preserve) {
73 assert_different_registers(tmp1, tmp2, extra);
74 tmp2 = extra;
75 }
76 assert_different_registers(preserve, tmp1, tmp2);
77 }
78
79
80
81 static void select_different_registers(Register preserve,
82 Register extra,
83 Register &tmp1,
84 Register &tmp2,
85 Register &tmp3) {
86 if (tmp1 == preserve) {
87 assert_different_registers(tmp1, tmp2, tmp3, extra);
88 tmp1 = extra;
89 } else if (tmp2 == preserve) {
90 assert_different_registers(tmp1, tmp2, tmp3, extra);
91 tmp2 = extra;
92 } else if (tmp3 == preserve) {
93 assert_different_registers(tmp1, tmp2, tmp3, extra);
94 tmp3 = extra;
95 }
96 assert_different_registers(preserve, tmp1, tmp2, tmp3);
97 }
98
99
100 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
101
102
103 LIR_Opr LIR_Assembler::receiverOpr() {
104 return FrameMap::receiver_opr;
105 }
106
107 LIR_Opr LIR_Assembler::osrBufferPointer() {
108 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
109 }
110
111 //--------------fpu register translations-----------------------
112
113
114 address LIR_Assembler::float_constant(float f) {
115 address const_addr = __ float_constant(f);
116 if (const_addr == NULL) {
117 bailout("const section overflow");
118 return __ code()->consts()->start();
119 } else {
120 return const_addr;
121 }
122 }
123
124
125 address LIR_Assembler::double_constant(double d) {
126 address const_addr = __ double_constant(d);
127 if (const_addr == NULL) {
128 bailout("const section overflow");
129 return __ code()->consts()->start();
130 } else {
131 return const_addr;
132 }
133 }
134
135 address LIR_Assembler::int_constant(jlong n) {
136 address const_addr = __ long_constant(n);
137 if (const_addr == NULL) {
138 bailout("const section overflow");
139 return __ code()->consts()->start();
140 } else {
141 return const_addr;
142 }
143 }
144
145 void LIR_Assembler::breakpoint() { Unimplemented(); }
146
147 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
148
149 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
150
151 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
152 //-------------------------------------------
153
154 static Register as_reg(LIR_Opr op) {
155 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
156 }
157
158 static jlong as_long(LIR_Opr data) {
159 jlong result;
160 switch (data->type()) {
161 case T_INT:
162 result = (data->as_jint());
163 break;
164 case T_LONG:
165 result = (data->as_jlong());
166 break;
167 default:
168 ShouldNotReachHere();
169 result = 0; // unreachable
170 }
171 return result;
172 }
173
174 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
175 Register base = addr->base()->as_pointer_register();
176 LIR_Opr opr = addr->index();
177 if (opr->is_cpu_register()) {
178 Register index;
179 if (opr->is_single_cpu())
180 index = opr->as_register();
181 else
182 index = opr->as_register_lo();
183 assert(addr->disp() == 0, "must be");
184 switch(opr->type()) {
185 case T_INT:
186 return Address(base, index, Address::sxtw(addr->scale()));
187 case T_LONG:
188 return Address(base, index, Address::lsl(addr->scale()));
189 default:
190 ShouldNotReachHere();
191 }
192 } else {
193 assert(addr->scale() == 0,
194 "expected for immediate operand, was: %d", addr->scale());
195 ptrdiff_t offset = ptrdiff_t(addr->disp());
196 // NOTE: Does not handle any 16 byte vector access.
197 const uint type_size = type2aelembytes(addr->type(), true);
198 return __ legitimize_address(Address(base, offset), type_size, tmp);
199 }
200 return Address();
201 }
202
203 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
204 ShouldNotReachHere();
205 return Address();
206 }
207
208 Address LIR_Assembler::as_Address(LIR_Address* addr) {
209 return as_Address(addr, rscratch1);
210 }
211
212 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
213 return as_Address(addr, rscratch1); // Ouch
214 // FIXME: This needs to be much more clever. See x86.
215 }
216
217 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
218 // not encodable as a base + (immediate) offset, generate an explicit address
219 // calculation to hold the address in a temporary register.
220 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) {
221 precond(size == 4 || size == 8);
222 Address addr = frame_map()->address_for_slot(index, adjust);
223 precond(addr.getMode() == Address::base_plus_offset);
224 precond(addr.base() == sp);
225 precond(addr.offset() > 0);
226 uint mask = size - 1;
227 assert((addr.offset() & mask) == 0, "scaled offsets only");
228 return __ legitimize_address(addr, size, tmp);
229 }
230
231 void LIR_Assembler::osr_entry() {
232 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
233 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
234 ValueStack* entry_state = osr_entry->state();
235 int number_of_locks = entry_state->locks_size();
236
237 // we jump here if osr happens with the interpreter
238 // state set up to continue at the beginning of the
239 // loop that triggered osr - in particular, we have
240 // the following registers setup:
241 //
242 // r2: osr buffer
243 //
244
245 // build frame
246 ciMethod* m = compilation()->method();
247 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
248
249 // OSR buffer is
250 //
251 // locals[nlocals-1..0]
252 // monitors[0..number_of_locks]
253 //
254 // locals is a direct copy of the interpreter frame so in the osr buffer
255 // so first slot in the local array is the last local from the interpreter
256 // and last slot is local[0] (receiver) from the interpreter
257 //
258 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
259 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
260 // in the interpreter frame (the method lock if a sync method)
261
262 // Initialize monitors in the compiled activation.
263 // r2: pointer to osr buffer
264 //
265 // All other registers are dead at this point and the locals will be
266 // copied into place by code emitted in the IR.
267
268 Register OSR_buf = osrBufferPointer()->as_pointer_register();
269 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
270 int monitor_offset = BytesPerWord * method()->max_locals() +
271 (2 * BytesPerWord) * (number_of_locks - 1);
272 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
273 // the OSR buffer using 2 word entries: first the lock and then
274 // the oop.
275 for (int i = 0; i < number_of_locks; i++) {
276 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
277 #ifdef ASSERT
278 // verify the interpreter's monitor has a non-null object
279 {
280 Label L;
281 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
282 __ cbnz(rscratch1, L);
283 __ stop("locked object is NULL");
284 __ bind(L);
285 }
286 #endif
287 __ ldp(r19, r20, Address(OSR_buf, slot_offset));
288 __ str(r19, frame_map()->address_for_monitor_lock(i));
289 __ str(r20, frame_map()->address_for_monitor_object(i));
290 }
291 }
292 }
293
294
295 // inline cache check; done before the frame is built.
296 int LIR_Assembler::check_icache() {
297 Register receiver = FrameMap::receiver_opr->as_register();
298 Register ic_klass = IC_Klass;
299 int start_offset = __ offset();
300 __ inline_cache_check(receiver, ic_klass);
301
302 // if icache check fails, then jump to runtime routine
303 // Note: RECEIVER must still contain the receiver!
304 Label dont;
305 __ br(Assembler::EQ, dont);
306 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
307
308 // We align the verified entry point unless the method body
309 // (including its inline cache check) will fit in a single 64-byte
310 // icache line.
311 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
312 // force alignment after the cache check.
313 __ align(CodeEntryAlignment);
314 }
315
316 __ bind(dont);
317 return start_offset;
318 }
319
320 void LIR_Assembler::clinit_barrier(ciMethod* method) {
321 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
322 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
323
324 Label L_skip_barrier;
325
326 __ mov_metadata(rscratch2, method->holder()->constant_encoding());
327 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
328 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
329 __ bind(L_skip_barrier);
330 }
331
332 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
333 if (o == NULL) {
334 __ mov(reg, zr);
335 } else {
336 __ movoop(reg, o);
337 }
338 }
339
340 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
341 address target = NULL;
342 relocInfo::relocType reloc_type = relocInfo::none;
343
344 switch (patching_id(info)) {
345 case PatchingStub::access_field_id:
346 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
347 reloc_type = relocInfo::section_word_type;
348 break;
349 case PatchingStub::load_klass_id:
350 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
351 reloc_type = relocInfo::metadata_type;
352 break;
353 case PatchingStub::load_mirror_id:
354 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
355 reloc_type = relocInfo::oop_type;
356 break;
357 case PatchingStub::load_appendix_id:
358 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
359 reloc_type = relocInfo::oop_type;
360 break;
361 default: ShouldNotReachHere();
362 }
363
364 __ far_call(RuntimeAddress(target));
365 add_call_info_here(info);
366 }
367
368 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
369 deoptimize_trap(info);
370 }
371
372
373 // This specifies the rsp decrement needed to build the frame
374 int LIR_Assembler::initial_frame_size_in_bytes() const {
375 // if rounding, must let FrameMap know!
376
377 return in_bytes(frame_map()->framesize_in_bytes());
378 }
379
380
381 int LIR_Assembler::emit_exception_handler() {
382 // generate code for exception handler
383 address handler_base = __ start_a_stub(exception_handler_size());
384 if (handler_base == NULL) {
385 // not enough space left for the handler
386 bailout("exception handler overflow");
387 return -1;
388 }
389
390 int offset = code_offset();
391
392 // the exception oop and pc are in r0, and r3
393 // no other registers need to be preserved, so invalidate them
394 __ invalidate_registers(false, true, true, false, true, true);
395
396 // check that there is really an exception
397 __ verify_not_null_oop(r0);
398
399 // search an exception handler (r0: exception oop, r3: throwing pc)
400 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
401 __ should_not_reach_here();
402 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
403 __ end_a_stub();
404
405 return offset;
406 }
407
408
409 // Emit the code to remove the frame from the stack in the exception
410 // unwind path.
411 int LIR_Assembler::emit_unwind_handler() {
412 #ifndef PRODUCT
413 if (CommentedAssembly) {
414 _masm->block_comment("Unwind handler");
415 }
416 #endif
417
418 int offset = code_offset();
419
420 // Fetch the exception from TLS and clear out exception related thread state
421 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
422 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
423 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
424
425 __ bind(_unwind_handler_entry);
426 __ verify_not_null_oop(r0);
427 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
428 __ mov(r19, r0); // Preserve the exception
429 }
430
431 // Perform needed unlocking
432 MonitorExitStub* stub = NULL;
433 if (method()->is_synchronized()) {
434 monitor_address(0, FrameMap::r0_opr);
435 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
436 if (UseHeavyMonitors) {
437 __ b(*stub->entry());
438 } else {
439 __ unlock_object(r5, r4, r0, *stub->entry());
440 }
441 __ bind(*stub->continuation());
442 }
443
444 if (compilation()->env()->dtrace_method_probes()) {
445 __ mov(c_rarg0, rthread);
446 __ mov_metadata(c_rarg1, method()->constant_encoding());
447 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
448 }
449
450 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
451 __ mov(r0, r19); // Restore the exception
452 }
453
454 // remove the activation and dispatch to the unwind handler
455 __ block_comment("remove_frame and dispatch to the unwind handler");
456 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
457 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
458
459 // Emit the slow path assembly
460 if (stub != NULL) {
461 stub->emit_code(this);
462 }
463
464 return offset;
465 }
466
467
468 int LIR_Assembler::emit_deopt_handler() {
469 // generate code for exception handler
470 address handler_base = __ start_a_stub(deopt_handler_size());
471 if (handler_base == NULL) {
472 // not enough space left for the handler
473 bailout("deopt handler overflow");
474 return -1;
475 }
476
477 int offset = code_offset();
478
479 __ adr(lr, pc());
480 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
481 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
482 __ end_a_stub();
483
484 return offset;
485 }
486
487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
488 _masm->code_section()->relocate(adr, relocInfo::poll_type);
489 int pc_offset = code_offset();
490 flush_debug_info(pc_offset);
491 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
492 if (info->exception_handlers() != NULL) {
493 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
494 }
495 }
496
497 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
498 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
499
500 ciMethod* method = compilation()->method();
501 if (InlineTypeReturnedAsFields && method->return_type()->is_inlinetype()) {
502 ciInlineKlass* vk = method->return_type()->as_inline_klass();
503 if (vk->can_be_returned_as_fields()) {
504 address unpack_handler = vk->unpack_handler();
505 assert(unpack_handler != NULL, "must be");
506 __ far_call(RuntimeAddress(unpack_handler));
507 // At this point, r0 points to the value object (for interpreter or C1 caller).
508 // The fields of the object are copied into registers (for C2 caller).
509 }
510 }
511
512 // Pop the stack before the safepoint code
513 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
514
515 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
516 __ reserved_stack_check();
517 }
518
519 code_stub->set_safepoint_offset(__ offset());
520 __ relocate(relocInfo::poll_return_type);
521 __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
522 __ ret(lr);
523 }
524
525 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
526 return (__ store_inline_type_fields_to_buf(vk, false));
527 }
528
529 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
530 guarantee(info != NULL, "Shouldn't be NULL");
531 __ get_polling_page(rscratch1, relocInfo::poll_type);
532 add_debug_info_for_branch(info); // This isn't just debug info:
533 // it's the oop map
534 __ read_polling_page(rscratch1, relocInfo::poll_type);
535 return __ offset();
536 }
537
538
539 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
540 if (from_reg == r31_sp)
541 from_reg = sp;
542 if (to_reg == r31_sp)
543 to_reg = sp;
544 __ mov(to_reg, from_reg);
545 }
546
547 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
548
549
550 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
551 assert(src->is_constant(), "should not call otherwise");
552 assert(dest->is_register(), "should not call otherwise");
553 LIR_Const* c = src->as_constant_ptr();
554
555 switch (c->type()) {
556 case T_INT: {
557 assert(patch_code == lir_patch_none, "no patching handled here");
558 __ movw(dest->as_register(), c->as_jint());
559 break;
560 }
561
562 case T_ADDRESS: {
563 assert(patch_code == lir_patch_none, "no patching handled here");
564 __ mov(dest->as_register(), c->as_jint());
565 break;
566 }
567
568 case T_LONG: {
569 assert(patch_code == lir_patch_none, "no patching handled here");
570 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
571 break;
572 }
573
574 case T_PRIMITIVE_OBJECT:
575 case T_OBJECT: {
576 if (patch_code != lir_patch_none) {
577 jobject2reg_with_patching(dest->as_register(), info);
578 } else {
579 jobject2reg(c->as_jobject(), dest->as_register());
580 }
581 break;
582 }
583
584 case T_METADATA: {
585 if (patch_code != lir_patch_none) {
586 klass2reg_with_patching(dest->as_register(), info);
587 } else {
588 __ mov_metadata(dest->as_register(), c->as_metadata());
589 }
590 break;
591 }
592
593 case T_FLOAT: {
594 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
595 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
596 } else {
597 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
598 __ ldrs(dest->as_float_reg(), Address(rscratch1));
599 }
600 break;
601 }
602
603 case T_DOUBLE: {
604 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
605 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
606 } else {
607 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
608 __ ldrd(dest->as_double_reg(), Address(rscratch1));
609 }
610 break;
611 }
612
613 default:
614 ShouldNotReachHere();
615 }
616 }
617
618 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
619 LIR_Const* c = src->as_constant_ptr();
620 switch (c->type()) {
621 case T_PRIMITIVE_OBJECT:
622 case T_OBJECT:
623 {
624 if (! c->as_jobject())
625 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
626 else {
627 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
628 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
629 }
630 }
631 break;
632 case T_ADDRESS:
633 {
634 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
635 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
636 }
637 case T_INT:
638 case T_FLOAT:
639 {
640 Register reg = zr;
641 if (c->as_jint_bits() == 0)
642 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
643 else {
644 __ movw(rscratch1, c->as_jint_bits());
645 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
646 }
647 }
648 break;
649 case T_LONG:
650 case T_DOUBLE:
651 {
652 Register reg = zr;
653 if (c->as_jlong_bits() == 0)
654 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
655 lo_word_offset_in_bytes));
656 else {
657 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
658 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
659 lo_word_offset_in_bytes));
660 }
661 }
662 break;
663 default:
664 ShouldNotReachHere();
665 }
666 }
667
668 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
669 assert(src->is_constant(), "should not call otherwise");
670 LIR_Const* c = src->as_constant_ptr();
671 LIR_Address* to_addr = dest->as_address_ptr();
672
673 void (Assembler::* insn)(Register Rt, const Address &adr);
674
675 switch (type) {
676 case T_ADDRESS:
677 assert(c->as_jint() == 0, "should be");
678 insn = &Assembler::str;
679 break;
680 case T_LONG:
681 assert(c->as_jlong() == 0, "should be");
682 insn = &Assembler::str;
683 break;
684 case T_INT:
685 assert(c->as_jint() == 0, "should be");
686 insn = &Assembler::strw;
687 break;
688 case T_PRIMITIVE_OBJECT:
689 case T_OBJECT:
690 case T_ARRAY:
691 // Non-null case is not handled on aarch64 but handled on x86
692 // FIXME: do we need to add it here?
693 assert(c->as_jobject() == 0, "should be");
694 if (UseCompressedOops && !wide) {
695 insn = &Assembler::strw;
696 } else {
697 insn = &Assembler::str;
698 }
699 break;
700 case T_CHAR:
701 case T_SHORT:
702 assert(c->as_jint() == 0, "should be");
703 insn = &Assembler::strh;
704 break;
705 case T_BOOLEAN:
706 case T_BYTE:
707 assert(c->as_jint() == 0, "should be");
708 insn = &Assembler::strb;
709 break;
710 default:
711 ShouldNotReachHere();
712 insn = &Assembler::str; // unreachable
713 }
714
715 if (info) add_debug_info_for_null_check_here(info);
716 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
717 }
718
719 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
720 assert(src->is_register(), "should not call otherwise");
721 assert(dest->is_register(), "should not call otherwise");
722
723 // move between cpu-registers
724 if (dest->is_single_cpu()) {
725 if (src->type() == T_LONG) {
726 // Can do LONG -> OBJECT
727 move_regs(src->as_register_lo(), dest->as_register());
728 return;
729 }
730 assert(src->is_single_cpu(), "must match");
731 if (src->type() == T_OBJECT || src->type() == T_PRIMITIVE_OBJECT) {
732 __ verify_oop(src->as_register());
733 }
734 move_regs(src->as_register(), dest->as_register());
735
736 } else if (dest->is_double_cpu()) {
737 if (is_reference_type(src->type())) {
738 // Surprising to me but we can see move of a long to t_object
739 __ verify_oop(src->as_register());
740 move_regs(src->as_register(), dest->as_register_lo());
741 return;
742 }
743 assert(src->is_double_cpu(), "must match");
744 Register f_lo = src->as_register_lo();
745 Register f_hi = src->as_register_hi();
746 Register t_lo = dest->as_register_lo();
747 Register t_hi = dest->as_register_hi();
748 assert(f_hi == f_lo, "must be same");
749 assert(t_hi == t_lo, "must be same");
750 move_regs(f_lo, t_lo);
751
752 } else if (dest->is_single_fpu()) {
753 __ fmovs(dest->as_float_reg(), src->as_float_reg());
754
755 } else if (dest->is_double_fpu()) {
756 __ fmovd(dest->as_double_reg(), src->as_double_reg());
757
758 } else {
759 ShouldNotReachHere();
760 }
761 }
762
763 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
764 precond(src->is_register() && dest->is_stack());
765
766 uint const c_sz32 = sizeof(uint32_t);
767 uint const c_sz64 = sizeof(uint64_t);
768
769 if (src->is_single_cpu()) {
770 int index = dest->single_stack_ix();
771 if (is_reference_type(type)) {
772 __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
773 __ verify_oop(src->as_register());
774 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
775 __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
776 } else {
777 __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1));
778 }
779
780 } else if (src->is_double_cpu()) {
781 int index = dest->double_stack_ix();
782 Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
783 __ str(src->as_register_lo(), dest_addr_LO);
784
785 } else if (src->is_single_fpu()) {
786 int index = dest->single_stack_ix();
787 __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
788
789 } else if (src->is_double_fpu()) {
790 int index = dest->double_stack_ix();
791 __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
792
793 } else {
794 ShouldNotReachHere();
795 }
796 }
797
798
799 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
800 LIR_Address* to_addr = dest->as_address_ptr();
801 PatchingStub* patch = NULL;
802 Register compressed_src = rscratch1;
803
804 if (patch_code != lir_patch_none) {
805 deoptimize_trap(info);
806 return;
807 }
808
809 if (is_reference_type(type)) {
810 __ verify_oop(src->as_register());
811
812 if (UseCompressedOops && !wide) {
813 __ encode_heap_oop(compressed_src, src->as_register());
814 } else {
815 compressed_src = src->as_register();
816 }
817 }
818
819 int null_check_here = code_offset();
820 switch (type) {
821 case T_FLOAT: {
822 __ strs(src->as_float_reg(), as_Address(to_addr));
823 break;
824 }
825
826 case T_DOUBLE: {
827 __ strd(src->as_double_reg(), as_Address(to_addr));
828 break;
829 }
830
831 case T_PRIMITIVE_OBJECT: // fall through
832 case T_ARRAY: // fall through
833 case T_OBJECT: // fall through
834 if (UseCompressedOops && !wide) {
835 __ strw(compressed_src, as_Address(to_addr, rscratch2));
836 } else {
837 __ str(compressed_src, as_Address(to_addr));
838 }
839 break;
840 case T_METADATA:
841 // We get here to store a method pointer to the stack to pass to
842 // a dtrace runtime call. This can't work on 64 bit with
843 // compressed klass ptrs: T_METADATA can be a compressed klass
844 // ptr or a 64 bit method pointer.
845 ShouldNotReachHere();
846 __ str(src->as_register(), as_Address(to_addr));
847 break;
848 case T_ADDRESS:
849 __ str(src->as_register(), as_Address(to_addr));
850 break;
851 case T_INT:
852 __ strw(src->as_register(), as_Address(to_addr));
853 break;
854
855 case T_LONG: {
856 __ str(src->as_register_lo(), as_Address_lo(to_addr));
857 break;
858 }
859
860 case T_BYTE: // fall through
861 case T_BOOLEAN: {
862 __ strb(src->as_register(), as_Address(to_addr));
863 break;
864 }
865
866 case T_CHAR: // fall through
867 case T_SHORT:
868 __ strh(src->as_register(), as_Address(to_addr));
869 break;
870
871 default:
872 ShouldNotReachHere();
873 }
874 if (info != NULL) {
875 add_debug_info_for_null_check(null_check_here, info);
876 }
877 }
878
879
880 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
881 precond(src->is_stack() && dest->is_register());
882
883 uint const c_sz32 = sizeof(uint32_t);
884 uint const c_sz64 = sizeof(uint64_t);
885
886 if (dest->is_single_cpu()) {
887 int index = src->single_stack_ix();
888 if (is_reference_type(type)) {
889 __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
890 __ verify_oop(dest->as_register());
891 } else if (type == T_METADATA || type == T_ADDRESS) {
892 __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
893 } else {
894 __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1));
895 }
896
897 } else if (dest->is_double_cpu()) {
898 int index = src->double_stack_ix();
899 Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
900 __ ldr(dest->as_register_lo(), src_addr_LO);
901
902 } else if (dest->is_single_fpu()) {
903 int index = src->single_stack_ix();
904 __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
905
906 } else if (dest->is_double_fpu()) {
907 int index = src->double_stack_ix();
908 __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
909
910 } else {
911 ShouldNotReachHere();
912 }
913 }
914
915
916 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
917 address target = NULL;
918 relocInfo::relocType reloc_type = relocInfo::none;
919
920 switch (patching_id(info)) {
921 case PatchingStub::access_field_id:
922 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
923 reloc_type = relocInfo::section_word_type;
924 break;
925 case PatchingStub::load_klass_id:
926 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
927 reloc_type = relocInfo::metadata_type;
928 break;
929 case PatchingStub::load_mirror_id:
930 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
931 reloc_type = relocInfo::oop_type;
932 break;
933 case PatchingStub::load_appendix_id:
934 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
935 reloc_type = relocInfo::oop_type;
936 break;
937 default: ShouldNotReachHere();
938 }
939
940 __ far_call(RuntimeAddress(target));
941 add_call_info_here(info);
942 }
943
944 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
945
946 LIR_Opr temp;
947 if (type == T_LONG || type == T_DOUBLE)
948 temp = FrameMap::rscratch1_long_opr;
949 else
950 temp = FrameMap::rscratch1_opr;
951
952 stack2reg(src, temp, src->type());
953 reg2stack(temp, dest, dest->type(), false);
954 }
955
956
957 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
958 LIR_Address* addr = src->as_address_ptr();
959 LIR_Address* from_addr = src->as_address_ptr();
960
961 if (addr->base()->type() == T_OBJECT || addr->base()->type() == T_PRIMITIVE_OBJECT) {
962 __ verify_oop(addr->base()->as_pointer_register());
963 }
964
965 if (patch_code != lir_patch_none) {
966 deoptimize_trap(info);
967 return;
968 }
969
970 if (info != NULL) {
971 add_debug_info_for_null_check_here(info);
972 }
973 int null_check_here = code_offset();
974 switch (type) {
975 case T_FLOAT: {
976 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
977 break;
978 }
979
980 case T_DOUBLE: {
981 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
982 break;
983 }
984
985 case T_PRIMITIVE_OBJECT: // fall through
986 case T_ARRAY: // fall through
987 case T_OBJECT: // fall through
988 if (UseCompressedOops && !wide) {
989 __ ldrw(dest->as_register(), as_Address(from_addr));
990 } else {
991 __ ldr(dest->as_register(), as_Address(from_addr));
992 }
993 break;
994 case T_METADATA:
995 // We get here to store a method pointer to the stack to pass to
996 // a dtrace runtime call. This can't work on 64 bit with
997 // compressed klass ptrs: T_METADATA can be a compressed klass
998 // ptr or a 64 bit method pointer.
999 ShouldNotReachHere();
1000 __ ldr(dest->as_register(), as_Address(from_addr));
1001 break;
1002 case T_ADDRESS:
1003 __ ldr(dest->as_register(), as_Address(from_addr));
1004 break;
1005 case T_INT:
1006 __ ldrw(dest->as_register(), as_Address(from_addr));
1007 break;
1008
1009 case T_LONG: {
1010 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1011 break;
1012 }
1013
1014 case T_BYTE:
1015 __ ldrsb(dest->as_register(), as_Address(from_addr));
1016 break;
1017 case T_BOOLEAN: {
1018 __ ldrb(dest->as_register(), as_Address(from_addr));
1019 break;
1020 }
1021
1022 case T_CHAR:
1023 __ ldrh(dest->as_register(), as_Address(from_addr));
1024 break;
1025 case T_SHORT:
1026 __ ldrsh(dest->as_register(), as_Address(from_addr));
1027 break;
1028
1029 default:
1030 ShouldNotReachHere();
1031 }
1032
1033 if (is_reference_type(type)) {
1034 if (UseCompressedOops && !wide) {
1035 __ decode_heap_oop(dest->as_register());
1036 }
1037
1038 if (!UseZGC) {
1039 // Load barrier has not yet been applied, so ZGC can't verify the oop here
1040 __ verify_oop(dest->as_register());
1041 }
1042 }
1043 }
1044
1045 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1046 assert(dst->is_cpu_register(), "must be");
1047 assert(dst->type() == src->type(), "must be");
1048
1049 if (src->is_cpu_register()) {
1050 reg2reg(src, dst);
1051 } else if (src->is_stack()) {
1052 stack2reg(src, dst, dst->type());
1053 } else if (src->is_constant()) {
1054 const2reg(src, dst, lir_patch_none, NULL);
1055 } else {
1056 ShouldNotReachHere();
1057 }
1058 }
1059
1060 int LIR_Assembler::array_element_size(BasicType type) const {
1061 int elem_size = type2aelembytes(type);
1062 return exact_log2(elem_size);
1063 }
1064
1065
1066 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1067 switch (op->code()) {
1068 case lir_idiv:
1069 case lir_irem:
1070 arithmetic_idiv(op->code(),
1071 op->in_opr1(),
1072 op->in_opr2(),
1073 op->in_opr3(),
1074 op->result_opr(),
1075 op->info());
1076 break;
1077 case lir_fmad:
1078 __ fmaddd(op->result_opr()->as_double_reg(),
1079 op->in_opr1()->as_double_reg(),
1080 op->in_opr2()->as_double_reg(),
1081 op->in_opr3()->as_double_reg());
1082 break;
1083 case lir_fmaf:
1084 __ fmadds(op->result_opr()->as_float_reg(),
1085 op->in_opr1()->as_float_reg(),
1086 op->in_opr2()->as_float_reg(),
1087 op->in_opr3()->as_float_reg());
1088 break;
1089 default: ShouldNotReachHere(); break;
1090 }
1091 }
1092
1093 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1094 #ifdef ASSERT
1095 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1096 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1097 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1098 #endif
1099
1100 if (op->cond() == lir_cond_always) {
1101 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1102 __ b(*(op->label()));
1103 } else {
1104 Assembler::Condition acond;
1105 if (op->code() == lir_cond_float_branch) {
1106 bool is_unordered = (op->ublock() == op->block());
1107 // Assembler::EQ does not permit unordered branches, so we add
1108 // another branch here. Likewise, Assembler::NE does not permit
1109 // ordered branches.
1110 if ((is_unordered && op->cond() == lir_cond_equal)
1111 || (!is_unordered && op->cond() == lir_cond_notEqual))
1112 __ br(Assembler::VS, *(op->ublock()->label()));
1113 switch(op->cond()) {
1114 case lir_cond_equal: acond = Assembler::EQ; break;
1115 case lir_cond_notEqual: acond = Assembler::NE; break;
1116 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1117 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1118 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1119 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1120 default: ShouldNotReachHere();
1121 acond = Assembler::EQ; // unreachable
1122 }
1123 } else {
1124 switch (op->cond()) {
1125 case lir_cond_equal: acond = Assembler::EQ; break;
1126 case lir_cond_notEqual: acond = Assembler::NE; break;
1127 case lir_cond_less: acond = Assembler::LT; break;
1128 case lir_cond_lessEqual: acond = Assembler::LE; break;
1129 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1130 case lir_cond_greater: acond = Assembler::GT; break;
1131 case lir_cond_belowEqual: acond = Assembler::LS; break;
1132 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1133 default: ShouldNotReachHere();
1134 acond = Assembler::EQ; // unreachable
1135 }
1136 }
1137 __ br(acond,*(op->label()));
1138 }
1139 }
1140
1141
1142
1143 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1144 LIR_Opr src = op->in_opr();
1145 LIR_Opr dest = op->result_opr();
1146
1147 switch (op->bytecode()) {
1148 case Bytecodes::_i2f:
1149 {
1150 __ scvtfws(dest->as_float_reg(), src->as_register());
1151 break;
1152 }
1153 case Bytecodes::_i2d:
1154 {
1155 __ scvtfwd(dest->as_double_reg(), src->as_register());
1156 break;
1157 }
1158 case Bytecodes::_l2d:
1159 {
1160 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1161 break;
1162 }
1163 case Bytecodes::_l2f:
1164 {
1165 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1166 break;
1167 }
1168 case Bytecodes::_f2d:
1169 {
1170 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1171 break;
1172 }
1173 case Bytecodes::_d2f:
1174 {
1175 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1176 break;
1177 }
1178 case Bytecodes::_i2c:
1179 {
1180 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1181 break;
1182 }
1183 case Bytecodes::_i2l:
1184 {
1185 __ sxtw(dest->as_register_lo(), src->as_register());
1186 break;
1187 }
1188 case Bytecodes::_i2s:
1189 {
1190 __ sxth(dest->as_register(), src->as_register());
1191 break;
1192 }
1193 case Bytecodes::_i2b:
1194 {
1195 __ sxtb(dest->as_register(), src->as_register());
1196 break;
1197 }
1198 case Bytecodes::_l2i:
1199 {
1200 _masm->block_comment("FIXME: This could be a no-op");
1201 __ uxtw(dest->as_register(), src->as_register_lo());
1202 break;
1203 }
1204 case Bytecodes::_d2l:
1205 {
1206 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1207 break;
1208 }
1209 case Bytecodes::_f2i:
1210 {
1211 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1212 break;
1213 }
1214 case Bytecodes::_f2l:
1215 {
1216 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1217 break;
1218 }
1219 case Bytecodes::_d2i:
1220 {
1221 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1222 break;
1223 }
1224 default: ShouldNotReachHere();
1225 }
1226 }
1227
1228 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1229 if (op->init_check()) {
1230 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1231 InstanceKlass::init_state_offset()));
1232 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1233 add_debug_info_for_null_check_here(op->stub()->info());
1234 __ br(Assembler::NE, *op->stub()->entry());
1235 }
1236 __ allocate_object(op->obj()->as_register(),
1237 op->tmp1()->as_register(),
1238 op->tmp2()->as_register(),
1239 op->header_size(),
1240 op->object_size(),
1241 op->klass()->as_register(),
1242 *op->stub()->entry());
1243 __ bind(*op->stub()->continuation());
1244 }
1245
1246 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1247 Register len = op->len()->as_register();
1248 __ uxtw(len, len);
1249
1250 if (UseSlowPath || op->type() == T_PRIMITIVE_OBJECT ||
1251 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1252 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1253 __ b(*op->stub()->entry());
1254 } else {
1255 Register tmp1 = op->tmp1()->as_register();
1256 Register tmp2 = op->tmp2()->as_register();
1257 Register tmp3 = op->tmp3()->as_register();
1258 if (len == tmp1) {
1259 tmp1 = tmp3;
1260 } else if (len == tmp2) {
1261 tmp2 = tmp3;
1262 } else if (len == tmp3) {
1263 // everything is ok
1264 } else {
1265 __ mov(tmp3, len);
1266 }
1267 __ allocate_array(op->obj()->as_register(),
1268 len,
1269 tmp1,
1270 tmp2,
1271 arrayOopDesc::header_size(op->type()),
1272 array_element_size(op->type()),
1273 op->klass()->as_register(),
1274 *op->stub()->entry());
1275 }
1276 __ bind(*op->stub()->continuation());
1277 }
1278
1279 void LIR_Assembler::type_profile_helper(Register mdo,
1280 ciMethodData *md, ciProfileData *data,
1281 Register recv, Label* update_done) {
1282 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1283 Label next_test;
1284 // See if the receiver is receiver[n].
1285 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1286 __ ldr(rscratch1, Address(rscratch2));
1287 __ cmp(recv, rscratch1);
1288 __ br(Assembler::NE, next_test);
1289 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1290 __ addptr(data_addr, DataLayout::counter_increment);
1291 __ b(*update_done);
1292 __ bind(next_test);
1293 }
1294
1295 // Didn't find receiver; find next empty slot and fill it in
1296 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1297 Label next_test;
1298 __ lea(rscratch2,
1299 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1300 Address recv_addr(rscratch2);
1301 __ ldr(rscratch1, recv_addr);
1302 __ cbnz(rscratch1, next_test);
1303 __ str(recv, recv_addr);
1304 __ mov(rscratch1, DataLayout::counter_increment);
1305 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1306 __ str(rscratch1, Address(rscratch2));
1307 __ b(*update_done);
1308 __ bind(next_test);
1309 }
1310 }
1311
1312 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1313 // we always need a stub for the failure case.
1314 CodeStub* stub = op->stub();
1315 Register obj = op->object()->as_register();
1316 Register k_RInfo = op->tmp1()->as_register();
1317 Register klass_RInfo = op->tmp2()->as_register();
1318 Register dst = op->result_opr()->as_register();
1319 ciKlass* k = op->klass();
1320 Register Rtmp1 = noreg;
1321
1322 // check if it needs to be profiled
1323 ciMethodData* md;
1324 ciProfileData* data;
1325
1326 const bool should_profile = op->should_profile();
1327
1328 if (should_profile) {
1329 ciMethod* method = op->profiled_method();
1330 assert(method != NULL, "Should have method");
1331 int bci = op->profiled_bci();
1332 md = method->method_data_or_null();
1333 assert(md != NULL, "Sanity");
1334 data = md->bci_to_data(bci);
1335 assert(data != NULL, "need data for type check");
1336 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1337 }
1338 Label profile_cast_success, profile_cast_failure;
1339 Label *success_target = should_profile ? &profile_cast_success : success;
1340 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1341
1342 if (obj == k_RInfo) {
1343 k_RInfo = dst;
1344 } else if (obj == klass_RInfo) {
1345 klass_RInfo = dst;
1346 }
1347 if (k->is_loaded() && !UseCompressedClassPointers) {
1348 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1349 } else {
1350 Rtmp1 = op->tmp3()->as_register();
1351 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1352 }
1353
1354 assert_different_registers(obj, k_RInfo, klass_RInfo);
1355
1356 if (op->need_null_check()) {
1357 if (should_profile) {
1358 Label not_null;
1359 __ cbnz(obj, not_null);
1360 // Object is null; update MDO and exit
1361 Register mdo = klass_RInfo;
1362 __ mov_metadata(mdo, md->constant_encoding());
1363 Address data_addr
1364 = __ form_address(rscratch2, mdo,
1365 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1366 0);
1367 __ ldrb(rscratch1, data_addr);
1368 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1369 __ strb(rscratch1, data_addr);
1370 __ b(*obj_is_null);
1371 __ bind(not_null);
1372 } else {
1373 __ cbz(obj, *obj_is_null);
1374 }
1375 }
1376
1377 if (!k->is_loaded()) {
1378 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1379 } else {
1380 __ mov_metadata(k_RInfo, k->constant_encoding());
1381 }
1382 __ verify_oop(obj);
1383
1384 if (op->fast_check()) {
1385 // get object class
1386 // not a safepoint as obj null check happens earlier
1387 __ load_klass(rscratch1, obj);
1388 __ cmp( rscratch1, k_RInfo);
1389
1390 __ br(Assembler::NE, *failure_target);
1391 // successful cast, fall through to profile or jump
1392 } else {
1393 // get object class
1394 // not a safepoint as obj null check happens earlier
1395 __ load_klass(klass_RInfo, obj);
1396 if (k->is_loaded()) {
1397 // See if we get an immediate positive hit
1398 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1399 __ cmp(k_RInfo, rscratch1);
1400 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1401 __ br(Assembler::NE, *failure_target);
1402 // successful cast, fall through to profile or jump
1403 } else {
1404 // See if we get an immediate positive hit
1405 __ br(Assembler::EQ, *success_target);
1406 // check for self
1407 __ cmp(klass_RInfo, k_RInfo);
1408 __ br(Assembler::EQ, *success_target);
1409
1410 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1411 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1412 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1413 // result is a boolean
1414 __ cbzw(klass_RInfo, *failure_target);
1415 // successful cast, fall through to profile or jump
1416 }
1417 } else {
1418 // perform the fast part of the checking logic
1419 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1420 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1421 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1422 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1423 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1424 // result is a boolean
1425 __ cbz(k_RInfo, *failure_target);
1426 // successful cast, fall through to profile or jump
1427 }
1428 }
1429 if (should_profile) {
1430 Register mdo = klass_RInfo, recv = k_RInfo;
1431 __ bind(profile_cast_success);
1432 __ mov_metadata(mdo, md->constant_encoding());
1433 __ load_klass(recv, obj);
1434 Label update_done;
1435 type_profile_helper(mdo, md, data, recv, success);
1436 __ b(*success);
1437
1438 __ bind(profile_cast_failure);
1439 __ mov_metadata(mdo, md->constant_encoding());
1440 Address counter_addr
1441 = __ form_address(rscratch2, mdo,
1442 md->byte_offset_of_slot(data, CounterData::count_offset()),
1443 0);
1444 __ ldr(rscratch1, counter_addr);
1445 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1446 __ str(rscratch1, counter_addr);
1447 __ b(*failure);
1448 }
1449 __ b(*success);
1450 }
1451
1452
1453 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1454 const bool should_profile = op->should_profile();
1455
1456 LIR_Code code = op->code();
1457 if (code == lir_store_check) {
1458 Register value = op->object()->as_register();
1459 Register array = op->array()->as_register();
1460 Register k_RInfo = op->tmp1()->as_register();
1461 Register klass_RInfo = op->tmp2()->as_register();
1462 Register Rtmp1 = op->tmp3()->as_register();
1463
1464 CodeStub* stub = op->stub();
1465
1466 // check if it needs to be profiled
1467 ciMethodData* md;
1468 ciProfileData* data;
1469
1470 if (should_profile) {
1471 ciMethod* method = op->profiled_method();
1472 assert(method != NULL, "Should have method");
1473 int bci = op->profiled_bci();
1474 md = method->method_data_or_null();
1475 assert(md != NULL, "Sanity");
1476 data = md->bci_to_data(bci);
1477 assert(data != NULL, "need data for type check");
1478 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1479 }
1480 Label profile_cast_success, profile_cast_failure, done;
1481 Label *success_target = should_profile ? &profile_cast_success : &done;
1482 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1483
1484 if (should_profile) {
1485 Label not_null;
1486 __ cbnz(value, not_null);
1487 // Object is null; update MDO and exit
1488 Register mdo = klass_RInfo;
1489 __ mov_metadata(mdo, md->constant_encoding());
1490 Address data_addr
1491 = __ form_address(rscratch2, mdo,
1492 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1493 0);
1494 __ ldrb(rscratch1, data_addr);
1495 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1496 __ strb(rscratch1, data_addr);
1497 __ b(done);
1498 __ bind(not_null);
1499 } else {
1500 __ cbz(value, done);
1501 }
1502
1503 add_debug_info_for_null_check_here(op->info_for_exception());
1504 __ load_klass(k_RInfo, array);
1505 __ load_klass(klass_RInfo, value);
1506
1507 // get instance klass (it's already uncompressed)
1508 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1509 // perform the fast part of the checking logic
1510 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1511 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1512 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1513 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1514 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1515 // result is a boolean
1516 __ cbzw(k_RInfo, *failure_target);
1517 // fall through to the success case
1518
1519 if (should_profile) {
1520 Register mdo = klass_RInfo, recv = k_RInfo;
1521 __ bind(profile_cast_success);
1522 __ mov_metadata(mdo, md->constant_encoding());
1523 __ load_klass(recv, value);
1524 Label update_done;
1525 type_profile_helper(mdo, md, data, recv, &done);
1526 __ b(done);
1527
1528 __ bind(profile_cast_failure);
1529 __ mov_metadata(mdo, md->constant_encoding());
1530 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1531 __ lea(rscratch2, counter_addr);
1532 __ ldr(rscratch1, Address(rscratch2));
1533 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1534 __ str(rscratch1, Address(rscratch2));
1535 __ b(*stub->entry());
1536 }
1537
1538 __ bind(done);
1539 } else if (code == lir_checkcast) {
1540 Register obj = op->object()->as_register();
1541 Register dst = op->result_opr()->as_register();
1542 Label success;
1543 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1544 __ bind(success);
1545 if (dst != obj) {
1546 __ mov(dst, obj);
1547 }
1548 } else if (code == lir_instanceof) {
1549 Register obj = op->object()->as_register();
1550 Register dst = op->result_opr()->as_register();
1551 Label success, failure, done;
1552 emit_typecheck_helper(op, &success, &failure, &failure);
1553 __ bind(failure);
1554 __ mov(dst, zr);
1555 __ b(done);
1556 __ bind(success);
1557 __ mov(dst, 1);
1558 __ bind(done);
1559 } else {
1560 ShouldNotReachHere();
1561 }
1562 }
1563
1564 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1565 // We are loading/storing from/to an array that *may* be flattened (the
1566 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1567 // If this array is flattened, take the slow path.
1568
1569 Register klass = op->tmp()->as_register();
1570 if (UseArrayMarkWordCheck) {
1571 __ test_flattened_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1572 } else {
1573 __ load_klass(klass, op->array()->as_register());
1574 __ ldrw(klass, Address(klass, Klass::layout_helper_offset()));
1575 __ tst(klass, Klass::_lh_array_tag_flat_value_bit_inplace);
1576 __ br(Assembler::NE, *op->stub()->entry());
1577 }
1578 if (!op->value()->is_illegal()) {
1579 // The array is not flattened, but it might be null-free. If we are storing
1580 // a null into a null-free array, take the slow path (which will throw NPE).
1581 Label skip;
1582 __ cbnz(op->value()->as_register(), skip);
1583 if (UseArrayMarkWordCheck) {
1584 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1585 } else {
1586 __ tst(klass, Klass::_lh_null_free_array_bit_inplace);
1587 __ br(Assembler::NE, *op->stub()->entry());
1588 }
1589 __ bind(skip);
1590 }
1591 }
1592
1593 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1594 // We are storing into an array that *may* be null-free (the declared type is
1595 // Object[], abstract[], interface[] or VT.ref[]).
1596 if (UseArrayMarkWordCheck) {
1597 Label test_mark_word;
1598 Register tmp = op->tmp()->as_register();
1599 __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1600 __ tst(tmp, markWord::unlocked_value);
1601 __ br(Assembler::NE, test_mark_word);
1602 __ load_prototype_header(tmp, op->array()->as_register());
1603 __ bind(test_mark_word);
1604 __ tst(tmp, markWord::null_free_array_bit_in_place);
1605 } else {
1606 Register klass = op->tmp()->as_register();
1607 __ load_klass(klass, op->array()->as_register());
1608 __ ldr(klass, Address(klass, Klass::layout_helper_offset()));
1609 __ tst(klass, Klass::_lh_null_free_array_bit_inplace);
1610 }
1611 }
1612
1613 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1614 Label L_oops_equal;
1615 Label L_oops_not_equal;
1616 Label L_end;
1617
1618 Register left = op->left()->as_register();
1619 Register right = op->right()->as_register();
1620
1621 __ cmp(left, right);
1622 __ br(Assembler::EQ, L_oops_equal);
1623
1624 // (1) Null check -- if one of the operands is null, the other must not be null (because
1625 // the two references are not equal), so they are not substitutable,
1626 // FIXME: do null check only if the operand is nullable
1627 {
1628 __ cbz(left, L_oops_not_equal);
1629 __ cbz(right, L_oops_not_equal);
1630 }
1631
1632 ciKlass* left_klass = op->left_klass();
1633 ciKlass* right_klass = op->right_klass();
1634
1635 // (2) Inline type check -- if either of the operands is not a inline type,
1636 // they are not substitutable. We do this only if we are not sure that the
1637 // operands are inline type
1638 if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
1639 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1640 Register tmp1 = op->tmp1()->as_register();
1641 __ mov(tmp1, markWord::inline_type_pattern);
1642 __ ldr(rscratch1, Address(left, oopDesc::mark_offset_in_bytes()));
1643 __ andr(tmp1, tmp1, rscratch1);
1644 __ ldr(rscratch1, Address(right, oopDesc::mark_offset_in_bytes()));
1645 __ andr(tmp1, tmp1, rscratch1);
1646 __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1647 __ br(Assembler::NE, L_oops_not_equal);
1648 }
1649
1650 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1651 if (left_klass != NULL && left_klass->is_inlinetype() && left_klass == right_klass) {
1652 // No need to load klass -- the operands are statically known to be the same inline klass.
1653 __ b(*op->stub()->entry());
1654 } else {
1655 Register left_klass_op = op->left_klass_op()->as_register();
1656 Register right_klass_op = op->right_klass_op()->as_register();
1657
1658 if (UseCompressedClassPointers) {
1659 __ ldrw(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1660 __ ldrw(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1661 __ cmpw(left_klass_op, right_klass_op);
1662 } else {
1663 __ ldr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1664 __ ldr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1665 __ cmp(left_klass_op, right_klass_op);
1666 }
1667
1668 __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1669 // fall through to L_oops_not_equal
1670 }
1671
1672 __ bind(L_oops_not_equal);
1673 move(op->not_equal_result(), op->result_opr());
1674 __ b(L_end);
1675
1676 __ bind(L_oops_equal);
1677 move(op->equal_result(), op->result_opr());
1678 __ b(L_end);
1679
1680 // We've returned from the stub. R0 contains 0x0 IFF the two
1681 // operands are not substitutable. (Don't compare against 0x1 in case the
1682 // C compiler is naughty)
1683 __ bind(*op->stub()->continuation());
1684 __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1685 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1686 // fall-through
1687 __ bind(L_end);
1688 }
1689
1690
1691 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1692 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1693 __ cset(rscratch1, Assembler::NE);
1694 __ membar(__ AnyAny);
1695 }
1696
1697 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1698 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1699 __ cset(rscratch1, Assembler::NE);
1700 __ membar(__ AnyAny);
1701 }
1702
1703
1704 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1705 assert(VM_Version::supports_cx8(), "wrong machine");
1706 Register addr;
1707 if (op->addr()->is_register()) {
1708 addr = as_reg(op->addr());
1709 } else {
1710 assert(op->addr()->is_address(), "what else?");
1711 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1712 assert(addr_ptr->disp() == 0, "need 0 disp");
1713 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1714 addr = as_reg(addr_ptr->base());
1715 }
1716 Register newval = as_reg(op->new_value());
1717 Register cmpval = as_reg(op->cmp_value());
1718
1719 if (op->code() == lir_cas_obj) {
1720 if (UseCompressedOops) {
1721 Register t1 = op->tmp1()->as_register();
1722 assert(op->tmp1()->is_valid(), "must be");
1723 __ encode_heap_oop(t1, cmpval);
1724 cmpval = t1;
1725 __ encode_heap_oop(rscratch2, newval);
1726 newval = rscratch2;
1727 casw(addr, newval, cmpval);
1728 } else {
1729 casl(addr, newval, cmpval);
1730 }
1731 } else if (op->code() == lir_cas_int) {
1732 casw(addr, newval, cmpval);
1733 } else {
1734 casl(addr, newval, cmpval);
1735 }
1736 }
1737
1738
1739 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1740 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1741 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on aarch64");
1742
1743 Assembler::Condition acond, ncond;
1744 switch (condition) {
1745 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1746 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1747 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1748 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1749 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1750 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1751 case lir_cond_belowEqual:
1752 case lir_cond_aboveEqual:
1753 default: ShouldNotReachHere();
1754 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1755 }
1756
1757 assert(result->is_single_cpu() || result->is_double_cpu(),
1758 "expect single register for result");
1759 if (opr1->is_constant() && opr2->is_constant()
1760 && opr1->type() == T_INT && opr2->type() == T_INT) {
1761 jint val1 = opr1->as_jint();
1762 jint val2 = opr2->as_jint();
1763 if (val1 == 0 && val2 == 1) {
1764 __ cset(result->as_register(), ncond);
1765 return;
1766 } else if (val1 == 1 && val2 == 0) {
1767 __ cset(result->as_register(), acond);
1768 return;
1769 }
1770 }
1771
1772 if (opr1->is_constant() && opr2->is_constant()
1773 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1774 jlong val1 = opr1->as_jlong();
1775 jlong val2 = opr2->as_jlong();
1776 if (val1 == 0 && val2 == 1) {
1777 __ cset(result->as_register_lo(), ncond);
1778 return;
1779 } else if (val1 == 1 && val2 == 0) {
1780 __ cset(result->as_register_lo(), acond);
1781 return;
1782 }
1783 }
1784
1785 if (opr1->is_stack()) {
1786 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1787 opr1 = FrameMap::rscratch1_opr;
1788 } else if (opr1->is_constant()) {
1789 LIR_Opr tmp
1790 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1791 const2reg(opr1, tmp, lir_patch_none, NULL);
1792 opr1 = tmp;
1793 }
1794
1795 if (opr2->is_stack()) {
1796 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1797 opr2 = FrameMap::rscratch2_opr;
1798 } else if (opr2->is_constant()) {
1799 LIR_Opr tmp
1800 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1801 const2reg(opr2, tmp, lir_patch_none, NULL);
1802 opr2 = tmp;
1803 }
1804
1805 if (result->type() == T_LONG)
1806 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1807 else
1808 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1809 }
1810
1811 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1812 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1813
1814 if (left->is_single_cpu()) {
1815 Register lreg = left->as_register();
1816 Register dreg = as_reg(dest);
1817
1818 if (right->is_single_cpu()) {
1819 // cpu register - cpu register
1820
1821 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1822 "should be");
1823 Register rreg = right->as_register();
1824 switch (code) {
1825 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1826 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1827 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1828 default: ShouldNotReachHere();
1829 }
1830
1831 } else if (right->is_double_cpu()) {
1832 Register rreg = right->as_register_lo();
1833 // single_cpu + double_cpu: can happen with obj+long
1834 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1835 switch (code) {
1836 case lir_add: __ add(dreg, lreg, rreg); break;
1837 case lir_sub: __ sub(dreg, lreg, rreg); break;
1838 default: ShouldNotReachHere();
1839 }
1840 } else if (right->is_constant()) {
1841 // cpu register - constant
1842 jlong c;
1843
1844 // FIXME. This is fugly: we really need to factor all this logic.
1845 switch(right->type()) {
1846 case T_LONG:
1847 c = right->as_constant_ptr()->as_jlong();
1848 break;
1849 case T_INT:
1850 case T_ADDRESS:
1851 c = right->as_constant_ptr()->as_jint();
1852 break;
1853 default:
1854 ShouldNotReachHere();
1855 c = 0; // unreachable
1856 break;
1857 }
1858
1859 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1860 if (c == 0 && dreg == lreg) {
1861 COMMENT("effective nop elided");
1862 return;
1863 }
1864 switch(left->type()) {
1865 case T_INT:
1866 switch (code) {
1867 case lir_add: __ addw(dreg, lreg, c); break;
1868 case lir_sub: __ subw(dreg, lreg, c); break;
1869 default: ShouldNotReachHere();
1870 }
1871 break;
1872 case T_OBJECT:
1873 case T_ADDRESS:
1874 switch (code) {
1875 case lir_add: __ add(dreg, lreg, c); break;
1876 case lir_sub: __ sub(dreg, lreg, c); break;
1877 default: ShouldNotReachHere();
1878 }
1879 break;
1880 default:
1881 ShouldNotReachHere();
1882 }
1883 } else {
1884 ShouldNotReachHere();
1885 }
1886
1887 } else if (left->is_double_cpu()) {
1888 Register lreg_lo = left->as_register_lo();
1889
1890 if (right->is_double_cpu()) {
1891 // cpu register - cpu register
1892 Register rreg_lo = right->as_register_lo();
1893 switch (code) {
1894 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1895 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1896 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1897 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1898 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1899 default:
1900 ShouldNotReachHere();
1901 }
1902
1903 } else if (right->is_constant()) {
1904 jlong c = right->as_constant_ptr()->as_jlong();
1905 Register dreg = as_reg(dest);
1906 switch (code) {
1907 case lir_add:
1908 case lir_sub:
1909 if (c == 0 && dreg == lreg_lo) {
1910 COMMENT("effective nop elided");
1911 return;
1912 }
1913 code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1914 break;
1915 case lir_div:
1916 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1917 if (c == 1) {
1918 // move lreg_lo to dreg if divisor is 1
1919 __ mov(dreg, lreg_lo);
1920 } else {
1921 unsigned int shift = log2i_exact(c);
1922 // use rscratch1 as intermediate result register
1923 __ asr(rscratch1, lreg_lo, 63);
1924 __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1925 __ asr(dreg, rscratch1, shift);
1926 }
1927 break;
1928 case lir_rem:
1929 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1930 if (c == 1) {
1931 // move 0 to dreg if divisor is 1
1932 __ mov(dreg, zr);
1933 } else {
1934 // use rscratch1 as intermediate result register
1935 __ negs(rscratch1, lreg_lo);
1936 __ andr(dreg, lreg_lo, c - 1);
1937 __ andr(rscratch1, rscratch1, c - 1);
1938 __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1939 }
1940 break;
1941 default:
1942 ShouldNotReachHere();
1943 }
1944 } else {
1945 ShouldNotReachHere();
1946 }
1947 } else if (left->is_single_fpu()) {
1948 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1949 switch (code) {
1950 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1951 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1952 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1953 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1954 default:
1955 ShouldNotReachHere();
1956 }
1957 } else if (left->is_double_fpu()) {
1958 if (right->is_double_fpu()) {
1959 // fpu register - fpu register
1960 switch (code) {
1961 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1962 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1963 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1964 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1965 default:
1966 ShouldNotReachHere();
1967 }
1968 } else {
1969 if (right->is_constant()) {
1970 ShouldNotReachHere();
1971 }
1972 ShouldNotReachHere();
1973 }
1974 } else if (left->is_single_stack() || left->is_address()) {
1975 assert(left == dest, "left and dest must be equal");
1976 ShouldNotReachHere();
1977 } else {
1978 ShouldNotReachHere();
1979 }
1980 }
1981
1982 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1983
1984
1985 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1986 switch(code) {
1987 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1988 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1989 default : ShouldNotReachHere();
1990 }
1991 }
1992
1993 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1994
1995 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1996 Register Rleft = left->is_single_cpu() ? left->as_register() :
1997 left->as_register_lo();
1998 if (dst->is_single_cpu()) {
1999 Register Rdst = dst->as_register();
2000 if (right->is_constant()) {
2001 switch (code) {
2002 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
2003 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
2004 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
2005 default: ShouldNotReachHere(); break;
2006 }
2007 } else {
2008 Register Rright = right->is_single_cpu() ? right->as_register() :
2009 right->as_register_lo();
2010 switch (code) {
2011 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
2012 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
2013 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
2014 default: ShouldNotReachHere(); break;
2015 }
2016 }
2017 } else {
2018 Register Rdst = dst->as_register_lo();
2019 if (right->is_constant()) {
2020 switch (code) {
2021 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
2022 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
2023 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
2024 default: ShouldNotReachHere(); break;
2025 }
2026 } else {
2027 Register Rright = right->is_single_cpu() ? right->as_register() :
2028 right->as_register_lo();
2029 switch (code) {
2030 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
2031 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
2032 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
2033 default: ShouldNotReachHere(); break;
2034 }
2035 }
2036 }
2037 }
2038
2039
2040
2041 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
2042
2043 // opcode check
2044 assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
2045 bool is_irem = (code == lir_irem);
2046
2047 // operand check
2048 assert(left->is_single_cpu(), "left must be register");
2049 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
2050 assert(result->is_single_cpu(), "result must be register");
2051 Register lreg = left->as_register();
2052 Register dreg = result->as_register();
2053
2054 // power-of-2 constant check and codegen
2055 if (right->is_constant()) {
2056 int c = right->as_constant_ptr()->as_jint();
2057 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
2058 if (is_irem) {
2059 if (c == 1) {
2060 // move 0 to dreg if divisor is 1
2061 __ movw(dreg, zr);
2062 } else {
2063 // use rscratch1 as intermediate result register
2064 __ negsw(rscratch1, lreg);
2065 __ andw(dreg, lreg, c - 1);
2066 __ andw(rscratch1, rscratch1, c - 1);
2067 __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
2068 }
2069 } else {
2070 if (c == 1) {
2071 // move lreg to dreg if divisor is 1
2072 __ movw(dreg, lreg);
2073 } else {
2074 unsigned int shift = exact_log2(c);
2075 // use rscratch1 as intermediate result register
2076 __ asrw(rscratch1, lreg, 31);
2077 __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
2078 __ asrw(dreg, rscratch1, shift);
2079 }
2080 }
2081 } else {
2082 Register rreg = right->as_register();
2083 __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
2084 }
2085 }
2086
2087
2088 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2089 if (opr1->is_constant() && opr2->is_single_cpu()) {
2090 // tableswitch
2091 Register reg = as_reg(opr2);
2092 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
2093 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
2094 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
2095 Register reg1 = as_reg(opr1);
2096 if (opr2->is_single_cpu()) {
2097 // cpu register - cpu register
2098 Register reg2 = opr2->as_register();
2099 if (is_reference_type(opr1->type())) {
2100 __ cmpoop(reg1, reg2);
2101 } else {
2102 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2103 __ cmpw(reg1, reg2);
2104 }
2105 return;
2106 }
2107 if (opr2->is_double_cpu()) {
2108 // cpu register - cpu register
2109 Register reg2 = opr2->as_register_lo();
2110 __ cmp(reg1, reg2);
2111 return;
2112 }
2113
2114 if (opr2->is_constant()) {
2115 bool is_32bit = false; // width of register operand
2116 jlong imm;
2117
2118 switch(opr2->type()) {
2119 case T_INT:
2120 imm = opr2->as_constant_ptr()->as_jint();
2121 is_32bit = true;
2122 break;
2123 case T_LONG:
2124 imm = opr2->as_constant_ptr()->as_jlong();
2125 break;
2126 case T_ADDRESS:
2127 imm = opr2->as_constant_ptr()->as_jint();
2128 break;
2129 case T_METADATA:
2130 imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
2131 break;
2132 case T_PRIMITIVE_OBJECT:
2133 case T_OBJECT:
2134 case T_ARRAY:
2135 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
2136 __ cmpoop(reg1, rscratch1);
2137 return;
2138 default:
2139 ShouldNotReachHere();
2140 imm = 0; // unreachable
2141 break;
2142 }
2143
2144 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
2145 if (is_32bit)
2146 __ cmpw(reg1, imm);
2147 else
2148 __ subs(zr, reg1, imm);
2149 return;
2150 } else {
2151 __ mov(rscratch1, imm);
2152 if (is_32bit)
2153 __ cmpw(reg1, rscratch1);
2154 else
2155 __ cmp(reg1, rscratch1);
2156 return;
2157 }
2158 } else
2159 ShouldNotReachHere();
2160 } else if (opr1->is_single_fpu()) {
2161 FloatRegister reg1 = opr1->as_float_reg();
2162 assert(opr2->is_single_fpu(), "expect single float register");
2163 FloatRegister reg2 = opr2->as_float_reg();
2164 __ fcmps(reg1, reg2);
2165 } else if (opr1->is_double_fpu()) {
2166 FloatRegister reg1 = opr1->as_double_reg();
2167 assert(opr2->is_double_fpu(), "expect double float register");
2168 FloatRegister reg2 = opr2->as_double_reg();
2169 __ fcmpd(reg1, reg2);
2170 } else {
2171 ShouldNotReachHere();
2172 }
2173 }
2174
2175 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2176 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2177 bool is_unordered_less = (code == lir_ucmp_fd2i);
2178 if (left->is_single_fpu()) {
2179 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2180 } else if (left->is_double_fpu()) {
2181 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2182 } else {
2183 ShouldNotReachHere();
2184 }
2185 } else if (code == lir_cmp_l2i) {
2186 Label done;
2187 __ cmp(left->as_register_lo(), right->as_register_lo());
2188 __ mov(dst->as_register(), (uint64_t)-1L);
2189 __ br(Assembler::LT, done);
2190 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2191 __ bind(done);
2192 } else {
2193 ShouldNotReachHere();
2194 }
2195 }
2196
2197
2198 void LIR_Assembler::align_call(LIR_Code code) { }
2199
2200
2201 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2202 address call = __ trampoline_call(Address(op->addr(), rtype));
2203 if (call == NULL) {
2204 bailout("trampoline stub overflow");
2205 return;
2206 }
2207 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2208 __ post_call_nop();
2209 }
2210
2211
2212 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2213 address call = __ ic_call(op->addr());
2214 if (call == NULL) {
2215 bailout("trampoline stub overflow");
2216 return;
2217 }
2218 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2219 __ post_call_nop();
2220 }
2221
2222 void LIR_Assembler::emit_static_call_stub() {
2223 address call_pc = __ pc();
2224 address stub = __ start_a_stub(call_stub_size());
2225 if (stub == NULL) {
2226 bailout("static call stub overflow");
2227 return;
2228 }
2229
2230 int start = __ offset();
2231
2232 __ relocate(static_stub_Relocation::spec(call_pc));
2233 __ emit_static_call_stub();
2234
2235 assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2236 <= call_stub_size(), "stub too big");
2237 __ end_a_stub();
2238 }
2239
2240
2241 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2242 assert(exceptionOop->as_register() == r0, "must match");
2243 assert(exceptionPC->as_register() == r3, "must match");
2244
2245 // exception object is not added to oop map by LinearScan
2246 // (LinearScan assumes that no oops are in fixed registers)
2247 info->add_register_oop(exceptionOop);
2248 Runtime1::StubID unwind_id;
2249
2250 // get current pc information
2251 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2252 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
2253 // As no instructions have been generated yet for this LIR node it's
2254 // possible that an oop map already exists for the current offset.
2255 // In that case insert an dummy NOP here to ensure all oop map PCs
2256 // are unique. See JDK-8237483.
2257 __ nop();
2258 }
2259 int pc_for_athrow_offset = __ offset();
2260 InternalAddress pc_for_athrow(__ pc());
2261 __ adr(exceptionPC->as_register(), pc_for_athrow);
2262 add_call_info(pc_for_athrow_offset, info); // for exception handler
2263
2264 __ verify_not_null_oop(r0);
2265 // search an exception handler (r0: exception oop, r3: throwing pc)
2266 if (compilation()->has_fpu_code()) {
2267 unwind_id = Runtime1::handle_exception_id;
2268 } else {
2269 unwind_id = Runtime1::handle_exception_nofpu_id;
2270 }
2271 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2272
2273 // FIXME: enough room for two byte trap ????
2274 __ nop();
2275 }
2276
2277
2278 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2279 assert(exceptionOop->as_register() == r0, "must match");
2280
2281 __ b(_unwind_handler_entry);
2282 }
2283
2284
2285 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2286 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2287 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2288
2289 switch (left->type()) {
2290 case T_INT: {
2291 switch (code) {
2292 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2293 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2294 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2295 default:
2296 ShouldNotReachHere();
2297 break;
2298 }
2299 break;
2300 case T_LONG:
2301 case T_PRIMITIVE_OBJECT:
2302 case T_ADDRESS:
2303 case T_OBJECT:
2304 switch (code) {
2305 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2306 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2307 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2308 default:
2309 ShouldNotReachHere();
2310 break;
2311 }
2312 break;
2313 default:
2314 ShouldNotReachHere();
2315 break;
2316 }
2317 }
2318 }
2319
2320
2321 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2322 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2323 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2324
2325 switch (left->type()) {
2326 case T_INT: {
2327 switch (code) {
2328 case lir_shl: __ lslw (dreg, lreg, count); break;
2329 case lir_shr: __ asrw (dreg, lreg, count); break;
2330 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2331 default:
2332 ShouldNotReachHere();
2333 break;
2334 }
2335 break;
2336 case T_LONG:
2337 case T_ADDRESS:
2338 case T_PRIMITIVE_OBJECT:
2339 case T_OBJECT:
2340 switch (code) {
2341 case lir_shl: __ lsl (dreg, lreg, count); break;
2342 case lir_shr: __ asr (dreg, lreg, count); break;
2343 case lir_ushr: __ lsr (dreg, lreg, count); break;
2344 default:
2345 ShouldNotReachHere();
2346 break;
2347 }
2348 break;
2349 default:
2350 ShouldNotReachHere();
2351 break;
2352 }
2353 }
2354 }
2355
2356
2357 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2358 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2359 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2360 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2361 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2362 }
2363
2364
2365 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2366 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2367 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2368 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2369 __ mov (rscratch1, c);
2370 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2371 }
2372
2373
2374 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2375 ShouldNotReachHere();
2376 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2377 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2378 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2379 __ lea(rscratch1, __ constant_oop_address(o));
2380 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2381 }
2382
2383 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2384 if (null_check) {
2385 __ cbz(obj, *slow_path->entry());
2386 }
2387 if (UseArrayMarkWordCheck) {
2388 if (is_dest) {
2389 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2390 } else {
2391 __ test_flattened_array_oop(obj, tmp, *slow_path->entry());
2392 }
2393 } else {
2394 __ load_klass(tmp, obj);
2395 __ ldr(tmp, Address(tmp, Klass::layout_helper_offset()));
2396 if (is_dest) {
2397 // Take the slow path if it's a null_free destination array, in case the source array contains NULLs.
2398 __ tst(tmp, Klass::_lh_null_free_array_bit_inplace);
2399 } else {
2400 __ tst(tmp, Klass::_lh_array_tag_flat_value_bit_inplace);
2401 }
2402 __ br(Assembler::NE, *slow_path->entry());
2403 }
2404 }
2405
2406 // This code replaces a call to arraycopy; no exception may
2407 // be thrown in this code, they must be thrown in the System.arraycopy
2408 // activation frame; we could save some checks if this would not be the case
2409 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2410 ciArrayKlass* default_type = op->expected_type();
2411 Register src = op->src()->as_register();
2412 Register dst = op->dst()->as_register();
2413 Register src_pos = op->src_pos()->as_register();
2414 Register dst_pos = op->dst_pos()->as_register();
2415 Register length = op->length()->as_register();
2416 Register tmp = op->tmp()->as_register();
2417
2418 CodeStub* stub = op->stub();
2419 int flags = op->flags();
2420 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2421 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2422
2423 if (flags & LIR_OpArrayCopy::always_slow_path) {
2424 __ b(*stub->entry());
2425 __ bind(*stub->continuation());
2426 return;
2427 }
2428
2429 // if we don't know anything, just go through the generic arraycopy
2430 if (default_type == NULL // || basic_type == T_OBJECT
2431 ) {
2432 Label done;
2433 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2434
2435 // Save the arguments in case the generic arraycopy fails and we
2436 // have to fall back to the JNI stub
2437 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2438 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2439 __ str(src, Address(sp, 4*BytesPerWord));
2440
2441 address copyfunc_addr = StubRoutines::generic_arraycopy();
2442 assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2443
2444 // The arguments are in java calling convention so we shift them
2445 // to C convention
2446 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2447 __ mov(c_rarg0, j_rarg0);
2448 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2449 __ mov(c_rarg1, j_rarg1);
2450 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2451 __ mov(c_rarg2, j_rarg2);
2452 assert_different_registers(c_rarg3, j_rarg4);
2453 __ mov(c_rarg3, j_rarg3);
2454 __ mov(c_rarg4, j_rarg4);
2455 #ifndef PRODUCT
2456 if (PrintC1Statistics) {
2457 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2458 }
2459 #endif
2460 __ far_call(RuntimeAddress(copyfunc_addr));
2461
2462 __ cbz(r0, *stub->continuation());
2463
2464 // Reload values from the stack so they are where the stub
2465 // expects them.
2466 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2467 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2468 __ ldr(src, Address(sp, 4*BytesPerWord));
2469
2470 // r0 is -1^K where K == partial copied count
2471 __ eonw(rscratch1, r0, zr);
2472 // adjust length down and src/end pos up by partial copied count
2473 __ subw(length, length, rscratch1);
2474 __ addw(src_pos, src_pos, rscratch1);
2475 __ addw(dst_pos, dst_pos, rscratch1);
2476 __ b(*stub->entry());
2477
2478 __ bind(*stub->continuation());
2479 return;
2480 }
2481
2482 // Handle inline type arrays
2483 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2484 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2485 }
2486
2487 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2488 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2489 }
2490
2491 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2492
2493 int elem_size = type2aelembytes(basic_type);
2494 int scale = exact_log2(elem_size);
2495
2496 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2497 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2498 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2499 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2500
2501 // test for NULL
2502 if (flags & LIR_OpArrayCopy::src_null_check) {
2503 __ cbz(src, *stub->entry());
2504 }
2505 if (flags & LIR_OpArrayCopy::dst_null_check) {
2506 __ cbz(dst, *stub->entry());
2507 }
2508
2509 // If the compiler was not able to prove that exact type of the source or the destination
2510 // of the arraycopy is an array type, check at runtime if the source or the destination is
2511 // an instance type.
2512 if (flags & LIR_OpArrayCopy::type_check) {
2513 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2514 __ load_klass(tmp, dst);
2515 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2516 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2517 __ br(Assembler::GE, *stub->entry());
2518 }
2519
2520 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2521 __ load_klass(tmp, src);
2522 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2523 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2524 __ br(Assembler::GE, *stub->entry());
2525 }
2526 }
2527
2528 // check if negative
2529 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2530 __ cmpw(src_pos, 0);
2531 __ br(Assembler::LT, *stub->entry());
2532 }
2533 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2534 __ cmpw(dst_pos, 0);
2535 __ br(Assembler::LT, *stub->entry());
2536 }
2537
2538 if (flags & LIR_OpArrayCopy::length_positive_check) {
2539 __ cmpw(length, 0);
2540 __ br(Assembler::LT, *stub->entry());
2541 }
2542
2543 if (flags & LIR_OpArrayCopy::src_range_check) {
2544 __ addw(tmp, src_pos, length);
2545 __ ldrw(rscratch1, src_length_addr);
2546 __ cmpw(tmp, rscratch1);
2547 __ br(Assembler::HI, *stub->entry());
2548 }
2549 if (flags & LIR_OpArrayCopy::dst_range_check) {
2550 __ addw(tmp, dst_pos, length);
2551 __ ldrw(rscratch1, dst_length_addr);
2552 __ cmpw(tmp, rscratch1);
2553 __ br(Assembler::HI, *stub->entry());
2554 }
2555
2556 if (flags & LIR_OpArrayCopy::type_check) {
2557 // We don't know the array types are compatible
2558 if (basic_type != T_OBJECT) {
2559 // Simple test for basic type arrays
2560 if (UseCompressedClassPointers) {
2561 __ ldrw(tmp, src_klass_addr);
2562 __ ldrw(rscratch1, dst_klass_addr);
2563 __ cmpw(tmp, rscratch1);
2564 } else {
2565 __ ldr(tmp, src_klass_addr);
2566 __ ldr(rscratch1, dst_klass_addr);
2567 __ cmp(tmp, rscratch1);
2568 }
2569 __ br(Assembler::NE, *stub->entry());
2570 } else {
2571 // For object arrays, if src is a sub class of dst then we can
2572 // safely do the copy.
2573 Label cont, slow;
2574
2575 #define PUSH(r1, r2) \
2576 stp(r1, r2, __ pre(sp, -2 * wordSize));
2577
2578 #define POP(r1, r2) \
2579 ldp(r1, r2, __ post(sp, 2 * wordSize));
2580
2581 __ PUSH(src, dst);
2582
2583 __ load_klass(src, src);
2584 __ load_klass(dst, dst);
2585
2586 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2587
2588 __ PUSH(src, dst);
2589 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2590 __ POP(src, dst);
2591
2592 __ cbnz(src, cont);
2593
2594 __ bind(slow);
2595 __ POP(src, dst);
2596
2597 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2598 if (copyfunc_addr != NULL) { // use stub if available
2599 // src is not a sub class of dst so we have to do a
2600 // per-element check.
2601
2602 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2603 if ((flags & mask) != mask) {
2604 // Check that at least both of them object arrays.
2605 assert(flags & mask, "one of the two should be known to be an object array");
2606
2607 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2608 __ load_klass(tmp, src);
2609 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2610 __ load_klass(tmp, dst);
2611 }
2612 int lh_offset = in_bytes(Klass::layout_helper_offset());
2613 Address klass_lh_addr(tmp, lh_offset);
2614 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2615 __ ldrw(rscratch1, klass_lh_addr);
2616 __ mov(rscratch2, objArray_lh);
2617 __ eorw(rscratch1, rscratch1, rscratch2);
2618 __ cbnzw(rscratch1, *stub->entry());
2619 }
2620
2621 // Spill because stubs can use any register they like and it's
2622 // easier to restore just those that we care about.
2623 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2624 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2625 __ str(src, Address(sp, 4*BytesPerWord));
2626
2627 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2628 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2629 assert_different_registers(c_rarg0, dst, dst_pos, length);
2630 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2631 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2632 assert_different_registers(c_rarg1, dst, length);
2633 __ uxtw(c_rarg2, length);
2634 assert_different_registers(c_rarg2, dst);
2635
2636 __ load_klass(c_rarg4, dst);
2637 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2638 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2639 __ far_call(RuntimeAddress(copyfunc_addr));
2640
2641 #ifndef PRODUCT
2642 if (PrintC1Statistics) {
2643 Label failed;
2644 __ cbnz(r0, failed);
2645 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2646 __ bind(failed);
2647 }
2648 #endif
2649
2650 __ cbz(r0, *stub->continuation());
2651
2652 #ifndef PRODUCT
2653 if (PrintC1Statistics) {
2654 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2655 }
2656 #endif
2657 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2658
2659 // Restore previously spilled arguments
2660 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2661 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2662 __ ldr(src, Address(sp, 4*BytesPerWord));
2663
2664 // return value is -1^K where K is partial copied count
2665 __ eonw(rscratch1, r0, zr);
2666 // adjust length down and src/end pos up by partial copied count
2667 __ subw(length, length, rscratch1);
2668 __ addw(src_pos, src_pos, rscratch1);
2669 __ addw(dst_pos, dst_pos, rscratch1);
2670 }
2671
2672 __ b(*stub->entry());
2673
2674 __ bind(cont);
2675 __ POP(src, dst);
2676 }
2677 }
2678
2679 #ifdef ASSERT
2680 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2681 // Sanity check the known type with the incoming class. For the
2682 // primitive case the types must match exactly with src.klass and
2683 // dst.klass each exactly matching the default type. For the
2684 // object array case, if no type check is needed then either the
2685 // dst type is exactly the expected type and the src type is a
2686 // subtype which we can't check or src is the same array as dst
2687 // but not necessarily exactly of type default_type.
2688 Label known_ok, halt;
2689 __ mov_metadata(tmp, default_type->constant_encoding());
2690 if (UseCompressedClassPointers) {
2691 __ encode_klass_not_null(tmp);
2692 }
2693
2694 if (basic_type != T_OBJECT) {
2695
2696 if (UseCompressedClassPointers) {
2697 __ ldrw(rscratch1, dst_klass_addr);
2698 __ cmpw(tmp, rscratch1);
2699 } else {
2700 __ ldr(rscratch1, dst_klass_addr);
2701 __ cmp(tmp, rscratch1);
2702 }
2703 __ br(Assembler::NE, halt);
2704 if (UseCompressedClassPointers) {
2705 __ ldrw(rscratch1, src_klass_addr);
2706 __ cmpw(tmp, rscratch1);
2707 } else {
2708 __ ldr(rscratch1, src_klass_addr);
2709 __ cmp(tmp, rscratch1);
2710 }
2711 __ br(Assembler::EQ, known_ok);
2712 } else {
2713 if (UseCompressedClassPointers) {
2714 __ ldrw(rscratch1, dst_klass_addr);
2715 __ cmpw(tmp, rscratch1);
2716 } else {
2717 __ ldr(rscratch1, dst_klass_addr);
2718 __ cmp(tmp, rscratch1);
2719 }
2720 __ br(Assembler::EQ, known_ok);
2721 __ cmp(src, dst);
2722 __ br(Assembler::EQ, known_ok);
2723 }
2724 __ bind(halt);
2725 __ stop("incorrect type information in arraycopy");
2726 __ bind(known_ok);
2727 }
2728 #endif
2729
2730 #ifndef PRODUCT
2731 if (PrintC1Statistics) {
2732 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2733 }
2734 #endif
2735
2736 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2737 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2738 assert_different_registers(c_rarg0, dst, dst_pos, length);
2739 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2740 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2741 assert_different_registers(c_rarg1, dst, length);
2742 __ uxtw(c_rarg2, length);
2743 assert_different_registers(c_rarg2, dst);
2744
2745 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2746 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2747 const char *name;
2748 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2749
2750 CodeBlob *cb = CodeCache::find_blob(entry);
2751 if (cb) {
2752 __ far_call(RuntimeAddress(entry));
2753 } else {
2754 __ call_VM_leaf(entry, 3);
2755 }
2756
2757 __ bind(*stub->continuation());
2758 }
2759
2760
2761
2762
2763 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2764 Register obj = op->obj_opr()->as_register(); // may not be an oop
2765 Register hdr = op->hdr_opr()->as_register();
2766 Register lock = op->lock_opr()->as_register();
2767 if (UseHeavyMonitors) {
2768 if (op->info() != NULL) {
2769 add_debug_info_for_null_check_here(op->info());
2770 __ null_check(obj, -1);
2771 }
2772 __ b(*op->stub()->entry());
2773 } else if (op->code() == lir_lock) {
2774 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2775 // add debug info for NullPointerException only if one is possible
2776 int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2777 if (op->info() != NULL) {
2778 add_debug_info_for_null_check(null_check_offset, op->info());
2779 }
2780 // done
2781 } else if (op->code() == lir_unlock) {
2782 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2783 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2784 } else {
2785 Unimplemented();
2786 }
2787 __ bind(*op->stub()->continuation());
2788 }
2789
2790 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2791 Register obj = op->obj()->as_pointer_register();
2792 Register result = op->result_opr()->as_pointer_register();
2793
2794 CodeEmitInfo* info = op->info();
2795 if (info != NULL) {
2796 add_debug_info_for_null_check_here(info);
2797 }
2798
2799 if (UseCompressedClassPointers) {
2800 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2801 __ decode_klass_not_null(result);
2802 } else {
2803 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2804 }
2805 }
2806
2807 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2808 ciMethod* method = op->profiled_method();
2809 int bci = op->profiled_bci();
2810 ciMethod* callee = op->profiled_callee();
2811
2812 // Update counter for all call types
2813 ciMethodData* md = method->method_data_or_null();
2814 assert(md != NULL, "Sanity");
2815 ciProfileData* data = md->bci_to_data(bci);
2816 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2817 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2818 Register mdo = op->mdo()->as_register();
2819 __ mov_metadata(mdo, md->constant_encoding());
2820 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2821 // Perform additional virtual call profiling for invokevirtual and
2822 // invokeinterface bytecodes
2823 if (op->should_profile_receiver_type()) {
2824 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2825 Register recv = op->recv()->as_register();
2826 assert_different_registers(mdo, recv);
2827 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2828 ciKlass* known_klass = op->known_holder();
2829 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2830 // We know the type that will be seen at this call site; we can
2831 // statically update the MethodData* rather than needing to do
2832 // dynamic tests on the receiver type
2833
2834 // NOTE: we should probably put a lock around this search to
2835 // avoid collisions by concurrent compilations
2836 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2837 uint i;
2838 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2839 ciKlass* receiver = vc_data->receiver(i);
2840 if (known_klass->equals(receiver)) {
2841 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2842 __ addptr(data_addr, DataLayout::counter_increment);
2843 return;
2844 }
2845 }
2846
2847 // Receiver type not found in profile data; select an empty slot
2848
2849 // Note that this is less efficient than it should be because it
2850 // always does a write to the receiver part of the
2851 // VirtualCallData rather than just the first time
2852 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2853 ciKlass* receiver = vc_data->receiver(i);
2854 if (receiver == NULL) {
2855 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2856 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2857 __ lea(rscratch2, recv_addr);
2858 __ str(rscratch1, Address(rscratch2));
2859 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2860 __ addptr(data_addr, DataLayout::counter_increment);
2861 return;
2862 }
2863 }
2864 } else {
2865 __ load_klass(recv, recv);
2866 Label update_done;
2867 type_profile_helper(mdo, md, data, recv, &update_done);
2868 // Receiver did not match any saved receiver and there is no empty row for it.
2869 // Increment total counter to indicate polymorphic case.
2870 __ addptr(counter_addr, DataLayout::counter_increment);
2871
2872 __ bind(update_done);
2873 }
2874 } else {
2875 // Static call
2876 __ addptr(counter_addr, DataLayout::counter_increment);
2877 }
2878 }
2879
2880
2881 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2882 Unimplemented();
2883 }
2884
2885
2886 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2887 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2888 }
2889
2890 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2891 assert(op->crc()->is_single_cpu(), "crc must be register");
2892 assert(op->val()->is_single_cpu(), "byte value must be register");
2893 assert(op->result_opr()->is_single_cpu(), "result must be register");
2894 Register crc = op->crc()->as_register();
2895 Register val = op->val()->as_register();
2896 Register res = op->result_opr()->as_register();
2897
2898 assert_different_registers(val, crc, res);
2899 uint64_t offset;
2900 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2901 __ add(res, res, offset);
2902
2903 __ mvnw(crc, crc); // ~crc
2904 __ update_byte_crc32(crc, val, res);
2905 __ mvnw(res, crc); // ~crc
2906 }
2907
2908 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2909 COMMENT("emit_profile_type {");
2910 Register obj = op->obj()->as_register();
2911 Register tmp = op->tmp()->as_pointer_register();
2912 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2913 ciKlass* exact_klass = op->exact_klass();
2914 intptr_t current_klass = op->current_klass();
2915 bool not_null = op->not_null();
2916 bool no_conflict = op->no_conflict();
2917
2918 Label update, next, none;
2919
2920 bool do_null = !not_null;
2921 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2922 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2923
2924 assert(do_null || do_update, "why are we here?");
2925 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2926 assert(mdo_addr.base() != rscratch1, "wrong register");
2927
2928 __ verify_oop(obj);
2929
2930 if (tmp != obj) {
2931 __ mov(tmp, obj);
2932 }
2933 if (do_null) {
2934 __ cbnz(tmp, update);
2935 if (!TypeEntries::was_null_seen(current_klass)) {
2936 __ ldr(rscratch2, mdo_addr);
2937 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2938 __ str(rscratch2, mdo_addr);
2939 }
2940 if (do_update) {
2941 #ifndef ASSERT
2942 __ b(next);
2943 }
2944 #else
2945 __ b(next);
2946 }
2947 } else {
2948 __ cbnz(tmp, update);
2949 __ stop("unexpected null obj");
2950 #endif
2951 }
2952
2953 __ bind(update);
2954
2955 if (do_update) {
2956 #ifdef ASSERT
2957 if (exact_klass != NULL) {
2958 Label ok;
2959 __ load_klass(tmp, tmp);
2960 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2961 __ eor(rscratch1, tmp, rscratch1);
2962 __ cbz(rscratch1, ok);
2963 __ stop("exact klass and actual klass differ");
2964 __ bind(ok);
2965 }
2966 #endif
2967 if (!no_conflict) {
2968 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2969 if (exact_klass != NULL) {
2970 __ mov_metadata(tmp, exact_klass->constant_encoding());
2971 } else {
2972 __ load_klass(tmp, tmp);
2973 }
2974
2975 __ ldr(rscratch2, mdo_addr);
2976 __ eor(tmp, tmp, rscratch2);
2977 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2978 // klass seen before, nothing to do. The unknown bit may have been
2979 // set already but no need to check.
2980 __ cbz(rscratch1, next);
2981
2982 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2983
2984 if (TypeEntries::is_type_none(current_klass)) {
2985 __ cbz(rscratch2, none);
2986 __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2987 __ br(Assembler::EQ, none);
2988 // There is a chance that the checks above (re-reading profiling
2989 // data from memory) fail if another thread has just set the
2990 // profiling to this obj's klass
2991 __ dmb(Assembler::ISHLD);
2992 __ ldr(rscratch2, mdo_addr);
2993 __ eor(tmp, tmp, rscratch2);
2994 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2995 __ cbz(rscratch1, next);
2996 }
2997 } else {
2998 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2999 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3000
3001 __ ldr(tmp, mdo_addr);
3002 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
3003 }
3004
3005 // different than before. Cannot keep accurate profile.
3006 __ ldr(rscratch2, mdo_addr);
3007 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
3008 __ str(rscratch2, mdo_addr);
3009
3010 if (TypeEntries::is_type_none(current_klass)) {
3011 __ b(next);
3012
3013 __ bind(none);
3014 // first time here. Set profile type.
3015 __ str(tmp, mdo_addr);
3016 }
3017 } else {
3018 // There's a single possible klass at this profile point
3019 assert(exact_klass != NULL, "should be");
3020 if (TypeEntries::is_type_none(current_klass)) {
3021 __ mov_metadata(tmp, exact_klass->constant_encoding());
3022 __ ldr(rscratch2, mdo_addr);
3023 __ eor(tmp, tmp, rscratch2);
3024 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
3025 __ cbz(rscratch1, next);
3026 #ifdef ASSERT
3027 {
3028 Label ok;
3029 __ ldr(rscratch1, mdo_addr);
3030 __ cbz(rscratch1, ok);
3031 __ cmp(rscratch1, (u1)TypeEntries::null_seen);
3032 __ br(Assembler::EQ, ok);
3033 // may have been set by another thread
3034 __ dmb(Assembler::ISHLD);
3035 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
3036 __ ldr(rscratch2, mdo_addr);
3037 __ eor(rscratch2, rscratch1, rscratch2);
3038 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
3039 __ cbz(rscratch2, ok);
3040
3041 __ stop("unexpected profiling mismatch");
3042 __ bind(ok);
3043 }
3044 #endif
3045 // first time here. Set profile type.
3046 __ str(tmp, mdo_addr);
3047 } else {
3048 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3049 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3050
3051 __ ldr(tmp, mdo_addr);
3052 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
3053
3054 __ orr(tmp, tmp, TypeEntries::type_unknown);
3055 __ str(tmp, mdo_addr);
3056 // FIXME: Write barrier needed here?
3057 }
3058 }
3059
3060 __ bind(next);
3061 }
3062 COMMENT("} emit_profile_type");
3063 }
3064
3065 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3066 Register obj = op->obj()->as_register();
3067 Register tmp = op->tmp()->as_pointer_register();
3068 bool not_null = op->not_null();
3069 int flag = op->flag();
3070
3071 Label not_inline_type;
3072 if (!not_null) {
3073 __ cbz(obj, not_inline_type);
3074 }
3075
3076 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3077
3078 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), rscratch2);
3079 __ ldrb(rscratch1, mdo_addr);
3080 __ orr(rscratch1, rscratch1, flag);
3081 __ strb(rscratch1, mdo_addr);
3082
3083 __ bind(not_inline_type);
3084 }
3085
3086 void LIR_Assembler::align_backward_branch_target() {
3087 }
3088
3089
3090 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3091 // tmp must be unused
3092 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3093
3094 if (left->is_single_cpu()) {
3095 assert(dest->is_single_cpu(), "expect single result reg");
3096 __ negw(dest->as_register(), left->as_register());
3097 } else if (left->is_double_cpu()) {
3098 assert(dest->is_double_cpu(), "expect double result reg");
3099 __ neg(dest->as_register_lo(), left->as_register_lo());
3100 } else if (left->is_single_fpu()) {
3101 assert(dest->is_single_fpu(), "expect single float result reg");
3102 __ fnegs(dest->as_float_reg(), left->as_float_reg());
3103 } else {
3104 assert(left->is_double_fpu(), "expect double float operand reg");
3105 assert(dest->is_double_fpu(), "expect double float result reg");
3106 __ fnegd(dest->as_double_reg(), left->as_double_reg());
3107 }
3108 }
3109
3110
3111 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3112 if (patch_code != lir_patch_none) {
3113 deoptimize_trap(info);
3114 return;
3115 }
3116
3117 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
3118 }
3119
3120
3121 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3122 assert(!tmp->is_valid(), "don't need temporary");
3123
3124 CodeBlob *cb = CodeCache::find_blob(dest);
3125 if (cb) {
3126 __ far_call(RuntimeAddress(dest));
3127 } else {
3128 __ mov(rscratch1, RuntimeAddress(dest));
3129 __ blr(rscratch1);
3130 }
3131
3132 if (info != NULL) {
3133 add_call_info_here(info);
3134 }
3135 __ post_call_nop();
3136 }
3137
3138 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3139 if (dest->is_address() || src->is_address()) {
3140 move_op(src, dest, type, lir_patch_none, info,
3141 /*pop_fpu_stack*/false, /*wide*/false);
3142 } else {
3143 ShouldNotReachHere();
3144 }
3145 }
3146
3147 #ifdef ASSERT
3148 // emit run-time assertion
3149 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3150 assert(op->code() == lir_assert, "must be");
3151
3152 if (op->in_opr1()->is_valid()) {
3153 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3154 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3155 } else {
3156 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3157 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3158 }
3159
3160 Label ok;
3161 if (op->condition() != lir_cond_always) {
3162 Assembler::Condition acond = Assembler::AL;
3163 switch (op->condition()) {
3164 case lir_cond_equal: acond = Assembler::EQ; break;
3165 case lir_cond_notEqual: acond = Assembler::NE; break;
3166 case lir_cond_less: acond = Assembler::LT; break;
3167 case lir_cond_lessEqual: acond = Assembler::LE; break;
3168 case lir_cond_greaterEqual: acond = Assembler::GE; break;
3169 case lir_cond_greater: acond = Assembler::GT; break;
3170 case lir_cond_belowEqual: acond = Assembler::LS; break;
3171 case lir_cond_aboveEqual: acond = Assembler::HS; break;
3172 default: ShouldNotReachHere();
3173 }
3174 __ br(acond, ok);
3175 }
3176 if (op->halt()) {
3177 const char* str = __ code_string(op->msg());
3178 __ stop(str);
3179 } else {
3180 breakpoint();
3181 }
3182 __ bind(ok);
3183 }
3184 #endif
3185
3186 #ifndef PRODUCT
3187 #define COMMENT(x) do { __ block_comment(x); } while (0)
3188 #else
3189 #define COMMENT(x)
3190 #endif
3191
3192 void LIR_Assembler::membar() {
3193 COMMENT("membar");
3194 __ membar(MacroAssembler::AnyAny);
3195 }
3196
3197 void LIR_Assembler::membar_acquire() {
3198 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3199 }
3200
3201 void LIR_Assembler::membar_release() {
3202 __ membar(Assembler::LoadStore|Assembler::StoreStore);
3203 }
3204
3205 void LIR_Assembler::membar_loadload() {
3206 __ membar(Assembler::LoadLoad);
3207 }
3208
3209 void LIR_Assembler::membar_storestore() {
3210 __ membar(MacroAssembler::StoreStore);
3211 }
3212
3213 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3214
3215 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3216
3217 void LIR_Assembler::on_spin_wait() {
3218 __ spin_wait();
3219 }
3220
3221 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3222 __ mov(result_reg->as_register(), rthread);
3223 }
3224
3225 void LIR_Assembler::check_orig_pc() {
3226 __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3227 __ cmp(rscratch2, (u1)NULL_WORD);
3228 }
3229
3230 void LIR_Assembler::peephole(LIR_List *lir) {
3231 #if 0
3232 if (tableswitch_count >= max_tableswitches)
3233 return;
3234
3235 /*
3236 This finite-state automaton recognizes sequences of compare-and-
3237 branch instructions. We will turn them into a tableswitch. You
3238 could argue that C1 really shouldn't be doing this sort of
3239 optimization, but without it the code is really horrible.
3240 */
3241
3242 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3243 int first_key, last_key = -2147483648;
3244 int next_key = 0;
3245 int start_insn = -1;
3246 int last_insn = -1;
3247 Register reg = noreg;
3248 LIR_Opr reg_opr;
3249 state = start_s;
3250
3251 LIR_OpList* inst = lir->instructions_list();
3252 for (int i = 0; i < inst->length(); i++) {
3253 LIR_Op* op = inst->at(i);
3254 switch (state) {
3255 case start_s:
3256 first_key = -1;
3257 start_insn = i;
3258 switch (op->code()) {
3259 case lir_cmp:
3260 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3261 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3262 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3263 && opr2->is_constant()
3264 && opr2->type() == T_INT) {
3265 reg_opr = opr1;
3266 reg = opr1->as_register();
3267 first_key = opr2->as_constant_ptr()->as_jint();
3268 next_key = first_key + 1;
3269 state = cmp_s;
3270 goto next_state;
3271 }
3272 break;
3273 }
3274 break;
3275 case cmp_s:
3276 switch (op->code()) {
3277 case lir_branch:
3278 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3279 state = beq_s;
3280 last_insn = i;
3281 goto next_state;
3282 }
3283 }
3284 state = start_s;
3285 break;
3286 case beq_s:
3287 switch (op->code()) {
3288 case lir_cmp: {
3289 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3290 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3291 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3292 && opr1->as_register() == reg
3293 && opr2->is_constant()
3294 && opr2->type() == T_INT
3295 && opr2->as_constant_ptr()->as_jint() == next_key) {
3296 last_key = next_key;
3297 next_key++;
3298 state = cmp_s;
3299 goto next_state;
3300 }
3301 }
3302 }
3303 last_key = next_key;
3304 state = start_s;
3305 break;
3306 default:
3307 assert(false, "impossible state");
3308 }
3309 if (state == start_s) {
3310 if (first_key < last_key - 5L && reg != noreg) {
3311 {
3312 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3313 // reg->encoding(),
3314 // start_insn, first_key, last_key);
3315 // for (int i = 0; i < inst->length(); i++) {
3316 // inst->at(i)->print();
3317 // tty->print("\n");
3318 // }
3319 // tty->print("\n");
3320 }
3321
3322 struct tableswitch *sw = &switches[tableswitch_count];
3323 sw->_insn_index = start_insn, sw->_first_key = first_key,
3324 sw->_last_key = last_key, sw->_reg = reg;
3325 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3326 {
3327 // Insert the new table of branches
3328 int offset = last_insn;
3329 for (int n = first_key; n < last_key; n++) {
3330 inst->insert_before
3331 (last_insn + 1,
3332 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3333 inst->at(offset)->as_OpBranch()->label()));
3334 offset -= 2, i++;
3335 }
3336 }
3337 // Delete all the old compare-and-branch instructions
3338 for (int n = first_key; n < last_key; n++) {
3339 inst->remove_at(start_insn);
3340 inst->remove_at(start_insn);
3341 }
3342 // Insert the tableswitch instruction
3343 inst->insert_before(start_insn,
3344 new LIR_Op2(lir_cmp, lir_cond_always,
3345 LIR_OprFact::intConst(tableswitch_count),
3346 reg_opr));
3347 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3348 tableswitch_count++;
3349 }
3350 reg = noreg;
3351 last_key = -2147483648;
3352 }
3353 next_state:
3354 ;
3355 }
3356 #endif
3357 }
3358
3359 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3360 Address addr = as_Address(src->as_address_ptr());
3361 BasicType type = src->type();
3362 bool is_oop = is_reference_type(type);
3363
3364 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3365 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3366
3367 switch(type) {
3368 case T_INT:
3369 xchg = &MacroAssembler::atomic_xchgalw;
3370 add = &MacroAssembler::atomic_addalw;
3371 break;
3372 case T_LONG:
3373 xchg = &MacroAssembler::atomic_xchgal;
3374 add = &MacroAssembler::atomic_addal;
3375 break;
3376 case T_PRIMITIVE_OBJECT:
3377 case T_OBJECT:
3378 case T_ARRAY:
3379 if (UseCompressedOops) {
3380 xchg = &MacroAssembler::atomic_xchgalw;
3381 add = &MacroAssembler::atomic_addalw;
3382 } else {
3383 xchg = &MacroAssembler::atomic_xchgal;
3384 add = &MacroAssembler::atomic_addal;
3385 }
3386 break;
3387 default:
3388 ShouldNotReachHere();
3389 xchg = &MacroAssembler::atomic_xchgal;
3390 add = &MacroAssembler::atomic_addal; // unreachable
3391 }
3392
3393 switch (code) {
3394 case lir_xadd:
3395 {
3396 RegisterOrConstant inc;
3397 Register tmp = as_reg(tmp_op);
3398 Register dst = as_reg(dest);
3399 if (data->is_constant()) {
3400 inc = RegisterOrConstant(as_long(data));
3401 assert_different_registers(dst, addr.base(), tmp,
3402 rscratch1, rscratch2);
3403 } else {
3404 inc = RegisterOrConstant(as_reg(data));
3405 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3406 rscratch1, rscratch2);
3407 }
3408 __ lea(tmp, addr);
3409 (_masm->*add)(dst, inc, tmp);
3410 break;
3411 }
3412 case lir_xchg:
3413 {
3414 Register tmp = tmp_op->as_register();
3415 Register obj = as_reg(data);
3416 Register dst = as_reg(dest);
3417 if (is_oop && UseCompressedOops) {
3418 __ encode_heap_oop(rscratch2, obj);
3419 obj = rscratch2;
3420 }
3421 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3422 __ lea(tmp, addr);
3423 (_masm->*xchg)(dst, obj, tmp);
3424 if (is_oop && UseCompressedOops) {
3425 __ decode_heap_oop(dst);
3426 }
3427 }
3428 break;
3429 default:
3430 ShouldNotReachHere();
3431 }
3432 __ membar(__ AnyAny);
3433 }
3434
3435 #undef __