1 /*
2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "code/aotCodeCache.hpp"
37 #include "code/compiledIC.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_aarch64.inline.hpp"
47
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register SYNC_header = r0; // synchronization header
57 const Register SHIFT_count = r0; // where count for shift operations must be
58
59 #define __ _masm->
60
61
62 static void select_different_registers(Register preserve,
63 Register extra,
64 Register &tmp1,
65 Register &tmp2) {
66 if (tmp1 == preserve) {
67 assert_different_registers(tmp1, tmp2, extra);
68 tmp1 = extra;
69 } else if (tmp2 == preserve) {
70 assert_different_registers(tmp1, tmp2, extra);
71 tmp2 = extra;
72 }
73 assert_different_registers(preserve, tmp1, tmp2);
74 }
75
76
77
78 static void select_different_registers(Register preserve,
79 Register extra,
80 Register &tmp1,
81 Register &tmp2,
82 Register &tmp3) {
83 if (tmp1 == preserve) {
84 assert_different_registers(tmp1, tmp2, tmp3, extra);
85 tmp1 = extra;
86 } else if (tmp2 == preserve) {
87 assert_different_registers(tmp1, tmp2, tmp3, extra);
88 tmp2 = extra;
89 } else if (tmp3 == preserve) {
90 assert_different_registers(tmp1, tmp2, tmp3, extra);
91 tmp3 = extra;
92 }
93 assert_different_registers(preserve, tmp1, tmp2, tmp3);
94 }
95
96
97 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
98
99
100 LIR_Opr LIR_Assembler::receiverOpr() {
101 return FrameMap::receiver_opr;
102 }
103
104 LIR_Opr LIR_Assembler::osrBufferPointer() {
105 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
106 }
107
108 //--------------fpu register translations-----------------------
109
110
111 address LIR_Assembler::float_constant(float f) {
112 address const_addr = __ float_constant(f);
113 if (const_addr == nullptr) {
114 bailout("const section overflow");
115 return __ code()->consts()->start();
116 } else {
117 return const_addr;
118 }
119 }
120
121
122 address LIR_Assembler::double_constant(double d) {
123 address const_addr = __ double_constant(d);
124 if (const_addr == nullptr) {
125 bailout("const section overflow");
126 return __ code()->consts()->start();
127 } else {
128 return const_addr;
129 }
130 }
131
132 address LIR_Assembler::int_constant(jlong n) {
133 address const_addr = __ long_constant(n);
134 if (const_addr == nullptr) {
135 bailout("const section overflow");
136 return __ code()->consts()->start();
137 } else {
138 return const_addr;
139 }
140 }
141
142 void LIR_Assembler::breakpoint() { Unimplemented(); }
143
144 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
145
146 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
147
148 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
149 //-------------------------------------------
150
151 static Register as_reg(LIR_Opr op) {
152 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
153 }
154
155 static jlong as_long(LIR_Opr data) {
156 jlong result;
157 switch (data->type()) {
158 case T_INT:
159 result = (data->as_jint());
160 break;
161 case T_LONG:
162 result = (data->as_jlong());
163 break;
164 default:
165 ShouldNotReachHere();
166 result = 0; // unreachable
167 }
168 return result;
169 }
170
171 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
172 Register base = addr->base()->as_pointer_register();
173 LIR_Opr opr = addr->index();
174 if (opr->is_cpu_register()) {
175 Register index;
176 if (opr->is_single_cpu())
177 index = opr->as_register();
178 else
179 index = opr->as_register_lo();
180 assert(addr->disp() == 0, "must be");
181 switch(opr->type()) {
182 case T_INT:
183 return Address(base, index, Address::sxtw(addr->scale()));
184 case T_LONG:
185 return Address(base, index, Address::lsl(addr->scale()));
186 default:
187 ShouldNotReachHere();
188 }
189 } else {
190 assert(addr->scale() == 0,
191 "expected for immediate operand, was: %d", addr->scale());
192 ptrdiff_t offset = ptrdiff_t(addr->disp());
193 // NOTE: Does not handle any 16 byte vector access.
194 const uint type_size = type2aelembytes(addr->type(), true);
195 return __ legitimize_address(Address(base, offset), type_size, tmp);
196 }
197 return Address();
198 }
199
200 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
201 ShouldNotReachHere();
202 return Address();
203 }
204
205 Address LIR_Assembler::as_Address(LIR_Address* addr) {
206 return as_Address(addr, rscratch1);
207 }
208
209 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
210 return as_Address(addr, rscratch1); // Ouch
211 // FIXME: This needs to be much more clever. See x86.
212 }
213
214 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
215 // not encodable as a base + (immediate) offset, generate an explicit address
216 // calculation to hold the address in a temporary register.
217 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) {
218 precond(size == 4 || size == 8);
219 Address addr = frame_map()->address_for_slot(index, adjust);
220 precond(addr.getMode() == Address::base_plus_offset);
221 precond(addr.base() == sp);
222 precond(addr.offset() > 0);
223 uint mask = size - 1;
224 assert((addr.offset() & mask) == 0, "scaled offsets only");
225 return __ legitimize_address(addr, size, tmp);
226 }
227
228 void LIR_Assembler::osr_entry() {
229 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
230 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
231 ValueStack* entry_state = osr_entry->state();
232 int number_of_locks = entry_state->locks_size();
233
234 // we jump here if osr happens with the interpreter
235 // state set up to continue at the beginning of the
236 // loop that triggered osr - in particular, we have
237 // the following registers setup:
238 //
239 // r2: osr buffer
240 //
241
242 // build frame
243 ciMethod* m = compilation()->method();
244 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
245
246 // OSR buffer is
247 //
248 // locals[nlocals-1..0]
249 // monitors[0..number_of_locks]
250 //
251 // locals is a direct copy of the interpreter frame so in the osr buffer
252 // so first slot in the local array is the last local from the interpreter
253 // and last slot is local[0] (receiver) from the interpreter
254 //
255 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
256 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
257 // in the interpreter frame (the method lock if a sync method)
258
259 // Initialize monitors in the compiled activation.
260 // r2: pointer to osr buffer
261 //
262 // All other registers are dead at this point and the locals will be
263 // copied into place by code emitted in the IR.
264
265 Register OSR_buf = osrBufferPointer()->as_pointer_register();
266 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
267 int monitor_offset = BytesPerWord * method()->max_locals() +
268 (2 * BytesPerWord) * (number_of_locks - 1);
269 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
270 // the OSR buffer using 2 word entries: first the lock and then
271 // the oop.
272 for (int i = 0; i < number_of_locks; i++) {
273 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
274 #ifdef ASSERT
275 // verify the interpreter's monitor has a non-null object
276 {
277 Label L;
278 __ ldr(rscratch1, __ form_address(rscratch1, OSR_buf, slot_offset + 1*BytesPerWord, 0));
279 __ cbnz(rscratch1, L);
280 __ stop("locked object is null");
281 __ bind(L);
282 }
283 #endif
284 __ ldr(r19, __ form_address(rscratch1, OSR_buf, slot_offset, 0));
285 __ ldr(r20, __ form_address(rscratch1, OSR_buf, slot_offset + BytesPerWord, 0));
286 __ str(r19, frame_map()->address_for_monitor_lock(i));
287 __ str(r20, frame_map()->address_for_monitor_object(i));
288 }
289 }
290 }
291
292
293 // inline cache check; done before the frame is built.
294 int LIR_Assembler::check_icache() {
295 return __ ic_check(CodeEntryAlignment);
296 }
297
298 void LIR_Assembler::clinit_barrier(ciMethod* method) {
299 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
300 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
301
302 Label L_skip_barrier;
303
304 __ mov_metadata(rscratch2, method->holder()->constant_encoding());
305 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
306 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
307 __ bind(L_skip_barrier);
308 }
309
310 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
311 if (o == nullptr) {
312 __ mov(reg, zr);
313 } else {
314 __ movoop(reg, o);
315 }
316 }
317
318 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
319 address target = nullptr;
320 relocInfo::relocType reloc_type = relocInfo::none;
321
322 switch (patching_id(info)) {
323 case PatchingStub::access_field_id:
324 target = Runtime1::entry_for(StubId::c1_access_field_patching_id);
325 reloc_type = relocInfo::section_word_type;
326 break;
327 case PatchingStub::load_klass_id:
328 target = Runtime1::entry_for(StubId::c1_load_klass_patching_id);
329 reloc_type = relocInfo::metadata_type;
330 break;
331 case PatchingStub::load_mirror_id:
332 target = Runtime1::entry_for(StubId::c1_load_mirror_patching_id);
333 reloc_type = relocInfo::oop_type;
334 break;
335 case PatchingStub::load_appendix_id:
336 target = Runtime1::entry_for(StubId::c1_load_appendix_patching_id);
337 reloc_type = relocInfo::oop_type;
338 break;
339 default: ShouldNotReachHere();
340 }
341
342 __ far_call(RuntimeAddress(target));
343 add_call_info_here(info);
344 }
345
346 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
347 deoptimize_trap(info);
348 }
349
350
351 // This specifies the rsp decrement needed to build the frame
352 int LIR_Assembler::initial_frame_size_in_bytes() const {
353 // if rounding, must let FrameMap know!
354
355 return in_bytes(frame_map()->framesize_in_bytes());
356 }
357
358
359 int LIR_Assembler::emit_exception_handler() {
360 // generate code for exception handler
361 address handler_base = __ start_a_stub(exception_handler_size());
362 if (handler_base == nullptr) {
363 // not enough space left for the handler
364 bailout("exception handler overflow");
365 return -1;
366 }
367
368 int offset = code_offset();
369
370 // the exception oop and pc are in r0, and r3
371 // no other registers need to be preserved, so invalidate them
372 __ invalidate_registers(false, true, true, false, true, true);
373
374 // check that there is really an exception
375 __ verify_not_null_oop(r0);
376
377 // search an exception handler (r0: exception oop, r3: throwing pc)
378 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
379 __ should_not_reach_here();
380 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
381 __ end_a_stub();
382
383 return offset;
384 }
385
386
387 // Emit the code to remove the frame from the stack in the exception
388 // unwind path.
389 int LIR_Assembler::emit_unwind_handler() {
390 #ifndef PRODUCT
391 if (CommentedAssembly) {
392 _masm->block_comment("Unwind handler");
393 }
394 #endif
395
396 int offset = code_offset();
397
398 // Fetch the exception from TLS and clear out exception related thread state
399 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
400 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
401 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
402
403 __ bind(_unwind_handler_entry);
404 __ verify_not_null_oop(r0);
405 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
406 __ mov(r19, r0); // Preserve the exception
407 }
408
409 // Perform needed unlocking
410 MonitorExitStub* stub = nullptr;
411 if (method()->is_synchronized()) {
412 monitor_address(0, FrameMap::r0_opr);
413 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
414 __ unlock_object(r5, r4, r0, r6, *stub->entry());
415 __ bind(*stub->continuation());
416 }
417
418 if (compilation()->env()->dtrace_method_probes()) {
419 __ mov(c_rarg0, rthread);
420 __ mov_metadata(c_rarg1, method()->constant_encoding());
421 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
422 }
423
424 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
425 __ mov(r0, r19); // Restore the exception
426 }
427
428 // remove the activation and dispatch to the unwind handler
429 __ block_comment("remove_frame and dispatch to the unwind handler");
430 __ remove_frame(initial_frame_size_in_bytes());
431 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
432
433 // Emit the slow path assembly
434 if (stub != nullptr) {
435 stub->emit_code(this);
436 }
437
438 return offset;
439 }
440
441
442 int LIR_Assembler::emit_deopt_handler() {
443 // generate code for exception handler
444 address handler_base = __ start_a_stub(deopt_handler_size());
445 if (handler_base == nullptr) {
446 // not enough space left for the handler
447 bailout("deopt handler overflow");
448 return -1;
449 }
450
451 int offset = code_offset();
452
453 Label start;
454 __ bind(start);
455
456 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
457
458 int entry_offset = __ offset();
459 __ b(start);
460
461 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
462 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
463 "out of bounds read in post-call NOP check");
464 __ end_a_stub();
465
466 return entry_offset;
467 }
468
469 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
470 _masm->code_section()->relocate(adr, relocInfo::poll_type);
471 int pc_offset = code_offset();
472 flush_debug_info(pc_offset);
473 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
474 if (info->exception_handlers() != nullptr) {
475 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
476 }
477 }
478
479 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
480 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
481
482 // Pop the stack before the safepoint code
483 __ remove_frame(initial_frame_size_in_bytes());
484
485 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
486 __ reserved_stack_check();
487 }
488
489 code_stub->set_safepoint_offset(__ offset());
490 __ relocate(relocInfo::poll_return_type);
491 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
492 __ ret(lr);
493 }
494
495 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
496 guarantee(info != nullptr, "Shouldn't be null");
497 __ get_polling_page(rscratch1, relocInfo::poll_type);
498 add_debug_info_for_branch(info); // This isn't just debug info:
499 // it's the oop map
500 __ read_polling_page(rscratch1, relocInfo::poll_type);
501 return __ offset();
502 }
503
504
505 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
506 if (from_reg == r31_sp)
507 from_reg = sp;
508 if (to_reg == r31_sp)
509 to_reg = sp;
510 __ mov(to_reg, from_reg);
511 }
512
513 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
514
515
516 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
517 assert(src->is_constant(), "should not call otherwise");
518 assert(dest->is_register(), "should not call otherwise");
519 LIR_Const* c = src->as_constant_ptr();
520
521 switch (c->type()) {
522 case T_INT: {
523 assert(patch_code == lir_patch_none, "no patching handled here");
524 __ movw(dest->as_register(), c->as_jint());
525 break;
526 }
527
528 case T_ADDRESS: {
529 assert(patch_code == lir_patch_none, "no patching handled here");
530 __ mov(dest->as_register(), c->as_jint());
531 break;
532 }
533
534 case T_LONG: {
535 assert(patch_code == lir_patch_none, "no patching handled here");
536 #if INCLUDE_CDS
537 if (AOTCodeCache::is_on_for_dump()) {
538 address b = c->as_pointer();
539 if (AOTRuntimeConstants::contains(b)) {
540 __ load_aotrc_address(dest->as_register_lo(), b);
541 break;
542 }
543 }
544 #endif
545 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
546 break;
547 }
548
549 case T_OBJECT: {
550 if (patch_code == lir_patch_none) {
551 jobject2reg(c->as_jobject(), dest->as_register());
552 } else {
553 jobject2reg_with_patching(dest->as_register(), info);
554 }
555 break;
556 }
557
558 case T_METADATA: {
559 if (patch_code != lir_patch_none) {
560 klass2reg_with_patching(dest->as_register(), info);
561 } else {
562 __ mov_metadata(dest->as_register(), c->as_metadata());
563 }
564 break;
565 }
566
567 case T_FLOAT: {
568 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
569 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
570 } else {
571 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
572 __ ldrs(dest->as_float_reg(), Address(rscratch1));
573 }
574 break;
575 }
576
577 case T_DOUBLE: {
578 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
579 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
580 } else {
581 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
582 __ ldrd(dest->as_double_reg(), Address(rscratch1));
583 }
584 break;
585 }
586
587 default:
588 ShouldNotReachHere();
589 }
590 }
591
592 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
593 LIR_Const* c = src->as_constant_ptr();
594 switch (c->type()) {
595 case T_OBJECT:
596 {
597 if (! c->as_jobject())
598 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
599 else {
600 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr);
601 reg2stack(FrameMap::rscratch1_opr, dest, c->type());
602 }
603 }
604 break;
605 case T_ADDRESS:
606 {
607 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr);
608 reg2stack(FrameMap::rscratch1_opr, dest, c->type());
609 }
610 case T_INT:
611 case T_FLOAT:
612 {
613 Register reg = zr;
614 if (c->as_jint_bits() == 0)
615 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
616 else {
617 __ movw(rscratch1, c->as_jint_bits());
618 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
619 }
620 }
621 break;
622 case T_LONG:
623 case T_DOUBLE:
624 {
625 Register reg = zr;
626 if (c->as_jlong_bits() == 0)
627 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
628 lo_word_offset_in_bytes));
629 else {
630 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
631 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
632 lo_word_offset_in_bytes));
633 }
634 }
635 break;
636 default:
637 ShouldNotReachHere();
638 }
639 }
640
641 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
642 assert(src->is_constant(), "should not call otherwise");
643 LIR_Const* c = src->as_constant_ptr();
644 LIR_Address* to_addr = dest->as_address_ptr();
645
646 void (Assembler::* insn)(Register Rt, const Address &adr);
647
648 switch (type) {
649 case T_ADDRESS:
650 assert(c->as_jint() == 0, "should be");
651 insn = &Assembler::str;
652 break;
653 case T_LONG:
654 assert(c->as_jlong() == 0, "should be");
655 insn = &Assembler::str;
656 break;
657 case T_INT:
658 assert(c->as_jint() == 0, "should be");
659 insn = &Assembler::strw;
660 break;
661 case T_OBJECT:
662 case T_ARRAY:
663 assert(c->as_jobject() == nullptr, "should be");
664 if (UseCompressedOops && !wide) {
665 insn = &Assembler::strw;
666 } else {
667 insn = &Assembler::str;
668 }
669 break;
670 case T_CHAR:
671 case T_SHORT:
672 assert(c->as_jint() == 0, "should be");
673 insn = &Assembler::strh;
674 break;
675 case T_BOOLEAN:
676 case T_BYTE:
677 assert(c->as_jint() == 0, "should be");
678 insn = &Assembler::strb;
679 break;
680 default:
681 ShouldNotReachHere();
682 insn = &Assembler::str; // unreachable
683 }
684
685 if (info) add_debug_info_for_null_check_here(info);
686 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
687 }
688
689 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
690 assert(src->is_register(), "should not call otherwise");
691 assert(dest->is_register(), "should not call otherwise");
692
693 // move between cpu-registers
694 if (dest->is_single_cpu()) {
695 if (src->type() == T_LONG) {
696 // Can do LONG -> OBJECT
697 move_regs(src->as_register_lo(), dest->as_register());
698 return;
699 }
700 assert(src->is_single_cpu(), "must match");
701 if (src->type() == T_OBJECT) {
702 __ verify_oop(src->as_register());
703 }
704 move_regs(src->as_register(), dest->as_register());
705
706 } else if (dest->is_double_cpu()) {
707 if (is_reference_type(src->type())) {
708 // Surprising to me but we can see move of a long to t_object
709 __ verify_oop(src->as_register());
710 move_regs(src->as_register(), dest->as_register_lo());
711 return;
712 }
713 assert(src->is_double_cpu(), "must match");
714 Register f_lo = src->as_register_lo();
715 Register f_hi = src->as_register_hi();
716 Register t_lo = dest->as_register_lo();
717 Register t_hi = dest->as_register_hi();
718 assert(f_hi == f_lo, "must be same");
719 assert(t_hi == t_lo, "must be same");
720 move_regs(f_lo, t_lo);
721
722 } else if (dest->is_single_fpu()) {
723 __ fmovs(dest->as_float_reg(), src->as_float_reg());
724
725 } else if (dest->is_double_fpu()) {
726 __ fmovd(dest->as_double_reg(), src->as_double_reg());
727
728 } else {
729 ShouldNotReachHere();
730 }
731 }
732
733 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
734 precond(src->is_register() && dest->is_stack());
735
736 uint const c_sz32 = sizeof(uint32_t);
737 uint const c_sz64 = sizeof(uint64_t);
738
739 if (src->is_single_cpu()) {
740 int index = dest->single_stack_ix();
741 if (is_reference_type(type)) {
742 __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
743 __ verify_oop(src->as_register());
744 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
745 __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
746 } else {
747 __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1));
748 }
749
750 } else if (src->is_double_cpu()) {
751 int index = dest->double_stack_ix();
752 Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
753 __ str(src->as_register_lo(), dest_addr_LO);
754
755 } else if (src->is_single_fpu()) {
756 int index = dest->single_stack_ix();
757 __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
758
759 } else if (src->is_double_fpu()) {
760 int index = dest->double_stack_ix();
761 __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
762
763 } else {
764 ShouldNotReachHere();
765 }
766 }
767
768
769 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
770 LIR_Address* to_addr = dest->as_address_ptr();
771 PatchingStub* patch = nullptr;
772 Register compressed_src = rscratch1;
773
774 if (patch_code != lir_patch_none) {
775 deoptimize_trap(info);
776 return;
777 }
778
779 if (is_reference_type(type)) {
780 __ verify_oop(src->as_register());
781
782 if (UseCompressedOops && !wide) {
783 __ encode_heap_oop(compressed_src, src->as_register());
784 } else {
785 compressed_src = src->as_register();
786 }
787 }
788
789 int null_check_here = code_offset();
790 switch (type) {
791 case T_FLOAT: {
792 __ strs(src->as_float_reg(), as_Address(to_addr));
793 break;
794 }
795
796 case T_DOUBLE: {
797 __ strd(src->as_double_reg(), as_Address(to_addr));
798 break;
799 }
800
801 case T_ARRAY: // fall through
802 case T_OBJECT: // fall through
803 if (UseCompressedOops && !wide) {
804 __ strw(compressed_src, as_Address(to_addr, rscratch2));
805 } else {
806 __ str(compressed_src, as_Address(to_addr));
807 }
808 break;
809 case T_METADATA:
810 // We get here to store a method pointer to the stack to pass to
811 // a dtrace runtime call. This can't work on 64 bit with
812 // compressed klass ptrs: T_METADATA can be a compressed klass
813 // ptr or a 64 bit method pointer.
814 ShouldNotReachHere();
815 __ str(src->as_register(), as_Address(to_addr));
816 break;
817 case T_ADDRESS:
818 __ str(src->as_register(), as_Address(to_addr));
819 break;
820 case T_INT:
821 __ strw(src->as_register(), as_Address(to_addr));
822 break;
823
824 case T_LONG: {
825 __ str(src->as_register_lo(), as_Address_lo(to_addr));
826 break;
827 }
828
829 case T_BYTE: // fall through
830 case T_BOOLEAN: {
831 __ strb(src->as_register(), as_Address(to_addr));
832 break;
833 }
834
835 case T_CHAR: // fall through
836 case T_SHORT:
837 __ strh(src->as_register(), as_Address(to_addr));
838 break;
839
840 default:
841 ShouldNotReachHere();
842 }
843 if (info != nullptr) {
844 add_debug_info_for_null_check(null_check_here, info);
845 }
846 }
847
848
849 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
850 precond(src->is_stack() && dest->is_register());
851
852 uint const c_sz32 = sizeof(uint32_t);
853 uint const c_sz64 = sizeof(uint64_t);
854
855 if (dest->is_single_cpu()) {
856 int index = src->single_stack_ix();
857 if (is_reference_type(type)) {
858 __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
859 __ verify_oop(dest->as_register());
860 } else if (type == T_METADATA || type == T_ADDRESS) {
861 __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
862 } else {
863 __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1));
864 }
865
866 } else if (dest->is_double_cpu()) {
867 int index = src->double_stack_ix();
868 Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
869 __ ldr(dest->as_register_lo(), src_addr_LO);
870
871 } else if (dest->is_single_fpu()) {
872 int index = src->single_stack_ix();
873 __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
874
875 } else if (dest->is_double_fpu()) {
876 int index = src->double_stack_ix();
877 __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
878
879 } else {
880 ShouldNotReachHere();
881 }
882 }
883
884
885 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
886 address target = nullptr;
887 relocInfo::relocType reloc_type = relocInfo::none;
888
889 switch (patching_id(info)) {
890 case PatchingStub::access_field_id:
891 target = Runtime1::entry_for(StubId::c1_access_field_patching_id);
892 reloc_type = relocInfo::section_word_type;
893 break;
894 case PatchingStub::load_klass_id:
895 target = Runtime1::entry_for(StubId::c1_load_klass_patching_id);
896 reloc_type = relocInfo::metadata_type;
897 break;
898 case PatchingStub::load_mirror_id:
899 target = Runtime1::entry_for(StubId::c1_load_mirror_patching_id);
900 reloc_type = relocInfo::oop_type;
901 break;
902 case PatchingStub::load_appendix_id:
903 target = Runtime1::entry_for(StubId::c1_load_appendix_patching_id);
904 reloc_type = relocInfo::oop_type;
905 break;
906 default: ShouldNotReachHere();
907 }
908
909 __ far_call(RuntimeAddress(target));
910 add_call_info_here(info);
911 }
912
913 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
914
915 LIR_Opr temp;
916 if (type == T_LONG || type == T_DOUBLE)
917 temp = FrameMap::rscratch1_long_opr;
918 else
919 temp = FrameMap::rscratch1_opr;
920
921 stack2reg(src, temp, src->type());
922 reg2stack(temp, dest, dest->type());
923 }
924
925
926 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
927 LIR_Address* addr = src->as_address_ptr();
928 LIR_Address* from_addr = src->as_address_ptr();
929
930 if (addr->base()->type() == T_OBJECT) {
931 __ verify_oop(addr->base()->as_pointer_register());
932 }
933
934 if (patch_code != lir_patch_none) {
935 deoptimize_trap(info);
936 return;
937 }
938
939 if (info != nullptr) {
940 add_debug_info_for_null_check_here(info);
941 }
942 int null_check_here = code_offset();
943 switch (type) {
944 case T_FLOAT: {
945 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
946 break;
947 }
948
949 case T_DOUBLE: {
950 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
951 break;
952 }
953
954 case T_ARRAY: // fall through
955 case T_OBJECT: // fall through
956 if (UseCompressedOops && !wide) {
957 __ ldrw(dest->as_register(), as_Address(from_addr));
958 } else {
959 __ ldr(dest->as_register(), as_Address(from_addr));
960 }
961 break;
962 case T_METADATA:
963 // We get here to store a method pointer to the stack to pass to
964 // a dtrace runtime call. This can't work on 64 bit with
965 // compressed klass ptrs: T_METADATA can be a compressed klass
966 // ptr or a 64 bit method pointer.
967 ShouldNotReachHere();
968 __ ldr(dest->as_register(), as_Address(from_addr));
969 break;
970 case T_ADDRESS:
971 __ ldr(dest->as_register(), as_Address(from_addr));
972 break;
973 case T_INT:
974 __ ldrw(dest->as_register(), as_Address(from_addr));
975 break;
976
977 case T_LONG: {
978 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
979 break;
980 }
981
982 case T_BYTE:
983 __ ldrsb(dest->as_register(), as_Address(from_addr));
984 break;
985 case T_BOOLEAN: {
986 __ ldrb(dest->as_register(), as_Address(from_addr));
987 break;
988 }
989
990 case T_CHAR:
991 __ ldrh(dest->as_register(), as_Address(from_addr));
992 break;
993 case T_SHORT:
994 __ ldrsh(dest->as_register(), as_Address(from_addr));
995 break;
996
997 default:
998 ShouldNotReachHere();
999 }
1000
1001 if (is_reference_type(type)) {
1002 if (UseCompressedOops && !wide) {
1003 __ decode_heap_oop(dest->as_register());
1004 }
1005
1006 __ verify_oop(dest->as_register());
1007 }
1008 }
1009
1010
1011 int LIR_Assembler::array_element_size(BasicType type) const {
1012 int elem_size = type2aelembytes(type);
1013 return exact_log2(elem_size);
1014 }
1015
1016
1017 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1018 switch (op->code()) {
1019 case lir_idiv:
1020 case lir_irem:
1021 arithmetic_idiv(op->code(),
1022 op->in_opr1(),
1023 op->in_opr2(),
1024 op->in_opr3(),
1025 op->result_opr(),
1026 op->info());
1027 break;
1028 case lir_fmad:
1029 __ fmaddd(op->result_opr()->as_double_reg(),
1030 op->in_opr1()->as_double_reg(),
1031 op->in_opr2()->as_double_reg(),
1032 op->in_opr3()->as_double_reg());
1033 break;
1034 case lir_fmaf:
1035 __ fmadds(op->result_opr()->as_float_reg(),
1036 op->in_opr1()->as_float_reg(),
1037 op->in_opr2()->as_float_reg(),
1038 op->in_opr3()->as_float_reg());
1039 break;
1040 default: ShouldNotReachHere(); break;
1041 }
1042 }
1043
1044 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1045 #ifdef ASSERT
1046 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1047 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1048 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1049 #endif
1050
1051 if (op->cond() == lir_cond_always) {
1052 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1053 __ b(*(op->label()));
1054 } else {
1055 Assembler::Condition acond;
1056 if (op->code() == lir_cond_float_branch) {
1057 bool is_unordered = (op->ublock() == op->block());
1058 // Assembler::EQ does not permit unordered branches, so we add
1059 // another branch here. Likewise, Assembler::NE does not permit
1060 // ordered branches.
1061 if ((is_unordered && op->cond() == lir_cond_equal)
1062 || (!is_unordered && op->cond() == lir_cond_notEqual))
1063 __ br(Assembler::VS, *(op->ublock()->label()));
1064 switch(op->cond()) {
1065 case lir_cond_equal: acond = Assembler::EQ; break;
1066 case lir_cond_notEqual: acond = Assembler::NE; break;
1067 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1068 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1069 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1070 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1071 default: ShouldNotReachHere();
1072 acond = Assembler::EQ; // unreachable
1073 }
1074 } else {
1075 switch (op->cond()) {
1076 case lir_cond_equal: acond = Assembler::EQ; break;
1077 case lir_cond_notEqual: acond = Assembler::NE; break;
1078 case lir_cond_less: acond = Assembler::LT; break;
1079 case lir_cond_lessEqual: acond = Assembler::LE; break;
1080 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1081 case lir_cond_greater: acond = Assembler::GT; break;
1082 case lir_cond_belowEqual: acond = Assembler::LS; break;
1083 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1084 default: ShouldNotReachHere();
1085 acond = Assembler::EQ; // unreachable
1086 }
1087 }
1088 __ br(acond,*(op->label()));
1089 }
1090 }
1091
1092
1093
1094 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1095 LIR_Opr src = op->in_opr();
1096 LIR_Opr dest = op->result_opr();
1097
1098 switch (op->bytecode()) {
1099 case Bytecodes::_i2f:
1100 {
1101 __ scvtfws(dest->as_float_reg(), src->as_register());
1102 break;
1103 }
1104 case Bytecodes::_i2d:
1105 {
1106 __ scvtfwd(dest->as_double_reg(), src->as_register());
1107 break;
1108 }
1109 case Bytecodes::_l2d:
1110 {
1111 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1112 break;
1113 }
1114 case Bytecodes::_l2f:
1115 {
1116 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1117 break;
1118 }
1119 case Bytecodes::_f2d:
1120 {
1121 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1122 break;
1123 }
1124 case Bytecodes::_d2f:
1125 {
1126 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1127 break;
1128 }
1129 case Bytecodes::_i2c:
1130 {
1131 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1132 break;
1133 }
1134 case Bytecodes::_i2l:
1135 {
1136 __ sxtw(dest->as_register_lo(), src->as_register());
1137 break;
1138 }
1139 case Bytecodes::_i2s:
1140 {
1141 __ sxth(dest->as_register(), src->as_register());
1142 break;
1143 }
1144 case Bytecodes::_i2b:
1145 {
1146 __ sxtb(dest->as_register(), src->as_register());
1147 break;
1148 }
1149 case Bytecodes::_l2i:
1150 {
1151 _masm->block_comment("FIXME: This could be a no-op");
1152 __ uxtw(dest->as_register(), src->as_register_lo());
1153 break;
1154 }
1155 case Bytecodes::_d2l:
1156 {
1157 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1158 break;
1159 }
1160 case Bytecodes::_f2i:
1161 {
1162 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1163 break;
1164 }
1165 case Bytecodes::_f2l:
1166 {
1167 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1168 break;
1169 }
1170 case Bytecodes::_d2i:
1171 {
1172 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1173 break;
1174 }
1175 default: ShouldNotReachHere();
1176 }
1177 }
1178
1179 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1180 if (op->init_check()) {
1181 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1182 __ ldarb(rscratch1, rscratch1);
1183 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1184 add_debug_info_for_null_check_here(op->stub()->info());
1185 __ br(Assembler::NE, *op->stub()->entry());
1186 }
1187 __ allocate_object(op->obj()->as_register(),
1188 op->tmp1()->as_register(),
1189 op->tmp2()->as_register(),
1190 op->header_size(),
1191 op->object_size(),
1192 op->klass()->as_register(),
1193 *op->stub()->entry());
1194 __ bind(*op->stub()->continuation());
1195 }
1196
1197 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1198 Register len = op->len()->as_register();
1199 __ uxtw(len, len);
1200
1201 if (UseSlowPath ||
1202 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1203 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1204 __ b(*op->stub()->entry());
1205 } else {
1206 Register tmp1 = op->tmp1()->as_register();
1207 Register tmp2 = op->tmp2()->as_register();
1208 Register tmp3 = op->tmp3()->as_register();
1209 if (len == tmp1) {
1210 tmp1 = tmp3;
1211 } else if (len == tmp2) {
1212 tmp2 = tmp3;
1213 } else if (len == tmp3) {
1214 // everything is ok
1215 } else {
1216 __ mov(tmp3, len);
1217 }
1218 __ allocate_array(op->obj()->as_register(),
1219 len,
1220 tmp1,
1221 tmp2,
1222 arrayOopDesc::base_offset_in_bytes(op->type()),
1223 array_element_size(op->type()),
1224 op->klass()->as_register(),
1225 *op->stub()->entry(),
1226 op->zero_array());
1227 }
1228 __ bind(*op->stub()->continuation());
1229 }
1230
1231 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md,
1232 ciProfileData *data, Register recv) {
1233
1234 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1235 __ profile_receiver_type(recv, mdo, mdp_offset);
1236 }
1237
1238 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1239 // we always need a stub for the failure case.
1240 CodeStub* stub = op->stub();
1241 Register obj = op->object()->as_register();
1242 Register k_RInfo = op->tmp1()->as_register();
1243 Register klass_RInfo = op->tmp2()->as_register();
1244 Register dst = op->result_opr()->as_register();
1245 ciKlass* k = op->klass();
1246 Register Rtmp1 = noreg;
1247
1248 // check if it needs to be profiled
1249 ciMethodData* md;
1250 ciProfileData* data;
1251
1252 const bool should_profile = op->should_profile();
1253
1254 if (should_profile) {
1255 ciMethod* method = op->profiled_method();
1256 assert(method != nullptr, "Should have method");
1257 int bci = op->profiled_bci();
1258 md = method->method_data_or_null();
1259 assert(md != nullptr, "Sanity");
1260 data = md->bci_to_data(bci);
1261 assert(data != nullptr, "need data for type check");
1262 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1263 }
1264 Label* success_target = success;
1265 Label* failure_target = failure;
1266
1267 if (obj == k_RInfo) {
1268 k_RInfo = dst;
1269 } else if (obj == klass_RInfo) {
1270 klass_RInfo = dst;
1271 }
1272 if (k->is_loaded() && !UseCompressedClassPointers) {
1273 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1274 } else {
1275 Rtmp1 = op->tmp3()->as_register();
1276 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1277 }
1278
1279 assert_different_registers(obj, k_RInfo, klass_RInfo);
1280
1281 if (should_profile) {
1282 Register mdo = klass_RInfo;
1283 __ mov_metadata(mdo, md->constant_encoding());
1284 Label not_null;
1285 __ cbnz(obj, not_null);
1286 // Object is null; update MDO and exit
1287 Address data_addr
1288 = __ form_address(rscratch2, mdo,
1289 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1290 0);
1291 __ ldrb(rscratch1, data_addr);
1292 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1293 __ strb(rscratch1, data_addr);
1294 __ b(*obj_is_null);
1295 __ bind(not_null);
1296
1297 Register recv = k_RInfo;
1298 __ load_klass(recv, obj);
1299 type_profile_helper(mdo, md, data, recv);
1300 } else {
1301 __ cbz(obj, *obj_is_null);
1302 }
1303
1304 if (!k->is_loaded()) {
1305 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1306 } else {
1307 __ mov_metadata(k_RInfo, k->constant_encoding());
1308 }
1309 __ verify_oop(obj);
1310
1311 if (op->fast_check()) {
1312 // get object class
1313 // not a safepoint as obj null check happens earlier
1314 __ load_klass(rscratch1, obj);
1315 __ cmp( rscratch1, k_RInfo);
1316
1317 __ br(Assembler::NE, *failure_target);
1318 // successful cast, fall through to profile or jump
1319 } else {
1320 // get object class
1321 // not a safepoint as obj null check happens earlier
1322 __ load_klass(klass_RInfo, obj);
1323 if (k->is_loaded()) {
1324 // See if we get an immediate positive hit
1325 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1326 __ cmp(k_RInfo, rscratch1);
1327 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1328 __ br(Assembler::NE, *failure_target);
1329 // successful cast, fall through to profile or jump
1330 } else {
1331 // See if we get an immediate positive hit
1332 __ br(Assembler::EQ, *success_target);
1333 // check for self
1334 __ cmp(klass_RInfo, k_RInfo);
1335 __ br(Assembler::EQ, *success_target);
1336
1337 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1338 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1339 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1340 // result is a boolean
1341 __ cbzw(klass_RInfo, *failure_target);
1342 // successful cast, fall through to profile or jump
1343 }
1344 } else {
1345 // perform the fast part of the checking logic
1346 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1347 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1348 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1349 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1350 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1351 // result is a boolean
1352 __ cbz(k_RInfo, *failure_target);
1353 // successful cast, fall through to profile or jump
1354 }
1355 }
1356 __ b(*success);
1357 }
1358
1359
1360 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1361 const bool should_profile = op->should_profile();
1362
1363 LIR_Code code = op->code();
1364 if (code == lir_store_check) {
1365 Register value = op->object()->as_register();
1366 Register array = op->array()->as_register();
1367 Register k_RInfo = op->tmp1()->as_register();
1368 Register klass_RInfo = op->tmp2()->as_register();
1369 Register Rtmp1 = op->tmp3()->as_register();
1370
1371 CodeStub* stub = op->stub();
1372
1373 // check if it needs to be profiled
1374 ciMethodData* md;
1375 ciProfileData* data;
1376
1377 if (should_profile) {
1378 ciMethod* method = op->profiled_method();
1379 assert(method != nullptr, "Should have method");
1380 int bci = op->profiled_bci();
1381 md = method->method_data_or_null();
1382 assert(md != nullptr, "Sanity");
1383 data = md->bci_to_data(bci);
1384 assert(data != nullptr, "need data for type check");
1385 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1386 }
1387 Label done;
1388 Label* success_target = &done;
1389 Label* failure_target = stub->entry();
1390
1391 if (should_profile) {
1392 Label not_null;
1393 Register mdo = klass_RInfo;
1394 __ mov_metadata(mdo, md->constant_encoding());
1395 __ cbnz(value, not_null);
1396 // Object is null; update MDO and exit
1397 Address data_addr
1398 = __ form_address(rscratch2, mdo,
1399 md->byte_offset_of_slot(data, DataLayout::flags_offset()), 0);
1400 __ ldrb(rscratch1, data_addr);
1401 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1402 __ strb(rscratch1, data_addr);
1403 __ b(done);
1404 __ bind(not_null);
1405
1406 Register recv = k_RInfo;
1407 __ load_klass(recv, value);
1408 type_profile_helper(mdo, md, data, recv);
1409 } else {
1410 __ cbz(value, done);
1411 }
1412
1413 add_debug_info_for_null_check_here(op->info_for_exception());
1414 __ load_klass(k_RInfo, array);
1415 __ load_klass(klass_RInfo, value);
1416
1417 // get instance klass (it's already uncompressed)
1418 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1419 // perform the fast part of the checking logic
1420 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1421 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1422 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1423 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1424 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1425 // result is a boolean
1426 __ cbzw(k_RInfo, *failure_target);
1427 // fall through to the success case
1428
1429 __ bind(done);
1430 } else if (code == lir_checkcast) {
1431 Register obj = op->object()->as_register();
1432 Register dst = op->result_opr()->as_register();
1433 Label success;
1434 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1435 __ bind(success);
1436 if (dst != obj) {
1437 __ mov(dst, obj);
1438 }
1439 } else if (code == lir_instanceof) {
1440 Register obj = op->object()->as_register();
1441 Register dst = op->result_opr()->as_register();
1442 Label success, failure, done;
1443 emit_typecheck_helper(op, &success, &failure, &failure);
1444 __ bind(failure);
1445 __ mov(dst, zr);
1446 __ b(done);
1447 __ bind(success);
1448 __ mov(dst, 1);
1449 __ bind(done);
1450 } else {
1451 ShouldNotReachHere();
1452 }
1453 }
1454
1455 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1456 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1457 __ cset(rscratch1, Assembler::NE);
1458 __ membar(__ AnyAny);
1459 }
1460
1461 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1462 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1463 __ cset(rscratch1, Assembler::NE);
1464 __ membar(__ AnyAny);
1465 }
1466
1467
1468 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1469 Register addr;
1470 if (op->addr()->is_register()) {
1471 addr = as_reg(op->addr());
1472 } else {
1473 assert(op->addr()->is_address(), "what else?");
1474 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1475 assert(addr_ptr->disp() == 0, "need 0 disp");
1476 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1477 addr = as_reg(addr_ptr->base());
1478 }
1479 Register newval = as_reg(op->new_value());
1480 Register cmpval = as_reg(op->cmp_value());
1481
1482 if (op->code() == lir_cas_obj) {
1483 if (UseCompressedOops) {
1484 Register t1 = op->tmp1()->as_register();
1485 assert(op->tmp1()->is_valid(), "must be");
1486 __ encode_heap_oop(t1, cmpval);
1487 cmpval = t1;
1488 __ encode_heap_oop(rscratch2, newval);
1489 newval = rscratch2;
1490 casw(addr, newval, cmpval);
1491 } else {
1492 casl(addr, newval, cmpval);
1493 }
1494 } else if (op->code() == lir_cas_int) {
1495 casw(addr, newval, cmpval);
1496 } else {
1497 casl(addr, newval, cmpval);
1498 }
1499 }
1500
1501
1502 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1503 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1504 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on aarch64");
1505
1506 Assembler::Condition acond, ncond;
1507 switch (condition) {
1508 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1509 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1510 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1511 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1512 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1513 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1514 case lir_cond_belowEqual:
1515 case lir_cond_aboveEqual:
1516 default: ShouldNotReachHere();
1517 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1518 }
1519
1520 assert(result->is_single_cpu() || result->is_double_cpu(),
1521 "expect single register for result");
1522 if (opr1->is_constant() && opr2->is_constant()
1523 && opr1->type() == T_INT && opr2->type() == T_INT) {
1524 jint val1 = opr1->as_jint();
1525 jint val2 = opr2->as_jint();
1526 if (val1 == 0 && val2 == 1) {
1527 __ cset(result->as_register(), ncond);
1528 return;
1529 } else if (val1 == 1 && val2 == 0) {
1530 __ cset(result->as_register(), acond);
1531 return;
1532 }
1533 }
1534
1535 if (opr1->is_constant() && opr2->is_constant()
1536 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1537 jlong val1 = opr1->as_jlong();
1538 jlong val2 = opr2->as_jlong();
1539 if (val1 == 0 && val2 == 1) {
1540 __ cset(result->as_register_lo(), ncond);
1541 return;
1542 } else if (val1 == 1 && val2 == 0) {
1543 __ cset(result->as_register_lo(), acond);
1544 return;
1545 }
1546 }
1547
1548 if (opr1->is_stack()) {
1549 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1550 opr1 = FrameMap::rscratch1_opr;
1551 } else if (opr1->is_constant()) {
1552 LIR_Opr tmp
1553 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1554 const2reg(opr1, tmp, lir_patch_none, nullptr);
1555 opr1 = tmp;
1556 }
1557
1558 if (opr2->is_stack()) {
1559 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1560 opr2 = FrameMap::rscratch2_opr;
1561 } else if (opr2->is_constant()) {
1562 LIR_Opr tmp
1563 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1564 const2reg(opr2, tmp, lir_patch_none, nullptr);
1565 opr2 = tmp;
1566 }
1567
1568 if (result->type() == T_LONG)
1569 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1570 else
1571 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1572 }
1573
1574 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1575 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1576
1577 if (left->is_single_cpu()) {
1578 Register lreg = left->as_register();
1579 Register dreg = as_reg(dest);
1580
1581 if (right->is_single_cpu()) {
1582 // cpu register - cpu register
1583
1584 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1585 "should be");
1586 Register rreg = right->as_register();
1587 switch (code) {
1588 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1589 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1590 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1591 default: ShouldNotReachHere();
1592 }
1593
1594 } else if (right->is_double_cpu()) {
1595 Register rreg = right->as_register_lo();
1596 // single_cpu + double_cpu: can happen with obj+long
1597 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1598 switch (code) {
1599 case lir_add: __ add(dreg, lreg, rreg); break;
1600 case lir_sub: __ sub(dreg, lreg, rreg); break;
1601 default: ShouldNotReachHere();
1602 }
1603 } else if (right->is_constant()) {
1604 // cpu register - constant
1605 jlong c;
1606
1607 // FIXME. This is fugly: we really need to factor all this logic.
1608 switch(right->type()) {
1609 case T_LONG:
1610 c = right->as_constant_ptr()->as_jlong();
1611 break;
1612 case T_INT:
1613 case T_ADDRESS:
1614 c = right->as_constant_ptr()->as_jint();
1615 break;
1616 default:
1617 ShouldNotReachHere();
1618 c = 0; // unreachable
1619 break;
1620 }
1621
1622 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1623 if (c == 0 && dreg == lreg) {
1624 COMMENT("effective nop elided");
1625 return;
1626 }
1627 switch(left->type()) {
1628 case T_INT:
1629 switch (code) {
1630 case lir_add: __ addw(dreg, lreg, c); break;
1631 case lir_sub: __ subw(dreg, lreg, c); break;
1632 default: ShouldNotReachHere();
1633 }
1634 break;
1635 case T_OBJECT:
1636 case T_ADDRESS:
1637 switch (code) {
1638 case lir_add: __ add(dreg, lreg, c); break;
1639 case lir_sub: __ sub(dreg, lreg, c); break;
1640 default: ShouldNotReachHere();
1641 }
1642 break;
1643 default:
1644 ShouldNotReachHere();
1645 }
1646 } else {
1647 ShouldNotReachHere();
1648 }
1649
1650 } else if (left->is_double_cpu()) {
1651 Register lreg_lo = left->as_register_lo();
1652
1653 if (right->is_double_cpu()) {
1654 // cpu register - cpu register
1655 Register rreg_lo = right->as_register_lo();
1656 switch (code) {
1657 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1658 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1659 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1660 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1661 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1662 default:
1663 ShouldNotReachHere();
1664 }
1665
1666 } else if (right->is_constant()) {
1667 jlong c = right->as_constant_ptr()->as_jlong();
1668 Register dreg = as_reg(dest);
1669 switch (code) {
1670 case lir_add:
1671 case lir_sub:
1672 if (c == 0 && dreg == lreg_lo) {
1673 COMMENT("effective nop elided");
1674 return;
1675 }
1676 code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1677 break;
1678 case lir_div:
1679 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1680 if (c == 1) {
1681 // move lreg_lo to dreg if divisor is 1
1682 __ mov(dreg, lreg_lo);
1683 } else {
1684 unsigned int shift = log2i_exact(c);
1685 // use rscratch1 as intermediate result register
1686 __ asr(rscratch1, lreg_lo, 63);
1687 __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1688 __ asr(dreg, rscratch1, shift);
1689 }
1690 break;
1691 case lir_rem:
1692 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1693 if (c == 1) {
1694 // move 0 to dreg if divisor is 1
1695 __ mov(dreg, zr);
1696 } else {
1697 // use rscratch1 as intermediate result register
1698 __ negs(rscratch1, lreg_lo);
1699 __ andr(dreg, lreg_lo, c - 1);
1700 __ andr(rscratch1, rscratch1, c - 1);
1701 __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1702 }
1703 break;
1704 default:
1705 ShouldNotReachHere();
1706 }
1707 } else {
1708 ShouldNotReachHere();
1709 }
1710 } else if (left->is_single_fpu()) {
1711 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1712 switch (code) {
1713 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1714 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1715 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1716 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1717 default:
1718 ShouldNotReachHere();
1719 }
1720 } else if (left->is_double_fpu()) {
1721 if (right->is_double_fpu()) {
1722 // fpu register - fpu register
1723 switch (code) {
1724 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1725 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1726 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1727 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1728 default:
1729 ShouldNotReachHere();
1730 }
1731 } else {
1732 if (right->is_constant()) {
1733 ShouldNotReachHere();
1734 }
1735 ShouldNotReachHere();
1736 }
1737 } else if (left->is_single_stack() || left->is_address()) {
1738 assert(left == dest, "left and dest must be equal");
1739 ShouldNotReachHere();
1740 } else {
1741 ShouldNotReachHere();
1742 }
1743 }
1744
1745 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1746 switch(code) {
1747 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1748 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1749 case lir_f2hf: __ flt_to_flt16(dest->as_register(), value->as_float_reg(), tmp->as_float_reg()); break;
1750 case lir_hf2f: __ flt16_to_flt(dest->as_float_reg(), value->as_register(), tmp->as_float_reg()); break;
1751 default : ShouldNotReachHere();
1752 }
1753 }
1754
1755 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1756
1757 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1758 Register Rleft = left->is_single_cpu() ? left->as_register() :
1759 left->as_register_lo();
1760 if (dst->is_single_cpu()) {
1761 Register Rdst = dst->as_register();
1762 if (right->is_constant()) {
1763 switch (code) {
1764 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1765 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1766 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1767 default: ShouldNotReachHere(); break;
1768 }
1769 } else {
1770 Register Rright = right->is_single_cpu() ? right->as_register() :
1771 right->as_register_lo();
1772 switch (code) {
1773 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1774 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1775 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1776 default: ShouldNotReachHere(); break;
1777 }
1778 }
1779 } else {
1780 Register Rdst = dst->as_register_lo();
1781 if (right->is_constant()) {
1782 switch (code) {
1783 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1784 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1785 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1786 default: ShouldNotReachHere(); break;
1787 }
1788 } else {
1789 Register Rright = right->is_single_cpu() ? right->as_register() :
1790 right->as_register_lo();
1791 switch (code) {
1792 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1793 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1794 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1795 default: ShouldNotReachHere(); break;
1796 }
1797 }
1798 }
1799 }
1800
1801
1802
1803 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
1804
1805 // opcode check
1806 assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
1807 bool is_irem = (code == lir_irem);
1808
1809 // operand check
1810 assert(left->is_single_cpu(), "left must be register");
1811 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
1812 assert(result->is_single_cpu(), "result must be register");
1813 Register lreg = left->as_register();
1814 Register dreg = result->as_register();
1815
1816 // power-of-2 constant check and codegen
1817 if (right->is_constant()) {
1818 int c = right->as_constant_ptr()->as_jint();
1819 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1820 if (is_irem) {
1821 if (c == 1) {
1822 // move 0 to dreg if divisor is 1
1823 __ movw(dreg, zr);
1824 } else {
1825 // use rscratch1 as intermediate result register
1826 __ negsw(rscratch1, lreg);
1827 __ andw(dreg, lreg, c - 1);
1828 __ andw(rscratch1, rscratch1, c - 1);
1829 __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
1830 }
1831 } else {
1832 if (c == 1) {
1833 // move lreg to dreg if divisor is 1
1834 __ movw(dreg, lreg);
1835 } else {
1836 unsigned int shift = exact_log2(c);
1837 // use rscratch1 as intermediate result register
1838 __ asrw(rscratch1, lreg, 31);
1839 __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
1840 __ asrw(dreg, rscratch1, shift);
1841 }
1842 }
1843 } else {
1844 Register rreg = right->as_register();
1845 __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
1846 }
1847 }
1848
1849
1850 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1851 if (opr1->is_constant() && opr2->is_single_cpu()) {
1852 // tableswitch
1853 Register reg = as_reg(opr2);
1854 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1855 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1856 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1857 Register reg1 = as_reg(opr1);
1858 if (opr2->is_single_cpu()) {
1859 // cpu register - cpu register
1860 Register reg2 = opr2->as_register();
1861 if (is_reference_type(opr1->type())) {
1862 __ cmpoop(reg1, reg2);
1863 } else {
1864 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1865 __ cmpw(reg1, reg2);
1866 }
1867 return;
1868 }
1869 if (opr2->is_double_cpu()) {
1870 // cpu register - cpu register
1871 Register reg2 = opr2->as_register_lo();
1872 __ cmp(reg1, reg2);
1873 return;
1874 }
1875
1876 if (opr2->is_constant()) {
1877 bool is_32bit = false; // width of register operand
1878 jlong imm;
1879
1880 switch(opr2->type()) {
1881 case T_INT:
1882 imm = opr2->as_constant_ptr()->as_jint();
1883 is_32bit = true;
1884 break;
1885 case T_LONG:
1886 imm = opr2->as_constant_ptr()->as_jlong();
1887 break;
1888 case T_ADDRESS:
1889 imm = opr2->as_constant_ptr()->as_jint();
1890 break;
1891 case T_METADATA:
1892 imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
1893 break;
1894 case T_OBJECT:
1895 case T_ARRAY:
1896 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1897 __ cmpoop(reg1, rscratch1);
1898 return;
1899 default:
1900 ShouldNotReachHere();
1901 imm = 0; // unreachable
1902 break;
1903 }
1904
1905 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1906 if (is_32bit)
1907 __ cmpw(reg1, imm);
1908 else
1909 __ subs(zr, reg1, imm);
1910 return;
1911 } else {
1912 __ mov(rscratch1, imm);
1913 if (is_32bit)
1914 __ cmpw(reg1, rscratch1);
1915 else
1916 __ cmp(reg1, rscratch1);
1917 return;
1918 }
1919 } else
1920 ShouldNotReachHere();
1921 } else if (opr1->is_single_fpu()) {
1922 FloatRegister reg1 = opr1->as_float_reg();
1923 assert(opr2->is_single_fpu(), "expect single float register");
1924 FloatRegister reg2 = opr2->as_float_reg();
1925 __ fcmps(reg1, reg2);
1926 } else if (opr1->is_double_fpu()) {
1927 FloatRegister reg1 = opr1->as_double_reg();
1928 assert(opr2->is_double_fpu(), "expect double float register");
1929 FloatRegister reg2 = opr2->as_double_reg();
1930 __ fcmpd(reg1, reg2);
1931 } else {
1932 ShouldNotReachHere();
1933 }
1934 }
1935
1936 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1937 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1938 bool is_unordered_less = (code == lir_ucmp_fd2i);
1939 if (left->is_single_fpu()) {
1940 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1941 } else if (left->is_double_fpu()) {
1942 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1943 } else {
1944 ShouldNotReachHere();
1945 }
1946 } else if (code == lir_cmp_l2i) {
1947 Label done;
1948 __ cmp(left->as_register_lo(), right->as_register_lo());
1949 __ mov(dst->as_register(), (uint64_t)-1L);
1950 __ br(Assembler::LT, done);
1951 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1952 __ bind(done);
1953 } else {
1954 ShouldNotReachHere();
1955 }
1956 }
1957
1958
1959 void LIR_Assembler::align_call(LIR_Code code) { }
1960
1961
1962 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1963 address call = __ trampoline_call(Address(op->addr(), rtype));
1964 if (call == nullptr) {
1965 bailout("trampoline stub overflow");
1966 return;
1967 }
1968 add_call_info(code_offset(), op->info());
1969 __ post_call_nop();
1970 }
1971
1972
1973 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1974 address call = __ ic_call(op->addr());
1975 if (call == nullptr) {
1976 bailout("trampoline stub overflow");
1977 return;
1978 }
1979 add_call_info(code_offset(), op->info());
1980 __ post_call_nop();
1981 }
1982
1983 void LIR_Assembler::emit_static_call_stub() {
1984 address call_pc = __ pc();
1985 address stub = __ start_a_stub(call_stub_size());
1986 if (stub == nullptr) {
1987 bailout("static call stub overflow");
1988 return;
1989 }
1990
1991 int start = __ offset();
1992
1993 __ relocate(static_stub_Relocation::spec(call_pc));
1994 __ emit_static_call_stub();
1995
1996 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
1997 <= call_stub_size(), "stub too big");
1998 __ end_a_stub();
1999 }
2000
2001
2002 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2003 assert(exceptionOop->as_register() == r0, "must match");
2004 assert(exceptionPC->as_register() == r3, "must match");
2005
2006 // exception object is not added to oop map by LinearScan
2007 // (LinearScan assumes that no oops are in fixed registers)
2008 info->add_register_oop(exceptionOop);
2009 StubId unwind_id;
2010
2011 // get current pc information
2012 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2013 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
2014 // As no instructions have been generated yet for this LIR node it's
2015 // possible that an oop map already exists for the current offset.
2016 // In that case insert an dummy NOP here to ensure all oop map PCs
2017 // are unique. See JDK-8237483.
2018 __ nop();
2019 }
2020 int pc_for_athrow_offset = __ offset();
2021 InternalAddress pc_for_athrow(__ pc());
2022 __ adr(exceptionPC->as_register(), pc_for_athrow);
2023 add_call_info(pc_for_athrow_offset, info); // for exception handler
2024
2025 __ verify_not_null_oop(r0);
2026 // search an exception handler (r0: exception oop, r3: throwing pc)
2027 if (compilation()->has_fpu_code()) {
2028 unwind_id = StubId::c1_handle_exception_id;
2029 } else {
2030 unwind_id = StubId::c1_handle_exception_nofpu_id;
2031 }
2032 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2033
2034 // FIXME: enough room for two byte trap ????
2035 __ nop();
2036 }
2037
2038
2039 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2040 assert(exceptionOop->as_register() == r0, "must match");
2041
2042 __ b(_unwind_handler_entry);
2043 }
2044
2045
2046 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2047 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2048 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2049
2050 switch (left->type()) {
2051 case T_INT: {
2052 switch (code) {
2053 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2054 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2055 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2056 default:
2057 ShouldNotReachHere();
2058 break;
2059 }
2060 break;
2061 case T_LONG:
2062 case T_ADDRESS:
2063 case T_OBJECT:
2064 switch (code) {
2065 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2066 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2067 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2068 default:
2069 ShouldNotReachHere();
2070 break;
2071 }
2072 break;
2073 default:
2074 ShouldNotReachHere();
2075 break;
2076 }
2077 }
2078 }
2079
2080
2081 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2082 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2083 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2084
2085 switch (left->type()) {
2086 case T_INT: {
2087 switch (code) {
2088 case lir_shl: __ lslw (dreg, lreg, count); break;
2089 case lir_shr: __ asrw (dreg, lreg, count); break;
2090 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2091 default:
2092 ShouldNotReachHere();
2093 break;
2094 }
2095 break;
2096 case T_LONG:
2097 case T_ADDRESS:
2098 case T_OBJECT:
2099 switch (code) {
2100 case lir_shl: __ lsl (dreg, lreg, count); break;
2101 case lir_shr: __ asr (dreg, lreg, count); break;
2102 case lir_ushr: __ lsr (dreg, lreg, count); break;
2103 default:
2104 ShouldNotReachHere();
2105 break;
2106 }
2107 break;
2108 default:
2109 ShouldNotReachHere();
2110 break;
2111 }
2112 }
2113 }
2114
2115
2116 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2117 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2118 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2119 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2120 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2121 }
2122
2123
2124 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2125 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2126 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2127 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2128 __ mov (rscratch1, c);
2129 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2130 }
2131
2132
2133 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2134 ShouldNotReachHere();
2135 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2136 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2137 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2138 __ lea(rscratch1, __ constant_oop_address(o));
2139 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2140 }
2141
2142
2143 // This code replaces a call to arraycopy; no exception may
2144 // be thrown in this code, they must be thrown in the System.arraycopy
2145 // activation frame; we could save some checks if this would not be the case
2146 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2147 ciArrayKlass* default_type = op->expected_type();
2148 Register src = op->src()->as_register();
2149 Register dst = op->dst()->as_register();
2150 Register src_pos = op->src_pos()->as_register();
2151 Register dst_pos = op->dst_pos()->as_register();
2152 Register length = op->length()->as_register();
2153 Register tmp = op->tmp()->as_register();
2154
2155 CodeStub* stub = op->stub();
2156 int flags = op->flags();
2157 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2158 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2159
2160 // if we don't know anything, just go through the generic arraycopy
2161 if (default_type == nullptr // || basic_type == T_OBJECT
2162 ) {
2163 Label done;
2164 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2165
2166 // Save the arguments in case the generic arraycopy fails and we
2167 // have to fall back to the JNI stub
2168 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2169 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2170 __ str(src, Address(sp, 4*BytesPerWord));
2171
2172 address copyfunc_addr = StubRoutines::generic_arraycopy();
2173 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2174
2175 // The arguments are in java calling convention so we shift them
2176 // to C convention
2177 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2178 __ mov(c_rarg0, j_rarg0);
2179 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2180 __ mov(c_rarg1, j_rarg1);
2181 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2182 __ mov(c_rarg2, j_rarg2);
2183 assert_different_registers(c_rarg3, j_rarg4);
2184 __ mov(c_rarg3, j_rarg3);
2185 __ mov(c_rarg4, j_rarg4);
2186 #ifndef PRODUCT
2187 if (PrintC1Statistics) {
2188 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2189 }
2190 #endif
2191 __ far_call(RuntimeAddress(copyfunc_addr));
2192
2193 __ cbz(r0, *stub->continuation());
2194
2195 // Reload values from the stack so they are where the stub
2196 // expects them.
2197 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2198 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2199 __ ldr(src, Address(sp, 4*BytesPerWord));
2200
2201 // r0 is -1^K where K == partial copied count
2202 __ eonw(rscratch1, r0, zr);
2203 // adjust length down and src/end pos up by partial copied count
2204 __ subw(length, length, rscratch1);
2205 __ addw(src_pos, src_pos, rscratch1);
2206 __ addw(dst_pos, dst_pos, rscratch1);
2207 __ b(*stub->entry());
2208
2209 __ bind(*stub->continuation());
2210 return;
2211 }
2212
2213 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2214
2215 int elem_size = type2aelembytes(basic_type);
2216 int scale = exact_log2(elem_size);
2217
2218 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2219 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2220
2221 // test for null
2222 if (flags & LIR_OpArrayCopy::src_null_check) {
2223 __ cbz(src, *stub->entry());
2224 }
2225 if (flags & LIR_OpArrayCopy::dst_null_check) {
2226 __ cbz(dst, *stub->entry());
2227 }
2228
2229 // If the compiler was not able to prove that exact type of the source or the destination
2230 // of the arraycopy is an array type, check at runtime if the source or the destination is
2231 // an instance type.
2232 if (flags & LIR_OpArrayCopy::type_check) {
2233 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2234 __ load_klass(tmp, dst);
2235 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2236 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2237 __ br(Assembler::GE, *stub->entry());
2238 }
2239
2240 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2241 __ load_klass(tmp, src);
2242 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2243 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2244 __ br(Assembler::GE, *stub->entry());
2245 }
2246 }
2247
2248 // check if negative
2249 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2250 __ cmpw(src_pos, 0);
2251 __ br(Assembler::LT, *stub->entry());
2252 }
2253 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2254 __ cmpw(dst_pos, 0);
2255 __ br(Assembler::LT, *stub->entry());
2256 }
2257
2258 if (flags & LIR_OpArrayCopy::length_positive_check) {
2259 __ cmpw(length, 0);
2260 __ br(Assembler::LT, *stub->entry());
2261 }
2262
2263 if (flags & LIR_OpArrayCopy::src_range_check) {
2264 __ addw(tmp, src_pos, length);
2265 __ ldrw(rscratch1, src_length_addr);
2266 __ cmpw(tmp, rscratch1);
2267 __ br(Assembler::HI, *stub->entry());
2268 }
2269 if (flags & LIR_OpArrayCopy::dst_range_check) {
2270 __ addw(tmp, dst_pos, length);
2271 __ ldrw(rscratch1, dst_length_addr);
2272 __ cmpw(tmp, rscratch1);
2273 __ br(Assembler::HI, *stub->entry());
2274 }
2275
2276 if (flags & LIR_OpArrayCopy::type_check) {
2277 // We don't know the array types are compatible
2278 if (basic_type != T_OBJECT) {
2279 // Simple test for basic type arrays
2280 __ cmp_klasses_from_objects(src, dst, tmp, rscratch1);
2281 __ br(Assembler::NE, *stub->entry());
2282 } else {
2283 // For object arrays, if src is a sub class of dst then we can
2284 // safely do the copy.
2285 Label cont, slow;
2286
2287 #define PUSH(r1, r2) \
2288 stp(r1, r2, __ pre(sp, -2 * wordSize));
2289
2290 #define POP(r1, r2) \
2291 ldp(r1, r2, __ post(sp, 2 * wordSize));
2292
2293 __ PUSH(src, dst);
2294
2295 __ load_klass(src, src);
2296 __ load_klass(dst, dst);
2297
2298 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2299
2300 __ PUSH(src, dst);
2301 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2302 __ POP(src, dst);
2303
2304 __ cbnz(src, cont);
2305
2306 __ bind(slow);
2307 __ POP(src, dst);
2308
2309 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2310 if (copyfunc_addr != nullptr) { // use stub if available
2311 // src is not a sub class of dst so we have to do a
2312 // per-element check.
2313
2314 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2315 if ((flags & mask) != mask) {
2316 // Check that at least both of them object arrays.
2317 assert(flags & mask, "one of the two should be known to be an object array");
2318
2319 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2320 __ load_klass(tmp, src);
2321 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2322 __ load_klass(tmp, dst);
2323 }
2324 int lh_offset = in_bytes(Klass::layout_helper_offset());
2325 Address klass_lh_addr(tmp, lh_offset);
2326 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2327 __ ldrw(rscratch1, klass_lh_addr);
2328 __ mov(rscratch2, objArray_lh);
2329 __ eorw(rscratch1, rscratch1, rscratch2);
2330 __ cbnzw(rscratch1, *stub->entry());
2331 }
2332
2333 // Spill because stubs can use any register they like and it's
2334 // easier to restore just those that we care about.
2335 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2336 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2337 __ str(src, Address(sp, 4*BytesPerWord));
2338
2339 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2340 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2341 assert_different_registers(c_rarg0, dst, dst_pos, length);
2342 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2343 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2344 assert_different_registers(c_rarg1, dst, length);
2345 __ uxtw(c_rarg2, length);
2346 assert_different_registers(c_rarg2, dst);
2347
2348 __ load_klass(c_rarg4, dst);
2349 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2350 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2351 __ far_call(RuntimeAddress(copyfunc_addr));
2352
2353 #ifndef PRODUCT
2354 if (PrintC1Statistics) {
2355 Label failed;
2356 __ cbnz(r0, failed);
2357 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2358 __ bind(failed);
2359 }
2360 #endif
2361
2362 __ cbz(r0, *stub->continuation());
2363
2364 #ifndef PRODUCT
2365 if (PrintC1Statistics) {
2366 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2367 }
2368 #endif
2369 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2370
2371 // Restore previously spilled arguments
2372 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2373 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2374 __ ldr(src, Address(sp, 4*BytesPerWord));
2375
2376 // return value is -1^K where K is partial copied count
2377 __ eonw(rscratch1, r0, zr);
2378 // adjust length down and src/end pos up by partial copied count
2379 __ subw(length, length, rscratch1);
2380 __ addw(src_pos, src_pos, rscratch1);
2381 __ addw(dst_pos, dst_pos, rscratch1);
2382 }
2383
2384 __ b(*stub->entry());
2385
2386 __ bind(cont);
2387 __ POP(src, dst);
2388 }
2389 }
2390
2391 #ifdef ASSERT
2392 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2393 // Sanity check the known type with the incoming class. For the
2394 // primitive case the types must match exactly with src.klass and
2395 // dst.klass each exactly matching the default type. For the
2396 // object array case, if no type check is needed then either the
2397 // dst type is exactly the expected type and the src type is a
2398 // subtype which we can't check or src is the same array as dst
2399 // but not necessarily exactly of type default_type.
2400 Label known_ok, halt;
2401 __ mov_metadata(tmp, default_type->constant_encoding());
2402
2403 if (basic_type != T_OBJECT) {
2404 __ cmp_klass(dst, tmp, rscratch1);
2405 __ br(Assembler::NE, halt);
2406 __ cmp_klass(src, tmp, rscratch1);
2407 __ br(Assembler::EQ, known_ok);
2408 } else {
2409 __ cmp_klass(dst, tmp, rscratch1);
2410 __ br(Assembler::EQ, known_ok);
2411 __ cmp(src, dst);
2412 __ br(Assembler::EQ, known_ok);
2413 }
2414 __ bind(halt);
2415 __ stop("incorrect type information in arraycopy");
2416 __ bind(known_ok);
2417 }
2418 #endif
2419
2420 #ifndef PRODUCT
2421 if (PrintC1Statistics) {
2422 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2423 }
2424 #endif
2425
2426 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2427 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2428 assert_different_registers(c_rarg0, dst, dst_pos, length);
2429 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2430 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2431 assert_different_registers(c_rarg1, dst, length);
2432 __ uxtw(c_rarg2, length);
2433 assert_different_registers(c_rarg2, dst);
2434
2435 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2436 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2437 const char *name;
2438 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2439
2440 CodeBlob *cb = CodeCache::find_blob(entry);
2441 if (cb) {
2442 __ far_call(RuntimeAddress(entry));
2443 } else {
2444 __ call_VM_leaf(entry, 3);
2445 }
2446
2447 if (stub != nullptr) {
2448 __ bind(*stub->continuation());
2449 }
2450 }
2451
2452
2453
2454
2455 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2456 Register obj = op->obj_opr()->as_register(); // may not be an oop
2457 Register hdr = op->hdr_opr()->as_register();
2458 Register lock = op->lock_opr()->as_register();
2459 Register temp = op->scratch_opr()->as_register();
2460 if (op->code() == lir_lock) {
2461 // add debug info for NullPointerException only if one is possible
2462 int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
2463 if (op->info() != nullptr) {
2464 add_debug_info_for_null_check(null_check_offset, op->info());
2465 }
2466 // done
2467 } else if (op->code() == lir_unlock) {
2468 __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
2469 } else {
2470 Unimplemented();
2471 }
2472 __ bind(*op->stub()->continuation());
2473 }
2474
2475 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2476 Register obj = op->obj()->as_pointer_register();
2477 Register result = op->result_opr()->as_pointer_register();
2478
2479 CodeEmitInfo* info = op->info();
2480 if (info != nullptr) {
2481 add_debug_info_for_null_check_here(info);
2482 }
2483
2484 __ load_klass(result, obj);
2485 }
2486
2487 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2488 ciMethod* method = op->profiled_method();
2489 int bci = op->profiled_bci();
2490 ciMethod* callee = op->profiled_callee();
2491
2492 // Update counter for all call types
2493 ciMethodData* md = method->method_data_or_null();
2494 assert(md != nullptr, "Sanity");
2495 ciProfileData* data = md->bci_to_data(bci);
2496 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2497 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2498 Register mdo = op->mdo()->as_register();
2499 __ mov_metadata(mdo, md->constant_encoding());
2500 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2501 // Perform additional virtual call profiling for invokevirtual and
2502 // invokeinterface bytecodes
2503 if (op->should_profile_receiver_type()) {
2504 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2505 Register recv = op->recv()->as_register();
2506 assert_different_registers(mdo, recv);
2507 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2508 ciKlass* known_klass = op->known_holder();
2509 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2510 // We know the type that will be seen at this call site; we can
2511 // statically update the MethodData* rather than needing to do
2512 // dynamic tests on the receiver type.
2513 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2514 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2515 ciKlass* receiver = vc_data->receiver(i);
2516 if (known_klass->equals(receiver)) {
2517 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2518 __ addptr(data_addr, DataLayout::counter_increment);
2519 return;
2520 }
2521 }
2522 // Receiver type is not found in profile data.
2523 // Fall back to runtime helper to handle the rest at runtime.
2524 __ mov_metadata(recv, known_klass->constant_encoding());
2525 } else {
2526 __ load_klass(recv, recv);
2527 }
2528 type_profile_helper(mdo, md, data, recv);
2529 } else {
2530 // Static call
2531 __ addptr(counter_addr, DataLayout::counter_increment);
2532 }
2533 }
2534
2535
2536 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2537 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2538 }
2539
2540 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2541 assert(op->crc()->is_single_cpu(), "crc must be register");
2542 assert(op->val()->is_single_cpu(), "byte value must be register");
2543 assert(op->result_opr()->is_single_cpu(), "result must be register");
2544 Register crc = op->crc()->as_register();
2545 Register val = op->val()->as_register();
2546 Register res = op->result_opr()->as_register();
2547
2548 assert_different_registers(val, crc, res);
2549 uint64_t offset;
2550 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2551 __ add(res, res, offset);
2552
2553 __ mvnw(crc, crc); // ~crc
2554 __ update_byte_crc32(crc, val, res);
2555 __ mvnw(res, crc); // ~crc
2556 }
2557
2558 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2559 COMMENT("emit_profile_type {");
2560 Register obj = op->obj()->as_register();
2561 Register tmp = op->tmp()->as_pointer_register();
2562 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2563 ciKlass* exact_klass = op->exact_klass();
2564 intptr_t current_klass = op->current_klass();
2565 bool not_null = op->not_null();
2566 bool no_conflict = op->no_conflict();
2567
2568 Label update, next, none;
2569
2570 bool do_null = !not_null;
2571 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2572 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2573
2574 assert(do_null || do_update, "why are we here?");
2575 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2576 assert(mdo_addr.base() != rscratch1, "wrong register");
2577
2578 __ verify_oop(obj);
2579
2580 if (tmp != obj) {
2581 assert_different_registers(obj, tmp, rscratch1, rscratch2, mdo_addr.base(), mdo_addr.index());
2582 __ mov(tmp, obj);
2583 } else {
2584 assert_different_registers(obj, rscratch1, rscratch2, mdo_addr.base(), mdo_addr.index());
2585 }
2586 if (do_null) {
2587 __ cbnz(tmp, update);
2588 if (!TypeEntries::was_null_seen(current_klass)) {
2589 __ ldr(rscratch2, mdo_addr);
2590 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2591 __ str(rscratch2, mdo_addr);
2592 }
2593 if (do_update) {
2594 #ifndef ASSERT
2595 __ b(next);
2596 }
2597 #else
2598 __ b(next);
2599 }
2600 } else {
2601 __ cbnz(tmp, update);
2602 __ stop("unexpected null obj");
2603 #endif
2604 }
2605
2606 __ bind(update);
2607
2608 if (do_update) {
2609 #ifdef ASSERT
2610 if (exact_klass != nullptr) {
2611 Label ok;
2612 __ load_klass(tmp, tmp);
2613 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2614 __ eor(rscratch1, tmp, rscratch1);
2615 __ cbz(rscratch1, ok);
2616 __ stop("exact klass and actual klass differ");
2617 __ bind(ok);
2618 }
2619 #endif
2620 if (!no_conflict) {
2621 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2622 if (exact_klass != nullptr) {
2623 __ mov_metadata(tmp, exact_klass->constant_encoding());
2624 } else {
2625 __ load_klass(tmp, tmp);
2626 }
2627
2628 __ ldr(rscratch2, mdo_addr);
2629 __ eor(tmp, tmp, rscratch2);
2630 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2631 // klass seen before, nothing to do. The unknown bit may have been
2632 // set already but no need to check.
2633 __ cbz(rscratch1, next);
2634
2635 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2636
2637 if (TypeEntries::is_type_none(current_klass)) {
2638 __ cbz(rscratch2, none);
2639 __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2640 __ br(Assembler::EQ, none);
2641 // There is a chance that the checks above
2642 // fail if another thread has just set the
2643 // profiling to this obj's klass
2644 __ dmb(Assembler::ISHLD);
2645 __ eor(tmp, tmp, rscratch2); // get back original value before XOR
2646 __ ldr(rscratch2, mdo_addr);
2647 __ eor(tmp, tmp, rscratch2);
2648 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2649 __ cbz(rscratch1, next);
2650 }
2651 } else {
2652 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2653 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2654
2655 __ ldr(tmp, mdo_addr);
2656 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2657 }
2658
2659 // different than before. Cannot keep accurate profile.
2660 __ ldr(rscratch2, mdo_addr);
2661 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2662 __ str(rscratch2, mdo_addr);
2663
2664 if (TypeEntries::is_type_none(current_klass)) {
2665 __ b(next);
2666
2667 __ bind(none);
2668 // first time here. Set profile type.
2669 __ str(tmp, mdo_addr);
2670 #ifdef ASSERT
2671 __ andr(tmp, tmp, TypeEntries::type_mask);
2672 __ verify_klass_ptr(tmp);
2673 #endif
2674 }
2675 } else {
2676 // There's a single possible klass at this profile point
2677 assert(exact_klass != nullptr, "should be");
2678 if (TypeEntries::is_type_none(current_klass)) {
2679 __ mov_metadata(tmp, exact_klass->constant_encoding());
2680 __ ldr(rscratch2, mdo_addr);
2681 __ eor(tmp, tmp, rscratch2);
2682 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2683 __ cbz(rscratch1, next);
2684 #ifdef ASSERT
2685 {
2686 Label ok;
2687 __ ldr(rscratch1, mdo_addr);
2688 __ cbz(rscratch1, ok);
2689 __ cmp(rscratch1, (u1)TypeEntries::null_seen);
2690 __ br(Assembler::EQ, ok);
2691 // may have been set by another thread
2692 __ dmb(Assembler::ISHLD);
2693 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2694 __ ldr(rscratch2, mdo_addr);
2695 __ eor(rscratch2, rscratch1, rscratch2);
2696 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2697 __ cbz(rscratch2, ok);
2698
2699 __ stop("unexpected profiling mismatch");
2700 __ bind(ok);
2701 }
2702 #endif
2703 // first time here. Set profile type.
2704 __ str(tmp, mdo_addr);
2705 #ifdef ASSERT
2706 __ andr(tmp, tmp, TypeEntries::type_mask);
2707 __ verify_klass_ptr(tmp);
2708 #endif
2709 } else {
2710 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2711 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2712
2713 __ ldr(tmp, mdo_addr);
2714 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2715
2716 __ orr(tmp, tmp, TypeEntries::type_unknown);
2717 __ str(tmp, mdo_addr);
2718 // FIXME: Write barrier needed here?
2719 }
2720 }
2721
2722 __ bind(next);
2723 }
2724 COMMENT("} emit_profile_type");
2725 }
2726
2727
2728 void LIR_Assembler::align_backward_branch_target() {
2729 }
2730
2731
2732 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2733 // tmp must be unused
2734 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2735
2736 if (left->is_single_cpu()) {
2737 assert(dest->is_single_cpu(), "expect single result reg");
2738 __ negw(dest->as_register(), left->as_register());
2739 } else if (left->is_double_cpu()) {
2740 assert(dest->is_double_cpu(), "expect double result reg");
2741 __ neg(dest->as_register_lo(), left->as_register_lo());
2742 } else if (left->is_single_fpu()) {
2743 assert(dest->is_single_fpu(), "expect single float result reg");
2744 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2745 } else {
2746 assert(left->is_double_fpu(), "expect double float operand reg");
2747 assert(dest->is_double_fpu(), "expect double float result reg");
2748 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2749 }
2750 }
2751
2752
2753 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2754 if (patch_code != lir_patch_none) {
2755 deoptimize_trap(info);
2756 return;
2757 }
2758
2759 __ lea(dest->as_pointer_register(), as_Address(addr->as_address_ptr()));
2760 }
2761
2762
2763 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2764 assert(!tmp->is_valid(), "don't need temporary");
2765
2766 CodeBlob *cb = CodeCache::find_blob(dest);
2767 if (cb) {
2768 __ far_call(RuntimeAddress(dest));
2769 } else {
2770 __ mov(rscratch1, RuntimeAddress(dest));
2771 __ blr(rscratch1);
2772 }
2773
2774 if (info != nullptr) {
2775 add_call_info_here(info);
2776 }
2777 __ post_call_nop();
2778 }
2779
2780 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2781 if (dest->is_address() || src->is_address()) {
2782 move_op(src, dest, type, lir_patch_none, info, /*wide*/false);
2783 } else {
2784 ShouldNotReachHere();
2785 }
2786 }
2787
2788 #ifdef ASSERT
2789 // emit run-time assertion
2790 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2791 assert(op->code() == lir_assert, "must be");
2792
2793 if (op->in_opr1()->is_valid()) {
2794 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2795 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2796 } else {
2797 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2798 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2799 }
2800
2801 Label ok;
2802 if (op->condition() != lir_cond_always) {
2803 Assembler::Condition acond = Assembler::AL;
2804 switch (op->condition()) {
2805 case lir_cond_equal: acond = Assembler::EQ; break;
2806 case lir_cond_notEqual: acond = Assembler::NE; break;
2807 case lir_cond_less: acond = Assembler::LT; break;
2808 case lir_cond_lessEqual: acond = Assembler::LE; break;
2809 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2810 case lir_cond_greater: acond = Assembler::GT; break;
2811 case lir_cond_belowEqual: acond = Assembler::LS; break;
2812 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2813 default: ShouldNotReachHere();
2814 }
2815 __ br(acond, ok);
2816 }
2817 if (op->halt()) {
2818 const char* str = __ code_string(op->msg());
2819 __ stop(str);
2820 } else {
2821 breakpoint();
2822 }
2823 __ bind(ok);
2824 }
2825 #endif
2826
2827 #ifndef PRODUCT
2828 #define COMMENT(x) do { __ block_comment(x); } while (0)
2829 #else
2830 #define COMMENT(x)
2831 #endif
2832
2833 void LIR_Assembler::membar() {
2834 COMMENT("membar");
2835 __ membar(MacroAssembler::AnyAny);
2836 }
2837
2838 void LIR_Assembler::membar_acquire() {
2839 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2840 }
2841
2842 void LIR_Assembler::membar_release() {
2843 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2844 }
2845
2846 void LIR_Assembler::membar_loadload() {
2847 __ membar(Assembler::LoadLoad);
2848 }
2849
2850 void LIR_Assembler::membar_storestore() {
2851 __ membar(MacroAssembler::StoreStore);
2852 }
2853
2854 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2855
2856 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2857
2858 void LIR_Assembler::on_spin_wait() {
2859 __ spin_wait();
2860 }
2861
2862 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2863 __ mov(result_reg->as_register(), rthread);
2864 }
2865
2866
2867 void LIR_Assembler::peephole(LIR_List *lir) {
2868 #if 0
2869 if (tableswitch_count >= max_tableswitches)
2870 return;
2871
2872 /*
2873 This finite-state automaton recognizes sequences of compare-and-
2874 branch instructions. We will turn them into a tableswitch. You
2875 could argue that C1 really shouldn't be doing this sort of
2876 optimization, but without it the code is really horrible.
2877 */
2878
2879 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2880 int first_key, last_key = -2147483648;
2881 int next_key = 0;
2882 int start_insn = -1;
2883 int last_insn = -1;
2884 Register reg = noreg;
2885 LIR_Opr reg_opr;
2886 state = start_s;
2887
2888 LIR_OpList* inst = lir->instructions_list();
2889 for (int i = 0; i < inst->length(); i++) {
2890 LIR_Op* op = inst->at(i);
2891 switch (state) {
2892 case start_s:
2893 first_key = -1;
2894 start_insn = i;
2895 switch (op->code()) {
2896 case lir_cmp:
2897 LIR_Opr opr1 = op->as_Op2()->in_opr1();
2898 LIR_Opr opr2 = op->as_Op2()->in_opr2();
2899 if (opr1->is_cpu_register() && opr1->is_single_cpu()
2900 && opr2->is_constant()
2901 && opr2->type() == T_INT) {
2902 reg_opr = opr1;
2903 reg = opr1->as_register();
2904 first_key = opr2->as_constant_ptr()->as_jint();
2905 next_key = first_key + 1;
2906 state = cmp_s;
2907 goto next_state;
2908 }
2909 break;
2910 }
2911 break;
2912 case cmp_s:
2913 switch (op->code()) {
2914 case lir_branch:
2915 if (op->as_OpBranch()->cond() == lir_cond_equal) {
2916 state = beq_s;
2917 last_insn = i;
2918 goto next_state;
2919 }
2920 }
2921 state = start_s;
2922 break;
2923 case beq_s:
2924 switch (op->code()) {
2925 case lir_cmp: {
2926 LIR_Opr opr1 = op->as_Op2()->in_opr1();
2927 LIR_Opr opr2 = op->as_Op2()->in_opr2();
2928 if (opr1->is_cpu_register() && opr1->is_single_cpu()
2929 && opr1->as_register() == reg
2930 && opr2->is_constant()
2931 && opr2->type() == T_INT
2932 && opr2->as_constant_ptr()->as_jint() == next_key) {
2933 last_key = next_key;
2934 next_key++;
2935 state = cmp_s;
2936 goto next_state;
2937 }
2938 }
2939 }
2940 last_key = next_key;
2941 state = start_s;
2942 break;
2943 default:
2944 assert(false, "impossible state");
2945 }
2946 if (state == start_s) {
2947 if (first_key < last_key - 5L && reg != noreg) {
2948 {
2949 // printf("found run register %d starting at insn %d low value %d high value %d\n",
2950 // reg->encoding(),
2951 // start_insn, first_key, last_key);
2952 // for (int i = 0; i < inst->length(); i++) {
2953 // inst->at(i)->print();
2954 // tty->print("\n");
2955 // }
2956 // tty->print("\n");
2957 }
2958
2959 struct tableswitch *sw = &switches[tableswitch_count];
2960 sw->_insn_index = start_insn, sw->_first_key = first_key,
2961 sw->_last_key = last_key, sw->_reg = reg;
2962 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
2963 {
2964 // Insert the new table of branches
2965 int offset = last_insn;
2966 for (int n = first_key; n < last_key; n++) {
2967 inst->insert_before
2968 (last_insn + 1,
2969 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
2970 inst->at(offset)->as_OpBranch()->label()));
2971 offset -= 2, i++;
2972 }
2973 }
2974 // Delete all the old compare-and-branch instructions
2975 for (int n = first_key; n < last_key; n++) {
2976 inst->remove_at(start_insn);
2977 inst->remove_at(start_insn);
2978 }
2979 // Insert the tableswitch instruction
2980 inst->insert_before(start_insn,
2981 new LIR_Op2(lir_cmp, lir_cond_always,
2982 LIR_OprFact::intConst(tableswitch_count),
2983 reg_opr));
2984 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
2985 tableswitch_count++;
2986 }
2987 reg = noreg;
2988 last_key = -2147483648;
2989 }
2990 next_state:
2991 ;
2992 }
2993 #endif
2994 }
2995
2996 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
2997 Address addr = as_Address(src->as_address_ptr());
2998 BasicType type = src->type();
2999 bool is_oop = is_reference_type(type);
3000
3001 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3002 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3003
3004 switch(type) {
3005 case T_INT:
3006 xchg = &MacroAssembler::atomic_xchgalw;
3007 add = &MacroAssembler::atomic_addalw;
3008 break;
3009 case T_LONG:
3010 xchg = &MacroAssembler::atomic_xchgal;
3011 add = &MacroAssembler::atomic_addal;
3012 break;
3013 case T_OBJECT:
3014 case T_ARRAY:
3015 if (UseCompressedOops) {
3016 xchg = &MacroAssembler::atomic_xchgalw;
3017 add = &MacroAssembler::atomic_addalw;
3018 } else {
3019 xchg = &MacroAssembler::atomic_xchgal;
3020 add = &MacroAssembler::atomic_addal;
3021 }
3022 break;
3023 default:
3024 ShouldNotReachHere();
3025 xchg = &MacroAssembler::atomic_xchgal;
3026 add = &MacroAssembler::atomic_addal; // unreachable
3027 }
3028
3029 switch (code) {
3030 case lir_xadd:
3031 {
3032 RegisterOrConstant inc;
3033 Register tmp = as_reg(tmp_op);
3034 Register dst = as_reg(dest);
3035 if (data->is_constant()) {
3036 inc = RegisterOrConstant(as_long(data));
3037 assert_different_registers(dst, addr.base(), tmp,
3038 rscratch1, rscratch2);
3039 } else {
3040 inc = RegisterOrConstant(as_reg(data));
3041 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3042 rscratch1, rscratch2);
3043 }
3044 __ lea(tmp, addr);
3045 (_masm->*add)(dst, inc, tmp);
3046 break;
3047 }
3048 case lir_xchg:
3049 {
3050 Register tmp = tmp_op->as_register();
3051 Register obj = as_reg(data);
3052 Register dst = as_reg(dest);
3053 if (is_oop && UseCompressedOops) {
3054 __ encode_heap_oop(rscratch2, obj);
3055 obj = rscratch2;
3056 }
3057 assert_different_registers(obj, addr.base(), tmp, rscratch1);
3058 assert_different_registers(dst, addr.base(), tmp, rscratch1);
3059 __ lea(tmp, addr);
3060 (_masm->*xchg)(dst, obj, tmp);
3061 if (is_oop && UseCompressedOops) {
3062 __ decode_heap_oop(dst);
3063 }
3064 }
3065 break;
3066 default:
3067 ShouldNotReachHere();
3068 }
3069 if(!UseLSE) {
3070 __ membar(__ AnyAny);
3071 }
3072 }
3073
3074 #undef __