1 /*
2 * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.inline.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInlineKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "ci/ciObjArrayKlass.hpp"
38 #include "code/aotCodeCache.hpp"
39 #include "code/compiledIC.hpp"
40 #include "gc/shared/collectedHeap.hpp"
41 #include "gc/shared/gc_globals.hpp"
42 #include "nativeInst_aarch64.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "oops/oop.inline.hpp"
45 #include "runtime/frame.inline.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "utilities/powerOfTwo.hpp"
49 #include "vmreg_aarch64.inline.hpp"
50
51
52 #ifndef PRODUCT
53 #define COMMENT(x) do { __ block_comment(x); } while (0)
54 #else
55 #define COMMENT(x)
56 #endif
57
58 NEEDS_CLEANUP // remove this definitions ?
59 const Register SYNC_header = r0; // synchronization header
60 const Register SHIFT_count = r0; // where count for shift operations must be
61
62 #define __ _masm->
63
64
65 static void select_different_registers(Register preserve,
66 Register extra,
67 Register &tmp1,
68 Register &tmp2) {
69 if (tmp1 == preserve) {
70 assert_different_registers(tmp1, tmp2, extra);
71 tmp1 = extra;
72 } else if (tmp2 == preserve) {
73 assert_different_registers(tmp1, tmp2, extra);
74 tmp2 = extra;
75 }
76 assert_different_registers(preserve, tmp1, tmp2);
77 }
78
79
80
81 static void select_different_registers(Register preserve,
82 Register extra,
83 Register &tmp1,
84 Register &tmp2,
85 Register &tmp3) {
86 if (tmp1 == preserve) {
87 assert_different_registers(tmp1, tmp2, tmp3, extra);
88 tmp1 = extra;
89 } else if (tmp2 == preserve) {
90 assert_different_registers(tmp1, tmp2, tmp3, extra);
91 tmp2 = extra;
92 } else if (tmp3 == preserve) {
93 assert_different_registers(tmp1, tmp2, tmp3, extra);
94 tmp3 = extra;
95 }
96 assert_different_registers(preserve, tmp1, tmp2, tmp3);
97 }
98
99
100 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
101
102
103 LIR_Opr LIR_Assembler::receiverOpr() {
104 return FrameMap::receiver_opr;
105 }
106
107 LIR_Opr LIR_Assembler::osrBufferPointer() {
108 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
109 }
110
111 //--------------fpu register translations-----------------------
112
113
114 address LIR_Assembler::float_constant(float f) {
115 address const_addr = __ float_constant(f);
116 if (const_addr == nullptr) {
117 bailout("const section overflow");
118 return __ code()->consts()->start();
119 } else {
120 return const_addr;
121 }
122 }
123
124
125 address LIR_Assembler::double_constant(double d) {
126 address const_addr = __ double_constant(d);
127 if (const_addr == nullptr) {
128 bailout("const section overflow");
129 return __ code()->consts()->start();
130 } else {
131 return const_addr;
132 }
133 }
134
135 address LIR_Assembler::int_constant(jlong n) {
136 address const_addr = __ long_constant(n);
137 if (const_addr == nullptr) {
138 bailout("const section overflow");
139 return __ code()->consts()->start();
140 } else {
141 return const_addr;
142 }
143 }
144
145 void LIR_Assembler::breakpoint() { Unimplemented(); }
146
147 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
148
149 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
150
151 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
152 //-------------------------------------------
153
154 static Register as_reg(LIR_Opr op) {
155 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
156 }
157
158 static jlong as_long(LIR_Opr data) {
159 jlong result;
160 switch (data->type()) {
161 case T_INT:
162 result = (data->as_jint());
163 break;
164 case T_LONG:
165 result = (data->as_jlong());
166 break;
167 default:
168 ShouldNotReachHere();
169 result = 0; // unreachable
170 }
171 return result;
172 }
173
174 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
175 Register base = addr->base()->as_pointer_register();
176 LIR_Opr opr = addr->index();
177 if (opr->is_cpu_register()) {
178 Register index;
179 if (opr->is_single_cpu())
180 index = opr->as_register();
181 else
182 index = opr->as_register_lo();
183 assert(addr->disp() == 0, "must be");
184 switch(opr->type()) {
185 case T_INT:
186 return Address(base, index, Address::sxtw(addr->scale()));
187 case T_LONG:
188 return Address(base, index, Address::lsl(addr->scale()));
189 default:
190 ShouldNotReachHere();
191 }
192 } else {
193 assert(addr->scale() == 0,
194 "expected for immediate operand, was: %d", addr->scale());
195 ptrdiff_t offset = ptrdiff_t(addr->disp());
196 // NOTE: Does not handle any 16 byte vector access.
197 const uint type_size = type2aelembytes(addr->type(), true);
198 return __ legitimize_address(Address(base, offset), type_size, tmp);
199 }
200 return Address();
201 }
202
203 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
204 ShouldNotReachHere();
205 return Address();
206 }
207
208 Address LIR_Assembler::as_Address(LIR_Address* addr) {
209 return as_Address(addr, rscratch1);
210 }
211
212 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
213 return as_Address(addr, rscratch1); // Ouch
214 // FIXME: This needs to be much more clever. See x86.
215 }
216
217 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
218 // not encodable as a base + (immediate) offset, generate an explicit address
219 // calculation to hold the address in a temporary register.
220 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) {
221 precond(size == 4 || size == 8);
222 Address addr = frame_map()->address_for_slot(index, adjust);
223 precond(addr.getMode() == Address::base_plus_offset);
224 precond(addr.base() == sp);
225 precond(addr.offset() > 0);
226 uint mask = size - 1;
227 assert((addr.offset() & mask) == 0, "scaled offsets only");
228 return __ legitimize_address(addr, size, tmp);
229 }
230
231 void LIR_Assembler::osr_entry() {
232 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
233 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
234 ValueStack* entry_state = osr_entry->state();
235 int number_of_locks = entry_state->locks_size();
236
237 // we jump here if osr happens with the interpreter
238 // state set up to continue at the beginning of the
239 // loop that triggered osr - in particular, we have
240 // the following registers setup:
241 //
242 // r2: osr buffer
243 //
244
245 // build frame
246 ciMethod* m = compilation()->method();
247 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
248
249 // OSR buffer is
250 //
251 // locals[nlocals-1..0]
252 // monitors[0..number_of_locks]
253 //
254 // locals is a direct copy of the interpreter frame so in the osr buffer
255 // so first slot in the local array is the last local from the interpreter
256 // and last slot is local[0] (receiver) from the interpreter
257 //
258 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
259 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
260 // in the interpreter frame (the method lock if a sync method)
261
262 // Initialize monitors in the compiled activation.
263 // r2: pointer to osr buffer
264 //
265 // All other registers are dead at this point and the locals will be
266 // copied into place by code emitted in the IR.
267
268 Register OSR_buf = osrBufferPointer()->as_pointer_register();
269 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
270 int monitor_offset = BytesPerWord * method()->max_locals() +
271 (2 * BytesPerWord) * (number_of_locks - 1);
272 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
273 // the OSR buffer using 2 word entries: first the lock and then
274 // the oop.
275 for (int i = 0; i < number_of_locks; i++) {
276 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
277 #ifdef ASSERT
278 // verify the interpreter's monitor has a non-null object
279 {
280 Label L;
281 __ ldr(rscratch1, __ form_address(rscratch1, OSR_buf, slot_offset + 1*BytesPerWord, 0));
282 __ cbnz(rscratch1, L);
283 __ stop("locked object is null");
284 __ bind(L);
285 }
286 #endif
287 __ ldr(r19, __ form_address(rscratch1, OSR_buf, slot_offset, 0));
288 __ ldr(r20, __ form_address(rscratch1, OSR_buf, slot_offset + BytesPerWord, 0));
289 __ str(r19, frame_map()->address_for_monitor_lock(i));
290 __ str(r20, frame_map()->address_for_monitor_object(i));
291 }
292 }
293 }
294
295
296 // inline cache check; done before the frame is built.
297 int LIR_Assembler::check_icache() {
298 return __ ic_check(CodeEntryAlignment);
299 }
300
301 void LIR_Assembler::clinit_barrier(ciMethod* method) {
302 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
303 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
304
305 Label L_skip_barrier;
306
307 __ mov_metadata(rscratch2, method->holder()->constant_encoding());
308 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
309 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
310 __ bind(L_skip_barrier);
311 }
312
313 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
314 if (o == nullptr) {
315 __ mov(reg, zr);
316 } else {
317 __ movoop(reg, o);
318 }
319 }
320
321 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
322 address target = nullptr;
323 relocInfo::relocType reloc_type = relocInfo::none;
324
325 switch (patching_id(info)) {
326 case PatchingStub::access_field_id:
327 target = Runtime1::entry_for(StubId::c1_access_field_patching_id);
328 reloc_type = relocInfo::section_word_type;
329 break;
330 case PatchingStub::load_klass_id:
331 target = Runtime1::entry_for(StubId::c1_load_klass_patching_id);
332 reloc_type = relocInfo::metadata_type;
333 break;
334 case PatchingStub::load_mirror_id:
335 target = Runtime1::entry_for(StubId::c1_load_mirror_patching_id);
336 reloc_type = relocInfo::oop_type;
337 break;
338 case PatchingStub::load_appendix_id:
339 target = Runtime1::entry_for(StubId::c1_load_appendix_patching_id);
340 reloc_type = relocInfo::oop_type;
341 break;
342 default: ShouldNotReachHere();
343 }
344
345 __ far_call(RuntimeAddress(target));
346 add_call_info_here(info);
347 }
348
349 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
350 deoptimize_trap(info);
351 }
352
353
354 // This specifies the rsp decrement needed to build the frame
355 int LIR_Assembler::initial_frame_size_in_bytes() const {
356 // if rounding, must let FrameMap know!
357
358 return in_bytes(frame_map()->framesize_in_bytes());
359 }
360
361
362 int LIR_Assembler::emit_exception_handler() {
363 // generate code for exception handler
364 address handler_base = __ start_a_stub(exception_handler_size());
365 if (handler_base == nullptr) {
366 // not enough space left for the handler
367 bailout("exception handler overflow");
368 return -1;
369 }
370
371 int offset = code_offset();
372
373 // the exception oop and pc are in r0, and r3
374 // no other registers need to be preserved, so invalidate them
375 __ invalidate_registers(false, true, true, false, true, true);
376
377 // check that there is really an exception
378 __ verify_not_null_oop(r0);
379
380 // search an exception handler (r0: exception oop, r3: throwing pc)
381 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
382 __ should_not_reach_here();
383 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
384 __ end_a_stub();
385
386 return offset;
387 }
388
389
390 // Emit the code to remove the frame from the stack in the exception
391 // unwind path.
392 int LIR_Assembler::emit_unwind_handler() {
393 #ifndef PRODUCT
394 if (CommentedAssembly) {
395 _masm->block_comment("Unwind handler");
396 }
397 #endif
398
399 int offset = code_offset();
400
401 // Fetch the exception from TLS and clear out exception related thread state
402 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
403 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
404 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
405
406 __ bind(_unwind_handler_entry);
407 __ verify_not_null_oop(r0);
408 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
409 __ mov(r19, r0); // Preserve the exception
410 }
411
412 // Perform needed unlocking
413 MonitorExitStub* stub = nullptr;
414 if (method()->is_synchronized()) {
415 monitor_address(0, FrameMap::r0_opr);
416 stub = new MonitorExitStub(FrameMap::r0_opr, 0);
417 __ unlock_object(r5, r4, r0, r6, *stub->entry());
418 __ bind(*stub->continuation());
419 }
420
421 if (compilation()->env()->dtrace_method_probes()) {
422 __ mov(c_rarg0, rthread);
423 __ mov_metadata(c_rarg1, method()->constant_encoding());
424 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
425 }
426
427 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
428 __ mov(r0, r19); // Restore the exception
429 }
430
431 // remove the activation and dispatch to the unwind handler
432 __ block_comment("remove_frame and dispatch to the unwind handler");
433 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
434 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
435
436 // Emit the slow path assembly
437 if (stub != nullptr) {
438 stub->emit_code(this);
439 }
440
441 return offset;
442 }
443
444
445 int LIR_Assembler::emit_deopt_handler() {
446 // generate code for exception handler
447 address handler_base = __ start_a_stub(deopt_handler_size());
448 if (handler_base == nullptr) {
449 // not enough space left for the handler
450 bailout("deopt handler overflow");
451 return -1;
452 }
453
454 int offset = code_offset();
455
456 Label start;
457 __ bind(start);
458
459 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
460
461 int entry_offset = __ offset();
462 __ b(start);
463
464 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
465 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
466 "out of bounds read in post-call NOP check");
467 __ end_a_stub();
468
469 return entry_offset;
470 }
471
472 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
473 _masm->code_section()->relocate(adr, relocInfo::poll_type);
474 int pc_offset = code_offset();
475 flush_debug_info(pc_offset);
476 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
477 if (info->exception_handlers() != nullptr) {
478 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
479 }
480 }
481
482 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
483 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
484
485 if (InlineTypeReturnedAsFields) {
486 // Check if we are returning an non-null inline type and load its fields into registers
487 ciType* return_type = compilation()->method()->return_type();
488 if (return_type->is_inlinetype()) {
489 ciInlineKlass* vk = return_type->as_inline_klass();
490 if (vk->can_be_returned_as_fields()) {
491 address unpack_handler = vk->unpack_handler();
492 assert(unpack_handler != nullptr, "must be");
493 __ far_call(RuntimeAddress(unpack_handler));
494 }
495 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
496 Label skip;
497 Label not_null;
498 __ cbnz(r0, not_null);
499 // Returned value is null, zero all return registers because they may belong to oop fields
500 __ mov(j_rarg1, zr);
501 __ mov(j_rarg2, zr);
502 __ mov(j_rarg3, zr);
503 __ mov(j_rarg4, zr);
504 __ mov(j_rarg5, zr);
505 __ mov(j_rarg6, zr);
506 __ mov(j_rarg7, zr);
507 __ b(skip);
508 __ bind(not_null);
509
510 // Check if we are returning an non-null inline type and load its fields into registers
511 __ test_oop_is_not_inline_type(r0, rscratch2, skip, /* can_be_null= */ false);
512
513 // Load fields from a buffered value with an inline class specific handler
514 __ load_klass(rscratch1 /*dst*/, r0 /*src*/);
515 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
516 __ ldr(rscratch1, Address(rscratch1, InlineKlass::unpack_handler_offset()));
517 // Unpack handler can be null if inline type is not scalarizable in returns
518 __ cbz(rscratch1, skip);
519 __ blr(rscratch1);
520
521 __ bind(skip);
522 }
523 // At this point, r0 points to the value object (for interpreter or C1 caller).
524 // The fields of the object are copied into registers (for C2 caller).
525 }
526
527 // Pop the stack before the safepoint code
528 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
529
530 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
531 __ reserved_stack_check();
532 }
533
534 code_stub->set_safepoint_offset(__ offset());
535 __ relocate(relocInfo::poll_return_type);
536 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
537 __ ret(lr);
538 }
539
540 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
541 return (__ store_inline_type_fields_to_buf(vk, false));
542 }
543
544 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
545 guarantee(info != nullptr, "Shouldn't be null");
546 __ get_polling_page(rscratch1, relocInfo::poll_type);
547 add_debug_info_for_branch(info); // This isn't just debug info:
548 // it's the oop map
549 __ read_polling_page(rscratch1, relocInfo::poll_type);
550 return __ offset();
551 }
552
553
554 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
555 if (from_reg == r31_sp)
556 from_reg = sp;
557 if (to_reg == r31_sp)
558 to_reg = sp;
559 __ mov(to_reg, from_reg);
560 }
561
562 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
563
564
565 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
566 assert(src->is_constant(), "should not call otherwise");
567 assert(dest->is_register(), "should not call otherwise");
568 LIR_Const* c = src->as_constant_ptr();
569
570 switch (c->type()) {
571 case T_INT: {
572 assert(patch_code == lir_patch_none, "no patching handled here");
573 __ movw(dest->as_register(), c->as_jint());
574 break;
575 }
576
577 case T_ADDRESS: {
578 assert(patch_code == lir_patch_none, "no patching handled here");
579 __ mov(dest->as_register(), c->as_jint());
580 break;
581 }
582
583 case T_LONG: {
584 assert(patch_code == lir_patch_none, "no patching handled here");
585 #if INCLUDE_CDS
586 if (AOTCodeCache::is_on_for_dump()) {
587 address b = c->as_pointer();
588 if (AOTRuntimeConstants::contains(b)) {
589 __ load_aotrc_address(dest->as_register_lo(), b);
590 break;
591 }
592 }
593 #endif
594 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
595 break;
596 }
597
598 case T_OBJECT: {
599 if (patch_code != lir_patch_none) {
600 jobject2reg_with_patching(dest->as_register(), info);
601 } else {
602 jobject2reg(c->as_jobject(), dest->as_register());
603 }
604 break;
605 }
606
607 case T_METADATA: {
608 if (patch_code != lir_patch_none) {
609 klass2reg_with_patching(dest->as_register(), info);
610 } else {
611 __ mov_metadata(dest->as_register(), c->as_metadata());
612 }
613 break;
614 }
615
616 case T_FLOAT: {
617 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
618 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
619 } else {
620 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
621 __ ldrs(dest->as_float_reg(), Address(rscratch1));
622 }
623 break;
624 }
625
626 case T_DOUBLE: {
627 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
628 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
629 } else {
630 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
631 __ ldrd(dest->as_double_reg(), Address(rscratch1));
632 }
633 break;
634 }
635
636 default:
637 ShouldNotReachHere();
638 }
639 }
640
641 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
642 LIR_Const* c = src->as_constant_ptr();
643 switch (c->type()) {
644 case T_OBJECT:
645 {
646 if (! c->as_jobject())
647 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
648 else {
649 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr);
650 reg2stack(FrameMap::rscratch1_opr, dest, c->type());
651 }
652 }
653 break;
654 case T_ADDRESS:
655 {
656 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr);
657 reg2stack(FrameMap::rscratch1_opr, dest, c->type());
658 }
659 case T_INT:
660 case T_FLOAT:
661 {
662 Register reg = zr;
663 if (c->as_jint_bits() == 0)
664 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
665 else {
666 __ movw(rscratch1, c->as_jint_bits());
667 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
668 }
669 }
670 break;
671 case T_LONG:
672 case T_DOUBLE:
673 {
674 Register reg = zr;
675 if (c->as_jlong_bits() == 0)
676 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
677 lo_word_offset_in_bytes));
678 else {
679 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
680 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
681 lo_word_offset_in_bytes));
682 }
683 }
684 break;
685 default:
686 ShouldNotReachHere();
687 }
688 }
689
690 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
691 assert(src->is_constant(), "should not call otherwise");
692 LIR_Const* c = src->as_constant_ptr();
693 LIR_Address* to_addr = dest->as_address_ptr();
694
695 void (Assembler::* insn)(Register Rt, const Address &adr);
696
697 switch (type) {
698 case T_ADDRESS:
699 assert(c->as_jint() == 0, "should be");
700 insn = &Assembler::str;
701 break;
702 case T_LONG:
703 assert(c->as_jlong() == 0, "should be");
704 insn = &Assembler::str;
705 break;
706 case T_INT:
707 assert(c->as_jint() == 0, "should be");
708 insn = &Assembler::strw;
709 break;
710 case T_OBJECT:
711 case T_ARRAY:
712 // Non-null case is not handled on aarch64 but handled on x86
713 // FIXME: do we need to add it here?
714 assert(c->as_jobject() == nullptr, "should be");
715 if (UseCompressedOops && !wide) {
716 insn = &Assembler::strw;
717 } else {
718 insn = &Assembler::str;
719 }
720 break;
721 case T_CHAR:
722 case T_SHORT:
723 assert(c->as_jint() == 0, "should be");
724 insn = &Assembler::strh;
725 break;
726 case T_BOOLEAN:
727 case T_BYTE:
728 assert(c->as_jint() == 0, "should be");
729 insn = &Assembler::strb;
730 break;
731 default:
732 ShouldNotReachHere();
733 insn = &Assembler::str; // unreachable
734 }
735
736 if (info) add_debug_info_for_null_check_here(info);
737 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
738 }
739
740 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
741 assert(src->is_register(), "should not call otherwise");
742 assert(dest->is_register(), "should not call otherwise");
743
744 // move between cpu-registers
745 if (dest->is_single_cpu()) {
746 if (src->type() == T_LONG) {
747 // Can do LONG -> OBJECT
748 move_regs(src->as_register_lo(), dest->as_register());
749 return;
750 }
751 assert(src->is_single_cpu(), "must match");
752 if (src->type() == T_OBJECT) {
753 __ verify_oop(src->as_register());
754 }
755 move_regs(src->as_register(), dest->as_register());
756
757 } else if (dest->is_double_cpu()) {
758 if (is_reference_type(src->type())) {
759 // Surprising to me but we can see move of a long to t_object
760 __ verify_oop(src->as_register());
761 move_regs(src->as_register(), dest->as_register_lo());
762 return;
763 }
764 assert(src->is_double_cpu(), "must match");
765 Register f_lo = src->as_register_lo();
766 Register f_hi = src->as_register_hi();
767 Register t_lo = dest->as_register_lo();
768 Register t_hi = dest->as_register_hi();
769 assert(f_hi == f_lo, "must be same");
770 assert(t_hi == t_lo, "must be same");
771 move_regs(f_lo, t_lo);
772
773 } else if (dest->is_single_fpu()) {
774 __ fmovs(dest->as_float_reg(), src->as_float_reg());
775
776 } else if (dest->is_double_fpu()) {
777 __ fmovd(dest->as_double_reg(), src->as_double_reg());
778
779 } else {
780 ShouldNotReachHere();
781 }
782 }
783
784 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
785 precond(src->is_register() && dest->is_stack());
786
787 uint const c_sz32 = sizeof(uint32_t);
788 uint const c_sz64 = sizeof(uint64_t);
789
790 if (src->is_single_cpu()) {
791 int index = dest->single_stack_ix();
792 if (is_reference_type(type)) {
793 __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
794 __ verify_oop(src->as_register());
795 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
796 __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
797 } else {
798 __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1));
799 }
800
801 } else if (src->is_double_cpu()) {
802 int index = dest->double_stack_ix();
803 Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
804 __ str(src->as_register_lo(), dest_addr_LO);
805
806 } else if (src->is_single_fpu()) {
807 int index = dest->single_stack_ix();
808 __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
809
810 } else if (src->is_double_fpu()) {
811 int index = dest->double_stack_ix();
812 __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
813
814 } else {
815 ShouldNotReachHere();
816 }
817 }
818
819
820 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
821 LIR_Address* to_addr = dest->as_address_ptr();
822 PatchingStub* patch = nullptr;
823 Register compressed_src = rscratch1;
824
825 if (patch_code != lir_patch_none) {
826 deoptimize_trap(info);
827 return;
828 }
829
830 if (is_reference_type(type)) {
831 __ verify_oop(src->as_register());
832
833 if (UseCompressedOops && !wide) {
834 __ encode_heap_oop(compressed_src, src->as_register());
835 } else {
836 compressed_src = src->as_register();
837 }
838 }
839
840 int null_check_here = code_offset();
841 switch (type) {
842 case T_FLOAT: {
843 __ strs(src->as_float_reg(), as_Address(to_addr));
844 break;
845 }
846
847 case T_DOUBLE: {
848 __ strd(src->as_double_reg(), as_Address(to_addr));
849 break;
850 }
851
852 case T_ARRAY: // fall through
853 case T_OBJECT: // fall through
854 if (UseCompressedOops && !wide) {
855 __ strw(compressed_src, as_Address(to_addr, rscratch2));
856 } else {
857 __ str(compressed_src, as_Address(to_addr));
858 }
859 break;
860 case T_METADATA:
861 // We get here to store a method pointer to the stack to pass to
862 // a dtrace runtime call. This can't work on 64 bit with
863 // compressed klass ptrs: T_METADATA can be a compressed klass
864 // ptr or a 64 bit method pointer.
865 ShouldNotReachHere();
866 __ str(src->as_register(), as_Address(to_addr));
867 break;
868 case T_ADDRESS:
869 __ str(src->as_register(), as_Address(to_addr));
870 break;
871 case T_INT:
872 __ strw(src->as_register(), as_Address(to_addr));
873 break;
874
875 case T_LONG: {
876 __ str(src->as_register_lo(), as_Address_lo(to_addr));
877 break;
878 }
879
880 case T_BYTE: // fall through
881 case T_BOOLEAN: {
882 __ strb(src->as_register(), as_Address(to_addr));
883 break;
884 }
885
886 case T_CHAR: // fall through
887 case T_SHORT:
888 __ strh(src->as_register(), as_Address(to_addr));
889 break;
890
891 default:
892 ShouldNotReachHere();
893 }
894 if (info != nullptr) {
895 add_debug_info_for_null_check(null_check_here, info);
896 }
897 }
898
899
900 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
901 precond(src->is_stack() && dest->is_register());
902
903 uint const c_sz32 = sizeof(uint32_t);
904 uint const c_sz64 = sizeof(uint64_t);
905
906 if (dest->is_single_cpu()) {
907 int index = src->single_stack_ix();
908 if (is_reference_type(type)) {
909 __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
910 __ verify_oop(dest->as_register());
911 } else if (type == T_METADATA || type == T_ADDRESS) {
912 __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
913 } else {
914 __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1));
915 }
916
917 } else if (dest->is_double_cpu()) {
918 int index = src->double_stack_ix();
919 Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
920 __ ldr(dest->as_register_lo(), src_addr_LO);
921
922 } else if (dest->is_single_fpu()) {
923 int index = src->single_stack_ix();
924 __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
925
926 } else if (dest->is_double_fpu()) {
927 int index = src->double_stack_ix();
928 __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
929
930 } else {
931 ShouldNotReachHere();
932 }
933 }
934
935
936 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
937 address target = nullptr;
938 relocInfo::relocType reloc_type = relocInfo::none;
939
940 switch (patching_id(info)) {
941 case PatchingStub::access_field_id:
942 target = Runtime1::entry_for(StubId::c1_access_field_patching_id);
943 reloc_type = relocInfo::section_word_type;
944 break;
945 case PatchingStub::load_klass_id:
946 target = Runtime1::entry_for(StubId::c1_load_klass_patching_id);
947 reloc_type = relocInfo::metadata_type;
948 break;
949 case PatchingStub::load_mirror_id:
950 target = Runtime1::entry_for(StubId::c1_load_mirror_patching_id);
951 reloc_type = relocInfo::oop_type;
952 break;
953 case PatchingStub::load_appendix_id:
954 target = Runtime1::entry_for(StubId::c1_load_appendix_patching_id);
955 reloc_type = relocInfo::oop_type;
956 break;
957 default: ShouldNotReachHere();
958 }
959
960 __ far_call(RuntimeAddress(target));
961 add_call_info_here(info);
962 }
963
964 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
965
966 LIR_Opr temp;
967 if (type == T_LONG || type == T_DOUBLE)
968 temp = FrameMap::rscratch1_long_opr;
969 else
970 temp = FrameMap::rscratch1_opr;
971
972 stack2reg(src, temp, src->type());
973 reg2stack(temp, dest, dest->type());
974 }
975
976
977 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
978 LIR_Address* addr = src->as_address_ptr();
979 LIR_Address* from_addr = src->as_address_ptr();
980
981 if (addr->base()->type() == T_OBJECT) {
982 __ verify_oop(addr->base()->as_pointer_register());
983 }
984
985 if (patch_code != lir_patch_none) {
986 deoptimize_trap(info);
987 return;
988 }
989
990 if (info != nullptr) {
991 add_debug_info_for_null_check_here(info);
992 }
993 int null_check_here = code_offset();
994 switch (type) {
995 case T_FLOAT: {
996 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
997 break;
998 }
999
1000 case T_DOUBLE: {
1001 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
1002 break;
1003 }
1004
1005 case T_ARRAY: // fall through
1006 case T_OBJECT: // fall through
1007 if (UseCompressedOops && !wide) {
1008 __ ldrw(dest->as_register(), as_Address(from_addr));
1009 } else {
1010 __ ldr(dest->as_register(), as_Address(from_addr));
1011 }
1012 break;
1013 case T_METADATA:
1014 // We get here to store a method pointer to the stack to pass to
1015 // a dtrace runtime call. This can't work on 64 bit with
1016 // compressed klass ptrs: T_METADATA can be a compressed klass
1017 // ptr or a 64 bit method pointer.
1018 ShouldNotReachHere();
1019 __ ldr(dest->as_register(), as_Address(from_addr));
1020 break;
1021 case T_ADDRESS:
1022 __ ldr(dest->as_register(), as_Address(from_addr));
1023 break;
1024 case T_INT:
1025 __ ldrw(dest->as_register(), as_Address(from_addr));
1026 break;
1027
1028 case T_LONG: {
1029 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1030 break;
1031 }
1032
1033 case T_BYTE:
1034 __ ldrsb(dest->as_register(), as_Address(from_addr));
1035 break;
1036 case T_BOOLEAN: {
1037 __ ldrb(dest->as_register(), as_Address(from_addr));
1038 break;
1039 }
1040
1041 case T_CHAR:
1042 __ ldrh(dest->as_register(), as_Address(from_addr));
1043 break;
1044 case T_SHORT:
1045 __ ldrsh(dest->as_register(), as_Address(from_addr));
1046 break;
1047
1048 default:
1049 ShouldNotReachHere();
1050 }
1051
1052 if (is_reference_type(type)) {
1053 if (UseCompressedOops && !wide) {
1054 __ decode_heap_oop(dest->as_register());
1055 }
1056
1057 __ verify_oop(dest->as_register());
1058 }
1059 }
1060
1061 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1062 assert(dst->is_cpu_register(), "must be");
1063 assert(dst->type() == src->type(), "must be");
1064
1065 if (src->is_cpu_register()) {
1066 reg2reg(src, dst);
1067 } else if (src->is_stack()) {
1068 stack2reg(src, dst, dst->type());
1069 } else if (src->is_constant()) {
1070 const2reg(src, dst, lir_patch_none, nullptr);
1071 } else {
1072 ShouldNotReachHere();
1073 }
1074 }
1075
1076 int LIR_Assembler::array_element_size(BasicType type) const {
1077 int elem_size = type2aelembytes(type);
1078 return exact_log2(elem_size);
1079 }
1080
1081
1082 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1083 switch (op->code()) {
1084 case lir_idiv:
1085 case lir_irem:
1086 arithmetic_idiv(op->code(),
1087 op->in_opr1(),
1088 op->in_opr2(),
1089 op->in_opr3(),
1090 op->result_opr(),
1091 op->info());
1092 break;
1093 case lir_fmad:
1094 __ fmaddd(op->result_opr()->as_double_reg(),
1095 op->in_opr1()->as_double_reg(),
1096 op->in_opr2()->as_double_reg(),
1097 op->in_opr3()->as_double_reg());
1098 break;
1099 case lir_fmaf:
1100 __ fmadds(op->result_opr()->as_float_reg(),
1101 op->in_opr1()->as_float_reg(),
1102 op->in_opr2()->as_float_reg(),
1103 op->in_opr3()->as_float_reg());
1104 break;
1105 default: ShouldNotReachHere(); break;
1106 }
1107 }
1108
1109 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1110 #ifdef ASSERT
1111 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1112 if (op->block() != nullptr) _branch_target_blocks.append(op->block());
1113 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1114 #endif
1115
1116 if (op->cond() == lir_cond_always) {
1117 if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1118 __ b(*(op->label()));
1119 } else {
1120 Assembler::Condition acond;
1121 if (op->code() == lir_cond_float_branch) {
1122 bool is_unordered = (op->ublock() == op->block());
1123 // Assembler::EQ does not permit unordered branches, so we add
1124 // another branch here. Likewise, Assembler::NE does not permit
1125 // ordered branches.
1126 if ((is_unordered && op->cond() == lir_cond_equal)
1127 || (!is_unordered && op->cond() == lir_cond_notEqual))
1128 __ br(Assembler::VS, *(op->ublock()->label()));
1129 switch(op->cond()) {
1130 case lir_cond_equal: acond = Assembler::EQ; break;
1131 case lir_cond_notEqual: acond = Assembler::NE; break;
1132 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1133 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1134 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1135 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1136 default: ShouldNotReachHere();
1137 acond = Assembler::EQ; // unreachable
1138 }
1139 } else {
1140 switch (op->cond()) {
1141 case lir_cond_equal: acond = Assembler::EQ; break;
1142 case lir_cond_notEqual: acond = Assembler::NE; break;
1143 case lir_cond_less: acond = Assembler::LT; break;
1144 case lir_cond_lessEqual: acond = Assembler::LE; break;
1145 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1146 case lir_cond_greater: acond = Assembler::GT; break;
1147 case lir_cond_belowEqual: acond = Assembler::LS; break;
1148 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1149 default: ShouldNotReachHere();
1150 acond = Assembler::EQ; // unreachable
1151 }
1152 }
1153 __ br(acond,*(op->label()));
1154 }
1155 }
1156
1157
1158
1159 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1160 LIR_Opr src = op->in_opr();
1161 LIR_Opr dest = op->result_opr();
1162
1163 switch (op->bytecode()) {
1164 case Bytecodes::_i2f:
1165 {
1166 __ scvtfws(dest->as_float_reg(), src->as_register());
1167 break;
1168 }
1169 case Bytecodes::_i2d:
1170 {
1171 __ scvtfwd(dest->as_double_reg(), src->as_register());
1172 break;
1173 }
1174 case Bytecodes::_l2d:
1175 {
1176 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1177 break;
1178 }
1179 case Bytecodes::_l2f:
1180 {
1181 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1182 break;
1183 }
1184 case Bytecodes::_f2d:
1185 {
1186 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1187 break;
1188 }
1189 case Bytecodes::_d2f:
1190 {
1191 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1192 break;
1193 }
1194 case Bytecodes::_i2c:
1195 {
1196 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1197 break;
1198 }
1199 case Bytecodes::_i2l:
1200 {
1201 __ sxtw(dest->as_register_lo(), src->as_register());
1202 break;
1203 }
1204 case Bytecodes::_i2s:
1205 {
1206 __ sxth(dest->as_register(), src->as_register());
1207 break;
1208 }
1209 case Bytecodes::_i2b:
1210 {
1211 __ sxtb(dest->as_register(), src->as_register());
1212 break;
1213 }
1214 case Bytecodes::_l2i:
1215 {
1216 _masm->block_comment("FIXME: This could be a no-op");
1217 __ uxtw(dest->as_register(), src->as_register_lo());
1218 break;
1219 }
1220 case Bytecodes::_d2l:
1221 {
1222 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1223 break;
1224 }
1225 case Bytecodes::_f2i:
1226 {
1227 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1228 break;
1229 }
1230 case Bytecodes::_f2l:
1231 {
1232 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1233 break;
1234 }
1235 case Bytecodes::_d2i:
1236 {
1237 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1238 break;
1239 }
1240 default: ShouldNotReachHere();
1241 }
1242 }
1243
1244 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1245 if (op->init_check()) {
1246 __ lea(rscratch1, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1247 __ ldarb(rscratch1, rscratch1);
1248 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1249 add_debug_info_for_null_check_here(op->stub()->info());
1250 __ br(Assembler::NE, *op->stub()->entry());
1251 }
1252 __ allocate_object(op->obj()->as_register(),
1253 op->tmp1()->as_register(),
1254 op->tmp2()->as_register(),
1255 op->header_size(),
1256 op->object_size(),
1257 op->klass()->as_register(),
1258 *op->stub()->entry());
1259 __ bind(*op->stub()->continuation());
1260 }
1261
1262 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1263 Register len = op->len()->as_register();
1264 __ uxtw(len, len);
1265
1266 if (UseSlowPath || op->always_slow_path() ||
1267 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1268 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1269 __ b(*op->stub()->entry());
1270 } else {
1271 Register tmp1 = op->tmp1()->as_register();
1272 Register tmp2 = op->tmp2()->as_register();
1273 Register tmp3 = op->tmp3()->as_register();
1274 if (len == tmp1) {
1275 tmp1 = tmp3;
1276 } else if (len == tmp2) {
1277 tmp2 = tmp3;
1278 } else if (len == tmp3) {
1279 // everything is ok
1280 } else {
1281 __ mov(tmp3, len);
1282 }
1283 __ allocate_array(op->obj()->as_register(),
1284 len,
1285 tmp1,
1286 tmp2,
1287 arrayOopDesc::base_offset_in_bytes(op->type()),
1288 array_element_size(op->type()),
1289 op->klass()->as_register(),
1290 *op->stub()->entry(),
1291 op->zero_array());
1292 }
1293 __ bind(*op->stub()->continuation());
1294 }
1295
1296 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md,
1297 ciProfileData *data, Register recv) {
1298
1299 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1300 __ profile_receiver_type(recv, mdo, mdp_offset);
1301 }
1302
1303 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1304 // we always need a stub for the failure case.
1305 CodeStub* stub = op->stub();
1306 Register obj = op->object()->as_register();
1307 Register k_RInfo = op->tmp1()->as_register();
1308 Register klass_RInfo = op->tmp2()->as_register();
1309 Register dst = op->result_opr()->as_register();
1310 ciKlass* k = op->klass();
1311 Register Rtmp1 = noreg;
1312
1313 // check if it needs to be profiled
1314 ciMethodData* md;
1315 ciProfileData* data;
1316
1317 const bool should_profile = op->should_profile();
1318
1319 if (should_profile) {
1320 ciMethod* method = op->profiled_method();
1321 assert(method != nullptr, "Should have method");
1322 int bci = op->profiled_bci();
1323 md = method->method_data_or_null();
1324 assert(md != nullptr, "Sanity");
1325 data = md->bci_to_data(bci);
1326 assert(data != nullptr, "need data for type check");
1327 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1328 }
1329 Label* success_target = success;
1330 Label* failure_target = failure;
1331
1332 if (obj == k_RInfo) {
1333 k_RInfo = dst;
1334 } else if (obj == klass_RInfo) {
1335 klass_RInfo = dst;
1336 }
1337 if (k->is_loaded() && !UseCompressedClassPointers) {
1338 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1339 } else {
1340 Rtmp1 = op->tmp3()->as_register();
1341 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1342 }
1343
1344 assert_different_registers(obj, k_RInfo, klass_RInfo);
1345
1346 if (op->need_null_check()) {
1347 if (should_profile) {
1348 Register mdo = klass_RInfo;
1349 __ mov_metadata(mdo, md->constant_encoding());
1350 Label not_null;
1351 __ cbnz(obj, not_null);
1352 // Object is null; update MDO and exit
1353 Address data_addr
1354 = __ form_address(rscratch2, mdo,
1355 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1356 0);
1357 __ ldrb(rscratch1, data_addr);
1358 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1359 __ strb(rscratch1, data_addr);
1360 __ b(*obj_is_null);
1361 __ bind(not_null);
1362
1363 Register recv = k_RInfo;
1364 __ load_klass(recv, obj);
1365 type_profile_helper(mdo, md, data, recv);
1366 } else {
1367 __ cbz(obj, *obj_is_null);
1368 }
1369 }
1370
1371 if (!k->is_loaded()) {
1372 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1373 } else {
1374 __ mov_metadata(k_RInfo, k->constant_encoding());
1375 }
1376 __ verify_oop(obj);
1377
1378 if (op->fast_check()) {
1379 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1380 // get object class
1381 // not a safepoint as obj null check happens earlier
1382 __ load_klass(rscratch1, obj);
1383 __ cmp( rscratch1, k_RInfo);
1384
1385 __ br(Assembler::NE, *failure_target);
1386 // successful cast, fall through to profile or jump
1387 } else {
1388 // get object class
1389 // not a safepoint as obj null check happens earlier
1390 __ load_klass(klass_RInfo, obj);
1391 if (k->is_loaded()) {
1392 // See if we get an immediate positive hit
1393 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1394 __ cmp(k_RInfo, rscratch1);
1395 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1396 __ br(Assembler::NE, *failure_target);
1397 // successful cast, fall through to profile or jump
1398 } else {
1399 // See if we get an immediate positive hit
1400 __ br(Assembler::EQ, *success_target);
1401 // check for self
1402 if (k->is_loaded() && k->is_obj_array_klass()) {
1403 // For a direct pointer comparison, we need the refined array klass pointer
1404 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1405 __ mov_metadata(rscratch1, k_refined->constant_encoding());
1406 __ cmp(klass_RInfo, rscratch1);
1407 } else {
1408 __ cmp(klass_RInfo, k_RInfo);
1409 }
1410 __ br(Assembler::EQ, *success_target);
1411
1412 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1413 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1414 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1415 // result is a boolean
1416 __ cbzw(klass_RInfo, *failure_target);
1417 // successful cast, fall through to profile or jump
1418 }
1419 } else {
1420 // perform the fast part of the checking logic
1421 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1422 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1423 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1424 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1425 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1426 // result is a boolean
1427 __ cbz(k_RInfo, *failure_target);
1428 // successful cast, fall through to profile or jump
1429 }
1430 }
1431 __ b(*success);
1432 }
1433
1434
1435 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1436 const bool should_profile = op->should_profile();
1437
1438 LIR_Code code = op->code();
1439 if (code == lir_store_check) {
1440 Register value = op->object()->as_register();
1441 Register array = op->array()->as_register();
1442 Register k_RInfo = op->tmp1()->as_register();
1443 Register klass_RInfo = op->tmp2()->as_register();
1444 Register Rtmp1 = op->tmp3()->as_register();
1445
1446 CodeStub* stub = op->stub();
1447
1448 // check if it needs to be profiled
1449 ciMethodData* md;
1450 ciProfileData* data;
1451
1452 if (should_profile) {
1453 ciMethod* method = op->profiled_method();
1454 assert(method != nullptr, "Should have method");
1455 int bci = op->profiled_bci();
1456 md = method->method_data_or_null();
1457 assert(md != nullptr, "Sanity");
1458 data = md->bci_to_data(bci);
1459 assert(data != nullptr, "need data for type check");
1460 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1461 }
1462 Label done;
1463 Label* success_target = &done;
1464 Label* failure_target = stub->entry();
1465
1466 if (should_profile) {
1467 Label not_null;
1468 Register mdo = klass_RInfo;
1469 __ mov_metadata(mdo, md->constant_encoding());
1470 __ cbnz(value, not_null);
1471 // Object is null; update MDO and exit
1472 Address data_addr
1473 = __ form_address(rscratch2, mdo,
1474 md->byte_offset_of_slot(data, DataLayout::flags_offset()), 0);
1475 __ ldrb(rscratch1, data_addr);
1476 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1477 __ strb(rscratch1, data_addr);
1478 __ b(done);
1479 __ bind(not_null);
1480
1481 Register recv = k_RInfo;
1482 __ load_klass(recv, value);
1483 type_profile_helper(mdo, md, data, recv);
1484 } else {
1485 __ cbz(value, done);
1486 }
1487
1488 add_debug_info_for_null_check_here(op->info_for_exception());
1489 __ load_klass(k_RInfo, array);
1490 __ load_klass(klass_RInfo, value);
1491
1492 // get instance klass (it's already uncompressed)
1493 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1494 // perform the fast part of the checking logic
1495 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1496 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1497 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1498 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1499 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1500 // result is a boolean
1501 __ cbzw(k_RInfo, *failure_target);
1502 // fall through to the success case
1503
1504 __ bind(done);
1505 } else if (code == lir_checkcast) {
1506 Register obj = op->object()->as_register();
1507 Register dst = op->result_opr()->as_register();
1508 Label success;
1509 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1510 __ bind(success);
1511 if (dst != obj) {
1512 __ mov(dst, obj);
1513 }
1514 } else if (code == lir_instanceof) {
1515 Register obj = op->object()->as_register();
1516 Register dst = op->result_opr()->as_register();
1517 Label success, failure, done;
1518 emit_typecheck_helper(op, &success, &failure, &failure);
1519 __ bind(failure);
1520 __ mov(dst, zr);
1521 __ b(done);
1522 __ bind(success);
1523 __ mov(dst, 1);
1524 __ bind(done);
1525 } else {
1526 ShouldNotReachHere();
1527 }
1528 }
1529
1530 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1531 // We are loading/storing from/to an array that *may* be a flat array (the
1532 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1533 // If this array is a flat array, take the slow path.
1534 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1535 if (!op->value()->is_illegal()) {
1536 // The array is not a flat array, but it might be null-free. If we are storing
1537 // a null into a null-free array, take the slow path (which will throw NPE).
1538 Label skip;
1539 __ cbnz(op->value()->as_register(), skip);
1540 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1541 __ bind(skip);
1542 }
1543 }
1544
1545 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1546 // We are storing into an array that *may* be null-free (the declared type is
1547 // Object[], abstract[], interface[] or VT.ref[]).
1548 Label test_mark_word;
1549 Register tmp = op->tmp()->as_register();
1550 __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1551 __ tst(tmp, markWord::unlocked_value);
1552 __ br(Assembler::NE, test_mark_word);
1553 __ load_prototype_header(tmp, op->array()->as_register());
1554 __ bind(test_mark_word);
1555 __ tst(tmp, markWord::null_free_array_bit_in_place);
1556 }
1557
1558 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1559 Label L_oops_equal;
1560 Label L_oops_not_equal;
1561 Label L_end;
1562
1563 Register left = op->left()->as_register();
1564 Register right = op->right()->as_register();
1565
1566 __ cmp(left, right);
1567 __ br(Assembler::EQ, L_oops_equal);
1568
1569 // (1) Null check -- if one of the operands is null, the other must not be null (because
1570 // the two references are not equal), so they are not substitutable,
1571 // FIXME: do null check only if the operand is nullable
1572 {
1573 __ cbz(left, L_oops_not_equal);
1574 __ cbz(right, L_oops_not_equal);
1575 }
1576
1577 ciKlass* left_klass = op->left_klass();
1578 ciKlass* right_klass = op->right_klass();
1579
1580 // (2) Inline type check -- if either of the operands is not a inline type,
1581 // they are not substitutable. We do this only if we are not sure that the
1582 // operands are inline type
1583 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1584 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1585 Register tmp1 = op->tmp1()->as_register();
1586 Register tmp2 = op->tmp2()->as_register();
1587 __ mov(tmp1, markWord::inline_type_pattern);
1588 __ ldr(tmp2, Address(left, oopDesc::mark_offset_in_bytes()));
1589 __ andr(tmp1, tmp1, tmp2);
1590 __ ldr(tmp2, Address(right, oopDesc::mark_offset_in_bytes()));
1591 __ andr(tmp1, tmp1, tmp2);
1592 __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1593 __ br(Assembler::NE, L_oops_not_equal);
1594 }
1595
1596 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1597 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1598 // No need to load klass -- the operands are statically known to be the same inline klass.
1599 __ b(*op->stub()->entry());
1600 } else {
1601 Register tmp1 = op->tmp1()->as_register();
1602 Register tmp2 = op->tmp2()->as_register();
1603 __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1604 __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1605 // fall through to L_oops_not_equal
1606 }
1607
1608 __ bind(L_oops_not_equal);
1609 move(op->not_equal_result(), op->result_opr());
1610 __ b(L_end);
1611
1612 __ bind(L_oops_equal);
1613 move(op->equal_result(), op->result_opr());
1614 __ b(L_end);
1615
1616 // We've returned from the stub. R0 contains 0x0 IFF the two
1617 // operands are not substitutable. (Don't compare against 0x1 in case the
1618 // C compiler is naughty)
1619 __ bind(*op->stub()->continuation());
1620 __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1621 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1622 // fall-through
1623 __ bind(L_end);
1624 }
1625
1626
1627 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1628 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1629 __ cset(rscratch1, Assembler::NE);
1630 __ membar(__ AnyAny);
1631 }
1632
1633 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1634 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1635 __ cset(rscratch1, Assembler::NE);
1636 __ membar(__ AnyAny);
1637 }
1638
1639
1640 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1641 Register addr;
1642 if (op->addr()->is_register()) {
1643 addr = as_reg(op->addr());
1644 } else {
1645 assert(op->addr()->is_address(), "what else?");
1646 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1647 assert(addr_ptr->disp() == 0, "need 0 disp");
1648 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1649 addr = as_reg(addr_ptr->base());
1650 }
1651 Register newval = as_reg(op->new_value());
1652 Register cmpval = as_reg(op->cmp_value());
1653
1654 if (op->code() == lir_cas_obj) {
1655 if (UseCompressedOops) {
1656 Register t1 = op->tmp1()->as_register();
1657 assert(op->tmp1()->is_valid(), "must be");
1658 __ encode_heap_oop(t1, cmpval);
1659 cmpval = t1;
1660 __ encode_heap_oop(rscratch2, newval);
1661 newval = rscratch2;
1662 casw(addr, newval, cmpval);
1663 } else {
1664 casl(addr, newval, cmpval);
1665 }
1666 } else if (op->code() == lir_cas_int) {
1667 casw(addr, newval, cmpval);
1668 } else {
1669 casl(addr, newval, cmpval);
1670 }
1671 }
1672
1673
1674 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1675 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1676 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on aarch64");
1677
1678 Assembler::Condition acond, ncond;
1679 switch (condition) {
1680 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1681 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1682 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1683 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1684 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1685 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1686 case lir_cond_belowEqual:
1687 case lir_cond_aboveEqual:
1688 default: ShouldNotReachHere();
1689 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1690 }
1691
1692 assert(result->is_single_cpu() || result->is_double_cpu(),
1693 "expect single register for result");
1694 if (opr1->is_constant() && opr2->is_constant()
1695 && opr1->type() == T_INT && opr2->type() == T_INT) {
1696 jint val1 = opr1->as_jint();
1697 jint val2 = opr2->as_jint();
1698 if (val1 == 0 && val2 == 1) {
1699 __ cset(result->as_register(), ncond);
1700 return;
1701 } else if (val1 == 1 && val2 == 0) {
1702 __ cset(result->as_register(), acond);
1703 return;
1704 }
1705 }
1706
1707 if (opr1->is_constant() && opr2->is_constant()
1708 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1709 jlong val1 = opr1->as_jlong();
1710 jlong val2 = opr2->as_jlong();
1711 if (val1 == 0 && val2 == 1) {
1712 __ cset(result->as_register_lo(), ncond);
1713 return;
1714 } else if (val1 == 1 && val2 == 0) {
1715 __ cset(result->as_register_lo(), acond);
1716 return;
1717 }
1718 }
1719
1720 if (opr1->is_stack()) {
1721 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1722 opr1 = FrameMap::rscratch1_opr;
1723 } else if (opr1->is_constant()) {
1724 LIR_Opr tmp
1725 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1726 const2reg(opr1, tmp, lir_patch_none, nullptr);
1727 opr1 = tmp;
1728 }
1729
1730 if (opr2->is_stack()) {
1731 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1732 opr2 = FrameMap::rscratch2_opr;
1733 } else if (opr2->is_constant()) {
1734 LIR_Opr tmp
1735 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1736 const2reg(opr2, tmp, lir_patch_none, nullptr);
1737 opr2 = tmp;
1738 }
1739
1740 if (result->type() == T_LONG)
1741 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1742 else
1743 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1744 }
1745
1746 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1747 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1748
1749 if (left->is_single_cpu()) {
1750 Register lreg = left->as_register();
1751 Register dreg = as_reg(dest);
1752
1753 if (right->is_single_cpu()) {
1754 // cpu register - cpu register
1755
1756 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1757 "should be");
1758 Register rreg = right->as_register();
1759 switch (code) {
1760 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1761 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1762 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1763 default: ShouldNotReachHere();
1764 }
1765
1766 } else if (right->is_double_cpu()) {
1767 Register rreg = right->as_register_lo();
1768 // single_cpu + double_cpu: can happen with obj+long
1769 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1770 switch (code) {
1771 case lir_add: __ add(dreg, lreg, rreg); break;
1772 case lir_sub: __ sub(dreg, lreg, rreg); break;
1773 default: ShouldNotReachHere();
1774 }
1775 } else if (right->is_constant()) {
1776 // cpu register - constant
1777 jlong c;
1778
1779 // FIXME. This is fugly: we really need to factor all this logic.
1780 switch(right->type()) {
1781 case T_LONG:
1782 c = right->as_constant_ptr()->as_jlong();
1783 break;
1784 case T_INT:
1785 case T_ADDRESS:
1786 c = right->as_constant_ptr()->as_jint();
1787 break;
1788 default:
1789 ShouldNotReachHere();
1790 c = 0; // unreachable
1791 break;
1792 }
1793
1794 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1795 if (c == 0 && dreg == lreg) {
1796 COMMENT("effective nop elided");
1797 return;
1798 }
1799 switch(left->type()) {
1800 case T_INT:
1801 switch (code) {
1802 case lir_add: __ addw(dreg, lreg, c); break;
1803 case lir_sub: __ subw(dreg, lreg, c); break;
1804 default: ShouldNotReachHere();
1805 }
1806 break;
1807 case T_OBJECT:
1808 case T_ADDRESS:
1809 switch (code) {
1810 case lir_add: __ add(dreg, lreg, c); break;
1811 case lir_sub: __ sub(dreg, lreg, c); break;
1812 default: ShouldNotReachHere();
1813 }
1814 break;
1815 default:
1816 ShouldNotReachHere();
1817 }
1818 } else {
1819 ShouldNotReachHere();
1820 }
1821
1822 } else if (left->is_double_cpu()) {
1823 Register lreg_lo = left->as_register_lo();
1824
1825 if (right->is_double_cpu()) {
1826 // cpu register - cpu register
1827 Register rreg_lo = right->as_register_lo();
1828 switch (code) {
1829 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1830 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1831 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1832 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1833 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1834 default:
1835 ShouldNotReachHere();
1836 }
1837
1838 } else if (right->is_constant()) {
1839 jlong c = right->as_constant_ptr()->as_jlong();
1840 Register dreg = as_reg(dest);
1841 switch (code) {
1842 case lir_add:
1843 case lir_sub:
1844 if (c == 0 && dreg == lreg_lo) {
1845 COMMENT("effective nop elided");
1846 return;
1847 }
1848 code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1849 break;
1850 case lir_div:
1851 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1852 if (c == 1) {
1853 // move lreg_lo to dreg if divisor is 1
1854 __ mov(dreg, lreg_lo);
1855 } else {
1856 unsigned int shift = log2i_exact(c);
1857 // use rscratch1 as intermediate result register
1858 __ asr(rscratch1, lreg_lo, 63);
1859 __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1860 __ asr(dreg, rscratch1, shift);
1861 }
1862 break;
1863 case lir_rem:
1864 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1865 if (c == 1) {
1866 // move 0 to dreg if divisor is 1
1867 __ mov(dreg, zr);
1868 } else {
1869 // use rscratch1 as intermediate result register
1870 __ negs(rscratch1, lreg_lo);
1871 __ andr(dreg, lreg_lo, c - 1);
1872 __ andr(rscratch1, rscratch1, c - 1);
1873 __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1874 }
1875 break;
1876 default:
1877 ShouldNotReachHere();
1878 }
1879 } else {
1880 ShouldNotReachHere();
1881 }
1882 } else if (left->is_single_fpu()) {
1883 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1884 switch (code) {
1885 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1886 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1887 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1888 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1889 default:
1890 ShouldNotReachHere();
1891 }
1892 } else if (left->is_double_fpu()) {
1893 if (right->is_double_fpu()) {
1894 // fpu register - fpu register
1895 switch (code) {
1896 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1897 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1898 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1899 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1900 default:
1901 ShouldNotReachHere();
1902 }
1903 } else {
1904 if (right->is_constant()) {
1905 ShouldNotReachHere();
1906 }
1907 ShouldNotReachHere();
1908 }
1909 } else if (left->is_single_stack() || left->is_address()) {
1910 assert(left == dest, "left and dest must be equal");
1911 ShouldNotReachHere();
1912 } else {
1913 ShouldNotReachHere();
1914 }
1915 }
1916
1917 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1918 switch(code) {
1919 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1920 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1921 case lir_f2hf: __ flt_to_flt16(dest->as_register(), value->as_float_reg(), tmp->as_float_reg()); break;
1922 case lir_hf2f: __ flt16_to_flt(dest->as_float_reg(), value->as_register(), tmp->as_float_reg()); break;
1923 default : ShouldNotReachHere();
1924 }
1925 }
1926
1927 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1928
1929 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1930 Register Rleft = left->is_single_cpu() ? left->as_register() :
1931 left->as_register_lo();
1932 if (dst->is_single_cpu()) {
1933 Register Rdst = dst->as_register();
1934 if (right->is_constant()) {
1935 switch (code) {
1936 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1937 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1938 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1939 default: ShouldNotReachHere(); break;
1940 }
1941 } else {
1942 Register Rright = right->is_single_cpu() ? right->as_register() :
1943 right->as_register_lo();
1944 switch (code) {
1945 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1946 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1947 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1948 default: ShouldNotReachHere(); break;
1949 }
1950 }
1951 } else {
1952 Register Rdst = dst->as_register_lo();
1953 if (right->is_constant()) {
1954 switch (code) {
1955 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1956 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1957 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1958 default: ShouldNotReachHere(); break;
1959 }
1960 } else {
1961 Register Rright = right->is_single_cpu() ? right->as_register() :
1962 right->as_register_lo();
1963 switch (code) {
1964 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1965 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1966 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1967 default: ShouldNotReachHere(); break;
1968 }
1969 }
1970 }
1971 }
1972
1973
1974
1975 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
1976
1977 // opcode check
1978 assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
1979 bool is_irem = (code == lir_irem);
1980
1981 // operand check
1982 assert(left->is_single_cpu(), "left must be register");
1983 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
1984 assert(result->is_single_cpu(), "result must be register");
1985 Register lreg = left->as_register();
1986 Register dreg = result->as_register();
1987
1988 // power-of-2 constant check and codegen
1989 if (right->is_constant()) {
1990 int c = right->as_constant_ptr()->as_jint();
1991 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1992 if (is_irem) {
1993 if (c == 1) {
1994 // move 0 to dreg if divisor is 1
1995 __ movw(dreg, zr);
1996 } else {
1997 // use rscratch1 as intermediate result register
1998 __ negsw(rscratch1, lreg);
1999 __ andw(dreg, lreg, c - 1);
2000 __ andw(rscratch1, rscratch1, c - 1);
2001 __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
2002 }
2003 } else {
2004 if (c == 1) {
2005 // move lreg to dreg if divisor is 1
2006 __ movw(dreg, lreg);
2007 } else {
2008 unsigned int shift = exact_log2(c);
2009 // use rscratch1 as intermediate result register
2010 __ asrw(rscratch1, lreg, 31);
2011 __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
2012 __ asrw(dreg, rscratch1, shift);
2013 }
2014 }
2015 } else {
2016 Register rreg = right->as_register();
2017 __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
2018 }
2019 }
2020
2021
2022 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2023 if (opr1->is_constant() && opr2->is_single_cpu()) {
2024 // tableswitch
2025 Register reg = as_reg(opr2);
2026 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
2027 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
2028 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
2029 Register reg1 = as_reg(opr1);
2030 if (opr2->is_single_cpu()) {
2031 // cpu register - cpu register
2032 Register reg2 = opr2->as_register();
2033 if (is_reference_type(opr1->type())) {
2034 __ cmpoop(reg1, reg2);
2035 } else {
2036 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2037 __ cmpw(reg1, reg2);
2038 }
2039 return;
2040 }
2041 if (opr2->is_double_cpu()) {
2042 // cpu register - cpu register
2043 Register reg2 = opr2->as_register_lo();
2044 __ cmp(reg1, reg2);
2045 return;
2046 }
2047
2048 if (opr2->is_constant()) {
2049 bool is_32bit = false; // width of register operand
2050 jlong imm;
2051
2052 switch(opr2->type()) {
2053 case T_INT:
2054 imm = opr2->as_constant_ptr()->as_jint();
2055 is_32bit = true;
2056 break;
2057 case T_LONG:
2058 imm = opr2->as_constant_ptr()->as_jlong();
2059 break;
2060 case T_ADDRESS:
2061 imm = opr2->as_constant_ptr()->as_jint();
2062 break;
2063 case T_METADATA:
2064 imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
2065 break;
2066 case T_OBJECT:
2067 case T_ARRAY:
2068 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
2069 __ cmpoop(reg1, rscratch1);
2070 return;
2071 default:
2072 ShouldNotReachHere();
2073 imm = 0; // unreachable
2074 break;
2075 }
2076
2077 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
2078 if (is_32bit)
2079 __ cmpw(reg1, imm);
2080 else
2081 __ subs(zr, reg1, imm);
2082 return;
2083 } else {
2084 __ mov(rscratch1, imm);
2085 if (is_32bit)
2086 __ cmpw(reg1, rscratch1);
2087 else
2088 __ cmp(reg1, rscratch1);
2089 return;
2090 }
2091 } else
2092 ShouldNotReachHere();
2093 } else if (opr1->is_single_fpu()) {
2094 FloatRegister reg1 = opr1->as_float_reg();
2095 assert(opr2->is_single_fpu(), "expect single float register");
2096 FloatRegister reg2 = opr2->as_float_reg();
2097 __ fcmps(reg1, reg2);
2098 } else if (opr1->is_double_fpu()) {
2099 FloatRegister reg1 = opr1->as_double_reg();
2100 assert(opr2->is_double_fpu(), "expect double float register");
2101 FloatRegister reg2 = opr2->as_double_reg();
2102 __ fcmpd(reg1, reg2);
2103 } else {
2104 ShouldNotReachHere();
2105 }
2106 }
2107
2108 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2109 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2110 bool is_unordered_less = (code == lir_ucmp_fd2i);
2111 if (left->is_single_fpu()) {
2112 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2113 } else if (left->is_double_fpu()) {
2114 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2115 } else {
2116 ShouldNotReachHere();
2117 }
2118 } else if (code == lir_cmp_l2i) {
2119 Label done;
2120 __ cmp(left->as_register_lo(), right->as_register_lo());
2121 __ mov(dst->as_register(), (uint64_t)-1L);
2122 __ br(Assembler::LT, done);
2123 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2124 __ bind(done);
2125 } else {
2126 ShouldNotReachHere();
2127 }
2128 }
2129
2130
2131 void LIR_Assembler::align_call(LIR_Code code) { }
2132
2133
2134 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2135 address call = __ trampoline_call(Address(op->addr(), rtype));
2136 if (call == nullptr) {
2137 bailout("trampoline stub overflow");
2138 return;
2139 }
2140 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2141 __ post_call_nop();
2142 }
2143
2144
2145 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2146 address call = __ ic_call(op->addr());
2147 if (call == nullptr) {
2148 bailout("trampoline stub overflow");
2149 return;
2150 }
2151 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2152 __ post_call_nop();
2153 }
2154
2155 void LIR_Assembler::emit_static_call_stub() {
2156 address call_pc = __ pc();
2157 address stub = __ start_a_stub(call_stub_size());
2158 if (stub == nullptr) {
2159 bailout("static call stub overflow");
2160 return;
2161 }
2162
2163 int start = __ offset();
2164
2165 __ relocate(static_stub_Relocation::spec(call_pc));
2166 __ emit_static_call_stub();
2167
2168 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2169 <= call_stub_size(), "stub too big");
2170 __ end_a_stub();
2171 }
2172
2173
2174 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2175 assert(exceptionOop->as_register() == r0, "must match");
2176 assert(exceptionPC->as_register() == r3, "must match");
2177
2178 // exception object is not added to oop map by LinearScan
2179 // (LinearScan assumes that no oops are in fixed registers)
2180 info->add_register_oop(exceptionOop);
2181 StubId unwind_id;
2182
2183 // get current pc information
2184 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2185 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
2186 // As no instructions have been generated yet for this LIR node it's
2187 // possible that an oop map already exists for the current offset.
2188 // In that case insert an dummy NOP here to ensure all oop map PCs
2189 // are unique. See JDK-8237483.
2190 __ nop();
2191 }
2192 int pc_for_athrow_offset = __ offset();
2193 InternalAddress pc_for_athrow(__ pc());
2194 __ adr(exceptionPC->as_register(), pc_for_athrow);
2195 add_call_info(pc_for_athrow_offset, info); // for exception handler
2196
2197 __ verify_not_null_oop(r0);
2198 // search an exception handler (r0: exception oop, r3: throwing pc)
2199 if (compilation()->has_fpu_code()) {
2200 unwind_id = StubId::c1_handle_exception_id;
2201 } else {
2202 unwind_id = StubId::c1_handle_exception_nofpu_id;
2203 }
2204 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2205
2206 // FIXME: enough room for two byte trap ????
2207 __ nop();
2208 }
2209
2210
2211 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2212 assert(exceptionOop->as_register() == r0, "must match");
2213
2214 __ b(_unwind_handler_entry);
2215 }
2216
2217
2218 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2219 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2220 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2221
2222 switch (left->type()) {
2223 case T_INT: {
2224 switch (code) {
2225 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2226 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2227 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2228 default:
2229 ShouldNotReachHere();
2230 break;
2231 }
2232 break;
2233 case T_LONG:
2234 case T_ADDRESS:
2235 case T_OBJECT:
2236 switch (code) {
2237 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2238 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2239 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2240 default:
2241 ShouldNotReachHere();
2242 break;
2243 }
2244 break;
2245 default:
2246 ShouldNotReachHere();
2247 break;
2248 }
2249 }
2250 }
2251
2252
2253 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2254 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2255 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2256
2257 switch (left->type()) {
2258 case T_INT: {
2259 switch (code) {
2260 case lir_shl: __ lslw (dreg, lreg, count); break;
2261 case lir_shr: __ asrw (dreg, lreg, count); break;
2262 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2263 default:
2264 ShouldNotReachHere();
2265 break;
2266 }
2267 break;
2268 case T_LONG:
2269 case T_ADDRESS:
2270 case T_OBJECT:
2271 switch (code) {
2272 case lir_shl: __ lsl (dreg, lreg, count); break;
2273 case lir_shr: __ asr (dreg, lreg, count); break;
2274 case lir_ushr: __ lsr (dreg, lreg, count); break;
2275 default:
2276 ShouldNotReachHere();
2277 break;
2278 }
2279 break;
2280 default:
2281 ShouldNotReachHere();
2282 break;
2283 }
2284 }
2285 }
2286
2287
2288 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2289 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2290 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2291 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2292 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2293 }
2294
2295
2296 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2297 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2298 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2299 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2300 __ mov (rscratch1, c);
2301 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2302 }
2303
2304
2305 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2306 ShouldNotReachHere();
2307 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2308 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2309 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2310 __ lea(rscratch1, __ constant_oop_address(o));
2311 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2312 }
2313
2314 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2315 if (null_check) {
2316 __ cbz(obj, *slow_path->entry());
2317 }
2318 if (is_dest) {
2319 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2320 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2321 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2322 } else {
2323 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2324 }
2325 }
2326
2327 // This code replaces a call to arraycopy; no exception may
2328 // be thrown in this code, they must be thrown in the System.arraycopy
2329 // activation frame; we could save some checks if this would not be the case
2330 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2331 ciArrayKlass* default_type = op->expected_type();
2332 Register src = op->src()->as_register();
2333 Register dst = op->dst()->as_register();
2334 Register src_pos = op->src_pos()->as_register();
2335 Register dst_pos = op->dst_pos()->as_register();
2336 Register length = op->length()->as_register();
2337 Register tmp = op->tmp()->as_register();
2338
2339 CodeStub* stub = op->stub();
2340 int flags = op->flags();
2341 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2342 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2343
2344 if (flags & LIR_OpArrayCopy::always_slow_path) {
2345 __ b(*stub->entry());
2346 __ bind(*stub->continuation());
2347 return;
2348 }
2349
2350 // if we don't know anything, just go through the generic arraycopy
2351 if (default_type == nullptr // || basic_type == T_OBJECT
2352 ) {
2353 Label done;
2354 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2355
2356 // Save the arguments in case the generic arraycopy fails and we
2357 // have to fall back to the JNI stub
2358 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2359 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2360 __ str(src, Address(sp, 4*BytesPerWord));
2361
2362 address copyfunc_addr = StubRoutines::generic_arraycopy();
2363 assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2364
2365 // The arguments are in java calling convention so we shift them
2366 // to C convention
2367 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2368 __ mov(c_rarg0, j_rarg0);
2369 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2370 __ mov(c_rarg1, j_rarg1);
2371 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2372 __ mov(c_rarg2, j_rarg2);
2373 assert_different_registers(c_rarg3, j_rarg4);
2374 __ mov(c_rarg3, j_rarg3);
2375 __ mov(c_rarg4, j_rarg4);
2376 #ifndef PRODUCT
2377 if (PrintC1Statistics) {
2378 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2379 }
2380 #endif
2381 __ far_call(RuntimeAddress(copyfunc_addr));
2382
2383 __ cbz(r0, *stub->continuation());
2384
2385 // Reload values from the stack so they are where the stub
2386 // expects them.
2387 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2388 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2389 __ ldr(src, Address(sp, 4*BytesPerWord));
2390
2391 // r0 is -1^K where K == partial copied count
2392 __ eonw(rscratch1, r0, zr);
2393 // adjust length down and src/end pos up by partial copied count
2394 __ subw(length, length, rscratch1);
2395 __ addw(src_pos, src_pos, rscratch1);
2396 __ addw(dst_pos, dst_pos, rscratch1);
2397 __ b(*stub->entry());
2398
2399 __ bind(*stub->continuation());
2400 return;
2401 }
2402
2403 // Handle inline type arrays
2404 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2405 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2406 }
2407 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2408 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2409 }
2410
2411 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2412
2413 int elem_size = type2aelembytes(basic_type);
2414 int scale = exact_log2(elem_size);
2415
2416 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2417 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2418
2419 // test for null
2420 if (flags & LIR_OpArrayCopy::src_null_check) {
2421 __ cbz(src, *stub->entry());
2422 }
2423 if (flags & LIR_OpArrayCopy::dst_null_check) {
2424 __ cbz(dst, *stub->entry());
2425 }
2426
2427 // If the compiler was not able to prove that exact type of the source or the destination
2428 // of the arraycopy is an array type, check at runtime if the source or the destination is
2429 // an instance type.
2430 if (flags & LIR_OpArrayCopy::type_check) {
2431 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2432 __ load_klass(tmp, dst);
2433 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2434 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2435 __ br(Assembler::GE, *stub->entry());
2436 }
2437
2438 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2439 __ load_klass(tmp, src);
2440 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2441 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2442 __ br(Assembler::GE, *stub->entry());
2443 }
2444 }
2445
2446 // check if negative
2447 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2448 __ cmpw(src_pos, 0);
2449 __ br(Assembler::LT, *stub->entry());
2450 }
2451 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2452 __ cmpw(dst_pos, 0);
2453 __ br(Assembler::LT, *stub->entry());
2454 }
2455
2456 if (flags & LIR_OpArrayCopy::length_positive_check) {
2457 __ cmpw(length, 0);
2458 __ br(Assembler::LT, *stub->entry());
2459 }
2460
2461 if (flags & LIR_OpArrayCopy::src_range_check) {
2462 __ addw(tmp, src_pos, length);
2463 __ ldrw(rscratch1, src_length_addr);
2464 __ cmpw(tmp, rscratch1);
2465 __ br(Assembler::HI, *stub->entry());
2466 }
2467 if (flags & LIR_OpArrayCopy::dst_range_check) {
2468 __ addw(tmp, dst_pos, length);
2469 __ ldrw(rscratch1, dst_length_addr);
2470 __ cmpw(tmp, rscratch1);
2471 __ br(Assembler::HI, *stub->entry());
2472 }
2473
2474 if (flags & LIR_OpArrayCopy::type_check) {
2475 // We don't know the array types are compatible
2476 if (basic_type != T_OBJECT) {
2477 // Simple test for basic type arrays
2478 __ cmp_klasses_from_objects(src, dst, tmp, rscratch1);
2479 __ br(Assembler::NE, *stub->entry());
2480 } else {
2481 // For object arrays, if src is a sub class of dst then we can
2482 // safely do the copy.
2483 Label cont, slow;
2484
2485 #define PUSH(r1, r2) \
2486 stp(r1, r2, __ pre(sp, -2 * wordSize));
2487
2488 #define POP(r1, r2) \
2489 ldp(r1, r2, __ post(sp, 2 * wordSize));
2490
2491 __ PUSH(src, dst);
2492
2493 __ load_klass(src, src);
2494 __ load_klass(dst, dst);
2495
2496 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2497
2498 __ PUSH(src, dst);
2499 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2500 __ POP(src, dst);
2501
2502 __ cbnz(src, cont);
2503
2504 __ bind(slow);
2505 __ POP(src, dst);
2506
2507 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2508 if (copyfunc_addr != nullptr) { // use stub if available
2509 // src is not a sub class of dst so we have to do a
2510 // per-element check.
2511
2512 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2513 if ((flags & mask) != mask) {
2514 // Check that at least both of them object arrays.
2515 assert(flags & mask, "one of the two should be known to be an object array");
2516
2517 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2518 __ load_klass(tmp, src);
2519 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2520 __ load_klass(tmp, dst);
2521 }
2522 int lh_offset = in_bytes(Klass::layout_helper_offset());
2523 Address klass_lh_addr(tmp, lh_offset);
2524 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2525 __ ldrw(rscratch1, klass_lh_addr);
2526 __ mov(rscratch2, objArray_lh);
2527 __ eorw(rscratch1, rscratch1, rscratch2);
2528 __ cbnzw(rscratch1, *stub->entry());
2529 }
2530
2531 // Spill because stubs can use any register they like and it's
2532 // easier to restore just those that we care about.
2533 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2534 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2535 __ str(src, Address(sp, 4*BytesPerWord));
2536
2537 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2538 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2539 assert_different_registers(c_rarg0, dst, dst_pos, length);
2540 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2541 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2542 assert_different_registers(c_rarg1, dst, length);
2543 __ uxtw(c_rarg2, length);
2544 assert_different_registers(c_rarg2, dst);
2545
2546 __ load_klass(c_rarg4, dst);
2547 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2548 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2549 __ far_call(RuntimeAddress(copyfunc_addr));
2550
2551 #ifndef PRODUCT
2552 if (PrintC1Statistics) {
2553 Label failed;
2554 __ cbnz(r0, failed);
2555 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2556 __ bind(failed);
2557 }
2558 #endif
2559
2560 __ cbz(r0, *stub->continuation());
2561
2562 #ifndef PRODUCT
2563 if (PrintC1Statistics) {
2564 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2565 }
2566 #endif
2567 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2568
2569 // Restore previously spilled arguments
2570 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2571 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2572 __ ldr(src, Address(sp, 4*BytesPerWord));
2573
2574 // return value is -1^K where K is partial copied count
2575 __ eonw(rscratch1, r0, zr);
2576 // adjust length down and src/end pos up by partial copied count
2577 __ subw(length, length, rscratch1);
2578 __ addw(src_pos, src_pos, rscratch1);
2579 __ addw(dst_pos, dst_pos, rscratch1);
2580 }
2581
2582 __ b(*stub->entry());
2583
2584 __ bind(cont);
2585 __ POP(src, dst);
2586 }
2587 }
2588
2589 #ifdef ASSERT
2590 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2591 // Sanity check the known type with the incoming class. For the
2592 // primitive case the types must match exactly with src.klass and
2593 // dst.klass each exactly matching the default type. For the
2594 // object array case, if no type check is needed then either the
2595 // dst type is exactly the expected type and the src type is a
2596 // subtype which we can't check or src is the same array as dst
2597 // but not necessarily exactly of type default_type.
2598 Label known_ok, halt;
2599 __ mov_metadata(tmp, default_type->constant_encoding());
2600
2601 if (basic_type != T_OBJECT) {
2602 __ cmp_klass(dst, tmp, rscratch1);
2603 __ br(Assembler::NE, halt);
2604 __ cmp_klass(src, tmp, rscratch1);
2605 __ br(Assembler::EQ, known_ok);
2606 } else {
2607 __ cmp_klass(dst, tmp, rscratch1);
2608 __ br(Assembler::EQ, known_ok);
2609 __ cmp(src, dst);
2610 __ br(Assembler::EQ, known_ok);
2611 }
2612 __ bind(halt);
2613 __ stop("incorrect type information in arraycopy");
2614 __ bind(known_ok);
2615 }
2616 #endif
2617
2618 #ifndef PRODUCT
2619 if (PrintC1Statistics) {
2620 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2621 }
2622 #endif
2623
2624 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2625 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2626 assert_different_registers(c_rarg0, dst, dst_pos, length);
2627 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2628 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2629 assert_different_registers(c_rarg1, dst, length);
2630 __ uxtw(c_rarg2, length);
2631 assert_different_registers(c_rarg2, dst);
2632
2633 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2634 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2635 const char *name;
2636 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2637
2638 CodeBlob *cb = CodeCache::find_blob(entry);
2639 if (cb) {
2640 __ far_call(RuntimeAddress(entry));
2641 } else {
2642 __ call_VM_leaf(entry, 3);
2643 }
2644
2645 if (stub != nullptr) {
2646 __ bind(*stub->continuation());
2647 }
2648 }
2649
2650
2651
2652
2653 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2654 Register obj = op->obj_opr()->as_register(); // may not be an oop
2655 Register hdr = op->hdr_opr()->as_register();
2656 Register lock = op->lock_opr()->as_register();
2657 Register temp = op->scratch_opr()->as_register();
2658 if (op->code() == lir_lock) {
2659 // add debug info for NullPointerException only if one is possible
2660 int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
2661 if (op->info() != nullptr) {
2662 add_debug_info_for_null_check(null_check_offset, op->info());
2663 }
2664 // done
2665 } else if (op->code() == lir_unlock) {
2666 __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
2667 } else {
2668 Unimplemented();
2669 }
2670 __ bind(*op->stub()->continuation());
2671 }
2672
2673 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2674 Register obj = op->obj()->as_pointer_register();
2675 Register result = op->result_opr()->as_pointer_register();
2676
2677 CodeEmitInfo* info = op->info();
2678 if (info != nullptr) {
2679 add_debug_info_for_null_check_here(info);
2680 }
2681
2682 __ load_klass(result, obj);
2683 }
2684
2685 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2686 ciMethod* method = op->profiled_method();
2687 int bci = op->profiled_bci();
2688 ciMethod* callee = op->profiled_callee();
2689
2690 // Update counter for all call types
2691 ciMethodData* md = method->method_data_or_null();
2692 assert(md != nullptr, "Sanity");
2693 ciProfileData* data = md->bci_to_data(bci);
2694 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2695 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2696 Register mdo = op->mdo()->as_register();
2697 __ mov_metadata(mdo, md->constant_encoding());
2698 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2699 // Perform additional virtual call profiling for invokevirtual and
2700 // invokeinterface bytecodes
2701 if (op->should_profile_receiver_type()) {
2702 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2703 Register recv = op->recv()->as_register();
2704 assert_different_registers(mdo, recv);
2705 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2706 ciKlass* known_klass = op->known_holder();
2707 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2708 // We know the type that will be seen at this call site; we can
2709 // statically update the MethodData* rather than needing to do
2710 // dynamic tests on the receiver type.
2711 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2712 for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2713 ciKlass* receiver = vc_data->receiver(i);
2714 if (known_klass->equals(receiver)) {
2715 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2716 __ addptr(data_addr, DataLayout::counter_increment);
2717 return;
2718 }
2719 }
2720 // Receiver type is not found in profile data.
2721 // Fall back to runtime helper to handle the rest at runtime.
2722 __ mov_metadata(recv, known_klass->constant_encoding());
2723 } else {
2724 __ load_klass(recv, recv);
2725 }
2726 type_profile_helper(mdo, md, data, recv);
2727 } else {
2728 // Static call
2729 __ addptr(counter_addr, DataLayout::counter_increment);
2730 }
2731 }
2732
2733
2734 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2735 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2736 }
2737
2738 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2739 assert(op->crc()->is_single_cpu(), "crc must be register");
2740 assert(op->val()->is_single_cpu(), "byte value must be register");
2741 assert(op->result_opr()->is_single_cpu(), "result must be register");
2742 Register crc = op->crc()->as_register();
2743 Register val = op->val()->as_register();
2744 Register res = op->result_opr()->as_register();
2745
2746 assert_different_registers(val, crc, res);
2747 uint64_t offset;
2748 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2749 __ add(res, res, offset);
2750
2751 __ mvnw(crc, crc); // ~crc
2752 __ update_byte_crc32(crc, val, res);
2753 __ mvnw(res, crc); // ~crc
2754 }
2755
2756 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2757 COMMENT("emit_profile_type {");
2758 Register obj = op->obj()->as_register();
2759 Register tmp = op->tmp()->as_pointer_register();
2760 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2761 ciKlass* exact_klass = op->exact_klass();
2762 intptr_t current_klass = op->current_klass();
2763 bool not_null = op->not_null();
2764 bool no_conflict = op->no_conflict();
2765
2766 Label update, next, none;
2767
2768 bool do_null = !not_null;
2769 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2770 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2771
2772 assert(do_null || do_update, "why are we here?");
2773 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2774 assert(mdo_addr.base() != rscratch1, "wrong register");
2775
2776 __ verify_oop(obj);
2777
2778 if (tmp != obj) {
2779 assert_different_registers(obj, tmp, rscratch1, rscratch2, mdo_addr.base(), mdo_addr.index());
2780 __ mov(tmp, obj);
2781 } else {
2782 assert_different_registers(obj, rscratch1, rscratch2, mdo_addr.base(), mdo_addr.index());
2783 }
2784 if (do_null) {
2785 __ cbnz(tmp, update);
2786 if (!TypeEntries::was_null_seen(current_klass)) {
2787 __ ldr(rscratch2, mdo_addr);
2788 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2789 __ str(rscratch2, mdo_addr);
2790 }
2791 if (do_update) {
2792 #ifndef ASSERT
2793 __ b(next);
2794 }
2795 #else
2796 __ b(next);
2797 }
2798 } else {
2799 __ cbnz(tmp, update);
2800 __ stop("unexpected null obj");
2801 #endif
2802 }
2803
2804 __ bind(update);
2805
2806 if (do_update) {
2807 #ifdef ASSERT
2808 if (exact_klass != nullptr) {
2809 Label ok;
2810 __ load_klass(tmp, tmp);
2811 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2812 __ eor(rscratch1, tmp, rscratch1);
2813 __ cbz(rscratch1, ok);
2814 __ stop("exact klass and actual klass differ");
2815 __ bind(ok);
2816 }
2817 #endif
2818 if (!no_conflict) {
2819 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2820 if (exact_klass != nullptr) {
2821 __ mov_metadata(tmp, exact_klass->constant_encoding());
2822 } else {
2823 __ load_klass(tmp, tmp);
2824 }
2825
2826 __ ldr(rscratch2, mdo_addr);
2827 __ eor(tmp, tmp, rscratch2);
2828 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2829 // klass seen before, nothing to do. The unknown bit may have been
2830 // set already but no need to check.
2831 __ cbz(rscratch1, next);
2832
2833 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2834
2835 if (TypeEntries::is_type_none(current_klass)) {
2836 __ cbz(rscratch2, none);
2837 __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2838 __ br(Assembler::EQ, none);
2839 // There is a chance that the checks above
2840 // fail if another thread has just set the
2841 // profiling to this obj's klass
2842 __ dmb(Assembler::ISHLD);
2843 __ eor(tmp, tmp, rscratch2); // get back original value before XOR
2844 __ ldr(rscratch2, mdo_addr);
2845 __ eor(tmp, tmp, rscratch2);
2846 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2847 __ cbz(rscratch1, next);
2848 }
2849 } else {
2850 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2851 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2852
2853 __ ldr(tmp, mdo_addr);
2854 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2855 }
2856
2857 // different than before. Cannot keep accurate profile.
2858 __ ldr(rscratch2, mdo_addr);
2859 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2860 __ str(rscratch2, mdo_addr);
2861
2862 if (TypeEntries::is_type_none(current_klass)) {
2863 __ b(next);
2864
2865 __ bind(none);
2866 // first time here. Set profile type.
2867 __ str(tmp, mdo_addr);
2868 #ifdef ASSERT
2869 __ andr(tmp, tmp, TypeEntries::type_mask);
2870 __ verify_klass_ptr(tmp);
2871 #endif
2872 }
2873 } else {
2874 // There's a single possible klass at this profile point
2875 assert(exact_klass != nullptr, "should be");
2876 if (TypeEntries::is_type_none(current_klass)) {
2877 __ mov_metadata(tmp, exact_klass->constant_encoding());
2878 __ ldr(rscratch2, mdo_addr);
2879 __ eor(tmp, tmp, rscratch2);
2880 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2881 __ cbz(rscratch1, next);
2882 #ifdef ASSERT
2883 {
2884 Label ok;
2885 __ ldr(rscratch1, mdo_addr);
2886 __ cbz(rscratch1, ok);
2887 __ cmp(rscratch1, (u1)TypeEntries::null_seen);
2888 __ br(Assembler::EQ, ok);
2889 // may have been set by another thread
2890 __ dmb(Assembler::ISHLD);
2891 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2892 __ ldr(rscratch2, mdo_addr);
2893 __ eor(rscratch2, rscratch1, rscratch2);
2894 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2895 __ cbz(rscratch2, ok);
2896
2897 __ stop("unexpected profiling mismatch");
2898 __ bind(ok);
2899 }
2900 #endif
2901 // first time here. Set profile type.
2902 __ str(tmp, mdo_addr);
2903 #ifdef ASSERT
2904 __ andr(tmp, tmp, TypeEntries::type_mask);
2905 __ verify_klass_ptr(tmp);
2906 #endif
2907 } else {
2908 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2909 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2910
2911 __ ldr(tmp, mdo_addr);
2912 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2913
2914 __ orr(tmp, tmp, TypeEntries::type_unknown);
2915 __ str(tmp, mdo_addr);
2916 // FIXME: Write barrier needed here?
2917 }
2918 }
2919
2920 __ bind(next);
2921 }
2922 COMMENT("} emit_profile_type");
2923 }
2924
2925 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
2926 Register obj = op->obj()->as_register();
2927 Register tmp = op->tmp()->as_pointer_register();
2928 bool not_null = op->not_null();
2929 int flag = op->flag();
2930
2931 Label not_inline_type;
2932 if (!not_null) {
2933 __ cbz(obj, not_inline_type);
2934 }
2935
2936 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
2937
2938 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), rscratch2);
2939 __ ldrb(rscratch1, mdo_addr);
2940 __ orr(rscratch1, rscratch1, flag);
2941 __ strb(rscratch1, mdo_addr);
2942
2943 __ bind(not_inline_type);
2944 }
2945
2946 void LIR_Assembler::align_backward_branch_target() {
2947 }
2948
2949
2950 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2951 // tmp must be unused
2952 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2953
2954 if (left->is_single_cpu()) {
2955 assert(dest->is_single_cpu(), "expect single result reg");
2956 __ negw(dest->as_register(), left->as_register());
2957 } else if (left->is_double_cpu()) {
2958 assert(dest->is_double_cpu(), "expect double result reg");
2959 __ neg(dest->as_register_lo(), left->as_register_lo());
2960 } else if (left->is_single_fpu()) {
2961 assert(dest->is_single_fpu(), "expect single float result reg");
2962 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2963 } else {
2964 assert(left->is_double_fpu(), "expect double float operand reg");
2965 assert(dest->is_double_fpu(), "expect double float result reg");
2966 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2967 }
2968 }
2969
2970
2971 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2972 if (patch_code != lir_patch_none) {
2973 deoptimize_trap(info);
2974 return;
2975 }
2976
2977 __ lea(dest->as_pointer_register(), as_Address(addr->as_address_ptr()));
2978 }
2979
2980
2981 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2982 assert(!tmp->is_valid(), "don't need temporary");
2983
2984 CodeBlob *cb = CodeCache::find_blob(dest);
2985 if (cb) {
2986 __ far_call(RuntimeAddress(dest));
2987 } else {
2988 __ mov(rscratch1, RuntimeAddress(dest));
2989 __ blr(rscratch1);
2990 }
2991
2992 if (info != nullptr) {
2993 add_call_info_here(info);
2994 }
2995 __ post_call_nop();
2996 }
2997
2998 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2999 if (dest->is_address() || src->is_address()) {
3000 move_op(src, dest, type, lir_patch_none, info, /*wide*/false);
3001 } else {
3002 ShouldNotReachHere();
3003 }
3004 }
3005
3006 #ifdef ASSERT
3007 // emit run-time assertion
3008 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3009 assert(op->code() == lir_assert, "must be");
3010
3011 if (op->in_opr1()->is_valid()) {
3012 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3013 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3014 } else {
3015 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3016 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3017 }
3018
3019 Label ok;
3020 if (op->condition() != lir_cond_always) {
3021 Assembler::Condition acond = Assembler::AL;
3022 switch (op->condition()) {
3023 case lir_cond_equal: acond = Assembler::EQ; break;
3024 case lir_cond_notEqual: acond = Assembler::NE; break;
3025 case lir_cond_less: acond = Assembler::LT; break;
3026 case lir_cond_lessEqual: acond = Assembler::LE; break;
3027 case lir_cond_greaterEqual: acond = Assembler::GE; break;
3028 case lir_cond_greater: acond = Assembler::GT; break;
3029 case lir_cond_belowEqual: acond = Assembler::LS; break;
3030 case lir_cond_aboveEqual: acond = Assembler::HS; break;
3031 default: ShouldNotReachHere();
3032 }
3033 __ br(acond, ok);
3034 }
3035 if (op->halt()) {
3036 const char* str = __ code_string(op->msg());
3037 __ stop(str);
3038 } else {
3039 breakpoint();
3040 }
3041 __ bind(ok);
3042 }
3043 #endif
3044
3045 #ifndef PRODUCT
3046 #define COMMENT(x) do { __ block_comment(x); } while (0)
3047 #else
3048 #define COMMENT(x)
3049 #endif
3050
3051 void LIR_Assembler::membar() {
3052 COMMENT("membar");
3053 __ membar(MacroAssembler::AnyAny);
3054 }
3055
3056 void LIR_Assembler::membar_acquire() {
3057 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3058 }
3059
3060 void LIR_Assembler::membar_release() {
3061 __ membar(Assembler::LoadStore|Assembler::StoreStore);
3062 }
3063
3064 void LIR_Assembler::membar_loadload() {
3065 __ membar(Assembler::LoadLoad);
3066 }
3067
3068 void LIR_Assembler::membar_storestore() {
3069 __ membar(MacroAssembler::StoreStore);
3070 }
3071
3072 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3073
3074 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3075
3076 void LIR_Assembler::on_spin_wait() {
3077 __ spin_wait();
3078 }
3079
3080 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3081 __ mov(result_reg->as_register(), rthread);
3082 }
3083
3084 void LIR_Assembler::check_orig_pc() {
3085 __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3086 __ cmp(rscratch2, (u1)NULL_WORD);
3087 }
3088
3089 void LIR_Assembler::peephole(LIR_List *lir) {
3090 #if 0
3091 if (tableswitch_count >= max_tableswitches)
3092 return;
3093
3094 /*
3095 This finite-state automaton recognizes sequences of compare-and-
3096 branch instructions. We will turn them into a tableswitch. You
3097 could argue that C1 really shouldn't be doing this sort of
3098 optimization, but without it the code is really horrible.
3099 */
3100
3101 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3102 int first_key, last_key = -2147483648;
3103 int next_key = 0;
3104 int start_insn = -1;
3105 int last_insn = -1;
3106 Register reg = noreg;
3107 LIR_Opr reg_opr;
3108 state = start_s;
3109
3110 LIR_OpList* inst = lir->instructions_list();
3111 for (int i = 0; i < inst->length(); i++) {
3112 LIR_Op* op = inst->at(i);
3113 switch (state) {
3114 case start_s:
3115 first_key = -1;
3116 start_insn = i;
3117 switch (op->code()) {
3118 case lir_cmp:
3119 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3120 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3121 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3122 && opr2->is_constant()
3123 && opr2->type() == T_INT) {
3124 reg_opr = opr1;
3125 reg = opr1->as_register();
3126 first_key = opr2->as_constant_ptr()->as_jint();
3127 next_key = first_key + 1;
3128 state = cmp_s;
3129 goto next_state;
3130 }
3131 break;
3132 }
3133 break;
3134 case cmp_s:
3135 switch (op->code()) {
3136 case lir_branch:
3137 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3138 state = beq_s;
3139 last_insn = i;
3140 goto next_state;
3141 }
3142 }
3143 state = start_s;
3144 break;
3145 case beq_s:
3146 switch (op->code()) {
3147 case lir_cmp: {
3148 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3149 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3150 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3151 && opr1->as_register() == reg
3152 && opr2->is_constant()
3153 && opr2->type() == T_INT
3154 && opr2->as_constant_ptr()->as_jint() == next_key) {
3155 last_key = next_key;
3156 next_key++;
3157 state = cmp_s;
3158 goto next_state;
3159 }
3160 }
3161 }
3162 last_key = next_key;
3163 state = start_s;
3164 break;
3165 default:
3166 assert(false, "impossible state");
3167 }
3168 if (state == start_s) {
3169 if (first_key < last_key - 5L && reg != noreg) {
3170 {
3171 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3172 // reg->encoding(),
3173 // start_insn, first_key, last_key);
3174 // for (int i = 0; i < inst->length(); i++) {
3175 // inst->at(i)->print();
3176 // tty->print("\n");
3177 // }
3178 // tty->print("\n");
3179 }
3180
3181 struct tableswitch *sw = &switches[tableswitch_count];
3182 sw->_insn_index = start_insn, sw->_first_key = first_key,
3183 sw->_last_key = last_key, sw->_reg = reg;
3184 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3185 {
3186 // Insert the new table of branches
3187 int offset = last_insn;
3188 for (int n = first_key; n < last_key; n++) {
3189 inst->insert_before
3190 (last_insn + 1,
3191 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3192 inst->at(offset)->as_OpBranch()->label()));
3193 offset -= 2, i++;
3194 }
3195 }
3196 // Delete all the old compare-and-branch instructions
3197 for (int n = first_key; n < last_key; n++) {
3198 inst->remove_at(start_insn);
3199 inst->remove_at(start_insn);
3200 }
3201 // Insert the tableswitch instruction
3202 inst->insert_before(start_insn,
3203 new LIR_Op2(lir_cmp, lir_cond_always,
3204 LIR_OprFact::intConst(tableswitch_count),
3205 reg_opr));
3206 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3207 tableswitch_count++;
3208 }
3209 reg = noreg;
3210 last_key = -2147483648;
3211 }
3212 next_state:
3213 ;
3214 }
3215 #endif
3216 }
3217
3218 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3219 Address addr = as_Address(src->as_address_ptr());
3220 BasicType type = src->type();
3221 bool is_oop = is_reference_type(type);
3222
3223 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3224 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3225
3226 switch(type) {
3227 case T_INT:
3228 xchg = &MacroAssembler::atomic_xchgalw;
3229 add = &MacroAssembler::atomic_addalw;
3230 break;
3231 case T_LONG:
3232 xchg = &MacroAssembler::atomic_xchgal;
3233 add = &MacroAssembler::atomic_addal;
3234 break;
3235 case T_OBJECT:
3236 case T_ARRAY:
3237 if (UseCompressedOops) {
3238 xchg = &MacroAssembler::atomic_xchgalw;
3239 add = &MacroAssembler::atomic_addalw;
3240 } else {
3241 xchg = &MacroAssembler::atomic_xchgal;
3242 add = &MacroAssembler::atomic_addal;
3243 }
3244 break;
3245 default:
3246 ShouldNotReachHere();
3247 xchg = &MacroAssembler::atomic_xchgal;
3248 add = &MacroAssembler::atomic_addal; // unreachable
3249 }
3250
3251 switch (code) {
3252 case lir_xadd:
3253 {
3254 RegisterOrConstant inc;
3255 Register tmp = as_reg(tmp_op);
3256 Register dst = as_reg(dest);
3257 if (data->is_constant()) {
3258 inc = RegisterOrConstant(as_long(data));
3259 assert_different_registers(dst, addr.base(), tmp,
3260 rscratch1, rscratch2);
3261 } else {
3262 inc = RegisterOrConstant(as_reg(data));
3263 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3264 rscratch1, rscratch2);
3265 }
3266 __ lea(tmp, addr);
3267 (_masm->*add)(dst, inc, tmp);
3268 break;
3269 }
3270 case lir_xchg:
3271 {
3272 Register tmp = tmp_op->as_register();
3273 Register obj = as_reg(data);
3274 Register dst = as_reg(dest);
3275 if (is_oop && UseCompressedOops) {
3276 __ encode_heap_oop(rscratch2, obj);
3277 obj = rscratch2;
3278 }
3279 assert_different_registers(obj, addr.base(), tmp, rscratch1);
3280 assert_different_registers(dst, addr.base(), tmp, rscratch1);
3281 __ lea(tmp, addr);
3282 (_masm->*xchg)(dst, obj, tmp);
3283 if (is_oop && UseCompressedOops) {
3284 __ decode_heap_oop(dst);
3285 }
3286 }
3287 break;
3288 default:
3289 ShouldNotReachHere();
3290 }
3291 if(!UseLSE) {
3292 __ membar(__ AnyAny);
3293 }
3294 }
3295
3296 #undef __