1 /*
2 * Copyright (c) 2013, Red Hat Inc.
3 * Copyright (c) 2000, 2020, Oracle and/or its affiliates.
4 * All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "gc_interface/collectedHeap.hpp"
38 #include "memory/barrierSet.hpp"
39 #include "memory/cardTableModRefBS.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "vmreg_aarch64.inline.hpp"
44
45
46
47 #ifndef PRODUCT
48 #define COMMENT(x) do { __ block_comment(x); } while (0)
49 #else
50 #define COMMENT(x)
51 #endif
52
53 NEEDS_CLEANUP // remove this definitions ?
54 const Register IC_Klass = rscratch2; // where the IC klass is cached
55 const Register SYNC_header = r0; // synchronization header
56 const Register SHIFT_count = r0; // where count for shift operations must be
57
58 #define __ _masm->
59
60
61 static void select_different_registers(Register preserve,
62 Register extra,
63 Register &tmp1,
64 Register &tmp2) {
65 if (tmp1 == preserve) {
66 assert_different_registers(tmp1, tmp2, extra);
67 tmp1 = extra;
68 } else if (tmp2 == preserve) {
69 assert_different_registers(tmp1, tmp2, extra);
70 tmp2 = extra;
71 }
72 assert_different_registers(preserve, tmp1, tmp2);
73 }
74
75
76
77 static void select_different_registers(Register preserve,
78 Register extra,
79 Register &tmp1,
80 Register &tmp2,
81 Register &tmp3) {
82 if (tmp1 == preserve) {
83 assert_different_registers(tmp1, tmp2, tmp3, extra);
84 tmp1 = extra;
85 } else if (tmp2 == preserve) {
86 assert_different_registers(tmp1, tmp2, tmp3, extra);
87 tmp2 = extra;
88 } else if (tmp3 == preserve) {
89 assert_different_registers(tmp1, tmp2, tmp3, extra);
90 tmp3 = extra;
91 }
92 assert_different_registers(preserve, tmp1, tmp2, tmp3);
93 }
94
95
96 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
97
98
99 LIR_Opr LIR_Assembler::receiverOpr() {
100 return FrameMap::receiver_opr;
101 }
102
103 LIR_Opr LIR_Assembler::osrBufferPointer() {
104 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
105 }
106
107 //--------------fpu register translations-----------------------
108
109
110 address LIR_Assembler::float_constant(float f) {
111 address const_addr = __ float_constant(f);
112 if (const_addr == NULL) {
113 bailout("const section overflow");
114 return __ code()->consts()->start();
115 } else {
116 return const_addr;
117 }
118 }
119
120
121 address LIR_Assembler::double_constant(double d) {
122 address const_addr = __ double_constant(d);
123 if (const_addr == NULL) {
124 bailout("const section overflow");
125 return __ code()->consts()->start();
126 } else {
127 return const_addr;
128 }
129 }
130
131 address LIR_Assembler::int_constant(jlong n) {
132 address const_addr = __ long_constant(n);
133 if (const_addr == NULL) {
134 bailout("const section overflow");
135 return __ code()->consts()->start();
136 } else {
137 return const_addr;
138 }
139 }
140
141 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
142
143 void LIR_Assembler::reset_FPU() { Unimplemented(); }
144
145 void LIR_Assembler::fpop() { Unimplemented(); }
146
147 void LIR_Assembler::fxch(int i) { Unimplemented(); }
148
149 void LIR_Assembler::fld(int i) { Unimplemented(); }
150
151 void LIR_Assembler::ffree(int i) { Unimplemented(); }
152
153 void LIR_Assembler::breakpoint() { Unimplemented(); }
154
155 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
156
157 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
158
159 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
160 //-------------------------------------------
161
162 static Register as_reg(LIR_Opr op) {
163 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
164 }
165
166 static jlong as_long(LIR_Opr data) {
167 jlong result;
168 switch (data->type()) {
169 case T_INT:
170 result = (data->as_jint());
171 break;
172 case T_LONG:
173 result = (data->as_jlong());
174 break;
175 default:
176 ShouldNotReachHere();
177 }
178 return result;
179 }
180
181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
182 Register base = addr->base()->as_pointer_register();
183 LIR_Opr opr = addr->index();
184 if (opr->is_cpu_register()) {
185 Register index;
186 if (opr->is_single_cpu())
187 index = opr->as_register();
188 else
189 index = opr->as_register_lo();
190 assert(addr->disp() == 0, "must be");
191 switch(opr->type()) {
192 case T_INT:
193 return Address(base, index, Address::sxtw(addr->scale()));
194 case T_LONG:
195 return Address(base, index, Address::lsl(addr->scale()));
196 default:
197 ShouldNotReachHere();
198 }
199 } else {
200 intptr_t addr_offset = intptr_t(addr->disp());
201 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
202 return Address(base, addr_offset, Address::lsl(addr->scale()));
203 else {
204 __ mov(tmp, addr_offset);
205 return Address(base, tmp, Address::lsl(addr->scale()));
206 }
207 }
208 return Address();
209 }
210
211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
212 ShouldNotReachHere();
213 return Address();
214 }
215
216 Address LIR_Assembler::as_Address(LIR_Address* addr) {
217 return as_Address(addr, rscratch1);
218 }
219
220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
221 return as_Address(addr, rscratch1); // Ouch
222 // FIXME: This needs to be much more clever. See x86.
223 }
224
225
226 void LIR_Assembler::osr_entry() {
227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
228 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
229 ValueStack* entry_state = osr_entry->state();
230 int number_of_locks = entry_state->locks_size();
231
232 // we jump here if osr happens with the interpreter
233 // state set up to continue at the beginning of the
234 // loop that triggered osr - in particular, we have
235 // the following registers setup:
236 //
237 // r2: osr buffer
238 //
239
240 // build frame
241 ciMethod* m = compilation()->method();
242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
243
244 // OSR buffer is
245 //
246 // locals[nlocals-1..0]
247 // monitors[0..number_of_locks]
248 //
249 // locals is a direct copy of the interpreter frame so in the osr buffer
250 // so first slot in the local array is the last local from the interpreter
251 // and last slot is local[0] (receiver) from the interpreter
252 //
253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
255 // in the interpreter frame (the method lock if a sync method)
256
257 // Initialize monitors in the compiled activation.
258 // r2: pointer to osr buffer
259 //
260 // All other registers are dead at this point and the locals will be
261 // copied into place by code emitted in the IR.
262
263 Register OSR_buf = osrBufferPointer()->as_pointer_register();
264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
265 int monitor_offset = BytesPerWord * method()->max_locals() +
266 (2 * BytesPerWord) * (number_of_locks - 1);
267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
268 // the OSR buffer using 2 word entries: first the lock and then
269 // the oop.
270 for (int i = 0; i < number_of_locks; i++) {
271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
272 #ifdef ASSERT
273 // verify the interpreter's monitor has a non-null object
274 {
275 Label L;
276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
277 __ cbnz(rscratch1, L);
278 __ stop("locked object is NULL");
279 __ bind(L);
280 }
281 #endif
282 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
283 __ str(r19, frame_map()->address_for_monitor_lock(i));
284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
285 __ str(r19, frame_map()->address_for_monitor_object(i));
286 }
287 }
288 }
289
290
291 // inline cache check; done before the frame is built.
292 int LIR_Assembler::check_icache() {
293 Register receiver = FrameMap::receiver_opr->as_register();
294 Register ic_klass = IC_Klass;
295 int start_offset = __ offset();
296 __ inline_cache_check(receiver, ic_klass);
297
298 // if icache check fails, then jump to runtime routine
299 // Note: RECEIVER must still contain the receiver!
300 Label dont;
301 __ br(Assembler::EQ, dont);
302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
303
304 // We align the verified entry point unless the method body
305 // (including its inline cache check) will fit in a single 64-byte
306 // icache line.
307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
308 // force alignment after the cache check.
309 __ align(CodeEntryAlignment);
310 }
311
312 __ bind(dont);
313 return start_offset;
314 }
315
316
317 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
318 if (o == NULL) {
319 __ mov(reg, zr);
320 } else {
321 __ movoop(reg, o, /*immediate*/true);
322 }
323 }
324
325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
326 address target = NULL;
327 relocInfo::relocType reloc_type = relocInfo::none;
328
329 switch (patching_id(info)) {
330 case PatchingStub::access_field_id:
331 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
332 reloc_type = relocInfo::section_word_type;
333 break;
334 case PatchingStub::load_klass_id:
335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
336 reloc_type = relocInfo::metadata_type;
337 break;
338 case PatchingStub::load_mirror_id:
339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
340 reloc_type = relocInfo::oop_type;
341 break;
342 case PatchingStub::load_appendix_id:
343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
344 reloc_type = relocInfo::oop_type;
345 break;
346 default: ShouldNotReachHere();
347 }
348
349 __ far_call(RuntimeAddress(target));
350 add_call_info_here(info);
351 }
352
353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
354 deoptimize_trap(info);
355 }
356
357
358 // This specifies the rsp decrement needed to build the frame
359 int LIR_Assembler::initial_frame_size_in_bytes() const {
360 // if rounding, must let FrameMap know!
361
362 // The frame_map records size in slots (32bit word)
363
364 // subtract two words to account for return address and link
365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
366 }
367
368
369 int LIR_Assembler::emit_exception_handler() {
370 // if the last instruction is a call (typically to do a throw which
371 // is coming at the end after block reordering) the return address
372 // must still point into the code area in order to avoid assertion
373 // failures when searching for the corresponding bci => add a nop
374 // (was bug 5/14/1999 - gri)
375 __ nop();
376
377 // generate code for exception handler
378 address handler_base = __ start_a_stub(exception_handler_size);
379 if (handler_base == NULL) {
380 // not enough space left for the handler
381 bailout("exception handler overflow");
382 return -1;
383 }
384
385 int offset = code_offset();
386
387 // the exception oop and pc are in r0, and r3
388 // no other registers need to be preserved, so invalidate them
389 __ invalidate_registers(false, true, true, false, true, true);
390
391 // check that there is really an exception
392 __ verify_not_null_oop(r0);
393
394 // search an exception handler (r0: exception oop, r3: throwing pc)
395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
396 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
397 __ end_a_stub();
398
399 return offset;
400 }
401
402
403 // Emit the code to remove the frame from the stack in the exception
404 // unwind path.
405 int LIR_Assembler::emit_unwind_handler() {
406 #ifndef PRODUCT
407 if (CommentedAssembly) {
408 _masm->block_comment("Unwind handler");
409 }
410 #endif
411
412 int offset = code_offset();
413
414 // Fetch the exception from TLS and clear out exception related thread state
415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
418
419 __ bind(_unwind_handler_entry);
420 __ verify_not_null_oop(r0);
421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
422 __ mov(r19, r0); // Preserve the exception
423 }
424
425 // Preform needed unlocking
426 MonitorExitStub* stub = NULL;
427 if (method()->is_synchronized()) {
428 monitor_address(0, FrameMap::r0_opr);
429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
430 __ unlock_object(r5, r4, r0, *stub->entry());
431 __ bind(*stub->continuation());
432 }
433
434 if (compilation()->env()->dtrace_method_probes()) {
435 __ mov(c_rarg0, rthread);
436 __ mov_metadata(c_rarg1, method()->constant_encoding());
437 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
438 }
439
440 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
441 __ mov(r0, r19); // Restore the exception
442 }
443
444 // remove the activation and dispatch to the unwind handler
445 __ block_comment("remove_frame and dispatch to the unwind handler");
446 __ remove_frame(initial_frame_size_in_bytes());
447 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
448
449 // Emit the slow path assembly
450 if (stub != NULL) {
451 stub->emit_code(this);
452 }
453
454 return offset;
455 }
456
457
458 int LIR_Assembler::emit_deopt_handler() {
459 // if the last instruction is a call (typically to do a throw which
460 // is coming at the end after block reordering) the return address
461 // must still point into the code area in order to avoid assertion
462 // failures when searching for the corresponding bci => add a nop
463 // (was bug 5/14/1999 - gri)
464 __ nop();
465
466 // generate code for exception handler
467 address handler_base = __ start_a_stub(deopt_handler_size);
468 if (handler_base == NULL) {
469 // not enough space left for the handler
470 bailout("deopt handler overflow");
471 return -1;
472 }
473
474 int offset = code_offset();
475
476 __ adr(lr, pc());
477 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
478 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
479 __ end_a_stub();
480
481 return offset;
482 }
483
484
485 // This is the fast version of java.lang.String.compare; it has not
486 // OSR-entry and therefore, we generate a slow version for OSR's
487 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
488 __ mov(r2, (address)__FUNCTION__);
489 __ call_Unimplemented();
490 }
491
492
493 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
494 _masm->code_section()->relocate(adr, relocInfo::poll_type);
495 int pc_offset = code_offset();
496 flush_debug_info(pc_offset);
497 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
498 if (info->exception_handlers() != NULL) {
499 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
500 }
501 }
502
503 // Rather than take a segfault when the polling page is protected,
504 // explicitly check for a safepoint in progress and if there is one,
505 // fake a call to the handler as if a segfault had been caught.
506 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
507 __ mov(rscratch1, SafepointSynchronize::address_of_state());
508 __ ldrb(rscratch1, Address(rscratch1));
509 Label nope, poll;
510 __ cbz(rscratch1, nope);
511 __ block_comment("safepoint");
512 __ enter();
513 __ push(0x3, sp); // r0 & r1
514 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
515 __ adr(r0, poll);
516 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
517 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
518 __ blr(rscratch1);
519 __ maybe_isb();
520 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
521 __ mov(rscratch1, r0);
522 __ pop(0x3, sp); // r0 & r1
523 __ leave();
524 __ br(rscratch1);
525 address polling_page(os::get_polling_page());
526 assert(os::is_poll_address(polling_page), "should be");
527 unsigned long off;
528 __ adrp(rscratch1, Address(polling_page, rtype), off);
529 __ bind(poll);
530 if (info)
531 add_debug_info_for_branch(info); // This isn't just debug info:
532 // it's the oop map
533 else
534 __ code_section()->relocate(pc(), rtype);
535 __ ldrw(zr, Address(rscratch1, off));
536 __ bind(nope);
537 }
538
539 void LIR_Assembler::return_op(LIR_Opr result) {
540 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
541 // Pop the stack before the safepoint code
542 __ remove_frame(initial_frame_size_in_bytes());
543 if (UseCompilerSafepoints) {
544 address polling_page(os::get_polling_page());
545 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
546 } else {
547 poll_for_safepoint(relocInfo::poll_return_type);
548 }
549 __ ret(lr);
550 }
551
552 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
553 address polling_page(os::get_polling_page());
554 if (UseCompilerSafepoints) {
555 guarantee(info != NULL, "Shouldn't be NULL");
556 assert(os::is_poll_address(polling_page), "should be");
557 unsigned long off;
558 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
559 assert(off == 0, "must be");
560 add_debug_info_for_branch(info); // This isn't just debug info:
561 // it's the oop map
562 __ read_polling_page(rscratch1, relocInfo::poll_type);
563 } else {
564 poll_for_safepoint(relocInfo::poll_type, info);
565 }
566
567 return __ offset();
568 }
569
570
571 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
572 if (from_reg == r31_sp)
573 from_reg = sp;
574 if (to_reg == r31_sp)
575 to_reg = sp;
576 __ mov(to_reg, from_reg);
577 }
578
579 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
580
581
582 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
583 assert(src->is_constant(), "should not call otherwise");
584 assert(dest->is_register(), "should not call otherwise");
585 LIR_Const* c = src->as_constant_ptr();
586
587 switch (c->type()) {
588 case T_INT: {
589 assert(patch_code == lir_patch_none, "no patching handled here");
590 __ movw(dest->as_register(), c->as_jint());
591 break;
592 }
593
594 case T_ADDRESS: {
595 assert(patch_code == lir_patch_none, "no patching handled here");
596 __ mov(dest->as_register(), c->as_jint());
597 break;
598 }
599
600 case T_LONG: {
601 assert(patch_code == lir_patch_none, "no patching handled here");
602 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
603 break;
604 }
605
606 case T_OBJECT: {
607 if (patch_code == lir_patch_none) {
608 jobject2reg(c->as_jobject(), dest->as_register());
609 } else {
610 jobject2reg_with_patching(dest->as_register(), info);
611 }
612 break;
613 }
614
615 case T_METADATA: {
616 if (patch_code != lir_patch_none) {
617 klass2reg_with_patching(dest->as_register(), info);
618 } else {
619 __ mov_metadata(dest->as_register(), c->as_metadata());
620 }
621 break;
622 }
623
624 case T_FLOAT: {
625 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
626 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
627 } else {
628 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
629 __ ldrs(dest->as_float_reg(), Address(rscratch1));
630 }
631 break;
632 }
633
634 case T_DOUBLE: {
635 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
636 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
637 } else {
638 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
639 __ ldrd(dest->as_double_reg(), Address(rscratch1));
640 }
641 break;
642 }
643
644 default:
645 ShouldNotReachHere();
646 }
647 }
648
649 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
650 LIR_Const* c = src->as_constant_ptr();
651 switch (c->type()) {
652 case T_OBJECT:
653 {
654 if (! c->as_jobject())
655 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
656 else {
657 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
658 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
659 }
660 }
661 break;
662 case T_ADDRESS:
663 {
664 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
665 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
666 }
667 case T_INT:
668 case T_FLOAT:
669 {
670 Register reg = zr;
671 if (c->as_jint_bits() == 0)
672 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
673 else {
674 __ movw(rscratch1, c->as_jint_bits());
675 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
676 }
677 }
678 break;
679 case T_LONG:
680 case T_DOUBLE:
681 {
682 Register reg = zr;
683 if (c->as_jlong_bits() == 0)
684 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
685 lo_word_offset_in_bytes));
686 else {
687 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
688 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
689 lo_word_offset_in_bytes));
690 }
691 }
692 break;
693 default:
694 ShouldNotReachHere();
695 }
696 }
697
698 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
699 assert(src->is_constant(), "should not call otherwise");
700 LIR_Const* c = src->as_constant_ptr();
701 LIR_Address* to_addr = dest->as_address_ptr();
702
703 void (Assembler::* insn)(Register Rt, const Address &adr);
704
705 switch (type) {
706 case T_ADDRESS:
707 assert(c->as_jint() == 0, "should be");
708 insn = &Assembler::str;
709 break;
710 case T_LONG:
711 assert(c->as_jlong() == 0, "should be");
712 insn = &Assembler::str;
713 break;
714 case T_INT:
715 assert(c->as_jint() == 0, "should be");
716 insn = &Assembler::strw;
717 break;
718 case T_OBJECT:
719 case T_ARRAY:
720 assert(c->as_jobject() == 0, "should be");
721 if (UseCompressedOops && !wide) {
722 insn = &Assembler::strw;
723 } else {
724 insn = &Assembler::str;
725 }
726 break;
727 case T_CHAR:
728 case T_SHORT:
729 assert(c->as_jint() == 0, "should be");
730 insn = &Assembler::strh;
731 break;
732 case T_BOOLEAN:
733 case T_BYTE:
734 assert(c->as_jint() == 0, "should be");
735 insn = &Assembler::strb;
736 break;
737 default:
738 ShouldNotReachHere();
739 }
740
741 if (info) add_debug_info_for_null_check_here(info);
742 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
743 }
744
745 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
746 assert(src->is_register(), "should not call otherwise");
747 assert(dest->is_register(), "should not call otherwise");
748
749 // move between cpu-registers
750 if (dest->is_single_cpu()) {
751 if (src->type() == T_LONG) {
752 // Can do LONG -> OBJECT
753 move_regs(src->as_register_lo(), dest->as_register());
754 return;
755 }
756 assert(src->is_single_cpu(), "must match");
757 if (src->type() == T_OBJECT) {
758 __ verify_oop(src->as_register());
759 }
760 move_regs(src->as_register(), dest->as_register());
761
762 } else if (dest->is_double_cpu()) {
763 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
764 // Surprising to me but we can see move of a long to t_object
765 __ verify_oop(src->as_register());
766 move_regs(src->as_register(), dest->as_register_lo());
767 return;
768 }
769 assert(src->is_double_cpu(), "must match");
770 Register f_lo = src->as_register_lo();
771 Register f_hi = src->as_register_hi();
772 Register t_lo = dest->as_register_lo();
773 Register t_hi = dest->as_register_hi();
774 assert(f_hi == f_lo, "must be same");
775 assert(t_hi == t_lo, "must be same");
776 move_regs(f_lo, t_lo);
777
778 } else if (dest->is_single_fpu()) {
779 __ fmovs(dest->as_float_reg(), src->as_float_reg());
780
781 } else if (dest->is_double_fpu()) {
782 __ fmovd(dest->as_double_reg(), src->as_double_reg());
783
784 } else {
785 ShouldNotReachHere();
786 }
787 }
788
789 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
790 if (src->is_single_cpu()) {
791 if (type == T_ARRAY || type == T_OBJECT) {
792 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
793 __ verify_oop(src->as_register());
794 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
795 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
796 } else {
797 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
798 }
799
800 } else if (src->is_double_cpu()) {
801 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
802 __ str(src->as_register_lo(), dest_addr_LO);
803
804 } else if (src->is_single_fpu()) {
805 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
806 __ strs(src->as_float_reg(), dest_addr);
807
808 } else if (src->is_double_fpu()) {
809 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
810 __ strd(src->as_double_reg(), dest_addr);
811
812 } else {
813 ShouldNotReachHere();
814 }
815
816 }
817
818
819 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
820 LIR_Address* to_addr = dest->as_address_ptr();
821 PatchingStub* patch = NULL;
822 Register compressed_src = rscratch1;
823
824 if (patch_code != lir_patch_none) {
825 deoptimize_trap(info);
826 return;
827 }
828
829 if (type == T_ARRAY || type == T_OBJECT) {
830 __ verify_oop(src->as_register());
831
832 if (UseCompressedOops && !wide) {
833 __ encode_heap_oop(compressed_src, src->as_register());
834 } else {
835 compressed_src = src->as_register();
836 }
837 }
838
839 int null_check_here = code_offset();
840 switch (type) {
841 case T_FLOAT: {
842 __ strs(src->as_float_reg(), as_Address(to_addr));
843 break;
844 }
845
846 case T_DOUBLE: {
847 __ strd(src->as_double_reg(), as_Address(to_addr));
848 break;
849 }
850
851 case T_ARRAY: // fall through
852 case T_OBJECT: // fall through
853 if (UseCompressedOops && !wide) {
854 __ strw(compressed_src, as_Address(to_addr, rscratch2));
855 } else {
856 __ str(compressed_src, as_Address(to_addr));
857 }
858 break;
859 case T_METADATA:
860 // We get here to store a method pointer to the stack to pass to
861 // a dtrace runtime call. This can't work on 64 bit with
862 // compressed klass ptrs: T_METADATA can be a compressed klass
863 // ptr or a 64 bit method pointer.
864 LP64_ONLY(ShouldNotReachHere());
865 __ str(src->as_register(), as_Address(to_addr));
866 break;
867 case T_ADDRESS:
868 __ str(src->as_register(), as_Address(to_addr));
869 break;
870 case T_INT:
871 __ strw(src->as_register(), as_Address(to_addr));
872 break;
873
874 case T_LONG: {
875 __ str(src->as_register_lo(), as_Address_lo(to_addr));
876 break;
877 }
878
879 case T_BYTE: // fall through
880 case T_BOOLEAN: {
881 __ strb(src->as_register(), as_Address(to_addr));
882 break;
883 }
884
885 case T_CHAR: // fall through
886 case T_SHORT:
887 __ strh(src->as_register(), as_Address(to_addr));
888 break;
889
890 default:
891 ShouldNotReachHere();
892 }
893 if (info != NULL) {
894 add_debug_info_for_null_check(null_check_here, info);
895 }
896 }
897
898
899 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
900 assert(src->is_stack(), "should not call otherwise");
901 assert(dest->is_register(), "should not call otherwise");
902
903 if (dest->is_single_cpu()) {
904 if (type == T_ARRAY || type == T_OBJECT) {
905 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
906 __ verify_oop(dest->as_register());
907 } else if (type == T_METADATA || type == T_ADDRESS) {
908 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
909 } else {
910 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
911 }
912
913 } else if (dest->is_double_cpu()) {
914 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
915 __ ldr(dest->as_register_lo(), src_addr_LO);
916
917 } else if (dest->is_single_fpu()) {
918 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
919 __ ldrs(dest->as_float_reg(), src_addr);
920
921 } else if (dest->is_double_fpu()) {
922 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
923 __ ldrd(dest->as_double_reg(), src_addr);
924
925 } else {
926 ShouldNotReachHere();
927 }
928 }
929
930
931 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
932 address target = NULL;
933 relocInfo::relocType reloc_type = relocInfo::none;
934
935 switch (patching_id(info)) {
936 case PatchingStub::access_field_id:
937 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
938 reloc_type = relocInfo::section_word_type;
939 break;
940 case PatchingStub::load_klass_id:
941 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
942 reloc_type = relocInfo::metadata_type;
943 break;
944 case PatchingStub::load_mirror_id:
945 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
946 reloc_type = relocInfo::oop_type;
947 break;
948 case PatchingStub::load_appendix_id:
949 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
950 reloc_type = relocInfo::oop_type;
951 break;
952 default: ShouldNotReachHere();
953 }
954
955 __ far_call(RuntimeAddress(target));
956 add_call_info_here(info);
957 }
958
959 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
960
961 LIR_Opr temp;
962 if (type == T_LONG || type == T_DOUBLE)
963 temp = FrameMap::rscratch1_long_opr;
964 else
965 temp = FrameMap::rscratch1_opr;
966
967 stack2reg(src, temp, src->type());
968 reg2stack(temp, dest, dest->type(), false);
969 }
970
971
972 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
973 LIR_Address* addr = src->as_address_ptr();
974 LIR_Address* from_addr = src->as_address_ptr();
975
976 if (addr->base()->type() == T_OBJECT) {
977 __ verify_oop(addr->base()->as_pointer_register());
978 }
979
980 if (patch_code != lir_patch_none) {
981 deoptimize_trap(info);
982 return;
983 }
984
985 if (info != NULL) {
986 add_debug_info_for_null_check_here(info);
987 }
988 int null_check_here = code_offset();
989 switch (type) {
990 case T_FLOAT: {
991 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
992 break;
993 }
994
995 case T_DOUBLE: {
996 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
997 break;
998 }
999
1000 case T_ARRAY: // fall through
1001 case T_OBJECT: // fall through
1002 if (UseCompressedOops && !wide) {
1003 __ ldrw(dest->as_register(), as_Address(from_addr));
1004 } else {
1005 __ ldr(dest->as_register(), as_Address(from_addr));
1006 }
1007 break;
1008 case T_METADATA:
1009 // We get here to store a method pointer to the stack to pass to
1010 // a dtrace runtime call. This can't work on 64 bit with
1011 // compressed klass ptrs: T_METADATA can be a compressed klass
1012 // ptr or a 64 bit method pointer.
1013 LP64_ONLY(ShouldNotReachHere());
1014 __ ldr(dest->as_register(), as_Address(from_addr));
1015 break;
1016 case T_ADDRESS:
1017 // FIXME: OMG this is a horrible kludge. Any offset from an
1018 // address that matches klass_offset_in_bytes() will be loaded
1019 // as a word, not a long.
1020 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1021 __ ldrw(dest->as_register(), as_Address(from_addr));
1022 } else {
1023 __ ldr(dest->as_register(), as_Address(from_addr));
1024 }
1025 break;
1026 case T_INT:
1027 __ ldrw(dest->as_register(), as_Address(from_addr));
1028 break;
1029
1030 case T_LONG: {
1031 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1032 break;
1033 }
1034
1035 case T_BYTE:
1036 __ ldrsb(dest->as_register(), as_Address(from_addr));
1037 break;
1038 case T_BOOLEAN: {
1039 __ ldrb(dest->as_register(), as_Address(from_addr));
1040 break;
1041 }
1042
1043 case T_CHAR:
1044 __ ldrh(dest->as_register(), as_Address(from_addr));
1045 break;
1046 case T_SHORT:
1047 __ ldrsh(dest->as_register(), as_Address(from_addr));
1048 break;
1049
1050 default:
1051 ShouldNotReachHere();
1052 }
1053
1054 if (type == T_ARRAY || type == T_OBJECT) {
1055 #ifdef _LP64
1056 if (UseCompressedOops && !wide) {
1057 __ decode_heap_oop(dest->as_register());
1058 }
1059 #endif
1060 __ verify_oop(dest->as_register());
1061 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1062 #ifdef _LP64
1063 if (UseCompressedClassPointers) {
1064 __ decode_klass_not_null(dest->as_register());
1065 }
1066 #endif
1067 }
1068 }
1069
1070
1071 void LIR_Assembler::prefetchr(LIR_Opr src) { Unimplemented(); }
1072
1073
1074 void LIR_Assembler::prefetchw(LIR_Opr src) { Unimplemented(); }
1075
1076
1077 int LIR_Assembler::array_element_size(BasicType type) const {
1078 int elem_size = type2aelembytes(type);
1079 return exact_log2(elem_size);
1080 }
1081
1082 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1083 Register Rdividend = op->in_opr1()->as_register();
1084 Register Rdivisor = op->in_opr2()->as_register();
1085 Register Rscratch = op->in_opr3()->as_register();
1086 Register Rresult = op->result_opr()->as_register();
1087 int divisor = -1;
1088
1089 /*
1090 TODO: For some reason, using the Rscratch that gets passed in is
1091 not possible because the register allocator does not see the tmp reg
1092 as used, and assignes it the same register as Rdividend. We use rscratch1
1093 instead.
1094
1095 assert(Rdividend != Rscratch, "");
1096 assert(Rdivisor != Rscratch, "");
1097 */
1098
1099 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1100 // convert division by a power of two into some shifts and logical operations
1101 }
1102
1103 if (op->code() == lir_irem) {
1104 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1105 } else if (op->code() == lir_idiv) {
1106 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1107 } else
1108 ShouldNotReachHere();
1109 }
1110
1111 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1112 #ifdef ASSERT
1113 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1114 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1115 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1116 #endif
1117
1118 if (op->cond() == lir_cond_always) {
1119 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1120 __ b(*(op->label()));
1121 } else {
1122 Assembler::Condition acond;
1123 if (op->code() == lir_cond_float_branch) {
1124 bool is_unordered = (op->ublock() == op->block());
1125 // Assembler::EQ does not permit unordered branches, so we add
1126 // another branch here. Likewise, Assembler::NE does not permit
1127 // ordered branches.
1128 if ((is_unordered && op->cond() == lir_cond_equal)
1129 || (!is_unordered && op->cond() == lir_cond_notEqual))
1130 __ br(Assembler::VS, *(op->ublock()->label()));
1131 switch(op->cond()) {
1132 case lir_cond_equal: acond = Assembler::EQ; break;
1133 case lir_cond_notEqual: acond = Assembler::NE; break;
1134 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1135 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1136 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1137 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1138 default: ShouldNotReachHere();
1139 }
1140 } else {
1141 switch (op->cond()) {
1142 case lir_cond_equal: acond = Assembler::EQ; break;
1143 case lir_cond_notEqual: acond = Assembler::NE; break;
1144 case lir_cond_less: acond = Assembler::LT; break;
1145 case lir_cond_lessEqual: acond = Assembler::LE; break;
1146 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1147 case lir_cond_greater: acond = Assembler::GT; break;
1148 case lir_cond_belowEqual: acond = Assembler::LS; break;
1149 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1150 default: ShouldNotReachHere();
1151 }
1152 }
1153 __ br(acond,*(op->label()));
1154 }
1155 }
1156
1157
1158
1159 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1160 LIR_Opr src = op->in_opr();
1161 LIR_Opr dest = op->result_opr();
1162
1163 switch (op->bytecode()) {
1164 case Bytecodes::_i2f:
1165 {
1166 __ scvtfws(dest->as_float_reg(), src->as_register());
1167 break;
1168 }
1169 case Bytecodes::_i2d:
1170 {
1171 __ scvtfwd(dest->as_double_reg(), src->as_register());
1172 break;
1173 }
1174 case Bytecodes::_l2d:
1175 {
1176 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1177 break;
1178 }
1179 case Bytecodes::_l2f:
1180 {
1181 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1182 break;
1183 }
1184 case Bytecodes::_f2d:
1185 {
1186 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1187 break;
1188 }
1189 case Bytecodes::_d2f:
1190 {
1191 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1192 break;
1193 }
1194 case Bytecodes::_i2c:
1195 {
1196 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1197 break;
1198 }
1199 case Bytecodes::_i2l:
1200 {
1201 __ sxtw(dest->as_register_lo(), src->as_register());
1202 break;
1203 }
1204 case Bytecodes::_i2s:
1205 {
1206 __ sxth(dest->as_register(), src->as_register());
1207 break;
1208 }
1209 case Bytecodes::_i2b:
1210 {
1211 __ sxtb(dest->as_register(), src->as_register());
1212 break;
1213 }
1214 case Bytecodes::_l2i:
1215 {
1216 _masm->block_comment("FIXME: This could be a no-op");
1217 __ uxtw(dest->as_register(), src->as_register_lo());
1218 break;
1219 }
1220 case Bytecodes::_d2l:
1221 {
1222 Register tmp = op->tmp1()->as_register();
1223 __ clear_fpsr();
1224 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1225 __ get_fpsr(tmp);
1226 __ tst(tmp, 1); // FPSCR.IOC
1227 __ br(Assembler::NE, *(op->stub()->entry()));
1228 __ bind(*op->stub()->continuation());
1229 break;
1230 }
1231 case Bytecodes::_f2i:
1232 {
1233 Register tmp = op->tmp1()->as_register();
1234 __ clear_fpsr();
1235 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1236 __ get_fpsr(tmp);
1237 __ tst(tmp, 1); // FPSCR.IOC
1238 __ br(Assembler::NE, *(op->stub()->entry()));
1239 __ bind(*op->stub()->continuation());
1240 break;
1241 }
1242 case Bytecodes::_f2l:
1243 {
1244 Register tmp = op->tmp1()->as_register();
1245 __ clear_fpsr();
1246 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1247 __ get_fpsr(tmp);
1248 __ tst(tmp, 1); // FPSCR.IOC
1249 __ br(Assembler::NE, *(op->stub()->entry()));
1250 __ bind(*op->stub()->continuation());
1251 break;
1252 }
1253 case Bytecodes::_d2i:
1254 {
1255 Register tmp = op->tmp1()->as_register();
1256 __ clear_fpsr();
1257 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1258 __ get_fpsr(tmp);
1259 __ tst(tmp, 1); // FPSCR.IOC
1260 __ br(Assembler::NE, *(op->stub()->entry()));
1261 __ bind(*op->stub()->continuation());
1262 break;
1263 }
1264 default: ShouldNotReachHere();
1265 }
1266 }
1267
1268 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1269 if (op->init_check()) {
1270 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1271 InstanceKlass::init_state_offset()));
1272 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1273 add_debug_info_for_null_check_here(op->stub()->info());
1274 __ br(Assembler::NE, *op->stub()->entry());
1275 }
1276 __ allocate_object(op->obj()->as_register(),
1277 op->tmp1()->as_register(),
1278 op->tmp2()->as_register(),
1279 op->header_size(),
1280 op->object_size(),
1281 op->klass()->as_register(),
1282 *op->stub()->entry());
1283 __ bind(*op->stub()->continuation());
1284 }
1285
1286 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1287 Register len = op->len()->as_register();
1288 __ uxtw(len, len);
1289
1290 if (UseSlowPath ||
1291 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1292 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1293 __ b(*op->stub()->entry());
1294 } else {
1295 Register tmp1 = op->tmp1()->as_register();
1296 Register tmp2 = op->tmp2()->as_register();
1297 Register tmp3 = op->tmp3()->as_register();
1298 if (len == tmp1) {
1299 tmp1 = tmp3;
1300 } else if (len == tmp2) {
1301 tmp2 = tmp3;
1302 } else if (len == tmp3) {
1303 // everything is ok
1304 } else {
1305 __ mov(tmp3, len);
1306 }
1307 __ allocate_array(op->obj()->as_register(),
1308 len,
1309 tmp1,
1310 tmp2,
1311 arrayOopDesc::header_size(op->type()),
1312 array_element_size(op->type()),
1313 op->klass()->as_register(),
1314 *op->stub()->entry());
1315 }
1316 __ bind(*op->stub()->continuation());
1317 }
1318
1319 void LIR_Assembler::type_profile_helper(Register mdo,
1320 ciMethodData *md, ciProfileData *data,
1321 Register recv, Label* update_done) {
1322 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1323 Label next_test;
1324 // See if the receiver is receiver[n].
1325 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1326 __ ldr(rscratch1, Address(rscratch2));
1327 __ cmp(recv, rscratch1);
1328 __ br(Assembler::NE, next_test);
1329 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1330 __ addptr(data_addr, DataLayout::counter_increment);
1331 __ b(*update_done);
1332 __ bind(next_test);
1333 }
1334
1335 // Didn't find receiver; find next empty slot and fill it in
1336 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1337 Label next_test;
1338 __ lea(rscratch2,
1339 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1340 Address recv_addr(rscratch2);
1341 __ ldr(rscratch1, recv_addr);
1342 __ cbnz(rscratch1, next_test);
1343 __ str(recv, recv_addr);
1344 __ mov(rscratch1, DataLayout::counter_increment);
1345 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1346 __ str(rscratch1, Address(rscratch2));
1347 __ b(*update_done);
1348 __ bind(next_test);
1349 }
1350 }
1351
1352 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1353 // we always need a stub for the failure case.
1354 CodeStub* stub = op->stub();
1355 Register obj = op->object()->as_register();
1356 Register k_RInfo = op->tmp1()->as_register();
1357 Register klass_RInfo = op->tmp2()->as_register();
1358 Register dst = op->result_opr()->as_register();
1359 ciKlass* k = op->klass();
1360 Register Rtmp1 = noreg;
1361
1362 // check if it needs to be profiled
1363 ciMethodData* md;
1364 ciProfileData* data;
1365
1366 if (op->should_profile()) {
1367 ciMethod* method = op->profiled_method();
1368 assert(method != NULL, "Should have method");
1369 int bci = op->profiled_bci();
1370 md = method->method_data_or_null();
1371 assert(md != NULL, "Sanity");
1372 data = md->bci_to_data(bci);
1373 assert(data != NULL, "need data for type check");
1374 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1375 }
1376 Label profile_cast_success, profile_cast_failure;
1377 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1378 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1379
1380 if (obj == k_RInfo) {
1381 k_RInfo = dst;
1382 } else if (obj == klass_RInfo) {
1383 klass_RInfo = dst;
1384 }
1385 if (k->is_loaded() && !UseCompressedClassPointers) {
1386 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1387 } else {
1388 Rtmp1 = op->tmp3()->as_register();
1389 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1390 }
1391
1392 assert_different_registers(obj, k_RInfo, klass_RInfo);
1393
1394 if (op->should_profile()) {
1395 Label not_null;
1396 __ cbnz(obj, not_null);
1397 // Object is null; update MDO and exit
1398 Register mdo = klass_RInfo;
1399 __ mov_metadata(mdo, md->constant_encoding());
1400 Address data_addr
1401 = __ form_address(rscratch2, mdo,
1402 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1403 LogBytesPerWord);
1404 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1405 __ ldr(rscratch1, data_addr);
1406 __ orr(rscratch1, rscratch1, header_bits);
1407 __ str(rscratch1, data_addr);
1408 __ b(*obj_is_null);
1409 __ bind(not_null);
1410 } else {
1411 __ cbz(obj, *obj_is_null);
1412 }
1413
1414 if (!k->is_loaded()) {
1415 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1416 } else {
1417 #ifdef _LP64
1418 __ mov_metadata(k_RInfo, k->constant_encoding());
1419 #endif // _LP64
1420 }
1421 __ verify_oop(obj);
1422
1423 if (op->fast_check()) {
1424 // get object class
1425 // not a safepoint as obj null check happens earlier
1426 __ load_klass(rscratch1, obj);
1427 __ cmp( rscratch1, k_RInfo);
1428
1429 __ br(Assembler::NE, *failure_target);
1430 // successful cast, fall through to profile or jump
1431 } else {
1432 // get object class
1433 // not a safepoint as obj null check happens earlier
1434 __ load_klass(klass_RInfo, obj);
1435 if (k->is_loaded()) {
1436 // See if we get an immediate positive hit
1437 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1438 __ cmp(k_RInfo, rscratch1);
1439 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1440 __ br(Assembler::NE, *failure_target);
1441 // successful cast, fall through to profile or jump
1442 } else {
1443 // See if we get an immediate positive hit
1444 __ br(Assembler::EQ, *success_target);
1445 // check for self
1446 __ cmp(klass_RInfo, k_RInfo);
1447 __ br(Assembler::EQ, *success_target);
1448
1449 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1450 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1451 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1452 // result is a boolean
1453 __ cbzw(klass_RInfo, *failure_target);
1454 // successful cast, fall through to profile or jump
1455 }
1456 } else {
1457 // perform the fast part of the checking logic
1458 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1459 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1460 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1461 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1462 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1463 // result is a boolean
1464 __ cbz(k_RInfo, *failure_target);
1465 // successful cast, fall through to profile or jump
1466 }
1467 }
1468 if (op->should_profile()) {
1469 Register mdo = klass_RInfo, recv = k_RInfo;
1470 __ bind(profile_cast_success);
1471 __ mov_metadata(mdo, md->constant_encoding());
1472 __ load_klass(recv, obj);
1473 Label update_done;
1474 type_profile_helper(mdo, md, data, recv, success);
1475 __ b(*success);
1476
1477 __ bind(profile_cast_failure);
1478 __ mov_metadata(mdo, md->constant_encoding());
1479 Address counter_addr
1480 = __ form_address(rscratch2, mdo,
1481 md->byte_offset_of_slot(data, CounterData::count_offset()),
1482 LogBytesPerWord);
1483 __ ldr(rscratch1, counter_addr);
1484 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1485 __ str(rscratch1, counter_addr);
1486 __ b(*failure);
1487 }
1488 __ b(*success);
1489 }
1490
1491
1492 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1493 LIR_Code code = op->code();
1494 if (code == lir_store_check) {
1495 Register value = op->object()->as_register();
1496 Register array = op->array()->as_register();
1497 Register k_RInfo = op->tmp1()->as_register();
1498 Register klass_RInfo = op->tmp2()->as_register();
1499 Register Rtmp1 = op->tmp3()->as_register();
1500
1501 CodeStub* stub = op->stub();
1502
1503 // check if it needs to be profiled
1504 ciMethodData* md;
1505 ciProfileData* data;
1506
1507 if (op->should_profile()) {
1508 ciMethod* method = op->profiled_method();
1509 assert(method != NULL, "Should have method");
1510 int bci = op->profiled_bci();
1511 md = method->method_data_or_null();
1512 assert(md != NULL, "Sanity");
1513 data = md->bci_to_data(bci);
1514 assert(data != NULL, "need data for type check");
1515 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1516 }
1517 Label profile_cast_success, profile_cast_failure, done;
1518 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1519 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1520
1521 if (op->should_profile()) {
1522 Label not_null;
1523 __ cbnz(value, not_null);
1524 // Object is null; update MDO and exit
1525 Register mdo = klass_RInfo;
1526 __ mov_metadata(mdo, md->constant_encoding());
1527 Address data_addr
1528 = __ form_address(rscratch2, mdo,
1529 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1530 LogBytesPerInt);
1531 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1532 __ ldrw(rscratch1, data_addr);
1533 __ orrw(rscratch1, rscratch1, header_bits);
1534 __ strw(rscratch1, data_addr);
1535 __ b(done);
1536 __ bind(not_null);
1537 } else {
1538 __ cbz(value, done);
1539 }
1540
1541 add_debug_info_for_null_check_here(op->info_for_exception());
1542 __ load_klass(k_RInfo, array);
1543 __ load_klass(klass_RInfo, value);
1544
1545 // get instance klass (it's already uncompressed)
1546 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1547 // perform the fast part of the checking logic
1548 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1549 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1550 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1551 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1552 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1553 // result is a boolean
1554 __ cbzw(k_RInfo, *failure_target);
1555 // fall through to the success case
1556
1557 if (op->should_profile()) {
1558 Register mdo = klass_RInfo, recv = k_RInfo;
1559 __ bind(profile_cast_success);
1560 __ mov_metadata(mdo, md->constant_encoding());
1561 __ load_klass(recv, value);
1562 Label update_done;
1563 type_profile_helper(mdo, md, data, recv, &done);
1564 __ b(done);
1565
1566 __ bind(profile_cast_failure);
1567 __ mov_metadata(mdo, md->constant_encoding());
1568 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1569 __ lea(rscratch2, counter_addr);
1570 __ ldr(rscratch1, Address(rscratch2));
1571 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1572 __ str(rscratch1, Address(rscratch2));
1573 __ b(*stub->entry());
1574 }
1575
1576 __ bind(done);
1577 } else if (code == lir_checkcast) {
1578 Register obj = op->object()->as_register();
1579 Register dst = op->result_opr()->as_register();
1580 Label success;
1581 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1582 __ bind(success);
1583 if (dst != obj) {
1584 __ mov(dst, obj);
1585 }
1586 } else if (code == lir_instanceof) {
1587 Register obj = op->object()->as_register();
1588 Register dst = op->result_opr()->as_register();
1589 Label success, failure, done;
1590 emit_typecheck_helper(op, &success, &failure, &failure);
1591 __ bind(failure);
1592 __ mov(dst, zr);
1593 __ b(done);
1594 __ bind(success);
1595 __ mov(dst, 1);
1596 __ bind(done);
1597 } else {
1598 ShouldNotReachHere();
1599 }
1600 }
1601
1602 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1603 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, rscratch1);
1604 __ cset(rscratch1, Assembler::NE);
1605 __ membar(__ AnyAny);
1606 }
1607
1608 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1609 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, rscratch1);
1610 __ cset(rscratch1, Assembler::NE);
1611 __ membar(__ AnyAny);
1612 }
1613
1614
1615 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1616 assert(VM_Version::supports_cx8(), "wrong machine");
1617 Register addr = as_reg(op->addr());
1618 Register newval = as_reg(op->new_value());
1619 Register cmpval = as_reg(op->cmp_value());
1620 Label succeed, fail, around;
1621
1622 if (op->code() == lir_cas_obj) {
1623 if (UseCompressedOops) {
1624 Register t1 = op->tmp1()->as_register();
1625 assert(op->tmp1()->is_valid(), "must be");
1626 __ encode_heap_oop(t1, cmpval);
1627 cmpval = t1;
1628 __ encode_heap_oop(rscratch2, newval);
1629 newval = rscratch2;
1630 casw(addr, newval, cmpval);
1631 } else {
1632 casl(addr, newval, cmpval);
1633 }
1634 } else if (op->code() == lir_cas_int) {
1635 casw(addr, newval, cmpval);
1636 } else {
1637 casl(addr, newval, cmpval);
1638 }
1639 }
1640
1641
1642 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1643
1644 Assembler::Condition acond, ncond;
1645 switch (condition) {
1646 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1647 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1648 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1649 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1650 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1651 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1652 case lir_cond_belowEqual: Unimplemented(); break;
1653 case lir_cond_aboveEqual: Unimplemented(); break;
1654 default: ShouldNotReachHere();
1655 }
1656
1657 assert(result->is_single_cpu() || result->is_double_cpu(),
1658 "expect single register for result");
1659 if (opr1->is_constant() && opr2->is_constant()
1660 && opr1->type() == T_INT && opr2->type() == T_INT) {
1661 jint val1 = opr1->as_jint();
1662 jint val2 = opr2->as_jint();
1663 if (val1 == 0 && val2 == 1) {
1664 __ cset(result->as_register(), ncond);
1665 return;
1666 } else if (val1 == 1 && val2 == 0) {
1667 __ cset(result->as_register(), acond);
1668 return;
1669 }
1670 }
1671
1672 if (opr1->is_constant() && opr2->is_constant()
1673 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1674 jlong val1 = opr1->as_jlong();
1675 jlong val2 = opr2->as_jlong();
1676 if (val1 == 0 && val2 == 1) {
1677 __ cset(result->as_register_lo(), ncond);
1678 return;
1679 } else if (val1 == 1 && val2 == 0) {
1680 __ cset(result->as_register_lo(), acond);
1681 return;
1682 }
1683 }
1684
1685 if (opr1->is_stack()) {
1686 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1687 opr1 = FrameMap::rscratch1_opr;
1688 } else if (opr1->is_constant()) {
1689 LIR_Opr tmp
1690 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1691 const2reg(opr1, tmp, lir_patch_none, NULL);
1692 opr1 = tmp;
1693 }
1694
1695 if (opr2->is_stack()) {
1696 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1697 opr2 = FrameMap::rscratch2_opr;
1698 } else if (opr2->is_constant()) {
1699 LIR_Opr tmp
1700 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1701 const2reg(opr2, tmp, lir_patch_none, NULL);
1702 opr2 = tmp;
1703 }
1704
1705 if (result->type() == T_LONG)
1706 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1707 else
1708 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1709 }
1710
1711 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1712 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1713
1714 if (left->is_single_cpu()) {
1715 Register lreg = left->as_register();
1716 Register dreg = as_reg(dest);
1717
1718 if (right->is_single_cpu()) {
1719 // cpu register - cpu register
1720
1721 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1722 "should be");
1723 Register rreg = right->as_register();
1724 switch (code) {
1725 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1726 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1727 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1728 default: ShouldNotReachHere();
1729 }
1730
1731 } else if (right->is_double_cpu()) {
1732 Register rreg = right->as_register_lo();
1733 // single_cpu + double_cpu: can happen with obj+long
1734 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1735 switch (code) {
1736 case lir_add: __ add(dreg, lreg, rreg); break;
1737 case lir_sub: __ sub(dreg, lreg, rreg); break;
1738 default: ShouldNotReachHere();
1739 }
1740 } else if (right->is_constant()) {
1741 // cpu register - constant
1742 jlong c;
1743
1744 // FIXME. This is fugly: we really need to factor all this logic.
1745 switch(right->type()) {
1746 case T_LONG:
1747 c = right->as_constant_ptr()->as_jlong();
1748 break;
1749 case T_INT:
1750 case T_ADDRESS:
1751 c = right->as_constant_ptr()->as_jint();
1752 break;
1753 default:
1754 ShouldNotReachHere();
1755 break;
1756 }
1757
1758 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1759 if (c == 0 && dreg == lreg) {
1760 COMMENT("effective nop elided");
1761 return;
1762 }
1763 switch(left->type()) {
1764 case T_INT:
1765 switch (code) {
1766 case lir_add: __ addw(dreg, lreg, c); break;
1767 case lir_sub: __ subw(dreg, lreg, c); break;
1768 default: ShouldNotReachHere();
1769 }
1770 break;
1771 case T_OBJECT:
1772 case T_ADDRESS:
1773 switch (code) {
1774 case lir_add: __ add(dreg, lreg, c); break;
1775 case lir_sub: __ sub(dreg, lreg, c); break;
1776 default: ShouldNotReachHere();
1777 }
1778 break;
1779 ShouldNotReachHere();
1780 }
1781 } else {
1782 ShouldNotReachHere();
1783 }
1784
1785 } else if (left->is_double_cpu()) {
1786 Register lreg_lo = left->as_register_lo();
1787
1788 if (right->is_double_cpu()) {
1789 // cpu register - cpu register
1790 Register rreg_lo = right->as_register_lo();
1791 switch (code) {
1792 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1793 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1794 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1795 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1796 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1797 default:
1798 ShouldNotReachHere();
1799 }
1800
1801 } else if (right->is_constant()) {
1802 jlong c = right->as_constant_ptr()->as_jlong_bits();
1803 Register dreg = as_reg(dest);
1804 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1805 if (c == 0 && dreg == lreg_lo) {
1806 COMMENT("effective nop elided");
1807 return;
1808 }
1809 switch (code) {
1810 case lir_add: __ add(dreg, lreg_lo, c); break;
1811 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1812 default:
1813 ShouldNotReachHere();
1814 }
1815 } else {
1816 ShouldNotReachHere();
1817 }
1818 } else if (left->is_single_fpu()) {
1819 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1820 switch (code) {
1821 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1822 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1823 case lir_mul_strictfp: // fall through
1824 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1825 case lir_div_strictfp: // fall through
1826 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1827 default:
1828 ShouldNotReachHere();
1829 }
1830 } else if (left->is_double_fpu()) {
1831 if (right->is_double_fpu()) {
1832 // fpu register - fpu register
1833 switch (code) {
1834 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1835 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1836 case lir_mul_strictfp: // fall through
1837 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1838 case lir_div_strictfp: // fall through
1839 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1840 default:
1841 ShouldNotReachHere();
1842 }
1843 } else {
1844 if (right->is_constant()) {
1845 ShouldNotReachHere();
1846 }
1847 ShouldNotReachHere();
1848 }
1849 } else if (left->is_single_stack() || left->is_address()) {
1850 assert(left == dest, "left and dest must be equal");
1851 ShouldNotReachHere();
1852 } else {
1853 ShouldNotReachHere();
1854 }
1855 }
1856
1857 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1858
1859
1860 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1861 switch(code) {
1862 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1863 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1864 default : ShouldNotReachHere();
1865 }
1866 }
1867
1868 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1869
1870 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1871 Register Rleft = left->is_single_cpu() ? left->as_register() :
1872 left->as_register_lo();
1873 if (dst->is_single_cpu()) {
1874 Register Rdst = dst->as_register();
1875 if (right->is_constant()) {
1876 switch (code) {
1877 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1878 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1879 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1880 default: ShouldNotReachHere(); break;
1881 }
1882 } else {
1883 Register Rright = right->is_single_cpu() ? right->as_register() :
1884 right->as_register_lo();
1885 switch (code) {
1886 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1887 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1888 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1889 default: ShouldNotReachHere(); break;
1890 }
1891 }
1892 } else {
1893 Register Rdst = dst->as_register_lo();
1894 if (right->is_constant()) {
1895 switch (code) {
1896 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1897 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1898 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1899 default: ShouldNotReachHere(); break;
1900 }
1901 } else {
1902 Register Rright = right->is_single_cpu() ? right->as_register() :
1903 right->as_register_lo();
1904 switch (code) {
1905 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1906 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1907 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1908 default: ShouldNotReachHere(); break;
1909 }
1910 }
1911 }
1912 }
1913
1914
1915
1916 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1917
1918
1919 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1920 if (opr1->is_constant() && opr2->is_single_cpu()) {
1921 // tableswitch
1922 Register reg = as_reg(opr2);
1923 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1924 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1925 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1926 Register reg1 = as_reg(opr1);
1927 if (opr2->is_single_cpu()) {
1928 // cpu register - cpu register
1929 Register reg2 = opr2->as_register();
1930 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1931 __ cmp(reg1, reg2);
1932 } else {
1933 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1934 __ cmpw(reg1, reg2);
1935 }
1936 return;
1937 }
1938 if (opr2->is_double_cpu()) {
1939 // cpu register - cpu register
1940 Register reg2 = opr2->as_register_lo();
1941 __ cmp(reg1, reg2);
1942 return;
1943 }
1944
1945 if (opr2->is_constant()) {
1946 bool is_32bit = false; // width of register operand
1947 jlong imm;
1948
1949 switch(opr2->type()) {
1950 case T_INT:
1951 imm = opr2->as_constant_ptr()->as_jint();
1952 is_32bit = true;
1953 break;
1954 case T_LONG:
1955 imm = opr2->as_constant_ptr()->as_jlong();
1956 break;
1957 case T_ADDRESS:
1958 imm = opr2->as_constant_ptr()->as_jint();
1959 break;
1960 case T_OBJECT:
1961 case T_ARRAY:
1962 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1963 break;
1964 default:
1965 ShouldNotReachHere();
1966 break;
1967 }
1968
1969 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1970 if (is_32bit)
1971 __ cmpw(reg1, imm);
1972 else
1973 __ cmp(reg1, imm);
1974 return;
1975 } else {
1976 __ mov(rscratch1, imm);
1977 if (is_32bit)
1978 __ cmpw(reg1, rscratch1);
1979 else
1980 __ cmp(reg1, rscratch1);
1981 return;
1982 }
1983 } else
1984 ShouldNotReachHere();
1985 } else if (opr1->is_single_fpu()) {
1986 FloatRegister reg1 = opr1->as_float_reg();
1987 assert(opr2->is_single_fpu(), "expect single float register");
1988 FloatRegister reg2 = opr2->as_float_reg();
1989 __ fcmps(reg1, reg2);
1990 } else if (opr1->is_double_fpu()) {
1991 FloatRegister reg1 = opr1->as_double_reg();
1992 assert(opr2->is_double_fpu(), "expect double float register");
1993 FloatRegister reg2 = opr2->as_double_reg();
1994 __ fcmpd(reg1, reg2);
1995 } else {
1996 ShouldNotReachHere();
1997 }
1998 }
1999
2000 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2001 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2002 bool is_unordered_less = (code == lir_ucmp_fd2i);
2003 if (left->is_single_fpu()) {
2004 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2005 } else if (left->is_double_fpu()) {
2006 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2007 } else {
2008 ShouldNotReachHere();
2009 }
2010 } else if (code == lir_cmp_l2i) {
2011 Label done;
2012 __ cmp(left->as_register_lo(), right->as_register_lo());
2013 __ mov(dst->as_register(), (u_int64_t)-1L);
2014 __ br(Assembler::LT, done);
2015 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2016 __ bind(done);
2017 } else {
2018 ShouldNotReachHere();
2019 }
2020 }
2021
2022
2023 void LIR_Assembler::align_call(LIR_Code code) { }
2024
2025
2026 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2027 address call = __ trampoline_call(Address(op->addr(), rtype));
2028 if (call == NULL) {
2029 bailout("trampoline stub overflow");
2030 return;
2031 }
2032 add_call_info(code_offset(), op->info());
2033 }
2034
2035
2036 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2037 address call = __ ic_call(op->addr());
2038 if (call == NULL) {
2039 bailout("trampoline stub overflow");
2040 return;
2041 }
2042 add_call_info(code_offset(), op->info());
2043 }
2044
2045
2046 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2047 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2048 ShouldNotReachHere();
2049 }
2050
2051
2052 void LIR_Assembler::emit_static_call_stub() {
2053 address call_pc = __ pc();
2054 address stub = __ start_a_stub(call_stub_size);
2055 if (stub == NULL) {
2056 bailout("static call stub overflow");
2057 return;
2058 }
2059
2060 int start = __ offset();
2061
2062 __ relocate(static_stub_Relocation::spec(call_pc));
2063 __ mov_metadata(rmethod, (Metadata*)NULL);
2064 __ movptr(rscratch1, 0);
2065 __ br(rscratch1);
2066
2067 assert(__ offset() - start <= call_stub_size, "stub too big");
2068 __ end_a_stub();
2069 }
2070
2071
2072 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2073 assert(exceptionOop->as_register() == r0, "must match");
2074 assert(exceptionPC->as_register() == r3, "must match");
2075
2076 // exception object is not added to oop map by LinearScan
2077 // (LinearScan assumes that no oops are in fixed registers)
2078 info->add_register_oop(exceptionOop);
2079 Runtime1::StubID unwind_id;
2080
2081 // get current pc information
2082 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2083 int pc_for_athrow_offset = __ offset();
2084 InternalAddress pc_for_athrow(__ pc());
2085 __ adr(exceptionPC->as_register(), pc_for_athrow);
2086 add_call_info(pc_for_athrow_offset, info); // for exception handler
2087
2088 __ verify_not_null_oop(r0);
2089 // search an exception handler (r0: exception oop, r3: throwing pc)
2090 if (compilation()->has_fpu_code()) {
2091 unwind_id = Runtime1::handle_exception_id;
2092 } else {
2093 unwind_id = Runtime1::handle_exception_nofpu_id;
2094 }
2095 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2096
2097 // FIXME: enough room for two byte trap ????
2098 __ nop();
2099 }
2100
2101
2102 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2103 assert(exceptionOop->as_register() == r0, "must match");
2104
2105 __ b(_unwind_handler_entry);
2106 }
2107
2108
2109 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2110 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2111 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2112
2113 switch (left->type()) {
2114 case T_INT: {
2115 switch (code) {
2116 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2117 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2118 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2119 default:
2120 ShouldNotReachHere();
2121 break;
2122 }
2123 break;
2124 case T_LONG:
2125 case T_ADDRESS:
2126 case T_OBJECT:
2127 switch (code) {
2128 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2129 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2130 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2131 default:
2132 ShouldNotReachHere();
2133 break;
2134 }
2135 break;
2136 default:
2137 ShouldNotReachHere();
2138 break;
2139 }
2140 }
2141 }
2142
2143
2144 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2145 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2146 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2147
2148 switch (left->type()) {
2149 case T_INT: {
2150 switch (code) {
2151 case lir_shl: __ lslw (dreg, lreg, count); break;
2152 case lir_shr: __ asrw (dreg, lreg, count); break;
2153 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2154 default:
2155 ShouldNotReachHere();
2156 break;
2157 }
2158 break;
2159 case T_LONG:
2160 case T_ADDRESS:
2161 case T_OBJECT:
2162 switch (code) {
2163 case lir_shl: __ lsl (dreg, lreg, count); break;
2164 case lir_shr: __ asr (dreg, lreg, count); break;
2165 case lir_ushr: __ lsr (dreg, lreg, count); break;
2166 default:
2167 ShouldNotReachHere();
2168 break;
2169 }
2170 break;
2171 default:
2172 ShouldNotReachHere();
2173 break;
2174 }
2175 }
2176 }
2177
2178
2179 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2180 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2181 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2182 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2183 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2184 }
2185
2186
2187 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2188 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2189 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2190 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2191 __ mov (rscratch1, c);
2192 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2193 }
2194
2195
2196 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2197 ShouldNotReachHere();
2198 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2199 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2200 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2201 __ lea(rscratch1, __ constant_oop_address(o));
2202 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2203 }
2204
2205
2206 // This code replaces a call to arraycopy; no exception may
2207 // be thrown in this code, they must be thrown in the System.arraycopy
2208 // activation frame; we could save some checks if this would not be the case
2209 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2210 ciArrayKlass* default_type = op->expected_type();
2211 Register src = op->src()->as_register();
2212 Register dst = op->dst()->as_register();
2213 Register src_pos = op->src_pos()->as_register();
2214 Register dst_pos = op->dst_pos()->as_register();
2215 Register length = op->length()->as_register();
2216 Register tmp = op->tmp()->as_register();
2217
2218 CodeStub* stub = op->stub();
2219 int flags = op->flags();
2220 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2221 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2222
2223 // if we don't know anything, just go through the generic arraycopy
2224 if (default_type == NULL // || basic_type == T_OBJECT
2225 ) {
2226 Label done;
2227 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2228
2229 // Save the arguments in case the generic arraycopy fails and we
2230 // have to fall back to the JNI stub
2231 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2232 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2233 __ str(src, Address(sp, 4*BytesPerWord));
2234
2235 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2236 address copyfunc_addr = StubRoutines::generic_arraycopy();
2237
2238 // The arguments are in java calling convention so we shift them
2239 // to C convention
2240 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2241 __ mov(c_rarg0, j_rarg0);
2242 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2243 __ mov(c_rarg1, j_rarg1);
2244 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2245 __ mov(c_rarg2, j_rarg2);
2246 assert_different_registers(c_rarg3, j_rarg4);
2247 __ mov(c_rarg3, j_rarg3);
2248 __ mov(c_rarg4, j_rarg4);
2249 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2250 __ mov(rscratch1, RuntimeAddress(C_entry));
2251 __ blr(rscratch1);
2252 } else {
2253 #ifndef PRODUCT
2254 if (PrintC1Statistics) {
2255 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2256 }
2257 #endif
2258 __ far_call(RuntimeAddress(copyfunc_addr));
2259 }
2260
2261 __ cbz(r0, *stub->continuation());
2262
2263 // Reload values from the stack so they are where the stub
2264 // expects them.
2265 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2266 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2267 __ ldr(src, Address(sp, 4*BytesPerWord));
2268
2269 if (copyfunc_addr != NULL) {
2270 // r0 is -1^K where K == partial copied count
2271 __ eonw(rscratch1, r0, zr);
2272 // adjust length down and src/end pos up by partial copied count
2273 __ subw(length, length, rscratch1);
2274 __ addw(src_pos, src_pos, rscratch1);
2275 __ addw(dst_pos, dst_pos, rscratch1);
2276 }
2277 __ b(*stub->entry());
2278
2279 __ bind(*stub->continuation());
2280 return;
2281 }
2282
2283 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2284
2285 int elem_size = type2aelembytes(basic_type);
2286 int shift_amount;
2287 int scale = exact_log2(elem_size);
2288
2289 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2290 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2291 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2292 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2293
2294 // test for NULL
2295 if (flags & LIR_OpArrayCopy::src_null_check) {
2296 __ cbz(src, *stub->entry());
2297 }
2298 if (flags & LIR_OpArrayCopy::dst_null_check) {
2299 __ cbz(dst, *stub->entry());
2300 }
2301
2302 // If the compiler was not able to prove that exact type of the source or the destination
2303 // of the arraycopy is an array type, check at runtime if the source or the destination is
2304 // an instance type.
2305 if (flags & LIR_OpArrayCopy::type_check) {
2306 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2307 __ load_klass(tmp, dst);
2308 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2309 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2310 __ br(Assembler::GE, *stub->entry());
2311 }
2312
2313 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2314 __ load_klass(tmp, src);
2315 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2316 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2317 __ br(Assembler::GE, *stub->entry());
2318 }
2319 }
2320
2321 // check if negative
2322 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2323 __ cmpw(src_pos, 0);
2324 __ br(Assembler::LT, *stub->entry());
2325 }
2326 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2327 __ cmpw(dst_pos, 0);
2328 __ br(Assembler::LT, *stub->entry());
2329 }
2330
2331 if (flags & LIR_OpArrayCopy::length_positive_check) {
2332 __ cmpw(length, 0);
2333 __ br(Assembler::LT, *stub->entry());
2334 }
2335
2336 if (flags & LIR_OpArrayCopy::src_range_check) {
2337 __ addw(tmp, src_pos, length);
2338 __ ldrw(rscratch1, src_length_addr);
2339 __ cmpw(tmp, rscratch1);
2340 __ br(Assembler::HI, *stub->entry());
2341 }
2342 if (flags & LIR_OpArrayCopy::dst_range_check) {
2343 __ addw(tmp, dst_pos, length);
2344 __ ldrw(rscratch1, dst_length_addr);
2345 __ cmpw(tmp, rscratch1);
2346 __ br(Assembler::HI, *stub->entry());
2347 }
2348
2349 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2350 // length_positive_check if the source of our length operand is an
2351 // arraylength. However, that arraylength might be zero, and the
2352 // stub that we're about to call contains an assertion that count !=
2353 // 0 . So we make this check purely in order not to trigger an
2354 // assertion failure.
2355 __ cbzw(length, *stub->continuation());
2356
2357 if (flags & LIR_OpArrayCopy::type_check) {
2358 // We don't know the array types are compatible
2359 if (basic_type != T_OBJECT) {
2360 // Simple test for basic type arrays
2361 if (UseCompressedClassPointers) {
2362 __ ldrw(tmp, src_klass_addr);
2363 __ ldrw(rscratch1, dst_klass_addr);
2364 __ cmpw(tmp, rscratch1);
2365 } else {
2366 __ ldr(tmp, src_klass_addr);
2367 __ ldr(rscratch1, dst_klass_addr);
2368 __ cmp(tmp, rscratch1);
2369 }
2370 __ br(Assembler::NE, *stub->entry());
2371 } else {
2372 // For object arrays, if src is a sub class of dst then we can
2373 // safely do the copy.
2374 Label cont, slow;
2375
2376 #define PUSH(r1, r2) \
2377 stp(r1, r2, __ pre(sp, -2 * wordSize));
2378
2379 #define POP(r1, r2) \
2380 ldp(r1, r2, __ post(sp, 2 * wordSize));
2381
2382 __ PUSH(src, dst);
2383
2384 __ load_klass(src, src);
2385 __ load_klass(dst, dst);
2386
2387 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2388
2389 __ PUSH(src, dst);
2390 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2391 __ POP(src, dst);
2392
2393 __ cbnz(src, cont);
2394
2395 __ bind(slow);
2396 __ POP(src, dst);
2397
2398 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2399 if (copyfunc_addr != NULL) { // use stub if available
2400 // src is not a sub class of dst so we have to do a
2401 // per-element check.
2402
2403 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2404 if ((flags & mask) != mask) {
2405 // Check that at least both of them object arrays.
2406 assert(flags & mask, "one of the two should be known to be an object array");
2407
2408 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2409 __ load_klass(tmp, src);
2410 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2411 __ load_klass(tmp, dst);
2412 }
2413 int lh_offset = in_bytes(Klass::layout_helper_offset());
2414 Address klass_lh_addr(tmp, lh_offset);
2415 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2416 __ ldrw(rscratch1, klass_lh_addr);
2417 __ mov(rscratch2, objArray_lh);
2418 __ eorw(rscratch1, rscratch1, rscratch2);
2419 __ cbnzw(rscratch1, *stub->entry());
2420 }
2421
2422 // Spill because stubs can use any register they like and it's
2423 // easier to restore just those that we care about.
2424 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2425 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2426 __ str(src, Address(sp, 4*BytesPerWord));
2427
2428 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2429 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2430 assert_different_registers(c_rarg0, dst, dst_pos, length);
2431 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2432 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2433 assert_different_registers(c_rarg1, dst, length);
2434 __ uxtw(c_rarg2, length);
2435 assert_different_registers(c_rarg2, dst);
2436
2437 __ load_klass(c_rarg4, dst);
2438 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2439 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2440 __ far_call(RuntimeAddress(copyfunc_addr));
2441
2442 #ifndef PRODUCT
2443 if (PrintC1Statistics) {
2444 Label failed;
2445 __ cbnz(r0, failed);
2446 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2447 __ bind(failed);
2448 }
2449 #endif
2450
2451 __ cbz(r0, *stub->continuation());
2452
2453 #ifndef PRODUCT
2454 if (PrintC1Statistics) {
2455 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2456 }
2457 #endif
2458 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2459
2460 // Restore previously spilled arguments
2461 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2462 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2463 __ ldr(src, Address(sp, 4*BytesPerWord));
2464
2465 // return value is -1^K where K is partial copied count
2466 __ eonw(rscratch1, r0, zr);
2467 // adjust length down and src/end pos up by partial copied count
2468 __ subw(length, length, rscratch1);
2469 __ addw(src_pos, src_pos, rscratch1);
2470 __ addw(dst_pos, dst_pos, rscratch1);
2471 }
2472
2473 __ b(*stub->entry());
2474
2475 __ bind(cont);
2476 __ POP(src, dst);
2477 }
2478 }
2479
2480 #ifdef ASSERT
2481 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2482 // Sanity check the known type with the incoming class. For the
2483 // primitive case the types must match exactly with src.klass and
2484 // dst.klass each exactly matching the default type. For the
2485 // object array case, if no type check is needed then either the
2486 // dst type is exactly the expected type and the src type is a
2487 // subtype which we can't check or src is the same array as dst
2488 // but not necessarily exactly of type default_type.
2489 Label known_ok, halt;
2490 __ mov_metadata(tmp, default_type->constant_encoding());
2491 #ifdef _LP64
2492 if (UseCompressedClassPointers) {
2493 __ encode_klass_not_null(tmp);
2494 }
2495 #endif
2496
2497 if (basic_type != T_OBJECT) {
2498
2499 if (UseCompressedClassPointers) {
2500 __ ldrw(rscratch1, dst_klass_addr);
2501 __ cmpw(tmp, rscratch1);
2502 } else {
2503 __ ldr(rscratch1, dst_klass_addr);
2504 __ cmp(tmp, rscratch1);
2505 }
2506 __ br(Assembler::NE, halt);
2507 if (UseCompressedClassPointers) {
2508 __ ldrw(rscratch1, src_klass_addr);
2509 __ cmpw(tmp, rscratch1);
2510 } else {
2511 __ ldr(rscratch1, src_klass_addr);
2512 __ cmp(tmp, rscratch1);
2513 }
2514 __ br(Assembler::EQ, known_ok);
2515 } else {
2516 if (UseCompressedClassPointers) {
2517 __ ldrw(rscratch1, dst_klass_addr);
2518 __ cmpw(tmp, rscratch1);
2519 } else {
2520 __ ldr(rscratch1, dst_klass_addr);
2521 __ cmp(tmp, rscratch1);
2522 }
2523 __ br(Assembler::EQ, known_ok);
2524 __ cmp(src, dst);
2525 __ br(Assembler::EQ, known_ok);
2526 }
2527 __ bind(halt);
2528 __ stop("incorrect type information in arraycopy");
2529 __ bind(known_ok);
2530 }
2531 #endif
2532
2533 #ifndef PRODUCT
2534 if (PrintC1Statistics) {
2535 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2536 }
2537 #endif
2538
2539 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2540 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2541 assert_different_registers(c_rarg0, dst, dst_pos, length);
2542 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2543 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2544 assert_different_registers(c_rarg1, dst, length);
2545 __ uxtw(c_rarg2, length);
2546 assert_different_registers(c_rarg2, dst);
2547
2548 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2549 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2550 const char *name;
2551 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2552
2553 CodeBlob *cb = CodeCache::find_blob(entry);
2554 if (cb) {
2555 __ far_call(RuntimeAddress(entry));
2556 } else {
2557 __ call_VM_leaf(entry, 3);
2558 }
2559
2560 __ bind(*stub->continuation());
2561 }
2562
2563
2564
2565
2566 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2567 Register obj = op->obj_opr()->as_register(); // may not be an oop
2568 Register hdr = op->hdr_opr()->as_register();
2569 Register lock = op->lock_opr()->as_register();
2570 if (!UseFastLocking) {
2571 __ b(*op->stub()->entry());
2572 } else if (op->code() == lir_lock) {
2573 Register scratch = noreg;
2574 if (UseBiasedLocking) {
2575 scratch = op->scratch_opr()->as_register();
2576 }
2577 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2578 // add debug info for NullPointerException only if one is possible
2579 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2580 if (op->info() != NULL) {
2581 add_debug_info_for_null_check(null_check_offset, op->info());
2582 }
2583 // done
2584 } else if (op->code() == lir_unlock) {
2585 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2586 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2587 } else {
2588 Unimplemented();
2589 }
2590 __ bind(*op->stub()->continuation());
2591 }
2592
2593
2594 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2595 ciMethod* method = op->profiled_method();
2596 int bci = op->profiled_bci();
2597 ciMethod* callee = op->profiled_callee();
2598
2599 // Update counter for all call types
2600 ciMethodData* md = method->method_data_or_null();
2601 assert(md != NULL, "Sanity");
2602 ciProfileData* data = md->bci_to_data(bci);
2603 assert(data->is_CounterData(), "need CounterData for calls");
2604 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2605 Register mdo = op->mdo()->as_register();
2606 __ mov_metadata(mdo, md->constant_encoding());
2607 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2608 Bytecodes::Code bc = method->java_code_at_bci(bci);
2609 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2610 // Perform additional virtual call profiling for invokevirtual and
2611 // invokeinterface bytecodes
2612 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2613 !callee_is_static && // required for optimized MH invokes
2614 C1ProfileVirtualCalls) {
2615 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2616 Register recv = op->recv()->as_register();
2617 assert_different_registers(mdo, recv);
2618 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2619 ciKlass* known_klass = op->known_holder();
2620 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2621 // We know the type that will be seen at this call site; we can
2622 // statically update the MethodData* rather than needing to do
2623 // dynamic tests on the receiver type
2624
2625 // NOTE: we should probably put a lock around this search to
2626 // avoid collisions by concurrent compilations
2627 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2628 uint i;
2629 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2630 ciKlass* receiver = vc_data->receiver(i);
2631 if (known_klass->equals(receiver)) {
2632 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2633 __ addptr(data_addr, DataLayout::counter_increment);
2634 return;
2635 }
2636 }
2637
2638 // Receiver type not found in profile data; select an empty slot
2639
2640 // Note that this is less efficient than it should be because it
2641 // always does a write to the receiver part of the
2642 // VirtualCallData rather than just the first time
2643 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2644 ciKlass* receiver = vc_data->receiver(i);
2645 if (receiver == NULL) {
2646 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2647 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2648 __ lea(rscratch2, recv_addr);
2649 __ str(rscratch1, Address(rscratch2));
2650 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2651 __ addptr(data_addr, DataLayout::counter_increment);
2652 return;
2653 }
2654 }
2655 } else {
2656 __ load_klass(recv, recv);
2657 Label update_done;
2658 type_profile_helper(mdo, md, data, recv, &update_done);
2659 // Receiver did not match any saved receiver and there is no empty row for it.
2660 // Increment total counter to indicate polymorphic case.
2661 __ addptr(counter_addr, DataLayout::counter_increment);
2662
2663 __ bind(update_done);
2664 }
2665 } else {
2666 // Static call
2667 __ addptr(counter_addr, DataLayout::counter_increment);
2668 }
2669 }
2670
2671
2672 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2673 Unimplemented();
2674 }
2675
2676
2677 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2678 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2679 }
2680
2681 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2682 assert(op->crc()->is_single_cpu(), "crc must be register");
2683 assert(op->val()->is_single_cpu(), "byte value must be register");
2684 assert(op->result_opr()->is_single_cpu(), "result must be register");
2685 Register crc = op->crc()->as_register();
2686 Register val = op->val()->as_register();
2687 Register res = op->result_opr()->as_register();
2688
2689 assert_different_registers(val, crc, res);
2690 unsigned long offset;
2691 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2692 if (offset) __ add(res, res, offset);
2693
2694 __ ornw(crc, zr, crc); // ~crc
2695 __ update_byte_crc32(crc, val, res);
2696 __ ornw(res, zr, crc); // ~crc
2697 }
2698
2699 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2700 COMMENT("emit_profile_type {");
2701 Register obj = op->obj()->as_register();
2702 Register tmp = op->tmp()->as_pointer_register();
2703 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2704 ciKlass* exact_klass = op->exact_klass();
2705 intptr_t current_klass = op->current_klass();
2706 bool not_null = op->not_null();
2707 bool no_conflict = op->no_conflict();
2708
2709 Label update, next, none;
2710
2711 bool do_null = !not_null;
2712 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2713 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2714
2715 assert(do_null || do_update, "why are we here?");
2716 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2717 assert(mdo_addr.base() != rscratch1, "wrong register");
2718
2719 __ verify_oop(obj);
2720
2721 if (tmp != obj) {
2722 __ mov(tmp, obj);
2723 }
2724 if (do_null) {
2725 __ cbnz(tmp, update);
2726 if (!TypeEntries::was_null_seen(current_klass)) {
2727 __ ldr(rscratch2, mdo_addr);
2728 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2729 __ str(rscratch2, mdo_addr);
2730 }
2731 if (do_update) {
2732 #ifndef ASSERT
2733 __ b(next);
2734 }
2735 #else
2736 __ b(next);
2737 }
2738 } else {
2739 __ cbnz(tmp, update);
2740 __ stop("unexpected null obj");
2741 #endif
2742 }
2743
2744 __ bind(update);
2745
2746 if (do_update) {
2747 #ifdef ASSERT
2748 if (exact_klass != NULL) {
2749 Label ok;
2750 __ load_klass(tmp, tmp);
2751 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2752 __ eor(rscratch1, tmp, rscratch1);
2753 __ cbz(rscratch1, ok);
2754 __ stop("exact klass and actual klass differ");
2755 __ bind(ok);
2756 }
2757 #endif
2758 if (!no_conflict) {
2759 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2760 if (exact_klass != NULL) {
2761 __ mov_metadata(tmp, exact_klass->constant_encoding());
2762 } else {
2763 __ load_klass(tmp, tmp);
2764 }
2765
2766 __ ldr(rscratch2, mdo_addr);
2767 __ eor(tmp, tmp, rscratch2);
2768 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2769 // klass seen before, nothing to do. The unknown bit may have been
2770 // set already but no need to check.
2771 __ cbz(rscratch1, next);
2772
2773 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2774 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2775
2776 if (TypeEntries::is_type_none(current_klass)) {
2777 __ cbz(rscratch2, none);
2778 __ cmp(rscratch2, TypeEntries::null_seen);
2779 __ br(Assembler::EQ, none);
2780 // There is a chance that the checks above (re-reading profiling
2781 // data from memory) fail if another thread has just set the
2782 // profiling to this obj's klass
2783 __ dmb(Assembler::ISHLD);
2784 __ ldr(rscratch2, mdo_addr);
2785 __ eor(tmp, tmp, rscratch2);
2786 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2787 __ cbz(rscratch1, next);
2788 }
2789 } else {
2790 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2791 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2792
2793 __ ldr(tmp, mdo_addr);
2794 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2795 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2796 }
2797
2798 // different than before. Cannot keep accurate profile.
2799 __ ldr(rscratch2, mdo_addr);
2800 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2801 __ str(rscratch2, mdo_addr);
2802
2803 if (TypeEntries::is_type_none(current_klass)) {
2804 __ b(next);
2805
2806 __ bind(none);
2807 // first time here. Set profile type.
2808 __ str(tmp, mdo_addr);
2809 }
2810 } else {
2811 // There's a single possible klass at this profile point
2812 assert(exact_klass != NULL, "should be");
2813 if (TypeEntries::is_type_none(current_klass)) {
2814 __ mov_metadata(tmp, exact_klass->constant_encoding());
2815 __ ldr(rscratch2, mdo_addr);
2816 __ eor(tmp, tmp, rscratch2);
2817 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2818 __ cbz(rscratch1, next);
2819 #ifdef ASSERT
2820 {
2821 Label ok;
2822 __ ldr(rscratch1, mdo_addr);
2823 __ cbz(rscratch1, ok);
2824 __ cmp(rscratch1, TypeEntries::null_seen);
2825 __ br(Assembler::EQ, ok);
2826 // may have been set by another thread
2827 __ dmb(Assembler::ISHLD);
2828 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2829 __ ldr(rscratch2, mdo_addr);
2830 __ eor(rscratch2, rscratch1, rscratch2);
2831 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2832 __ cbz(rscratch2, ok);
2833
2834 __ stop("unexpected profiling mismatch");
2835 __ bind(ok);
2836 }
2837 #endif
2838 // first time here. Set profile type.
2839 __ ldr(tmp, mdo_addr);
2840 } else {
2841 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2842 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2843
2844 __ ldr(tmp, mdo_addr);
2845 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2846 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2847
2848 __ orr(tmp, tmp, TypeEntries::type_unknown);
2849 __ str(tmp, mdo_addr);
2850 // FIXME: Write barrier needed here?
2851 }
2852 }
2853
2854 __ bind(next);
2855 }
2856 COMMENT("} emit_profile_type");
2857 }
2858
2859
2860 void LIR_Assembler::align_backward_branch_target() {
2861 }
2862
2863
2864 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2865 if (left->is_single_cpu()) {
2866 assert(dest->is_single_cpu(), "expect single result reg");
2867 __ negw(dest->as_register(), left->as_register());
2868 } else if (left->is_double_cpu()) {
2869 assert(dest->is_double_cpu(), "expect double result reg");
2870 __ neg(dest->as_register_lo(), left->as_register_lo());
2871 } else if (left->is_single_fpu()) {
2872 assert(dest->is_single_fpu(), "expect single float result reg");
2873 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2874 } else {
2875 assert(left->is_double_fpu(), "expect double float operand reg");
2876 assert(dest->is_double_fpu(), "expect double float result reg");
2877 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2878 }
2879 }
2880
2881
2882 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2883 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2884 }
2885
2886
2887 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2888 assert(!tmp->is_valid(), "don't need temporary");
2889
2890 CodeBlob *cb = CodeCache::find_blob(dest);
2891 if (cb) {
2892 __ far_call(RuntimeAddress(dest));
2893 } else {
2894 __ mov(rscratch1, RuntimeAddress(dest));
2895 __ blr(rscratch1);
2896 }
2897
2898 if (info != NULL) {
2899 add_call_info_here(info);
2900 }
2901 __ maybe_isb();
2902 }
2903
2904 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2905 if (dest->is_address() || src->is_address()) {
2906 move_op(src, dest, type, lir_patch_none, info,
2907 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2908 } else {
2909 ShouldNotReachHere();
2910 }
2911 }
2912
2913 #ifdef ASSERT
2914 // emit run-time assertion
2915 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2916 assert(op->code() == lir_assert, "must be");
2917
2918 if (op->in_opr1()->is_valid()) {
2919 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2920 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2921 } else {
2922 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2923 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2924 }
2925
2926 Label ok;
2927 if (op->condition() != lir_cond_always) {
2928 Assembler::Condition acond = Assembler::AL;
2929 switch (op->condition()) {
2930 case lir_cond_equal: acond = Assembler::EQ; break;
2931 case lir_cond_notEqual: acond = Assembler::NE; break;
2932 case lir_cond_less: acond = Assembler::LT; break;
2933 case lir_cond_lessEqual: acond = Assembler::LE; break;
2934 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2935 case lir_cond_greater: acond = Assembler::GT; break;
2936 case lir_cond_belowEqual: acond = Assembler::LS; break;
2937 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2938 default: ShouldNotReachHere();
2939 }
2940 __ br(acond, ok);
2941 }
2942 if (op->halt()) {
2943 const char* str = __ code_string(op->msg());
2944 __ stop(str);
2945 } else {
2946 breakpoint();
2947 }
2948 __ bind(ok);
2949 }
2950 #endif
2951
2952 #ifndef PRODUCT
2953 #define COMMENT(x) do { __ block_comment(x); } while (0)
2954 #else
2955 #define COMMENT(x)
2956 #endif
2957
2958 void LIR_Assembler::membar() {
2959 COMMENT("membar");
2960 __ membar(MacroAssembler::AnyAny);
2961 }
2962
2963 void LIR_Assembler::membar_acquire() {
2964 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2965 }
2966
2967 void LIR_Assembler::membar_release() {
2968 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2969 }
2970
2971 void LIR_Assembler::membar_loadload() {
2972 __ membar(Assembler::LoadLoad);
2973 }
2974
2975 void LIR_Assembler::membar_storestore() {
2976 __ membar(MacroAssembler::StoreStore);
2977 }
2978
2979 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2980
2981 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2982
2983 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2984 __ mov(result_reg->as_register(), rthread);
2985 }
2986
2987
2988 void LIR_Assembler::peephole(LIR_List *lir) {
2989 #if 0
2990 if (tableswitch_count >= max_tableswitches)
2991 return;
2992
2993 /*
2994 This finite-state automaton recognizes sequences of compare-and-
2995 branch instructions. We will turn them into a tableswitch. You
2996 could argue that C1 really shouldn't be doing this sort of
2997 optimization, but without it the code is really horrible.
2998 */
2999
3000 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3001 int first_key, last_key = -2147483648;
3002 int next_key = 0;
3003 int start_insn = -1;
3004 int last_insn = -1;
3005 Register reg = noreg;
3006 LIR_Opr reg_opr;
3007 state = start_s;
3008
3009 LIR_OpList* inst = lir->instructions_list();
3010 for (int i = 0; i < inst->length(); i++) {
3011 LIR_Op* op = inst->at(i);
3012 switch (state) {
3013 case start_s:
3014 first_key = -1;
3015 start_insn = i;
3016 switch (op->code()) {
3017 case lir_cmp:
3018 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3019 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3020 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3021 && opr2->is_constant()
3022 && opr2->type() == T_INT) {
3023 reg_opr = opr1;
3024 reg = opr1->as_register();
3025 first_key = opr2->as_constant_ptr()->as_jint();
3026 next_key = first_key + 1;
3027 state = cmp_s;
3028 goto next_state;
3029 }
3030 break;
3031 }
3032 break;
3033 case cmp_s:
3034 switch (op->code()) {
3035 case lir_branch:
3036 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3037 state = beq_s;
3038 last_insn = i;
3039 goto next_state;
3040 }
3041 }
3042 state = start_s;
3043 break;
3044 case beq_s:
3045 switch (op->code()) {
3046 case lir_cmp: {
3047 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3048 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3049 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3050 && opr1->as_register() == reg
3051 && opr2->is_constant()
3052 && opr2->type() == T_INT
3053 && opr2->as_constant_ptr()->as_jint() == next_key) {
3054 last_key = next_key;
3055 next_key++;
3056 state = cmp_s;
3057 goto next_state;
3058 }
3059 }
3060 }
3061 last_key = next_key;
3062 state = start_s;
3063 break;
3064 default:
3065 assert(false, "impossible state");
3066 }
3067 if (state == start_s) {
3068 if (first_key < last_key - 5L && reg != noreg) {
3069 {
3070 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3071 // reg->encoding(),
3072 // start_insn, first_key, last_key);
3073 // for (int i = 0; i < inst->length(); i++) {
3074 // inst->at(i)->print();
3075 // tty->print("\n");
3076 // }
3077 // tty->print("\n");
3078 }
3079
3080 struct tableswitch *sw = &switches[tableswitch_count];
3081 sw->_insn_index = start_insn, sw->_first_key = first_key,
3082 sw->_last_key = last_key, sw->_reg = reg;
3083 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3084 {
3085 // Insert the new table of branches
3086 int offset = last_insn;
3087 for (int n = first_key; n < last_key; n++) {
3088 inst->insert_before
3089 (last_insn + 1,
3090 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3091 inst->at(offset)->as_OpBranch()->label()));
3092 offset -= 2, i++;
3093 }
3094 }
3095 // Delete all the old compare-and-branch instructions
3096 for (int n = first_key; n < last_key; n++) {
3097 inst->remove_at(start_insn);
3098 inst->remove_at(start_insn);
3099 }
3100 // Insert the tableswitch instruction
3101 inst->insert_before(start_insn,
3102 new LIR_Op2(lir_cmp, lir_cond_always,
3103 LIR_OprFact::intConst(tableswitch_count),
3104 reg_opr));
3105 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3106 tableswitch_count++;
3107 }
3108 reg = noreg;
3109 last_key = -2147483648;
3110 }
3111 next_state:
3112 ;
3113 }
3114 #endif
3115 }
3116
3117 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3118 Address addr = as_Address(src->as_address_ptr());
3119 BasicType type = src->type();
3120 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3121
3122 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3123 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3124
3125 switch(type) {
3126 case T_INT:
3127 xchg = &MacroAssembler::atomic_xchgalw;
3128 add = &MacroAssembler::atomic_addalw;
3129 break;
3130 case T_LONG:
3131 xchg = &MacroAssembler::atomic_xchgal;
3132 add = &MacroAssembler::atomic_addal;
3133 break;
3134 case T_OBJECT:
3135 case T_ARRAY:
3136 if (UseCompressedOops) {
3137 xchg = &MacroAssembler::atomic_xchgalw;
3138 add = &MacroAssembler::atomic_addalw;
3139 } else {
3140 xchg = &MacroAssembler::atomic_xchgal;
3141 add = &MacroAssembler::atomic_addal;
3142 }
3143 break;
3144 default:
3145 ShouldNotReachHere();
3146 xchg = &MacroAssembler::atomic_xchgal;
3147 add = &MacroAssembler::atomic_addal; // unreachable
3148 }
3149
3150 switch (code) {
3151 case lir_xadd:
3152 {
3153 RegisterOrConstant inc;
3154 Register tmp = as_reg(tmp_op);
3155 Register dst = as_reg(dest);
3156 if (data->is_constant()) {
3157 inc = RegisterOrConstant(as_long(data));
3158 assert_different_registers(dst, addr.base(), tmp,
3159 rscratch1, rscratch2);
3160 } else {
3161 inc = RegisterOrConstant(as_reg(data));
3162 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3163 rscratch1, rscratch2);
3164 }
3165 __ lea(tmp, addr);
3166 (_masm->*add)(dst, inc, tmp);
3167 break;
3168 }
3169 case lir_xchg:
3170 {
3171 Register tmp = tmp_op->as_register();
3172 Register obj = as_reg(data);
3173 Register dst = as_reg(dest);
3174 if (is_oop && UseCompressedOops) {
3175 __ encode_heap_oop(rscratch2, obj);
3176 obj = rscratch2;
3177 }
3178 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3179 __ lea(tmp, addr);
3180 (_masm->*xchg)(dst, obj, tmp);
3181 if (is_oop && UseCompressedOops) {
3182 __ decode_heap_oop(dst);
3183 }
3184 }
3185 break;
3186 default:
3187 ShouldNotReachHere();
3188 }
3189 __ membar(__ AnyAny);
3190 }
3191
3192 #undef __