1 /* 2 * Copyright (c) 2013, Red Hat Inc. 3 * Copyright (c) 2000, 2020, Oracle and/or its affiliates. 4 * All rights reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "asm/assembler.hpp" 29 #include "c1/c1_CodeStubs.hpp" 30 #include "c1/c1_Compilation.hpp" 31 #include "c1/c1_LIRAssembler.hpp" 32 #include "c1/c1_MacroAssembler.hpp" 33 #include "c1/c1_Runtime1.hpp" 34 #include "c1/c1_ValueStack.hpp" 35 #include "ci/ciArrayKlass.hpp" 36 #include "ci/ciInstance.hpp" 37 #include "gc_interface/collectedHeap.hpp" 38 #include "memory/barrierSet.hpp" 39 #include "memory/cardTableModRefBS.hpp" 40 #include "nativeInst_aarch64.hpp" 41 #include "oops/objArrayKlass.hpp" 42 #include "runtime/sharedRuntime.hpp" 43 #include "vmreg_aarch64.inline.hpp" 44 45 46 47 #ifndef PRODUCT 48 #define COMMENT(x) do { __ block_comment(x); } while (0) 49 #else 50 #define COMMENT(x) 51 #endif 52 53 NEEDS_CLEANUP // remove this definitions ? 54 const Register IC_Klass = rscratch2; // where the IC klass is cached 55 const Register SYNC_header = r0; // synchronization header 56 const Register SHIFT_count = r0; // where count for shift operations must be 57 58 #define __ _masm-> 59 60 61 static void select_different_registers(Register preserve, 62 Register extra, 63 Register &tmp1, 64 Register &tmp2) { 65 if (tmp1 == preserve) { 66 assert_different_registers(tmp1, tmp2, extra); 67 tmp1 = extra; 68 } else if (tmp2 == preserve) { 69 assert_different_registers(tmp1, tmp2, extra); 70 tmp2 = extra; 71 } 72 assert_different_registers(preserve, tmp1, tmp2); 73 } 74 75 76 77 static void select_different_registers(Register preserve, 78 Register extra, 79 Register &tmp1, 80 Register &tmp2, 81 Register &tmp3) { 82 if (tmp1 == preserve) { 83 assert_different_registers(tmp1, tmp2, tmp3, extra); 84 tmp1 = extra; 85 } else if (tmp2 == preserve) { 86 assert_different_registers(tmp1, tmp2, tmp3, extra); 87 tmp2 = extra; 88 } else if (tmp3 == preserve) { 89 assert_different_registers(tmp1, tmp2, tmp3, extra); 90 tmp3 = extra; 91 } 92 assert_different_registers(preserve, tmp1, tmp2, tmp3); 93 } 94 95 96 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } 97 98 99 LIR_Opr LIR_Assembler::receiverOpr() { 100 return FrameMap::receiver_opr; 101 } 102 103 LIR_Opr LIR_Assembler::osrBufferPointer() { 104 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 105 } 106 107 //--------------fpu register translations----------------------- 108 109 110 address LIR_Assembler::float_constant(float f) { 111 address const_addr = __ float_constant(f); 112 if (const_addr == NULL) { 113 bailout("const section overflow"); 114 return __ code()->consts()->start(); 115 } else { 116 return const_addr; 117 } 118 } 119 120 121 address LIR_Assembler::double_constant(double d) { 122 address const_addr = __ double_constant(d); 123 if (const_addr == NULL) { 124 bailout("const section overflow"); 125 return __ code()->consts()->start(); 126 } else { 127 return const_addr; 128 } 129 } 130 131 address LIR_Assembler::int_constant(jlong n) { 132 address const_addr = __ long_constant(n); 133 if (const_addr == NULL) { 134 bailout("const section overflow"); 135 return __ code()->consts()->start(); 136 } else { 137 return const_addr; 138 } 139 } 140 141 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); } 142 143 void LIR_Assembler::reset_FPU() { Unimplemented(); } 144 145 void LIR_Assembler::fpop() { Unimplemented(); } 146 147 void LIR_Assembler::fxch(int i) { Unimplemented(); } 148 149 void LIR_Assembler::fld(int i) { Unimplemented(); } 150 151 void LIR_Assembler::ffree(int i) { Unimplemented(); } 152 153 void LIR_Assembler::breakpoint() { Unimplemented(); } 154 155 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); } 156 157 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); } 158 159 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; } 160 //------------------------------------------- 161 162 static Register as_reg(LIR_Opr op) { 163 return op->is_double_cpu() ? op->as_register_lo() : op->as_register(); 164 } 165 166 static jlong as_long(LIR_Opr data) { 167 jlong result; 168 switch (data->type()) { 169 case T_INT: 170 result = (data->as_jint()); 171 break; 172 case T_LONG: 173 result = (data->as_jlong()); 174 break; 175 default: 176 ShouldNotReachHere(); 177 } 178 return result; 179 } 180 181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 182 Register base = addr->base()->as_pointer_register(); 183 LIR_Opr opr = addr->index(); 184 if (opr->is_cpu_register()) { 185 Register index; 186 if (opr->is_single_cpu()) 187 index = opr->as_register(); 188 else 189 index = opr->as_register_lo(); 190 assert(addr->disp() == 0, "must be"); 191 switch(opr->type()) { 192 case T_INT: 193 return Address(base, index, Address::sxtw(addr->scale())); 194 case T_LONG: 195 return Address(base, index, Address::lsl(addr->scale())); 196 default: 197 ShouldNotReachHere(); 198 } 199 } else { 200 intptr_t addr_offset = intptr_t(addr->disp()); 201 if (Address::offset_ok_for_immed(addr_offset, addr->scale())) 202 return Address(base, addr_offset, Address::lsl(addr->scale())); 203 else { 204 __ mov(tmp, addr_offset); 205 return Address(base, tmp, Address::lsl(addr->scale())); 206 } 207 } 208 return Address(); 209 } 210 211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 212 ShouldNotReachHere(); 213 return Address(); 214 } 215 216 Address LIR_Assembler::as_Address(LIR_Address* addr) { 217 return as_Address(addr, rscratch1); 218 } 219 220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 221 return as_Address(addr, rscratch1); // Ouch 222 // FIXME: This needs to be much more clever. See x86. 223 } 224 225 226 void LIR_Assembler::osr_entry() { 227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 228 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 229 ValueStack* entry_state = osr_entry->state(); 230 int number_of_locks = entry_state->locks_size(); 231 232 // we jump here if osr happens with the interpreter 233 // state set up to continue at the beginning of the 234 // loop that triggered osr - in particular, we have 235 // the following registers setup: 236 // 237 // r2: osr buffer 238 // 239 240 // build frame 241 ciMethod* m = compilation()->method(); 242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 243 244 // OSR buffer is 245 // 246 // locals[nlocals-1..0] 247 // monitors[0..number_of_locks] 248 // 249 // locals is a direct copy of the interpreter frame so in the osr buffer 250 // so first slot in the local array is the last local from the interpreter 251 // and last slot is local[0] (receiver) from the interpreter 252 // 253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 255 // in the interpreter frame (the method lock if a sync method) 256 257 // Initialize monitors in the compiled activation. 258 // r2: pointer to osr buffer 259 // 260 // All other registers are dead at this point and the locals will be 261 // copied into place by code emitted in the IR. 262 263 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 265 int monitor_offset = BytesPerWord * method()->max_locals() + 266 (2 * BytesPerWord) * (number_of_locks - 1); 267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 268 // the OSR buffer using 2 word entries: first the lock and then 269 // the oop. 270 for (int i = 0; i < number_of_locks; i++) { 271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 272 #ifdef ASSERT 273 // verify the interpreter's monitor has a non-null object 274 { 275 Label L; 276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 277 __ cbnz(rscratch1, L); 278 __ stop("locked object is NULL"); 279 __ bind(L); 280 } 281 #endif 282 __ ldr(r19, Address(OSR_buf, slot_offset + 0)); 283 __ str(r19, frame_map()->address_for_monitor_lock(i)); 284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 285 __ str(r19, frame_map()->address_for_monitor_object(i)); 286 } 287 } 288 } 289 290 291 // inline cache check; done before the frame is built. 292 int LIR_Assembler::check_icache() { 293 Register receiver = FrameMap::receiver_opr->as_register(); 294 Register ic_klass = IC_Klass; 295 int start_offset = __ offset(); 296 __ inline_cache_check(receiver, ic_klass); 297 298 // if icache check fails, then jump to runtime routine 299 // Note: RECEIVER must still contain the receiver! 300 Label dont; 301 __ br(Assembler::EQ, dont); 302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 303 304 // We align the verified entry point unless the method body 305 // (including its inline cache check) will fit in a single 64-byte 306 // icache line. 307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) { 308 // force alignment after the cache check. 309 __ align(CodeEntryAlignment); 310 } 311 312 __ bind(dont); 313 return start_offset; 314 } 315 316 317 void LIR_Assembler::jobject2reg(jobject o, Register reg) { 318 if (o == NULL) { 319 __ mov(reg, zr); 320 } else { 321 __ movoop(reg, o, /*immediate*/true); 322 } 323 } 324 325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) { 326 address target = NULL; 327 relocInfo::relocType reloc_type = relocInfo::none; 328 329 switch (patching_id(info)) { 330 case PatchingStub::access_field_id: 331 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 332 reloc_type = relocInfo::section_word_type; 333 break; 334 case PatchingStub::load_klass_id: 335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 336 reloc_type = relocInfo::metadata_type; 337 break; 338 case PatchingStub::load_mirror_id: 339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 340 reloc_type = relocInfo::oop_type; 341 break; 342 case PatchingStub::load_appendix_id: 343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 344 reloc_type = relocInfo::oop_type; 345 break; 346 default: ShouldNotReachHere(); 347 } 348 349 __ far_call(RuntimeAddress(target)); 350 add_call_info_here(info); 351 } 352 353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) { 354 deoptimize_trap(info); 355 } 356 357 358 // This specifies the rsp decrement needed to build the frame 359 int LIR_Assembler::initial_frame_size_in_bytes() const { 360 // if rounding, must let FrameMap know! 361 362 // The frame_map records size in slots (32bit word) 363 364 // subtract two words to account for return address and link 365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size; 366 } 367 368 369 int LIR_Assembler::emit_exception_handler() { 370 // if the last instruction is a call (typically to do a throw which 371 // is coming at the end after block reordering) the return address 372 // must still point into the code area in order to avoid assertion 373 // failures when searching for the corresponding bci => add a nop 374 // (was bug 5/14/1999 - gri) 375 __ nop(); 376 377 // generate code for exception handler 378 address handler_base = __ start_a_stub(exception_handler_size); 379 if (handler_base == NULL) { 380 // not enough space left for the handler 381 bailout("exception handler overflow"); 382 return -1; 383 } 384 385 int offset = code_offset(); 386 387 // the exception oop and pc are in r0, and r3 388 // no other registers need to be preserved, so invalidate them 389 __ invalidate_registers(false, true, true, false, true, true); 390 391 // check that there is really an exception 392 __ verify_not_null_oop(r0); 393 394 // search an exception handler (r0: exception oop, r3: throwing pc) 395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); 396 guarantee(code_offset() - offset <= exception_handler_size, "overflow"); 397 __ end_a_stub(); 398 399 return offset; 400 } 401 402 403 // Emit the code to remove the frame from the stack in the exception 404 // unwind path. 405 int LIR_Assembler::emit_unwind_handler() { 406 #ifndef PRODUCT 407 if (CommentedAssembly) { 408 _masm->block_comment("Unwind handler"); 409 } 410 #endif 411 412 int offset = code_offset(); 413 414 // Fetch the exception from TLS and clear out exception related thread state 415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset())); 416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset())); 417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset())); 418 419 __ bind(_unwind_handler_entry); 420 __ verify_not_null_oop(r0); 421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 422 __ mov(r19, r0); // Preserve the exception 423 } 424 425 // Preform needed unlocking 426 MonitorExitStub* stub = NULL; 427 if (method()->is_synchronized()) { 428 monitor_address(0, FrameMap::r0_opr); 429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0); 430 __ unlock_object(r5, r4, r0, *stub->entry()); 431 __ bind(*stub->continuation()); 432 } 433 434 if (compilation()->env()->dtrace_method_probes()) { 435 __ mov(c_rarg0, rthread); 436 __ mov_metadata(c_rarg1, method()->constant_encoding()); 437 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1); 438 } 439 440 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 441 __ mov(r0, r19); // Restore the exception 442 } 443 444 // remove the activation and dispatch to the unwind handler 445 __ block_comment("remove_frame and dispatch to the unwind handler"); 446 __ remove_frame(initial_frame_size_in_bytes()); 447 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id))); 448 449 // Emit the slow path assembly 450 if (stub != NULL) { 451 stub->emit_code(this); 452 } 453 454 return offset; 455 } 456 457 458 int LIR_Assembler::emit_deopt_handler() { 459 // if the last instruction is a call (typically to do a throw which 460 // is coming at the end after block reordering) the return address 461 // must still point into the code area in order to avoid assertion 462 // failures when searching for the corresponding bci => add a nop 463 // (was bug 5/14/1999 - gri) 464 __ nop(); 465 466 // generate code for exception handler 467 address handler_base = __ start_a_stub(deopt_handler_size); 468 if (handler_base == NULL) { 469 // not enough space left for the handler 470 bailout("deopt handler overflow"); 471 return -1; 472 } 473 474 int offset = code_offset(); 475 476 __ adr(lr, pc()); 477 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 478 guarantee(code_offset() - offset <= deopt_handler_size, "overflow"); 479 __ end_a_stub(); 480 481 return offset; 482 } 483 484 485 // This is the fast version of java.lang.String.compare; it has not 486 // OSR-entry and therefore, we generate a slow version for OSR's 487 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) { 488 __ mov(r2, (address)__FUNCTION__); 489 __ call_Unimplemented(); 490 } 491 492 493 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) { 494 _masm->code_section()->relocate(adr, relocInfo::poll_type); 495 int pc_offset = code_offset(); 496 flush_debug_info(pc_offset); 497 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset); 498 if (info->exception_handlers() != NULL) { 499 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers()); 500 } 501 } 502 503 // Rather than take a segfault when the polling page is protected, 504 // explicitly check for a safepoint in progress and if there is one, 505 // fake a call to the handler as if a segfault had been caught. 506 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) { 507 __ mov(rscratch1, SafepointSynchronize::address_of_state()); 508 __ ldrb(rscratch1, Address(rscratch1)); 509 Label nope, poll; 510 __ cbz(rscratch1, nope); 511 __ block_comment("safepoint"); 512 __ enter(); 513 __ push(0x3, sp); // r0 & r1 514 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1 515 __ adr(r0, poll); 516 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset())); 517 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub)); 518 __ blr(rscratch1); 519 __ maybe_isb(); 520 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1 521 __ mov(rscratch1, r0); 522 __ pop(0x3, sp); // r0 & r1 523 __ leave(); 524 __ br(rscratch1); 525 address polling_page(os::get_polling_page()); 526 assert(os::is_poll_address(polling_page), "should be"); 527 unsigned long off; 528 __ adrp(rscratch1, Address(polling_page, rtype), off); 529 __ bind(poll); 530 if (info) 531 add_debug_info_for_branch(info); // This isn't just debug info: 532 // it's the oop map 533 else 534 __ code_section()->relocate(pc(), rtype); 535 __ ldrw(zr, Address(rscratch1, off)); 536 __ bind(nope); 537 } 538 539 void LIR_Assembler::return_op(LIR_Opr result) { 540 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,"); 541 // Pop the stack before the safepoint code 542 __ remove_frame(initial_frame_size_in_bytes()); 543 if (UseCompilerSafepoints) { 544 address polling_page(os::get_polling_page()); 545 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type); 546 } else { 547 poll_for_safepoint(relocInfo::poll_return_type); 548 } 549 __ ret(lr); 550 } 551 552 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 553 address polling_page(os::get_polling_page()); 554 if (UseCompilerSafepoints) { 555 guarantee(info != NULL, "Shouldn't be NULL"); 556 assert(os::is_poll_address(polling_page), "should be"); 557 unsigned long off; 558 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off); 559 assert(off == 0, "must be"); 560 add_debug_info_for_branch(info); // This isn't just debug info: 561 // it's the oop map 562 __ read_polling_page(rscratch1, relocInfo::poll_type); 563 } else { 564 poll_for_safepoint(relocInfo::poll_type, info); 565 } 566 567 return __ offset(); 568 } 569 570 571 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 572 if (from_reg == r31_sp) 573 from_reg = sp; 574 if (to_reg == r31_sp) 575 to_reg = sp; 576 __ mov(to_reg, from_reg); 577 } 578 579 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); } 580 581 582 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 583 assert(src->is_constant(), "should not call otherwise"); 584 assert(dest->is_register(), "should not call otherwise"); 585 LIR_Const* c = src->as_constant_ptr(); 586 587 switch (c->type()) { 588 case T_INT: { 589 assert(patch_code == lir_patch_none, "no patching handled here"); 590 __ movw(dest->as_register(), c->as_jint()); 591 break; 592 } 593 594 case T_ADDRESS: { 595 assert(patch_code == lir_patch_none, "no patching handled here"); 596 __ mov(dest->as_register(), c->as_jint()); 597 break; 598 } 599 600 case T_LONG: { 601 assert(patch_code == lir_patch_none, "no patching handled here"); 602 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong()); 603 break; 604 } 605 606 case T_OBJECT: { 607 if (patch_code == lir_patch_none) { 608 jobject2reg(c->as_jobject(), dest->as_register()); 609 } else { 610 jobject2reg_with_patching(dest->as_register(), info); 611 } 612 break; 613 } 614 615 case T_METADATA: { 616 if (patch_code != lir_patch_none) { 617 klass2reg_with_patching(dest->as_register(), info); 618 } else { 619 __ mov_metadata(dest->as_register(), c->as_metadata()); 620 } 621 break; 622 } 623 624 case T_FLOAT: { 625 if (__ operand_valid_for_float_immediate(c->as_jfloat())) { 626 __ fmovs(dest->as_float_reg(), (c->as_jfloat())); 627 } else { 628 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat()))); 629 __ ldrs(dest->as_float_reg(), Address(rscratch1)); 630 } 631 break; 632 } 633 634 case T_DOUBLE: { 635 if (__ operand_valid_for_float_immediate(c->as_jdouble())) { 636 __ fmovd(dest->as_double_reg(), (c->as_jdouble())); 637 } else { 638 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble()))); 639 __ ldrd(dest->as_double_reg(), Address(rscratch1)); 640 } 641 break; 642 } 643 644 default: 645 ShouldNotReachHere(); 646 } 647 } 648 649 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 650 LIR_Const* c = src->as_constant_ptr(); 651 switch (c->type()) { 652 case T_OBJECT: 653 { 654 if (! c->as_jobject()) 655 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 656 else { 657 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL); 658 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 659 } 660 } 661 break; 662 case T_ADDRESS: 663 { 664 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL); 665 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 666 } 667 case T_INT: 668 case T_FLOAT: 669 { 670 Register reg = zr; 671 if (c->as_jint_bits() == 0) 672 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 673 else { 674 __ movw(rscratch1, c->as_jint_bits()); 675 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix())); 676 } 677 } 678 break; 679 case T_LONG: 680 case T_DOUBLE: 681 { 682 Register reg = zr; 683 if (c->as_jlong_bits() == 0) 684 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(), 685 lo_word_offset_in_bytes)); 686 else { 687 __ mov(rscratch1, (intptr_t)c->as_jlong_bits()); 688 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(), 689 lo_word_offset_in_bytes)); 690 } 691 } 692 break; 693 default: 694 ShouldNotReachHere(); 695 } 696 } 697 698 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 699 assert(src->is_constant(), "should not call otherwise"); 700 LIR_Const* c = src->as_constant_ptr(); 701 LIR_Address* to_addr = dest->as_address_ptr(); 702 703 void (Assembler::* insn)(Register Rt, const Address &adr); 704 705 switch (type) { 706 case T_ADDRESS: 707 assert(c->as_jint() == 0, "should be"); 708 insn = &Assembler::str; 709 break; 710 case T_LONG: 711 assert(c->as_jlong() == 0, "should be"); 712 insn = &Assembler::str; 713 break; 714 case T_INT: 715 assert(c->as_jint() == 0, "should be"); 716 insn = &Assembler::strw; 717 break; 718 case T_OBJECT: 719 case T_ARRAY: 720 assert(c->as_jobject() == 0, "should be"); 721 if (UseCompressedOops && !wide) { 722 insn = &Assembler::strw; 723 } else { 724 insn = &Assembler::str; 725 } 726 break; 727 case T_CHAR: 728 case T_SHORT: 729 assert(c->as_jint() == 0, "should be"); 730 insn = &Assembler::strh; 731 break; 732 case T_BOOLEAN: 733 case T_BYTE: 734 assert(c->as_jint() == 0, "should be"); 735 insn = &Assembler::strb; 736 break; 737 default: 738 ShouldNotReachHere(); 739 } 740 741 if (info) add_debug_info_for_null_check_here(info); 742 (_masm->*insn)(zr, as_Address(to_addr, rscratch1)); 743 } 744 745 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 746 assert(src->is_register(), "should not call otherwise"); 747 assert(dest->is_register(), "should not call otherwise"); 748 749 // move between cpu-registers 750 if (dest->is_single_cpu()) { 751 if (src->type() == T_LONG) { 752 // Can do LONG -> OBJECT 753 move_regs(src->as_register_lo(), dest->as_register()); 754 return; 755 } 756 assert(src->is_single_cpu(), "must match"); 757 if (src->type() == T_OBJECT) { 758 __ verify_oop(src->as_register()); 759 } 760 move_regs(src->as_register(), dest->as_register()); 761 762 } else if (dest->is_double_cpu()) { 763 if (src->type() == T_OBJECT || src->type() == T_ARRAY) { 764 // Surprising to me but we can see move of a long to t_object 765 __ verify_oop(src->as_register()); 766 move_regs(src->as_register(), dest->as_register_lo()); 767 return; 768 } 769 assert(src->is_double_cpu(), "must match"); 770 Register f_lo = src->as_register_lo(); 771 Register f_hi = src->as_register_hi(); 772 Register t_lo = dest->as_register_lo(); 773 Register t_hi = dest->as_register_hi(); 774 assert(f_hi == f_lo, "must be same"); 775 assert(t_hi == t_lo, "must be same"); 776 move_regs(f_lo, t_lo); 777 778 } else if (dest->is_single_fpu()) { 779 __ fmovs(dest->as_float_reg(), src->as_float_reg()); 780 781 } else if (dest->is_double_fpu()) { 782 __ fmovd(dest->as_double_reg(), src->as_double_reg()); 783 784 } else { 785 ShouldNotReachHere(); 786 } 787 } 788 789 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 790 if (src->is_single_cpu()) { 791 if (type == T_ARRAY || type == T_OBJECT) { 792 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 793 __ verify_oop(src->as_register()); 794 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) { 795 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 796 } else { 797 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix())); 798 } 799 800 } else if (src->is_double_cpu()) { 801 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes); 802 __ str(src->as_register_lo(), dest_addr_LO); 803 804 } else if (src->is_single_fpu()) { 805 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix()); 806 __ strs(src->as_float_reg(), dest_addr); 807 808 } else if (src->is_double_fpu()) { 809 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix()); 810 __ strd(src->as_double_reg(), dest_addr); 811 812 } else { 813 ShouldNotReachHere(); 814 } 815 816 } 817 818 819 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) { 820 LIR_Address* to_addr = dest->as_address_ptr(); 821 PatchingStub* patch = NULL; 822 Register compressed_src = rscratch1; 823 824 if (patch_code != lir_patch_none) { 825 deoptimize_trap(info); 826 return; 827 } 828 829 if (type == T_ARRAY || type == T_OBJECT) { 830 __ verify_oop(src->as_register()); 831 832 if (UseCompressedOops && !wide) { 833 __ encode_heap_oop(compressed_src, src->as_register()); 834 } else { 835 compressed_src = src->as_register(); 836 } 837 } 838 839 int null_check_here = code_offset(); 840 switch (type) { 841 case T_FLOAT: { 842 __ strs(src->as_float_reg(), as_Address(to_addr)); 843 break; 844 } 845 846 case T_DOUBLE: { 847 __ strd(src->as_double_reg(), as_Address(to_addr)); 848 break; 849 } 850 851 case T_ARRAY: // fall through 852 case T_OBJECT: // fall through 853 if (UseCompressedOops && !wide) { 854 __ strw(compressed_src, as_Address(to_addr, rscratch2)); 855 } else { 856 __ str(compressed_src, as_Address(to_addr)); 857 } 858 break; 859 case T_METADATA: 860 // We get here to store a method pointer to the stack to pass to 861 // a dtrace runtime call. This can't work on 64 bit with 862 // compressed klass ptrs: T_METADATA can be a compressed klass 863 // ptr or a 64 bit method pointer. 864 LP64_ONLY(ShouldNotReachHere()); 865 __ str(src->as_register(), as_Address(to_addr)); 866 break; 867 case T_ADDRESS: 868 __ str(src->as_register(), as_Address(to_addr)); 869 break; 870 case T_INT: 871 __ strw(src->as_register(), as_Address(to_addr)); 872 break; 873 874 case T_LONG: { 875 __ str(src->as_register_lo(), as_Address_lo(to_addr)); 876 break; 877 } 878 879 case T_BYTE: // fall through 880 case T_BOOLEAN: { 881 __ strb(src->as_register(), as_Address(to_addr)); 882 break; 883 } 884 885 case T_CHAR: // fall through 886 case T_SHORT: 887 __ strh(src->as_register(), as_Address(to_addr)); 888 break; 889 890 default: 891 ShouldNotReachHere(); 892 } 893 if (info != NULL) { 894 add_debug_info_for_null_check(null_check_here, info); 895 } 896 } 897 898 899 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 900 assert(src->is_stack(), "should not call otherwise"); 901 assert(dest->is_register(), "should not call otherwise"); 902 903 if (dest->is_single_cpu()) { 904 if (type == T_ARRAY || type == T_OBJECT) { 905 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 906 __ verify_oop(dest->as_register()); 907 } else if (type == T_METADATA || type == T_ADDRESS) { 908 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 909 } else { 910 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 911 } 912 913 } else if (dest->is_double_cpu()) { 914 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes); 915 __ ldr(dest->as_register_lo(), src_addr_LO); 916 917 } else if (dest->is_single_fpu()) { 918 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix()); 919 __ ldrs(dest->as_float_reg(), src_addr); 920 921 } else if (dest->is_double_fpu()) { 922 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix()); 923 __ ldrd(dest->as_double_reg(), src_addr); 924 925 } else { 926 ShouldNotReachHere(); 927 } 928 } 929 930 931 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 932 address target = NULL; 933 relocInfo::relocType reloc_type = relocInfo::none; 934 935 switch (patching_id(info)) { 936 case PatchingStub::access_field_id: 937 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 938 reloc_type = relocInfo::section_word_type; 939 break; 940 case PatchingStub::load_klass_id: 941 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 942 reloc_type = relocInfo::metadata_type; 943 break; 944 case PatchingStub::load_mirror_id: 945 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 946 reloc_type = relocInfo::oop_type; 947 break; 948 case PatchingStub::load_appendix_id: 949 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 950 reloc_type = relocInfo::oop_type; 951 break; 952 default: ShouldNotReachHere(); 953 } 954 955 __ far_call(RuntimeAddress(target)); 956 add_call_info_here(info); 957 } 958 959 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 960 961 LIR_Opr temp; 962 if (type == T_LONG || type == T_DOUBLE) 963 temp = FrameMap::rscratch1_long_opr; 964 else 965 temp = FrameMap::rscratch1_opr; 966 967 stack2reg(src, temp, src->type()); 968 reg2stack(temp, dest, dest->type(), false); 969 } 970 971 972 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) { 973 LIR_Address* addr = src->as_address_ptr(); 974 LIR_Address* from_addr = src->as_address_ptr(); 975 976 if (addr->base()->type() == T_OBJECT) { 977 __ verify_oop(addr->base()->as_pointer_register()); 978 } 979 980 if (patch_code != lir_patch_none) { 981 deoptimize_trap(info); 982 return; 983 } 984 985 if (info != NULL) { 986 add_debug_info_for_null_check_here(info); 987 } 988 int null_check_here = code_offset(); 989 switch (type) { 990 case T_FLOAT: { 991 __ ldrs(dest->as_float_reg(), as_Address(from_addr)); 992 break; 993 } 994 995 case T_DOUBLE: { 996 __ ldrd(dest->as_double_reg(), as_Address(from_addr)); 997 break; 998 } 999 1000 case T_ARRAY: // fall through 1001 case T_OBJECT: // fall through 1002 if (UseCompressedOops && !wide) { 1003 __ ldrw(dest->as_register(), as_Address(from_addr)); 1004 } else { 1005 __ ldr(dest->as_register(), as_Address(from_addr)); 1006 } 1007 break; 1008 case T_METADATA: 1009 // We get here to store a method pointer to the stack to pass to 1010 // a dtrace runtime call. This can't work on 64 bit with 1011 // compressed klass ptrs: T_METADATA can be a compressed klass 1012 // ptr or a 64 bit method pointer. 1013 LP64_ONLY(ShouldNotReachHere()); 1014 __ ldr(dest->as_register(), as_Address(from_addr)); 1015 break; 1016 case T_ADDRESS: 1017 // FIXME: OMG this is a horrible kludge. Any offset from an 1018 // address that matches klass_offset_in_bytes() will be loaded 1019 // as a word, not a long. 1020 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1021 __ ldrw(dest->as_register(), as_Address(from_addr)); 1022 } else { 1023 __ ldr(dest->as_register(), as_Address(from_addr)); 1024 } 1025 break; 1026 case T_INT: 1027 __ ldrw(dest->as_register(), as_Address(from_addr)); 1028 break; 1029 1030 case T_LONG: { 1031 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr)); 1032 break; 1033 } 1034 1035 case T_BYTE: 1036 __ ldrsb(dest->as_register(), as_Address(from_addr)); 1037 break; 1038 case T_BOOLEAN: { 1039 __ ldrb(dest->as_register(), as_Address(from_addr)); 1040 break; 1041 } 1042 1043 case T_CHAR: 1044 __ ldrh(dest->as_register(), as_Address(from_addr)); 1045 break; 1046 case T_SHORT: 1047 __ ldrsh(dest->as_register(), as_Address(from_addr)); 1048 break; 1049 1050 default: 1051 ShouldNotReachHere(); 1052 } 1053 1054 if (type == T_ARRAY || type == T_OBJECT) { 1055 #ifdef _LP64 1056 if (UseCompressedOops && !wide) { 1057 __ decode_heap_oop(dest->as_register()); 1058 } 1059 #endif 1060 __ verify_oop(dest->as_register()); 1061 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) { 1062 #ifdef _LP64 1063 if (UseCompressedClassPointers) { 1064 __ decode_klass_not_null(dest->as_register()); 1065 } 1066 #endif 1067 } 1068 } 1069 1070 1071 void LIR_Assembler::prefetchr(LIR_Opr src) { Unimplemented(); } 1072 1073 1074 void LIR_Assembler::prefetchw(LIR_Opr src) { Unimplemented(); } 1075 1076 1077 int LIR_Assembler::array_element_size(BasicType type) const { 1078 int elem_size = type2aelembytes(type); 1079 return exact_log2(elem_size); 1080 } 1081 1082 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1083 Register Rdividend = op->in_opr1()->as_register(); 1084 Register Rdivisor = op->in_opr2()->as_register(); 1085 Register Rscratch = op->in_opr3()->as_register(); 1086 Register Rresult = op->result_opr()->as_register(); 1087 int divisor = -1; 1088 1089 /* 1090 TODO: For some reason, using the Rscratch that gets passed in is 1091 not possible because the register allocator does not see the tmp reg 1092 as used, and assignes it the same register as Rdividend. We use rscratch1 1093 instead. 1094 1095 assert(Rdividend != Rscratch, ""); 1096 assert(Rdivisor != Rscratch, ""); 1097 */ 1098 1099 if (Rdivisor == noreg && is_power_of_2(divisor)) { 1100 // convert division by a power of two into some shifts and logical operations 1101 } 1102 1103 if (op->code() == lir_irem) { 1104 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1); 1105 } else if (op->code() == lir_idiv) { 1106 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1); 1107 } else 1108 ShouldNotReachHere(); 1109 } 1110 1111 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1112 #ifdef ASSERT 1113 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label"); 1114 if (op->block() != NULL) _branch_target_blocks.append(op->block()); 1115 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock()); 1116 #endif 1117 1118 if (op->cond() == lir_cond_always) { 1119 if (op->info() != NULL) add_debug_info_for_branch(op->info()); 1120 __ b(*(op->label())); 1121 } else { 1122 Assembler::Condition acond; 1123 if (op->code() == lir_cond_float_branch) { 1124 bool is_unordered = (op->ublock() == op->block()); 1125 // Assembler::EQ does not permit unordered branches, so we add 1126 // another branch here. Likewise, Assembler::NE does not permit 1127 // ordered branches. 1128 if ((is_unordered && op->cond() == lir_cond_equal) 1129 || (!is_unordered && op->cond() == lir_cond_notEqual)) 1130 __ br(Assembler::VS, *(op->ublock()->label())); 1131 switch(op->cond()) { 1132 case lir_cond_equal: acond = Assembler::EQ; break; 1133 case lir_cond_notEqual: acond = Assembler::NE; break; 1134 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break; 1135 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break; 1136 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break; 1137 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break; 1138 default: ShouldNotReachHere(); 1139 } 1140 } else { 1141 switch (op->cond()) { 1142 case lir_cond_equal: acond = Assembler::EQ; break; 1143 case lir_cond_notEqual: acond = Assembler::NE; break; 1144 case lir_cond_less: acond = Assembler::LT; break; 1145 case lir_cond_lessEqual: acond = Assembler::LE; break; 1146 case lir_cond_greaterEqual: acond = Assembler::GE; break; 1147 case lir_cond_greater: acond = Assembler::GT; break; 1148 case lir_cond_belowEqual: acond = Assembler::LS; break; 1149 case lir_cond_aboveEqual: acond = Assembler::HS; break; 1150 default: ShouldNotReachHere(); 1151 } 1152 } 1153 __ br(acond,*(op->label())); 1154 } 1155 } 1156 1157 1158 1159 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1160 LIR_Opr src = op->in_opr(); 1161 LIR_Opr dest = op->result_opr(); 1162 1163 switch (op->bytecode()) { 1164 case Bytecodes::_i2f: 1165 { 1166 __ scvtfws(dest->as_float_reg(), src->as_register()); 1167 break; 1168 } 1169 case Bytecodes::_i2d: 1170 { 1171 __ scvtfwd(dest->as_double_reg(), src->as_register()); 1172 break; 1173 } 1174 case Bytecodes::_l2d: 1175 { 1176 __ scvtfd(dest->as_double_reg(), src->as_register_lo()); 1177 break; 1178 } 1179 case Bytecodes::_l2f: 1180 { 1181 __ scvtfs(dest->as_float_reg(), src->as_register_lo()); 1182 break; 1183 } 1184 case Bytecodes::_f2d: 1185 { 1186 __ fcvts(dest->as_double_reg(), src->as_float_reg()); 1187 break; 1188 } 1189 case Bytecodes::_d2f: 1190 { 1191 __ fcvtd(dest->as_float_reg(), src->as_double_reg()); 1192 break; 1193 } 1194 case Bytecodes::_i2c: 1195 { 1196 __ ubfx(dest->as_register(), src->as_register(), 0, 16); 1197 break; 1198 } 1199 case Bytecodes::_i2l: 1200 { 1201 __ sxtw(dest->as_register_lo(), src->as_register()); 1202 break; 1203 } 1204 case Bytecodes::_i2s: 1205 { 1206 __ sxth(dest->as_register(), src->as_register()); 1207 break; 1208 } 1209 case Bytecodes::_i2b: 1210 { 1211 __ sxtb(dest->as_register(), src->as_register()); 1212 break; 1213 } 1214 case Bytecodes::_l2i: 1215 { 1216 _masm->block_comment("FIXME: This could be a no-op"); 1217 __ uxtw(dest->as_register(), src->as_register_lo()); 1218 break; 1219 } 1220 case Bytecodes::_d2l: 1221 { 1222 Register tmp = op->tmp1()->as_register(); 1223 __ clear_fpsr(); 1224 __ fcvtzd(dest->as_register_lo(), src->as_double_reg()); 1225 __ get_fpsr(tmp); 1226 __ tst(tmp, 1); // FPSCR.IOC 1227 __ br(Assembler::NE, *(op->stub()->entry())); 1228 __ bind(*op->stub()->continuation()); 1229 break; 1230 } 1231 case Bytecodes::_f2i: 1232 { 1233 Register tmp = op->tmp1()->as_register(); 1234 __ clear_fpsr(); 1235 __ fcvtzsw(dest->as_register(), src->as_float_reg()); 1236 __ get_fpsr(tmp); 1237 __ tst(tmp, 1); // FPSCR.IOC 1238 __ br(Assembler::NE, *(op->stub()->entry())); 1239 __ bind(*op->stub()->continuation()); 1240 break; 1241 } 1242 case Bytecodes::_f2l: 1243 { 1244 Register tmp = op->tmp1()->as_register(); 1245 __ clear_fpsr(); 1246 __ fcvtzs(dest->as_register_lo(), src->as_float_reg()); 1247 __ get_fpsr(tmp); 1248 __ tst(tmp, 1); // FPSCR.IOC 1249 __ br(Assembler::NE, *(op->stub()->entry())); 1250 __ bind(*op->stub()->continuation()); 1251 break; 1252 } 1253 case Bytecodes::_d2i: 1254 { 1255 Register tmp = op->tmp1()->as_register(); 1256 __ clear_fpsr(); 1257 __ fcvtzdw(dest->as_register(), src->as_double_reg()); 1258 __ get_fpsr(tmp); 1259 __ tst(tmp, 1); // FPSCR.IOC 1260 __ br(Assembler::NE, *(op->stub()->entry())); 1261 __ bind(*op->stub()->continuation()); 1262 break; 1263 } 1264 default: ShouldNotReachHere(); 1265 } 1266 } 1267 1268 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1269 if (op->init_check()) { 1270 __ ldrb(rscratch1, Address(op->klass()->as_register(), 1271 InstanceKlass::init_state_offset())); 1272 __ cmpw(rscratch1, InstanceKlass::fully_initialized); 1273 add_debug_info_for_null_check_here(op->stub()->info()); 1274 __ br(Assembler::NE, *op->stub()->entry()); 1275 } 1276 __ allocate_object(op->obj()->as_register(), 1277 op->tmp1()->as_register(), 1278 op->tmp2()->as_register(), 1279 op->header_size(), 1280 op->object_size(), 1281 op->klass()->as_register(), 1282 *op->stub()->entry()); 1283 __ bind(*op->stub()->continuation()); 1284 } 1285 1286 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1287 Register len = op->len()->as_register(); 1288 __ uxtw(len, len); 1289 1290 if (UseSlowPath || 1291 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) || 1292 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) { 1293 __ b(*op->stub()->entry()); 1294 } else { 1295 Register tmp1 = op->tmp1()->as_register(); 1296 Register tmp2 = op->tmp2()->as_register(); 1297 Register tmp3 = op->tmp3()->as_register(); 1298 if (len == tmp1) { 1299 tmp1 = tmp3; 1300 } else if (len == tmp2) { 1301 tmp2 = tmp3; 1302 } else if (len == tmp3) { 1303 // everything is ok 1304 } else { 1305 __ mov(tmp3, len); 1306 } 1307 __ allocate_array(op->obj()->as_register(), 1308 len, 1309 tmp1, 1310 tmp2, 1311 arrayOopDesc::header_size(op->type()), 1312 array_element_size(op->type()), 1313 op->klass()->as_register(), 1314 *op->stub()->entry()); 1315 } 1316 __ bind(*op->stub()->continuation()); 1317 } 1318 1319 void LIR_Assembler::type_profile_helper(Register mdo, 1320 ciMethodData *md, ciProfileData *data, 1321 Register recv, Label* update_done) { 1322 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1323 Label next_test; 1324 // See if the receiver is receiver[n]. 1325 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1326 __ ldr(rscratch1, Address(rscratch2)); 1327 __ cmp(recv, rscratch1); 1328 __ br(Assembler::NE, next_test); 1329 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); 1330 __ addptr(data_addr, DataLayout::counter_increment); 1331 __ b(*update_done); 1332 __ bind(next_test); 1333 } 1334 1335 // Didn't find receiver; find next empty slot and fill it in 1336 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1337 Label next_test; 1338 __ lea(rscratch2, 1339 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1340 Address recv_addr(rscratch2); 1341 __ ldr(rscratch1, recv_addr); 1342 __ cbnz(rscratch1, next_test); 1343 __ str(recv, recv_addr); 1344 __ mov(rscratch1, DataLayout::counter_increment); 1345 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)))); 1346 __ str(rscratch1, Address(rscratch2)); 1347 __ b(*update_done); 1348 __ bind(next_test); 1349 } 1350 } 1351 1352 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1353 // we always need a stub for the failure case. 1354 CodeStub* stub = op->stub(); 1355 Register obj = op->object()->as_register(); 1356 Register k_RInfo = op->tmp1()->as_register(); 1357 Register klass_RInfo = op->tmp2()->as_register(); 1358 Register dst = op->result_opr()->as_register(); 1359 ciKlass* k = op->klass(); 1360 Register Rtmp1 = noreg; 1361 1362 // check if it needs to be profiled 1363 ciMethodData* md; 1364 ciProfileData* data; 1365 1366 if (op->should_profile()) { 1367 ciMethod* method = op->profiled_method(); 1368 assert(method != NULL, "Should have method"); 1369 int bci = op->profiled_bci(); 1370 md = method->method_data_or_null(); 1371 assert(md != NULL, "Sanity"); 1372 data = md->bci_to_data(bci); 1373 assert(data != NULL, "need data for type check"); 1374 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1375 } 1376 Label profile_cast_success, profile_cast_failure; 1377 Label *success_target = op->should_profile() ? &profile_cast_success : success; 1378 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure; 1379 1380 if (obj == k_RInfo) { 1381 k_RInfo = dst; 1382 } else if (obj == klass_RInfo) { 1383 klass_RInfo = dst; 1384 } 1385 if (k->is_loaded() && !UseCompressedClassPointers) { 1386 select_different_registers(obj, dst, k_RInfo, klass_RInfo); 1387 } else { 1388 Rtmp1 = op->tmp3()->as_register(); 1389 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1390 } 1391 1392 assert_different_registers(obj, k_RInfo, klass_RInfo); 1393 1394 if (op->should_profile()) { 1395 Label not_null; 1396 __ cbnz(obj, not_null); 1397 // Object is null; update MDO and exit 1398 Register mdo = klass_RInfo; 1399 __ mov_metadata(mdo, md->constant_encoding()); 1400 Address data_addr 1401 = __ form_address(rscratch2, mdo, 1402 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()), 1403 LogBytesPerWord); 1404 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant()); 1405 __ ldr(rscratch1, data_addr); 1406 __ orr(rscratch1, rscratch1, header_bits); 1407 __ str(rscratch1, data_addr); 1408 __ b(*obj_is_null); 1409 __ bind(not_null); 1410 } else { 1411 __ cbz(obj, *obj_is_null); 1412 } 1413 1414 if (!k->is_loaded()) { 1415 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1416 } else { 1417 #ifdef _LP64 1418 __ mov_metadata(k_RInfo, k->constant_encoding()); 1419 #endif // _LP64 1420 } 1421 __ verify_oop(obj); 1422 1423 if (op->fast_check()) { 1424 // get object class 1425 // not a safepoint as obj null check happens earlier 1426 __ load_klass(rscratch1, obj); 1427 __ cmp( rscratch1, k_RInfo); 1428 1429 __ br(Assembler::NE, *failure_target); 1430 // successful cast, fall through to profile or jump 1431 } else { 1432 // get object class 1433 // not a safepoint as obj null check happens earlier 1434 __ load_klass(klass_RInfo, obj); 1435 if (k->is_loaded()) { 1436 // See if we get an immediate positive hit 1437 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset()))); 1438 __ cmp(k_RInfo, rscratch1); 1439 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1440 __ br(Assembler::NE, *failure_target); 1441 // successful cast, fall through to profile or jump 1442 } else { 1443 // See if we get an immediate positive hit 1444 __ br(Assembler::EQ, *success_target); 1445 // check for self 1446 __ cmp(klass_RInfo, k_RInfo); 1447 __ br(Assembler::EQ, *success_target); 1448 1449 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1450 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1451 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1452 // result is a boolean 1453 __ cbzw(klass_RInfo, *failure_target); 1454 // successful cast, fall through to profile or jump 1455 } 1456 } else { 1457 // perform the fast part of the checking logic 1458 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1459 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1460 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1461 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1462 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1463 // result is a boolean 1464 __ cbz(k_RInfo, *failure_target); 1465 // successful cast, fall through to profile or jump 1466 } 1467 } 1468 if (op->should_profile()) { 1469 Register mdo = klass_RInfo, recv = k_RInfo; 1470 __ bind(profile_cast_success); 1471 __ mov_metadata(mdo, md->constant_encoding()); 1472 __ load_klass(recv, obj); 1473 Label update_done; 1474 type_profile_helper(mdo, md, data, recv, success); 1475 __ b(*success); 1476 1477 __ bind(profile_cast_failure); 1478 __ mov_metadata(mdo, md->constant_encoding()); 1479 Address counter_addr 1480 = __ form_address(rscratch2, mdo, 1481 md->byte_offset_of_slot(data, CounterData::count_offset()), 1482 LogBytesPerWord); 1483 __ ldr(rscratch1, counter_addr); 1484 __ sub(rscratch1, rscratch1, DataLayout::counter_increment); 1485 __ str(rscratch1, counter_addr); 1486 __ b(*failure); 1487 } 1488 __ b(*success); 1489 } 1490 1491 1492 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1493 LIR_Code code = op->code(); 1494 if (code == lir_store_check) { 1495 Register value = op->object()->as_register(); 1496 Register array = op->array()->as_register(); 1497 Register k_RInfo = op->tmp1()->as_register(); 1498 Register klass_RInfo = op->tmp2()->as_register(); 1499 Register Rtmp1 = op->tmp3()->as_register(); 1500 1501 CodeStub* stub = op->stub(); 1502 1503 // check if it needs to be profiled 1504 ciMethodData* md; 1505 ciProfileData* data; 1506 1507 if (op->should_profile()) { 1508 ciMethod* method = op->profiled_method(); 1509 assert(method != NULL, "Should have method"); 1510 int bci = op->profiled_bci(); 1511 md = method->method_data_or_null(); 1512 assert(md != NULL, "Sanity"); 1513 data = md->bci_to_data(bci); 1514 assert(data != NULL, "need data for type check"); 1515 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1516 } 1517 Label profile_cast_success, profile_cast_failure, done; 1518 Label *success_target = op->should_profile() ? &profile_cast_success : &done; 1519 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry(); 1520 1521 if (op->should_profile()) { 1522 Label not_null; 1523 __ cbnz(value, not_null); 1524 // Object is null; update MDO and exit 1525 Register mdo = klass_RInfo; 1526 __ mov_metadata(mdo, md->constant_encoding()); 1527 Address data_addr 1528 = __ form_address(rscratch2, mdo, 1529 md->byte_offset_of_slot(data, DataLayout::header_offset()), 1530 LogBytesPerInt); 1531 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant()); 1532 __ ldrw(rscratch1, data_addr); 1533 __ orrw(rscratch1, rscratch1, header_bits); 1534 __ strw(rscratch1, data_addr); 1535 __ b(done); 1536 __ bind(not_null); 1537 } else { 1538 __ cbz(value, done); 1539 } 1540 1541 add_debug_info_for_null_check_here(op->info_for_exception()); 1542 __ load_klass(k_RInfo, array); 1543 __ load_klass(klass_RInfo, value); 1544 1545 // get instance klass (it's already uncompressed) 1546 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1547 // perform the fast part of the checking logic 1548 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1549 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1550 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1551 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1552 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1553 // result is a boolean 1554 __ cbzw(k_RInfo, *failure_target); 1555 // fall through to the success case 1556 1557 if (op->should_profile()) { 1558 Register mdo = klass_RInfo, recv = k_RInfo; 1559 __ bind(profile_cast_success); 1560 __ mov_metadata(mdo, md->constant_encoding()); 1561 __ load_klass(recv, value); 1562 Label update_done; 1563 type_profile_helper(mdo, md, data, recv, &done); 1564 __ b(done); 1565 1566 __ bind(profile_cast_failure); 1567 __ mov_metadata(mdo, md->constant_encoding()); 1568 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1569 __ lea(rscratch2, counter_addr); 1570 __ ldr(rscratch1, Address(rscratch2)); 1571 __ sub(rscratch1, rscratch1, DataLayout::counter_increment); 1572 __ str(rscratch1, Address(rscratch2)); 1573 __ b(*stub->entry()); 1574 } 1575 1576 __ bind(done); 1577 } else if (code == lir_checkcast) { 1578 Register obj = op->object()->as_register(); 1579 Register dst = op->result_opr()->as_register(); 1580 Label success; 1581 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1582 __ bind(success); 1583 if (dst != obj) { 1584 __ mov(dst, obj); 1585 } 1586 } else if (code == lir_instanceof) { 1587 Register obj = op->object()->as_register(); 1588 Register dst = op->result_opr()->as_register(); 1589 Label success, failure, done; 1590 emit_typecheck_helper(op, &success, &failure, &failure); 1591 __ bind(failure); 1592 __ mov(dst, zr); 1593 __ b(done); 1594 __ bind(success); 1595 __ mov(dst, 1); 1596 __ bind(done); 1597 } else { 1598 ShouldNotReachHere(); 1599 } 1600 } 1601 1602 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { 1603 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, rscratch1); 1604 __ cset(rscratch1, Assembler::NE); 1605 __ membar(__ AnyAny); 1606 } 1607 1608 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { 1609 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, rscratch1); 1610 __ cset(rscratch1, Assembler::NE); 1611 __ membar(__ AnyAny); 1612 } 1613 1614 1615 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1616 assert(VM_Version::supports_cx8(), "wrong machine"); 1617 Register addr = as_reg(op->addr()); 1618 Register newval = as_reg(op->new_value()); 1619 Register cmpval = as_reg(op->cmp_value()); 1620 Label succeed, fail, around; 1621 1622 if (op->code() == lir_cas_obj) { 1623 if (UseCompressedOops) { 1624 Register t1 = op->tmp1()->as_register(); 1625 assert(op->tmp1()->is_valid(), "must be"); 1626 __ encode_heap_oop(t1, cmpval); 1627 cmpval = t1; 1628 __ encode_heap_oop(rscratch2, newval); 1629 newval = rscratch2; 1630 casw(addr, newval, cmpval); 1631 } else { 1632 casl(addr, newval, cmpval); 1633 } 1634 } else if (op->code() == lir_cas_int) { 1635 casw(addr, newval, cmpval); 1636 } else { 1637 casl(addr, newval, cmpval); 1638 } 1639 } 1640 1641 1642 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) { 1643 1644 Assembler::Condition acond, ncond; 1645 switch (condition) { 1646 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break; 1647 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break; 1648 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break; 1649 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break; 1650 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break; 1651 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break; 1652 case lir_cond_belowEqual: Unimplemented(); break; 1653 case lir_cond_aboveEqual: Unimplemented(); break; 1654 default: ShouldNotReachHere(); 1655 } 1656 1657 assert(result->is_single_cpu() || result->is_double_cpu(), 1658 "expect single register for result"); 1659 if (opr1->is_constant() && opr2->is_constant() 1660 && opr1->type() == T_INT && opr2->type() == T_INT) { 1661 jint val1 = opr1->as_jint(); 1662 jint val2 = opr2->as_jint(); 1663 if (val1 == 0 && val2 == 1) { 1664 __ cset(result->as_register(), ncond); 1665 return; 1666 } else if (val1 == 1 && val2 == 0) { 1667 __ cset(result->as_register(), acond); 1668 return; 1669 } 1670 } 1671 1672 if (opr1->is_constant() && opr2->is_constant() 1673 && opr1->type() == T_LONG && opr2->type() == T_LONG) { 1674 jlong val1 = opr1->as_jlong(); 1675 jlong val2 = opr2->as_jlong(); 1676 if (val1 == 0 && val2 == 1) { 1677 __ cset(result->as_register_lo(), ncond); 1678 return; 1679 } else if (val1 == 1 && val2 == 0) { 1680 __ cset(result->as_register_lo(), acond); 1681 return; 1682 } 1683 } 1684 1685 if (opr1->is_stack()) { 1686 stack2reg(opr1, FrameMap::rscratch1_opr, result->type()); 1687 opr1 = FrameMap::rscratch1_opr; 1688 } else if (opr1->is_constant()) { 1689 LIR_Opr tmp 1690 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr; 1691 const2reg(opr1, tmp, lir_patch_none, NULL); 1692 opr1 = tmp; 1693 } 1694 1695 if (opr2->is_stack()) { 1696 stack2reg(opr2, FrameMap::rscratch2_opr, result->type()); 1697 opr2 = FrameMap::rscratch2_opr; 1698 } else if (opr2->is_constant()) { 1699 LIR_Opr tmp 1700 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr; 1701 const2reg(opr2, tmp, lir_patch_none, NULL); 1702 opr2 = tmp; 1703 } 1704 1705 if (result->type() == T_LONG) 1706 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond); 1707 else 1708 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond); 1709 } 1710 1711 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 1712 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method"); 1713 1714 if (left->is_single_cpu()) { 1715 Register lreg = left->as_register(); 1716 Register dreg = as_reg(dest); 1717 1718 if (right->is_single_cpu()) { 1719 // cpu register - cpu register 1720 1721 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT, 1722 "should be"); 1723 Register rreg = right->as_register(); 1724 switch (code) { 1725 case lir_add: __ addw (dest->as_register(), lreg, rreg); break; 1726 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break; 1727 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break; 1728 default: ShouldNotReachHere(); 1729 } 1730 1731 } else if (right->is_double_cpu()) { 1732 Register rreg = right->as_register_lo(); 1733 // single_cpu + double_cpu: can happen with obj+long 1734 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1735 switch (code) { 1736 case lir_add: __ add(dreg, lreg, rreg); break; 1737 case lir_sub: __ sub(dreg, lreg, rreg); break; 1738 default: ShouldNotReachHere(); 1739 } 1740 } else if (right->is_constant()) { 1741 // cpu register - constant 1742 jlong c; 1743 1744 // FIXME. This is fugly: we really need to factor all this logic. 1745 switch(right->type()) { 1746 case T_LONG: 1747 c = right->as_constant_ptr()->as_jlong(); 1748 break; 1749 case T_INT: 1750 case T_ADDRESS: 1751 c = right->as_constant_ptr()->as_jint(); 1752 break; 1753 default: 1754 ShouldNotReachHere(); 1755 break; 1756 } 1757 1758 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1759 if (c == 0 && dreg == lreg) { 1760 COMMENT("effective nop elided"); 1761 return; 1762 } 1763 switch(left->type()) { 1764 case T_INT: 1765 switch (code) { 1766 case lir_add: __ addw(dreg, lreg, c); break; 1767 case lir_sub: __ subw(dreg, lreg, c); break; 1768 default: ShouldNotReachHere(); 1769 } 1770 break; 1771 case T_OBJECT: 1772 case T_ADDRESS: 1773 switch (code) { 1774 case lir_add: __ add(dreg, lreg, c); break; 1775 case lir_sub: __ sub(dreg, lreg, c); break; 1776 default: ShouldNotReachHere(); 1777 } 1778 break; 1779 ShouldNotReachHere(); 1780 } 1781 } else { 1782 ShouldNotReachHere(); 1783 } 1784 1785 } else if (left->is_double_cpu()) { 1786 Register lreg_lo = left->as_register_lo(); 1787 1788 if (right->is_double_cpu()) { 1789 // cpu register - cpu register 1790 Register rreg_lo = right->as_register_lo(); 1791 switch (code) { 1792 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1793 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1794 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1795 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break; 1796 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break; 1797 default: 1798 ShouldNotReachHere(); 1799 } 1800 1801 } else if (right->is_constant()) { 1802 jlong c = right->as_constant_ptr()->as_jlong_bits(); 1803 Register dreg = as_reg(dest); 1804 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1805 if (c == 0 && dreg == lreg_lo) { 1806 COMMENT("effective nop elided"); 1807 return; 1808 } 1809 switch (code) { 1810 case lir_add: __ add(dreg, lreg_lo, c); break; 1811 case lir_sub: __ sub(dreg, lreg_lo, c); break; 1812 default: 1813 ShouldNotReachHere(); 1814 } 1815 } else { 1816 ShouldNotReachHere(); 1817 } 1818 } else if (left->is_single_fpu()) { 1819 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register"); 1820 switch (code) { 1821 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1822 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1823 case lir_mul_strictfp: // fall through 1824 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1825 case lir_div_strictfp: // fall through 1826 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1827 default: 1828 ShouldNotReachHere(); 1829 } 1830 } else if (left->is_double_fpu()) { 1831 if (right->is_double_fpu()) { 1832 // fpu register - fpu register 1833 switch (code) { 1834 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1835 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1836 case lir_mul_strictfp: // fall through 1837 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1838 case lir_div_strictfp: // fall through 1839 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1840 default: 1841 ShouldNotReachHere(); 1842 } 1843 } else { 1844 if (right->is_constant()) { 1845 ShouldNotReachHere(); 1846 } 1847 ShouldNotReachHere(); 1848 } 1849 } else if (left->is_single_stack() || left->is_address()) { 1850 assert(left == dest, "left and dest must be equal"); 1851 ShouldNotReachHere(); 1852 } else { 1853 ShouldNotReachHere(); 1854 } 1855 } 1856 1857 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); } 1858 1859 1860 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { 1861 switch(code) { 1862 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break; 1863 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break; 1864 default : ShouldNotReachHere(); 1865 } 1866 } 1867 1868 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 1869 1870 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register"); 1871 Register Rleft = left->is_single_cpu() ? left->as_register() : 1872 left->as_register_lo(); 1873 if (dst->is_single_cpu()) { 1874 Register Rdst = dst->as_register(); 1875 if (right->is_constant()) { 1876 switch (code) { 1877 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break; 1878 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break; 1879 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break; 1880 default: ShouldNotReachHere(); break; 1881 } 1882 } else { 1883 Register Rright = right->is_single_cpu() ? right->as_register() : 1884 right->as_register_lo(); 1885 switch (code) { 1886 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break; 1887 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break; 1888 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break; 1889 default: ShouldNotReachHere(); break; 1890 } 1891 } 1892 } else { 1893 Register Rdst = dst->as_register_lo(); 1894 if (right->is_constant()) { 1895 switch (code) { 1896 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break; 1897 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break; 1898 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break; 1899 default: ShouldNotReachHere(); break; 1900 } 1901 } else { 1902 Register Rright = right->is_single_cpu() ? right->as_register() : 1903 right->as_register_lo(); 1904 switch (code) { 1905 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break; 1906 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break; 1907 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break; 1908 default: ShouldNotReachHere(); break; 1909 } 1910 } 1911 } 1912 } 1913 1914 1915 1916 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); } 1917 1918 1919 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 1920 if (opr1->is_constant() && opr2->is_single_cpu()) { 1921 // tableswitch 1922 Register reg = as_reg(opr2); 1923 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()]; 1924 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after); 1925 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) { 1926 Register reg1 = as_reg(opr1); 1927 if (opr2->is_single_cpu()) { 1928 // cpu register - cpu register 1929 Register reg2 = opr2->as_register(); 1930 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) { 1931 __ cmp(reg1, reg2); 1932 } else { 1933 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?"); 1934 __ cmpw(reg1, reg2); 1935 } 1936 return; 1937 } 1938 if (opr2->is_double_cpu()) { 1939 // cpu register - cpu register 1940 Register reg2 = opr2->as_register_lo(); 1941 __ cmp(reg1, reg2); 1942 return; 1943 } 1944 1945 if (opr2->is_constant()) { 1946 bool is_32bit = false; // width of register operand 1947 jlong imm; 1948 1949 switch(opr2->type()) { 1950 case T_INT: 1951 imm = opr2->as_constant_ptr()->as_jint(); 1952 is_32bit = true; 1953 break; 1954 case T_LONG: 1955 imm = opr2->as_constant_ptr()->as_jlong(); 1956 break; 1957 case T_ADDRESS: 1958 imm = opr2->as_constant_ptr()->as_jint(); 1959 break; 1960 case T_OBJECT: 1961 case T_ARRAY: 1962 imm = jlong(opr2->as_constant_ptr()->as_jobject()); 1963 break; 1964 default: 1965 ShouldNotReachHere(); 1966 break; 1967 } 1968 1969 if (Assembler::operand_valid_for_add_sub_immediate(imm)) { 1970 if (is_32bit) 1971 __ cmpw(reg1, imm); 1972 else 1973 __ cmp(reg1, imm); 1974 return; 1975 } else { 1976 __ mov(rscratch1, imm); 1977 if (is_32bit) 1978 __ cmpw(reg1, rscratch1); 1979 else 1980 __ cmp(reg1, rscratch1); 1981 return; 1982 } 1983 } else 1984 ShouldNotReachHere(); 1985 } else if (opr1->is_single_fpu()) { 1986 FloatRegister reg1 = opr1->as_float_reg(); 1987 assert(opr2->is_single_fpu(), "expect single float register"); 1988 FloatRegister reg2 = opr2->as_float_reg(); 1989 __ fcmps(reg1, reg2); 1990 } else if (opr1->is_double_fpu()) { 1991 FloatRegister reg1 = opr1->as_double_reg(); 1992 assert(opr2->is_double_fpu(), "expect double float register"); 1993 FloatRegister reg2 = opr2->as_double_reg(); 1994 __ fcmpd(reg1, reg2); 1995 } else { 1996 ShouldNotReachHere(); 1997 } 1998 } 1999 2000 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){ 2001 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 2002 bool is_unordered_less = (code == lir_ucmp_fd2i); 2003 if (left->is_single_fpu()) { 2004 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register()); 2005 } else if (left->is_double_fpu()) { 2006 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register()); 2007 } else { 2008 ShouldNotReachHere(); 2009 } 2010 } else if (code == lir_cmp_l2i) { 2011 Label done; 2012 __ cmp(left->as_register_lo(), right->as_register_lo()); 2013 __ mov(dst->as_register(), (u_int64_t)-1L); 2014 __ br(Assembler::LT, done); 2015 __ csinc(dst->as_register(), zr, zr, Assembler::EQ); 2016 __ bind(done); 2017 } else { 2018 ShouldNotReachHere(); 2019 } 2020 } 2021 2022 2023 void LIR_Assembler::align_call(LIR_Code code) { } 2024 2025 2026 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 2027 address call = __ trampoline_call(Address(op->addr(), rtype)); 2028 if (call == NULL) { 2029 bailout("trampoline stub overflow"); 2030 return; 2031 } 2032 add_call_info(code_offset(), op->info()); 2033 } 2034 2035 2036 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 2037 address call = __ ic_call(op->addr()); 2038 if (call == NULL) { 2039 bailout("trampoline stub overflow"); 2040 return; 2041 } 2042 add_call_info(code_offset(), op->info()); 2043 } 2044 2045 2046 /* Currently, vtable-dispatch is only enabled for sparc platforms */ 2047 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) { 2048 ShouldNotReachHere(); 2049 } 2050 2051 2052 void LIR_Assembler::emit_static_call_stub() { 2053 address call_pc = __ pc(); 2054 address stub = __ start_a_stub(call_stub_size); 2055 if (stub == NULL) { 2056 bailout("static call stub overflow"); 2057 return; 2058 } 2059 2060 int start = __ offset(); 2061 2062 __ relocate(static_stub_Relocation::spec(call_pc)); 2063 __ mov_metadata(rmethod, (Metadata*)NULL); 2064 __ movptr(rscratch1, 0); 2065 __ br(rscratch1); 2066 2067 assert(__ offset() - start <= call_stub_size, "stub too big"); 2068 __ end_a_stub(); 2069 } 2070 2071 2072 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 2073 assert(exceptionOop->as_register() == r0, "must match"); 2074 assert(exceptionPC->as_register() == r3, "must match"); 2075 2076 // exception object is not added to oop map by LinearScan 2077 // (LinearScan assumes that no oops are in fixed registers) 2078 info->add_register_oop(exceptionOop); 2079 Runtime1::StubID unwind_id; 2080 2081 // get current pc information 2082 // pc is only needed if the method has an exception handler, the unwind code does not need it. 2083 int pc_for_athrow_offset = __ offset(); 2084 InternalAddress pc_for_athrow(__ pc()); 2085 __ adr(exceptionPC->as_register(), pc_for_athrow); 2086 add_call_info(pc_for_athrow_offset, info); // for exception handler 2087 2088 __ verify_not_null_oop(r0); 2089 // search an exception handler (r0: exception oop, r3: throwing pc) 2090 if (compilation()->has_fpu_code()) { 2091 unwind_id = Runtime1::handle_exception_id; 2092 } else { 2093 unwind_id = Runtime1::handle_exception_nofpu_id; 2094 } 2095 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 2096 2097 // FIXME: enough room for two byte trap ???? 2098 __ nop(); 2099 } 2100 2101 2102 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 2103 assert(exceptionOop->as_register() == r0, "must match"); 2104 2105 __ b(_unwind_handler_entry); 2106 } 2107 2108 2109 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 2110 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2111 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2112 2113 switch (left->type()) { 2114 case T_INT: { 2115 switch (code) { 2116 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break; 2117 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break; 2118 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break; 2119 default: 2120 ShouldNotReachHere(); 2121 break; 2122 } 2123 break; 2124 case T_LONG: 2125 case T_ADDRESS: 2126 case T_OBJECT: 2127 switch (code) { 2128 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break; 2129 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break; 2130 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break; 2131 default: 2132 ShouldNotReachHere(); 2133 break; 2134 } 2135 break; 2136 default: 2137 ShouldNotReachHere(); 2138 break; 2139 } 2140 } 2141 } 2142 2143 2144 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 2145 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2146 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2147 2148 switch (left->type()) { 2149 case T_INT: { 2150 switch (code) { 2151 case lir_shl: __ lslw (dreg, lreg, count); break; 2152 case lir_shr: __ asrw (dreg, lreg, count); break; 2153 case lir_ushr: __ lsrw (dreg, lreg, count); break; 2154 default: 2155 ShouldNotReachHere(); 2156 break; 2157 } 2158 break; 2159 case T_LONG: 2160 case T_ADDRESS: 2161 case T_OBJECT: 2162 switch (code) { 2163 case lir_shl: __ lsl (dreg, lreg, count); break; 2164 case lir_shr: __ asr (dreg, lreg, count); break; 2165 case lir_ushr: __ lsr (dreg, lreg, count); break; 2166 default: 2167 ShouldNotReachHere(); 2168 break; 2169 } 2170 break; 2171 default: 2172 ShouldNotReachHere(); 2173 break; 2174 } 2175 } 2176 } 2177 2178 2179 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 2180 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2181 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2182 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2183 __ str (r, Address(sp, offset_from_rsp_in_bytes)); 2184 } 2185 2186 2187 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 2188 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2189 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2190 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2191 __ mov (rscratch1, c); 2192 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2193 } 2194 2195 2196 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) { 2197 ShouldNotReachHere(); 2198 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2199 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2200 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2201 __ lea(rscratch1, __ constant_oop_address(o)); 2202 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2203 } 2204 2205 2206 // This code replaces a call to arraycopy; no exception may 2207 // be thrown in this code, they must be thrown in the System.arraycopy 2208 // activation frame; we could save some checks if this would not be the case 2209 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 2210 ciArrayKlass* default_type = op->expected_type(); 2211 Register src = op->src()->as_register(); 2212 Register dst = op->dst()->as_register(); 2213 Register src_pos = op->src_pos()->as_register(); 2214 Register dst_pos = op->dst_pos()->as_register(); 2215 Register length = op->length()->as_register(); 2216 Register tmp = op->tmp()->as_register(); 2217 2218 CodeStub* stub = op->stub(); 2219 int flags = op->flags(); 2220 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL; 2221 if (basic_type == T_ARRAY) basic_type = T_OBJECT; 2222 2223 // if we don't know anything, just go through the generic arraycopy 2224 if (default_type == NULL // || basic_type == T_OBJECT 2225 ) { 2226 Label done; 2227 assert(src == r1 && src_pos == r2, "mismatch in calling convention"); 2228 2229 // Save the arguments in case the generic arraycopy fails and we 2230 // have to fall back to the JNI stub 2231 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2232 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2233 __ str(src, Address(sp, 4*BytesPerWord)); 2234 2235 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy); 2236 address copyfunc_addr = StubRoutines::generic_arraycopy(); 2237 2238 // The arguments are in java calling convention so we shift them 2239 // to C convention 2240 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4); 2241 __ mov(c_rarg0, j_rarg0); 2242 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4); 2243 __ mov(c_rarg1, j_rarg1); 2244 assert_different_registers(c_rarg2, j_rarg3, j_rarg4); 2245 __ mov(c_rarg2, j_rarg2); 2246 assert_different_registers(c_rarg3, j_rarg4); 2247 __ mov(c_rarg3, j_rarg3); 2248 __ mov(c_rarg4, j_rarg4); 2249 if (copyfunc_addr == NULL) { // Use C version if stub was not generated 2250 __ mov(rscratch1, RuntimeAddress(C_entry)); 2251 __ blr(rscratch1); 2252 } else { 2253 #ifndef PRODUCT 2254 if (PrintC1Statistics) { 2255 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); 2256 } 2257 #endif 2258 __ far_call(RuntimeAddress(copyfunc_addr)); 2259 } 2260 2261 __ cbz(r0, *stub->continuation()); 2262 2263 // Reload values from the stack so they are where the stub 2264 // expects them. 2265 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2266 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2267 __ ldr(src, Address(sp, 4*BytesPerWord)); 2268 2269 if (copyfunc_addr != NULL) { 2270 // r0 is -1^K where K == partial copied count 2271 __ eonw(rscratch1, r0, zr); 2272 // adjust length down and src/end pos up by partial copied count 2273 __ subw(length, length, rscratch1); 2274 __ addw(src_pos, src_pos, rscratch1); 2275 __ addw(dst_pos, dst_pos, rscratch1); 2276 } 2277 __ b(*stub->entry()); 2278 2279 __ bind(*stub->continuation()); 2280 return; 2281 } 2282 2283 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point"); 2284 2285 int elem_size = type2aelembytes(basic_type); 2286 int shift_amount; 2287 int scale = exact_log2(elem_size); 2288 2289 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); 2290 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); 2291 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); 2292 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); 2293 2294 // test for NULL 2295 if (flags & LIR_OpArrayCopy::src_null_check) { 2296 __ cbz(src, *stub->entry()); 2297 } 2298 if (flags & LIR_OpArrayCopy::dst_null_check) { 2299 __ cbz(dst, *stub->entry()); 2300 } 2301 2302 // If the compiler was not able to prove that exact type of the source or the destination 2303 // of the arraycopy is an array type, check at runtime if the source or the destination is 2304 // an instance type. 2305 if (flags & LIR_OpArrayCopy::type_check) { 2306 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) { 2307 __ load_klass(tmp, dst); 2308 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2309 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2310 __ br(Assembler::GE, *stub->entry()); 2311 } 2312 2313 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) { 2314 __ load_klass(tmp, src); 2315 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2316 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2317 __ br(Assembler::GE, *stub->entry()); 2318 } 2319 } 2320 2321 // check if negative 2322 if (flags & LIR_OpArrayCopy::src_pos_positive_check) { 2323 __ cmpw(src_pos, 0); 2324 __ br(Assembler::LT, *stub->entry()); 2325 } 2326 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) { 2327 __ cmpw(dst_pos, 0); 2328 __ br(Assembler::LT, *stub->entry()); 2329 } 2330 2331 if (flags & LIR_OpArrayCopy::length_positive_check) { 2332 __ cmpw(length, 0); 2333 __ br(Assembler::LT, *stub->entry()); 2334 } 2335 2336 if (flags & LIR_OpArrayCopy::src_range_check) { 2337 __ addw(tmp, src_pos, length); 2338 __ ldrw(rscratch1, src_length_addr); 2339 __ cmpw(tmp, rscratch1); 2340 __ br(Assembler::HI, *stub->entry()); 2341 } 2342 if (flags & LIR_OpArrayCopy::dst_range_check) { 2343 __ addw(tmp, dst_pos, length); 2344 __ ldrw(rscratch1, dst_length_addr); 2345 __ cmpw(tmp, rscratch1); 2346 __ br(Assembler::HI, *stub->entry()); 2347 } 2348 2349 // FIXME: The logic in LIRGenerator::arraycopy_helper clears 2350 // length_positive_check if the source of our length operand is an 2351 // arraylength. However, that arraylength might be zero, and the 2352 // stub that we're about to call contains an assertion that count != 2353 // 0 . So we make this check purely in order not to trigger an 2354 // assertion failure. 2355 __ cbzw(length, *stub->continuation()); 2356 2357 if (flags & LIR_OpArrayCopy::type_check) { 2358 // We don't know the array types are compatible 2359 if (basic_type != T_OBJECT) { 2360 // Simple test for basic type arrays 2361 if (UseCompressedClassPointers) { 2362 __ ldrw(tmp, src_klass_addr); 2363 __ ldrw(rscratch1, dst_klass_addr); 2364 __ cmpw(tmp, rscratch1); 2365 } else { 2366 __ ldr(tmp, src_klass_addr); 2367 __ ldr(rscratch1, dst_klass_addr); 2368 __ cmp(tmp, rscratch1); 2369 } 2370 __ br(Assembler::NE, *stub->entry()); 2371 } else { 2372 // For object arrays, if src is a sub class of dst then we can 2373 // safely do the copy. 2374 Label cont, slow; 2375 2376 #define PUSH(r1, r2) \ 2377 stp(r1, r2, __ pre(sp, -2 * wordSize)); 2378 2379 #define POP(r1, r2) \ 2380 ldp(r1, r2, __ post(sp, 2 * wordSize)); 2381 2382 __ PUSH(src, dst); 2383 2384 __ load_klass(src, src); 2385 __ load_klass(dst, dst); 2386 2387 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL); 2388 2389 __ PUSH(src, dst); 2390 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 2391 __ POP(src, dst); 2392 2393 __ cbnz(src, cont); 2394 2395 __ bind(slow); 2396 __ POP(src, dst); 2397 2398 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 2399 if (copyfunc_addr != NULL) { // use stub if available 2400 // src is not a sub class of dst so we have to do a 2401 // per-element check. 2402 2403 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 2404 if ((flags & mask) != mask) { 2405 // Check that at least both of them object arrays. 2406 assert(flags & mask, "one of the two should be known to be an object array"); 2407 2408 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 2409 __ load_klass(tmp, src); 2410 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 2411 __ load_klass(tmp, dst); 2412 } 2413 int lh_offset = in_bytes(Klass::layout_helper_offset()); 2414 Address klass_lh_addr(tmp, lh_offset); 2415 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 2416 __ ldrw(rscratch1, klass_lh_addr); 2417 __ mov(rscratch2, objArray_lh); 2418 __ eorw(rscratch1, rscratch1, rscratch2); 2419 __ cbnzw(rscratch1, *stub->entry()); 2420 } 2421 2422 // Spill because stubs can use any register they like and it's 2423 // easier to restore just those that we care about. 2424 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2425 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2426 __ str(src, Address(sp, 4*BytesPerWord)); 2427 2428 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2429 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2430 assert_different_registers(c_rarg0, dst, dst_pos, length); 2431 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2432 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2433 assert_different_registers(c_rarg1, dst, length); 2434 __ uxtw(c_rarg2, length); 2435 assert_different_registers(c_rarg2, dst); 2436 2437 __ load_klass(c_rarg4, dst); 2438 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset())); 2439 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset())); 2440 __ far_call(RuntimeAddress(copyfunc_addr)); 2441 2442 #ifndef PRODUCT 2443 if (PrintC1Statistics) { 2444 Label failed; 2445 __ cbnz(r0, failed); 2446 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt)); 2447 __ bind(failed); 2448 } 2449 #endif 2450 2451 __ cbz(r0, *stub->continuation()); 2452 2453 #ifndef PRODUCT 2454 if (PrintC1Statistics) { 2455 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt)); 2456 } 2457 #endif 2458 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1); 2459 2460 // Restore previously spilled arguments 2461 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2462 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2463 __ ldr(src, Address(sp, 4*BytesPerWord)); 2464 2465 // return value is -1^K where K is partial copied count 2466 __ eonw(rscratch1, r0, zr); 2467 // adjust length down and src/end pos up by partial copied count 2468 __ subw(length, length, rscratch1); 2469 __ addw(src_pos, src_pos, rscratch1); 2470 __ addw(dst_pos, dst_pos, rscratch1); 2471 } 2472 2473 __ b(*stub->entry()); 2474 2475 __ bind(cont); 2476 __ POP(src, dst); 2477 } 2478 } 2479 2480 #ifdef ASSERT 2481 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { 2482 // Sanity check the known type with the incoming class. For the 2483 // primitive case the types must match exactly with src.klass and 2484 // dst.klass each exactly matching the default type. For the 2485 // object array case, if no type check is needed then either the 2486 // dst type is exactly the expected type and the src type is a 2487 // subtype which we can't check or src is the same array as dst 2488 // but not necessarily exactly of type default_type. 2489 Label known_ok, halt; 2490 __ mov_metadata(tmp, default_type->constant_encoding()); 2491 #ifdef _LP64 2492 if (UseCompressedClassPointers) { 2493 __ encode_klass_not_null(tmp); 2494 } 2495 #endif 2496 2497 if (basic_type != T_OBJECT) { 2498 2499 if (UseCompressedClassPointers) { 2500 __ ldrw(rscratch1, dst_klass_addr); 2501 __ cmpw(tmp, rscratch1); 2502 } else { 2503 __ ldr(rscratch1, dst_klass_addr); 2504 __ cmp(tmp, rscratch1); 2505 } 2506 __ br(Assembler::NE, halt); 2507 if (UseCompressedClassPointers) { 2508 __ ldrw(rscratch1, src_klass_addr); 2509 __ cmpw(tmp, rscratch1); 2510 } else { 2511 __ ldr(rscratch1, src_klass_addr); 2512 __ cmp(tmp, rscratch1); 2513 } 2514 __ br(Assembler::EQ, known_ok); 2515 } else { 2516 if (UseCompressedClassPointers) { 2517 __ ldrw(rscratch1, dst_klass_addr); 2518 __ cmpw(tmp, rscratch1); 2519 } else { 2520 __ ldr(rscratch1, dst_klass_addr); 2521 __ cmp(tmp, rscratch1); 2522 } 2523 __ br(Assembler::EQ, known_ok); 2524 __ cmp(src, dst); 2525 __ br(Assembler::EQ, known_ok); 2526 } 2527 __ bind(halt); 2528 __ stop("incorrect type information in arraycopy"); 2529 __ bind(known_ok); 2530 } 2531 #endif 2532 2533 #ifndef PRODUCT 2534 if (PrintC1Statistics) { 2535 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type))); 2536 } 2537 #endif 2538 2539 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2540 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2541 assert_different_registers(c_rarg0, dst, dst_pos, length); 2542 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2543 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2544 assert_different_registers(c_rarg1, dst, length); 2545 __ uxtw(c_rarg2, length); 2546 assert_different_registers(c_rarg2, dst); 2547 2548 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 2549 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 2550 const char *name; 2551 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 2552 2553 CodeBlob *cb = CodeCache::find_blob(entry); 2554 if (cb) { 2555 __ far_call(RuntimeAddress(entry)); 2556 } else { 2557 __ call_VM_leaf(entry, 3); 2558 } 2559 2560 __ bind(*stub->continuation()); 2561 } 2562 2563 2564 2565 2566 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 2567 Register obj = op->obj_opr()->as_register(); // may not be an oop 2568 Register hdr = op->hdr_opr()->as_register(); 2569 Register lock = op->lock_opr()->as_register(); 2570 if (!UseFastLocking) { 2571 __ b(*op->stub()->entry()); 2572 } else if (op->code() == lir_lock) { 2573 Register scratch = noreg; 2574 if (UseBiasedLocking) { 2575 scratch = op->scratch_opr()->as_register(); 2576 } 2577 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2578 // add debug info for NullPointerException only if one is possible 2579 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry()); 2580 if (op->info() != NULL) { 2581 add_debug_info_for_null_check(null_check_offset, op->info()); 2582 } 2583 // done 2584 } else if (op->code() == lir_unlock) { 2585 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2586 __ unlock_object(hdr, obj, lock, *op->stub()->entry()); 2587 } else { 2588 Unimplemented(); 2589 } 2590 __ bind(*op->stub()->continuation()); 2591 } 2592 2593 2594 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 2595 ciMethod* method = op->profiled_method(); 2596 int bci = op->profiled_bci(); 2597 ciMethod* callee = op->profiled_callee(); 2598 2599 // Update counter for all call types 2600 ciMethodData* md = method->method_data_or_null(); 2601 assert(md != NULL, "Sanity"); 2602 ciProfileData* data = md->bci_to_data(bci); 2603 assert(data->is_CounterData(), "need CounterData for calls"); 2604 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 2605 Register mdo = op->mdo()->as_register(); 2606 __ mov_metadata(mdo, md->constant_encoding()); 2607 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 2608 Bytecodes::Code bc = method->java_code_at_bci(bci); 2609 const bool callee_is_static = callee->is_loaded() && callee->is_static(); 2610 // Perform additional virtual call profiling for invokevirtual and 2611 // invokeinterface bytecodes 2612 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) && 2613 !callee_is_static && // required for optimized MH invokes 2614 C1ProfileVirtualCalls) { 2615 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 2616 Register recv = op->recv()->as_register(); 2617 assert_different_registers(mdo, recv); 2618 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 2619 ciKlass* known_klass = op->known_holder(); 2620 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) { 2621 // We know the type that will be seen at this call site; we can 2622 // statically update the MethodData* rather than needing to do 2623 // dynamic tests on the receiver type 2624 2625 // NOTE: we should probably put a lock around this search to 2626 // avoid collisions by concurrent compilations 2627 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 2628 uint i; 2629 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2630 ciKlass* receiver = vc_data->receiver(i); 2631 if (known_klass->equals(receiver)) { 2632 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2633 __ addptr(data_addr, DataLayout::counter_increment); 2634 return; 2635 } 2636 } 2637 2638 // Receiver type not found in profile data; select an empty slot 2639 2640 // Note that this is less efficient than it should be because it 2641 // always does a write to the receiver part of the 2642 // VirtualCallData rather than just the first time 2643 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2644 ciKlass* receiver = vc_data->receiver(i); 2645 if (receiver == NULL) { 2646 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); 2647 __ mov_metadata(rscratch1, known_klass->constant_encoding()); 2648 __ lea(rscratch2, recv_addr); 2649 __ str(rscratch1, Address(rscratch2)); 2650 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2651 __ addptr(data_addr, DataLayout::counter_increment); 2652 return; 2653 } 2654 } 2655 } else { 2656 __ load_klass(recv, recv); 2657 Label update_done; 2658 type_profile_helper(mdo, md, data, recv, &update_done); 2659 // Receiver did not match any saved receiver and there is no empty row for it. 2660 // Increment total counter to indicate polymorphic case. 2661 __ addptr(counter_addr, DataLayout::counter_increment); 2662 2663 __ bind(update_done); 2664 } 2665 } else { 2666 // Static call 2667 __ addptr(counter_addr, DataLayout::counter_increment); 2668 } 2669 } 2670 2671 2672 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 2673 Unimplemented(); 2674 } 2675 2676 2677 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 2678 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 2679 } 2680 2681 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 2682 assert(op->crc()->is_single_cpu(), "crc must be register"); 2683 assert(op->val()->is_single_cpu(), "byte value must be register"); 2684 assert(op->result_opr()->is_single_cpu(), "result must be register"); 2685 Register crc = op->crc()->as_register(); 2686 Register val = op->val()->as_register(); 2687 Register res = op->result_opr()->as_register(); 2688 2689 assert_different_registers(val, crc, res); 2690 unsigned long offset; 2691 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset); 2692 if (offset) __ add(res, res, offset); 2693 2694 __ ornw(crc, zr, crc); // ~crc 2695 __ update_byte_crc32(crc, val, res); 2696 __ ornw(res, zr, crc); // ~crc 2697 } 2698 2699 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 2700 COMMENT("emit_profile_type {"); 2701 Register obj = op->obj()->as_register(); 2702 Register tmp = op->tmp()->as_pointer_register(); 2703 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 2704 ciKlass* exact_klass = op->exact_klass(); 2705 intptr_t current_klass = op->current_klass(); 2706 bool not_null = op->not_null(); 2707 bool no_conflict = op->no_conflict(); 2708 2709 Label update, next, none; 2710 2711 bool do_null = !not_null; 2712 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 2713 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 2714 2715 assert(do_null || do_update, "why are we here?"); 2716 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 2717 assert(mdo_addr.base() != rscratch1, "wrong register"); 2718 2719 __ verify_oop(obj); 2720 2721 if (tmp != obj) { 2722 __ mov(tmp, obj); 2723 } 2724 if (do_null) { 2725 __ cbnz(tmp, update); 2726 if (!TypeEntries::was_null_seen(current_klass)) { 2727 __ ldr(rscratch2, mdo_addr); 2728 __ orr(rscratch2, rscratch2, TypeEntries::null_seen); 2729 __ str(rscratch2, mdo_addr); 2730 } 2731 if (do_update) { 2732 #ifndef ASSERT 2733 __ b(next); 2734 } 2735 #else 2736 __ b(next); 2737 } 2738 } else { 2739 __ cbnz(tmp, update); 2740 __ stop("unexpected null obj"); 2741 #endif 2742 } 2743 2744 __ bind(update); 2745 2746 if (do_update) { 2747 #ifdef ASSERT 2748 if (exact_klass != NULL) { 2749 Label ok; 2750 __ load_klass(tmp, tmp); 2751 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2752 __ eor(rscratch1, tmp, rscratch1); 2753 __ cbz(rscratch1, ok); 2754 __ stop("exact klass and actual klass differ"); 2755 __ bind(ok); 2756 } 2757 #endif 2758 if (!no_conflict) { 2759 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) { 2760 if (exact_klass != NULL) { 2761 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2762 } else { 2763 __ load_klass(tmp, tmp); 2764 } 2765 2766 __ ldr(rscratch2, mdo_addr); 2767 __ eor(tmp, tmp, rscratch2); 2768 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2769 // klass seen before, nothing to do. The unknown bit may have been 2770 // set already but no need to check. 2771 __ cbz(rscratch1, next); 2772 2773 __ andr(rscratch1, tmp, TypeEntries::type_unknown); 2774 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore. 2775 2776 if (TypeEntries::is_type_none(current_klass)) { 2777 __ cbz(rscratch2, none); 2778 __ cmp(rscratch2, TypeEntries::null_seen); 2779 __ br(Assembler::EQ, none); 2780 // There is a chance that the checks above (re-reading profiling 2781 // data from memory) fail if another thread has just set the 2782 // profiling to this obj's klass 2783 __ dmb(Assembler::ISHLD); 2784 __ ldr(rscratch2, mdo_addr); 2785 __ eor(tmp, tmp, rscratch2); 2786 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2787 __ cbz(rscratch1, next); 2788 } 2789 } else { 2790 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 2791 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 2792 2793 __ ldr(tmp, mdo_addr); 2794 __ andr(rscratch1, tmp, TypeEntries::type_unknown); 2795 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore. 2796 } 2797 2798 // different than before. Cannot keep accurate profile. 2799 __ ldr(rscratch2, mdo_addr); 2800 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown); 2801 __ str(rscratch2, mdo_addr); 2802 2803 if (TypeEntries::is_type_none(current_klass)) { 2804 __ b(next); 2805 2806 __ bind(none); 2807 // first time here. Set profile type. 2808 __ str(tmp, mdo_addr); 2809 } 2810 } else { 2811 // There's a single possible klass at this profile point 2812 assert(exact_klass != NULL, "should be"); 2813 if (TypeEntries::is_type_none(current_klass)) { 2814 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2815 __ ldr(rscratch2, mdo_addr); 2816 __ eor(tmp, tmp, rscratch2); 2817 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2818 __ cbz(rscratch1, next); 2819 #ifdef ASSERT 2820 { 2821 Label ok; 2822 __ ldr(rscratch1, mdo_addr); 2823 __ cbz(rscratch1, ok); 2824 __ cmp(rscratch1, TypeEntries::null_seen); 2825 __ br(Assembler::EQ, ok); 2826 // may have been set by another thread 2827 __ dmb(Assembler::ISHLD); 2828 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2829 __ ldr(rscratch2, mdo_addr); 2830 __ eor(rscratch2, rscratch1, rscratch2); 2831 __ andr(rscratch2, rscratch2, TypeEntries::type_mask); 2832 __ cbz(rscratch2, ok); 2833 2834 __ stop("unexpected profiling mismatch"); 2835 __ bind(ok); 2836 } 2837 #endif 2838 // first time here. Set profile type. 2839 __ ldr(tmp, mdo_addr); 2840 } else { 2841 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 2842 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 2843 2844 __ ldr(tmp, mdo_addr); 2845 __ andr(rscratch1, tmp, TypeEntries::type_unknown); 2846 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore. 2847 2848 __ orr(tmp, tmp, TypeEntries::type_unknown); 2849 __ str(tmp, mdo_addr); 2850 // FIXME: Write barrier needed here? 2851 } 2852 } 2853 2854 __ bind(next); 2855 } 2856 COMMENT("} emit_profile_type"); 2857 } 2858 2859 2860 void LIR_Assembler::align_backward_branch_target() { 2861 } 2862 2863 2864 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) { 2865 if (left->is_single_cpu()) { 2866 assert(dest->is_single_cpu(), "expect single result reg"); 2867 __ negw(dest->as_register(), left->as_register()); 2868 } else if (left->is_double_cpu()) { 2869 assert(dest->is_double_cpu(), "expect double result reg"); 2870 __ neg(dest->as_register_lo(), left->as_register_lo()); 2871 } else if (left->is_single_fpu()) { 2872 assert(dest->is_single_fpu(), "expect single float result reg"); 2873 __ fnegs(dest->as_float_reg(), left->as_float_reg()); 2874 } else { 2875 assert(left->is_double_fpu(), "expect double float operand reg"); 2876 assert(dest->is_double_fpu(), "expect double float result reg"); 2877 __ fnegd(dest->as_double_reg(), left->as_double_reg()); 2878 } 2879 } 2880 2881 2882 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) { 2883 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr())); 2884 } 2885 2886 2887 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 2888 assert(!tmp->is_valid(), "don't need temporary"); 2889 2890 CodeBlob *cb = CodeCache::find_blob(dest); 2891 if (cb) { 2892 __ far_call(RuntimeAddress(dest)); 2893 } else { 2894 __ mov(rscratch1, RuntimeAddress(dest)); 2895 __ blr(rscratch1); 2896 } 2897 2898 if (info != NULL) { 2899 add_call_info_here(info); 2900 } 2901 __ maybe_isb(); 2902 } 2903 2904 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 2905 if (dest->is_address() || src->is_address()) { 2906 move_op(src, dest, type, lir_patch_none, info, 2907 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false); 2908 } else { 2909 ShouldNotReachHere(); 2910 } 2911 } 2912 2913 #ifdef ASSERT 2914 // emit run-time assertion 2915 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 2916 assert(op->code() == lir_assert, "must be"); 2917 2918 if (op->in_opr1()->is_valid()) { 2919 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 2920 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 2921 } else { 2922 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 2923 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 2924 } 2925 2926 Label ok; 2927 if (op->condition() != lir_cond_always) { 2928 Assembler::Condition acond = Assembler::AL; 2929 switch (op->condition()) { 2930 case lir_cond_equal: acond = Assembler::EQ; break; 2931 case lir_cond_notEqual: acond = Assembler::NE; break; 2932 case lir_cond_less: acond = Assembler::LT; break; 2933 case lir_cond_lessEqual: acond = Assembler::LE; break; 2934 case lir_cond_greaterEqual: acond = Assembler::GE; break; 2935 case lir_cond_greater: acond = Assembler::GT; break; 2936 case lir_cond_belowEqual: acond = Assembler::LS; break; 2937 case lir_cond_aboveEqual: acond = Assembler::HS; break; 2938 default: ShouldNotReachHere(); 2939 } 2940 __ br(acond, ok); 2941 } 2942 if (op->halt()) { 2943 const char* str = __ code_string(op->msg()); 2944 __ stop(str); 2945 } else { 2946 breakpoint(); 2947 } 2948 __ bind(ok); 2949 } 2950 #endif 2951 2952 #ifndef PRODUCT 2953 #define COMMENT(x) do { __ block_comment(x); } while (0) 2954 #else 2955 #define COMMENT(x) 2956 #endif 2957 2958 void LIR_Assembler::membar() { 2959 COMMENT("membar"); 2960 __ membar(MacroAssembler::AnyAny); 2961 } 2962 2963 void LIR_Assembler::membar_acquire() { 2964 __ membar(Assembler::LoadLoad|Assembler::LoadStore); 2965 } 2966 2967 void LIR_Assembler::membar_release() { 2968 __ membar(Assembler::LoadStore|Assembler::StoreStore); 2969 } 2970 2971 void LIR_Assembler::membar_loadload() { 2972 __ membar(Assembler::LoadLoad); 2973 } 2974 2975 void LIR_Assembler::membar_storestore() { 2976 __ membar(MacroAssembler::StoreStore); 2977 } 2978 2979 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); } 2980 2981 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); } 2982 2983 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 2984 __ mov(result_reg->as_register(), rthread); 2985 } 2986 2987 2988 void LIR_Assembler::peephole(LIR_List *lir) { 2989 #if 0 2990 if (tableswitch_count >= max_tableswitches) 2991 return; 2992 2993 /* 2994 This finite-state automaton recognizes sequences of compare-and- 2995 branch instructions. We will turn them into a tableswitch. You 2996 could argue that C1 really shouldn't be doing this sort of 2997 optimization, but without it the code is really horrible. 2998 */ 2999 3000 enum { start_s, cmp1_s, beq_s, cmp_s } state; 3001 int first_key, last_key = -2147483648; 3002 int next_key = 0; 3003 int start_insn = -1; 3004 int last_insn = -1; 3005 Register reg = noreg; 3006 LIR_Opr reg_opr; 3007 state = start_s; 3008 3009 LIR_OpList* inst = lir->instructions_list(); 3010 for (int i = 0; i < inst->length(); i++) { 3011 LIR_Op* op = inst->at(i); 3012 switch (state) { 3013 case start_s: 3014 first_key = -1; 3015 start_insn = i; 3016 switch (op->code()) { 3017 case lir_cmp: 3018 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3019 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3020 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3021 && opr2->is_constant() 3022 && opr2->type() == T_INT) { 3023 reg_opr = opr1; 3024 reg = opr1->as_register(); 3025 first_key = opr2->as_constant_ptr()->as_jint(); 3026 next_key = first_key + 1; 3027 state = cmp_s; 3028 goto next_state; 3029 } 3030 break; 3031 } 3032 break; 3033 case cmp_s: 3034 switch (op->code()) { 3035 case lir_branch: 3036 if (op->as_OpBranch()->cond() == lir_cond_equal) { 3037 state = beq_s; 3038 last_insn = i; 3039 goto next_state; 3040 } 3041 } 3042 state = start_s; 3043 break; 3044 case beq_s: 3045 switch (op->code()) { 3046 case lir_cmp: { 3047 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3048 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3049 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3050 && opr1->as_register() == reg 3051 && opr2->is_constant() 3052 && opr2->type() == T_INT 3053 && opr2->as_constant_ptr()->as_jint() == next_key) { 3054 last_key = next_key; 3055 next_key++; 3056 state = cmp_s; 3057 goto next_state; 3058 } 3059 } 3060 } 3061 last_key = next_key; 3062 state = start_s; 3063 break; 3064 default: 3065 assert(false, "impossible state"); 3066 } 3067 if (state == start_s) { 3068 if (first_key < last_key - 5L && reg != noreg) { 3069 { 3070 // printf("found run register %d starting at insn %d low value %d high value %d\n", 3071 // reg->encoding(), 3072 // start_insn, first_key, last_key); 3073 // for (int i = 0; i < inst->length(); i++) { 3074 // inst->at(i)->print(); 3075 // tty->print("\n"); 3076 // } 3077 // tty->print("\n"); 3078 } 3079 3080 struct tableswitch *sw = &switches[tableswitch_count]; 3081 sw->_insn_index = start_insn, sw->_first_key = first_key, 3082 sw->_last_key = last_key, sw->_reg = reg; 3083 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after)); 3084 { 3085 // Insert the new table of branches 3086 int offset = last_insn; 3087 for (int n = first_key; n < last_key; n++) { 3088 inst->insert_before 3089 (last_insn + 1, 3090 new LIR_OpBranch(lir_cond_always, T_ILLEGAL, 3091 inst->at(offset)->as_OpBranch()->label())); 3092 offset -= 2, i++; 3093 } 3094 } 3095 // Delete all the old compare-and-branch instructions 3096 for (int n = first_key; n < last_key; n++) { 3097 inst->remove_at(start_insn); 3098 inst->remove_at(start_insn); 3099 } 3100 // Insert the tableswitch instruction 3101 inst->insert_before(start_insn, 3102 new LIR_Op2(lir_cmp, lir_cond_always, 3103 LIR_OprFact::intConst(tableswitch_count), 3104 reg_opr)); 3105 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches)); 3106 tableswitch_count++; 3107 } 3108 reg = noreg; 3109 last_key = -2147483648; 3110 } 3111 next_state: 3112 ; 3113 } 3114 #endif 3115 } 3116 3117 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { 3118 Address addr = as_Address(src->as_address_ptr()); 3119 BasicType type = src->type(); 3120 bool is_oop = type == T_OBJECT || type == T_ARRAY; 3121 3122 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr); 3123 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr); 3124 3125 switch(type) { 3126 case T_INT: 3127 xchg = &MacroAssembler::atomic_xchgalw; 3128 add = &MacroAssembler::atomic_addalw; 3129 break; 3130 case T_LONG: 3131 xchg = &MacroAssembler::atomic_xchgal; 3132 add = &MacroAssembler::atomic_addal; 3133 break; 3134 case T_OBJECT: 3135 case T_ARRAY: 3136 if (UseCompressedOops) { 3137 xchg = &MacroAssembler::atomic_xchgalw; 3138 add = &MacroAssembler::atomic_addalw; 3139 } else { 3140 xchg = &MacroAssembler::atomic_xchgal; 3141 add = &MacroAssembler::atomic_addal; 3142 } 3143 break; 3144 default: 3145 ShouldNotReachHere(); 3146 xchg = &MacroAssembler::atomic_xchgal; 3147 add = &MacroAssembler::atomic_addal; // unreachable 3148 } 3149 3150 switch (code) { 3151 case lir_xadd: 3152 { 3153 RegisterOrConstant inc; 3154 Register tmp = as_reg(tmp_op); 3155 Register dst = as_reg(dest); 3156 if (data->is_constant()) { 3157 inc = RegisterOrConstant(as_long(data)); 3158 assert_different_registers(dst, addr.base(), tmp, 3159 rscratch1, rscratch2); 3160 } else { 3161 inc = RegisterOrConstant(as_reg(data)); 3162 assert_different_registers(inc.as_register(), dst, addr.base(), tmp, 3163 rscratch1, rscratch2); 3164 } 3165 __ lea(tmp, addr); 3166 (_masm->*add)(dst, inc, tmp); 3167 break; 3168 } 3169 case lir_xchg: 3170 { 3171 Register tmp = tmp_op->as_register(); 3172 Register obj = as_reg(data); 3173 Register dst = as_reg(dest); 3174 if (is_oop && UseCompressedOops) { 3175 __ encode_heap_oop(rscratch2, obj); 3176 obj = rscratch2; 3177 } 3178 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst); 3179 __ lea(tmp, addr); 3180 (_masm->*xchg)(dst, obj, tmp); 3181 if (is_oop && UseCompressedOops) { 3182 __ decode_heap_oop(dst); 3183 } 3184 } 3185 break; 3186 default: 3187 ShouldNotReachHere(); 3188 } 3189 __ membar(__ AnyAny); 3190 } 3191 3192 #undef __