1 /* 2 * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "asm/macroAssembler.inline.hpp" 28 #include "asm/assembler.hpp" 29 #include "c1/c1_CodeStubs.hpp" 30 #include "c1/c1_Compilation.hpp" 31 #include "c1/c1_LIRAssembler.hpp" 32 #include "c1/c1_MacroAssembler.hpp" 33 #include "c1/c1_Runtime1.hpp" 34 #include "c1/c1_ValueStack.hpp" 35 #include "ci/ciArrayKlass.hpp" 36 #include "ci/ciInstance.hpp" 37 #include "ci/ciUtilities.hpp" 38 #include "code/SCCache.hpp" 39 #include "code/compiledIC.hpp" 40 #include "gc/shared/collectedHeap.hpp" 41 #include "gc/shared/gc_globals.hpp" 42 #include "nativeInst_aarch64.hpp" 43 #include "oops/objArrayKlass.hpp" 44 #include "runtime/frame.inline.hpp" 45 #include "runtime/sharedRuntime.hpp" 46 #include "runtime/stubRoutines.hpp" 47 #include "utilities/powerOfTwo.hpp" 48 #include "vmreg_aarch64.inline.hpp" 49 50 51 #ifndef PRODUCT 52 #define COMMENT(x) do { __ block_comment(x); } while (0) 53 #else 54 #define COMMENT(x) 55 #endif 56 57 NEEDS_CLEANUP // remove this definitions ? 58 const Register SYNC_header = r0; // synchronization header 59 const Register SHIFT_count = r0; // where count for shift operations must be 60 61 #define __ _masm-> 62 63 64 static void select_different_registers(Register preserve, 65 Register extra, 66 Register &tmp1, 67 Register &tmp2) { 68 if (tmp1 == preserve) { 69 assert_different_registers(tmp1, tmp2, extra); 70 tmp1 = extra; 71 } else if (tmp2 == preserve) { 72 assert_different_registers(tmp1, tmp2, extra); 73 tmp2 = extra; 74 } 75 assert_different_registers(preserve, tmp1, tmp2); 76 } 77 78 79 80 static void select_different_registers(Register preserve, 81 Register extra, 82 Register &tmp1, 83 Register &tmp2, 84 Register &tmp3) { 85 if (tmp1 == preserve) { 86 assert_different_registers(tmp1, tmp2, tmp3, extra); 87 tmp1 = extra; 88 } else if (tmp2 == preserve) { 89 assert_different_registers(tmp1, tmp2, tmp3, extra); 90 tmp2 = extra; 91 } else if (tmp3 == preserve) { 92 assert_different_registers(tmp1, tmp2, tmp3, extra); 93 tmp3 = extra; 94 } 95 assert_different_registers(preserve, tmp1, tmp2, tmp3); 96 } 97 98 99 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } 100 101 102 LIR_Opr LIR_Assembler::receiverOpr() { 103 return FrameMap::receiver_opr; 104 } 105 106 LIR_Opr LIR_Assembler::osrBufferPointer() { 107 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 108 } 109 110 //--------------fpu register translations----------------------- 111 112 113 address LIR_Assembler::float_constant(float f) { 114 address const_addr = __ float_constant(f); 115 if (const_addr == nullptr) { 116 bailout("const section overflow"); 117 return __ code()->consts()->start(); 118 } else { 119 return const_addr; 120 } 121 } 122 123 124 address LIR_Assembler::double_constant(double d) { 125 address const_addr = __ double_constant(d); 126 if (const_addr == nullptr) { 127 bailout("const section overflow"); 128 return __ code()->consts()->start(); 129 } else { 130 return const_addr; 131 } 132 } 133 134 address LIR_Assembler::int_constant(jlong n) { 135 address const_addr = __ long_constant(n); 136 if (const_addr == nullptr) { 137 bailout("const section overflow"); 138 return __ code()->consts()->start(); 139 } else { 140 return const_addr; 141 } 142 } 143 144 void LIR_Assembler::breakpoint() { Unimplemented(); } 145 146 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); } 147 148 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); } 149 150 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; } 151 //------------------------------------------- 152 153 static Register as_reg(LIR_Opr op) { 154 return op->is_double_cpu() ? op->as_register_lo() : op->as_register(); 155 } 156 157 static jlong as_long(LIR_Opr data) { 158 jlong result; 159 switch (data->type()) { 160 case T_INT: 161 result = (data->as_jint()); 162 break; 163 case T_LONG: 164 result = (data->as_jlong()); 165 break; 166 default: 167 ShouldNotReachHere(); 168 result = 0; // unreachable 169 } 170 return result; 171 } 172 173 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 174 Register base = addr->base()->as_pointer_register(); 175 LIR_Opr opr = addr->index(); 176 if (opr->is_cpu_register()) { 177 Register index; 178 if (opr->is_single_cpu()) 179 index = opr->as_register(); 180 else 181 index = opr->as_register_lo(); 182 assert(addr->disp() == 0, "must be"); 183 switch(opr->type()) { 184 case T_INT: 185 return Address(base, index, Address::sxtw(addr->scale())); 186 case T_LONG: 187 return Address(base, index, Address::lsl(addr->scale())); 188 default: 189 ShouldNotReachHere(); 190 } 191 } else { 192 assert(addr->scale() == 0, 193 "expected for immediate operand, was: %d", addr->scale()); 194 ptrdiff_t offset = ptrdiff_t(addr->disp()); 195 // NOTE: Does not handle any 16 byte vector access. 196 const uint type_size = type2aelembytes(addr->type(), true); 197 return __ legitimize_address(Address(base, offset), type_size, tmp); 198 } 199 return Address(); 200 } 201 202 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 203 ShouldNotReachHere(); 204 return Address(); 205 } 206 207 Address LIR_Assembler::as_Address(LIR_Address* addr) { 208 return as_Address(addr, rscratch1); 209 } 210 211 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 212 return as_Address(addr, rscratch1); // Ouch 213 // FIXME: This needs to be much more clever. See x86. 214 } 215 216 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is 217 // not encodable as a base + (immediate) offset, generate an explicit address 218 // calculation to hold the address in a temporary register. 219 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) { 220 precond(size == 4 || size == 8); 221 Address addr = frame_map()->address_for_slot(index, adjust); 222 precond(addr.getMode() == Address::base_plus_offset); 223 precond(addr.base() == sp); 224 precond(addr.offset() > 0); 225 uint mask = size - 1; 226 assert((addr.offset() & mask) == 0, "scaled offsets only"); 227 return __ legitimize_address(addr, size, tmp); 228 } 229 230 void LIR_Assembler::osr_entry() { 231 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 232 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 233 ValueStack* entry_state = osr_entry->state(); 234 int number_of_locks = entry_state->locks_size(); 235 236 // we jump here if osr happens with the interpreter 237 // state set up to continue at the beginning of the 238 // loop that triggered osr - in particular, we have 239 // the following registers setup: 240 // 241 // r2: osr buffer 242 // 243 244 // build frame 245 ciMethod* m = compilation()->method(); 246 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 247 248 // OSR buffer is 249 // 250 // locals[nlocals-1..0] 251 // monitors[0..number_of_locks] 252 // 253 // locals is a direct copy of the interpreter frame so in the osr buffer 254 // so first slot in the local array is the last local from the interpreter 255 // and last slot is local[0] (receiver) from the interpreter 256 // 257 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 258 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 259 // in the interpreter frame (the method lock if a sync method) 260 261 // Initialize monitors in the compiled activation. 262 // r2: pointer to osr buffer 263 // 264 // All other registers are dead at this point and the locals will be 265 // copied into place by code emitted in the IR. 266 267 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 268 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 269 int monitor_offset = BytesPerWord * method()->max_locals() + 270 (2 * BytesPerWord) * (number_of_locks - 1); 271 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 272 // the OSR buffer using 2 word entries: first the lock and then 273 // the oop. 274 for (int i = 0; i < number_of_locks; i++) { 275 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 276 #ifdef ASSERT 277 // verify the interpreter's monitor has a non-null object 278 { 279 Label L; 280 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 281 __ cbnz(rscratch1, L); 282 __ stop("locked object is null"); 283 __ bind(L); 284 } 285 #endif 286 __ ldr(r19, Address(OSR_buf, slot_offset)); 287 __ ldr(r20, Address(OSR_buf, slot_offset + BytesPerWord)); 288 __ str(r19, frame_map()->address_for_monitor_lock(i)); 289 __ str(r20, frame_map()->address_for_monitor_object(i)); 290 } 291 } 292 } 293 294 295 // inline cache check; done before the frame is built. 296 int LIR_Assembler::check_icache() { 297 return __ ic_check(CodeEntryAlignment); 298 } 299 300 void LIR_Assembler::clinit_barrier(ciMethod* method) { 301 assert(VM_Version::supports_fast_class_init_checks(), "sanity"); 302 assert(!method->holder()->is_not_initialized(), "initialization should have been started"); 303 304 Label L_skip_barrier; 305 306 __ mov_metadata(rscratch2, method->holder()->constant_encoding()); 307 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/); 308 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 309 __ bind(L_skip_barrier); 310 } 311 312 void LIR_Assembler::jobject2reg(jobject o, Register reg) { 313 if (o == nullptr) { 314 __ mov(reg, zr); 315 } else { 316 __ movoop(reg, o); 317 } 318 } 319 320 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) { 321 address target = nullptr; 322 relocInfo::relocType reloc_type = relocInfo::none; 323 324 switch (patching_id(info)) { 325 case PatchingStub::access_field_id: 326 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 327 reloc_type = relocInfo::section_word_type; 328 break; 329 case PatchingStub::load_klass_id: 330 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 331 reloc_type = relocInfo::metadata_type; 332 break; 333 case PatchingStub::load_mirror_id: 334 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 335 reloc_type = relocInfo::oop_type; 336 break; 337 case PatchingStub::load_appendix_id: 338 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 339 reloc_type = relocInfo::oop_type; 340 break; 341 default: ShouldNotReachHere(); 342 } 343 344 __ far_call(RuntimeAddress(target)); 345 add_call_info_here(info); 346 } 347 348 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) { 349 deoptimize_trap(info); 350 } 351 352 353 // This specifies the rsp decrement needed to build the frame 354 int LIR_Assembler::initial_frame_size_in_bytes() const { 355 // if rounding, must let FrameMap know! 356 357 return in_bytes(frame_map()->framesize_in_bytes()); 358 } 359 360 361 int LIR_Assembler::emit_exception_handler() { 362 // generate code for exception handler 363 address handler_base = __ start_a_stub(exception_handler_size()); 364 if (handler_base == nullptr) { 365 // not enough space left for the handler 366 bailout("exception handler overflow"); 367 return -1; 368 } 369 370 int offset = code_offset(); 371 372 // the exception oop and pc are in r0, and r3 373 // no other registers need to be preserved, so invalidate them 374 __ invalidate_registers(false, true, true, false, true, true); 375 376 // check that there is really an exception 377 __ verify_not_null_oop(r0); 378 379 // search an exception handler (r0: exception oop, r3: throwing pc) 380 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); 381 __ should_not_reach_here(); 382 guarantee(code_offset() - offset <= exception_handler_size(), "overflow"); 383 __ end_a_stub(); 384 385 return offset; 386 } 387 388 389 // Emit the code to remove the frame from the stack in the exception 390 // unwind path. 391 int LIR_Assembler::emit_unwind_handler() { 392 #ifndef PRODUCT 393 if (CommentedAssembly) { 394 _masm->block_comment("Unwind handler"); 395 } 396 #endif 397 398 int offset = code_offset(); 399 400 // Fetch the exception from TLS and clear out exception related thread state 401 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset())); 402 __ str(zr, Address(rthread, JavaThread::exception_oop_offset())); 403 __ str(zr, Address(rthread, JavaThread::exception_pc_offset())); 404 405 __ bind(_unwind_handler_entry); 406 __ verify_not_null_oop(r0); 407 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 408 __ mov(r19, r0); // Preserve the exception 409 } 410 411 // Perform needed unlocking 412 MonitorExitStub* stub = nullptr; 413 if (method()->is_synchronized()) { 414 monitor_address(0, FrameMap::r0_opr); 415 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0); 416 if (LockingMode == LM_MONITOR) { 417 __ b(*stub->entry()); 418 } else { 419 __ unlock_object(r5, r4, r0, r6, *stub->entry()); 420 } 421 __ bind(*stub->continuation()); 422 } 423 424 if (compilation()->env()->dtrace_method_probes()) { 425 __ mov(c_rarg0, rthread); 426 __ mov_metadata(c_rarg1, method()->constant_encoding()); 427 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1); 428 } 429 430 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 431 __ mov(r0, r19); // Restore the exception 432 } 433 434 // remove the activation and dispatch to the unwind handler 435 __ block_comment("remove_frame and dispatch to the unwind handler"); 436 __ remove_frame(initial_frame_size_in_bytes()); 437 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id))); 438 439 // Emit the slow path assembly 440 if (stub != nullptr) { 441 stub->emit_code(this); 442 } 443 444 return offset; 445 } 446 447 448 int LIR_Assembler::emit_deopt_handler() { 449 // generate code for exception handler 450 address handler_base = __ start_a_stub(deopt_handler_size()); 451 if (handler_base == nullptr) { 452 // not enough space left for the handler 453 bailout("deopt handler overflow"); 454 return -1; 455 } 456 457 int offset = code_offset(); 458 459 __ adr(lr, pc()); 460 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 461 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); 462 __ end_a_stub(); 463 464 return offset; 465 } 466 467 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) { 468 _masm->code_section()->relocate(adr, relocInfo::poll_type); 469 int pc_offset = code_offset(); 470 flush_debug_info(pc_offset); 471 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset); 472 if (info->exception_handlers() != nullptr) { 473 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers()); 474 } 475 } 476 477 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { 478 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,"); 479 480 // Pop the stack before the safepoint code 481 __ remove_frame(initial_frame_size_in_bytes()); 482 483 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) { 484 __ reserved_stack_check(); 485 } 486 487 code_stub->set_safepoint_offset(__ offset()); 488 __ relocate(relocInfo::poll_return_type); 489 __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */); 490 __ ret(lr); 491 } 492 493 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 494 guarantee(info != nullptr, "Shouldn't be null"); 495 __ get_polling_page(rscratch1, relocInfo::poll_type); 496 add_debug_info_for_branch(info); // This isn't just debug info: 497 // it's the oop map 498 __ read_polling_page(rscratch1, relocInfo::poll_type); 499 return __ offset(); 500 } 501 502 503 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 504 if (from_reg == r31_sp) 505 from_reg = sp; 506 if (to_reg == r31_sp) 507 to_reg = sp; 508 __ mov(to_reg, from_reg); 509 } 510 511 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); } 512 513 514 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 515 assert(src->is_constant(), "should not call otherwise"); 516 assert(dest->is_register(), "should not call otherwise"); 517 LIR_Const* c = src->as_constant_ptr(); 518 519 switch (c->type()) { 520 case T_INT: { 521 assert(patch_code == lir_patch_none, "no patching handled here"); 522 __ movw(dest->as_register(), c->as_jint()); 523 break; 524 } 525 526 case T_ADDRESS: { 527 assert(patch_code == lir_patch_none, "no patching handled here"); 528 __ mov(dest->as_register(), c->as_jint()); 529 break; 530 } 531 532 case T_LONG: { 533 assert(patch_code == lir_patch_none, "no patching handled here"); 534 if (SCCache::is_on_for_write()) { 535 // SCA needs relocation info for card table base 536 address b = c->as_pointer(); 537 if (is_card_table_address(b)) { 538 __ lea(dest->as_register_lo(), ExternalAddress(b)); 539 break; 540 } 541 } 542 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong()); 543 break; 544 } 545 546 case T_OBJECT: { 547 if (patch_code == lir_patch_none) { 548 jobject2reg(c->as_jobject(), dest->as_register()); 549 } else { 550 jobject2reg_with_patching(dest->as_register(), info); 551 } 552 break; 553 } 554 555 case T_METADATA: { 556 if (patch_code != lir_patch_none) { 557 klass2reg_with_patching(dest->as_register(), info); 558 } else { 559 __ mov_metadata(dest->as_register(), c->as_metadata()); 560 } 561 break; 562 } 563 564 case T_FLOAT: { 565 if (__ operand_valid_for_float_immediate(c->as_jfloat())) { 566 __ fmovs(dest->as_float_reg(), (c->as_jfloat())); 567 } else { 568 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat()))); 569 __ ldrs(dest->as_float_reg(), Address(rscratch1)); 570 } 571 break; 572 } 573 574 case T_DOUBLE: { 575 if (__ operand_valid_for_float_immediate(c->as_jdouble())) { 576 __ fmovd(dest->as_double_reg(), (c->as_jdouble())); 577 } else { 578 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble()))); 579 __ ldrd(dest->as_double_reg(), Address(rscratch1)); 580 } 581 break; 582 } 583 584 default: 585 ShouldNotReachHere(); 586 } 587 } 588 589 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 590 LIR_Const* c = src->as_constant_ptr(); 591 switch (c->type()) { 592 case T_OBJECT: 593 { 594 if (! c->as_jobject()) 595 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 596 else { 597 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr); 598 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 599 } 600 } 601 break; 602 case T_ADDRESS: 603 { 604 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr); 605 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false); 606 } 607 case T_INT: 608 case T_FLOAT: 609 { 610 Register reg = zr; 611 if (c->as_jint_bits() == 0) 612 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 613 else { 614 __ movw(rscratch1, c->as_jint_bits()); 615 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix())); 616 } 617 } 618 break; 619 case T_LONG: 620 case T_DOUBLE: 621 { 622 Register reg = zr; 623 if (c->as_jlong_bits() == 0) 624 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(), 625 lo_word_offset_in_bytes)); 626 else { 627 __ mov(rscratch1, (intptr_t)c->as_jlong_bits()); 628 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(), 629 lo_word_offset_in_bytes)); 630 } 631 } 632 break; 633 default: 634 ShouldNotReachHere(); 635 } 636 } 637 638 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 639 assert(src->is_constant(), "should not call otherwise"); 640 LIR_Const* c = src->as_constant_ptr(); 641 LIR_Address* to_addr = dest->as_address_ptr(); 642 643 void (Assembler::* insn)(Register Rt, const Address &adr); 644 645 switch (type) { 646 case T_ADDRESS: 647 assert(c->as_jint() == 0, "should be"); 648 insn = &Assembler::str; 649 break; 650 case T_LONG: 651 assert(c->as_jlong() == 0, "should be"); 652 insn = &Assembler::str; 653 break; 654 case T_INT: 655 assert(c->as_jint() == 0, "should be"); 656 insn = &Assembler::strw; 657 break; 658 case T_OBJECT: 659 case T_ARRAY: 660 assert(c->as_jobject() == 0, "should be"); 661 if (UseCompressedOops && !wide) { 662 insn = &Assembler::strw; 663 } else { 664 insn = &Assembler::str; 665 } 666 break; 667 case T_CHAR: 668 case T_SHORT: 669 assert(c->as_jint() == 0, "should be"); 670 insn = &Assembler::strh; 671 break; 672 case T_BOOLEAN: 673 case T_BYTE: 674 assert(c->as_jint() == 0, "should be"); 675 insn = &Assembler::strb; 676 break; 677 default: 678 ShouldNotReachHere(); 679 insn = &Assembler::str; // unreachable 680 } 681 682 if (info) add_debug_info_for_null_check_here(info); 683 (_masm->*insn)(zr, as_Address(to_addr, rscratch1)); 684 } 685 686 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 687 assert(src->is_register(), "should not call otherwise"); 688 assert(dest->is_register(), "should not call otherwise"); 689 690 // move between cpu-registers 691 if (dest->is_single_cpu()) { 692 if (src->type() == T_LONG) { 693 // Can do LONG -> OBJECT 694 move_regs(src->as_register_lo(), dest->as_register()); 695 return; 696 } 697 assert(src->is_single_cpu(), "must match"); 698 if (src->type() == T_OBJECT) { 699 __ verify_oop(src->as_register()); 700 } 701 move_regs(src->as_register(), dest->as_register()); 702 703 } else if (dest->is_double_cpu()) { 704 if (is_reference_type(src->type())) { 705 // Surprising to me but we can see move of a long to t_object 706 __ verify_oop(src->as_register()); 707 move_regs(src->as_register(), dest->as_register_lo()); 708 return; 709 } 710 assert(src->is_double_cpu(), "must match"); 711 Register f_lo = src->as_register_lo(); 712 Register f_hi = src->as_register_hi(); 713 Register t_lo = dest->as_register_lo(); 714 Register t_hi = dest->as_register_hi(); 715 assert(f_hi == f_lo, "must be same"); 716 assert(t_hi == t_lo, "must be same"); 717 move_regs(f_lo, t_lo); 718 719 } else if (dest->is_single_fpu()) { 720 __ fmovs(dest->as_float_reg(), src->as_float_reg()); 721 722 } else if (dest->is_double_fpu()) { 723 __ fmovd(dest->as_double_reg(), src->as_double_reg()); 724 725 } else { 726 ShouldNotReachHere(); 727 } 728 } 729 730 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 731 precond(src->is_register() && dest->is_stack()); 732 733 uint const c_sz32 = sizeof(uint32_t); 734 uint const c_sz64 = sizeof(uint64_t); 735 736 if (src->is_single_cpu()) { 737 int index = dest->single_stack_ix(); 738 if (is_reference_type(type)) { 739 __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1)); 740 __ verify_oop(src->as_register()); 741 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) { 742 __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1)); 743 } else { 744 __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1)); 745 } 746 747 } else if (src->is_double_cpu()) { 748 int index = dest->double_stack_ix(); 749 Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes); 750 __ str(src->as_register_lo(), dest_addr_LO); 751 752 } else if (src->is_single_fpu()) { 753 int index = dest->single_stack_ix(); 754 __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1)); 755 756 } else if (src->is_double_fpu()) { 757 int index = dest->double_stack_ix(); 758 __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1)); 759 760 } else { 761 ShouldNotReachHere(); 762 } 763 } 764 765 766 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) { 767 LIR_Address* to_addr = dest->as_address_ptr(); 768 PatchingStub* patch = nullptr; 769 Register compressed_src = rscratch1; 770 771 if (patch_code != lir_patch_none) { 772 deoptimize_trap(info); 773 return; 774 } 775 776 if (is_reference_type(type)) { 777 __ verify_oop(src->as_register()); 778 779 if (UseCompressedOops && !wide) { 780 __ encode_heap_oop(compressed_src, src->as_register()); 781 } else { 782 compressed_src = src->as_register(); 783 } 784 } 785 786 int null_check_here = code_offset(); 787 switch (type) { 788 case T_FLOAT: { 789 __ strs(src->as_float_reg(), as_Address(to_addr)); 790 break; 791 } 792 793 case T_DOUBLE: { 794 __ strd(src->as_double_reg(), as_Address(to_addr)); 795 break; 796 } 797 798 case T_ARRAY: // fall through 799 case T_OBJECT: // fall through 800 if (UseCompressedOops && !wide) { 801 __ strw(compressed_src, as_Address(to_addr, rscratch2)); 802 } else { 803 __ str(compressed_src, as_Address(to_addr)); 804 } 805 break; 806 case T_METADATA: 807 // We get here to store a method pointer to the stack to pass to 808 // a dtrace runtime call. This can't work on 64 bit with 809 // compressed klass ptrs: T_METADATA can be a compressed klass 810 // ptr or a 64 bit method pointer. 811 ShouldNotReachHere(); 812 __ str(src->as_register(), as_Address(to_addr)); 813 break; 814 case T_ADDRESS: 815 __ str(src->as_register(), as_Address(to_addr)); 816 break; 817 case T_INT: 818 __ strw(src->as_register(), as_Address(to_addr)); 819 break; 820 821 case T_LONG: { 822 __ str(src->as_register_lo(), as_Address_lo(to_addr)); 823 break; 824 } 825 826 case T_BYTE: // fall through 827 case T_BOOLEAN: { 828 __ strb(src->as_register(), as_Address(to_addr)); 829 break; 830 } 831 832 case T_CHAR: // fall through 833 case T_SHORT: 834 __ strh(src->as_register(), as_Address(to_addr)); 835 break; 836 837 default: 838 ShouldNotReachHere(); 839 } 840 if (info != nullptr) { 841 add_debug_info_for_null_check(null_check_here, info); 842 } 843 } 844 845 846 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 847 precond(src->is_stack() && dest->is_register()); 848 849 uint const c_sz32 = sizeof(uint32_t); 850 uint const c_sz64 = sizeof(uint64_t); 851 852 if (dest->is_single_cpu()) { 853 int index = src->single_stack_ix(); 854 if (is_reference_type(type)) { 855 __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1)); 856 __ verify_oop(dest->as_register()); 857 } else if (type == T_METADATA || type == T_ADDRESS) { 858 __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1)); 859 } else { 860 __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1)); 861 } 862 863 } else if (dest->is_double_cpu()) { 864 int index = src->double_stack_ix(); 865 Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes); 866 __ ldr(dest->as_register_lo(), src_addr_LO); 867 868 } else if (dest->is_single_fpu()) { 869 int index = src->single_stack_ix(); 870 __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1)); 871 872 } else if (dest->is_double_fpu()) { 873 int index = src->double_stack_ix(); 874 __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1)); 875 876 } else { 877 ShouldNotReachHere(); 878 } 879 } 880 881 882 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 883 address target = nullptr; 884 relocInfo::relocType reloc_type = relocInfo::none; 885 886 switch (patching_id(info)) { 887 case PatchingStub::access_field_id: 888 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 889 reloc_type = relocInfo::section_word_type; 890 break; 891 case PatchingStub::load_klass_id: 892 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 893 reloc_type = relocInfo::metadata_type; 894 break; 895 case PatchingStub::load_mirror_id: 896 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 897 reloc_type = relocInfo::oop_type; 898 break; 899 case PatchingStub::load_appendix_id: 900 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 901 reloc_type = relocInfo::oop_type; 902 break; 903 default: ShouldNotReachHere(); 904 } 905 906 __ far_call(RuntimeAddress(target)); 907 add_call_info_here(info); 908 } 909 910 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 911 912 LIR_Opr temp; 913 if (type == T_LONG || type == T_DOUBLE) 914 temp = FrameMap::rscratch1_long_opr; 915 else 916 temp = FrameMap::rscratch1_opr; 917 918 stack2reg(src, temp, src->type()); 919 reg2stack(temp, dest, dest->type(), false); 920 } 921 922 923 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) { 924 LIR_Address* addr = src->as_address_ptr(); 925 LIR_Address* from_addr = src->as_address_ptr(); 926 927 if (addr->base()->type() == T_OBJECT) { 928 __ verify_oop(addr->base()->as_pointer_register()); 929 } 930 931 if (patch_code != lir_patch_none) { 932 deoptimize_trap(info); 933 return; 934 } 935 936 if (info != nullptr) { 937 add_debug_info_for_null_check_here(info); 938 } 939 int null_check_here = code_offset(); 940 switch (type) { 941 case T_FLOAT: { 942 __ ldrs(dest->as_float_reg(), as_Address(from_addr)); 943 break; 944 } 945 946 case T_DOUBLE: { 947 __ ldrd(dest->as_double_reg(), as_Address(from_addr)); 948 break; 949 } 950 951 case T_ARRAY: // fall through 952 case T_OBJECT: // fall through 953 if (UseCompressedOops && !wide) { 954 __ ldrw(dest->as_register(), as_Address(from_addr)); 955 } else { 956 __ ldr(dest->as_register(), as_Address(from_addr)); 957 } 958 break; 959 case T_METADATA: 960 // We get here to store a method pointer to the stack to pass to 961 // a dtrace runtime call. This can't work on 64 bit with 962 // compressed klass ptrs: T_METADATA can be a compressed klass 963 // ptr or a 64 bit method pointer. 964 ShouldNotReachHere(); 965 __ ldr(dest->as_register(), as_Address(from_addr)); 966 break; 967 case T_ADDRESS: 968 __ ldr(dest->as_register(), as_Address(from_addr)); 969 break; 970 case T_INT: 971 __ ldrw(dest->as_register(), as_Address(from_addr)); 972 break; 973 974 case T_LONG: { 975 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr)); 976 break; 977 } 978 979 case T_BYTE: 980 __ ldrsb(dest->as_register(), as_Address(from_addr)); 981 break; 982 case T_BOOLEAN: { 983 __ ldrb(dest->as_register(), as_Address(from_addr)); 984 break; 985 } 986 987 case T_CHAR: 988 __ ldrh(dest->as_register(), as_Address(from_addr)); 989 break; 990 case T_SHORT: 991 __ ldrsh(dest->as_register(), as_Address(from_addr)); 992 break; 993 994 default: 995 ShouldNotReachHere(); 996 } 997 998 if (is_reference_type(type)) { 999 if (UseCompressedOops && !wide) { 1000 __ decode_heap_oop(dest->as_register()); 1001 } 1002 1003 if (!(UseZGC && !ZGenerational)) { 1004 // Load barrier has not yet been applied, so ZGC can't verify the oop here 1005 __ verify_oop(dest->as_register()); 1006 } 1007 } 1008 } 1009 1010 1011 int LIR_Assembler::array_element_size(BasicType type) const { 1012 int elem_size = type2aelembytes(type); 1013 return exact_log2(elem_size); 1014 } 1015 1016 1017 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1018 switch (op->code()) { 1019 case lir_idiv: 1020 case lir_irem: 1021 arithmetic_idiv(op->code(), 1022 op->in_opr1(), 1023 op->in_opr2(), 1024 op->in_opr3(), 1025 op->result_opr(), 1026 op->info()); 1027 break; 1028 case lir_fmad: 1029 __ fmaddd(op->result_opr()->as_double_reg(), 1030 op->in_opr1()->as_double_reg(), 1031 op->in_opr2()->as_double_reg(), 1032 op->in_opr3()->as_double_reg()); 1033 break; 1034 case lir_fmaf: 1035 __ fmadds(op->result_opr()->as_float_reg(), 1036 op->in_opr1()->as_float_reg(), 1037 op->in_opr2()->as_float_reg(), 1038 op->in_opr3()->as_float_reg()); 1039 break; 1040 default: ShouldNotReachHere(); break; 1041 } 1042 } 1043 1044 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1045 #ifdef ASSERT 1046 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label"); 1047 if (op->block() != nullptr) _branch_target_blocks.append(op->block()); 1048 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock()); 1049 #endif 1050 1051 if (op->cond() == lir_cond_always) { 1052 if (op->info() != nullptr) add_debug_info_for_branch(op->info()); 1053 __ b(*(op->label())); 1054 } else { 1055 Assembler::Condition acond; 1056 if (op->code() == lir_cond_float_branch) { 1057 bool is_unordered = (op->ublock() == op->block()); 1058 // Assembler::EQ does not permit unordered branches, so we add 1059 // another branch here. Likewise, Assembler::NE does not permit 1060 // ordered branches. 1061 if ((is_unordered && op->cond() == lir_cond_equal) 1062 || (!is_unordered && op->cond() == lir_cond_notEqual)) 1063 __ br(Assembler::VS, *(op->ublock()->label())); 1064 switch(op->cond()) { 1065 case lir_cond_equal: acond = Assembler::EQ; break; 1066 case lir_cond_notEqual: acond = Assembler::NE; break; 1067 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break; 1068 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break; 1069 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break; 1070 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break; 1071 default: ShouldNotReachHere(); 1072 acond = Assembler::EQ; // unreachable 1073 } 1074 } else { 1075 switch (op->cond()) { 1076 case lir_cond_equal: acond = Assembler::EQ; break; 1077 case lir_cond_notEqual: acond = Assembler::NE; break; 1078 case lir_cond_less: acond = Assembler::LT; break; 1079 case lir_cond_lessEqual: acond = Assembler::LE; break; 1080 case lir_cond_greaterEqual: acond = Assembler::GE; break; 1081 case lir_cond_greater: acond = Assembler::GT; break; 1082 case lir_cond_belowEqual: acond = Assembler::LS; break; 1083 case lir_cond_aboveEqual: acond = Assembler::HS; break; 1084 default: ShouldNotReachHere(); 1085 acond = Assembler::EQ; // unreachable 1086 } 1087 } 1088 __ br(acond,*(op->label())); 1089 } 1090 } 1091 1092 1093 1094 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1095 LIR_Opr src = op->in_opr(); 1096 LIR_Opr dest = op->result_opr(); 1097 1098 switch (op->bytecode()) { 1099 case Bytecodes::_i2f: 1100 { 1101 __ scvtfws(dest->as_float_reg(), src->as_register()); 1102 break; 1103 } 1104 case Bytecodes::_i2d: 1105 { 1106 __ scvtfwd(dest->as_double_reg(), src->as_register()); 1107 break; 1108 } 1109 case Bytecodes::_l2d: 1110 { 1111 __ scvtfd(dest->as_double_reg(), src->as_register_lo()); 1112 break; 1113 } 1114 case Bytecodes::_l2f: 1115 { 1116 __ scvtfs(dest->as_float_reg(), src->as_register_lo()); 1117 break; 1118 } 1119 case Bytecodes::_f2d: 1120 { 1121 __ fcvts(dest->as_double_reg(), src->as_float_reg()); 1122 break; 1123 } 1124 case Bytecodes::_d2f: 1125 { 1126 __ fcvtd(dest->as_float_reg(), src->as_double_reg()); 1127 break; 1128 } 1129 case Bytecodes::_i2c: 1130 { 1131 __ ubfx(dest->as_register(), src->as_register(), 0, 16); 1132 break; 1133 } 1134 case Bytecodes::_i2l: 1135 { 1136 __ sxtw(dest->as_register_lo(), src->as_register()); 1137 break; 1138 } 1139 case Bytecodes::_i2s: 1140 { 1141 __ sxth(dest->as_register(), src->as_register()); 1142 break; 1143 } 1144 case Bytecodes::_i2b: 1145 { 1146 __ sxtb(dest->as_register(), src->as_register()); 1147 break; 1148 } 1149 case Bytecodes::_l2i: 1150 { 1151 _masm->block_comment("FIXME: This could be a no-op"); 1152 __ uxtw(dest->as_register(), src->as_register_lo()); 1153 break; 1154 } 1155 case Bytecodes::_d2l: 1156 { 1157 __ fcvtzd(dest->as_register_lo(), src->as_double_reg()); 1158 break; 1159 } 1160 case Bytecodes::_f2i: 1161 { 1162 __ fcvtzsw(dest->as_register(), src->as_float_reg()); 1163 break; 1164 } 1165 case Bytecodes::_f2l: 1166 { 1167 __ fcvtzs(dest->as_register_lo(), src->as_float_reg()); 1168 break; 1169 } 1170 case Bytecodes::_d2i: 1171 { 1172 __ fcvtzdw(dest->as_register(), src->as_double_reg()); 1173 break; 1174 } 1175 default: ShouldNotReachHere(); 1176 } 1177 } 1178 1179 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1180 if (op->init_check()) { 1181 __ ldrb(rscratch1, Address(op->klass()->as_register(), 1182 InstanceKlass::init_state_offset())); 1183 __ cmpw(rscratch1, InstanceKlass::fully_initialized); 1184 add_debug_info_for_null_check_here(op->stub()->info()); 1185 __ br(Assembler::NE, *op->stub()->entry()); 1186 } 1187 __ allocate_object(op->obj()->as_register(), 1188 op->tmp1()->as_register(), 1189 op->tmp2()->as_register(), 1190 op->header_size(), 1191 op->object_size(), 1192 op->klass()->as_register(), 1193 *op->stub()->entry()); 1194 __ bind(*op->stub()->continuation()); 1195 } 1196 1197 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1198 Register len = op->len()->as_register(); 1199 __ uxtw(len, len); 1200 1201 if (UseSlowPath || 1202 (!UseFastNewObjectArray && is_reference_type(op->type())) || 1203 (!UseFastNewTypeArray && !is_reference_type(op->type()))) { 1204 __ b(*op->stub()->entry()); 1205 } else { 1206 Register tmp1 = op->tmp1()->as_register(); 1207 Register tmp2 = op->tmp2()->as_register(); 1208 Register tmp3 = op->tmp3()->as_register(); 1209 if (len == tmp1) { 1210 tmp1 = tmp3; 1211 } else if (len == tmp2) { 1212 tmp2 = tmp3; 1213 } else if (len == tmp3) { 1214 // everything is ok 1215 } else { 1216 __ mov(tmp3, len); 1217 } 1218 __ allocate_array(op->obj()->as_register(), 1219 len, 1220 tmp1, 1221 tmp2, 1222 arrayOopDesc::base_offset_in_bytes(op->type()), 1223 array_element_size(op->type()), 1224 op->klass()->as_register(), 1225 *op->stub()->entry(), 1226 op->zero_array()); 1227 } 1228 __ bind(*op->stub()->continuation()); 1229 } 1230 1231 void LIR_Assembler::type_profile_helper(Register mdo, 1232 ciMethodData *md, ciProfileData *data, 1233 Register recv, Label* update_done) { 1234 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1235 Label next_test; 1236 // See if the receiver is receiver[n]. 1237 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1238 __ ldr(rscratch1, Address(rscratch2)); 1239 __ cmp(recv, rscratch1); 1240 __ br(Assembler::NE, next_test); 1241 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); 1242 __ addptr(data_addr, DataLayout::counter_increment); 1243 __ b(*update_done); 1244 __ bind(next_test); 1245 } 1246 1247 // Didn't find receiver; find next empty slot and fill it in 1248 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1249 Label next_test; 1250 __ lea(rscratch2, 1251 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1252 Address recv_addr(rscratch2); 1253 __ ldr(rscratch1, recv_addr); 1254 __ cbnz(rscratch1, next_test); 1255 __ str(recv, recv_addr); 1256 __ mov(rscratch1, DataLayout::counter_increment); 1257 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)))); 1258 __ str(rscratch1, Address(rscratch2)); 1259 __ b(*update_done); 1260 __ bind(next_test); 1261 } 1262 } 1263 1264 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1265 // we always need a stub for the failure case. 1266 CodeStub* stub = op->stub(); 1267 Register obj = op->object()->as_register(); 1268 Register k_RInfo = op->tmp1()->as_register(); 1269 Register klass_RInfo = op->tmp2()->as_register(); 1270 Register dst = op->result_opr()->as_register(); 1271 ciKlass* k = op->klass(); 1272 Register Rtmp1 = noreg; 1273 1274 // check if it needs to be profiled 1275 ciMethodData* md; 1276 ciProfileData* data; 1277 1278 const bool should_profile = op->should_profile(); 1279 1280 if (should_profile) { 1281 ciMethod* method = op->profiled_method(); 1282 assert(method != nullptr, "Should have method"); 1283 int bci = op->profiled_bci(); 1284 md = method->method_data_or_null(); 1285 assert(md != nullptr, "Sanity"); 1286 data = md->bci_to_data(bci); 1287 assert(data != nullptr, "need data for type check"); 1288 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1289 } 1290 Label* success_target = success; 1291 Label* failure_target = failure; 1292 1293 if (obj == k_RInfo) { 1294 k_RInfo = dst; 1295 } else if (obj == klass_RInfo) { 1296 klass_RInfo = dst; 1297 } 1298 if (k->is_loaded() && !UseCompressedClassPointers) { 1299 select_different_registers(obj, dst, k_RInfo, klass_RInfo); 1300 } else { 1301 Rtmp1 = op->tmp3()->as_register(); 1302 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1303 } 1304 1305 assert_different_registers(obj, k_RInfo, klass_RInfo); 1306 1307 if (should_profile) { 1308 Register mdo = klass_RInfo; 1309 __ mov_metadata(mdo, md->constant_encoding()); 1310 Label not_null; 1311 __ cbnz(obj, not_null); 1312 // Object is null; update MDO and exit 1313 Address data_addr 1314 = __ form_address(rscratch2, mdo, 1315 md->byte_offset_of_slot(data, DataLayout::flags_offset()), 1316 0); 1317 __ ldrb(rscratch1, data_addr); 1318 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant()); 1319 __ strb(rscratch1, data_addr); 1320 __ b(*obj_is_null); 1321 __ bind(not_null); 1322 1323 Label update_done; 1324 Register recv = k_RInfo; 1325 __ load_klass(recv, obj); 1326 type_profile_helper(mdo, md, data, recv, &update_done); 1327 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1328 __ addptr(counter_addr, DataLayout::counter_increment); 1329 1330 __ bind(update_done); 1331 } else { 1332 __ cbz(obj, *obj_is_null); 1333 } 1334 1335 if (!k->is_loaded()) { 1336 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1337 } else { 1338 __ mov_metadata(k_RInfo, k->constant_encoding()); 1339 } 1340 __ verify_oop(obj); 1341 1342 if (op->fast_check()) { 1343 // get object class 1344 // not a safepoint as obj null check happens earlier 1345 __ load_klass(rscratch1, obj); 1346 __ cmp( rscratch1, k_RInfo); 1347 1348 __ br(Assembler::NE, *failure_target); 1349 // successful cast, fall through to profile or jump 1350 } else { 1351 // get object class 1352 // not a safepoint as obj null check happens earlier 1353 __ load_klass(klass_RInfo, obj); 1354 if (k->is_loaded()) { 1355 // See if we get an immediate positive hit 1356 __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset()))); 1357 __ cmp(k_RInfo, rscratch1); 1358 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1359 __ br(Assembler::NE, *failure_target); 1360 // successful cast, fall through to profile or jump 1361 } else { 1362 // See if we get an immediate positive hit 1363 __ br(Assembler::EQ, *success_target); 1364 // check for self 1365 __ cmp(klass_RInfo, k_RInfo); 1366 __ br(Assembler::EQ, *success_target); 1367 1368 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1369 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1370 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1371 // result is a boolean 1372 __ cbzw(klass_RInfo, *failure_target); 1373 // successful cast, fall through to profile or jump 1374 } 1375 } else { 1376 // perform the fast part of the checking logic 1377 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr); 1378 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1379 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1380 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1381 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1382 // result is a boolean 1383 __ cbz(k_RInfo, *failure_target); 1384 // successful cast, fall through to profile or jump 1385 } 1386 } 1387 __ b(*success); 1388 } 1389 1390 1391 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1392 const bool should_profile = op->should_profile(); 1393 1394 LIR_Code code = op->code(); 1395 if (code == lir_store_check) { 1396 Register value = op->object()->as_register(); 1397 Register array = op->array()->as_register(); 1398 Register k_RInfo = op->tmp1()->as_register(); 1399 Register klass_RInfo = op->tmp2()->as_register(); 1400 Register Rtmp1 = op->tmp3()->as_register(); 1401 1402 CodeStub* stub = op->stub(); 1403 1404 // check if it needs to be profiled 1405 ciMethodData* md; 1406 ciProfileData* data; 1407 1408 if (should_profile) { 1409 ciMethod* method = op->profiled_method(); 1410 assert(method != nullptr, "Should have method"); 1411 int bci = op->profiled_bci(); 1412 md = method->method_data_or_null(); 1413 assert(md != nullptr, "Sanity"); 1414 data = md->bci_to_data(bci); 1415 assert(data != nullptr, "need data for type check"); 1416 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1417 } 1418 Label done; 1419 Label* success_target = &done; 1420 Label* failure_target = stub->entry(); 1421 1422 if (should_profile) { 1423 Label not_null; 1424 Register mdo = klass_RInfo; 1425 __ mov_metadata(mdo, md->constant_encoding()); 1426 __ cbnz(value, not_null); 1427 // Object is null; update MDO and exit 1428 Address data_addr 1429 = __ form_address(rscratch2, mdo, 1430 md->byte_offset_of_slot(data, DataLayout::flags_offset()), 1431 0); 1432 __ ldrb(rscratch1, data_addr); 1433 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant()); 1434 __ strb(rscratch1, data_addr); 1435 __ b(done); 1436 __ bind(not_null); 1437 1438 Label update_done; 1439 Register recv = k_RInfo; 1440 __ load_klass(recv, value); 1441 type_profile_helper(mdo, md, data, recv, &update_done); 1442 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1443 __ addptr(counter_addr, DataLayout::counter_increment); 1444 __ bind(update_done); 1445 } else { 1446 __ cbz(value, done); 1447 } 1448 1449 add_debug_info_for_null_check_here(op->info_for_exception()); 1450 __ load_klass(k_RInfo, array); 1451 __ load_klass(klass_RInfo, value); 1452 1453 // get instance klass (it's already uncompressed) 1454 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1455 // perform the fast part of the checking logic 1456 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr); 1457 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1458 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize))); 1459 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1460 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize))); 1461 // result is a boolean 1462 __ cbzw(k_RInfo, *failure_target); 1463 // fall through to the success case 1464 1465 __ bind(done); 1466 } else if (code == lir_checkcast) { 1467 Register obj = op->object()->as_register(); 1468 Register dst = op->result_opr()->as_register(); 1469 Label success; 1470 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1471 __ bind(success); 1472 if (dst != obj) { 1473 __ mov(dst, obj); 1474 } 1475 } else if (code == lir_instanceof) { 1476 Register obj = op->object()->as_register(); 1477 Register dst = op->result_opr()->as_register(); 1478 Label success, failure, done; 1479 emit_typecheck_helper(op, &success, &failure, &failure); 1480 __ bind(failure); 1481 __ mov(dst, zr); 1482 __ b(done); 1483 __ bind(success); 1484 __ mov(dst, 1); 1485 __ bind(done); 1486 } else { 1487 ShouldNotReachHere(); 1488 } 1489 } 1490 1491 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { 1492 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); 1493 __ cset(rscratch1, Assembler::NE); 1494 __ membar(__ AnyAny); 1495 } 1496 1497 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { 1498 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1); 1499 __ cset(rscratch1, Assembler::NE); 1500 __ membar(__ AnyAny); 1501 } 1502 1503 1504 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1505 Register addr; 1506 if (op->addr()->is_register()) { 1507 addr = as_reg(op->addr()); 1508 } else { 1509 assert(op->addr()->is_address(), "what else?"); 1510 LIR_Address* addr_ptr = op->addr()->as_address_ptr(); 1511 assert(addr_ptr->disp() == 0, "need 0 disp"); 1512 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index"); 1513 addr = as_reg(addr_ptr->base()); 1514 } 1515 Register newval = as_reg(op->new_value()); 1516 Register cmpval = as_reg(op->cmp_value()); 1517 1518 if (op->code() == lir_cas_obj) { 1519 if (UseCompressedOops) { 1520 Register t1 = op->tmp1()->as_register(); 1521 assert(op->tmp1()->is_valid(), "must be"); 1522 __ encode_heap_oop(t1, cmpval); 1523 cmpval = t1; 1524 __ encode_heap_oop(rscratch2, newval); 1525 newval = rscratch2; 1526 casw(addr, newval, cmpval); 1527 } else { 1528 casl(addr, newval, cmpval); 1529 } 1530 } else if (op->code() == lir_cas_int) { 1531 casw(addr, newval, cmpval); 1532 } else { 1533 casl(addr, newval, cmpval); 1534 } 1535 } 1536 1537 1538 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type, 1539 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) { 1540 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on aarch64"); 1541 1542 Assembler::Condition acond, ncond; 1543 switch (condition) { 1544 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break; 1545 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break; 1546 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break; 1547 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break; 1548 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break; 1549 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break; 1550 case lir_cond_belowEqual: 1551 case lir_cond_aboveEqual: 1552 default: ShouldNotReachHere(); 1553 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable 1554 } 1555 1556 assert(result->is_single_cpu() || result->is_double_cpu(), 1557 "expect single register for result"); 1558 if (opr1->is_constant() && opr2->is_constant() 1559 && opr1->type() == T_INT && opr2->type() == T_INT) { 1560 jint val1 = opr1->as_jint(); 1561 jint val2 = opr2->as_jint(); 1562 if (val1 == 0 && val2 == 1) { 1563 __ cset(result->as_register(), ncond); 1564 return; 1565 } else if (val1 == 1 && val2 == 0) { 1566 __ cset(result->as_register(), acond); 1567 return; 1568 } 1569 } 1570 1571 if (opr1->is_constant() && opr2->is_constant() 1572 && opr1->type() == T_LONG && opr2->type() == T_LONG) { 1573 jlong val1 = opr1->as_jlong(); 1574 jlong val2 = opr2->as_jlong(); 1575 if (val1 == 0 && val2 == 1) { 1576 __ cset(result->as_register_lo(), ncond); 1577 return; 1578 } else if (val1 == 1 && val2 == 0) { 1579 __ cset(result->as_register_lo(), acond); 1580 return; 1581 } 1582 } 1583 1584 if (opr1->is_stack()) { 1585 stack2reg(opr1, FrameMap::rscratch1_opr, result->type()); 1586 opr1 = FrameMap::rscratch1_opr; 1587 } else if (opr1->is_constant()) { 1588 LIR_Opr tmp 1589 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr; 1590 const2reg(opr1, tmp, lir_patch_none, nullptr); 1591 opr1 = tmp; 1592 } 1593 1594 if (opr2->is_stack()) { 1595 stack2reg(opr2, FrameMap::rscratch2_opr, result->type()); 1596 opr2 = FrameMap::rscratch2_opr; 1597 } else if (opr2->is_constant()) { 1598 LIR_Opr tmp 1599 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr; 1600 const2reg(opr2, tmp, lir_patch_none, nullptr); 1601 opr2 = tmp; 1602 } 1603 1604 if (result->type() == T_LONG) 1605 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond); 1606 else 1607 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond); 1608 } 1609 1610 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 1611 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method"); 1612 1613 if (left->is_single_cpu()) { 1614 Register lreg = left->as_register(); 1615 Register dreg = as_reg(dest); 1616 1617 if (right->is_single_cpu()) { 1618 // cpu register - cpu register 1619 1620 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT, 1621 "should be"); 1622 Register rreg = right->as_register(); 1623 switch (code) { 1624 case lir_add: __ addw (dest->as_register(), lreg, rreg); break; 1625 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break; 1626 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break; 1627 default: ShouldNotReachHere(); 1628 } 1629 1630 } else if (right->is_double_cpu()) { 1631 Register rreg = right->as_register_lo(); 1632 // single_cpu + double_cpu: can happen with obj+long 1633 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1634 switch (code) { 1635 case lir_add: __ add(dreg, lreg, rreg); break; 1636 case lir_sub: __ sub(dreg, lreg, rreg); break; 1637 default: ShouldNotReachHere(); 1638 } 1639 } else if (right->is_constant()) { 1640 // cpu register - constant 1641 jlong c; 1642 1643 // FIXME. This is fugly: we really need to factor all this logic. 1644 switch(right->type()) { 1645 case T_LONG: 1646 c = right->as_constant_ptr()->as_jlong(); 1647 break; 1648 case T_INT: 1649 case T_ADDRESS: 1650 c = right->as_constant_ptr()->as_jint(); 1651 break; 1652 default: 1653 ShouldNotReachHere(); 1654 c = 0; // unreachable 1655 break; 1656 } 1657 1658 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op"); 1659 if (c == 0 && dreg == lreg) { 1660 COMMENT("effective nop elided"); 1661 return; 1662 } 1663 switch(left->type()) { 1664 case T_INT: 1665 switch (code) { 1666 case lir_add: __ addw(dreg, lreg, c); break; 1667 case lir_sub: __ subw(dreg, lreg, c); break; 1668 default: ShouldNotReachHere(); 1669 } 1670 break; 1671 case T_OBJECT: 1672 case T_ADDRESS: 1673 switch (code) { 1674 case lir_add: __ add(dreg, lreg, c); break; 1675 case lir_sub: __ sub(dreg, lreg, c); break; 1676 default: ShouldNotReachHere(); 1677 } 1678 break; 1679 default: 1680 ShouldNotReachHere(); 1681 } 1682 } else { 1683 ShouldNotReachHere(); 1684 } 1685 1686 } else if (left->is_double_cpu()) { 1687 Register lreg_lo = left->as_register_lo(); 1688 1689 if (right->is_double_cpu()) { 1690 // cpu register - cpu register 1691 Register rreg_lo = right->as_register_lo(); 1692 switch (code) { 1693 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1694 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1695 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break; 1696 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break; 1697 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break; 1698 default: 1699 ShouldNotReachHere(); 1700 } 1701 1702 } else if (right->is_constant()) { 1703 jlong c = right->as_constant_ptr()->as_jlong(); 1704 Register dreg = as_reg(dest); 1705 switch (code) { 1706 case lir_add: 1707 case lir_sub: 1708 if (c == 0 && dreg == lreg_lo) { 1709 COMMENT("effective nop elided"); 1710 return; 1711 } 1712 code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c); 1713 break; 1714 case lir_div: 1715 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant"); 1716 if (c == 1) { 1717 // move lreg_lo to dreg if divisor is 1 1718 __ mov(dreg, lreg_lo); 1719 } else { 1720 unsigned int shift = log2i_exact(c); 1721 // use rscratch1 as intermediate result register 1722 __ asr(rscratch1, lreg_lo, 63); 1723 __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift); 1724 __ asr(dreg, rscratch1, shift); 1725 } 1726 break; 1727 case lir_rem: 1728 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant"); 1729 if (c == 1) { 1730 // move 0 to dreg if divisor is 1 1731 __ mov(dreg, zr); 1732 } else { 1733 // use rscratch1 as intermediate result register 1734 __ negs(rscratch1, lreg_lo); 1735 __ andr(dreg, lreg_lo, c - 1); 1736 __ andr(rscratch1, rscratch1, c - 1); 1737 __ csneg(dreg, dreg, rscratch1, Assembler::MI); 1738 } 1739 break; 1740 default: 1741 ShouldNotReachHere(); 1742 } 1743 } else { 1744 ShouldNotReachHere(); 1745 } 1746 } else if (left->is_single_fpu()) { 1747 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register"); 1748 switch (code) { 1749 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1750 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1751 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1752 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break; 1753 default: 1754 ShouldNotReachHere(); 1755 } 1756 } else if (left->is_double_fpu()) { 1757 if (right->is_double_fpu()) { 1758 // fpu register - fpu register 1759 switch (code) { 1760 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1761 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1762 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1763 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break; 1764 default: 1765 ShouldNotReachHere(); 1766 } 1767 } else { 1768 if (right->is_constant()) { 1769 ShouldNotReachHere(); 1770 } 1771 ShouldNotReachHere(); 1772 } 1773 } else if (left->is_single_stack() || left->is_address()) { 1774 assert(left == dest, "left and dest must be equal"); 1775 ShouldNotReachHere(); 1776 } else { 1777 ShouldNotReachHere(); 1778 } 1779 } 1780 1781 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); } 1782 1783 1784 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) { 1785 switch(code) { 1786 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break; 1787 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break; 1788 case lir_f2hf: __ flt_to_flt16(dest->as_register(), value->as_float_reg(), tmp->as_float_reg()); break; 1789 case lir_hf2f: __ flt16_to_flt(dest->as_float_reg(), value->as_register(), tmp->as_float_reg()); break; 1790 default : ShouldNotReachHere(); 1791 } 1792 } 1793 1794 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 1795 1796 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register"); 1797 Register Rleft = left->is_single_cpu() ? left->as_register() : 1798 left->as_register_lo(); 1799 if (dst->is_single_cpu()) { 1800 Register Rdst = dst->as_register(); 1801 if (right->is_constant()) { 1802 switch (code) { 1803 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break; 1804 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break; 1805 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break; 1806 default: ShouldNotReachHere(); break; 1807 } 1808 } else { 1809 Register Rright = right->is_single_cpu() ? right->as_register() : 1810 right->as_register_lo(); 1811 switch (code) { 1812 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break; 1813 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break; 1814 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break; 1815 default: ShouldNotReachHere(); break; 1816 } 1817 } 1818 } else { 1819 Register Rdst = dst->as_register_lo(); 1820 if (right->is_constant()) { 1821 switch (code) { 1822 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break; 1823 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break; 1824 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break; 1825 default: ShouldNotReachHere(); break; 1826 } 1827 } else { 1828 Register Rright = right->is_single_cpu() ? right->as_register() : 1829 right->as_register_lo(); 1830 switch (code) { 1831 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break; 1832 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break; 1833 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break; 1834 default: ShouldNotReachHere(); break; 1835 } 1836 } 1837 } 1838 } 1839 1840 1841 1842 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) { 1843 1844 // opcode check 1845 assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem"); 1846 bool is_irem = (code == lir_irem); 1847 1848 // operand check 1849 assert(left->is_single_cpu(), "left must be register"); 1850 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant"); 1851 assert(result->is_single_cpu(), "result must be register"); 1852 Register lreg = left->as_register(); 1853 Register dreg = result->as_register(); 1854 1855 // power-of-2 constant check and codegen 1856 if (right->is_constant()) { 1857 int c = right->as_constant_ptr()->as_jint(); 1858 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant"); 1859 if (is_irem) { 1860 if (c == 1) { 1861 // move 0 to dreg if divisor is 1 1862 __ movw(dreg, zr); 1863 } else { 1864 // use rscratch1 as intermediate result register 1865 __ negsw(rscratch1, lreg); 1866 __ andw(dreg, lreg, c - 1); 1867 __ andw(rscratch1, rscratch1, c - 1); 1868 __ csnegw(dreg, dreg, rscratch1, Assembler::MI); 1869 } 1870 } else { 1871 if (c == 1) { 1872 // move lreg to dreg if divisor is 1 1873 __ movw(dreg, lreg); 1874 } else { 1875 unsigned int shift = exact_log2(c); 1876 // use rscratch1 as intermediate result register 1877 __ asrw(rscratch1, lreg, 31); 1878 __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift); 1879 __ asrw(dreg, rscratch1, shift); 1880 } 1881 } 1882 } else { 1883 Register rreg = right->as_register(); 1884 __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1); 1885 } 1886 } 1887 1888 1889 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 1890 if (opr1->is_constant() && opr2->is_single_cpu()) { 1891 // tableswitch 1892 Register reg = as_reg(opr2); 1893 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()]; 1894 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after); 1895 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) { 1896 Register reg1 = as_reg(opr1); 1897 if (opr2->is_single_cpu()) { 1898 // cpu register - cpu register 1899 Register reg2 = opr2->as_register(); 1900 if (is_reference_type(opr1->type())) { 1901 __ cmpoop(reg1, reg2); 1902 } else { 1903 assert(!is_reference_type(opr2->type()), "cmp int, oop?"); 1904 __ cmpw(reg1, reg2); 1905 } 1906 return; 1907 } 1908 if (opr2->is_double_cpu()) { 1909 // cpu register - cpu register 1910 Register reg2 = opr2->as_register_lo(); 1911 __ cmp(reg1, reg2); 1912 return; 1913 } 1914 1915 if (opr2->is_constant()) { 1916 bool is_32bit = false; // width of register operand 1917 jlong imm; 1918 1919 switch(opr2->type()) { 1920 case T_INT: 1921 imm = opr2->as_constant_ptr()->as_jint(); 1922 is_32bit = true; 1923 break; 1924 case T_LONG: 1925 imm = opr2->as_constant_ptr()->as_jlong(); 1926 break; 1927 case T_ADDRESS: 1928 imm = opr2->as_constant_ptr()->as_jint(); 1929 break; 1930 case T_METADATA: 1931 imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata()); 1932 break; 1933 case T_OBJECT: 1934 case T_ARRAY: 1935 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1); 1936 __ cmpoop(reg1, rscratch1); 1937 return; 1938 default: 1939 ShouldNotReachHere(); 1940 imm = 0; // unreachable 1941 break; 1942 } 1943 1944 if (Assembler::operand_valid_for_add_sub_immediate(imm)) { 1945 if (is_32bit) 1946 __ cmpw(reg1, imm); 1947 else 1948 __ subs(zr, reg1, imm); 1949 return; 1950 } else { 1951 __ mov(rscratch1, imm); 1952 if (is_32bit) 1953 __ cmpw(reg1, rscratch1); 1954 else 1955 __ cmp(reg1, rscratch1); 1956 return; 1957 } 1958 } else 1959 ShouldNotReachHere(); 1960 } else if (opr1->is_single_fpu()) { 1961 FloatRegister reg1 = opr1->as_float_reg(); 1962 assert(opr2->is_single_fpu(), "expect single float register"); 1963 FloatRegister reg2 = opr2->as_float_reg(); 1964 __ fcmps(reg1, reg2); 1965 } else if (opr1->is_double_fpu()) { 1966 FloatRegister reg1 = opr1->as_double_reg(); 1967 assert(opr2->is_double_fpu(), "expect double float register"); 1968 FloatRegister reg2 = opr2->as_double_reg(); 1969 __ fcmpd(reg1, reg2); 1970 } else { 1971 ShouldNotReachHere(); 1972 } 1973 } 1974 1975 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){ 1976 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 1977 bool is_unordered_less = (code == lir_ucmp_fd2i); 1978 if (left->is_single_fpu()) { 1979 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register()); 1980 } else if (left->is_double_fpu()) { 1981 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register()); 1982 } else { 1983 ShouldNotReachHere(); 1984 } 1985 } else if (code == lir_cmp_l2i) { 1986 Label done; 1987 __ cmp(left->as_register_lo(), right->as_register_lo()); 1988 __ mov(dst->as_register(), (uint64_t)-1L); 1989 __ br(Assembler::LT, done); 1990 __ csinc(dst->as_register(), zr, zr, Assembler::EQ); 1991 __ bind(done); 1992 } else { 1993 ShouldNotReachHere(); 1994 } 1995 } 1996 1997 1998 void LIR_Assembler::align_call(LIR_Code code) { } 1999 2000 2001 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 2002 address call = __ trampoline_call(Address(op->addr(), rtype)); 2003 if (call == nullptr) { 2004 bailout("trampoline stub overflow"); 2005 return; 2006 } 2007 add_call_info(code_offset(), op->info()); 2008 __ post_call_nop(); 2009 } 2010 2011 2012 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 2013 address call = __ ic_call(op->addr()); 2014 if (call == nullptr) { 2015 bailout("trampoline stub overflow"); 2016 return; 2017 } 2018 add_call_info(code_offset(), op->info()); 2019 __ post_call_nop(); 2020 } 2021 2022 void LIR_Assembler::emit_static_call_stub() { 2023 address call_pc = __ pc(); 2024 address stub = __ start_a_stub(call_stub_size()); 2025 if (stub == nullptr) { 2026 bailout("static call stub overflow"); 2027 return; 2028 } 2029 2030 int start = __ offset(); 2031 2032 __ relocate(static_stub_Relocation::spec(call_pc)); 2033 __ emit_static_call_stub(); 2034 2035 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size() 2036 <= call_stub_size(), "stub too big"); 2037 __ end_a_stub(); 2038 } 2039 2040 2041 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 2042 assert(exceptionOop->as_register() == r0, "must match"); 2043 assert(exceptionPC->as_register() == r3, "must match"); 2044 2045 // exception object is not added to oop map by LinearScan 2046 // (LinearScan assumes that no oops are in fixed registers) 2047 info->add_register_oop(exceptionOop); 2048 Runtime1::StubID unwind_id; 2049 2050 // get current pc information 2051 // pc is only needed if the method has an exception handler, the unwind code does not need it. 2052 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) { 2053 // As no instructions have been generated yet for this LIR node it's 2054 // possible that an oop map already exists for the current offset. 2055 // In that case insert an dummy NOP here to ensure all oop map PCs 2056 // are unique. See JDK-8237483. 2057 __ nop(); 2058 } 2059 int pc_for_athrow_offset = __ offset(); 2060 InternalAddress pc_for_athrow(__ pc()); 2061 __ adr(exceptionPC->as_register(), pc_for_athrow); 2062 add_call_info(pc_for_athrow_offset, info); // for exception handler 2063 2064 __ verify_not_null_oop(r0); 2065 // search an exception handler (r0: exception oop, r3: throwing pc) 2066 if (compilation()->has_fpu_code()) { 2067 unwind_id = Runtime1::handle_exception_id; 2068 } else { 2069 unwind_id = Runtime1::handle_exception_nofpu_id; 2070 } 2071 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 2072 2073 // FIXME: enough room for two byte trap ???? 2074 __ nop(); 2075 } 2076 2077 2078 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 2079 assert(exceptionOop->as_register() == r0, "must match"); 2080 2081 __ b(_unwind_handler_entry); 2082 } 2083 2084 2085 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 2086 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2087 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2088 2089 switch (left->type()) { 2090 case T_INT: { 2091 switch (code) { 2092 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break; 2093 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break; 2094 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break; 2095 default: 2096 ShouldNotReachHere(); 2097 break; 2098 } 2099 break; 2100 case T_LONG: 2101 case T_ADDRESS: 2102 case T_OBJECT: 2103 switch (code) { 2104 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break; 2105 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break; 2106 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break; 2107 default: 2108 ShouldNotReachHere(); 2109 break; 2110 } 2111 break; 2112 default: 2113 ShouldNotReachHere(); 2114 break; 2115 } 2116 } 2117 } 2118 2119 2120 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 2121 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 2122 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 2123 2124 switch (left->type()) { 2125 case T_INT: { 2126 switch (code) { 2127 case lir_shl: __ lslw (dreg, lreg, count); break; 2128 case lir_shr: __ asrw (dreg, lreg, count); break; 2129 case lir_ushr: __ lsrw (dreg, lreg, count); break; 2130 default: 2131 ShouldNotReachHere(); 2132 break; 2133 } 2134 break; 2135 case T_LONG: 2136 case T_ADDRESS: 2137 case T_OBJECT: 2138 switch (code) { 2139 case lir_shl: __ lsl (dreg, lreg, count); break; 2140 case lir_shr: __ asr (dreg, lreg, count); break; 2141 case lir_ushr: __ lsr (dreg, lreg, count); break; 2142 default: 2143 ShouldNotReachHere(); 2144 break; 2145 } 2146 break; 2147 default: 2148 ShouldNotReachHere(); 2149 break; 2150 } 2151 } 2152 } 2153 2154 2155 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 2156 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2157 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2158 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2159 __ str (r, Address(sp, offset_from_rsp_in_bytes)); 2160 } 2161 2162 2163 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 2164 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2165 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2166 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2167 __ mov (rscratch1, c); 2168 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2169 } 2170 2171 2172 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) { 2173 ShouldNotReachHere(); 2174 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2175 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2176 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2177 __ lea(rscratch1, __ constant_oop_address(o)); 2178 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes)); 2179 } 2180 2181 2182 // This code replaces a call to arraycopy; no exception may 2183 // be thrown in this code, they must be thrown in the System.arraycopy 2184 // activation frame; we could save some checks if this would not be the case 2185 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 2186 ciArrayKlass* default_type = op->expected_type(); 2187 Register src = op->src()->as_register(); 2188 Register dst = op->dst()->as_register(); 2189 Register src_pos = op->src_pos()->as_register(); 2190 Register dst_pos = op->dst_pos()->as_register(); 2191 Register length = op->length()->as_register(); 2192 Register tmp = op->tmp()->as_register(); 2193 2194 CodeStub* stub = op->stub(); 2195 int flags = op->flags(); 2196 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL; 2197 if (is_reference_type(basic_type)) basic_type = T_OBJECT; 2198 2199 // if we don't know anything, just go through the generic arraycopy 2200 if (default_type == nullptr // || basic_type == T_OBJECT 2201 ) { 2202 Label done; 2203 assert(src == r1 && src_pos == r2, "mismatch in calling convention"); 2204 2205 // Save the arguments in case the generic arraycopy fails and we 2206 // have to fall back to the JNI stub 2207 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2208 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2209 __ str(src, Address(sp, 4*BytesPerWord)); 2210 2211 address copyfunc_addr = StubRoutines::generic_arraycopy(); 2212 assert(copyfunc_addr != nullptr, "generic arraycopy stub required"); 2213 2214 // The arguments are in java calling convention so we shift them 2215 // to C convention 2216 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4); 2217 __ mov(c_rarg0, j_rarg0); 2218 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4); 2219 __ mov(c_rarg1, j_rarg1); 2220 assert_different_registers(c_rarg2, j_rarg3, j_rarg4); 2221 __ mov(c_rarg2, j_rarg2); 2222 assert_different_registers(c_rarg3, j_rarg4); 2223 __ mov(c_rarg3, j_rarg3); 2224 __ mov(c_rarg4, j_rarg4); 2225 #ifndef PRODUCT 2226 if (PrintC1Statistics) { 2227 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); 2228 } 2229 #endif 2230 __ far_call(RuntimeAddress(copyfunc_addr)); 2231 2232 __ cbz(r0, *stub->continuation()); 2233 2234 // Reload values from the stack so they are where the stub 2235 // expects them. 2236 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2237 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2238 __ ldr(src, Address(sp, 4*BytesPerWord)); 2239 2240 // r0 is -1^K where K == partial copied count 2241 __ eonw(rscratch1, r0, zr); 2242 // adjust length down and src/end pos up by partial copied count 2243 __ subw(length, length, rscratch1); 2244 __ addw(src_pos, src_pos, rscratch1); 2245 __ addw(dst_pos, dst_pos, rscratch1); 2246 __ b(*stub->entry()); 2247 2248 __ bind(*stub->continuation()); 2249 return; 2250 } 2251 2252 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point"); 2253 2254 int elem_size = type2aelembytes(basic_type); 2255 int scale = exact_log2(elem_size); 2256 2257 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); 2258 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); 2259 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); 2260 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); 2261 2262 // test for null 2263 if (flags & LIR_OpArrayCopy::src_null_check) { 2264 __ cbz(src, *stub->entry()); 2265 } 2266 if (flags & LIR_OpArrayCopy::dst_null_check) { 2267 __ cbz(dst, *stub->entry()); 2268 } 2269 2270 // If the compiler was not able to prove that exact type of the source or the destination 2271 // of the arraycopy is an array type, check at runtime if the source or the destination is 2272 // an instance type. 2273 if (flags & LIR_OpArrayCopy::type_check) { 2274 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) { 2275 __ load_klass(tmp, dst); 2276 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2277 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2278 __ br(Assembler::GE, *stub->entry()); 2279 } 2280 2281 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) { 2282 __ load_klass(tmp, src); 2283 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset()))); 2284 __ cmpw(rscratch1, Klass::_lh_neutral_value); 2285 __ br(Assembler::GE, *stub->entry()); 2286 } 2287 } 2288 2289 // check if negative 2290 if (flags & LIR_OpArrayCopy::src_pos_positive_check) { 2291 __ cmpw(src_pos, 0); 2292 __ br(Assembler::LT, *stub->entry()); 2293 } 2294 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) { 2295 __ cmpw(dst_pos, 0); 2296 __ br(Assembler::LT, *stub->entry()); 2297 } 2298 2299 if (flags & LIR_OpArrayCopy::length_positive_check) { 2300 __ cmpw(length, 0); 2301 __ br(Assembler::LT, *stub->entry()); 2302 } 2303 2304 if (flags & LIR_OpArrayCopy::src_range_check) { 2305 __ addw(tmp, src_pos, length); 2306 __ ldrw(rscratch1, src_length_addr); 2307 __ cmpw(tmp, rscratch1); 2308 __ br(Assembler::HI, *stub->entry()); 2309 } 2310 if (flags & LIR_OpArrayCopy::dst_range_check) { 2311 __ addw(tmp, dst_pos, length); 2312 __ ldrw(rscratch1, dst_length_addr); 2313 __ cmpw(tmp, rscratch1); 2314 __ br(Assembler::HI, *stub->entry()); 2315 } 2316 2317 if (flags & LIR_OpArrayCopy::type_check) { 2318 // We don't know the array types are compatible 2319 if (basic_type != T_OBJECT) { 2320 // Simple test for basic type arrays 2321 if (UseCompressedClassPointers) { 2322 __ ldrw(tmp, src_klass_addr); 2323 __ ldrw(rscratch1, dst_klass_addr); 2324 __ cmpw(tmp, rscratch1); 2325 } else { 2326 __ ldr(tmp, src_klass_addr); 2327 __ ldr(rscratch1, dst_klass_addr); 2328 __ cmp(tmp, rscratch1); 2329 } 2330 __ br(Assembler::NE, *stub->entry()); 2331 } else { 2332 // For object arrays, if src is a sub class of dst then we can 2333 // safely do the copy. 2334 Label cont, slow; 2335 2336 #define PUSH(r1, r2) \ 2337 stp(r1, r2, __ pre(sp, -2 * wordSize)); 2338 2339 #define POP(r1, r2) \ 2340 ldp(r1, r2, __ post(sp, 2 * wordSize)); 2341 2342 __ PUSH(src, dst); 2343 2344 __ load_klass(src, src); 2345 __ load_klass(dst, dst); 2346 2347 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr); 2348 2349 __ PUSH(src, dst); 2350 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 2351 __ POP(src, dst); 2352 2353 __ cbnz(src, cont); 2354 2355 __ bind(slow); 2356 __ POP(src, dst); 2357 2358 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 2359 if (copyfunc_addr != nullptr) { // use stub if available 2360 // src is not a sub class of dst so we have to do a 2361 // per-element check. 2362 2363 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 2364 if ((flags & mask) != mask) { 2365 // Check that at least both of them object arrays. 2366 assert(flags & mask, "one of the two should be known to be an object array"); 2367 2368 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 2369 __ load_klass(tmp, src); 2370 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 2371 __ load_klass(tmp, dst); 2372 } 2373 int lh_offset = in_bytes(Klass::layout_helper_offset()); 2374 Address klass_lh_addr(tmp, lh_offset); 2375 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 2376 __ ldrw(rscratch1, klass_lh_addr); 2377 __ mov(rscratch2, objArray_lh); 2378 __ eorw(rscratch1, rscratch1, rscratch2); 2379 __ cbnzw(rscratch1, *stub->entry()); 2380 } 2381 2382 // Spill because stubs can use any register they like and it's 2383 // easier to restore just those that we care about. 2384 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2385 __ stp(length, src_pos, Address(sp, 2*BytesPerWord)); 2386 __ str(src, Address(sp, 4*BytesPerWord)); 2387 2388 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2389 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2390 assert_different_registers(c_rarg0, dst, dst_pos, length); 2391 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2392 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2393 assert_different_registers(c_rarg1, dst, length); 2394 __ uxtw(c_rarg2, length); 2395 assert_different_registers(c_rarg2, dst); 2396 2397 __ load_klass(c_rarg4, dst); 2398 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset())); 2399 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset())); 2400 __ far_call(RuntimeAddress(copyfunc_addr)); 2401 2402 #ifndef PRODUCT 2403 if (PrintC1Statistics) { 2404 Label failed; 2405 __ cbnz(r0, failed); 2406 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt)); 2407 __ bind(failed); 2408 } 2409 #endif 2410 2411 __ cbz(r0, *stub->continuation()); 2412 2413 #ifndef PRODUCT 2414 if (PrintC1Statistics) { 2415 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt)); 2416 } 2417 #endif 2418 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1); 2419 2420 // Restore previously spilled arguments 2421 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord)); 2422 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord)); 2423 __ ldr(src, Address(sp, 4*BytesPerWord)); 2424 2425 // return value is -1^K where K is partial copied count 2426 __ eonw(rscratch1, r0, zr); 2427 // adjust length down and src/end pos up by partial copied count 2428 __ subw(length, length, rscratch1); 2429 __ addw(src_pos, src_pos, rscratch1); 2430 __ addw(dst_pos, dst_pos, rscratch1); 2431 } 2432 2433 __ b(*stub->entry()); 2434 2435 __ bind(cont); 2436 __ POP(src, dst); 2437 } 2438 } 2439 2440 #ifdef ASSERT 2441 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { 2442 // Sanity check the known type with the incoming class. For the 2443 // primitive case the types must match exactly with src.klass and 2444 // dst.klass each exactly matching the default type. For the 2445 // object array case, if no type check is needed then either the 2446 // dst type is exactly the expected type and the src type is a 2447 // subtype which we can't check or src is the same array as dst 2448 // but not necessarily exactly of type default_type. 2449 Label known_ok, halt; 2450 __ mov_metadata(tmp, default_type->constant_encoding()); 2451 if (UseCompressedClassPointers) { 2452 __ encode_klass_not_null(tmp); 2453 } 2454 2455 if (basic_type != T_OBJECT) { 2456 2457 if (UseCompressedClassPointers) { 2458 __ ldrw(rscratch1, dst_klass_addr); 2459 __ cmpw(tmp, rscratch1); 2460 } else { 2461 __ ldr(rscratch1, dst_klass_addr); 2462 __ cmp(tmp, rscratch1); 2463 } 2464 __ br(Assembler::NE, halt); 2465 if (UseCompressedClassPointers) { 2466 __ ldrw(rscratch1, src_klass_addr); 2467 __ cmpw(tmp, rscratch1); 2468 } else { 2469 __ ldr(rscratch1, src_klass_addr); 2470 __ cmp(tmp, rscratch1); 2471 } 2472 __ br(Assembler::EQ, known_ok); 2473 } else { 2474 if (UseCompressedClassPointers) { 2475 __ ldrw(rscratch1, dst_klass_addr); 2476 __ cmpw(tmp, rscratch1); 2477 } else { 2478 __ ldr(rscratch1, dst_klass_addr); 2479 __ cmp(tmp, rscratch1); 2480 } 2481 __ br(Assembler::EQ, known_ok); 2482 __ cmp(src, dst); 2483 __ br(Assembler::EQ, known_ok); 2484 } 2485 __ bind(halt); 2486 __ stop("incorrect type information in arraycopy"); 2487 __ bind(known_ok); 2488 } 2489 #endif 2490 2491 #ifndef PRODUCT 2492 if (PrintC1Statistics) { 2493 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type))); 2494 } 2495 #endif 2496 2497 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale))); 2498 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type)); 2499 assert_different_registers(c_rarg0, dst, dst_pos, length); 2500 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale))); 2501 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type)); 2502 assert_different_registers(c_rarg1, dst, length); 2503 __ uxtw(c_rarg2, length); 2504 assert_different_registers(c_rarg2, dst); 2505 2506 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 2507 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 2508 const char *name; 2509 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 2510 2511 CodeBlob *cb = CodeCache::find_blob(entry); 2512 if (cb) { 2513 __ far_call(RuntimeAddress(entry)); 2514 } else { 2515 __ call_VM_leaf(entry, 3); 2516 } 2517 2518 if (stub != nullptr) { 2519 __ bind(*stub->continuation()); 2520 } 2521 } 2522 2523 2524 2525 2526 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 2527 Register obj = op->obj_opr()->as_register(); // may not be an oop 2528 Register hdr = op->hdr_opr()->as_register(); 2529 Register lock = op->lock_opr()->as_register(); 2530 Register temp = op->scratch_opr()->as_register(); 2531 if (LockingMode == LM_MONITOR) { 2532 if (op->info() != nullptr) { 2533 add_debug_info_for_null_check_here(op->info()); 2534 __ null_check(obj, -1); 2535 } 2536 __ b(*op->stub()->entry()); 2537 } else if (op->code() == lir_lock) { 2538 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2539 // add debug info for NullPointerException only if one is possible 2540 int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry()); 2541 if (op->info() != nullptr) { 2542 add_debug_info_for_null_check(null_check_offset, op->info()); 2543 } 2544 // done 2545 } else if (op->code() == lir_unlock) { 2546 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 2547 __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry()); 2548 } else { 2549 Unimplemented(); 2550 } 2551 __ bind(*op->stub()->continuation()); 2552 } 2553 2554 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { 2555 Register obj = op->obj()->as_pointer_register(); 2556 Register result = op->result_opr()->as_pointer_register(); 2557 2558 CodeEmitInfo* info = op->info(); 2559 if (info != nullptr) { 2560 add_debug_info_for_null_check_here(info); 2561 } 2562 2563 if (UseCompressedClassPointers) { 2564 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes())); 2565 __ decode_klass_not_null(result); 2566 } else { 2567 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes())); 2568 } 2569 } 2570 2571 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 2572 ciMethod* method = op->profiled_method(); 2573 int bci = op->profiled_bci(); 2574 ciMethod* callee = op->profiled_callee(); 2575 2576 // Update counter for all call types 2577 ciMethodData* md = method->method_data_or_null(); 2578 assert(md != nullptr, "Sanity"); 2579 ciProfileData* data = md->bci_to_data(bci); 2580 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls"); 2581 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 2582 Register mdo = op->mdo()->as_register(); 2583 __ mov_metadata(mdo, md->constant_encoding()); 2584 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 2585 // Perform additional virtual call profiling for invokevirtual and 2586 // invokeinterface bytecodes 2587 if (op->should_profile_receiver_type()) { 2588 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 2589 Register recv = op->recv()->as_register(); 2590 assert_different_registers(mdo, recv); 2591 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 2592 ciKlass* known_klass = op->known_holder(); 2593 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) { 2594 // We know the type that will be seen at this call site; we can 2595 // statically update the MethodData* rather than needing to do 2596 // dynamic tests on the receiver type 2597 2598 // NOTE: we should probably put a lock around this search to 2599 // avoid collisions by concurrent compilations 2600 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 2601 uint i; 2602 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2603 ciKlass* receiver = vc_data->receiver(i); 2604 if (known_klass->equals(receiver)) { 2605 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2606 __ addptr(data_addr, DataLayout::counter_increment); 2607 return; 2608 } 2609 } 2610 2611 // Receiver type not found in profile data; select an empty slot 2612 2613 // Note that this is less efficient than it should be because it 2614 // always does a write to the receiver part of the 2615 // VirtualCallData rather than just the first time 2616 for (i = 0; i < VirtualCallData::row_limit(); i++) { 2617 ciKlass* receiver = vc_data->receiver(i); 2618 if (receiver == nullptr) { 2619 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); 2620 __ mov_metadata(rscratch1, known_klass->constant_encoding()); 2621 __ lea(rscratch2, recv_addr); 2622 __ str(rscratch1, Address(rscratch2)); 2623 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 2624 __ addptr(data_addr, DataLayout::counter_increment); 2625 return; 2626 } 2627 } 2628 } else { 2629 __ load_klass(recv, recv); 2630 Label update_done; 2631 type_profile_helper(mdo, md, data, recv, &update_done); 2632 // Receiver did not match any saved receiver and there is no empty row for it. 2633 // Increment total counter to indicate polymorphic case. 2634 __ addptr(counter_addr, DataLayout::counter_increment); 2635 2636 __ bind(update_done); 2637 } 2638 } else { 2639 // Static call 2640 __ addptr(counter_addr, DataLayout::counter_increment); 2641 } 2642 } 2643 2644 2645 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 2646 Unimplemented(); 2647 } 2648 2649 2650 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 2651 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 2652 } 2653 2654 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 2655 assert(op->crc()->is_single_cpu(), "crc must be register"); 2656 assert(op->val()->is_single_cpu(), "byte value must be register"); 2657 assert(op->result_opr()->is_single_cpu(), "result must be register"); 2658 Register crc = op->crc()->as_register(); 2659 Register val = op->val()->as_register(); 2660 Register res = op->result_opr()->as_register(); 2661 2662 assert_different_registers(val, crc, res); 2663 uint64_t offset; 2664 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset); 2665 __ add(res, res, offset); 2666 2667 __ mvnw(crc, crc); // ~crc 2668 __ update_byte_crc32(crc, val, res); 2669 __ mvnw(res, crc); // ~crc 2670 } 2671 2672 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 2673 COMMENT("emit_profile_type {"); 2674 Register obj = op->obj()->as_register(); 2675 Register tmp = op->tmp()->as_pointer_register(); 2676 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 2677 ciKlass* exact_klass = op->exact_klass(); 2678 intptr_t current_klass = op->current_klass(); 2679 bool not_null = op->not_null(); 2680 bool no_conflict = op->no_conflict(); 2681 2682 Label update, next, none; 2683 2684 bool do_null = !not_null; 2685 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 2686 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 2687 2688 assert(do_null || do_update, "why are we here?"); 2689 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 2690 assert(mdo_addr.base() != rscratch1, "wrong register"); 2691 2692 __ verify_oop(obj); 2693 2694 if (tmp != obj) { 2695 assert_different_registers(obj, tmp, rscratch1, rscratch2, mdo_addr.base(), mdo_addr.index()); 2696 __ mov(tmp, obj); 2697 } else { 2698 assert_different_registers(obj, rscratch1, rscratch2, mdo_addr.base(), mdo_addr.index()); 2699 } 2700 if (do_null) { 2701 __ cbnz(tmp, update); 2702 if (!TypeEntries::was_null_seen(current_klass)) { 2703 __ ldr(rscratch2, mdo_addr); 2704 __ orr(rscratch2, rscratch2, TypeEntries::null_seen); 2705 __ str(rscratch2, mdo_addr); 2706 } 2707 if (do_update) { 2708 #ifndef ASSERT 2709 __ b(next); 2710 } 2711 #else 2712 __ b(next); 2713 } 2714 } else { 2715 __ cbnz(tmp, update); 2716 __ stop("unexpected null obj"); 2717 #endif 2718 } 2719 2720 __ bind(update); 2721 2722 if (do_update) { 2723 #ifdef ASSERT 2724 if (exact_klass != nullptr) { 2725 Label ok; 2726 __ load_klass(tmp, tmp); 2727 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2728 __ eor(rscratch1, tmp, rscratch1); 2729 __ cbz(rscratch1, ok); 2730 __ stop("exact klass and actual klass differ"); 2731 __ bind(ok); 2732 } 2733 #endif 2734 if (!no_conflict) { 2735 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) { 2736 if (exact_klass != nullptr) { 2737 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2738 } else { 2739 __ load_klass(tmp, tmp); 2740 } 2741 2742 __ ldr(rscratch2, mdo_addr); 2743 __ eor(tmp, tmp, rscratch2); 2744 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2745 // klass seen before, nothing to do. The unknown bit may have been 2746 // set already but no need to check. 2747 __ cbz(rscratch1, next); 2748 2749 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2750 2751 if (TypeEntries::is_type_none(current_klass)) { 2752 __ cbz(rscratch2, none); 2753 __ cmp(rscratch2, (u1)TypeEntries::null_seen); 2754 __ br(Assembler::EQ, none); 2755 // There is a chance that the checks above 2756 // fail if another thread has just set the 2757 // profiling to this obj's klass 2758 __ dmb(Assembler::ISHLD); 2759 __ eor(tmp, tmp, rscratch2); // get back original value before XOR 2760 __ ldr(rscratch2, mdo_addr); 2761 __ eor(tmp, tmp, rscratch2); 2762 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2763 __ cbz(rscratch1, next); 2764 } 2765 } else { 2766 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr && 2767 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 2768 2769 __ ldr(tmp, mdo_addr); 2770 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2771 } 2772 2773 // different than before. Cannot keep accurate profile. 2774 __ ldr(rscratch2, mdo_addr); 2775 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown); 2776 __ str(rscratch2, mdo_addr); 2777 2778 if (TypeEntries::is_type_none(current_klass)) { 2779 __ b(next); 2780 2781 __ bind(none); 2782 // first time here. Set profile type. 2783 __ str(tmp, mdo_addr); 2784 #ifdef ASSERT 2785 __ andr(tmp, tmp, TypeEntries::type_mask); 2786 __ verify_klass_ptr(tmp); 2787 #endif 2788 } 2789 } else { 2790 // There's a single possible klass at this profile point 2791 assert(exact_klass != nullptr, "should be"); 2792 if (TypeEntries::is_type_none(current_klass)) { 2793 __ mov_metadata(tmp, exact_klass->constant_encoding()); 2794 __ ldr(rscratch2, mdo_addr); 2795 __ eor(tmp, tmp, rscratch2); 2796 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask); 2797 __ cbz(rscratch1, next); 2798 #ifdef ASSERT 2799 { 2800 Label ok; 2801 __ ldr(rscratch1, mdo_addr); 2802 __ cbz(rscratch1, ok); 2803 __ cmp(rscratch1, (u1)TypeEntries::null_seen); 2804 __ br(Assembler::EQ, ok); 2805 // may have been set by another thread 2806 __ dmb(Assembler::ISHLD); 2807 __ mov_metadata(rscratch1, exact_klass->constant_encoding()); 2808 __ ldr(rscratch2, mdo_addr); 2809 __ eor(rscratch2, rscratch1, rscratch2); 2810 __ andr(rscratch2, rscratch2, TypeEntries::type_mask); 2811 __ cbz(rscratch2, ok); 2812 2813 __ stop("unexpected profiling mismatch"); 2814 __ bind(ok); 2815 } 2816 #endif 2817 // first time here. Set profile type. 2818 __ str(tmp, mdo_addr); 2819 #ifdef ASSERT 2820 __ andr(tmp, tmp, TypeEntries::type_mask); 2821 __ verify_klass_ptr(tmp); 2822 #endif 2823 } else { 2824 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr && 2825 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 2826 2827 __ ldr(tmp, mdo_addr); 2828 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore. 2829 2830 __ orr(tmp, tmp, TypeEntries::type_unknown); 2831 __ str(tmp, mdo_addr); 2832 // FIXME: Write barrier needed here? 2833 } 2834 } 2835 2836 __ bind(next); 2837 } 2838 COMMENT("} emit_profile_type"); 2839 } 2840 2841 2842 void LIR_Assembler::align_backward_branch_target() { 2843 } 2844 2845 2846 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) { 2847 // tmp must be unused 2848 assert(tmp->is_illegal(), "wasting a register if tmp is allocated"); 2849 2850 if (left->is_single_cpu()) { 2851 assert(dest->is_single_cpu(), "expect single result reg"); 2852 __ negw(dest->as_register(), left->as_register()); 2853 } else if (left->is_double_cpu()) { 2854 assert(dest->is_double_cpu(), "expect double result reg"); 2855 __ neg(dest->as_register_lo(), left->as_register_lo()); 2856 } else if (left->is_single_fpu()) { 2857 assert(dest->is_single_fpu(), "expect single float result reg"); 2858 __ fnegs(dest->as_float_reg(), left->as_float_reg()); 2859 } else { 2860 assert(left->is_double_fpu(), "expect double float operand reg"); 2861 assert(dest->is_double_fpu(), "expect double float result reg"); 2862 __ fnegd(dest->as_double_reg(), left->as_double_reg()); 2863 } 2864 } 2865 2866 2867 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 2868 if (patch_code != lir_patch_none) { 2869 deoptimize_trap(info); 2870 return; 2871 } 2872 2873 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr())); 2874 } 2875 2876 2877 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 2878 assert(!tmp->is_valid(), "don't need temporary"); 2879 2880 CodeBlob *cb = CodeCache::find_blob(dest); 2881 if (cb) { 2882 __ far_call(RuntimeAddress(dest)); 2883 } else { 2884 __ mov(rscratch1, RuntimeAddress(dest)); 2885 __ blr(rscratch1); 2886 } 2887 2888 if (info != nullptr) { 2889 add_call_info_here(info); 2890 } 2891 __ post_call_nop(); 2892 } 2893 2894 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 2895 if (dest->is_address() || src->is_address()) { 2896 move_op(src, dest, type, lir_patch_none, info, 2897 /*pop_fpu_stack*/false, /*wide*/false); 2898 } else { 2899 ShouldNotReachHere(); 2900 } 2901 } 2902 2903 #ifdef ASSERT 2904 // emit run-time assertion 2905 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 2906 assert(op->code() == lir_assert, "must be"); 2907 2908 if (op->in_opr1()->is_valid()) { 2909 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 2910 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 2911 } else { 2912 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 2913 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 2914 } 2915 2916 Label ok; 2917 if (op->condition() != lir_cond_always) { 2918 Assembler::Condition acond = Assembler::AL; 2919 switch (op->condition()) { 2920 case lir_cond_equal: acond = Assembler::EQ; break; 2921 case lir_cond_notEqual: acond = Assembler::NE; break; 2922 case lir_cond_less: acond = Assembler::LT; break; 2923 case lir_cond_lessEqual: acond = Assembler::LE; break; 2924 case lir_cond_greaterEqual: acond = Assembler::GE; break; 2925 case lir_cond_greater: acond = Assembler::GT; break; 2926 case lir_cond_belowEqual: acond = Assembler::LS; break; 2927 case lir_cond_aboveEqual: acond = Assembler::HS; break; 2928 default: ShouldNotReachHere(); 2929 } 2930 __ br(acond, ok); 2931 } 2932 if (op->halt()) { 2933 const char* str = __ code_string(op->msg()); 2934 __ stop(str); 2935 } else { 2936 breakpoint(); 2937 } 2938 __ bind(ok); 2939 } 2940 #endif 2941 2942 #ifndef PRODUCT 2943 #define COMMENT(x) do { __ block_comment(x); } while (0) 2944 #else 2945 #define COMMENT(x) 2946 #endif 2947 2948 void LIR_Assembler::membar() { 2949 COMMENT("membar"); 2950 __ membar(MacroAssembler::AnyAny); 2951 } 2952 2953 void LIR_Assembler::membar_acquire() { 2954 __ membar(Assembler::LoadLoad|Assembler::LoadStore); 2955 } 2956 2957 void LIR_Assembler::membar_release() { 2958 __ membar(Assembler::LoadStore|Assembler::StoreStore); 2959 } 2960 2961 void LIR_Assembler::membar_loadload() { 2962 __ membar(Assembler::LoadLoad); 2963 } 2964 2965 void LIR_Assembler::membar_storestore() { 2966 __ membar(MacroAssembler::StoreStore); 2967 } 2968 2969 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); } 2970 2971 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); } 2972 2973 void LIR_Assembler::on_spin_wait() { 2974 __ spin_wait(); 2975 } 2976 2977 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 2978 __ mov(result_reg->as_register(), rthread); 2979 } 2980 2981 2982 void LIR_Assembler::peephole(LIR_List *lir) { 2983 #if 0 2984 if (tableswitch_count >= max_tableswitches) 2985 return; 2986 2987 /* 2988 This finite-state automaton recognizes sequences of compare-and- 2989 branch instructions. We will turn them into a tableswitch. You 2990 could argue that C1 really shouldn't be doing this sort of 2991 optimization, but without it the code is really horrible. 2992 */ 2993 2994 enum { start_s, cmp1_s, beq_s, cmp_s } state; 2995 int first_key, last_key = -2147483648; 2996 int next_key = 0; 2997 int start_insn = -1; 2998 int last_insn = -1; 2999 Register reg = noreg; 3000 LIR_Opr reg_opr; 3001 state = start_s; 3002 3003 LIR_OpList* inst = lir->instructions_list(); 3004 for (int i = 0; i < inst->length(); i++) { 3005 LIR_Op* op = inst->at(i); 3006 switch (state) { 3007 case start_s: 3008 first_key = -1; 3009 start_insn = i; 3010 switch (op->code()) { 3011 case lir_cmp: 3012 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3013 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3014 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3015 && opr2->is_constant() 3016 && opr2->type() == T_INT) { 3017 reg_opr = opr1; 3018 reg = opr1->as_register(); 3019 first_key = opr2->as_constant_ptr()->as_jint(); 3020 next_key = first_key + 1; 3021 state = cmp_s; 3022 goto next_state; 3023 } 3024 break; 3025 } 3026 break; 3027 case cmp_s: 3028 switch (op->code()) { 3029 case lir_branch: 3030 if (op->as_OpBranch()->cond() == lir_cond_equal) { 3031 state = beq_s; 3032 last_insn = i; 3033 goto next_state; 3034 } 3035 } 3036 state = start_s; 3037 break; 3038 case beq_s: 3039 switch (op->code()) { 3040 case lir_cmp: { 3041 LIR_Opr opr1 = op->as_Op2()->in_opr1(); 3042 LIR_Opr opr2 = op->as_Op2()->in_opr2(); 3043 if (opr1->is_cpu_register() && opr1->is_single_cpu() 3044 && opr1->as_register() == reg 3045 && opr2->is_constant() 3046 && opr2->type() == T_INT 3047 && opr2->as_constant_ptr()->as_jint() == next_key) { 3048 last_key = next_key; 3049 next_key++; 3050 state = cmp_s; 3051 goto next_state; 3052 } 3053 } 3054 } 3055 last_key = next_key; 3056 state = start_s; 3057 break; 3058 default: 3059 assert(false, "impossible state"); 3060 } 3061 if (state == start_s) { 3062 if (first_key < last_key - 5L && reg != noreg) { 3063 { 3064 // printf("found run register %d starting at insn %d low value %d high value %d\n", 3065 // reg->encoding(), 3066 // start_insn, first_key, last_key); 3067 // for (int i = 0; i < inst->length(); i++) { 3068 // inst->at(i)->print(); 3069 // tty->print("\n"); 3070 // } 3071 // tty->print("\n"); 3072 } 3073 3074 struct tableswitch *sw = &switches[tableswitch_count]; 3075 sw->_insn_index = start_insn, sw->_first_key = first_key, 3076 sw->_last_key = last_key, sw->_reg = reg; 3077 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after)); 3078 { 3079 // Insert the new table of branches 3080 int offset = last_insn; 3081 for (int n = first_key; n < last_key; n++) { 3082 inst->insert_before 3083 (last_insn + 1, 3084 new LIR_OpBranch(lir_cond_always, T_ILLEGAL, 3085 inst->at(offset)->as_OpBranch()->label())); 3086 offset -= 2, i++; 3087 } 3088 } 3089 // Delete all the old compare-and-branch instructions 3090 for (int n = first_key; n < last_key; n++) { 3091 inst->remove_at(start_insn); 3092 inst->remove_at(start_insn); 3093 } 3094 // Insert the tableswitch instruction 3095 inst->insert_before(start_insn, 3096 new LIR_Op2(lir_cmp, lir_cond_always, 3097 LIR_OprFact::intConst(tableswitch_count), 3098 reg_opr)); 3099 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches)); 3100 tableswitch_count++; 3101 } 3102 reg = noreg; 3103 last_key = -2147483648; 3104 } 3105 next_state: 3106 ; 3107 } 3108 #endif 3109 } 3110 3111 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { 3112 Address addr = as_Address(src->as_address_ptr()); 3113 BasicType type = src->type(); 3114 bool is_oop = is_reference_type(type); 3115 3116 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr); 3117 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr); 3118 3119 switch(type) { 3120 case T_INT: 3121 xchg = &MacroAssembler::atomic_xchgalw; 3122 add = &MacroAssembler::atomic_addalw; 3123 break; 3124 case T_LONG: 3125 xchg = &MacroAssembler::atomic_xchgal; 3126 add = &MacroAssembler::atomic_addal; 3127 break; 3128 case T_OBJECT: 3129 case T_ARRAY: 3130 if (UseCompressedOops) { 3131 xchg = &MacroAssembler::atomic_xchgalw; 3132 add = &MacroAssembler::atomic_addalw; 3133 } else { 3134 xchg = &MacroAssembler::atomic_xchgal; 3135 add = &MacroAssembler::atomic_addal; 3136 } 3137 break; 3138 default: 3139 ShouldNotReachHere(); 3140 xchg = &MacroAssembler::atomic_xchgal; 3141 add = &MacroAssembler::atomic_addal; // unreachable 3142 } 3143 3144 switch (code) { 3145 case lir_xadd: 3146 { 3147 RegisterOrConstant inc; 3148 Register tmp = as_reg(tmp_op); 3149 Register dst = as_reg(dest); 3150 if (data->is_constant()) { 3151 inc = RegisterOrConstant(as_long(data)); 3152 assert_different_registers(dst, addr.base(), tmp, 3153 rscratch1, rscratch2); 3154 } else { 3155 inc = RegisterOrConstant(as_reg(data)); 3156 assert_different_registers(inc.as_register(), dst, addr.base(), tmp, 3157 rscratch1, rscratch2); 3158 } 3159 __ lea(tmp, addr); 3160 (_masm->*add)(dst, inc, tmp); 3161 break; 3162 } 3163 case lir_xchg: 3164 { 3165 Register tmp = tmp_op->as_register(); 3166 Register obj = as_reg(data); 3167 Register dst = as_reg(dest); 3168 if (is_oop && UseCompressedOops) { 3169 __ encode_heap_oop(rscratch2, obj); 3170 obj = rscratch2; 3171 } 3172 assert_different_registers(obj, addr.base(), tmp, rscratch1); 3173 assert_different_registers(dst, addr.base(), tmp, rscratch1); 3174 __ lea(tmp, addr); 3175 (_masm->*xchg)(dst, obj, tmp); 3176 if (is_oop && UseCompressedOops) { 3177 __ decode_heap_oop(dst); 3178 } 3179 } 3180 break; 3181 default: 3182 ShouldNotReachHere(); 3183 } 3184 __ membar(__ AnyAny); 3185 } 3186 3187 #undef __