1 /* 2 * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "asm/macroAssembler.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #include "c1/c1_CodeStubs.hpp" 28 #include "c1/c1_Compilation.hpp" 29 #include "c1/c1_LIRAssembler.hpp" 30 #include "c1/c1_MacroAssembler.hpp" 31 #include "c1/c1_Runtime1.hpp" 32 #include "c1/c1_ValueStack.hpp" 33 #include "ci/ciArrayKlass.hpp" 34 #include "ci/ciInlineKlass.hpp" 35 #include "ci/ciInstance.hpp" 36 #include "compiler/oopMap.hpp" 37 #include "gc/shared/collectedHeap.hpp" 38 #include "gc/shared/gc_globals.hpp" 39 #include "nativeInst_x86.hpp" 40 #include "oops/oop.inline.hpp" 41 #include "oops/objArrayKlass.hpp" 42 #include "runtime/frame.inline.hpp" 43 #include "runtime/safepointMechanism.hpp" 44 #include "runtime/sharedRuntime.hpp" 45 #include "runtime/stubRoutines.hpp" 46 #include "utilities/powerOfTwo.hpp" 47 #include "vmreg_x86.inline.hpp" 48 49 50 // These masks are used to provide 128-bit aligned bitmasks to the XMM 51 // instructions, to allow sign-masking or sign-bit flipping. They allow 52 // fast versions of NegF/NegD and AbsF/AbsD. 53 54 // Note: 'double' and 'long long' have 32-bits alignment on x86. 55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) { 56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address 57 // of 128-bits operands for SSE instructions. 58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF))); 59 // Store the value to a 128-bits operand. 60 operand[0] = lo; 61 operand[1] = hi; 62 return operand; 63 } 64 65 // Buffer for 128-bits masks used by SSE instructions. 66 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment) 67 68 // Static initialization during VM startup. 69 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF)); 70 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF)); 71 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000)); 72 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000)); 73 74 75 NEEDS_CLEANUP // remove this definitions ? 76 const Register SYNC_header = rax; // synchronization header 77 const Register SHIFT_count = rcx; // where count for shift operations must be 78 79 #define __ _masm-> 80 81 82 static void select_different_registers(Register preserve, 83 Register extra, 84 Register &tmp1, 85 Register &tmp2) { 86 if (tmp1 == preserve) { 87 assert_different_registers(tmp1, tmp2, extra); 88 tmp1 = extra; 89 } else if (tmp2 == preserve) { 90 assert_different_registers(tmp1, tmp2, extra); 91 tmp2 = extra; 92 } 93 assert_different_registers(preserve, tmp1, tmp2); 94 } 95 96 97 98 static void select_different_registers(Register preserve, 99 Register extra, 100 Register &tmp1, 101 Register &tmp2, 102 Register &tmp3) { 103 if (tmp1 == preserve) { 104 assert_different_registers(tmp1, tmp2, tmp3, extra); 105 tmp1 = extra; 106 } else if (tmp2 == preserve) { 107 assert_different_registers(tmp1, tmp2, tmp3, extra); 108 tmp2 = extra; 109 } else if (tmp3 == preserve) { 110 assert_different_registers(tmp1, tmp2, tmp3, extra); 111 tmp3 = extra; 112 } 113 assert_different_registers(preserve, tmp1, tmp2, tmp3); 114 } 115 116 117 118 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { 119 if (opr->is_constant()) { 120 LIR_Const* constant = opr->as_constant_ptr(); 121 switch (constant->type()) { 122 case T_INT: { 123 return true; 124 } 125 126 default: 127 return false; 128 } 129 } 130 return false; 131 } 132 133 134 LIR_Opr LIR_Assembler::receiverOpr() { 135 return FrameMap::receiver_opr; 136 } 137 138 LIR_Opr LIR_Assembler::osrBufferPointer() { 139 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 140 } 141 142 //--------------fpu register translations----------------------- 143 144 145 address LIR_Assembler::float_constant(float f) { 146 address const_addr = __ float_constant(f); 147 if (const_addr == nullptr) { 148 bailout("const section overflow"); 149 return __ code()->consts()->start(); 150 } else { 151 return const_addr; 152 } 153 } 154 155 156 address LIR_Assembler::double_constant(double d) { 157 address const_addr = __ double_constant(d); 158 if (const_addr == nullptr) { 159 bailout("const section overflow"); 160 return __ code()->consts()->start(); 161 } else { 162 return const_addr; 163 } 164 } 165 166 #ifndef _LP64 167 void LIR_Assembler::fpop() { 168 __ fpop(); 169 } 170 171 void LIR_Assembler::fxch(int i) { 172 __ fxch(i); 173 } 174 175 void LIR_Assembler::fld(int i) { 176 __ fld_s(i); 177 } 178 179 void LIR_Assembler::ffree(int i) { 180 __ ffree(i); 181 } 182 #endif // !_LP64 183 184 void LIR_Assembler::breakpoint() { 185 __ int3(); 186 } 187 188 void LIR_Assembler::push(LIR_Opr opr) { 189 if (opr->is_single_cpu()) { 190 __ push_reg(opr->as_register()); 191 } else if (opr->is_double_cpu()) { 192 NOT_LP64(__ push_reg(opr->as_register_hi())); 193 __ push_reg(opr->as_register_lo()); 194 } else if (opr->is_stack()) { 195 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix())); 196 } else if (opr->is_constant()) { 197 LIR_Const* const_opr = opr->as_constant_ptr(); 198 if (const_opr->type() == T_OBJECT) { 199 __ push_oop(const_opr->as_jobject(), rscratch1); 200 } else if (const_opr->type() == T_INT) { 201 __ push_jint(const_opr->as_jint()); 202 } else { 203 ShouldNotReachHere(); 204 } 205 206 } else { 207 ShouldNotReachHere(); 208 } 209 } 210 211 void LIR_Assembler::pop(LIR_Opr opr) { 212 if (opr->is_single_cpu()) { 213 __ pop_reg(opr->as_register()); 214 } else { 215 ShouldNotReachHere(); 216 } 217 } 218 219 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { 220 return addr->base()->is_illegal() && addr->index()->is_illegal(); 221 } 222 223 //------------------------------------------- 224 225 Address LIR_Assembler::as_Address(LIR_Address* addr) { 226 return as_Address(addr, rscratch1); 227 } 228 229 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 230 if (addr->base()->is_illegal()) { 231 assert(addr->index()->is_illegal(), "must be illegal too"); 232 AddressLiteral laddr((address)addr->disp(), relocInfo::none); 233 if (! __ reachable(laddr)) { 234 __ movptr(tmp, laddr.addr()); 235 Address res(tmp, 0); 236 return res; 237 } else { 238 return __ as_Address(laddr); 239 } 240 } 241 242 Register base = addr->base()->as_pointer_register(); 243 244 if (addr->index()->is_illegal()) { 245 return Address( base, addr->disp()); 246 } else if (addr->index()->is_cpu_register()) { 247 Register index = addr->index()->as_pointer_register(); 248 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp()); 249 } else if (addr->index()->is_constant()) { 250 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp(); 251 assert(Assembler::is_simm32(addr_offset), "must be"); 252 253 return Address(base, addr_offset); 254 } else { 255 Unimplemented(); 256 return Address(); 257 } 258 } 259 260 261 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 262 Address base = as_Address(addr); 263 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord); 264 } 265 266 267 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 268 return as_Address(addr); 269 } 270 271 272 void LIR_Assembler::osr_entry() { 273 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 274 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 275 ValueStack* entry_state = osr_entry->state(); 276 int number_of_locks = entry_state->locks_size(); 277 278 // we jump here if osr happens with the interpreter 279 // state set up to continue at the beginning of the 280 // loop that triggered osr - in particular, we have 281 // the following registers setup: 282 // 283 // rcx: osr buffer 284 // 285 286 // build frame 287 ciMethod* m = compilation()->method(); 288 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 289 290 // OSR buffer is 291 // 292 // locals[nlocals-1..0] 293 // monitors[0..number_of_locks] 294 // 295 // locals is a direct copy of the interpreter frame so in the osr buffer 296 // so first slot in the local array is the last local from the interpreter 297 // and last slot is local[0] (receiver) from the interpreter 298 // 299 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 300 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 301 // in the interpreter frame (the method lock if a sync method) 302 303 // Initialize monitors in the compiled activation. 304 // rcx: pointer to osr buffer 305 // 306 // All other registers are dead at this point and the locals will be 307 // copied into place by code emitted in the IR. 308 309 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 310 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 311 int monitor_offset = BytesPerWord * method()->max_locals() + 312 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1); 313 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 314 // the OSR buffer using 2 word entries: first the lock and then 315 // the oop. 316 for (int i = 0; i < number_of_locks; i++) { 317 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 318 #ifdef ASSERT 319 // verify the interpreter's monitor has a non-null object 320 { 321 Label L; 322 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD); 323 __ jcc(Assembler::notZero, L); 324 __ stop("locked object is null"); 325 __ bind(L); 326 } 327 #endif 328 __ movptr(rbx, Address(OSR_buf, slot_offset + 0)); 329 __ movptr(frame_map()->address_for_monitor_lock(i), rbx); 330 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 331 __ movptr(frame_map()->address_for_monitor_object(i), rbx); 332 } 333 } 334 } 335 336 337 // inline cache check; done before the frame is built. 338 int LIR_Assembler::check_icache() { 339 return __ ic_check(CodeEntryAlignment); 340 } 341 342 void LIR_Assembler::clinit_barrier(ciMethod* method) { 343 assert(VM_Version::supports_fast_class_init_checks(), "sanity"); 344 assert(!method->holder()->is_not_initialized(), "initialization should have been started"); 345 346 Label L_skip_barrier; 347 Register klass = rscratch1; 348 Register thread = LP64_ONLY( r15_thread ) NOT_LP64( noreg ); 349 assert(thread != noreg, "x86_32 not implemented"); 350 351 __ mov_metadata(klass, method->holder()->constant_encoding()); 352 __ clinit_barrier(klass, thread, &L_skip_barrier /*L_fast_path*/); 353 354 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 355 356 __ bind(L_skip_barrier); 357 } 358 359 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) { 360 jobject o = nullptr; 361 PatchingStub* patch = new PatchingStub(_masm, patching_id(info)); 362 __ movoop(reg, o); 363 patching_epilog(patch, lir_patch_normal, reg, info); 364 } 365 366 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 367 Metadata* o = nullptr; 368 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id); 369 __ mov_metadata(reg, o); 370 patching_epilog(patch, lir_patch_normal, reg, info); 371 } 372 373 // This specifies the rsp decrement needed to build the frame 374 int LIR_Assembler::initial_frame_size_in_bytes() const { 375 // if rounding, must let FrameMap know! 376 377 // The frame_map records size in slots (32bit word) 378 379 // subtract two words to account for return address and link 380 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size; 381 } 382 383 384 int LIR_Assembler::emit_exception_handler() { 385 // generate code for exception handler 386 address handler_base = __ start_a_stub(exception_handler_size()); 387 if (handler_base == nullptr) { 388 // not enough space left for the handler 389 bailout("exception handler overflow"); 390 return -1; 391 } 392 393 int offset = code_offset(); 394 395 // the exception oop and pc are in rax, and rdx 396 // no other registers need to be preserved, so invalidate them 397 __ invalidate_registers(false, true, true, false, true, true); 398 399 // check that there is really an exception 400 __ verify_not_null_oop(rax); 401 402 // search an exception handler (rax: exception oop, rdx: throwing pc) 403 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::handle_exception_from_callee_id))); 404 __ should_not_reach_here(); 405 guarantee(code_offset() - offset <= exception_handler_size(), "overflow"); 406 __ end_a_stub(); 407 408 return offset; 409 } 410 411 412 // Emit the code to remove the frame from the stack in the exception 413 // unwind path. 414 int LIR_Assembler::emit_unwind_handler() { 415 #ifndef PRODUCT 416 if (CommentedAssembly) { 417 _masm->block_comment("Unwind handler"); 418 } 419 #endif 420 421 int offset = code_offset(); 422 423 // Fetch the exception from TLS and clear out exception related thread state 424 Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); 425 NOT_LP64(__ get_thread(thread)); 426 __ movptr(rax, Address(thread, JavaThread::exception_oop_offset())); 427 __ movptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD); 428 __ movptr(Address(thread, JavaThread::exception_pc_offset()), NULL_WORD); 429 430 __ bind(_unwind_handler_entry); 431 __ verify_not_null_oop(rax); 432 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 433 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved) 434 } 435 436 // Perform needed unlocking 437 MonitorExitStub* stub = nullptr; 438 if (method()->is_synchronized()) { 439 monitor_address(0, FrameMap::rax_opr); 440 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0); 441 if (LockingMode == LM_MONITOR) { 442 __ jmp(*stub->entry()); 443 } else { 444 __ unlock_object(rdi, rsi, rax, *stub->entry()); 445 } 446 __ bind(*stub->continuation()); 447 } 448 449 if (compilation()->env()->dtrace_method_probes()) { 450 #ifdef _LP64 451 __ mov(rdi, r15_thread); 452 __ mov_metadata(rsi, method()->constant_encoding()); 453 #else 454 __ get_thread(rax); 455 __ movptr(Address(rsp, 0), rax); 456 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg); 457 #endif 458 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit))); 459 } 460 461 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 462 __ mov(rax, rbx); // Restore the exception 463 } 464 465 // remove the activation and dispatch to the unwind handler 466 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair()); 467 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id))); 468 469 // Emit the slow path assembly 470 if (stub != nullptr) { 471 stub->emit_code(this); 472 } 473 474 return offset; 475 } 476 477 478 int LIR_Assembler::emit_deopt_handler() { 479 // generate code for exception handler 480 address handler_base = __ start_a_stub(deopt_handler_size()); 481 if (handler_base == nullptr) { 482 // not enough space left for the handler 483 bailout("deopt handler overflow"); 484 return -1; 485 } 486 487 int offset = code_offset(); 488 InternalAddress here(__ pc()); 489 490 __ pushptr(here.addr(), rscratch1); 491 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 492 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); 493 __ end_a_stub(); 494 495 return offset; 496 } 497 498 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { 499 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,"); 500 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) { 501 assert(result->fpu() == 0, "result must already be on TOS"); 502 } 503 if (InlineTypeReturnedAsFields) { 504 #ifndef _LP64 505 Unimplemented(); 506 #endif 507 // Check if we are returning an non-null inline type and load its fields into registers 508 ciType* return_type = compilation()->method()->return_type(); 509 if (return_type->is_inlinetype()) { 510 ciInlineKlass* vk = return_type->as_inline_klass(); 511 if (vk->can_be_returned_as_fields()) { 512 address unpack_handler = vk->unpack_handler(); 513 assert(unpack_handler != nullptr, "must be"); 514 __ call(RuntimeAddress(unpack_handler)); 515 } 516 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) { 517 Label skip; 518 __ test_oop_is_not_inline_type(rax, rscratch1, skip); 519 520 // Load fields from a buffered value with an inline class specific handler 521 __ load_klass(rdi, rax, rscratch1); 522 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset())); 523 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset())); 524 // Unpack handler can be null if inline type is not scalarizable in returns 525 __ testptr(rdi, rdi); 526 __ jcc(Assembler::zero, skip); 527 __ call(rdi); 528 529 __ bind(skip); 530 } 531 // At this point, rax points to the value object (for interpreter or C1 caller). 532 // The fields of the object are copied into registers (for C2 caller). 533 } 534 535 // Pop the stack before the safepoint code 536 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair()); 537 538 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) { 539 __ reserved_stack_check(); 540 } 541 542 // Note: we do not need to round double result; float result has the right precision 543 // the poll sets the condition code, but no data registers 544 545 #ifdef _LP64 546 const Register thread = r15_thread; 547 #else 548 const Register thread = rbx; 549 __ get_thread(thread); 550 #endif 551 code_stub->set_safepoint_offset(__ offset()); 552 __ relocate(relocInfo::poll_return_type); 553 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */); 554 __ ret(0); 555 } 556 557 558 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) { 559 return (__ store_inline_type_fields_to_buf(vk, false)); 560 } 561 562 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 563 guarantee(info != nullptr, "Shouldn't be null"); 564 int offset = __ offset(); 565 #ifdef _LP64 566 const Register poll_addr = rscratch1; 567 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset())); 568 #else 569 assert(tmp->is_cpu_register(), "needed"); 570 const Register poll_addr = tmp->as_register(); 571 __ get_thread(poll_addr); 572 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset()))); 573 #endif 574 add_debug_info_for_branch(info); 575 __ relocate(relocInfo::poll_type); 576 address pre_pc = __ pc(); 577 __ testl(rax, Address(poll_addr, 0)); 578 address post_pc = __ pc(); 579 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length"); 580 return offset; 581 } 582 583 584 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 585 if (from_reg != to_reg) __ mov(to_reg, from_reg); 586 } 587 588 void LIR_Assembler::swap_reg(Register a, Register b) { 589 __ xchgptr(a, b); 590 } 591 592 593 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 594 assert(src->is_constant(), "should not call otherwise"); 595 assert(dest->is_register(), "should not call otherwise"); 596 LIR_Const* c = src->as_constant_ptr(); 597 598 switch (c->type()) { 599 case T_INT: { 600 assert(patch_code == lir_patch_none, "no patching handled here"); 601 __ movl(dest->as_register(), c->as_jint()); 602 break; 603 } 604 605 case T_ADDRESS: { 606 assert(patch_code == lir_patch_none, "no patching handled here"); 607 __ movptr(dest->as_register(), c->as_jint()); 608 break; 609 } 610 611 case T_LONG: { 612 assert(patch_code == lir_patch_none, "no patching handled here"); 613 #ifdef _LP64 614 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong()); 615 #else 616 __ movptr(dest->as_register_lo(), c->as_jint_lo()); 617 __ movptr(dest->as_register_hi(), c->as_jint_hi()); 618 #endif // _LP64 619 break; 620 } 621 622 case T_OBJECT: { 623 if (patch_code != lir_patch_none) { 624 jobject2reg_with_patching(dest->as_register(), info); 625 } else { 626 __ movoop(dest->as_register(), c->as_jobject()); 627 } 628 break; 629 } 630 631 case T_METADATA: { 632 if (patch_code != lir_patch_none) { 633 klass2reg_with_patching(dest->as_register(), info); 634 } else { 635 __ mov_metadata(dest->as_register(), c->as_metadata()); 636 } 637 break; 638 } 639 640 case T_FLOAT: { 641 if (dest->is_single_xmm()) { 642 if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_float()) { 643 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg()); 644 } else { 645 __ movflt(dest->as_xmm_float_reg(), 646 InternalAddress(float_constant(c->as_jfloat()))); 647 } 648 } else { 649 #ifndef _LP64 650 assert(dest->is_single_fpu(), "must be"); 651 assert(dest->fpu_regnr() == 0, "dest must be TOS"); 652 if (c->is_zero_float()) { 653 __ fldz(); 654 } else if (c->is_one_float()) { 655 __ fld1(); 656 } else { 657 __ fld_s (InternalAddress(float_constant(c->as_jfloat()))); 658 } 659 #else 660 ShouldNotReachHere(); 661 #endif // !_LP64 662 } 663 break; 664 } 665 666 case T_DOUBLE: { 667 if (dest->is_double_xmm()) { 668 if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_double()) { 669 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg()); 670 } else { 671 __ movdbl(dest->as_xmm_double_reg(), 672 InternalAddress(double_constant(c->as_jdouble()))); 673 } 674 } else { 675 #ifndef _LP64 676 assert(dest->is_double_fpu(), "must be"); 677 assert(dest->fpu_regnrLo() == 0, "dest must be TOS"); 678 if (c->is_zero_double()) { 679 __ fldz(); 680 } else if (c->is_one_double()) { 681 __ fld1(); 682 } else { 683 __ fld_d (InternalAddress(double_constant(c->as_jdouble()))); 684 } 685 #else 686 ShouldNotReachHere(); 687 #endif // !_LP64 688 } 689 break; 690 } 691 692 default: 693 ShouldNotReachHere(); 694 } 695 } 696 697 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 698 assert(src->is_constant(), "should not call otherwise"); 699 assert(dest->is_stack(), "should not call otherwise"); 700 LIR_Const* c = src->as_constant_ptr(); 701 702 switch (c->type()) { 703 case T_INT: // fall through 704 case T_FLOAT: 705 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits()); 706 break; 707 708 case T_ADDRESS: 709 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits()); 710 break; 711 712 case T_OBJECT: 713 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1); 714 break; 715 716 case T_LONG: // fall through 717 case T_DOUBLE: 718 #ifdef _LP64 719 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(), 720 lo_word_offset_in_bytes), 721 (intptr_t)c->as_jlong_bits(), 722 rscratch1); 723 #else 724 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(), 725 lo_word_offset_in_bytes), c->as_jint_lo_bits()); 726 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(), 727 hi_word_offset_in_bytes), c->as_jint_hi_bits()); 728 #endif // _LP64 729 break; 730 731 default: 732 ShouldNotReachHere(); 733 } 734 } 735 736 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 737 assert(src->is_constant(), "should not call otherwise"); 738 assert(dest->is_address(), "should not call otherwise"); 739 LIR_Const* c = src->as_constant_ptr(); 740 LIR_Address* addr = dest->as_address_ptr(); 741 742 int null_check_here = code_offset(); 743 switch (type) { 744 case T_INT: // fall through 745 case T_FLOAT: 746 __ movl(as_Address(addr), c->as_jint_bits()); 747 break; 748 749 case T_ADDRESS: 750 __ movptr(as_Address(addr), c->as_jint_bits()); 751 break; 752 753 case T_OBJECT: // fall through 754 case T_ARRAY: 755 if (c->as_jobject() == nullptr) { 756 if (UseCompressedOops && !wide) { 757 __ movl(as_Address(addr), NULL_WORD); 758 } else { 759 #ifdef _LP64 760 __ xorptr(rscratch1, rscratch1); 761 null_check_here = code_offset(); 762 __ movptr(as_Address(addr), rscratch1); 763 #else 764 __ movptr(as_Address(addr), NULL_WORD); 765 #endif 766 } 767 } else { 768 if (is_literal_address(addr)) { 769 ShouldNotReachHere(); 770 __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1); 771 } else { 772 #ifdef _LP64 773 __ movoop(rscratch1, c->as_jobject()); 774 if (UseCompressedOops && !wide) { 775 __ encode_heap_oop(rscratch1); 776 null_check_here = code_offset(); 777 __ movl(as_Address_lo(addr), rscratch1); 778 } else { 779 null_check_here = code_offset(); 780 __ movptr(as_Address_lo(addr), rscratch1); 781 } 782 #else 783 __ movoop(as_Address(addr), c->as_jobject(), noreg); 784 #endif 785 } 786 } 787 break; 788 789 case T_LONG: // fall through 790 case T_DOUBLE: 791 #ifdef _LP64 792 if (is_literal_address(addr)) { 793 ShouldNotReachHere(); 794 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits()); 795 } else { 796 __ movptr(r10, (intptr_t)c->as_jlong_bits()); 797 null_check_here = code_offset(); 798 __ movptr(as_Address_lo(addr), r10); 799 } 800 #else 801 // Always reachable in 32bit so this doesn't produce useless move literal 802 __ movptr(as_Address_hi(addr), c->as_jint_hi_bits()); 803 __ movptr(as_Address_lo(addr), c->as_jint_lo_bits()); 804 #endif // _LP64 805 break; 806 807 case T_BOOLEAN: // fall through 808 case T_BYTE: 809 __ movb(as_Address(addr), c->as_jint() & 0xFF); 810 break; 811 812 case T_CHAR: // fall through 813 case T_SHORT: 814 __ movw(as_Address(addr), c->as_jint() & 0xFFFF); 815 break; 816 817 default: 818 ShouldNotReachHere(); 819 }; 820 821 if (info != nullptr) { 822 add_debug_info_for_null_check(null_check_here, info); 823 } 824 } 825 826 827 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 828 assert(src->is_register(), "should not call otherwise"); 829 assert(dest->is_register(), "should not call otherwise"); 830 831 // move between cpu-registers 832 if (dest->is_single_cpu()) { 833 #ifdef _LP64 834 if (src->type() == T_LONG) { 835 // Can do LONG -> OBJECT 836 move_regs(src->as_register_lo(), dest->as_register()); 837 return; 838 } 839 #endif 840 assert(src->is_single_cpu(), "must match"); 841 if (src->type() == T_OBJECT) { 842 __ verify_oop(src->as_register()); 843 } 844 move_regs(src->as_register(), dest->as_register()); 845 846 } else if (dest->is_double_cpu()) { 847 #ifdef _LP64 848 if (is_reference_type(src->type())) { 849 // Surprising to me but we can see move of a long to t_object 850 __ verify_oop(src->as_register()); 851 move_regs(src->as_register(), dest->as_register_lo()); 852 return; 853 } 854 #endif 855 assert(src->is_double_cpu(), "must match"); 856 Register f_lo = src->as_register_lo(); 857 Register f_hi = src->as_register_hi(); 858 Register t_lo = dest->as_register_lo(); 859 Register t_hi = dest->as_register_hi(); 860 #ifdef _LP64 861 assert(f_hi == f_lo, "must be same"); 862 assert(t_hi == t_lo, "must be same"); 863 move_regs(f_lo, t_lo); 864 #else 865 assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation"); 866 867 868 if (f_lo == t_hi && f_hi == t_lo) { 869 swap_reg(f_lo, f_hi); 870 } else if (f_hi == t_lo) { 871 assert(f_lo != t_hi, "overwriting register"); 872 move_regs(f_hi, t_hi); 873 move_regs(f_lo, t_lo); 874 } else { 875 assert(f_hi != t_lo, "overwriting register"); 876 move_regs(f_lo, t_lo); 877 move_regs(f_hi, t_hi); 878 } 879 #endif // LP64 880 881 #ifndef _LP64 882 // special moves from fpu-register to xmm-register 883 // necessary for method results 884 } else if (src->is_single_xmm() && !dest->is_single_xmm()) { 885 __ movflt(Address(rsp, 0), src->as_xmm_float_reg()); 886 __ fld_s(Address(rsp, 0)); 887 } else if (src->is_double_xmm() && !dest->is_double_xmm()) { 888 __ movdbl(Address(rsp, 0), src->as_xmm_double_reg()); 889 __ fld_d(Address(rsp, 0)); 890 } else if (dest->is_single_xmm() && !src->is_single_xmm()) { 891 __ fstp_s(Address(rsp, 0)); 892 __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0)); 893 } else if (dest->is_double_xmm() && !src->is_double_xmm()) { 894 __ fstp_d(Address(rsp, 0)); 895 __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0)); 896 #endif // !_LP64 897 898 // move between xmm-registers 899 } else if (dest->is_single_xmm()) { 900 assert(src->is_single_xmm(), "must match"); 901 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg()); 902 } else if (dest->is_double_xmm()) { 903 assert(src->is_double_xmm(), "must match"); 904 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg()); 905 906 #ifndef _LP64 907 // move between fpu-registers (no instruction necessary because of fpu-stack) 908 } else if (dest->is_single_fpu() || dest->is_double_fpu()) { 909 assert(src->is_single_fpu() || src->is_double_fpu(), "must match"); 910 assert(src->fpu() == dest->fpu(), "currently should be nothing to do"); 911 #endif // !_LP64 912 913 } else { 914 ShouldNotReachHere(); 915 } 916 } 917 918 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 919 assert(src->is_register(), "should not call otherwise"); 920 assert(dest->is_stack(), "should not call otherwise"); 921 922 if (src->is_single_cpu()) { 923 Address dst = frame_map()->address_for_slot(dest->single_stack_ix()); 924 if (is_reference_type(type)) { 925 __ verify_oop(src->as_register()); 926 __ movptr (dst, src->as_register()); 927 } else if (type == T_METADATA || type == T_ADDRESS) { 928 __ movptr (dst, src->as_register()); 929 } else { 930 __ movl (dst, src->as_register()); 931 } 932 933 } else if (src->is_double_cpu()) { 934 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes); 935 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes); 936 __ movptr (dstLO, src->as_register_lo()); 937 NOT_LP64(__ movptr (dstHI, src->as_register_hi())); 938 939 } else if (src->is_single_xmm()) { 940 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix()); 941 __ movflt(dst_addr, src->as_xmm_float_reg()); 942 943 } else if (src->is_double_xmm()) { 944 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix()); 945 __ movdbl(dst_addr, src->as_xmm_double_reg()); 946 947 #ifndef _LP64 948 } else if (src->is_single_fpu()) { 949 assert(src->fpu_regnr() == 0, "argument must be on TOS"); 950 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix()); 951 if (pop_fpu_stack) __ fstp_s (dst_addr); 952 else __ fst_s (dst_addr); 953 954 } else if (src->is_double_fpu()) { 955 assert(src->fpu_regnrLo() == 0, "argument must be on TOS"); 956 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix()); 957 if (pop_fpu_stack) __ fstp_d (dst_addr); 958 else __ fst_d (dst_addr); 959 #endif // !_LP64 960 961 } else { 962 ShouldNotReachHere(); 963 } 964 } 965 966 967 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) { 968 LIR_Address* to_addr = dest->as_address_ptr(); 969 PatchingStub* patch = nullptr; 970 Register compressed_src = rscratch1; 971 972 if (is_reference_type(type)) { 973 __ verify_oop(src->as_register()); 974 #ifdef _LP64 975 if (UseCompressedOops && !wide) { 976 __ movptr(compressed_src, src->as_register()); 977 __ encode_heap_oop(compressed_src); 978 if (patch_code != lir_patch_none) { 979 info->oop_map()->set_narrowoop(compressed_src->as_VMReg()); 980 } 981 } 982 #endif 983 } 984 985 if (patch_code != lir_patch_none) { 986 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 987 Address toa = as_Address(to_addr); 988 assert(toa.disp() != 0, "must have"); 989 } 990 991 int null_check_here = code_offset(); 992 switch (type) { 993 case T_FLOAT: { 994 #ifdef _LP64 995 assert(src->is_single_xmm(), "not a float"); 996 __ movflt(as_Address(to_addr), src->as_xmm_float_reg()); 997 #else 998 if (src->is_single_xmm()) { 999 __ movflt(as_Address(to_addr), src->as_xmm_float_reg()); 1000 } else { 1001 assert(src->is_single_fpu(), "must be"); 1002 assert(src->fpu_regnr() == 0, "argument must be on TOS"); 1003 if (pop_fpu_stack) __ fstp_s(as_Address(to_addr)); 1004 else __ fst_s (as_Address(to_addr)); 1005 } 1006 #endif // _LP64 1007 break; 1008 } 1009 1010 case T_DOUBLE: { 1011 #ifdef _LP64 1012 assert(src->is_double_xmm(), "not a double"); 1013 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg()); 1014 #else 1015 if (src->is_double_xmm()) { 1016 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg()); 1017 } else { 1018 assert(src->is_double_fpu(), "must be"); 1019 assert(src->fpu_regnrLo() == 0, "argument must be on TOS"); 1020 if (pop_fpu_stack) __ fstp_d(as_Address(to_addr)); 1021 else __ fst_d (as_Address(to_addr)); 1022 } 1023 #endif // _LP64 1024 break; 1025 } 1026 1027 case T_ARRAY: // fall through 1028 case T_OBJECT: // fall through 1029 if (UseCompressedOops && !wide) { 1030 __ movl(as_Address(to_addr), compressed_src); 1031 } else { 1032 __ movptr(as_Address(to_addr), src->as_register()); 1033 } 1034 break; 1035 case T_METADATA: 1036 // We get here to store a method pointer to the stack to pass to 1037 // a dtrace runtime call. This can't work on 64 bit with 1038 // compressed klass ptrs: T_METADATA can be a compressed klass 1039 // ptr or a 64 bit method pointer. 1040 LP64_ONLY(ShouldNotReachHere()); 1041 __ movptr(as_Address(to_addr), src->as_register()); 1042 break; 1043 case T_ADDRESS: 1044 __ movptr(as_Address(to_addr), src->as_register()); 1045 break; 1046 case T_INT: 1047 __ movl(as_Address(to_addr), src->as_register()); 1048 break; 1049 1050 case T_LONG: { 1051 Register from_lo = src->as_register_lo(); 1052 Register from_hi = src->as_register_hi(); 1053 #ifdef _LP64 1054 __ movptr(as_Address_lo(to_addr), from_lo); 1055 #else 1056 Register base = to_addr->base()->as_register(); 1057 Register index = noreg; 1058 if (to_addr->index()->is_register()) { 1059 index = to_addr->index()->as_register(); 1060 } 1061 if (base == from_lo || index == from_lo) { 1062 assert(base != from_hi, "can't be"); 1063 assert(index == noreg || (index != base && index != from_hi), "can't handle this"); 1064 __ movl(as_Address_hi(to_addr), from_hi); 1065 if (patch != nullptr) { 1066 patching_epilog(patch, lir_patch_high, base, info); 1067 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1068 patch_code = lir_patch_low; 1069 } 1070 __ movl(as_Address_lo(to_addr), from_lo); 1071 } else { 1072 assert(index == noreg || (index != base && index != from_lo), "can't handle this"); 1073 __ movl(as_Address_lo(to_addr), from_lo); 1074 if (patch != nullptr) { 1075 patching_epilog(patch, lir_patch_low, base, info); 1076 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1077 patch_code = lir_patch_high; 1078 } 1079 __ movl(as_Address_hi(to_addr), from_hi); 1080 } 1081 #endif // _LP64 1082 break; 1083 } 1084 1085 case T_BYTE: // fall through 1086 case T_BOOLEAN: { 1087 Register src_reg = src->as_register(); 1088 Address dst_addr = as_Address(to_addr); 1089 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6"); 1090 __ movb(dst_addr, src_reg); 1091 break; 1092 } 1093 1094 case T_CHAR: // fall through 1095 case T_SHORT: 1096 __ movw(as_Address(to_addr), src->as_register()); 1097 break; 1098 1099 default: 1100 ShouldNotReachHere(); 1101 } 1102 if (info != nullptr) { 1103 add_debug_info_for_null_check(null_check_here, info); 1104 } 1105 1106 if (patch_code != lir_patch_none) { 1107 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info); 1108 } 1109 } 1110 1111 1112 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 1113 assert(src->is_stack(), "should not call otherwise"); 1114 assert(dest->is_register(), "should not call otherwise"); 1115 1116 if (dest->is_single_cpu()) { 1117 if (is_reference_type(type)) { 1118 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 1119 __ verify_oop(dest->as_register()); 1120 } else if (type == T_METADATA || type == T_ADDRESS) { 1121 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 1122 } else { 1123 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 1124 } 1125 1126 } else if (dest->is_double_cpu()) { 1127 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes); 1128 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes); 1129 __ movptr(dest->as_register_lo(), src_addr_LO); 1130 NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI)); 1131 1132 } else if (dest->is_single_xmm()) { 1133 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix()); 1134 __ movflt(dest->as_xmm_float_reg(), src_addr); 1135 1136 } else if (dest->is_double_xmm()) { 1137 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix()); 1138 __ movdbl(dest->as_xmm_double_reg(), src_addr); 1139 1140 #ifndef _LP64 1141 } else if (dest->is_single_fpu()) { 1142 assert(dest->fpu_regnr() == 0, "dest must be TOS"); 1143 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix()); 1144 __ fld_s(src_addr); 1145 1146 } else if (dest->is_double_fpu()) { 1147 assert(dest->fpu_regnrLo() == 0, "dest must be TOS"); 1148 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix()); 1149 __ fld_d(src_addr); 1150 #endif // _LP64 1151 1152 } else { 1153 ShouldNotReachHere(); 1154 } 1155 } 1156 1157 1158 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 1159 if (src->is_single_stack()) { 1160 if (is_reference_type(type)) { 1161 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix())); 1162 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix())); 1163 } else { 1164 #ifndef _LP64 1165 __ pushl(frame_map()->address_for_slot(src ->single_stack_ix())); 1166 __ popl (frame_map()->address_for_slot(dest->single_stack_ix())); 1167 #else 1168 //no pushl on 64bits 1169 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix())); 1170 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1); 1171 #endif 1172 } 1173 1174 } else if (src->is_double_stack()) { 1175 #ifdef _LP64 1176 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix())); 1177 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix())); 1178 #else 1179 __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0)); 1180 // push and pop the part at src + wordSize, adding wordSize for the previous push 1181 __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize)); 1182 __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize)); 1183 __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0)); 1184 #endif // _LP64 1185 1186 } else { 1187 ShouldNotReachHere(); 1188 } 1189 } 1190 1191 1192 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) { 1193 assert(src->is_address(), "should not call otherwise"); 1194 assert(dest->is_register(), "should not call otherwise"); 1195 1196 LIR_Address* addr = src->as_address_ptr(); 1197 Address from_addr = as_Address(addr); 1198 1199 if (addr->base()->type() == T_OBJECT) { 1200 __ verify_oop(addr->base()->as_pointer_register()); 1201 } 1202 1203 switch (type) { 1204 case T_BOOLEAN: // fall through 1205 case T_BYTE: // fall through 1206 case T_CHAR: // fall through 1207 case T_SHORT: 1208 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) { 1209 // on pre P6 processors we may get partial register stalls 1210 // so blow away the value of to_rinfo before loading a 1211 // partial word into it. Do it here so that it precedes 1212 // the potential patch point below. 1213 __ xorptr(dest->as_register(), dest->as_register()); 1214 } 1215 break; 1216 default: 1217 break; 1218 } 1219 1220 PatchingStub* patch = nullptr; 1221 if (patch_code != lir_patch_none) { 1222 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1223 assert(from_addr.disp() != 0, "must have"); 1224 } 1225 if (info != nullptr) { 1226 add_debug_info_for_null_check_here(info); 1227 } 1228 1229 switch (type) { 1230 case T_FLOAT: { 1231 if (dest->is_single_xmm()) { 1232 __ movflt(dest->as_xmm_float_reg(), from_addr); 1233 } else { 1234 #ifndef _LP64 1235 assert(dest->is_single_fpu(), "must be"); 1236 assert(dest->fpu_regnr() == 0, "dest must be TOS"); 1237 __ fld_s(from_addr); 1238 #else 1239 ShouldNotReachHere(); 1240 #endif // !LP64 1241 } 1242 break; 1243 } 1244 1245 case T_DOUBLE: { 1246 if (dest->is_double_xmm()) { 1247 __ movdbl(dest->as_xmm_double_reg(), from_addr); 1248 } else { 1249 #ifndef _LP64 1250 assert(dest->is_double_fpu(), "must be"); 1251 assert(dest->fpu_regnrLo() == 0, "dest must be TOS"); 1252 __ fld_d(from_addr); 1253 #else 1254 ShouldNotReachHere(); 1255 #endif // !LP64 1256 } 1257 break; 1258 } 1259 1260 case T_OBJECT: // fall through 1261 case T_ARRAY: // fall through 1262 if (UseCompressedOops && !wide) { 1263 __ movl(dest->as_register(), from_addr); 1264 } else { 1265 __ movptr(dest->as_register(), from_addr); 1266 } 1267 break; 1268 1269 case T_ADDRESS: 1270 __ movptr(dest->as_register(), from_addr); 1271 break; 1272 case T_INT: 1273 __ movl(dest->as_register(), from_addr); 1274 break; 1275 1276 case T_LONG: { 1277 Register to_lo = dest->as_register_lo(); 1278 Register to_hi = dest->as_register_hi(); 1279 #ifdef _LP64 1280 __ movptr(to_lo, as_Address_lo(addr)); 1281 #else 1282 Register base = addr->base()->as_register(); 1283 Register index = noreg; 1284 if (addr->index()->is_register()) { 1285 index = addr->index()->as_register(); 1286 } 1287 if ((base == to_lo && index == to_hi) || 1288 (base == to_hi && index == to_lo)) { 1289 // addresses with 2 registers are only formed as a result of 1290 // array access so this code will never have to deal with 1291 // patches or null checks. 1292 assert(info == nullptr && patch == nullptr, "must be"); 1293 __ lea(to_hi, as_Address(addr)); 1294 __ movl(to_lo, Address(to_hi, 0)); 1295 __ movl(to_hi, Address(to_hi, BytesPerWord)); 1296 } else if (base == to_lo || index == to_lo) { 1297 assert(base != to_hi, "can't be"); 1298 assert(index == noreg || (index != base && index != to_hi), "can't handle this"); 1299 __ movl(to_hi, as_Address_hi(addr)); 1300 if (patch != nullptr) { 1301 patching_epilog(patch, lir_patch_high, base, info); 1302 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1303 patch_code = lir_patch_low; 1304 } 1305 __ movl(to_lo, as_Address_lo(addr)); 1306 } else { 1307 assert(index == noreg || (index != base && index != to_lo), "can't handle this"); 1308 __ movl(to_lo, as_Address_lo(addr)); 1309 if (patch != nullptr) { 1310 patching_epilog(patch, lir_patch_low, base, info); 1311 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1312 patch_code = lir_patch_high; 1313 } 1314 __ movl(to_hi, as_Address_hi(addr)); 1315 } 1316 #endif // _LP64 1317 break; 1318 } 1319 1320 case T_BOOLEAN: // fall through 1321 case T_BYTE: { 1322 Register dest_reg = dest->as_register(); 1323 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6"); 1324 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) { 1325 __ movsbl(dest_reg, from_addr); 1326 } else { 1327 __ movb(dest_reg, from_addr); 1328 __ shll(dest_reg, 24); 1329 __ sarl(dest_reg, 24); 1330 } 1331 break; 1332 } 1333 1334 case T_CHAR: { 1335 Register dest_reg = dest->as_register(); 1336 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6"); 1337 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) { 1338 __ movzwl(dest_reg, from_addr); 1339 } else { 1340 __ movw(dest_reg, from_addr); 1341 } 1342 break; 1343 } 1344 1345 case T_SHORT: { 1346 Register dest_reg = dest->as_register(); 1347 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) { 1348 __ movswl(dest_reg, from_addr); 1349 } else { 1350 __ movw(dest_reg, from_addr); 1351 __ shll(dest_reg, 16); 1352 __ sarl(dest_reg, 16); 1353 } 1354 break; 1355 } 1356 1357 default: 1358 ShouldNotReachHere(); 1359 } 1360 1361 if (patch != nullptr) { 1362 patching_epilog(patch, patch_code, addr->base()->as_register(), info); 1363 } 1364 1365 if (is_reference_type(type)) { 1366 #ifdef _LP64 1367 if (UseCompressedOops && !wide) { 1368 __ decode_heap_oop(dest->as_register()); 1369 } 1370 #endif 1371 1372 __ verify_oop(dest->as_register()); 1373 } 1374 } 1375 1376 1377 NEEDS_CLEANUP; // This could be static? 1378 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const { 1379 int elem_size = type2aelembytes(type); 1380 switch (elem_size) { 1381 case 1: return Address::times_1; 1382 case 2: return Address::times_2; 1383 case 4: return Address::times_4; 1384 case 8: return Address::times_8; 1385 } 1386 ShouldNotReachHere(); 1387 return Address::no_scale; 1388 } 1389 1390 1391 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1392 switch (op->code()) { 1393 case lir_idiv: 1394 case lir_irem: 1395 arithmetic_idiv(op->code(), 1396 op->in_opr1(), 1397 op->in_opr2(), 1398 op->in_opr3(), 1399 op->result_opr(), 1400 op->info()); 1401 break; 1402 case lir_fmad: 1403 __ fmad(op->result_opr()->as_xmm_double_reg(), 1404 op->in_opr1()->as_xmm_double_reg(), 1405 op->in_opr2()->as_xmm_double_reg(), 1406 op->in_opr3()->as_xmm_double_reg()); 1407 break; 1408 case lir_fmaf: 1409 __ fmaf(op->result_opr()->as_xmm_float_reg(), 1410 op->in_opr1()->as_xmm_float_reg(), 1411 op->in_opr2()->as_xmm_float_reg(), 1412 op->in_opr3()->as_xmm_float_reg()); 1413 break; 1414 default: ShouldNotReachHere(); break; 1415 } 1416 } 1417 1418 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1419 #ifdef ASSERT 1420 assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label"); 1421 if (op->block() != nullptr) _branch_target_blocks.append(op->block()); 1422 if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock()); 1423 #endif 1424 1425 if (op->cond() == lir_cond_always) { 1426 if (op->info() != nullptr) add_debug_info_for_branch(op->info()); 1427 __ jmp (*(op->label())); 1428 } else { 1429 Assembler::Condition acond = Assembler::zero; 1430 if (op->code() == lir_cond_float_branch) { 1431 assert(op->ublock() != nullptr, "must have unordered successor"); 1432 __ jcc(Assembler::parity, *(op->ublock()->label())); 1433 switch(op->cond()) { 1434 case lir_cond_equal: acond = Assembler::equal; break; 1435 case lir_cond_notEqual: acond = Assembler::notEqual; break; 1436 case lir_cond_less: acond = Assembler::below; break; 1437 case lir_cond_lessEqual: acond = Assembler::belowEqual; break; 1438 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break; 1439 case lir_cond_greater: acond = Assembler::above; break; 1440 default: ShouldNotReachHere(); 1441 } 1442 } else { 1443 switch (op->cond()) { 1444 case lir_cond_equal: acond = Assembler::equal; break; 1445 case lir_cond_notEqual: acond = Assembler::notEqual; break; 1446 case lir_cond_less: acond = Assembler::less; break; 1447 case lir_cond_lessEqual: acond = Assembler::lessEqual; break; 1448 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break; 1449 case lir_cond_greater: acond = Assembler::greater; break; 1450 case lir_cond_belowEqual: acond = Assembler::belowEqual; break; 1451 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break; 1452 default: ShouldNotReachHere(); 1453 } 1454 } 1455 __ jcc(acond,*(op->label())); 1456 } 1457 } 1458 1459 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1460 LIR_Opr src = op->in_opr(); 1461 LIR_Opr dest = op->result_opr(); 1462 1463 switch (op->bytecode()) { 1464 case Bytecodes::_i2l: 1465 #ifdef _LP64 1466 __ movl2ptr(dest->as_register_lo(), src->as_register()); 1467 #else 1468 move_regs(src->as_register(), dest->as_register_lo()); 1469 move_regs(src->as_register(), dest->as_register_hi()); 1470 __ sarl(dest->as_register_hi(), 31); 1471 #endif // LP64 1472 break; 1473 1474 case Bytecodes::_l2i: 1475 #ifdef _LP64 1476 __ movl(dest->as_register(), src->as_register_lo()); 1477 #else 1478 move_regs(src->as_register_lo(), dest->as_register()); 1479 #endif 1480 break; 1481 1482 case Bytecodes::_i2b: 1483 move_regs(src->as_register(), dest->as_register()); 1484 __ sign_extend_byte(dest->as_register()); 1485 break; 1486 1487 case Bytecodes::_i2c: 1488 move_regs(src->as_register(), dest->as_register()); 1489 __ andl(dest->as_register(), 0xFFFF); 1490 break; 1491 1492 case Bytecodes::_i2s: 1493 move_regs(src->as_register(), dest->as_register()); 1494 __ sign_extend_short(dest->as_register()); 1495 break; 1496 1497 1498 #ifdef _LP64 1499 case Bytecodes::_f2d: 1500 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg()); 1501 break; 1502 1503 case Bytecodes::_d2f: 1504 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg()); 1505 break; 1506 1507 case Bytecodes::_i2f: 1508 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register()); 1509 break; 1510 1511 case Bytecodes::_i2d: 1512 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register()); 1513 break; 1514 1515 case Bytecodes::_l2f: 1516 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo()); 1517 break; 1518 1519 case Bytecodes::_l2d: 1520 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo()); 1521 break; 1522 1523 case Bytecodes::_f2i: 1524 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg()); 1525 break; 1526 1527 case Bytecodes::_d2i: 1528 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg()); 1529 break; 1530 1531 case Bytecodes::_f2l: 1532 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg()); 1533 break; 1534 1535 case Bytecodes::_d2l: 1536 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg()); 1537 break; 1538 #else 1539 case Bytecodes::_f2d: 1540 case Bytecodes::_d2f: 1541 if (dest->is_single_xmm()) { 1542 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg()); 1543 } else if (dest->is_double_xmm()) { 1544 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg()); 1545 } else { 1546 assert(src->fpu() == dest->fpu(), "register must be equal"); 1547 // do nothing (float result is rounded later through spilling) 1548 } 1549 break; 1550 1551 case Bytecodes::_i2f: 1552 case Bytecodes::_i2d: 1553 if (dest->is_single_xmm()) { 1554 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register()); 1555 } else if (dest->is_double_xmm()) { 1556 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register()); 1557 } else { 1558 assert(dest->fpu() == 0, "result must be on TOS"); 1559 __ movl(Address(rsp, 0), src->as_register()); 1560 __ fild_s(Address(rsp, 0)); 1561 } 1562 break; 1563 1564 case Bytecodes::_l2f: 1565 case Bytecodes::_l2d: 1566 assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)"); 1567 assert(dest->fpu() == 0, "result must be on TOS"); 1568 __ movptr(Address(rsp, 0), src->as_register_lo()); 1569 __ movl(Address(rsp, BytesPerWord), src->as_register_hi()); 1570 __ fild_d(Address(rsp, 0)); 1571 // float result is rounded later through spilling 1572 break; 1573 1574 case Bytecodes::_f2i: 1575 case Bytecodes::_d2i: 1576 if (src->is_single_xmm()) { 1577 __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg()); 1578 } else if (src->is_double_xmm()) { 1579 __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg()); 1580 } else { 1581 assert(src->fpu() == 0, "input must be on TOS"); 1582 __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_trunc())); 1583 __ fist_s(Address(rsp, 0)); 1584 __ movl(dest->as_register(), Address(rsp, 0)); 1585 __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std())); 1586 } 1587 // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 1588 assert(op->stub() != nullptr, "stub required"); 1589 __ cmpl(dest->as_register(), 0x80000000); 1590 __ jcc(Assembler::equal, *op->stub()->entry()); 1591 __ bind(*op->stub()->continuation()); 1592 break; 1593 1594 case Bytecodes::_f2l: 1595 case Bytecodes::_d2l: 1596 assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)"); 1597 assert(src->fpu() == 0, "input must be on TOS"); 1598 assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers"); 1599 1600 // instruction sequence too long to inline it here 1601 { 1602 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::fpu2long_stub_id))); 1603 } 1604 break; 1605 #endif // _LP64 1606 1607 default: ShouldNotReachHere(); 1608 } 1609 } 1610 1611 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1612 if (op->init_check()) { 1613 add_debug_info_for_null_check_here(op->stub()->info()); 1614 // init_state needs acquire, but x86 is TSO, and so we are already good. 1615 __ cmpb(Address(op->klass()->as_register(), 1616 InstanceKlass::init_state_offset()), 1617 InstanceKlass::fully_initialized); 1618 __ jcc(Assembler::notEqual, *op->stub()->entry()); 1619 } 1620 __ allocate_object(op->obj()->as_register(), 1621 op->tmp1()->as_register(), 1622 op->tmp2()->as_register(), 1623 op->header_size(), 1624 op->object_size(), 1625 op->klass()->as_register(), 1626 *op->stub()->entry()); 1627 __ bind(*op->stub()->continuation()); 1628 } 1629 1630 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1631 Register len = op->len()->as_register(); 1632 LP64_ONLY( __ movslq(len, len); ) 1633 1634 if (UseSlowPath || op->is_null_free() || 1635 (!UseFastNewObjectArray && is_reference_type(op->type())) || 1636 (!UseFastNewTypeArray && !is_reference_type(op->type()))) { 1637 __ jmp(*op->stub()->entry()); 1638 } else { 1639 Register tmp1 = op->tmp1()->as_register(); 1640 Register tmp2 = op->tmp2()->as_register(); 1641 Register tmp3 = op->tmp3()->as_register(); 1642 if (len == tmp1) { 1643 tmp1 = tmp3; 1644 } else if (len == tmp2) { 1645 tmp2 = tmp3; 1646 } else if (len == tmp3) { 1647 // everything is ok 1648 } else { 1649 __ mov(tmp3, len); 1650 } 1651 __ allocate_array(op->obj()->as_register(), 1652 len, 1653 tmp1, 1654 tmp2, 1655 arrayOopDesc::base_offset_in_bytes(op->type()), 1656 array_element_size(op->type()), 1657 op->klass()->as_register(), 1658 *op->stub()->entry(), 1659 op->zero_array()); 1660 } 1661 __ bind(*op->stub()->continuation()); 1662 } 1663 1664 void LIR_Assembler::type_profile_helper(Register mdo, 1665 ciMethodData *md, ciProfileData *data, 1666 Register recv, Label* update_done) { 1667 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1668 Label next_test; 1669 // See if the receiver is receiver[n]. 1670 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1671 __ jccb(Assembler::notEqual, next_test); 1672 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); 1673 __ addptr(data_addr, DataLayout::counter_increment); 1674 __ jmp(*update_done); 1675 __ bind(next_test); 1676 } 1677 1678 // Didn't find receiver; find next empty slot and fill it in 1679 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1680 Label next_test; 1681 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))); 1682 __ cmpptr(recv_addr, NULL_WORD); 1683 __ jccb(Assembler::notEqual, next_test); 1684 __ movptr(recv_addr, recv); 1685 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment); 1686 __ jmp(*update_done); 1687 __ bind(next_test); 1688 } 1689 } 1690 1691 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1692 // we always need a stub for the failure case. 1693 CodeStub* stub = op->stub(); 1694 Register obj = op->object()->as_register(); 1695 Register k_RInfo = op->tmp1()->as_register(); 1696 Register klass_RInfo = op->tmp2()->as_register(); 1697 Register dst = op->result_opr()->as_register(); 1698 ciKlass* k = op->klass(); 1699 Register Rtmp1 = noreg; 1700 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 1701 1702 // check if it needs to be profiled 1703 ciMethodData* md = nullptr; 1704 ciProfileData* data = nullptr; 1705 1706 if (op->should_profile()) { 1707 ciMethod* method = op->profiled_method(); 1708 assert(method != nullptr, "Should have method"); 1709 int bci = op->profiled_bci(); 1710 md = method->method_data_or_null(); 1711 assert(md != nullptr, "Sanity"); 1712 data = md->bci_to_data(bci); 1713 assert(data != nullptr, "need data for type check"); 1714 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1715 } 1716 Label* success_target = success; 1717 Label* failure_target = failure; 1718 1719 if (obj == k_RInfo) { 1720 k_RInfo = dst; 1721 } else if (obj == klass_RInfo) { 1722 klass_RInfo = dst; 1723 } 1724 if (k->is_loaded() && !UseCompressedClassPointers) { 1725 select_different_registers(obj, dst, k_RInfo, klass_RInfo); 1726 } else { 1727 Rtmp1 = op->tmp3()->as_register(); 1728 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1729 } 1730 1731 assert_different_registers(obj, k_RInfo, klass_RInfo); 1732 1733 if (op->need_null_check()) { 1734 __ testptr(obj, obj); 1735 if (op->should_profile()) { 1736 Label not_null; 1737 Register mdo = klass_RInfo; 1738 __ mov_metadata(mdo, md->constant_encoding()); 1739 __ jccb(Assembler::notEqual, not_null); 1740 // Object is null; update MDO and exit 1741 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset())); 1742 int header_bits = BitData::null_seen_byte_constant(); 1743 __ orb(data_addr, header_bits); 1744 __ jmp(*obj_is_null); 1745 __ bind(not_null); 1746 1747 Label update_done; 1748 Register recv = k_RInfo; 1749 __ load_klass(recv, obj, tmp_load_klass); 1750 type_profile_helper(mdo, md, data, recv, &update_done); 1751 1752 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1753 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment); 1754 1755 __ bind(update_done); 1756 } else { 1757 __ jcc(Assembler::equal, *obj_is_null); 1758 } 1759 } 1760 1761 if (!k->is_loaded()) { 1762 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1763 } else { 1764 #ifdef _LP64 1765 __ mov_metadata(k_RInfo, k->constant_encoding()); 1766 #endif // _LP64 1767 } 1768 __ verify_oop(obj); 1769 1770 if (op->fast_check()) { 1771 // get object class 1772 // not a safepoint as obj null check happens earlier 1773 #ifdef _LP64 1774 if (UseCompressedClassPointers) { 1775 __ load_klass(Rtmp1, obj, tmp_load_klass); 1776 __ cmpptr(k_RInfo, Rtmp1); 1777 } else { 1778 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes())); 1779 } 1780 #else 1781 if (k->is_loaded()) { 1782 __ cmpklass(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding()); 1783 } else { 1784 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes())); 1785 } 1786 #endif 1787 __ jcc(Assembler::notEqual, *failure_target); 1788 // successful cast, fall through to profile or jump 1789 } else { 1790 // get object class 1791 // not a safepoint as obj null check happens earlier 1792 __ load_klass(klass_RInfo, obj, tmp_load_klass); 1793 if (k->is_loaded()) { 1794 // See if we get an immediate positive hit 1795 #ifdef _LP64 1796 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset())); 1797 #else 1798 __ cmpklass(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding()); 1799 #endif // _LP64 1800 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1801 __ jcc(Assembler::notEqual, *failure_target); 1802 // successful cast, fall through to profile or jump 1803 } else { 1804 // See if we get an immediate positive hit 1805 __ jcc(Assembler::equal, *success_target); 1806 // check for self 1807 #ifdef _LP64 1808 __ cmpptr(klass_RInfo, k_RInfo); 1809 #else 1810 __ cmpklass(klass_RInfo, k->constant_encoding()); 1811 #endif // _LP64 1812 __ jcc(Assembler::equal, *success_target); 1813 1814 __ push(klass_RInfo); 1815 #ifdef _LP64 1816 __ push(k_RInfo); 1817 #else 1818 __ pushklass(k->constant_encoding(), noreg); 1819 #endif // _LP64 1820 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id))); 1821 __ pop(klass_RInfo); 1822 __ pop(klass_RInfo); 1823 // result is a boolean 1824 __ testl(klass_RInfo, klass_RInfo); 1825 __ jcc(Assembler::equal, *failure_target); 1826 // successful cast, fall through to profile or jump 1827 } 1828 } else { 1829 // perform the fast part of the checking logic 1830 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr); 1831 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1832 __ push(klass_RInfo); 1833 __ push(k_RInfo); 1834 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id))); 1835 __ pop(klass_RInfo); 1836 __ pop(k_RInfo); 1837 // result is a boolean 1838 __ testl(k_RInfo, k_RInfo); 1839 __ jcc(Assembler::equal, *failure_target); 1840 // successful cast, fall through to profile or jump 1841 } 1842 } 1843 __ jmp(*success); 1844 } 1845 1846 1847 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1848 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 1849 LIR_Code code = op->code(); 1850 if (code == lir_store_check) { 1851 Register value = op->object()->as_register(); 1852 Register array = op->array()->as_register(); 1853 Register k_RInfo = op->tmp1()->as_register(); 1854 Register klass_RInfo = op->tmp2()->as_register(); 1855 Register Rtmp1 = op->tmp3()->as_register(); 1856 1857 CodeStub* stub = op->stub(); 1858 1859 // check if it needs to be profiled 1860 ciMethodData* md = nullptr; 1861 ciProfileData* data = nullptr; 1862 1863 if (op->should_profile()) { 1864 ciMethod* method = op->profiled_method(); 1865 assert(method != nullptr, "Should have method"); 1866 int bci = op->profiled_bci(); 1867 md = method->method_data_or_null(); 1868 assert(md != nullptr, "Sanity"); 1869 data = md->bci_to_data(bci); 1870 assert(data != nullptr, "need data for type check"); 1871 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1872 } 1873 Label done; 1874 Label* success_target = &done; 1875 Label* failure_target = stub->entry(); 1876 1877 __ testptr(value, value); 1878 if (op->should_profile()) { 1879 Label not_null; 1880 Register mdo = klass_RInfo; 1881 __ mov_metadata(mdo, md->constant_encoding()); 1882 __ jccb(Assembler::notEqual, not_null); 1883 // Object is null; update MDO and exit 1884 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset())); 1885 int header_bits = BitData::null_seen_byte_constant(); 1886 __ orb(data_addr, header_bits); 1887 __ jmp(done); 1888 __ bind(not_null); 1889 1890 Label update_done; 1891 Register recv = k_RInfo; 1892 __ load_klass(recv, value, tmp_load_klass); 1893 type_profile_helper(mdo, md, data, recv, &update_done); 1894 1895 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1896 __ addptr(counter_addr, DataLayout::counter_increment); 1897 __ bind(update_done); 1898 } else { 1899 __ jcc(Assembler::equal, done); 1900 } 1901 1902 add_debug_info_for_null_check_here(op->info_for_exception()); 1903 __ load_klass(k_RInfo, array, tmp_load_klass); 1904 __ load_klass(klass_RInfo, value, tmp_load_klass); 1905 1906 // get instance klass (it's already uncompressed) 1907 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1908 // perform the fast part of the checking logic 1909 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr); 1910 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1911 __ push(klass_RInfo); 1912 __ push(k_RInfo); 1913 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id))); 1914 __ pop(klass_RInfo); 1915 __ pop(k_RInfo); 1916 // result is a boolean 1917 __ testl(k_RInfo, k_RInfo); 1918 __ jcc(Assembler::equal, *failure_target); 1919 // fall through to the success case 1920 1921 __ bind(done); 1922 } else 1923 if (code == lir_checkcast) { 1924 Register obj = op->object()->as_register(); 1925 Register dst = op->result_opr()->as_register(); 1926 Label success; 1927 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1928 __ bind(success); 1929 if (dst != obj) { 1930 __ mov(dst, obj); 1931 } 1932 } else 1933 if (code == lir_instanceof) { 1934 Register obj = op->object()->as_register(); 1935 Register dst = op->result_opr()->as_register(); 1936 Label success, failure, done; 1937 emit_typecheck_helper(op, &success, &failure, &failure); 1938 __ bind(failure); 1939 __ xorptr(dst, dst); 1940 __ jmpb(done); 1941 __ bind(success); 1942 __ movptr(dst, 1); 1943 __ bind(done); 1944 } else { 1945 ShouldNotReachHere(); 1946 } 1947 1948 } 1949 1950 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) { 1951 // We are loading/storing from/to an array that *may* be a flat array (the 1952 // declared type is Object[], abstract[], interface[] or VT.ref[]). 1953 // If this array is a flat array, take the slow path. 1954 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry()); 1955 if (!op->value()->is_illegal()) { 1956 // The array is not a flat array, but it might be null-free. If we are storing 1957 // a null into a null-free array, take the slow path (which will throw NPE). 1958 Label skip; 1959 __ cmpptr(op->value()->as_register(), NULL_WORD); 1960 __ jcc(Assembler::notEqual, skip); 1961 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry()); 1962 __ bind(skip); 1963 } 1964 } 1965 1966 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) { 1967 // We are storing into an array that *may* be null-free (the declared type is 1968 // Object[], abstract[], interface[] or VT.ref[]). 1969 Label test_mark_word; 1970 Register tmp = op->tmp()->as_register(); 1971 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes())); 1972 __ testl(tmp, markWord::unlocked_value); 1973 __ jccb(Assembler::notZero, test_mark_word); 1974 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1); 1975 __ bind(test_mark_word); 1976 __ testl(tmp, markWord::null_free_array_bit_in_place); 1977 } 1978 1979 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) { 1980 Label L_oops_equal; 1981 Label L_oops_not_equal; 1982 Label L_end; 1983 1984 Register left = op->left()->as_register(); 1985 Register right = op->right()->as_register(); 1986 1987 __ cmpptr(left, right); 1988 __ jcc(Assembler::equal, L_oops_equal); 1989 1990 // (1) Null check -- if one of the operands is null, the other must not be null (because 1991 // the two references are not equal), so they are not substitutable, 1992 // FIXME: do null check only if the operand is nullable 1993 __ testptr(left, right); 1994 __ jcc(Assembler::zero, L_oops_not_equal); 1995 1996 ciKlass* left_klass = op->left_klass(); 1997 ciKlass* right_klass = op->right_klass(); 1998 1999 // (2) Inline type check -- if either of the operands is not a inline type, 2000 // they are not substitutable. We do this only if we are not sure that the 2001 // operands are inline type 2002 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node. 2003 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) { 2004 Register tmp1 = op->tmp1()->as_register(); 2005 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern); 2006 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes())); 2007 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes())); 2008 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern); 2009 __ jcc(Assembler::notEqual, L_oops_not_equal); 2010 } 2011 2012 // (3) Same klass check: if the operands are of different klasses, they are not substitutable. 2013 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) { 2014 // No need to load klass -- the operands are statically known to be the same inline klass. 2015 __ jmp(*op->stub()->entry()); 2016 } else { 2017 Register left_klass_op = op->left_klass_op()->as_register(); 2018 Register right_klass_op = op->right_klass_op()->as_register(); 2019 2020 if (UseCompressedClassPointers) { 2021 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes())); 2022 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes())); 2023 __ cmpl(left_klass_op, right_klass_op); 2024 } else { 2025 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes())); 2026 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes())); 2027 __ cmpptr(left_klass_op, right_klass_op); 2028 } 2029 2030 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check 2031 // fall through to L_oops_not_equal 2032 } 2033 2034 __ bind(L_oops_not_equal); 2035 move(op->not_equal_result(), op->result_opr()); 2036 __ jmp(L_end); 2037 2038 __ bind(L_oops_equal); 2039 move(op->equal_result(), op->result_opr()); 2040 __ jmp(L_end); 2041 2042 // We've returned from the stub. RAX contains 0x0 IFF the two 2043 // operands are not substitutable. (Don't compare against 0x1 in case the 2044 // C compiler is naughty) 2045 __ bind(*op->stub()->continuation()); 2046 __ cmpl(rax, 0); 2047 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal 2048 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal 2049 // fall-through 2050 __ bind(L_end); 2051 } 2052 2053 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 2054 if (LP64_ONLY(false &&) op->code() == lir_cas_long) { 2055 assert(op->cmp_value()->as_register_lo() == rax, "wrong register"); 2056 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register"); 2057 assert(op->new_value()->as_register_lo() == rbx, "wrong register"); 2058 assert(op->new_value()->as_register_hi() == rcx, "wrong register"); 2059 Register addr = op->addr()->as_register(); 2060 __ lock(); 2061 NOT_LP64(__ cmpxchg8(Address(addr, 0))); 2062 2063 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) { 2064 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");) 2065 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo()); 2066 Register newval = op->new_value()->as_register(); 2067 Register cmpval = op->cmp_value()->as_register(); 2068 assert(cmpval == rax, "wrong register"); 2069 assert(newval != noreg, "new val must be register"); 2070 assert(cmpval != newval, "cmp and new values must be in different registers"); 2071 assert(cmpval != addr, "cmp and addr must be in different registers"); 2072 assert(newval != addr, "new value and addr must be in different registers"); 2073 2074 if ( op->code() == lir_cas_obj) { 2075 #ifdef _LP64 2076 if (UseCompressedOops) { 2077 __ encode_heap_oop(cmpval); 2078 __ mov(rscratch1, newval); 2079 __ encode_heap_oop(rscratch1); 2080 __ lock(); 2081 // cmpval (rax) is implicitly used by this instruction 2082 __ cmpxchgl(rscratch1, Address(addr, 0)); 2083 } else 2084 #endif 2085 { 2086 __ lock(); 2087 __ cmpxchgptr(newval, Address(addr, 0)); 2088 } 2089 } else { 2090 assert(op->code() == lir_cas_int, "lir_cas_int expected"); 2091 __ lock(); 2092 __ cmpxchgl(newval, Address(addr, 0)); 2093 } 2094 #ifdef _LP64 2095 } else if (op->code() == lir_cas_long) { 2096 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo()); 2097 Register newval = op->new_value()->as_register_lo(); 2098 Register cmpval = op->cmp_value()->as_register_lo(); 2099 assert(cmpval == rax, "wrong register"); 2100 assert(newval != noreg, "new val must be register"); 2101 assert(cmpval != newval, "cmp and new values must be in different registers"); 2102 assert(cmpval != addr, "cmp and addr must be in different registers"); 2103 assert(newval != addr, "new value and addr must be in different registers"); 2104 __ lock(); 2105 __ cmpxchgq(newval, Address(addr, 0)); 2106 #endif // _LP64 2107 } else { 2108 Unimplemented(); 2109 } 2110 } 2111 2112 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) { 2113 assert(dst->is_cpu_register(), "must be"); 2114 assert(dst->type() == src->type(), "must be"); 2115 2116 if (src->is_cpu_register()) { 2117 reg2reg(src, dst); 2118 } else if (src->is_stack()) { 2119 stack2reg(src, dst, dst->type()); 2120 } else if (src->is_constant()) { 2121 const2reg(src, dst, lir_patch_none, nullptr); 2122 } else { 2123 ShouldNotReachHere(); 2124 } 2125 } 2126 2127 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type, 2128 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) { 2129 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86"); 2130 2131 Assembler::Condition acond, ncond; 2132 switch (condition) { 2133 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break; 2134 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break; 2135 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break; 2136 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break; 2137 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break; 2138 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break; 2139 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break; 2140 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break; 2141 default: acond = Assembler::equal; ncond = Assembler::notEqual; 2142 ShouldNotReachHere(); 2143 } 2144 2145 if (opr1->is_cpu_register()) { 2146 reg2reg(opr1, result); 2147 } else if (opr1->is_stack()) { 2148 stack2reg(opr1, result, result->type()); 2149 } else if (opr1->is_constant()) { 2150 const2reg(opr1, result, lir_patch_none, nullptr); 2151 } else { 2152 ShouldNotReachHere(); 2153 } 2154 2155 if (VM_Version::supports_cmov() && !opr2->is_constant()) { 2156 // optimized version that does not require a branch 2157 if (opr2->is_single_cpu()) { 2158 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move"); 2159 __ cmov(ncond, result->as_register(), opr2->as_register()); 2160 } else if (opr2->is_double_cpu()) { 2161 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move"); 2162 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move"); 2163 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo()); 2164 NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());) 2165 } else if (opr2->is_single_stack()) { 2166 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix())); 2167 } else if (opr2->is_double_stack()) { 2168 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes)); 2169 NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));) 2170 } else { 2171 ShouldNotReachHere(); 2172 } 2173 2174 } else { 2175 Label skip; 2176 __ jccb(acond, skip); 2177 if (opr2->is_cpu_register()) { 2178 reg2reg(opr2, result); 2179 } else if (opr2->is_stack()) { 2180 stack2reg(opr2, result, result->type()); 2181 } else if (opr2->is_constant()) { 2182 const2reg(opr2, result, lir_patch_none, nullptr); 2183 } else { 2184 ShouldNotReachHere(); 2185 } 2186 __ bind(skip); 2187 } 2188 } 2189 2190 2191 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 2192 assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method"); 2193 2194 if (left->is_single_cpu()) { 2195 assert(left == dest, "left and dest must be equal"); 2196 Register lreg = left->as_register(); 2197 2198 if (right->is_single_cpu()) { 2199 // cpu register - cpu register 2200 Register rreg = right->as_register(); 2201 switch (code) { 2202 case lir_add: __ addl (lreg, rreg); break; 2203 case lir_sub: __ subl (lreg, rreg); break; 2204 case lir_mul: __ imull(lreg, rreg); break; 2205 default: ShouldNotReachHere(); 2206 } 2207 2208 } else if (right->is_stack()) { 2209 // cpu register - stack 2210 Address raddr = frame_map()->address_for_slot(right->single_stack_ix()); 2211 switch (code) { 2212 case lir_add: __ addl(lreg, raddr); break; 2213 case lir_sub: __ subl(lreg, raddr); break; 2214 default: ShouldNotReachHere(); 2215 } 2216 2217 } else if (right->is_constant()) { 2218 // cpu register - constant 2219 jint c = right->as_constant_ptr()->as_jint(); 2220 switch (code) { 2221 case lir_add: { 2222 __ incrementl(lreg, c); 2223 break; 2224 } 2225 case lir_sub: { 2226 __ decrementl(lreg, c); 2227 break; 2228 } 2229 default: ShouldNotReachHere(); 2230 } 2231 2232 } else { 2233 ShouldNotReachHere(); 2234 } 2235 2236 } else if (left->is_double_cpu()) { 2237 assert(left == dest, "left and dest must be equal"); 2238 Register lreg_lo = left->as_register_lo(); 2239 Register lreg_hi = left->as_register_hi(); 2240 2241 if (right->is_double_cpu()) { 2242 // cpu register - cpu register 2243 Register rreg_lo = right->as_register_lo(); 2244 Register rreg_hi = right->as_register_hi(); 2245 NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi)); 2246 LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo)); 2247 switch (code) { 2248 case lir_add: 2249 __ addptr(lreg_lo, rreg_lo); 2250 NOT_LP64(__ adcl(lreg_hi, rreg_hi)); 2251 break; 2252 case lir_sub: 2253 __ subptr(lreg_lo, rreg_lo); 2254 NOT_LP64(__ sbbl(lreg_hi, rreg_hi)); 2255 break; 2256 case lir_mul: 2257 #ifdef _LP64 2258 __ imulq(lreg_lo, rreg_lo); 2259 #else 2260 assert(lreg_lo == rax && lreg_hi == rdx, "must be"); 2261 __ imull(lreg_hi, rreg_lo); 2262 __ imull(rreg_hi, lreg_lo); 2263 __ addl (rreg_hi, lreg_hi); 2264 __ mull (rreg_lo); 2265 __ addl (lreg_hi, rreg_hi); 2266 #endif // _LP64 2267 break; 2268 default: 2269 ShouldNotReachHere(); 2270 } 2271 2272 } else if (right->is_constant()) { 2273 // cpu register - constant 2274 #ifdef _LP64 2275 jlong c = right->as_constant_ptr()->as_jlong_bits(); 2276 __ movptr(r10, (intptr_t) c); 2277 switch (code) { 2278 case lir_add: 2279 __ addptr(lreg_lo, r10); 2280 break; 2281 case lir_sub: 2282 __ subptr(lreg_lo, r10); 2283 break; 2284 default: 2285 ShouldNotReachHere(); 2286 } 2287 #else 2288 jint c_lo = right->as_constant_ptr()->as_jint_lo(); 2289 jint c_hi = right->as_constant_ptr()->as_jint_hi(); 2290 switch (code) { 2291 case lir_add: 2292 __ addptr(lreg_lo, c_lo); 2293 __ adcl(lreg_hi, c_hi); 2294 break; 2295 case lir_sub: 2296 __ subptr(lreg_lo, c_lo); 2297 __ sbbl(lreg_hi, c_hi); 2298 break; 2299 default: 2300 ShouldNotReachHere(); 2301 } 2302 #endif // _LP64 2303 2304 } else { 2305 ShouldNotReachHere(); 2306 } 2307 2308 } else if (left->is_single_xmm()) { 2309 assert(left == dest, "left and dest must be equal"); 2310 XMMRegister lreg = left->as_xmm_float_reg(); 2311 2312 if (right->is_single_xmm()) { 2313 XMMRegister rreg = right->as_xmm_float_reg(); 2314 switch (code) { 2315 case lir_add: __ addss(lreg, rreg); break; 2316 case lir_sub: __ subss(lreg, rreg); break; 2317 case lir_mul: __ mulss(lreg, rreg); break; 2318 case lir_div: __ divss(lreg, rreg); break; 2319 default: ShouldNotReachHere(); 2320 } 2321 } else { 2322 Address raddr; 2323 if (right->is_single_stack()) { 2324 raddr = frame_map()->address_for_slot(right->single_stack_ix()); 2325 } else if (right->is_constant()) { 2326 // hack for now 2327 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat()))); 2328 } else { 2329 ShouldNotReachHere(); 2330 } 2331 switch (code) { 2332 case lir_add: __ addss(lreg, raddr); break; 2333 case lir_sub: __ subss(lreg, raddr); break; 2334 case lir_mul: __ mulss(lreg, raddr); break; 2335 case lir_div: __ divss(lreg, raddr); break; 2336 default: ShouldNotReachHere(); 2337 } 2338 } 2339 2340 } else if (left->is_double_xmm()) { 2341 assert(left == dest, "left and dest must be equal"); 2342 2343 XMMRegister lreg = left->as_xmm_double_reg(); 2344 if (right->is_double_xmm()) { 2345 XMMRegister rreg = right->as_xmm_double_reg(); 2346 switch (code) { 2347 case lir_add: __ addsd(lreg, rreg); break; 2348 case lir_sub: __ subsd(lreg, rreg); break; 2349 case lir_mul: __ mulsd(lreg, rreg); break; 2350 case lir_div: __ divsd(lreg, rreg); break; 2351 default: ShouldNotReachHere(); 2352 } 2353 } else { 2354 Address raddr; 2355 if (right->is_double_stack()) { 2356 raddr = frame_map()->address_for_slot(right->double_stack_ix()); 2357 } else if (right->is_constant()) { 2358 // hack for now 2359 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble()))); 2360 } else { 2361 ShouldNotReachHere(); 2362 } 2363 switch (code) { 2364 case lir_add: __ addsd(lreg, raddr); break; 2365 case lir_sub: __ subsd(lreg, raddr); break; 2366 case lir_mul: __ mulsd(lreg, raddr); break; 2367 case lir_div: __ divsd(lreg, raddr); break; 2368 default: ShouldNotReachHere(); 2369 } 2370 } 2371 2372 #ifndef _LP64 2373 } else if (left->is_single_fpu()) { 2374 assert(dest->is_single_fpu(), "fpu stack allocation required"); 2375 2376 if (right->is_single_fpu()) { 2377 arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack); 2378 2379 } else { 2380 assert(left->fpu_regnr() == 0, "left must be on TOS"); 2381 assert(dest->fpu_regnr() == 0, "dest must be on TOS"); 2382 2383 Address raddr; 2384 if (right->is_single_stack()) { 2385 raddr = frame_map()->address_for_slot(right->single_stack_ix()); 2386 } else if (right->is_constant()) { 2387 address const_addr = float_constant(right->as_jfloat()); 2388 assert(const_addr != nullptr, "incorrect float/double constant maintenance"); 2389 // hack for now 2390 raddr = __ as_Address(InternalAddress(const_addr)); 2391 } else { 2392 ShouldNotReachHere(); 2393 } 2394 2395 switch (code) { 2396 case lir_add: __ fadd_s(raddr); break; 2397 case lir_sub: __ fsub_s(raddr); break; 2398 case lir_mul: __ fmul_s(raddr); break; 2399 case lir_div: __ fdiv_s(raddr); break; 2400 default: ShouldNotReachHere(); 2401 } 2402 } 2403 2404 } else if (left->is_double_fpu()) { 2405 assert(dest->is_double_fpu(), "fpu stack allocation required"); 2406 2407 if (code == lir_mul || code == lir_div) { 2408 // Double values require special handling for strictfp mul/div on x86 2409 __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias1())); 2410 __ fmulp(left->fpu_regnrLo() + 1); 2411 } 2412 2413 if (right->is_double_fpu()) { 2414 arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack); 2415 2416 } else { 2417 assert(left->fpu_regnrLo() == 0, "left must be on TOS"); 2418 assert(dest->fpu_regnrLo() == 0, "dest must be on TOS"); 2419 2420 Address raddr; 2421 if (right->is_double_stack()) { 2422 raddr = frame_map()->address_for_slot(right->double_stack_ix()); 2423 } else if (right->is_constant()) { 2424 // hack for now 2425 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble()))); 2426 } else { 2427 ShouldNotReachHere(); 2428 } 2429 2430 switch (code) { 2431 case lir_add: __ fadd_d(raddr); break; 2432 case lir_sub: __ fsub_d(raddr); break; 2433 case lir_mul: __ fmul_d(raddr); break; 2434 case lir_div: __ fdiv_d(raddr); break; 2435 default: ShouldNotReachHere(); 2436 } 2437 } 2438 2439 if (code == lir_mul || code == lir_div) { 2440 // Double values require special handling for strictfp mul/div on x86 2441 __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias2())); 2442 __ fmulp(dest->fpu_regnrLo() + 1); 2443 } 2444 #endif // !_LP64 2445 2446 } else if (left->is_single_stack() || left->is_address()) { 2447 assert(left == dest, "left and dest must be equal"); 2448 2449 Address laddr; 2450 if (left->is_single_stack()) { 2451 laddr = frame_map()->address_for_slot(left->single_stack_ix()); 2452 } else if (left->is_address()) { 2453 laddr = as_Address(left->as_address_ptr()); 2454 } else { 2455 ShouldNotReachHere(); 2456 } 2457 2458 if (right->is_single_cpu()) { 2459 Register rreg = right->as_register(); 2460 switch (code) { 2461 case lir_add: __ addl(laddr, rreg); break; 2462 case lir_sub: __ subl(laddr, rreg); break; 2463 default: ShouldNotReachHere(); 2464 } 2465 } else if (right->is_constant()) { 2466 jint c = right->as_constant_ptr()->as_jint(); 2467 switch (code) { 2468 case lir_add: { 2469 __ incrementl(laddr, c); 2470 break; 2471 } 2472 case lir_sub: { 2473 __ decrementl(laddr, c); 2474 break; 2475 } 2476 default: ShouldNotReachHere(); 2477 } 2478 } else { 2479 ShouldNotReachHere(); 2480 } 2481 2482 } else { 2483 ShouldNotReachHere(); 2484 } 2485 } 2486 2487 #ifndef _LP64 2488 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { 2489 assert(pop_fpu_stack || (left_index == dest_index || right_index == dest_index), "invalid LIR"); 2490 assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR"); 2491 assert(left_index == 0 || right_index == 0, "either must be on top of stack"); 2492 2493 bool left_is_tos = (left_index == 0); 2494 bool dest_is_tos = (dest_index == 0); 2495 int non_tos_index = (left_is_tos ? right_index : left_index); 2496 2497 switch (code) { 2498 case lir_add: 2499 if (pop_fpu_stack) __ faddp(non_tos_index); 2500 else if (dest_is_tos) __ fadd (non_tos_index); 2501 else __ fadda(non_tos_index); 2502 break; 2503 2504 case lir_sub: 2505 if (left_is_tos) { 2506 if (pop_fpu_stack) __ fsubrp(non_tos_index); 2507 else if (dest_is_tos) __ fsub (non_tos_index); 2508 else __ fsubra(non_tos_index); 2509 } else { 2510 if (pop_fpu_stack) __ fsubp (non_tos_index); 2511 else if (dest_is_tos) __ fsubr (non_tos_index); 2512 else __ fsuba (non_tos_index); 2513 } 2514 break; 2515 2516 case lir_mul: 2517 if (pop_fpu_stack) __ fmulp(non_tos_index); 2518 else if (dest_is_tos) __ fmul (non_tos_index); 2519 else __ fmula(non_tos_index); 2520 break; 2521 2522 case lir_div: 2523 if (left_is_tos) { 2524 if (pop_fpu_stack) __ fdivrp(non_tos_index); 2525 else if (dest_is_tos) __ fdiv (non_tos_index); 2526 else __ fdivra(non_tos_index); 2527 } else { 2528 if (pop_fpu_stack) __ fdivp (non_tos_index); 2529 else if (dest_is_tos) __ fdivr (non_tos_index); 2530 else __ fdiva (non_tos_index); 2531 } 2532 break; 2533 2534 case lir_rem: 2535 assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation"); 2536 __ fremr(noreg); 2537 break; 2538 2539 default: 2540 ShouldNotReachHere(); 2541 } 2542 } 2543 #endif // _LP64 2544 2545 2546 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) { 2547 if (value->is_double_xmm()) { 2548 switch(code) { 2549 case lir_abs : 2550 { 2551 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) { 2552 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); 2553 } 2554 assert(!tmp->is_valid(), "do not need temporary"); 2555 __ andpd(dest->as_xmm_double_reg(), 2556 ExternalAddress((address)double_signmask_pool), 2557 rscratch1); 2558 } 2559 break; 2560 2561 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break; 2562 // all other intrinsics are not available in the SSE instruction set, so FPU is used 2563 default : ShouldNotReachHere(); 2564 } 2565 2566 #ifndef _LP64 2567 } else if (value->is_double_fpu()) { 2568 assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS"); 2569 switch(code) { 2570 case lir_abs : __ fabs() ; break; 2571 case lir_sqrt : __ fsqrt(); break; 2572 default : ShouldNotReachHere(); 2573 } 2574 #endif // !_LP64 2575 } else if (code == lir_f2hf) { 2576 __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg()); 2577 } else if (code == lir_hf2f) { 2578 __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register()); 2579 } else { 2580 Unimplemented(); 2581 } 2582 } 2583 2584 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 2585 // assert(left->destroys_register(), "check"); 2586 if (left->is_single_cpu()) { 2587 Register reg = left->as_register(); 2588 if (right->is_constant()) { 2589 int val = right->as_constant_ptr()->as_jint(); 2590 switch (code) { 2591 case lir_logic_and: __ andl (reg, val); break; 2592 case lir_logic_or: __ orl (reg, val); break; 2593 case lir_logic_xor: __ xorl (reg, val); break; 2594 default: ShouldNotReachHere(); 2595 } 2596 } else if (right->is_stack()) { 2597 // added support for stack operands 2598 Address raddr = frame_map()->address_for_slot(right->single_stack_ix()); 2599 switch (code) { 2600 case lir_logic_and: __ andl (reg, raddr); break; 2601 case lir_logic_or: __ orl (reg, raddr); break; 2602 case lir_logic_xor: __ xorl (reg, raddr); break; 2603 default: ShouldNotReachHere(); 2604 } 2605 } else { 2606 Register rright = right->as_register(); 2607 switch (code) { 2608 case lir_logic_and: __ andptr (reg, rright); break; 2609 case lir_logic_or : __ orptr (reg, rright); break; 2610 case lir_logic_xor: __ xorptr (reg, rright); break; 2611 default: ShouldNotReachHere(); 2612 } 2613 } 2614 move_regs(reg, dst->as_register()); 2615 } else { 2616 Register l_lo = left->as_register_lo(); 2617 Register l_hi = left->as_register_hi(); 2618 if (right->is_constant()) { 2619 #ifdef _LP64 2620 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong()); 2621 switch (code) { 2622 case lir_logic_and: 2623 __ andq(l_lo, rscratch1); 2624 break; 2625 case lir_logic_or: 2626 __ orq(l_lo, rscratch1); 2627 break; 2628 case lir_logic_xor: 2629 __ xorq(l_lo, rscratch1); 2630 break; 2631 default: ShouldNotReachHere(); 2632 } 2633 #else 2634 int r_lo = right->as_constant_ptr()->as_jint_lo(); 2635 int r_hi = right->as_constant_ptr()->as_jint_hi(); 2636 switch (code) { 2637 case lir_logic_and: 2638 __ andl(l_lo, r_lo); 2639 __ andl(l_hi, r_hi); 2640 break; 2641 case lir_logic_or: 2642 __ orl(l_lo, r_lo); 2643 __ orl(l_hi, r_hi); 2644 break; 2645 case lir_logic_xor: 2646 __ xorl(l_lo, r_lo); 2647 __ xorl(l_hi, r_hi); 2648 break; 2649 default: ShouldNotReachHere(); 2650 } 2651 #endif // _LP64 2652 } else { 2653 #ifdef _LP64 2654 Register r_lo; 2655 if (is_reference_type(right->type())) { 2656 r_lo = right->as_register(); 2657 } else { 2658 r_lo = right->as_register_lo(); 2659 } 2660 #else 2661 Register r_lo = right->as_register_lo(); 2662 Register r_hi = right->as_register_hi(); 2663 assert(l_lo != r_hi, "overwriting registers"); 2664 #endif 2665 switch (code) { 2666 case lir_logic_and: 2667 __ andptr(l_lo, r_lo); 2668 NOT_LP64(__ andptr(l_hi, r_hi);) 2669 break; 2670 case lir_logic_or: 2671 __ orptr(l_lo, r_lo); 2672 NOT_LP64(__ orptr(l_hi, r_hi);) 2673 break; 2674 case lir_logic_xor: 2675 __ xorptr(l_lo, r_lo); 2676 NOT_LP64(__ xorptr(l_hi, r_hi);) 2677 break; 2678 default: ShouldNotReachHere(); 2679 } 2680 } 2681 2682 Register dst_lo = dst->as_register_lo(); 2683 Register dst_hi = dst->as_register_hi(); 2684 2685 #ifdef _LP64 2686 move_regs(l_lo, dst_lo); 2687 #else 2688 if (dst_lo == l_hi) { 2689 assert(dst_hi != l_lo, "overwriting registers"); 2690 move_regs(l_hi, dst_hi); 2691 move_regs(l_lo, dst_lo); 2692 } else { 2693 assert(dst_lo != l_hi, "overwriting registers"); 2694 move_regs(l_lo, dst_lo); 2695 move_regs(l_hi, dst_hi); 2696 } 2697 #endif // _LP64 2698 } 2699 } 2700 2701 2702 // we assume that rax, and rdx can be overwritten 2703 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { 2704 2705 assert(left->is_single_cpu(), "left must be register"); 2706 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant"); 2707 assert(result->is_single_cpu(), "result must be register"); 2708 2709 // assert(left->destroys_register(), "check"); 2710 // assert(right->destroys_register(), "check"); 2711 2712 Register lreg = left->as_register(); 2713 Register dreg = result->as_register(); 2714 2715 if (right->is_constant()) { 2716 jint divisor = right->as_constant_ptr()->as_jint(); 2717 assert(divisor > 0 && is_power_of_2(divisor), "must be"); 2718 if (code == lir_idiv) { 2719 assert(lreg == rax, "must be rax,"); 2720 assert(temp->as_register() == rdx, "tmp register must be rdx"); 2721 __ cdql(); // sign extend into rdx:rax 2722 if (divisor == 2) { 2723 __ subl(lreg, rdx); 2724 } else { 2725 __ andl(rdx, divisor - 1); 2726 __ addl(lreg, rdx); 2727 } 2728 __ sarl(lreg, log2i_exact(divisor)); 2729 move_regs(lreg, dreg); 2730 } else if (code == lir_irem) { 2731 Label done; 2732 __ mov(dreg, lreg); 2733 __ andl(dreg, 0x80000000 | (divisor - 1)); 2734 __ jcc(Assembler::positive, done); 2735 __ decrement(dreg); 2736 __ orl(dreg, ~(divisor - 1)); 2737 __ increment(dreg); 2738 __ bind(done); 2739 } else { 2740 ShouldNotReachHere(); 2741 } 2742 } else { 2743 Register rreg = right->as_register(); 2744 assert(lreg == rax, "left register must be rax,"); 2745 assert(rreg != rdx, "right register must not be rdx"); 2746 assert(temp->as_register() == rdx, "tmp register must be rdx"); 2747 2748 move_regs(lreg, rax); 2749 2750 int idivl_offset = __ corrected_idivl(rreg); 2751 if (ImplicitDiv0Checks) { 2752 add_debug_info_for_div0(idivl_offset, info); 2753 } 2754 if (code == lir_irem) { 2755 move_regs(rdx, dreg); // result is in rdx 2756 } else { 2757 move_regs(rax, dreg); 2758 } 2759 } 2760 } 2761 2762 2763 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 2764 if (opr1->is_single_cpu()) { 2765 Register reg1 = opr1->as_register(); 2766 if (opr2->is_single_cpu()) { 2767 // cpu register - cpu register 2768 if (is_reference_type(opr1->type())) { 2769 __ cmpoop(reg1, opr2->as_register()); 2770 } else { 2771 assert(!is_reference_type(opr2->type()), "cmp int, oop?"); 2772 __ cmpl(reg1, opr2->as_register()); 2773 } 2774 } else if (opr2->is_stack()) { 2775 // cpu register - stack 2776 if (is_reference_type(opr1->type())) { 2777 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix())); 2778 } else { 2779 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix())); 2780 } 2781 } else if (opr2->is_constant()) { 2782 // cpu register - constant 2783 LIR_Const* c = opr2->as_constant_ptr(); 2784 if (c->type() == T_INT) { 2785 jint i = c->as_jint(); 2786 if (i == 0) { 2787 __ testl(reg1, reg1); 2788 } else { 2789 __ cmpl(reg1, i); 2790 } 2791 } else if (c->type() == T_METADATA) { 2792 // All we need for now is a comparison with null for equality. 2793 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops"); 2794 Metadata* m = c->as_metadata(); 2795 if (m == nullptr) { 2796 __ testptr(reg1, reg1); 2797 } else { 2798 ShouldNotReachHere(); 2799 } 2800 } else if (is_reference_type(c->type())) { 2801 // In 64bit oops are single register 2802 jobject o = c->as_jobject(); 2803 if (o == nullptr) { 2804 __ testptr(reg1, reg1); 2805 } else { 2806 __ cmpoop(reg1, o, rscratch1); 2807 } 2808 } else { 2809 fatal("unexpected type: %s", basictype_to_str(c->type())); 2810 } 2811 // cpu register - address 2812 } else if (opr2->is_address()) { 2813 if (op->info() != nullptr) { 2814 add_debug_info_for_null_check_here(op->info()); 2815 } 2816 __ cmpl(reg1, as_Address(opr2->as_address_ptr())); 2817 } else { 2818 ShouldNotReachHere(); 2819 } 2820 2821 } else if(opr1->is_double_cpu()) { 2822 Register xlo = opr1->as_register_lo(); 2823 Register xhi = opr1->as_register_hi(); 2824 if (opr2->is_double_cpu()) { 2825 #ifdef _LP64 2826 __ cmpptr(xlo, opr2->as_register_lo()); 2827 #else 2828 // cpu register - cpu register 2829 Register ylo = opr2->as_register_lo(); 2830 Register yhi = opr2->as_register_hi(); 2831 __ subl(xlo, ylo); 2832 __ sbbl(xhi, yhi); 2833 if (condition == lir_cond_equal || condition == lir_cond_notEqual) { 2834 __ orl(xhi, xlo); 2835 } 2836 #endif // _LP64 2837 } else if (opr2->is_constant()) { 2838 // cpu register - constant 0 2839 assert(opr2->as_jlong() == (jlong)0, "only handles zero"); 2840 #ifdef _LP64 2841 __ cmpptr(xlo, (int32_t)opr2->as_jlong()); 2842 #else 2843 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case"); 2844 __ orl(xhi, xlo); 2845 #endif // _LP64 2846 } else { 2847 ShouldNotReachHere(); 2848 } 2849 2850 } else if (opr1->is_single_xmm()) { 2851 XMMRegister reg1 = opr1->as_xmm_float_reg(); 2852 if (opr2->is_single_xmm()) { 2853 // xmm register - xmm register 2854 __ ucomiss(reg1, opr2->as_xmm_float_reg()); 2855 } else if (opr2->is_stack()) { 2856 // xmm register - stack 2857 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix())); 2858 } else if (opr2->is_constant()) { 2859 // xmm register - constant 2860 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat()))); 2861 } else if (opr2->is_address()) { 2862 // xmm register - address 2863 if (op->info() != nullptr) { 2864 add_debug_info_for_null_check_here(op->info()); 2865 } 2866 __ ucomiss(reg1, as_Address(opr2->as_address_ptr())); 2867 } else { 2868 ShouldNotReachHere(); 2869 } 2870 2871 } else if (opr1->is_double_xmm()) { 2872 XMMRegister reg1 = opr1->as_xmm_double_reg(); 2873 if (opr2->is_double_xmm()) { 2874 // xmm register - xmm register 2875 __ ucomisd(reg1, opr2->as_xmm_double_reg()); 2876 } else if (opr2->is_stack()) { 2877 // xmm register - stack 2878 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix())); 2879 } else if (opr2->is_constant()) { 2880 // xmm register - constant 2881 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble()))); 2882 } else if (opr2->is_address()) { 2883 // xmm register - address 2884 if (op->info() != nullptr) { 2885 add_debug_info_for_null_check_here(op->info()); 2886 } 2887 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address())); 2888 } else { 2889 ShouldNotReachHere(); 2890 } 2891 2892 #ifndef _LP64 2893 } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) { 2894 assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)"); 2895 assert(opr2->is_fpu_register(), "both must be registers"); 2896 __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1); 2897 #endif // LP64 2898 2899 } else if (opr1->is_address() && opr2->is_constant()) { 2900 LIR_Const* c = opr2->as_constant_ptr(); 2901 #ifdef _LP64 2902 if (is_reference_type(c->type())) { 2903 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse"); 2904 __ movoop(rscratch1, c->as_jobject()); 2905 } 2906 #endif // LP64 2907 if (op->info() != nullptr) { 2908 add_debug_info_for_null_check_here(op->info()); 2909 } 2910 // special case: address - constant 2911 LIR_Address* addr = opr1->as_address_ptr(); 2912 if (c->type() == T_INT) { 2913 __ cmpl(as_Address(addr), c->as_jint()); 2914 } else if (is_reference_type(c->type())) { 2915 #ifdef _LP64 2916 // %%% Make this explode if addr isn't reachable until we figure out a 2917 // better strategy by giving noreg as the temp for as_Address 2918 __ cmpoop(rscratch1, as_Address(addr, noreg)); 2919 #else 2920 __ cmpoop(as_Address(addr), c->as_jobject()); 2921 #endif // _LP64 2922 } else { 2923 ShouldNotReachHere(); 2924 } 2925 2926 } else { 2927 ShouldNotReachHere(); 2928 } 2929 } 2930 2931 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) { 2932 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 2933 if (left->is_single_xmm()) { 2934 assert(right->is_single_xmm(), "must match"); 2935 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i); 2936 } else if (left->is_double_xmm()) { 2937 assert(right->is_double_xmm(), "must match"); 2938 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i); 2939 2940 } else { 2941 #ifdef _LP64 2942 ShouldNotReachHere(); 2943 #else 2944 assert(left->is_single_fpu() || left->is_double_fpu(), "must be"); 2945 assert(right->is_single_fpu() || right->is_double_fpu(), "must match"); 2946 2947 assert(left->fpu() == 0, "left must be on TOS"); 2948 __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(), 2949 op->fpu_pop_count() > 0, op->fpu_pop_count() > 1); 2950 #endif // LP64 2951 } 2952 } else { 2953 assert(code == lir_cmp_l2i, "check"); 2954 #ifdef _LP64 2955 Label done; 2956 Register dest = dst->as_register(); 2957 __ cmpptr(left->as_register_lo(), right->as_register_lo()); 2958 __ movl(dest, -1); 2959 __ jccb(Assembler::less, done); 2960 __ setb(Assembler::notZero, dest); 2961 __ movzbl(dest, dest); 2962 __ bind(done); 2963 #else 2964 __ lcmp2int(left->as_register_hi(), 2965 left->as_register_lo(), 2966 right->as_register_hi(), 2967 right->as_register_lo()); 2968 move_regs(left->as_register_hi(), dst->as_register()); 2969 #endif // _LP64 2970 } 2971 } 2972 2973 2974 void LIR_Assembler::align_call(LIR_Code code) { 2975 // make sure that the displacement word of the call ends up word aligned 2976 int offset = __ offset(); 2977 switch (code) { 2978 case lir_static_call: 2979 case lir_optvirtual_call: 2980 case lir_dynamic_call: 2981 offset += NativeCall::displacement_offset; 2982 break; 2983 case lir_icvirtual_call: 2984 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex; 2985 break; 2986 default: ShouldNotReachHere(); 2987 } 2988 __ align(BytesPerWord, offset); 2989 } 2990 2991 2992 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 2993 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0, 2994 "must be aligned"); 2995 __ call(AddressLiteral(op->addr(), rtype)); 2996 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields()); 2997 __ post_call_nop(); 2998 } 2999 3000 3001 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 3002 __ ic_call(op->addr()); 3003 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields()); 3004 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0, 3005 "must be aligned"); 3006 __ post_call_nop(); 3007 } 3008 3009 3010 void LIR_Assembler::emit_static_call_stub() { 3011 address call_pc = __ pc(); 3012 address stub = __ start_a_stub(call_stub_size()); 3013 if (stub == nullptr) { 3014 bailout("static call stub overflow"); 3015 return; 3016 } 3017 3018 int start = __ offset(); 3019 3020 // make sure that the displacement word of the call ends up word aligned 3021 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset); 3022 __ relocate(static_stub_Relocation::spec(call_pc)); 3023 __ mov_metadata(rbx, (Metadata*)nullptr); 3024 // must be set to -1 at code generation time 3025 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned"); 3026 // On 64bit this will die since it will take a movq & jmp, must be only a jmp 3027 __ jump(RuntimeAddress(__ pc())); 3028 3029 assert(__ offset() - start <= call_stub_size(), "stub too big"); 3030 __ end_a_stub(); 3031 } 3032 3033 3034 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 3035 assert(exceptionOop->as_register() == rax, "must match"); 3036 assert(exceptionPC->as_register() == rdx, "must match"); 3037 3038 // exception object is not added to oop map by LinearScan 3039 // (LinearScan assumes that no oops are in fixed registers) 3040 info->add_register_oop(exceptionOop); 3041 C1StubId unwind_id; 3042 3043 // get current pc information 3044 // pc is only needed if the method has an exception handler, the unwind code does not need it. 3045 int pc_for_athrow_offset = __ offset(); 3046 InternalAddress pc_for_athrow(__ pc()); 3047 __ lea(exceptionPC->as_register(), pc_for_athrow); 3048 add_call_info(pc_for_athrow_offset, info); // for exception handler 3049 3050 __ verify_not_null_oop(rax); 3051 // search an exception handler (rax: exception oop, rdx: throwing pc) 3052 if (compilation()->has_fpu_code()) { 3053 unwind_id = C1StubId::handle_exception_id; 3054 } else { 3055 unwind_id = C1StubId::handle_exception_nofpu_id; 3056 } 3057 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 3058 3059 // enough room for two byte trap 3060 __ nop(); 3061 } 3062 3063 3064 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 3065 assert(exceptionOop->as_register() == rax, "must match"); 3066 3067 __ jmp(_unwind_handler_entry); 3068 } 3069 3070 3071 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 3072 3073 // optimized version for linear scan: 3074 // * count must be already in ECX (guaranteed by LinearScan) 3075 // * left and dest must be equal 3076 // * tmp must be unused 3077 assert(count->as_register() == SHIFT_count, "count must be in ECX"); 3078 assert(left == dest, "left and dest must be equal"); 3079 assert(tmp->is_illegal(), "wasting a register if tmp is allocated"); 3080 3081 if (left->is_single_cpu()) { 3082 Register value = left->as_register(); 3083 assert(value != SHIFT_count, "left cannot be ECX"); 3084 3085 switch (code) { 3086 case lir_shl: __ shll(value); break; 3087 case lir_shr: __ sarl(value); break; 3088 case lir_ushr: __ shrl(value); break; 3089 default: ShouldNotReachHere(); 3090 } 3091 } else if (left->is_double_cpu()) { 3092 Register lo = left->as_register_lo(); 3093 Register hi = left->as_register_hi(); 3094 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX"); 3095 #ifdef _LP64 3096 switch (code) { 3097 case lir_shl: __ shlptr(lo); break; 3098 case lir_shr: __ sarptr(lo); break; 3099 case lir_ushr: __ shrptr(lo); break; 3100 default: ShouldNotReachHere(); 3101 } 3102 #else 3103 3104 switch (code) { 3105 case lir_shl: __ lshl(hi, lo); break; 3106 case lir_shr: __ lshr(hi, lo, true); break; 3107 case lir_ushr: __ lshr(hi, lo, false); break; 3108 default: ShouldNotReachHere(); 3109 } 3110 #endif // LP64 3111 } else { 3112 ShouldNotReachHere(); 3113 } 3114 } 3115 3116 3117 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 3118 if (dest->is_single_cpu()) { 3119 // first move left into dest so that left is not destroyed by the shift 3120 Register value = dest->as_register(); 3121 count = count & 0x1F; // Java spec 3122 3123 move_regs(left->as_register(), value); 3124 switch (code) { 3125 case lir_shl: __ shll(value, count); break; 3126 case lir_shr: __ sarl(value, count); break; 3127 case lir_ushr: __ shrl(value, count); break; 3128 default: ShouldNotReachHere(); 3129 } 3130 } else if (dest->is_double_cpu()) { 3131 #ifndef _LP64 3132 Unimplemented(); 3133 #else 3134 // first move left into dest so that left is not destroyed by the shift 3135 Register value = dest->as_register_lo(); 3136 count = count & 0x1F; // Java spec 3137 3138 move_regs(left->as_register_lo(), value); 3139 switch (code) { 3140 case lir_shl: __ shlptr(value, count); break; 3141 case lir_shr: __ sarptr(value, count); break; 3142 case lir_ushr: __ shrptr(value, count); break; 3143 default: ShouldNotReachHere(); 3144 } 3145 #endif // _LP64 3146 } else { 3147 ShouldNotReachHere(); 3148 } 3149 } 3150 3151 3152 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 3153 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 3154 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 3155 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 3156 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r); 3157 } 3158 3159 3160 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 3161 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 3162 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 3163 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 3164 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c); 3165 } 3166 3167 3168 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) { 3169 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 3170 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 3171 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 3172 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1); 3173 } 3174 3175 3176 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) { 3177 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 3178 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 3179 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 3180 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1); 3181 } 3182 3183 3184 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) { 3185 if (null_check) { 3186 __ testptr(obj, obj); 3187 __ jcc(Assembler::zero, *slow_path->entry()); 3188 } 3189 if (is_dest) { 3190 __ test_null_free_array_oop(obj, tmp, *slow_path->entry()); 3191 } else { 3192 __ test_flat_array_oop(obj, tmp, *slow_path->entry()); 3193 } 3194 } 3195 3196 3197 // This code replaces a call to arraycopy; no exception may 3198 // be thrown in this code, they must be thrown in the System.arraycopy 3199 // activation frame; we could save some checks if this would not be the case 3200 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 3201 ciArrayKlass* default_type = op->expected_type(); 3202 Register src = op->src()->as_register(); 3203 Register dst = op->dst()->as_register(); 3204 Register src_pos = op->src_pos()->as_register(); 3205 Register dst_pos = op->dst_pos()->as_register(); 3206 Register length = op->length()->as_register(); 3207 Register tmp = op->tmp()->as_register(); 3208 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 3209 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg; 3210 3211 CodeStub* stub = op->stub(); 3212 int flags = op->flags(); 3213 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL; 3214 if (is_reference_type(basic_type)) basic_type = T_OBJECT; 3215 3216 if (flags & LIR_OpArrayCopy::always_slow_path) { 3217 __ jmp(*stub->entry()); 3218 __ bind(*stub->continuation()); 3219 return; 3220 } 3221 3222 // if we don't know anything, just go through the generic arraycopy 3223 if (default_type == nullptr) { 3224 // save outgoing arguments on stack in case call to System.arraycopy is needed 3225 // HACK ALERT. This code used to push the parameters in a hardwired fashion 3226 // for interpreter calling conventions. Now we have to do it in new style conventions. 3227 // For the moment until C1 gets the new register allocator I just force all the 3228 // args to the right place (except the register args) and then on the back side 3229 // reload the register args properly if we go slow path. Yuck 3230 3231 // These are proper for the calling convention 3232 store_parameter(length, 2); 3233 store_parameter(dst_pos, 1); 3234 store_parameter(dst, 0); 3235 3236 // these are just temporary placements until we need to reload 3237 store_parameter(src_pos, 3); 3238 store_parameter(src, 4); 3239 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");) 3240 3241 address copyfunc_addr = StubRoutines::generic_arraycopy(); 3242 assert(copyfunc_addr != nullptr, "generic arraycopy stub required"); 3243 3244 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint 3245 #ifdef _LP64 3246 // The arguments are in java calling convention so we can trivially shift them to C 3247 // convention 3248 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4); 3249 __ mov(c_rarg0, j_rarg0); 3250 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4); 3251 __ mov(c_rarg1, j_rarg1); 3252 assert_different_registers(c_rarg2, j_rarg3, j_rarg4); 3253 __ mov(c_rarg2, j_rarg2); 3254 assert_different_registers(c_rarg3, j_rarg4); 3255 __ mov(c_rarg3, j_rarg3); 3256 #ifdef _WIN64 3257 // Allocate abi space for args but be sure to keep stack aligned 3258 __ subptr(rsp, 6*wordSize); 3259 store_parameter(j_rarg4, 4); 3260 #ifndef PRODUCT 3261 if (PrintC1Statistics) { 3262 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1); 3263 } 3264 #endif 3265 __ call(RuntimeAddress(copyfunc_addr)); 3266 __ addptr(rsp, 6*wordSize); 3267 #else 3268 __ mov(c_rarg4, j_rarg4); 3269 #ifndef PRODUCT 3270 if (PrintC1Statistics) { 3271 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1); 3272 } 3273 #endif 3274 __ call(RuntimeAddress(copyfunc_addr)); 3275 #endif // _WIN64 3276 #else 3277 __ push(length); 3278 __ push(dst_pos); 3279 __ push(dst); 3280 __ push(src_pos); 3281 __ push(src); 3282 3283 #ifndef PRODUCT 3284 if (PrintC1Statistics) { 3285 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1); 3286 } 3287 #endif 3288 __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack 3289 3290 #endif // _LP64 3291 3292 __ testl(rax, rax); 3293 __ jcc(Assembler::equal, *stub->continuation()); 3294 3295 __ mov(tmp, rax); 3296 __ xorl(tmp, -1); 3297 3298 // Reload values from the stack so they are where the stub 3299 // expects them. 3300 __ movptr (dst, Address(rsp, 0*BytesPerWord)); 3301 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord)); 3302 __ movptr (length, Address(rsp, 2*BytesPerWord)); 3303 __ movptr (src_pos, Address(rsp, 3*BytesPerWord)); 3304 __ movptr (src, Address(rsp, 4*BytesPerWord)); 3305 3306 __ subl(length, tmp); 3307 __ addl(src_pos, tmp); 3308 __ addl(dst_pos, tmp); 3309 __ jmp(*stub->entry()); 3310 3311 __ bind(*stub->continuation()); 3312 return; 3313 } 3314 3315 // Handle inline type arrays 3316 if (flags & LIR_OpArrayCopy::src_inlinetype_check) { 3317 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check)); 3318 } 3319 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) { 3320 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check)); 3321 } 3322 3323 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point"); 3324 3325 int elem_size = type2aelembytes(basic_type); 3326 Address::ScaleFactor scale; 3327 3328 switch (elem_size) { 3329 case 1 : 3330 scale = Address::times_1; 3331 break; 3332 case 2 : 3333 scale = Address::times_2; 3334 break; 3335 case 4 : 3336 scale = Address::times_4; 3337 break; 3338 case 8 : 3339 scale = Address::times_8; 3340 break; 3341 default: 3342 scale = Address::no_scale; 3343 ShouldNotReachHere(); 3344 } 3345 3346 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); 3347 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); 3348 3349 // length and pos's are all sign extended at this point on 64bit 3350 3351 // test for null 3352 if (flags & LIR_OpArrayCopy::src_null_check) { 3353 __ testptr(src, src); 3354 __ jcc(Assembler::zero, *stub->entry()); 3355 } 3356 if (flags & LIR_OpArrayCopy::dst_null_check) { 3357 __ testptr(dst, dst); 3358 __ jcc(Assembler::zero, *stub->entry()); 3359 } 3360 3361 // If the compiler was not able to prove that exact type of the source or the destination 3362 // of the arraycopy is an array type, check at runtime if the source or the destination is 3363 // an instance type. 3364 if (flags & LIR_OpArrayCopy::type_check) { 3365 if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 3366 __ load_klass(tmp, dst, tmp_load_klass); 3367 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value); 3368 __ jcc(Assembler::greaterEqual, *stub->entry()); 3369 } 3370 3371 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 3372 __ load_klass(tmp, src, tmp_load_klass); 3373 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value); 3374 __ jcc(Assembler::greaterEqual, *stub->entry()); 3375 } 3376 } 3377 3378 // check if negative 3379 if (flags & LIR_OpArrayCopy::src_pos_positive_check) { 3380 __ testl(src_pos, src_pos); 3381 __ jcc(Assembler::less, *stub->entry()); 3382 } 3383 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) { 3384 __ testl(dst_pos, dst_pos); 3385 __ jcc(Assembler::less, *stub->entry()); 3386 } 3387 3388 if (flags & LIR_OpArrayCopy::src_range_check) { 3389 __ lea(tmp, Address(src_pos, length, Address::times_1, 0)); 3390 __ cmpl(tmp, src_length_addr); 3391 __ jcc(Assembler::above, *stub->entry()); 3392 } 3393 if (flags & LIR_OpArrayCopy::dst_range_check) { 3394 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0)); 3395 __ cmpl(tmp, dst_length_addr); 3396 __ jcc(Assembler::above, *stub->entry()); 3397 } 3398 3399 if (flags & LIR_OpArrayCopy::length_positive_check) { 3400 __ testl(length, length); 3401 __ jcc(Assembler::less, *stub->entry()); 3402 } 3403 3404 #ifdef _LP64 3405 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null 3406 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null 3407 #endif 3408 3409 if (flags & LIR_OpArrayCopy::type_check) { 3410 // We don't know the array types are compatible 3411 if (basic_type != T_OBJECT) { 3412 // Simple test for basic type arrays 3413 __ cmp_klasses_from_objects(src, dst, tmp, tmp2); 3414 __ jcc(Assembler::notEqual, *stub->entry()); 3415 } else { 3416 // For object arrays, if src is a sub class of dst then we can 3417 // safely do the copy. 3418 Label cont, slow; 3419 3420 __ push(src); 3421 __ push(dst); 3422 3423 __ load_klass(src, src, tmp_load_klass); 3424 __ load_klass(dst, dst, tmp_load_klass); 3425 3426 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr); 3427 3428 __ push(src); 3429 __ push(dst); 3430 __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id))); 3431 __ pop(dst); 3432 __ pop(src); 3433 3434 __ testl(src, src); 3435 __ jcc(Assembler::notEqual, cont); 3436 3437 __ bind(slow); 3438 __ pop(dst); 3439 __ pop(src); 3440 3441 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 3442 if (copyfunc_addr != nullptr) { // use stub if available 3443 // src is not a sub class of dst so we have to do a 3444 // per-element check. 3445 3446 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 3447 if ((flags & mask) != mask) { 3448 // Check that at least both of them object arrays. 3449 assert(flags & mask, "one of the two should be known to be an object array"); 3450 3451 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 3452 __ load_klass(tmp, src, tmp_load_klass); 3453 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 3454 __ load_klass(tmp, dst, tmp_load_klass); 3455 } 3456 int lh_offset = in_bytes(Klass::layout_helper_offset()); 3457 Address klass_lh_addr(tmp, lh_offset); 3458 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 3459 __ cmpl(klass_lh_addr, objArray_lh); 3460 __ jcc(Assembler::notEqual, *stub->entry()); 3461 } 3462 3463 // Spill because stubs can use any register they like and it's 3464 // easier to restore just those that we care about. 3465 store_parameter(dst, 0); 3466 store_parameter(dst_pos, 1); 3467 store_parameter(length, 2); 3468 store_parameter(src_pos, 3); 3469 store_parameter(src, 4); 3470 3471 #ifndef _LP64 3472 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); 3473 __ movptr(tmp, dst_klass_addr); 3474 __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset())); 3475 __ push(tmp); 3476 __ movl(tmp, Address(tmp, Klass::super_check_offset_offset())); 3477 __ push(tmp); 3478 __ push(length); 3479 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3480 __ push(tmp); 3481 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3482 __ push(tmp); 3483 3484 __ call_VM_leaf(copyfunc_addr, 5); 3485 #else 3486 __ movl2ptr(length, length); //higher 32bits must be null 3487 3488 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3489 assert_different_registers(c_rarg0, dst, dst_pos, length); 3490 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3491 assert_different_registers(c_rarg1, dst, length); 3492 3493 __ mov(c_rarg2, length); 3494 assert_different_registers(c_rarg2, dst); 3495 3496 #ifdef _WIN64 3497 // Allocate abi space for args but be sure to keep stack aligned 3498 __ subptr(rsp, 6*wordSize); 3499 __ load_klass(c_rarg3, dst, tmp_load_klass); 3500 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset())); 3501 store_parameter(c_rarg3, 4); 3502 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset())); 3503 __ call(RuntimeAddress(copyfunc_addr)); 3504 __ addptr(rsp, 6*wordSize); 3505 #else 3506 __ load_klass(c_rarg4, dst, tmp_load_klass); 3507 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset())); 3508 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset())); 3509 __ call(RuntimeAddress(copyfunc_addr)); 3510 #endif 3511 3512 #endif 3513 3514 #ifndef PRODUCT 3515 if (PrintC1Statistics) { 3516 Label failed; 3517 __ testl(rax, rax); 3518 __ jcc(Assembler::notZero, failed); 3519 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1); 3520 __ bind(failed); 3521 } 3522 #endif 3523 3524 __ testl(rax, rax); 3525 __ jcc(Assembler::zero, *stub->continuation()); 3526 3527 #ifndef PRODUCT 3528 if (PrintC1Statistics) { 3529 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1); 3530 } 3531 #endif 3532 3533 __ mov(tmp, rax); 3534 3535 __ xorl(tmp, -1); 3536 3537 // Restore previously spilled arguments 3538 __ movptr (dst, Address(rsp, 0*BytesPerWord)); 3539 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord)); 3540 __ movptr (length, Address(rsp, 2*BytesPerWord)); 3541 __ movptr (src_pos, Address(rsp, 3*BytesPerWord)); 3542 __ movptr (src, Address(rsp, 4*BytesPerWord)); 3543 3544 3545 __ subl(length, tmp); 3546 __ addl(src_pos, tmp); 3547 __ addl(dst_pos, tmp); 3548 } 3549 3550 __ jmp(*stub->entry()); 3551 3552 __ bind(cont); 3553 __ pop(dst); 3554 __ pop(src); 3555 } 3556 } 3557 3558 #ifdef ASSERT 3559 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { 3560 // Sanity check the known type with the incoming class. For the 3561 // primitive case the types must match exactly with src.klass and 3562 // dst.klass each exactly matching the default type. For the 3563 // object array case, if no type check is needed then either the 3564 // dst type is exactly the expected type and the src type is a 3565 // subtype which we can't check or src is the same array as dst 3566 // but not necessarily exactly of type default_type. 3567 Label known_ok, halt; 3568 __ mov_metadata(tmp, default_type->constant_encoding()); 3569 #ifdef _LP64 3570 if (UseCompressedClassPointers) { 3571 __ encode_klass_not_null(tmp, rscratch1); 3572 } 3573 #endif 3574 3575 if (basic_type != T_OBJECT) { 3576 __ cmp_klass(tmp, dst, tmp2); 3577 __ jcc(Assembler::notEqual, halt); 3578 __ cmp_klass(tmp, src, tmp2); 3579 __ jcc(Assembler::equal, known_ok); 3580 } else { 3581 __ cmp_klass(tmp, dst, tmp2); 3582 __ jcc(Assembler::equal, known_ok); 3583 __ cmpptr(src, dst); 3584 __ jcc(Assembler::equal, known_ok); 3585 } 3586 __ bind(halt); 3587 __ stop("incorrect type information in arraycopy"); 3588 __ bind(known_ok); 3589 } 3590 #endif 3591 3592 #ifndef PRODUCT 3593 if (PrintC1Statistics) { 3594 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1); 3595 } 3596 #endif 3597 3598 #ifdef _LP64 3599 assert_different_registers(c_rarg0, dst, dst_pos, length); 3600 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3601 assert_different_registers(c_rarg1, length); 3602 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3603 __ mov(c_rarg2, length); 3604 3605 #else 3606 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3607 store_parameter(tmp, 0); 3608 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3609 store_parameter(tmp, 1); 3610 store_parameter(length, 2); 3611 #endif // _LP64 3612 3613 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 3614 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 3615 const char *name; 3616 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 3617 __ call_VM_leaf(entry, 0); 3618 3619 if (stub != nullptr) { 3620 __ bind(*stub->continuation()); 3621 } 3622 } 3623 3624 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 3625 assert(op->crc()->is_single_cpu(), "crc must be register"); 3626 assert(op->val()->is_single_cpu(), "byte value must be register"); 3627 assert(op->result_opr()->is_single_cpu(), "result must be register"); 3628 Register crc = op->crc()->as_register(); 3629 Register val = op->val()->as_register(); 3630 Register res = op->result_opr()->as_register(); 3631 3632 assert_different_registers(val, crc, res); 3633 3634 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr())); 3635 __ notl(crc); // ~crc 3636 __ update_byte_crc32(crc, val, res); 3637 __ notl(crc); // ~crc 3638 __ mov(res, crc); 3639 } 3640 3641 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 3642 Register obj = op->obj_opr()->as_register(); // may not be an oop 3643 Register hdr = op->hdr_opr()->as_register(); 3644 Register lock = op->lock_opr()->as_register(); 3645 if (LockingMode == LM_MONITOR) { 3646 if (op->info() != nullptr) { 3647 add_debug_info_for_null_check_here(op->info()); 3648 __ null_check(obj); 3649 } 3650 __ jmp(*op->stub()->entry()); 3651 } else if (op->code() == lir_lock) { 3652 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 3653 Register tmp = LockingMode == LM_LIGHTWEIGHT ? op->scratch_opr()->as_register() : noreg; 3654 // add debug info for NullPointerException only if one is possible 3655 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry()); 3656 if (op->info() != nullptr) { 3657 add_debug_info_for_null_check(null_check_offset, op->info()); 3658 } 3659 // done 3660 } else if (op->code() == lir_unlock) { 3661 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 3662 __ unlock_object(hdr, obj, lock, *op->stub()->entry()); 3663 } else { 3664 Unimplemented(); 3665 } 3666 __ bind(*op->stub()->continuation()); 3667 } 3668 3669 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { 3670 Register obj = op->obj()->as_pointer_register(); 3671 Register result = op->result_opr()->as_pointer_register(); 3672 3673 CodeEmitInfo* info = op->info(); 3674 if (info != nullptr) { 3675 add_debug_info_for_null_check_here(info); 3676 } 3677 3678 __ load_klass(result, obj, rscratch1); 3679 } 3680 3681 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 3682 ciMethod* method = op->profiled_method(); 3683 int bci = op->profiled_bci(); 3684 ciMethod* callee = op->profiled_callee(); 3685 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 3686 3687 // Update counter for all call types 3688 ciMethodData* md = method->method_data_or_null(); 3689 assert(md != nullptr, "Sanity"); 3690 ciProfileData* data = md->bci_to_data(bci); 3691 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls"); 3692 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 3693 Register mdo = op->mdo()->as_register(); 3694 __ mov_metadata(mdo, md->constant_encoding()); 3695 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 3696 // Perform additional virtual call profiling for invokevirtual and 3697 // invokeinterface bytecodes 3698 if (op->should_profile_receiver_type()) { 3699 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 3700 Register recv = op->recv()->as_register(); 3701 assert_different_registers(mdo, recv); 3702 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 3703 ciKlass* known_klass = op->known_holder(); 3704 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) { 3705 // We know the type that will be seen at this call site; we can 3706 // statically update the MethodData* rather than needing to do 3707 // dynamic tests on the receiver type 3708 3709 // NOTE: we should probably put a lock around this search to 3710 // avoid collisions by concurrent compilations 3711 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 3712 uint i; 3713 for (i = 0; i < VirtualCallData::row_limit(); i++) { 3714 ciKlass* receiver = vc_data->receiver(i); 3715 if (known_klass->equals(receiver)) { 3716 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 3717 __ addptr(data_addr, DataLayout::counter_increment); 3718 return; 3719 } 3720 } 3721 3722 // Receiver type not found in profile data; select an empty slot 3723 3724 // Note that this is less efficient than it should be because it 3725 // always does a write to the receiver part of the 3726 // VirtualCallData rather than just the first time 3727 for (i = 0; i < VirtualCallData::row_limit(); i++) { 3728 ciKlass* receiver = vc_data->receiver(i); 3729 if (receiver == nullptr) { 3730 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); 3731 __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1); 3732 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 3733 __ addptr(data_addr, DataLayout::counter_increment); 3734 return; 3735 } 3736 } 3737 } else { 3738 __ load_klass(recv, recv, tmp_load_klass); 3739 Label update_done; 3740 type_profile_helper(mdo, md, data, recv, &update_done); 3741 // Receiver did not match any saved receiver and there is no empty row for it. 3742 // Increment total counter to indicate polymorphic case. 3743 __ addptr(counter_addr, DataLayout::counter_increment); 3744 3745 __ bind(update_done); 3746 } 3747 } else { 3748 // Static call 3749 __ addptr(counter_addr, DataLayout::counter_increment); 3750 } 3751 } 3752 3753 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 3754 Register obj = op->obj()->as_register(); 3755 Register tmp = op->tmp()->as_pointer_register(); 3756 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 3757 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 3758 ciKlass* exact_klass = op->exact_klass(); 3759 intptr_t current_klass = op->current_klass(); 3760 bool not_null = op->not_null(); 3761 bool no_conflict = op->no_conflict(); 3762 3763 Label update, next, none; 3764 3765 bool do_null = !not_null; 3766 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 3767 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 3768 3769 assert(do_null || do_update, "why are we here?"); 3770 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 3771 3772 __ verify_oop(obj); 3773 3774 #ifdef ASSERT 3775 if (obj == tmp) { 3776 #ifdef _LP64 3777 assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index()); 3778 #else 3779 assert_different_registers(obj, mdo_addr.base(), mdo_addr.index()); 3780 #endif 3781 } else { 3782 #ifdef _LP64 3783 assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index()); 3784 #else 3785 assert_different_registers(obj, tmp, mdo_addr.base(), mdo_addr.index()); 3786 #endif 3787 } 3788 #endif 3789 if (do_null) { 3790 __ testptr(obj, obj); 3791 __ jccb(Assembler::notZero, update); 3792 if (!TypeEntries::was_null_seen(current_klass)) { 3793 __ testptr(mdo_addr, TypeEntries::null_seen); 3794 #ifndef ASSERT 3795 __ jccb(Assembler::notZero, next); // already set 3796 #else 3797 __ jcc(Assembler::notZero, next); // already set 3798 #endif 3799 // atomic update to prevent overwriting Klass* with 0 3800 __ lock(); 3801 __ orptr(mdo_addr, TypeEntries::null_seen); 3802 } 3803 if (do_update) { 3804 #ifndef ASSERT 3805 __ jmpb(next); 3806 } 3807 #else 3808 __ jmp(next); 3809 } 3810 } else { 3811 __ testptr(obj, obj); 3812 __ jcc(Assembler::notZero, update); 3813 __ stop("unexpected null obj"); 3814 #endif 3815 } 3816 3817 __ bind(update); 3818 3819 if (do_update) { 3820 #ifdef ASSERT 3821 if (exact_klass != nullptr) { 3822 Label ok; 3823 __ load_klass(tmp, obj, tmp_load_klass); 3824 __ push(tmp); 3825 __ mov_metadata(tmp, exact_klass->constant_encoding()); 3826 __ cmpptr(tmp, Address(rsp, 0)); 3827 __ jcc(Assembler::equal, ok); 3828 __ stop("exact klass and actual klass differ"); 3829 __ bind(ok); 3830 __ pop(tmp); 3831 } 3832 #endif 3833 if (!no_conflict) { 3834 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) { 3835 if (exact_klass != nullptr) { 3836 __ mov_metadata(tmp, exact_klass->constant_encoding()); 3837 } else { 3838 __ load_klass(tmp, obj, tmp_load_klass); 3839 } 3840 #ifdef _LP64 3841 __ mov(rscratch1, tmp); // save original value before XOR 3842 #endif 3843 __ xorptr(tmp, mdo_addr); 3844 __ testptr(tmp, TypeEntries::type_klass_mask); 3845 // klass seen before, nothing to do. The unknown bit may have been 3846 // set already but no need to check. 3847 __ jccb(Assembler::zero, next); 3848 3849 __ testptr(tmp, TypeEntries::type_unknown); 3850 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore. 3851 3852 if (TypeEntries::is_type_none(current_klass)) { 3853 __ testptr(mdo_addr, TypeEntries::type_mask); 3854 __ jccb(Assembler::zero, none); 3855 #ifdef _LP64 3856 // There is a chance that the checks above (re-reading profiling 3857 // data from memory) fail if another thread has just set the 3858 // profiling to this obj's klass 3859 __ mov(tmp, rscratch1); // get back original value before XOR 3860 __ xorptr(tmp, mdo_addr); 3861 __ testptr(tmp, TypeEntries::type_klass_mask); 3862 __ jccb(Assembler::zero, next); 3863 #endif 3864 } 3865 } else { 3866 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr && 3867 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 3868 3869 __ testptr(mdo_addr, TypeEntries::type_unknown); 3870 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore. 3871 } 3872 3873 // different than before. Cannot keep accurate profile. 3874 __ orptr(mdo_addr, TypeEntries::type_unknown); 3875 3876 if (TypeEntries::is_type_none(current_klass)) { 3877 __ jmpb(next); 3878 3879 __ bind(none); 3880 // first time here. Set profile type. 3881 __ movptr(mdo_addr, tmp); 3882 #ifdef ASSERT 3883 __ andptr(tmp, TypeEntries::type_klass_mask); 3884 __ verify_klass_ptr(tmp); 3885 #endif 3886 } 3887 } else { 3888 // There's a single possible klass at this profile point 3889 assert(exact_klass != nullptr, "should be"); 3890 if (TypeEntries::is_type_none(current_klass)) { 3891 __ mov_metadata(tmp, exact_klass->constant_encoding()); 3892 __ xorptr(tmp, mdo_addr); 3893 __ testptr(tmp, TypeEntries::type_klass_mask); 3894 #ifdef ASSERT 3895 __ jcc(Assembler::zero, next); 3896 3897 { 3898 Label ok; 3899 __ push(tmp); 3900 __ testptr(mdo_addr, TypeEntries::type_mask); 3901 __ jcc(Assembler::zero, ok); 3902 // may have been set by another thread 3903 __ mov_metadata(tmp, exact_klass->constant_encoding()); 3904 __ xorptr(tmp, mdo_addr); 3905 __ testptr(tmp, TypeEntries::type_mask); 3906 __ jcc(Assembler::zero, ok); 3907 3908 __ stop("unexpected profiling mismatch"); 3909 __ bind(ok); 3910 __ pop(tmp); 3911 } 3912 #else 3913 __ jccb(Assembler::zero, next); 3914 #endif 3915 // first time here. Set profile type. 3916 __ movptr(mdo_addr, tmp); 3917 #ifdef ASSERT 3918 __ andptr(tmp, TypeEntries::type_klass_mask); 3919 __ verify_klass_ptr(tmp); 3920 #endif 3921 } else { 3922 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr && 3923 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 3924 3925 __ testptr(mdo_addr, TypeEntries::type_unknown); 3926 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore. 3927 3928 __ orptr(mdo_addr, TypeEntries::type_unknown); 3929 } 3930 } 3931 } 3932 __ bind(next); 3933 } 3934 3935 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { 3936 Register obj = op->obj()->as_register(); 3937 Register tmp = op->tmp()->as_pointer_register(); 3938 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 3939 bool not_null = op->not_null(); 3940 int flag = op->flag(); 3941 3942 Label not_inline_type; 3943 if (!not_null) { 3944 __ testptr(obj, obj); 3945 __ jccb(Assembler::zero, not_inline_type); 3946 } 3947 3948 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type); 3949 3950 __ orb(mdo_addr, flag); 3951 3952 __ bind(not_inline_type); 3953 } 3954 3955 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 3956 Unimplemented(); 3957 } 3958 3959 3960 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 3961 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 3962 } 3963 3964 3965 void LIR_Assembler::align_backward_branch_target() { 3966 __ align(BytesPerWord); 3967 } 3968 3969 3970 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) { 3971 if (left->is_single_cpu()) { 3972 __ negl(left->as_register()); 3973 move_regs(left->as_register(), dest->as_register()); 3974 3975 } else if (left->is_double_cpu()) { 3976 Register lo = left->as_register_lo(); 3977 #ifdef _LP64 3978 Register dst = dest->as_register_lo(); 3979 __ movptr(dst, lo); 3980 __ negptr(dst); 3981 #else 3982 Register hi = left->as_register_hi(); 3983 __ lneg(hi, lo); 3984 if (dest->as_register_lo() == hi) { 3985 assert(dest->as_register_hi() != lo, "destroying register"); 3986 move_regs(hi, dest->as_register_hi()); 3987 move_regs(lo, dest->as_register_lo()); 3988 } else { 3989 move_regs(lo, dest->as_register_lo()); 3990 move_regs(hi, dest->as_register_hi()); 3991 } 3992 #endif // _LP64 3993 3994 } else if (dest->is_single_xmm()) { 3995 assert(!tmp->is_valid(), "do not need temporary"); 3996 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) { 3997 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg()); 3998 } 3999 __ xorps(dest->as_xmm_float_reg(), 4000 ExternalAddress((address)float_signflip_pool), 4001 rscratch1); 4002 } else if (dest->is_double_xmm()) { 4003 assert(!tmp->is_valid(), "do not need temporary"); 4004 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) { 4005 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg()); 4006 } 4007 __ xorpd(dest->as_xmm_double_reg(), 4008 ExternalAddress((address)double_signflip_pool), 4009 rscratch1); 4010 #ifndef _LP64 4011 } else if (left->is_single_fpu() || left->is_double_fpu()) { 4012 assert(left->fpu() == 0, "arg must be on TOS"); 4013 assert(dest->fpu() == 0, "dest must be TOS"); 4014 __ fchs(); 4015 #endif // !_LP64 4016 4017 } else { 4018 ShouldNotReachHere(); 4019 } 4020 } 4021 4022 4023 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 4024 assert(src->is_address(), "must be an address"); 4025 assert(dest->is_register(), "must be a register"); 4026 4027 PatchingStub* patch = nullptr; 4028 if (patch_code != lir_patch_none) { 4029 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 4030 } 4031 4032 Register reg = dest->as_pointer_register(); 4033 LIR_Address* addr = src->as_address_ptr(); 4034 __ lea(reg, as_Address(addr)); 4035 4036 if (patch != nullptr) { 4037 patching_epilog(patch, patch_code, addr->base()->as_register(), info); 4038 } 4039 } 4040 4041 4042 4043 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 4044 assert(!tmp->is_valid(), "don't need temporary"); 4045 __ call(RuntimeAddress(dest)); 4046 if (info != nullptr) { 4047 add_call_info_here(info); 4048 } 4049 __ post_call_nop(); 4050 } 4051 4052 4053 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 4054 assert(type == T_LONG, "only for volatile long fields"); 4055 4056 if (info != nullptr) { 4057 add_debug_info_for_null_check_here(info); 4058 } 4059 4060 if (src->is_double_xmm()) { 4061 if (dest->is_double_cpu()) { 4062 #ifdef _LP64 4063 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg()); 4064 #else 4065 __ movdl(dest->as_register_lo(), src->as_xmm_double_reg()); 4066 __ psrlq(src->as_xmm_double_reg(), 32); 4067 __ movdl(dest->as_register_hi(), src->as_xmm_double_reg()); 4068 #endif // _LP64 4069 } else if (dest->is_double_stack()) { 4070 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg()); 4071 } else if (dest->is_address()) { 4072 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg()); 4073 } else { 4074 ShouldNotReachHere(); 4075 } 4076 4077 } else if (dest->is_double_xmm()) { 4078 if (src->is_double_stack()) { 4079 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix())); 4080 } else if (src->is_address()) { 4081 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr())); 4082 } else { 4083 ShouldNotReachHere(); 4084 } 4085 4086 #ifndef _LP64 4087 } else if (src->is_double_fpu()) { 4088 assert(src->fpu_regnrLo() == 0, "must be TOS"); 4089 if (dest->is_double_stack()) { 4090 __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix())); 4091 } else if (dest->is_address()) { 4092 __ fistp_d(as_Address(dest->as_address_ptr())); 4093 } else { 4094 ShouldNotReachHere(); 4095 } 4096 4097 } else if (dest->is_double_fpu()) { 4098 assert(dest->fpu_regnrLo() == 0, "must be TOS"); 4099 if (src->is_double_stack()) { 4100 __ fild_d(frame_map()->address_for_slot(src->double_stack_ix())); 4101 } else if (src->is_address()) { 4102 __ fild_d(as_Address(src->as_address_ptr())); 4103 } else { 4104 ShouldNotReachHere(); 4105 } 4106 #endif // !_LP64 4107 4108 } else { 4109 ShouldNotReachHere(); 4110 } 4111 } 4112 4113 #ifdef ASSERT 4114 // emit run-time assertion 4115 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 4116 assert(op->code() == lir_assert, "must be"); 4117 4118 if (op->in_opr1()->is_valid()) { 4119 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 4120 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 4121 } else { 4122 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 4123 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 4124 } 4125 4126 Label ok; 4127 if (op->condition() != lir_cond_always) { 4128 Assembler::Condition acond = Assembler::zero; 4129 switch (op->condition()) { 4130 case lir_cond_equal: acond = Assembler::equal; break; 4131 case lir_cond_notEqual: acond = Assembler::notEqual; break; 4132 case lir_cond_less: acond = Assembler::less; break; 4133 case lir_cond_lessEqual: acond = Assembler::lessEqual; break; 4134 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break; 4135 case lir_cond_greater: acond = Assembler::greater; break; 4136 case lir_cond_belowEqual: acond = Assembler::belowEqual; break; 4137 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break; 4138 default: ShouldNotReachHere(); 4139 } 4140 __ jcc(acond, ok); 4141 } 4142 if (op->halt()) { 4143 const char* str = __ code_string(op->msg()); 4144 __ stop(str); 4145 } else { 4146 breakpoint(); 4147 } 4148 __ bind(ok); 4149 } 4150 #endif 4151 4152 void LIR_Assembler::membar() { 4153 // QQQ sparc TSO uses this, 4154 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad)); 4155 } 4156 4157 void LIR_Assembler::membar_acquire() { 4158 // No x86 machines currently require load fences 4159 } 4160 4161 void LIR_Assembler::membar_release() { 4162 // No x86 machines currently require store fences 4163 } 4164 4165 void LIR_Assembler::membar_loadload() { 4166 // no-op 4167 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload)); 4168 } 4169 4170 void LIR_Assembler::membar_storestore() { 4171 // no-op 4172 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore)); 4173 } 4174 4175 void LIR_Assembler::membar_loadstore() { 4176 // no-op 4177 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore)); 4178 } 4179 4180 void LIR_Assembler::membar_storeload() { 4181 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); 4182 } 4183 4184 void LIR_Assembler::on_spin_wait() { 4185 __ pause (); 4186 } 4187 4188 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 4189 assert(result_reg->is_register(), "check"); 4190 #ifdef _LP64 4191 // __ get_thread(result_reg->as_register_lo()); 4192 __ mov(result_reg->as_register(), r15_thread); 4193 #else 4194 __ get_thread(result_reg->as_register()); 4195 #endif // _LP64 4196 } 4197 4198 void LIR_Assembler::check_orig_pc() { 4199 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD); 4200 } 4201 4202 void LIR_Assembler::peephole(LIR_List*) { 4203 // do nothing for now 4204 } 4205 4206 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) { 4207 assert(data == dest, "xchg/xadd uses only 2 operands"); 4208 4209 if (data->type() == T_INT) { 4210 if (code == lir_xadd) { 4211 __ lock(); 4212 __ xaddl(as_Address(src->as_address_ptr()), data->as_register()); 4213 } else { 4214 __ xchgl(data->as_register(), as_Address(src->as_address_ptr())); 4215 } 4216 } else if (data->is_oop()) { 4217 assert (code == lir_xchg, "xadd for oops"); 4218 Register obj = data->as_register(); 4219 #ifdef _LP64 4220 if (UseCompressedOops) { 4221 __ encode_heap_oop(obj); 4222 __ xchgl(obj, as_Address(src->as_address_ptr())); 4223 __ decode_heap_oop(obj); 4224 } else { 4225 __ xchgptr(obj, as_Address(src->as_address_ptr())); 4226 } 4227 #else 4228 __ xchgl(obj, as_Address(src->as_address_ptr())); 4229 #endif 4230 } else if (data->type() == T_LONG) { 4231 #ifdef _LP64 4232 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register"); 4233 if (code == lir_xadd) { 4234 __ lock(); 4235 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo()); 4236 } else { 4237 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr())); 4238 } 4239 #else 4240 ShouldNotReachHere(); 4241 #endif 4242 } else { 4243 ShouldNotReachHere(); 4244 } 4245 } 4246 4247 #undef __