1 /* 2 * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.hpp" 27 #include "asm/macroAssembler.inline.hpp" 28 #include "c1/c1_CodeStubs.hpp" 29 #include "c1/c1_Compilation.hpp" 30 #include "c1/c1_LIRAssembler.hpp" 31 #include "c1/c1_MacroAssembler.hpp" 32 #include "c1/c1_Runtime1.hpp" 33 #include "c1/c1_ValueStack.hpp" 34 #include "ci/ciArrayKlass.hpp" 35 #include "ci/ciInstance.hpp" 36 #include "compiler/oopMap.hpp" 37 #include "gc/shared/collectedHeap.hpp" 38 #include "gc/shared/gc_globals.hpp" 39 #include "nativeInst_x86.hpp" 40 #include "oops/objArrayKlass.hpp" 41 #include "runtime/frame.inline.hpp" 42 #include "runtime/safepointMechanism.hpp" 43 #include "runtime/sharedRuntime.hpp" 44 #include "runtime/stubRoutines.hpp" 45 #include "utilities/powerOfTwo.hpp" 46 #include "vmreg_x86.inline.hpp" 47 48 49 // These masks are used to provide 128-bit aligned bitmasks to the XMM 50 // instructions, to allow sign-masking or sign-bit flipping. They allow 51 // fast versions of NegF/NegD and AbsF/AbsD. 52 53 // Note: 'double' and 'long long' have 32-bits alignment on x86. 54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) { 55 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address 56 // of 128-bits operands for SSE instructions. 57 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF))); 58 // Store the value to a 128-bits operand. 59 operand[0] = lo; 60 operand[1] = hi; 61 return operand; 62 } 63 64 // Buffer for 128-bits masks used by SSE instructions. 65 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment) 66 67 // Static initialization during VM startup. 68 static jlong *float_signmask_pool = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF)); 69 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF)); 70 static jlong *float_signflip_pool = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000)); 71 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000)); 72 73 74 NEEDS_CLEANUP // remove this definitions ? 75 const Register IC_Klass = rax; // where the IC klass is cached 76 const Register SYNC_header = rax; // synchronization header 77 const Register SHIFT_count = rcx; // where count for shift operations must be 78 79 #define __ _masm-> 80 81 82 static void select_different_registers(Register preserve, 83 Register extra, 84 Register &tmp1, 85 Register &tmp2) { 86 if (tmp1 == preserve) { 87 assert_different_registers(tmp1, tmp2, extra); 88 tmp1 = extra; 89 } else if (tmp2 == preserve) { 90 assert_different_registers(tmp1, tmp2, extra); 91 tmp2 = extra; 92 } 93 assert_different_registers(preserve, tmp1, tmp2); 94 } 95 96 97 98 static void select_different_registers(Register preserve, 99 Register extra, 100 Register &tmp1, 101 Register &tmp2, 102 Register &tmp3) { 103 if (tmp1 == preserve) { 104 assert_different_registers(tmp1, tmp2, tmp3, extra); 105 tmp1 = extra; 106 } else if (tmp2 == preserve) { 107 assert_different_registers(tmp1, tmp2, tmp3, extra); 108 tmp2 = extra; 109 } else if (tmp3 == preserve) { 110 assert_different_registers(tmp1, tmp2, tmp3, extra); 111 tmp3 = extra; 112 } 113 assert_different_registers(preserve, tmp1, tmp2, tmp3); 114 } 115 116 117 118 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { 119 if (opr->is_constant()) { 120 LIR_Const* constant = opr->as_constant_ptr(); 121 switch (constant->type()) { 122 case T_INT: { 123 return true; 124 } 125 126 default: 127 return false; 128 } 129 } 130 return false; 131 } 132 133 134 LIR_Opr LIR_Assembler::receiverOpr() { 135 return FrameMap::receiver_opr; 136 } 137 138 LIR_Opr LIR_Assembler::osrBufferPointer() { 139 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 140 } 141 142 //--------------fpu register translations----------------------- 143 144 145 address LIR_Assembler::float_constant(float f) { 146 address const_addr = __ float_constant(f); 147 if (const_addr == NULL) { 148 bailout("const section overflow"); 149 return __ code()->consts()->start(); 150 } else { 151 return const_addr; 152 } 153 } 154 155 156 address LIR_Assembler::double_constant(double d) { 157 address const_addr = __ double_constant(d); 158 if (const_addr == NULL) { 159 bailout("const section overflow"); 160 return __ code()->consts()->start(); 161 } else { 162 return const_addr; 163 } 164 } 165 166 #ifndef _LP64 167 void LIR_Assembler::fpop() { 168 __ fpop(); 169 } 170 171 void LIR_Assembler::fxch(int i) { 172 __ fxch(i); 173 } 174 175 void LIR_Assembler::fld(int i) { 176 __ fld_s(i); 177 } 178 179 void LIR_Assembler::ffree(int i) { 180 __ ffree(i); 181 } 182 #endif // !_LP64 183 184 void LIR_Assembler::breakpoint() { 185 __ int3(); 186 } 187 188 void LIR_Assembler::push(LIR_Opr opr) { 189 if (opr->is_single_cpu()) { 190 __ push_reg(opr->as_register()); 191 } else if (opr->is_double_cpu()) { 192 NOT_LP64(__ push_reg(opr->as_register_hi())); 193 __ push_reg(opr->as_register_lo()); 194 } else if (opr->is_stack()) { 195 __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix())); 196 } else if (opr->is_constant()) { 197 LIR_Const* const_opr = opr->as_constant_ptr(); 198 if (const_opr->type() == T_OBJECT) { 199 __ push_oop(const_opr->as_jobject()); 200 } else if (const_opr->type() == T_INT) { 201 __ push_jint(const_opr->as_jint()); 202 } else { 203 ShouldNotReachHere(); 204 } 205 206 } else { 207 ShouldNotReachHere(); 208 } 209 } 210 211 void LIR_Assembler::pop(LIR_Opr opr) { 212 if (opr->is_single_cpu()) { 213 __ pop_reg(opr->as_register()); 214 } else { 215 ShouldNotReachHere(); 216 } 217 } 218 219 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { 220 return addr->base()->is_illegal() && addr->index()->is_illegal(); 221 } 222 223 //------------------------------------------- 224 225 Address LIR_Assembler::as_Address(LIR_Address* addr) { 226 return as_Address(addr, rscratch1); 227 } 228 229 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 230 if (addr->base()->is_illegal()) { 231 assert(addr->index()->is_illegal(), "must be illegal too"); 232 AddressLiteral laddr((address)addr->disp(), relocInfo::none); 233 if (! __ reachable(laddr)) { 234 __ movptr(tmp, laddr.addr()); 235 Address res(tmp, 0); 236 return res; 237 } else { 238 return __ as_Address(laddr); 239 } 240 } 241 242 Register base = addr->base()->as_pointer_register(); 243 244 if (addr->index()->is_illegal()) { 245 return Address( base, addr->disp()); 246 } else if (addr->index()->is_cpu_register()) { 247 Register index = addr->index()->as_pointer_register(); 248 return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp()); 249 } else if (addr->index()->is_constant()) { 250 intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp(); 251 assert(Assembler::is_simm32(addr_offset), "must be"); 252 253 return Address(base, addr_offset); 254 } else { 255 Unimplemented(); 256 return Address(); 257 } 258 } 259 260 261 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 262 Address base = as_Address(addr); 263 return Address(base._base, base._index, base._scale, base._disp + BytesPerWord); 264 } 265 266 267 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 268 return as_Address(addr); 269 } 270 271 272 void LIR_Assembler::osr_entry() { 273 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 274 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 275 ValueStack* entry_state = osr_entry->state(); 276 int number_of_locks = entry_state->locks_size(); 277 278 // we jump here if osr happens with the interpreter 279 // state set up to continue at the beginning of the 280 // loop that triggered osr - in particular, we have 281 // the following registers setup: 282 // 283 // rcx: osr buffer 284 // 285 286 // build frame 287 ciMethod* m = compilation()->method(); 288 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 289 290 // OSR buffer is 291 // 292 // locals[nlocals-1..0] 293 // monitors[0..number_of_locks] 294 // 295 // locals is a direct copy of the interpreter frame so in the osr buffer 296 // so first slot in the local array is the last local from the interpreter 297 // and last slot is local[0] (receiver) from the interpreter 298 // 299 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 300 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 301 // in the interpreter frame (the method lock if a sync method) 302 303 // Initialize monitors in the compiled activation. 304 // rcx: pointer to osr buffer 305 // 306 // All other registers are dead at this point and the locals will be 307 // copied into place by code emitted in the IR. 308 309 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 310 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 311 int monitor_offset = BytesPerWord * method()->max_locals() + 312 (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1); 313 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 314 // the OSR buffer using 2 word entries: first the lock and then 315 // the oop. 316 for (int i = 0; i < number_of_locks; i++) { 317 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 318 #ifdef ASSERT 319 // verify the interpreter's monitor has a non-null object 320 { 321 Label L; 322 __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD); 323 __ jcc(Assembler::notZero, L); 324 __ stop("locked object is NULL"); 325 __ bind(L); 326 } 327 #endif 328 __ movptr(rbx, Address(OSR_buf, slot_offset + 0)); 329 __ movptr(frame_map()->address_for_monitor_lock(i), rbx); 330 __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord)); 331 __ movptr(frame_map()->address_for_monitor_object(i), rbx); 332 } 333 } 334 } 335 336 337 // inline cache check; done before the frame is built. 338 int LIR_Assembler::check_icache() { 339 Register receiver = FrameMap::receiver_opr->as_register(); 340 Register ic_klass = IC_Klass; 341 const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9); 342 const bool do_post_padding = VerifyOops || UseCompressedClassPointers; 343 if (!do_post_padding) { 344 // insert some nops so that the verified entry point is aligned on CodeEntryAlignment 345 __ align(CodeEntryAlignment, __ offset() + ic_cmp_size); 346 } 347 int offset = __ offset(); 348 __ inline_cache_check(receiver, IC_Klass); 349 assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct"); 350 if (do_post_padding) { 351 // force alignment after the cache check. 352 // It's been verified to be aligned if !VerifyOops 353 __ align(CodeEntryAlignment); 354 } 355 return offset; 356 } 357 358 void LIR_Assembler::clinit_barrier(ciMethod* method) { 359 assert(VM_Version::supports_fast_class_init_checks(), "sanity"); 360 assert(!method->holder()->is_not_initialized(), "initialization should have been started"); 361 362 Label L_skip_barrier; 363 Register klass = rscratch1; 364 Register thread = LP64_ONLY( r15_thread ) NOT_LP64( noreg ); 365 assert(thread != noreg, "x86_32 not implemented"); 366 367 __ mov_metadata(klass, method->holder()->constant_encoding()); 368 __ clinit_barrier(klass, thread, &L_skip_barrier /*L_fast_path*/); 369 370 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 371 372 __ bind(L_skip_barrier); 373 } 374 375 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) { 376 jobject o = NULL; 377 PatchingStub* patch = new PatchingStub(_masm, patching_id(info)); 378 __ movoop(reg, o); 379 patching_epilog(patch, lir_patch_normal, reg, info); 380 } 381 382 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 383 Metadata* o = NULL; 384 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id); 385 __ mov_metadata(reg, o); 386 patching_epilog(patch, lir_patch_normal, reg, info); 387 } 388 389 // This specifies the rsp decrement needed to build the frame 390 int LIR_Assembler::initial_frame_size_in_bytes() const { 391 // if rounding, must let FrameMap know! 392 393 // The frame_map records size in slots (32bit word) 394 395 // subtract two words to account for return address and link 396 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size; 397 } 398 399 400 int LIR_Assembler::emit_exception_handler() { 401 // if the last instruction is a call (typically to do a throw which 402 // is coming at the end after block reordering) the return address 403 // must still point into the code area in order to avoid assertion 404 // failures when searching for the corresponding bci => add a nop 405 // (was bug 5/14/1999 - gri) 406 __ nop(); 407 408 // generate code for exception handler 409 address handler_base = __ start_a_stub(exception_handler_size()); 410 if (handler_base == NULL) { 411 // not enough space left for the handler 412 bailout("exception handler overflow"); 413 return -1; 414 } 415 416 int offset = code_offset(); 417 418 // the exception oop and pc are in rax, and rdx 419 // no other registers need to be preserved, so invalidate them 420 __ invalidate_registers(false, true, true, false, true, true); 421 422 // check that there is really an exception 423 __ verify_not_null_oop(rax); 424 425 // search an exception handler (rax: exception oop, rdx: throwing pc) 426 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); 427 __ should_not_reach_here(); 428 guarantee(code_offset() - offset <= exception_handler_size(), "overflow"); 429 __ end_a_stub(); 430 431 return offset; 432 } 433 434 435 // Emit the code to remove the frame from the stack in the exception 436 // unwind path. 437 int LIR_Assembler::emit_unwind_handler() { 438 #ifndef PRODUCT 439 if (CommentedAssembly) { 440 _masm->block_comment("Unwind handler"); 441 } 442 #endif 443 444 int offset = code_offset(); 445 446 // Fetch the exception from TLS and clear out exception related thread state 447 Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); 448 NOT_LP64(__ get_thread(rsi)); 449 __ movptr(rax, Address(thread, JavaThread::exception_oop_offset())); 450 __ movptr(Address(thread, JavaThread::exception_oop_offset()), (intptr_t)NULL_WORD); 451 __ movptr(Address(thread, JavaThread::exception_pc_offset()), (intptr_t)NULL_WORD); 452 453 __ bind(_unwind_handler_entry); 454 __ verify_not_null_oop(rax); 455 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 456 __ mov(rbx, rax); // Preserve the exception (rbx is always callee-saved) 457 } 458 459 // Perform needed unlocking 460 MonitorExitStub* stub = NULL; 461 if (method()->is_synchronized()) { 462 monitor_address(0, FrameMap::rax_opr); 463 stub = new MonitorExitStub(FrameMap::rax_opr, true, 0); 464 if (UseHeavyMonitors) { 465 __ jmp(*stub->entry()); 466 } else { 467 __ unlock_object(rdi, rsi, rax, *stub->entry()); 468 } 469 __ bind(*stub->continuation()); 470 } 471 472 if (compilation()->env()->dtrace_method_probes()) { 473 #ifdef _LP64 474 __ mov(rdi, r15_thread); 475 __ mov_metadata(rsi, method()->constant_encoding()); 476 #else 477 __ get_thread(rax); 478 __ movptr(Address(rsp, 0), rax); 479 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding()); 480 #endif 481 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit))); 482 } 483 484 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 485 __ mov(rax, rbx); // Restore the exception 486 } 487 488 // remove the activation and dispatch to the unwind handler 489 __ remove_frame(initial_frame_size_in_bytes()); 490 __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id))); 491 492 // Emit the slow path assembly 493 if (stub != NULL) { 494 stub->emit_code(this); 495 } 496 497 return offset; 498 } 499 500 501 int LIR_Assembler::emit_deopt_handler() { 502 // if the last instruction is a call (typically to do a throw which 503 // is coming at the end after block reordering) the return address 504 // must still point into the code area in order to avoid assertion 505 // failures when searching for the corresponding bci => add a nop 506 // (was bug 5/14/1999 - gri) 507 __ nop(); 508 509 // generate code for exception handler 510 address handler_base = __ start_a_stub(deopt_handler_size()); 511 if (handler_base == NULL) { 512 // not enough space left for the handler 513 bailout("deopt handler overflow"); 514 return -1; 515 } 516 517 int offset = code_offset(); 518 InternalAddress here(__ pc()); 519 520 __ pushptr(here.addr()); 521 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 522 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); 523 __ end_a_stub(); 524 525 return offset; 526 } 527 528 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { 529 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,"); 530 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) { 531 assert(result->fpu() == 0, "result must already be on TOS"); 532 } 533 534 // Pop the stack before the safepoint code 535 __ remove_frame(initial_frame_size_in_bytes()); 536 537 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) { 538 __ reserved_stack_check(); 539 } 540 541 // Note: we do not need to round double result; float result has the right precision 542 // the poll sets the condition code, but no data registers 543 544 #ifdef _LP64 545 const Register thread = r15_thread; 546 #else 547 const Register thread = rbx; 548 __ get_thread(thread); 549 #endif 550 code_stub->set_safepoint_offset(__ offset()); 551 __ relocate(relocInfo::poll_return_type); 552 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */); 553 __ ret(0); 554 } 555 556 557 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 558 guarantee(info != NULL, "Shouldn't be NULL"); 559 int offset = __ offset(); 560 #ifdef _LP64 561 const Register poll_addr = rscratch1; 562 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset())); 563 #else 564 assert(tmp->is_cpu_register(), "needed"); 565 const Register poll_addr = tmp->as_register(); 566 __ get_thread(poll_addr); 567 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset()))); 568 #endif 569 add_debug_info_for_branch(info); 570 __ relocate(relocInfo::poll_type); 571 address pre_pc = __ pc(); 572 __ testl(rax, Address(poll_addr, 0)); 573 address post_pc = __ pc(); 574 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length"); 575 return offset; 576 } 577 578 579 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 580 if (from_reg != to_reg) __ mov(to_reg, from_reg); 581 } 582 583 void LIR_Assembler::swap_reg(Register a, Register b) { 584 __ xchgptr(a, b); 585 } 586 587 588 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 589 assert(src->is_constant(), "should not call otherwise"); 590 assert(dest->is_register(), "should not call otherwise"); 591 LIR_Const* c = src->as_constant_ptr(); 592 593 switch (c->type()) { 594 case T_INT: { 595 assert(patch_code == lir_patch_none, "no patching handled here"); 596 __ movl(dest->as_register(), c->as_jint()); 597 break; 598 } 599 600 case T_ADDRESS: { 601 assert(patch_code == lir_patch_none, "no patching handled here"); 602 __ movptr(dest->as_register(), c->as_jint()); 603 break; 604 } 605 606 case T_LONG: { 607 assert(patch_code == lir_patch_none, "no patching handled here"); 608 #ifdef _LP64 609 __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong()); 610 #else 611 __ movptr(dest->as_register_lo(), c->as_jint_lo()); 612 __ movptr(dest->as_register_hi(), c->as_jint_hi()); 613 #endif // _LP64 614 break; 615 } 616 617 case T_OBJECT: { 618 if (patch_code != lir_patch_none) { 619 jobject2reg_with_patching(dest->as_register(), info); 620 } else { 621 __ movoop(dest->as_register(), c->as_jobject()); 622 } 623 break; 624 } 625 626 case T_METADATA: { 627 if (patch_code != lir_patch_none) { 628 klass2reg_with_patching(dest->as_register(), info); 629 } else { 630 __ mov_metadata(dest->as_register(), c->as_metadata()); 631 } 632 break; 633 } 634 635 case T_FLOAT: { 636 if (dest->is_single_xmm()) { 637 if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_float()) { 638 __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg()); 639 } else { 640 __ movflt(dest->as_xmm_float_reg(), 641 InternalAddress(float_constant(c->as_jfloat()))); 642 } 643 } else { 644 #ifndef _LP64 645 assert(dest->is_single_fpu(), "must be"); 646 assert(dest->fpu_regnr() == 0, "dest must be TOS"); 647 if (c->is_zero_float()) { 648 __ fldz(); 649 } else if (c->is_one_float()) { 650 __ fld1(); 651 } else { 652 __ fld_s (InternalAddress(float_constant(c->as_jfloat()))); 653 } 654 #else 655 ShouldNotReachHere(); 656 #endif // !_LP64 657 } 658 break; 659 } 660 661 case T_DOUBLE: { 662 if (dest->is_double_xmm()) { 663 if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_double()) { 664 __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg()); 665 } else { 666 __ movdbl(dest->as_xmm_double_reg(), 667 InternalAddress(double_constant(c->as_jdouble()))); 668 } 669 } else { 670 #ifndef _LP64 671 assert(dest->is_double_fpu(), "must be"); 672 assert(dest->fpu_regnrLo() == 0, "dest must be TOS"); 673 if (c->is_zero_double()) { 674 __ fldz(); 675 } else if (c->is_one_double()) { 676 __ fld1(); 677 } else { 678 __ fld_d (InternalAddress(double_constant(c->as_jdouble()))); 679 } 680 #else 681 ShouldNotReachHere(); 682 #endif // !_LP64 683 } 684 break; 685 } 686 687 default: 688 ShouldNotReachHere(); 689 } 690 } 691 692 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 693 assert(src->is_constant(), "should not call otherwise"); 694 assert(dest->is_stack(), "should not call otherwise"); 695 LIR_Const* c = src->as_constant_ptr(); 696 697 switch (c->type()) { 698 case T_INT: // fall through 699 case T_FLOAT: 700 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits()); 701 break; 702 703 case T_ADDRESS: 704 __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits()); 705 break; 706 707 case T_OBJECT: 708 __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject()); 709 break; 710 711 case T_LONG: // fall through 712 case T_DOUBLE: 713 #ifdef _LP64 714 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(), 715 lo_word_offset_in_bytes), (intptr_t)c->as_jlong_bits()); 716 #else 717 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(), 718 lo_word_offset_in_bytes), c->as_jint_lo_bits()); 719 __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(), 720 hi_word_offset_in_bytes), c->as_jint_hi_bits()); 721 #endif // _LP64 722 break; 723 724 default: 725 ShouldNotReachHere(); 726 } 727 } 728 729 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 730 assert(src->is_constant(), "should not call otherwise"); 731 assert(dest->is_address(), "should not call otherwise"); 732 LIR_Const* c = src->as_constant_ptr(); 733 LIR_Address* addr = dest->as_address_ptr(); 734 735 int null_check_here = code_offset(); 736 switch (type) { 737 case T_INT: // fall through 738 case T_FLOAT: 739 __ movl(as_Address(addr), c->as_jint_bits()); 740 break; 741 742 case T_ADDRESS: 743 __ movptr(as_Address(addr), c->as_jint_bits()); 744 break; 745 746 case T_OBJECT: // fall through 747 case T_ARRAY: 748 if (c->as_jobject() == NULL) { 749 if (UseCompressedOops && !wide) { 750 __ movl(as_Address(addr), (int32_t)NULL_WORD); 751 } else { 752 #ifdef _LP64 753 __ xorptr(rscratch1, rscratch1); 754 null_check_here = code_offset(); 755 __ movptr(as_Address(addr), rscratch1); 756 #else 757 __ movptr(as_Address(addr), NULL_WORD); 758 #endif 759 } 760 } else { 761 if (is_literal_address(addr)) { 762 ShouldNotReachHere(); 763 __ movoop(as_Address(addr, noreg), c->as_jobject()); 764 } else { 765 #ifdef _LP64 766 __ movoop(rscratch1, c->as_jobject()); 767 if (UseCompressedOops && !wide) { 768 __ encode_heap_oop(rscratch1); 769 null_check_here = code_offset(); 770 __ movl(as_Address_lo(addr), rscratch1); 771 } else { 772 null_check_here = code_offset(); 773 __ movptr(as_Address_lo(addr), rscratch1); 774 } 775 #else 776 __ movoop(as_Address(addr), c->as_jobject()); 777 #endif 778 } 779 } 780 break; 781 782 case T_LONG: // fall through 783 case T_DOUBLE: 784 #ifdef _LP64 785 if (is_literal_address(addr)) { 786 ShouldNotReachHere(); 787 __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits()); 788 } else { 789 __ movptr(r10, (intptr_t)c->as_jlong_bits()); 790 null_check_here = code_offset(); 791 __ movptr(as_Address_lo(addr), r10); 792 } 793 #else 794 // Always reachable in 32bit so this doesn't produce useless move literal 795 __ movptr(as_Address_hi(addr), c->as_jint_hi_bits()); 796 __ movptr(as_Address_lo(addr), c->as_jint_lo_bits()); 797 #endif // _LP64 798 break; 799 800 case T_BOOLEAN: // fall through 801 case T_BYTE: 802 __ movb(as_Address(addr), c->as_jint() & 0xFF); 803 break; 804 805 case T_CHAR: // fall through 806 case T_SHORT: 807 __ movw(as_Address(addr), c->as_jint() & 0xFFFF); 808 break; 809 810 default: 811 ShouldNotReachHere(); 812 }; 813 814 if (info != NULL) { 815 add_debug_info_for_null_check(null_check_here, info); 816 } 817 } 818 819 820 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 821 assert(src->is_register(), "should not call otherwise"); 822 assert(dest->is_register(), "should not call otherwise"); 823 824 // move between cpu-registers 825 if (dest->is_single_cpu()) { 826 #ifdef _LP64 827 if (src->type() == T_LONG) { 828 // Can do LONG -> OBJECT 829 move_regs(src->as_register_lo(), dest->as_register()); 830 return; 831 } 832 #endif 833 assert(src->is_single_cpu(), "must match"); 834 if (src->type() == T_OBJECT) { 835 __ verify_oop(src->as_register()); 836 } 837 move_regs(src->as_register(), dest->as_register()); 838 839 } else if (dest->is_double_cpu()) { 840 #ifdef _LP64 841 if (is_reference_type(src->type())) { 842 // Surprising to me but we can see move of a long to t_object 843 __ verify_oop(src->as_register()); 844 move_regs(src->as_register(), dest->as_register_lo()); 845 return; 846 } 847 #endif 848 assert(src->is_double_cpu(), "must match"); 849 Register f_lo = src->as_register_lo(); 850 Register f_hi = src->as_register_hi(); 851 Register t_lo = dest->as_register_lo(); 852 Register t_hi = dest->as_register_hi(); 853 #ifdef _LP64 854 assert(f_hi == f_lo, "must be same"); 855 assert(t_hi == t_lo, "must be same"); 856 move_regs(f_lo, t_lo); 857 #else 858 assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation"); 859 860 861 if (f_lo == t_hi && f_hi == t_lo) { 862 swap_reg(f_lo, f_hi); 863 } else if (f_hi == t_lo) { 864 assert(f_lo != t_hi, "overwriting register"); 865 move_regs(f_hi, t_hi); 866 move_regs(f_lo, t_lo); 867 } else { 868 assert(f_hi != t_lo, "overwriting register"); 869 move_regs(f_lo, t_lo); 870 move_regs(f_hi, t_hi); 871 } 872 #endif // LP64 873 874 #ifndef _LP64 875 // special moves from fpu-register to xmm-register 876 // necessary for method results 877 } else if (src->is_single_xmm() && !dest->is_single_xmm()) { 878 __ movflt(Address(rsp, 0), src->as_xmm_float_reg()); 879 __ fld_s(Address(rsp, 0)); 880 } else if (src->is_double_xmm() && !dest->is_double_xmm()) { 881 __ movdbl(Address(rsp, 0), src->as_xmm_double_reg()); 882 __ fld_d(Address(rsp, 0)); 883 } else if (dest->is_single_xmm() && !src->is_single_xmm()) { 884 __ fstp_s(Address(rsp, 0)); 885 __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0)); 886 } else if (dest->is_double_xmm() && !src->is_double_xmm()) { 887 __ fstp_d(Address(rsp, 0)); 888 __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0)); 889 #endif // !_LP64 890 891 // move between xmm-registers 892 } else if (dest->is_single_xmm()) { 893 assert(src->is_single_xmm(), "must match"); 894 __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg()); 895 } else if (dest->is_double_xmm()) { 896 assert(src->is_double_xmm(), "must match"); 897 __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg()); 898 899 #ifndef _LP64 900 // move between fpu-registers (no instruction necessary because of fpu-stack) 901 } else if (dest->is_single_fpu() || dest->is_double_fpu()) { 902 assert(src->is_single_fpu() || src->is_double_fpu(), "must match"); 903 assert(src->fpu() == dest->fpu(), "currently should be nothing to do"); 904 #endif // !_LP64 905 906 } else { 907 ShouldNotReachHere(); 908 } 909 } 910 911 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 912 assert(src->is_register(), "should not call otherwise"); 913 assert(dest->is_stack(), "should not call otherwise"); 914 915 if (src->is_single_cpu()) { 916 Address dst = frame_map()->address_for_slot(dest->single_stack_ix()); 917 if (is_reference_type(type)) { 918 __ verify_oop(src->as_register()); 919 __ movptr (dst, src->as_register()); 920 } else if (type == T_METADATA || type == T_ADDRESS) { 921 __ movptr (dst, src->as_register()); 922 } else { 923 __ movl (dst, src->as_register()); 924 } 925 926 } else if (src->is_double_cpu()) { 927 Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes); 928 Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes); 929 __ movptr (dstLO, src->as_register_lo()); 930 NOT_LP64(__ movptr (dstHI, src->as_register_hi())); 931 932 } else if (src->is_single_xmm()) { 933 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix()); 934 __ movflt(dst_addr, src->as_xmm_float_reg()); 935 936 } else if (src->is_double_xmm()) { 937 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix()); 938 __ movdbl(dst_addr, src->as_xmm_double_reg()); 939 940 #ifndef _LP64 941 } else if (src->is_single_fpu()) { 942 assert(src->fpu_regnr() == 0, "argument must be on TOS"); 943 Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix()); 944 if (pop_fpu_stack) __ fstp_s (dst_addr); 945 else __ fst_s (dst_addr); 946 947 } else if (src->is_double_fpu()) { 948 assert(src->fpu_regnrLo() == 0, "argument must be on TOS"); 949 Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix()); 950 if (pop_fpu_stack) __ fstp_d (dst_addr); 951 else __ fst_d (dst_addr); 952 #endif // !_LP64 953 954 } else { 955 ShouldNotReachHere(); 956 } 957 } 958 959 960 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) { 961 LIR_Address* to_addr = dest->as_address_ptr(); 962 PatchingStub* patch = NULL; 963 Register compressed_src = rscratch1; 964 965 if (is_reference_type(type)) { 966 __ verify_oop(src->as_register()); 967 #ifdef _LP64 968 if (UseCompressedOops && !wide) { 969 __ movptr(compressed_src, src->as_register()); 970 __ encode_heap_oop(compressed_src); 971 if (patch_code != lir_patch_none) { 972 info->oop_map()->set_narrowoop(compressed_src->as_VMReg()); 973 } 974 } 975 #endif 976 } 977 978 if (patch_code != lir_patch_none) { 979 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 980 Address toa = as_Address(to_addr); 981 assert(toa.disp() != 0, "must have"); 982 } 983 984 int null_check_here = code_offset(); 985 switch (type) { 986 case T_FLOAT: { 987 #ifdef _LP64 988 assert(src->is_single_xmm(), "not a float"); 989 __ movflt(as_Address(to_addr), src->as_xmm_float_reg()); 990 #else 991 if (src->is_single_xmm()) { 992 __ movflt(as_Address(to_addr), src->as_xmm_float_reg()); 993 } else { 994 assert(src->is_single_fpu(), "must be"); 995 assert(src->fpu_regnr() == 0, "argument must be on TOS"); 996 if (pop_fpu_stack) __ fstp_s(as_Address(to_addr)); 997 else __ fst_s (as_Address(to_addr)); 998 } 999 #endif // _LP64 1000 break; 1001 } 1002 1003 case T_DOUBLE: { 1004 #ifdef _LP64 1005 assert(src->is_double_xmm(), "not a double"); 1006 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg()); 1007 #else 1008 if (src->is_double_xmm()) { 1009 __ movdbl(as_Address(to_addr), src->as_xmm_double_reg()); 1010 } else { 1011 assert(src->is_double_fpu(), "must be"); 1012 assert(src->fpu_regnrLo() == 0, "argument must be on TOS"); 1013 if (pop_fpu_stack) __ fstp_d(as_Address(to_addr)); 1014 else __ fst_d (as_Address(to_addr)); 1015 } 1016 #endif // _LP64 1017 break; 1018 } 1019 1020 case T_ARRAY: // fall through 1021 case T_OBJECT: // fall through 1022 if (UseCompressedOops && !wide) { 1023 __ movl(as_Address(to_addr), compressed_src); 1024 } else { 1025 __ movptr(as_Address(to_addr), src->as_register()); 1026 } 1027 break; 1028 case T_METADATA: 1029 // We get here to store a method pointer to the stack to pass to 1030 // a dtrace runtime call. This can't work on 64 bit with 1031 // compressed klass ptrs: T_METADATA can be a compressed klass 1032 // ptr or a 64 bit method pointer. 1033 LP64_ONLY(ShouldNotReachHere()); 1034 __ movptr(as_Address(to_addr), src->as_register()); 1035 break; 1036 case T_ADDRESS: 1037 __ movptr(as_Address(to_addr), src->as_register()); 1038 break; 1039 case T_INT: 1040 __ movl(as_Address(to_addr), src->as_register()); 1041 break; 1042 1043 case T_LONG: { 1044 Register from_lo = src->as_register_lo(); 1045 Register from_hi = src->as_register_hi(); 1046 #ifdef _LP64 1047 __ movptr(as_Address_lo(to_addr), from_lo); 1048 #else 1049 Register base = to_addr->base()->as_register(); 1050 Register index = noreg; 1051 if (to_addr->index()->is_register()) { 1052 index = to_addr->index()->as_register(); 1053 } 1054 if (base == from_lo || index == from_lo) { 1055 assert(base != from_hi, "can't be"); 1056 assert(index == noreg || (index != base && index != from_hi), "can't handle this"); 1057 __ movl(as_Address_hi(to_addr), from_hi); 1058 if (patch != NULL) { 1059 patching_epilog(patch, lir_patch_high, base, info); 1060 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1061 patch_code = lir_patch_low; 1062 } 1063 __ movl(as_Address_lo(to_addr), from_lo); 1064 } else { 1065 assert(index == noreg || (index != base && index != from_lo), "can't handle this"); 1066 __ movl(as_Address_lo(to_addr), from_lo); 1067 if (patch != NULL) { 1068 patching_epilog(patch, lir_patch_low, base, info); 1069 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1070 patch_code = lir_patch_high; 1071 } 1072 __ movl(as_Address_hi(to_addr), from_hi); 1073 } 1074 #endif // _LP64 1075 break; 1076 } 1077 1078 case T_BYTE: // fall through 1079 case T_BOOLEAN: { 1080 Register src_reg = src->as_register(); 1081 Address dst_addr = as_Address(to_addr); 1082 assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6"); 1083 __ movb(dst_addr, src_reg); 1084 break; 1085 } 1086 1087 case T_CHAR: // fall through 1088 case T_SHORT: 1089 __ movw(as_Address(to_addr), src->as_register()); 1090 break; 1091 1092 default: 1093 ShouldNotReachHere(); 1094 } 1095 if (info != NULL) { 1096 add_debug_info_for_null_check(null_check_here, info); 1097 } 1098 1099 if (patch_code != lir_patch_none) { 1100 patching_epilog(patch, patch_code, to_addr->base()->as_register(), info); 1101 } 1102 } 1103 1104 1105 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 1106 assert(src->is_stack(), "should not call otherwise"); 1107 assert(dest->is_register(), "should not call otherwise"); 1108 1109 if (dest->is_single_cpu()) { 1110 if (is_reference_type(type)) { 1111 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 1112 __ verify_oop(dest->as_register()); 1113 } else if (type == T_METADATA || type == T_ADDRESS) { 1114 __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 1115 } else { 1116 __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix())); 1117 } 1118 1119 } else if (dest->is_double_cpu()) { 1120 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes); 1121 Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes); 1122 __ movptr(dest->as_register_lo(), src_addr_LO); 1123 NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI)); 1124 1125 } else if (dest->is_single_xmm()) { 1126 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix()); 1127 __ movflt(dest->as_xmm_float_reg(), src_addr); 1128 1129 } else if (dest->is_double_xmm()) { 1130 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix()); 1131 __ movdbl(dest->as_xmm_double_reg(), src_addr); 1132 1133 #ifndef _LP64 1134 } else if (dest->is_single_fpu()) { 1135 assert(dest->fpu_regnr() == 0, "dest must be TOS"); 1136 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix()); 1137 __ fld_s(src_addr); 1138 1139 } else if (dest->is_double_fpu()) { 1140 assert(dest->fpu_regnrLo() == 0, "dest must be TOS"); 1141 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix()); 1142 __ fld_d(src_addr); 1143 #endif // _LP64 1144 1145 } else { 1146 ShouldNotReachHere(); 1147 } 1148 } 1149 1150 1151 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 1152 if (src->is_single_stack()) { 1153 if (is_reference_type(type)) { 1154 __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix())); 1155 __ popptr (frame_map()->address_for_slot(dest->single_stack_ix())); 1156 } else { 1157 #ifndef _LP64 1158 __ pushl(frame_map()->address_for_slot(src ->single_stack_ix())); 1159 __ popl (frame_map()->address_for_slot(dest->single_stack_ix())); 1160 #else 1161 //no pushl on 64bits 1162 __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix())); 1163 __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1); 1164 #endif 1165 } 1166 1167 } else if (src->is_double_stack()) { 1168 #ifdef _LP64 1169 __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix())); 1170 __ popptr (frame_map()->address_for_slot(dest->double_stack_ix())); 1171 #else 1172 __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0)); 1173 // push and pop the part at src + wordSize, adding wordSize for the previous push 1174 __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize)); 1175 __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize)); 1176 __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0)); 1177 #endif // _LP64 1178 1179 } else { 1180 ShouldNotReachHere(); 1181 } 1182 } 1183 1184 1185 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) { 1186 assert(src->is_address(), "should not call otherwise"); 1187 assert(dest->is_register(), "should not call otherwise"); 1188 1189 LIR_Address* addr = src->as_address_ptr(); 1190 Address from_addr = as_Address(addr); 1191 1192 if (addr->base()->type() == T_OBJECT) { 1193 __ verify_oop(addr->base()->as_pointer_register()); 1194 } 1195 1196 switch (type) { 1197 case T_BOOLEAN: // fall through 1198 case T_BYTE: // fall through 1199 case T_CHAR: // fall through 1200 case T_SHORT: 1201 if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) { 1202 // on pre P6 processors we may get partial register stalls 1203 // so blow away the value of to_rinfo before loading a 1204 // partial word into it. Do it here so that it precedes 1205 // the potential patch point below. 1206 __ xorptr(dest->as_register(), dest->as_register()); 1207 } 1208 break; 1209 default: 1210 break; 1211 } 1212 1213 PatchingStub* patch = NULL; 1214 if (patch_code != lir_patch_none) { 1215 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1216 assert(from_addr.disp() != 0, "must have"); 1217 } 1218 if (info != NULL) { 1219 add_debug_info_for_null_check_here(info); 1220 } 1221 1222 switch (type) { 1223 case T_FLOAT: { 1224 if (dest->is_single_xmm()) { 1225 __ movflt(dest->as_xmm_float_reg(), from_addr); 1226 } else { 1227 #ifndef _LP64 1228 assert(dest->is_single_fpu(), "must be"); 1229 assert(dest->fpu_regnr() == 0, "dest must be TOS"); 1230 __ fld_s(from_addr); 1231 #else 1232 ShouldNotReachHere(); 1233 #endif // !LP64 1234 } 1235 break; 1236 } 1237 1238 case T_DOUBLE: { 1239 if (dest->is_double_xmm()) { 1240 __ movdbl(dest->as_xmm_double_reg(), from_addr); 1241 } else { 1242 #ifndef _LP64 1243 assert(dest->is_double_fpu(), "must be"); 1244 assert(dest->fpu_regnrLo() == 0, "dest must be TOS"); 1245 __ fld_d(from_addr); 1246 #else 1247 ShouldNotReachHere(); 1248 #endif // !LP64 1249 } 1250 break; 1251 } 1252 1253 case T_OBJECT: // fall through 1254 case T_ARRAY: // fall through 1255 if (UseCompressedOops && !wide) { 1256 __ movl(dest->as_register(), from_addr); 1257 } else { 1258 __ movptr(dest->as_register(), from_addr); 1259 } 1260 break; 1261 1262 case T_ADDRESS: 1263 __ movptr(dest->as_register(), from_addr); 1264 break; 1265 case T_INT: 1266 __ movl(dest->as_register(), from_addr); 1267 break; 1268 1269 case T_LONG: { 1270 Register to_lo = dest->as_register_lo(); 1271 Register to_hi = dest->as_register_hi(); 1272 #ifdef _LP64 1273 __ movptr(to_lo, as_Address_lo(addr)); 1274 #else 1275 Register base = addr->base()->as_register(); 1276 Register index = noreg; 1277 if (addr->index()->is_register()) { 1278 index = addr->index()->as_register(); 1279 } 1280 if ((base == to_lo && index == to_hi) || 1281 (base == to_hi && index == to_lo)) { 1282 // addresses with 2 registers are only formed as a result of 1283 // array access so this code will never have to deal with 1284 // patches or null checks. 1285 assert(info == NULL && patch == NULL, "must be"); 1286 __ lea(to_hi, as_Address(addr)); 1287 __ movl(to_lo, Address(to_hi, 0)); 1288 __ movl(to_hi, Address(to_hi, BytesPerWord)); 1289 } else if (base == to_lo || index == to_lo) { 1290 assert(base != to_hi, "can't be"); 1291 assert(index == noreg || (index != base && index != to_hi), "can't handle this"); 1292 __ movl(to_hi, as_Address_hi(addr)); 1293 if (patch != NULL) { 1294 patching_epilog(patch, lir_patch_high, base, info); 1295 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1296 patch_code = lir_patch_low; 1297 } 1298 __ movl(to_lo, as_Address_lo(addr)); 1299 } else { 1300 assert(index == noreg || (index != base && index != to_lo), "can't handle this"); 1301 __ movl(to_lo, as_Address_lo(addr)); 1302 if (patch != NULL) { 1303 patching_epilog(patch, lir_patch_low, base, info); 1304 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 1305 patch_code = lir_patch_high; 1306 } 1307 __ movl(to_hi, as_Address_hi(addr)); 1308 } 1309 #endif // _LP64 1310 break; 1311 } 1312 1313 case T_BOOLEAN: // fall through 1314 case T_BYTE: { 1315 Register dest_reg = dest->as_register(); 1316 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6"); 1317 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) { 1318 __ movsbl(dest_reg, from_addr); 1319 } else { 1320 __ movb(dest_reg, from_addr); 1321 __ shll(dest_reg, 24); 1322 __ sarl(dest_reg, 24); 1323 } 1324 break; 1325 } 1326 1327 case T_CHAR: { 1328 Register dest_reg = dest->as_register(); 1329 assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6"); 1330 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) { 1331 __ movzwl(dest_reg, from_addr); 1332 } else { 1333 __ movw(dest_reg, from_addr); 1334 } 1335 break; 1336 } 1337 1338 case T_SHORT: { 1339 Register dest_reg = dest->as_register(); 1340 if (VM_Version::is_P6() || from_addr.uses(dest_reg)) { 1341 __ movswl(dest_reg, from_addr); 1342 } else { 1343 __ movw(dest_reg, from_addr); 1344 __ shll(dest_reg, 16); 1345 __ sarl(dest_reg, 16); 1346 } 1347 break; 1348 } 1349 1350 default: 1351 ShouldNotReachHere(); 1352 } 1353 1354 if (patch != NULL) { 1355 patching_epilog(patch, patch_code, addr->base()->as_register(), info); 1356 } 1357 1358 if (is_reference_type(type)) { 1359 #ifdef _LP64 1360 if (UseCompressedOops && !wide) { 1361 __ decode_heap_oop(dest->as_register()); 1362 } 1363 #endif 1364 1365 // Load barrier has not yet been applied, so ZGC can't verify the oop here 1366 if (!UseZGC) { 1367 __ verify_oop(dest->as_register()); 1368 } 1369 } 1370 } 1371 1372 1373 NEEDS_CLEANUP; // This could be static? 1374 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const { 1375 int elem_size = type2aelembytes(type); 1376 switch (elem_size) { 1377 case 1: return Address::times_1; 1378 case 2: return Address::times_2; 1379 case 4: return Address::times_4; 1380 case 8: return Address::times_8; 1381 } 1382 ShouldNotReachHere(); 1383 return Address::no_scale; 1384 } 1385 1386 1387 void LIR_Assembler::emit_op3(LIR_Op3* op) { 1388 switch (op->code()) { 1389 case lir_idiv: 1390 case lir_irem: 1391 arithmetic_idiv(op->code(), 1392 op->in_opr1(), 1393 op->in_opr2(), 1394 op->in_opr3(), 1395 op->result_opr(), 1396 op->info()); 1397 break; 1398 case lir_fmad: 1399 __ fmad(op->result_opr()->as_xmm_double_reg(), 1400 op->in_opr1()->as_xmm_double_reg(), 1401 op->in_opr2()->as_xmm_double_reg(), 1402 op->in_opr3()->as_xmm_double_reg()); 1403 break; 1404 case lir_fmaf: 1405 __ fmaf(op->result_opr()->as_xmm_float_reg(), 1406 op->in_opr1()->as_xmm_float_reg(), 1407 op->in_opr2()->as_xmm_float_reg(), 1408 op->in_opr3()->as_xmm_float_reg()); 1409 break; 1410 default: ShouldNotReachHere(); break; 1411 } 1412 } 1413 1414 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 1415 #ifdef ASSERT 1416 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label"); 1417 if (op->block() != NULL) _branch_target_blocks.append(op->block()); 1418 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock()); 1419 #endif 1420 1421 if (op->cond() == lir_cond_always) { 1422 if (op->info() != NULL) add_debug_info_for_branch(op->info()); 1423 __ jmp (*(op->label())); 1424 } else { 1425 Assembler::Condition acond = Assembler::zero; 1426 if (op->code() == lir_cond_float_branch) { 1427 assert(op->ublock() != NULL, "must have unordered successor"); 1428 __ jcc(Assembler::parity, *(op->ublock()->label())); 1429 switch(op->cond()) { 1430 case lir_cond_equal: acond = Assembler::equal; break; 1431 case lir_cond_notEqual: acond = Assembler::notEqual; break; 1432 case lir_cond_less: acond = Assembler::below; break; 1433 case lir_cond_lessEqual: acond = Assembler::belowEqual; break; 1434 case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break; 1435 case lir_cond_greater: acond = Assembler::above; break; 1436 default: ShouldNotReachHere(); 1437 } 1438 } else { 1439 switch (op->cond()) { 1440 case lir_cond_equal: acond = Assembler::equal; break; 1441 case lir_cond_notEqual: acond = Assembler::notEqual; break; 1442 case lir_cond_less: acond = Assembler::less; break; 1443 case lir_cond_lessEqual: acond = Assembler::lessEqual; break; 1444 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break; 1445 case lir_cond_greater: acond = Assembler::greater; break; 1446 case lir_cond_belowEqual: acond = Assembler::belowEqual; break; 1447 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break; 1448 default: ShouldNotReachHere(); 1449 } 1450 } 1451 __ jcc(acond,*(op->label())); 1452 } 1453 } 1454 1455 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 1456 LIR_Opr src = op->in_opr(); 1457 LIR_Opr dest = op->result_opr(); 1458 1459 switch (op->bytecode()) { 1460 case Bytecodes::_i2l: 1461 #ifdef _LP64 1462 __ movl2ptr(dest->as_register_lo(), src->as_register()); 1463 #else 1464 move_regs(src->as_register(), dest->as_register_lo()); 1465 move_regs(src->as_register(), dest->as_register_hi()); 1466 __ sarl(dest->as_register_hi(), 31); 1467 #endif // LP64 1468 break; 1469 1470 case Bytecodes::_l2i: 1471 #ifdef _LP64 1472 __ movl(dest->as_register(), src->as_register_lo()); 1473 #else 1474 move_regs(src->as_register_lo(), dest->as_register()); 1475 #endif 1476 break; 1477 1478 case Bytecodes::_i2b: 1479 move_regs(src->as_register(), dest->as_register()); 1480 __ sign_extend_byte(dest->as_register()); 1481 break; 1482 1483 case Bytecodes::_i2c: 1484 move_regs(src->as_register(), dest->as_register()); 1485 __ andl(dest->as_register(), 0xFFFF); 1486 break; 1487 1488 case Bytecodes::_i2s: 1489 move_regs(src->as_register(), dest->as_register()); 1490 __ sign_extend_short(dest->as_register()); 1491 break; 1492 1493 1494 #ifdef _LP64 1495 case Bytecodes::_f2d: 1496 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg()); 1497 break; 1498 1499 case Bytecodes::_d2f: 1500 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg()); 1501 break; 1502 1503 case Bytecodes::_i2f: 1504 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register()); 1505 break; 1506 1507 case Bytecodes::_i2d: 1508 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register()); 1509 break; 1510 1511 case Bytecodes::_l2f: 1512 __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo()); 1513 break; 1514 1515 case Bytecodes::_l2d: 1516 __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo()); 1517 break; 1518 1519 case Bytecodes::_f2i: 1520 __ convert_f2i(dest->as_register(), src->as_xmm_float_reg()); 1521 break; 1522 1523 case Bytecodes::_d2i: 1524 __ convert_d2i(dest->as_register(), src->as_xmm_double_reg()); 1525 break; 1526 1527 case Bytecodes::_f2l: 1528 __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg()); 1529 break; 1530 1531 case Bytecodes::_d2l: 1532 __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg()); 1533 break; 1534 #else 1535 case Bytecodes::_f2d: 1536 case Bytecodes::_d2f: 1537 if (dest->is_single_xmm()) { 1538 __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg()); 1539 } else if (dest->is_double_xmm()) { 1540 __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg()); 1541 } else { 1542 assert(src->fpu() == dest->fpu(), "register must be equal"); 1543 // do nothing (float result is rounded later through spilling) 1544 } 1545 break; 1546 1547 case Bytecodes::_i2f: 1548 case Bytecodes::_i2d: 1549 if (dest->is_single_xmm()) { 1550 __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register()); 1551 } else if (dest->is_double_xmm()) { 1552 __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register()); 1553 } else { 1554 assert(dest->fpu() == 0, "result must be on TOS"); 1555 __ movl(Address(rsp, 0), src->as_register()); 1556 __ fild_s(Address(rsp, 0)); 1557 } 1558 break; 1559 1560 case Bytecodes::_l2f: 1561 case Bytecodes::_l2d: 1562 assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)"); 1563 assert(dest->fpu() == 0, "result must be on TOS"); 1564 __ movptr(Address(rsp, 0), src->as_register_lo()); 1565 __ movl(Address(rsp, BytesPerWord), src->as_register_hi()); 1566 __ fild_d(Address(rsp, 0)); 1567 // float result is rounded later through spilling 1568 break; 1569 1570 case Bytecodes::_f2i: 1571 case Bytecodes::_d2i: 1572 if (src->is_single_xmm()) { 1573 __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg()); 1574 } else if (src->is_double_xmm()) { 1575 __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg()); 1576 } else { 1577 assert(src->fpu() == 0, "input must be on TOS"); 1578 __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_trunc())); 1579 __ fist_s(Address(rsp, 0)); 1580 __ movl(dest->as_register(), Address(rsp, 0)); 1581 __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std())); 1582 } 1583 // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub 1584 assert(op->stub() != NULL, "stub required"); 1585 __ cmpl(dest->as_register(), 0x80000000); 1586 __ jcc(Assembler::equal, *op->stub()->entry()); 1587 __ bind(*op->stub()->continuation()); 1588 break; 1589 1590 case Bytecodes::_f2l: 1591 case Bytecodes::_d2l: 1592 assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)"); 1593 assert(src->fpu() == 0, "input must be on TOS"); 1594 assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers"); 1595 1596 // instruction sequence too long to inline it here 1597 { 1598 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id))); 1599 } 1600 break; 1601 #endif // _LP64 1602 1603 default: ShouldNotReachHere(); 1604 } 1605 } 1606 1607 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1608 if (op->init_check()) { 1609 add_debug_info_for_null_check_here(op->stub()->info()); 1610 __ cmpb(Address(op->klass()->as_register(), 1611 InstanceKlass::init_state_offset()), 1612 InstanceKlass::fully_initialized); 1613 __ jcc(Assembler::notEqual, *op->stub()->entry()); 1614 } 1615 __ allocate_object(op->obj()->as_register(), 1616 op->tmp1()->as_register(), 1617 op->tmp2()->as_register(), 1618 op->header_size(), 1619 op->object_size(), 1620 op->klass()->as_register(), 1621 *op->stub()->entry()); 1622 __ bind(*op->stub()->continuation()); 1623 } 1624 1625 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1626 Register len = op->len()->as_register(); 1627 LP64_ONLY( __ movslq(len, len); ) 1628 1629 if (UseSlowPath || 1630 (!UseFastNewObjectArray && is_reference_type(op->type())) || 1631 (!UseFastNewTypeArray && !is_reference_type(op->type()))) { 1632 __ jmp(*op->stub()->entry()); 1633 } else { 1634 Register tmp1 = op->tmp1()->as_register(); 1635 Register tmp2 = op->tmp2()->as_register(); 1636 Register tmp3 = op->tmp3()->as_register(); 1637 if (len == tmp1) { 1638 tmp1 = tmp3; 1639 } else if (len == tmp2) { 1640 tmp2 = tmp3; 1641 } else if (len == tmp3) { 1642 // everything is ok 1643 } else { 1644 __ mov(tmp3, len); 1645 } 1646 __ allocate_array(op->obj()->as_register(), 1647 len, 1648 tmp1, 1649 tmp2, 1650 arrayOopDesc::header_size(op->type()), 1651 array_element_size(op->type()), 1652 op->klass()->as_register(), 1653 *op->stub()->entry()); 1654 } 1655 __ bind(*op->stub()->continuation()); 1656 } 1657 1658 void LIR_Assembler::type_profile_helper(Register mdo, 1659 ciMethodData *md, ciProfileData *data, 1660 Register recv, Label* update_done) { 1661 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1662 Label next_test; 1663 // See if the receiver is receiver[n]. 1664 __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1665 __ jccb(Assembler::notEqual, next_test); 1666 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); 1667 __ addptr(data_addr, DataLayout::counter_increment); 1668 __ jmp(*update_done); 1669 __ bind(next_test); 1670 } 1671 1672 // Didn't find receiver; find next empty slot and fill it in 1673 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1674 Label next_test; 1675 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))); 1676 __ cmpptr(recv_addr, (intptr_t)NULL_WORD); 1677 __ jccb(Assembler::notEqual, next_test); 1678 __ movptr(recv_addr, recv); 1679 __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment); 1680 __ jmp(*update_done); 1681 __ bind(next_test); 1682 } 1683 } 1684 1685 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1686 // we always need a stub for the failure case. 1687 CodeStub* stub = op->stub(); 1688 Register obj = op->object()->as_register(); 1689 Register k_RInfo = op->tmp1()->as_register(); 1690 Register klass_RInfo = op->tmp2()->as_register(); 1691 Register dst = op->result_opr()->as_register(); 1692 ciKlass* k = op->klass(); 1693 Register Rtmp1 = noreg; 1694 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 1695 1696 // check if it needs to be profiled 1697 ciMethodData* md = NULL; 1698 ciProfileData* data = NULL; 1699 1700 if (op->should_profile()) { 1701 ciMethod* method = op->profiled_method(); 1702 assert(method != NULL, "Should have method"); 1703 int bci = op->profiled_bci(); 1704 md = method->method_data_or_null(); 1705 assert(md != NULL, "Sanity"); 1706 data = md->bci_to_data(bci); 1707 assert(data != NULL, "need data for type check"); 1708 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1709 } 1710 Label profile_cast_success, profile_cast_failure; 1711 Label *success_target = op->should_profile() ? &profile_cast_success : success; 1712 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure; 1713 1714 if (obj == k_RInfo) { 1715 k_RInfo = dst; 1716 } else if (obj == klass_RInfo) { 1717 klass_RInfo = dst; 1718 } 1719 if (k->is_loaded() && !UseCompressedClassPointers) { 1720 select_different_registers(obj, dst, k_RInfo, klass_RInfo); 1721 } else { 1722 Rtmp1 = op->tmp3()->as_register(); 1723 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1724 } 1725 1726 assert_different_registers(obj, k_RInfo, klass_RInfo); 1727 1728 __ cmpptr(obj, (int32_t)NULL_WORD); 1729 if (op->should_profile()) { 1730 Label not_null; 1731 __ jccb(Assembler::notEqual, not_null); 1732 // Object is null; update MDO and exit 1733 Register mdo = klass_RInfo; 1734 __ mov_metadata(mdo, md->constant_encoding()); 1735 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset())); 1736 int header_bits = BitData::null_seen_byte_constant(); 1737 __ orb(data_addr, header_bits); 1738 __ jmp(*obj_is_null); 1739 __ bind(not_null); 1740 } else { 1741 __ jcc(Assembler::equal, *obj_is_null); 1742 } 1743 1744 if (!k->is_loaded()) { 1745 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1746 } else { 1747 #ifdef _LP64 1748 __ mov_metadata(k_RInfo, k->constant_encoding()); 1749 #endif // _LP64 1750 } 1751 __ verify_oop(obj); 1752 1753 if (op->fast_check()) { 1754 // get object class 1755 // not a safepoint as obj null check happens earlier 1756 #ifdef _LP64 1757 if (UseCompressedClassPointers) { 1758 __ load_klass(Rtmp1, obj, tmp_load_klass); 1759 __ cmpptr(k_RInfo, Rtmp1); 1760 } else { 1761 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes())); 1762 } 1763 #else 1764 if (k->is_loaded()) { 1765 __ cmpklass(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding()); 1766 } else { 1767 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes())); 1768 } 1769 #endif 1770 __ jcc(Assembler::notEqual, *failure_target); 1771 // successful cast, fall through to profile or jump 1772 } else { 1773 // get object class 1774 // not a safepoint as obj null check happens earlier 1775 __ load_klass(klass_RInfo, obj, tmp_load_klass); 1776 if (k->is_loaded()) { 1777 // See if we get an immediate positive hit 1778 #ifdef _LP64 1779 __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset())); 1780 #else 1781 __ cmpklass(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding()); 1782 #endif // _LP64 1783 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1784 __ jcc(Assembler::notEqual, *failure_target); 1785 // successful cast, fall through to profile or jump 1786 } else { 1787 // See if we get an immediate positive hit 1788 __ jcc(Assembler::equal, *success_target); 1789 // check for self 1790 #ifdef _LP64 1791 __ cmpptr(klass_RInfo, k_RInfo); 1792 #else 1793 __ cmpklass(klass_RInfo, k->constant_encoding()); 1794 #endif // _LP64 1795 __ jcc(Assembler::equal, *success_target); 1796 1797 __ push(klass_RInfo); 1798 #ifdef _LP64 1799 __ push(k_RInfo); 1800 #else 1801 __ pushklass(k->constant_encoding()); 1802 #endif // _LP64 1803 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1804 __ pop(klass_RInfo); 1805 __ pop(klass_RInfo); 1806 // result is a boolean 1807 __ cmpl(klass_RInfo, 0); 1808 __ jcc(Assembler::equal, *failure_target); 1809 // successful cast, fall through to profile or jump 1810 } 1811 } else { 1812 // perform the fast part of the checking logic 1813 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1814 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1815 __ push(klass_RInfo); 1816 __ push(k_RInfo); 1817 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1818 __ pop(klass_RInfo); 1819 __ pop(k_RInfo); 1820 // result is a boolean 1821 __ cmpl(k_RInfo, 0); 1822 __ jcc(Assembler::equal, *failure_target); 1823 // successful cast, fall through to profile or jump 1824 } 1825 } 1826 if (op->should_profile()) { 1827 Register mdo = klass_RInfo, recv = k_RInfo; 1828 __ bind(profile_cast_success); 1829 __ mov_metadata(mdo, md->constant_encoding()); 1830 __ load_klass(recv, obj, tmp_load_klass); 1831 type_profile_helper(mdo, md, data, recv, success); 1832 __ jmp(*success); 1833 1834 __ bind(profile_cast_failure); 1835 __ mov_metadata(mdo, md->constant_encoding()); 1836 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1837 __ subptr(counter_addr, DataLayout::counter_increment); 1838 __ jmp(*failure); 1839 } 1840 __ jmp(*success); 1841 } 1842 1843 1844 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1845 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 1846 LIR_Code code = op->code(); 1847 if (code == lir_store_check) { 1848 Register value = op->object()->as_register(); 1849 Register array = op->array()->as_register(); 1850 Register k_RInfo = op->tmp1()->as_register(); 1851 Register klass_RInfo = op->tmp2()->as_register(); 1852 Register Rtmp1 = op->tmp3()->as_register(); 1853 1854 CodeStub* stub = op->stub(); 1855 1856 // check if it needs to be profiled 1857 ciMethodData* md = NULL; 1858 ciProfileData* data = NULL; 1859 1860 if (op->should_profile()) { 1861 ciMethod* method = op->profiled_method(); 1862 assert(method != NULL, "Should have method"); 1863 int bci = op->profiled_bci(); 1864 md = method->method_data_or_null(); 1865 assert(md != NULL, "Sanity"); 1866 data = md->bci_to_data(bci); 1867 assert(data != NULL, "need data for type check"); 1868 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1869 } 1870 Label profile_cast_success, profile_cast_failure, done; 1871 Label *success_target = op->should_profile() ? &profile_cast_success : &done; 1872 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry(); 1873 1874 __ cmpptr(value, (int32_t)NULL_WORD); 1875 if (op->should_profile()) { 1876 Label not_null; 1877 __ jccb(Assembler::notEqual, not_null); 1878 // Object is null; update MDO and exit 1879 Register mdo = klass_RInfo; 1880 __ mov_metadata(mdo, md->constant_encoding()); 1881 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset())); 1882 int header_bits = BitData::null_seen_byte_constant(); 1883 __ orb(data_addr, header_bits); 1884 __ jmp(done); 1885 __ bind(not_null); 1886 } else { 1887 __ jcc(Assembler::equal, done); 1888 } 1889 1890 add_debug_info_for_null_check_here(op->info_for_exception()); 1891 __ load_klass(k_RInfo, array, tmp_load_klass); 1892 __ load_klass(klass_RInfo, value, tmp_load_klass); 1893 1894 // get instance klass (it's already uncompressed) 1895 __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 1896 // perform the fast part of the checking logic 1897 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL); 1898 // call out-of-line instance of __ check_klass_subtype_slow_path(...): 1899 __ push(klass_RInfo); 1900 __ push(k_RInfo); 1901 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1902 __ pop(klass_RInfo); 1903 __ pop(k_RInfo); 1904 // result is a boolean 1905 __ cmpl(k_RInfo, 0); 1906 __ jcc(Assembler::equal, *failure_target); 1907 // fall through to the success case 1908 1909 if (op->should_profile()) { 1910 Register mdo = klass_RInfo, recv = k_RInfo; 1911 __ bind(profile_cast_success); 1912 __ mov_metadata(mdo, md->constant_encoding()); 1913 __ load_klass(recv, value, tmp_load_klass); 1914 type_profile_helper(mdo, md, data, recv, &done); 1915 __ jmpb(done); 1916 1917 __ bind(profile_cast_failure); 1918 __ mov_metadata(mdo, md->constant_encoding()); 1919 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1920 __ subptr(counter_addr, DataLayout::counter_increment); 1921 __ jmp(*stub->entry()); 1922 } 1923 1924 __ bind(done); 1925 } else 1926 if (code == lir_checkcast) { 1927 Register obj = op->object()->as_register(); 1928 Register dst = op->result_opr()->as_register(); 1929 Label success; 1930 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1931 __ bind(success); 1932 if (dst != obj) { 1933 __ mov(dst, obj); 1934 } 1935 } else 1936 if (code == lir_instanceof) { 1937 Register obj = op->object()->as_register(); 1938 Register dst = op->result_opr()->as_register(); 1939 Label success, failure, done; 1940 emit_typecheck_helper(op, &success, &failure, &failure); 1941 __ bind(failure); 1942 __ xorptr(dst, dst); 1943 __ jmpb(done); 1944 __ bind(success); 1945 __ movptr(dst, 1); 1946 __ bind(done); 1947 } else { 1948 ShouldNotReachHere(); 1949 } 1950 1951 } 1952 1953 1954 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1955 if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) { 1956 assert(op->cmp_value()->as_register_lo() == rax, "wrong register"); 1957 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register"); 1958 assert(op->new_value()->as_register_lo() == rbx, "wrong register"); 1959 assert(op->new_value()->as_register_hi() == rcx, "wrong register"); 1960 Register addr = op->addr()->as_register(); 1961 __ lock(); 1962 NOT_LP64(__ cmpxchg8(Address(addr, 0))); 1963 1964 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) { 1965 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");) 1966 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo()); 1967 Register newval = op->new_value()->as_register(); 1968 Register cmpval = op->cmp_value()->as_register(); 1969 assert(cmpval == rax, "wrong register"); 1970 assert(newval != NULL, "new val must be register"); 1971 assert(cmpval != newval, "cmp and new values must be in different registers"); 1972 assert(cmpval != addr, "cmp and addr must be in different registers"); 1973 assert(newval != addr, "new value and addr must be in different registers"); 1974 1975 if ( op->code() == lir_cas_obj) { 1976 #ifdef _LP64 1977 if (UseCompressedOops) { 1978 __ encode_heap_oop(cmpval); 1979 __ mov(rscratch1, newval); 1980 __ encode_heap_oop(rscratch1); 1981 __ lock(); 1982 // cmpval (rax) is implicitly used by this instruction 1983 __ cmpxchgl(rscratch1, Address(addr, 0)); 1984 } else 1985 #endif 1986 { 1987 __ lock(); 1988 __ cmpxchgptr(newval, Address(addr, 0)); 1989 } 1990 } else { 1991 assert(op->code() == lir_cas_int, "lir_cas_int expected"); 1992 __ lock(); 1993 __ cmpxchgl(newval, Address(addr, 0)); 1994 } 1995 #ifdef _LP64 1996 } else if (op->code() == lir_cas_long) { 1997 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo()); 1998 Register newval = op->new_value()->as_register_lo(); 1999 Register cmpval = op->cmp_value()->as_register_lo(); 2000 assert(cmpval == rax, "wrong register"); 2001 assert(newval != NULL, "new val must be register"); 2002 assert(cmpval != newval, "cmp and new values must be in different registers"); 2003 assert(cmpval != addr, "cmp and addr must be in different registers"); 2004 assert(newval != addr, "new value and addr must be in different registers"); 2005 __ lock(); 2006 __ cmpxchgq(newval, Address(addr, 0)); 2007 #endif // _LP64 2008 } else { 2009 Unimplemented(); 2010 } 2011 } 2012 2013 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type, 2014 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) { 2015 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86"); 2016 2017 Assembler::Condition acond, ncond; 2018 switch (condition) { 2019 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break; 2020 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break; 2021 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break; 2022 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break; 2023 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break; 2024 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break; 2025 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break; 2026 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break; 2027 default: acond = Assembler::equal; ncond = Assembler::notEqual; 2028 ShouldNotReachHere(); 2029 } 2030 2031 if (opr1->is_cpu_register()) { 2032 reg2reg(opr1, result); 2033 } else if (opr1->is_stack()) { 2034 stack2reg(opr1, result, result->type()); 2035 } else if (opr1->is_constant()) { 2036 const2reg(opr1, result, lir_patch_none, NULL); 2037 } else { 2038 ShouldNotReachHere(); 2039 } 2040 2041 if (VM_Version::supports_cmov() && !opr2->is_constant()) { 2042 // optimized version that does not require a branch 2043 if (opr2->is_single_cpu()) { 2044 assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move"); 2045 __ cmov(ncond, result->as_register(), opr2->as_register()); 2046 } else if (opr2->is_double_cpu()) { 2047 assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move"); 2048 assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move"); 2049 __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo()); 2050 NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());) 2051 } else if (opr2->is_single_stack()) { 2052 __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix())); 2053 } else if (opr2->is_double_stack()) { 2054 __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes)); 2055 NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));) 2056 } else { 2057 ShouldNotReachHere(); 2058 } 2059 2060 } else { 2061 Label skip; 2062 __ jcc (acond, skip); 2063 if (opr2->is_cpu_register()) { 2064 reg2reg(opr2, result); 2065 } else if (opr2->is_stack()) { 2066 stack2reg(opr2, result, result->type()); 2067 } else if (opr2->is_constant()) { 2068 const2reg(opr2, result, lir_patch_none, NULL); 2069 } else { 2070 ShouldNotReachHere(); 2071 } 2072 __ bind(skip); 2073 } 2074 } 2075 2076 2077 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) { 2078 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method"); 2079 2080 if (left->is_single_cpu()) { 2081 assert(left == dest, "left and dest must be equal"); 2082 Register lreg = left->as_register(); 2083 2084 if (right->is_single_cpu()) { 2085 // cpu register - cpu register 2086 Register rreg = right->as_register(); 2087 switch (code) { 2088 case lir_add: __ addl (lreg, rreg); break; 2089 case lir_sub: __ subl (lreg, rreg); break; 2090 case lir_mul: __ imull(lreg, rreg); break; 2091 default: ShouldNotReachHere(); 2092 } 2093 2094 } else if (right->is_stack()) { 2095 // cpu register - stack 2096 Address raddr = frame_map()->address_for_slot(right->single_stack_ix()); 2097 switch (code) { 2098 case lir_add: __ addl(lreg, raddr); break; 2099 case lir_sub: __ subl(lreg, raddr); break; 2100 default: ShouldNotReachHere(); 2101 } 2102 2103 } else if (right->is_constant()) { 2104 // cpu register - constant 2105 jint c = right->as_constant_ptr()->as_jint(); 2106 switch (code) { 2107 case lir_add: { 2108 __ incrementl(lreg, c); 2109 break; 2110 } 2111 case lir_sub: { 2112 __ decrementl(lreg, c); 2113 break; 2114 } 2115 default: ShouldNotReachHere(); 2116 } 2117 2118 } else { 2119 ShouldNotReachHere(); 2120 } 2121 2122 } else if (left->is_double_cpu()) { 2123 assert(left == dest, "left and dest must be equal"); 2124 Register lreg_lo = left->as_register_lo(); 2125 Register lreg_hi = left->as_register_hi(); 2126 2127 if (right->is_double_cpu()) { 2128 // cpu register - cpu register 2129 Register rreg_lo = right->as_register_lo(); 2130 Register rreg_hi = right->as_register_hi(); 2131 NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi)); 2132 LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo)); 2133 switch (code) { 2134 case lir_add: 2135 __ addptr(lreg_lo, rreg_lo); 2136 NOT_LP64(__ adcl(lreg_hi, rreg_hi)); 2137 break; 2138 case lir_sub: 2139 __ subptr(lreg_lo, rreg_lo); 2140 NOT_LP64(__ sbbl(lreg_hi, rreg_hi)); 2141 break; 2142 case lir_mul: 2143 #ifdef _LP64 2144 __ imulq(lreg_lo, rreg_lo); 2145 #else 2146 assert(lreg_lo == rax && lreg_hi == rdx, "must be"); 2147 __ imull(lreg_hi, rreg_lo); 2148 __ imull(rreg_hi, lreg_lo); 2149 __ addl (rreg_hi, lreg_hi); 2150 __ mull (rreg_lo); 2151 __ addl (lreg_hi, rreg_hi); 2152 #endif // _LP64 2153 break; 2154 default: 2155 ShouldNotReachHere(); 2156 } 2157 2158 } else if (right->is_constant()) { 2159 // cpu register - constant 2160 #ifdef _LP64 2161 jlong c = right->as_constant_ptr()->as_jlong_bits(); 2162 __ movptr(r10, (intptr_t) c); 2163 switch (code) { 2164 case lir_add: 2165 __ addptr(lreg_lo, r10); 2166 break; 2167 case lir_sub: 2168 __ subptr(lreg_lo, r10); 2169 break; 2170 default: 2171 ShouldNotReachHere(); 2172 } 2173 #else 2174 jint c_lo = right->as_constant_ptr()->as_jint_lo(); 2175 jint c_hi = right->as_constant_ptr()->as_jint_hi(); 2176 switch (code) { 2177 case lir_add: 2178 __ addptr(lreg_lo, c_lo); 2179 __ adcl(lreg_hi, c_hi); 2180 break; 2181 case lir_sub: 2182 __ subptr(lreg_lo, c_lo); 2183 __ sbbl(lreg_hi, c_hi); 2184 break; 2185 default: 2186 ShouldNotReachHere(); 2187 } 2188 #endif // _LP64 2189 2190 } else { 2191 ShouldNotReachHere(); 2192 } 2193 2194 } else if (left->is_single_xmm()) { 2195 assert(left == dest, "left and dest must be equal"); 2196 XMMRegister lreg = left->as_xmm_float_reg(); 2197 2198 if (right->is_single_xmm()) { 2199 XMMRegister rreg = right->as_xmm_float_reg(); 2200 switch (code) { 2201 case lir_add: __ addss(lreg, rreg); break; 2202 case lir_sub: __ subss(lreg, rreg); break; 2203 case lir_mul: __ mulss(lreg, rreg); break; 2204 case lir_div: __ divss(lreg, rreg); break; 2205 default: ShouldNotReachHere(); 2206 } 2207 } else { 2208 Address raddr; 2209 if (right->is_single_stack()) { 2210 raddr = frame_map()->address_for_slot(right->single_stack_ix()); 2211 } else if (right->is_constant()) { 2212 // hack for now 2213 raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat()))); 2214 } else { 2215 ShouldNotReachHere(); 2216 } 2217 switch (code) { 2218 case lir_add: __ addss(lreg, raddr); break; 2219 case lir_sub: __ subss(lreg, raddr); break; 2220 case lir_mul: __ mulss(lreg, raddr); break; 2221 case lir_div: __ divss(lreg, raddr); break; 2222 default: ShouldNotReachHere(); 2223 } 2224 } 2225 2226 } else if (left->is_double_xmm()) { 2227 assert(left == dest, "left and dest must be equal"); 2228 2229 XMMRegister lreg = left->as_xmm_double_reg(); 2230 if (right->is_double_xmm()) { 2231 XMMRegister rreg = right->as_xmm_double_reg(); 2232 switch (code) { 2233 case lir_add: __ addsd(lreg, rreg); break; 2234 case lir_sub: __ subsd(lreg, rreg); break; 2235 case lir_mul: __ mulsd(lreg, rreg); break; 2236 case lir_div: __ divsd(lreg, rreg); break; 2237 default: ShouldNotReachHere(); 2238 } 2239 } else { 2240 Address raddr; 2241 if (right->is_double_stack()) { 2242 raddr = frame_map()->address_for_slot(right->double_stack_ix()); 2243 } else if (right->is_constant()) { 2244 // hack for now 2245 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble()))); 2246 } else { 2247 ShouldNotReachHere(); 2248 } 2249 switch (code) { 2250 case lir_add: __ addsd(lreg, raddr); break; 2251 case lir_sub: __ subsd(lreg, raddr); break; 2252 case lir_mul: __ mulsd(lreg, raddr); break; 2253 case lir_div: __ divsd(lreg, raddr); break; 2254 default: ShouldNotReachHere(); 2255 } 2256 } 2257 2258 #ifndef _LP64 2259 } else if (left->is_single_fpu()) { 2260 assert(dest->is_single_fpu(), "fpu stack allocation required"); 2261 2262 if (right->is_single_fpu()) { 2263 arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack); 2264 2265 } else { 2266 assert(left->fpu_regnr() == 0, "left must be on TOS"); 2267 assert(dest->fpu_regnr() == 0, "dest must be on TOS"); 2268 2269 Address raddr; 2270 if (right->is_single_stack()) { 2271 raddr = frame_map()->address_for_slot(right->single_stack_ix()); 2272 } else if (right->is_constant()) { 2273 address const_addr = float_constant(right->as_jfloat()); 2274 assert(const_addr != NULL, "incorrect float/double constant maintenance"); 2275 // hack for now 2276 raddr = __ as_Address(InternalAddress(const_addr)); 2277 } else { 2278 ShouldNotReachHere(); 2279 } 2280 2281 switch (code) { 2282 case lir_add: __ fadd_s(raddr); break; 2283 case lir_sub: __ fsub_s(raddr); break; 2284 case lir_mul: __ fmul_s(raddr); break; 2285 case lir_div: __ fdiv_s(raddr); break; 2286 default: ShouldNotReachHere(); 2287 } 2288 } 2289 2290 } else if (left->is_double_fpu()) { 2291 assert(dest->is_double_fpu(), "fpu stack allocation required"); 2292 2293 if (code == lir_mul || code == lir_div) { 2294 // Double values require special handling for strictfp mul/div on x86 2295 __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias1())); 2296 __ fmulp(left->fpu_regnrLo() + 1); 2297 } 2298 2299 if (right->is_double_fpu()) { 2300 arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack); 2301 2302 } else { 2303 assert(left->fpu_regnrLo() == 0, "left must be on TOS"); 2304 assert(dest->fpu_regnrLo() == 0, "dest must be on TOS"); 2305 2306 Address raddr; 2307 if (right->is_double_stack()) { 2308 raddr = frame_map()->address_for_slot(right->double_stack_ix()); 2309 } else if (right->is_constant()) { 2310 // hack for now 2311 raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble()))); 2312 } else { 2313 ShouldNotReachHere(); 2314 } 2315 2316 switch (code) { 2317 case lir_add: __ fadd_d(raddr); break; 2318 case lir_sub: __ fsub_d(raddr); break; 2319 case lir_mul: __ fmul_d(raddr); break; 2320 case lir_div: __ fdiv_d(raddr); break; 2321 default: ShouldNotReachHere(); 2322 } 2323 } 2324 2325 if (code == lir_mul || code == lir_div) { 2326 // Double values require special handling for strictfp mul/div on x86 2327 __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias2())); 2328 __ fmulp(dest->fpu_regnrLo() + 1); 2329 } 2330 #endif // !_LP64 2331 2332 } else if (left->is_single_stack() || left->is_address()) { 2333 assert(left == dest, "left and dest must be equal"); 2334 2335 Address laddr; 2336 if (left->is_single_stack()) { 2337 laddr = frame_map()->address_for_slot(left->single_stack_ix()); 2338 } else if (left->is_address()) { 2339 laddr = as_Address(left->as_address_ptr()); 2340 } else { 2341 ShouldNotReachHere(); 2342 } 2343 2344 if (right->is_single_cpu()) { 2345 Register rreg = right->as_register(); 2346 switch (code) { 2347 case lir_add: __ addl(laddr, rreg); break; 2348 case lir_sub: __ subl(laddr, rreg); break; 2349 default: ShouldNotReachHere(); 2350 } 2351 } else if (right->is_constant()) { 2352 jint c = right->as_constant_ptr()->as_jint(); 2353 switch (code) { 2354 case lir_add: { 2355 __ incrementl(laddr, c); 2356 break; 2357 } 2358 case lir_sub: { 2359 __ decrementl(laddr, c); 2360 break; 2361 } 2362 default: ShouldNotReachHere(); 2363 } 2364 } else { 2365 ShouldNotReachHere(); 2366 } 2367 2368 } else { 2369 ShouldNotReachHere(); 2370 } 2371 } 2372 2373 #ifndef _LP64 2374 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { 2375 assert(pop_fpu_stack || (left_index == dest_index || right_index == dest_index), "invalid LIR"); 2376 assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR"); 2377 assert(left_index == 0 || right_index == 0, "either must be on top of stack"); 2378 2379 bool left_is_tos = (left_index == 0); 2380 bool dest_is_tos = (dest_index == 0); 2381 int non_tos_index = (left_is_tos ? right_index : left_index); 2382 2383 switch (code) { 2384 case lir_add: 2385 if (pop_fpu_stack) __ faddp(non_tos_index); 2386 else if (dest_is_tos) __ fadd (non_tos_index); 2387 else __ fadda(non_tos_index); 2388 break; 2389 2390 case lir_sub: 2391 if (left_is_tos) { 2392 if (pop_fpu_stack) __ fsubrp(non_tos_index); 2393 else if (dest_is_tos) __ fsub (non_tos_index); 2394 else __ fsubra(non_tos_index); 2395 } else { 2396 if (pop_fpu_stack) __ fsubp (non_tos_index); 2397 else if (dest_is_tos) __ fsubr (non_tos_index); 2398 else __ fsuba (non_tos_index); 2399 } 2400 break; 2401 2402 case lir_mul: 2403 if (pop_fpu_stack) __ fmulp(non_tos_index); 2404 else if (dest_is_tos) __ fmul (non_tos_index); 2405 else __ fmula(non_tos_index); 2406 break; 2407 2408 case lir_div: 2409 if (left_is_tos) { 2410 if (pop_fpu_stack) __ fdivrp(non_tos_index); 2411 else if (dest_is_tos) __ fdiv (non_tos_index); 2412 else __ fdivra(non_tos_index); 2413 } else { 2414 if (pop_fpu_stack) __ fdivp (non_tos_index); 2415 else if (dest_is_tos) __ fdivr (non_tos_index); 2416 else __ fdiva (non_tos_index); 2417 } 2418 break; 2419 2420 case lir_rem: 2421 assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation"); 2422 __ fremr(noreg); 2423 break; 2424 2425 default: 2426 ShouldNotReachHere(); 2427 } 2428 } 2429 #endif // _LP64 2430 2431 2432 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) { 2433 if (value->is_double_xmm()) { 2434 switch(code) { 2435 case lir_abs : 2436 { 2437 #ifdef _LP64 2438 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) { 2439 assert(tmp->is_valid(), "need temporary"); 2440 __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2); 2441 } else 2442 #endif 2443 { 2444 if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) { 2445 __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); 2446 } 2447 assert(!tmp->is_valid(), "do not need temporary"); 2448 __ andpd(dest->as_xmm_double_reg(), 2449 ExternalAddress((address)double_signmask_pool)); 2450 } 2451 } 2452 break; 2453 2454 case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break; 2455 // all other intrinsics are not available in the SSE instruction set, so FPU is used 2456 default : ShouldNotReachHere(); 2457 } 2458 2459 #ifndef _LP64 2460 } else if (value->is_double_fpu()) { 2461 assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS"); 2462 switch(code) { 2463 case lir_abs : __ fabs() ; break; 2464 case lir_sqrt : __ fsqrt(); break; 2465 default : ShouldNotReachHere(); 2466 } 2467 #endif // !_LP64 2468 } else { 2469 Unimplemented(); 2470 } 2471 } 2472 2473 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 2474 // assert(left->destroys_register(), "check"); 2475 if (left->is_single_cpu()) { 2476 Register reg = left->as_register(); 2477 if (right->is_constant()) { 2478 int val = right->as_constant_ptr()->as_jint(); 2479 switch (code) { 2480 case lir_logic_and: __ andl (reg, val); break; 2481 case lir_logic_or: __ orl (reg, val); break; 2482 case lir_logic_xor: __ xorl (reg, val); break; 2483 default: ShouldNotReachHere(); 2484 } 2485 } else if (right->is_stack()) { 2486 // added support for stack operands 2487 Address raddr = frame_map()->address_for_slot(right->single_stack_ix()); 2488 switch (code) { 2489 case lir_logic_and: __ andl (reg, raddr); break; 2490 case lir_logic_or: __ orl (reg, raddr); break; 2491 case lir_logic_xor: __ xorl (reg, raddr); break; 2492 default: ShouldNotReachHere(); 2493 } 2494 } else { 2495 Register rright = right->as_register(); 2496 switch (code) { 2497 case lir_logic_and: __ andptr (reg, rright); break; 2498 case lir_logic_or : __ orptr (reg, rright); break; 2499 case lir_logic_xor: __ xorptr (reg, rright); break; 2500 default: ShouldNotReachHere(); 2501 } 2502 } 2503 move_regs(reg, dst->as_register()); 2504 } else { 2505 Register l_lo = left->as_register_lo(); 2506 Register l_hi = left->as_register_hi(); 2507 if (right->is_constant()) { 2508 #ifdef _LP64 2509 __ mov64(rscratch1, right->as_constant_ptr()->as_jlong()); 2510 switch (code) { 2511 case lir_logic_and: 2512 __ andq(l_lo, rscratch1); 2513 break; 2514 case lir_logic_or: 2515 __ orq(l_lo, rscratch1); 2516 break; 2517 case lir_logic_xor: 2518 __ xorq(l_lo, rscratch1); 2519 break; 2520 default: ShouldNotReachHere(); 2521 } 2522 #else 2523 int r_lo = right->as_constant_ptr()->as_jint_lo(); 2524 int r_hi = right->as_constant_ptr()->as_jint_hi(); 2525 switch (code) { 2526 case lir_logic_and: 2527 __ andl(l_lo, r_lo); 2528 __ andl(l_hi, r_hi); 2529 break; 2530 case lir_logic_or: 2531 __ orl(l_lo, r_lo); 2532 __ orl(l_hi, r_hi); 2533 break; 2534 case lir_logic_xor: 2535 __ xorl(l_lo, r_lo); 2536 __ xorl(l_hi, r_hi); 2537 break; 2538 default: ShouldNotReachHere(); 2539 } 2540 #endif // _LP64 2541 } else { 2542 #ifdef _LP64 2543 Register r_lo; 2544 if (is_reference_type(right->type())) { 2545 r_lo = right->as_register(); 2546 } else { 2547 r_lo = right->as_register_lo(); 2548 } 2549 #else 2550 Register r_lo = right->as_register_lo(); 2551 Register r_hi = right->as_register_hi(); 2552 assert(l_lo != r_hi, "overwriting registers"); 2553 #endif 2554 switch (code) { 2555 case lir_logic_and: 2556 __ andptr(l_lo, r_lo); 2557 NOT_LP64(__ andptr(l_hi, r_hi);) 2558 break; 2559 case lir_logic_or: 2560 __ orptr(l_lo, r_lo); 2561 NOT_LP64(__ orptr(l_hi, r_hi);) 2562 break; 2563 case lir_logic_xor: 2564 __ xorptr(l_lo, r_lo); 2565 NOT_LP64(__ xorptr(l_hi, r_hi);) 2566 break; 2567 default: ShouldNotReachHere(); 2568 } 2569 } 2570 2571 Register dst_lo = dst->as_register_lo(); 2572 Register dst_hi = dst->as_register_hi(); 2573 2574 #ifdef _LP64 2575 move_regs(l_lo, dst_lo); 2576 #else 2577 if (dst_lo == l_hi) { 2578 assert(dst_hi != l_lo, "overwriting registers"); 2579 move_regs(l_hi, dst_hi); 2580 move_regs(l_lo, dst_lo); 2581 } else { 2582 assert(dst_lo != l_hi, "overwriting registers"); 2583 move_regs(l_lo, dst_lo); 2584 move_regs(l_hi, dst_hi); 2585 } 2586 #endif // _LP64 2587 } 2588 } 2589 2590 2591 // we assume that rax, and rdx can be overwritten 2592 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { 2593 2594 assert(left->is_single_cpu(), "left must be register"); 2595 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant"); 2596 assert(result->is_single_cpu(), "result must be register"); 2597 2598 // assert(left->destroys_register(), "check"); 2599 // assert(right->destroys_register(), "check"); 2600 2601 Register lreg = left->as_register(); 2602 Register dreg = result->as_register(); 2603 2604 if (right->is_constant()) { 2605 jint divisor = right->as_constant_ptr()->as_jint(); 2606 assert(divisor > 0 && is_power_of_2(divisor), "must be"); 2607 if (code == lir_idiv) { 2608 assert(lreg == rax, "must be rax,"); 2609 assert(temp->as_register() == rdx, "tmp register must be rdx"); 2610 __ cdql(); // sign extend into rdx:rax 2611 if (divisor == 2) { 2612 __ subl(lreg, rdx); 2613 } else { 2614 __ andl(rdx, divisor - 1); 2615 __ addl(lreg, rdx); 2616 } 2617 __ sarl(lreg, log2i_exact(divisor)); 2618 move_regs(lreg, dreg); 2619 } else if (code == lir_irem) { 2620 Label done; 2621 __ mov(dreg, lreg); 2622 __ andl(dreg, 0x80000000 | (divisor - 1)); 2623 __ jcc(Assembler::positive, done); 2624 __ decrement(dreg); 2625 __ orl(dreg, ~(divisor - 1)); 2626 __ increment(dreg); 2627 __ bind(done); 2628 } else { 2629 ShouldNotReachHere(); 2630 } 2631 } else { 2632 Register rreg = right->as_register(); 2633 assert(lreg == rax, "left register must be rax,"); 2634 assert(rreg != rdx, "right register must not be rdx"); 2635 assert(temp->as_register() == rdx, "tmp register must be rdx"); 2636 2637 move_regs(lreg, rax); 2638 2639 int idivl_offset = __ corrected_idivl(rreg); 2640 if (ImplicitDiv0Checks) { 2641 add_debug_info_for_div0(idivl_offset, info); 2642 } 2643 if (code == lir_irem) { 2644 move_regs(rdx, dreg); // result is in rdx 2645 } else { 2646 move_regs(rax, dreg); 2647 } 2648 } 2649 } 2650 2651 2652 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) { 2653 if (opr1->is_single_cpu()) { 2654 Register reg1 = opr1->as_register(); 2655 if (opr2->is_single_cpu()) { 2656 // cpu register - cpu register 2657 if (is_reference_type(opr1->type())) { 2658 __ cmpoop(reg1, opr2->as_register()); 2659 } else { 2660 assert(!is_reference_type(opr2->type()), "cmp int, oop?"); 2661 __ cmpl(reg1, opr2->as_register()); 2662 } 2663 } else if (opr2->is_stack()) { 2664 // cpu register - stack 2665 if (is_reference_type(opr1->type())) { 2666 __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix())); 2667 } else { 2668 __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix())); 2669 } 2670 } else if (opr2->is_constant()) { 2671 // cpu register - constant 2672 LIR_Const* c = opr2->as_constant_ptr(); 2673 if (c->type() == T_INT) { 2674 __ cmpl(reg1, c->as_jint()); 2675 } else if (c->type() == T_METADATA) { 2676 // All we need for now is a comparison with NULL for equality. 2677 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops"); 2678 Metadata* m = c->as_metadata(); 2679 if (m == NULL) { 2680 __ cmpptr(reg1, (int32_t)0); 2681 } else { 2682 ShouldNotReachHere(); 2683 } 2684 } else if (is_reference_type(c->type())) { 2685 // In 64bit oops are single register 2686 jobject o = c->as_jobject(); 2687 if (o == NULL) { 2688 __ cmpptr(reg1, (int32_t)NULL_WORD); 2689 } else { 2690 __ cmpoop(reg1, o); 2691 } 2692 } else { 2693 fatal("unexpected type: %s", basictype_to_str(c->type())); 2694 } 2695 // cpu register - address 2696 } else if (opr2->is_address()) { 2697 if (op->info() != NULL) { 2698 add_debug_info_for_null_check_here(op->info()); 2699 } 2700 __ cmpl(reg1, as_Address(opr2->as_address_ptr())); 2701 } else { 2702 ShouldNotReachHere(); 2703 } 2704 2705 } else if(opr1->is_double_cpu()) { 2706 Register xlo = opr1->as_register_lo(); 2707 Register xhi = opr1->as_register_hi(); 2708 if (opr2->is_double_cpu()) { 2709 #ifdef _LP64 2710 __ cmpptr(xlo, opr2->as_register_lo()); 2711 #else 2712 // cpu register - cpu register 2713 Register ylo = opr2->as_register_lo(); 2714 Register yhi = opr2->as_register_hi(); 2715 __ subl(xlo, ylo); 2716 __ sbbl(xhi, yhi); 2717 if (condition == lir_cond_equal || condition == lir_cond_notEqual) { 2718 __ orl(xhi, xlo); 2719 } 2720 #endif // _LP64 2721 } else if (opr2->is_constant()) { 2722 // cpu register - constant 0 2723 assert(opr2->as_jlong() == (jlong)0, "only handles zero"); 2724 #ifdef _LP64 2725 __ cmpptr(xlo, (int32_t)opr2->as_jlong()); 2726 #else 2727 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case"); 2728 __ orl(xhi, xlo); 2729 #endif // _LP64 2730 } else { 2731 ShouldNotReachHere(); 2732 } 2733 2734 } else if (opr1->is_single_xmm()) { 2735 XMMRegister reg1 = opr1->as_xmm_float_reg(); 2736 if (opr2->is_single_xmm()) { 2737 // xmm register - xmm register 2738 __ ucomiss(reg1, opr2->as_xmm_float_reg()); 2739 } else if (opr2->is_stack()) { 2740 // xmm register - stack 2741 __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix())); 2742 } else if (opr2->is_constant()) { 2743 // xmm register - constant 2744 __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat()))); 2745 } else if (opr2->is_address()) { 2746 // xmm register - address 2747 if (op->info() != NULL) { 2748 add_debug_info_for_null_check_here(op->info()); 2749 } 2750 __ ucomiss(reg1, as_Address(opr2->as_address_ptr())); 2751 } else { 2752 ShouldNotReachHere(); 2753 } 2754 2755 } else if (opr1->is_double_xmm()) { 2756 XMMRegister reg1 = opr1->as_xmm_double_reg(); 2757 if (opr2->is_double_xmm()) { 2758 // xmm register - xmm register 2759 __ ucomisd(reg1, opr2->as_xmm_double_reg()); 2760 } else if (opr2->is_stack()) { 2761 // xmm register - stack 2762 __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix())); 2763 } else if (opr2->is_constant()) { 2764 // xmm register - constant 2765 __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble()))); 2766 } else if (opr2->is_address()) { 2767 // xmm register - address 2768 if (op->info() != NULL) { 2769 add_debug_info_for_null_check_here(op->info()); 2770 } 2771 __ ucomisd(reg1, as_Address(opr2->pointer()->as_address())); 2772 } else { 2773 ShouldNotReachHere(); 2774 } 2775 2776 #ifndef _LP64 2777 } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) { 2778 assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)"); 2779 assert(opr2->is_fpu_register(), "both must be registers"); 2780 __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1); 2781 #endif // LP64 2782 2783 } else if (opr1->is_address() && opr2->is_constant()) { 2784 LIR_Const* c = opr2->as_constant_ptr(); 2785 #ifdef _LP64 2786 if (is_reference_type(c->type())) { 2787 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse"); 2788 __ movoop(rscratch1, c->as_jobject()); 2789 } 2790 #endif // LP64 2791 if (op->info() != NULL) { 2792 add_debug_info_for_null_check_here(op->info()); 2793 } 2794 // special case: address - constant 2795 LIR_Address* addr = opr1->as_address_ptr(); 2796 if (c->type() == T_INT) { 2797 __ cmpl(as_Address(addr), c->as_jint()); 2798 } else if (is_reference_type(c->type())) { 2799 #ifdef _LP64 2800 // %%% Make this explode if addr isn't reachable until we figure out a 2801 // better strategy by giving noreg as the temp for as_Address 2802 __ cmpoop(rscratch1, as_Address(addr, noreg)); 2803 #else 2804 __ cmpoop(as_Address(addr), c->as_jobject()); 2805 #endif // _LP64 2806 } else { 2807 ShouldNotReachHere(); 2808 } 2809 2810 } else { 2811 ShouldNotReachHere(); 2812 } 2813 } 2814 2815 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) { 2816 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 2817 if (left->is_single_xmm()) { 2818 assert(right->is_single_xmm(), "must match"); 2819 __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i); 2820 } else if (left->is_double_xmm()) { 2821 assert(right->is_double_xmm(), "must match"); 2822 __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i); 2823 2824 } else { 2825 #ifdef _LP64 2826 ShouldNotReachHere(); 2827 #else 2828 assert(left->is_single_fpu() || left->is_double_fpu(), "must be"); 2829 assert(right->is_single_fpu() || right->is_double_fpu(), "must match"); 2830 2831 assert(left->fpu() == 0, "left must be on TOS"); 2832 __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(), 2833 op->fpu_pop_count() > 0, op->fpu_pop_count() > 1); 2834 #endif // LP64 2835 } 2836 } else { 2837 assert(code == lir_cmp_l2i, "check"); 2838 #ifdef _LP64 2839 Label done; 2840 Register dest = dst->as_register(); 2841 __ cmpptr(left->as_register_lo(), right->as_register_lo()); 2842 __ movl(dest, -1); 2843 __ jccb(Assembler::less, done); 2844 __ set_byte_if_not_zero(dest); 2845 __ movzbl(dest, dest); 2846 __ bind(done); 2847 #else 2848 __ lcmp2int(left->as_register_hi(), 2849 left->as_register_lo(), 2850 right->as_register_hi(), 2851 right->as_register_lo()); 2852 move_regs(left->as_register_hi(), dst->as_register()); 2853 #endif // _LP64 2854 } 2855 } 2856 2857 2858 void LIR_Assembler::align_call(LIR_Code code) { 2859 // make sure that the displacement word of the call ends up word aligned 2860 int offset = __ offset(); 2861 switch (code) { 2862 case lir_static_call: 2863 case lir_optvirtual_call: 2864 case lir_dynamic_call: 2865 offset += NativeCall::displacement_offset; 2866 break; 2867 case lir_icvirtual_call: 2868 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size; 2869 break; 2870 default: ShouldNotReachHere(); 2871 } 2872 __ align(BytesPerWord, offset); 2873 } 2874 2875 2876 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 2877 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0, 2878 "must be aligned"); 2879 __ call(AddressLiteral(op->addr(), rtype)); 2880 add_call_info(code_offset(), op->info()); 2881 } 2882 2883 2884 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 2885 __ ic_call(op->addr()); 2886 add_call_info(code_offset(), op->info()); 2887 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0, 2888 "must be aligned"); 2889 } 2890 2891 2892 void LIR_Assembler::emit_static_call_stub() { 2893 address call_pc = __ pc(); 2894 address stub = __ start_a_stub(call_stub_size()); 2895 if (stub == NULL) { 2896 bailout("static call stub overflow"); 2897 return; 2898 } 2899 2900 int start = __ offset(); 2901 2902 // make sure that the displacement word of the call ends up word aligned 2903 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset); 2904 __ relocate(static_stub_Relocation::spec(call_pc)); 2905 __ mov_metadata(rbx, (Metadata*)NULL); 2906 // must be set to -1 at code generation time 2907 assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned"); 2908 // On 64bit this will die since it will take a movq & jmp, must be only a jmp 2909 __ jump(RuntimeAddress(__ pc())); 2910 2911 assert(__ offset() - start <= call_stub_size(), "stub too big"); 2912 __ end_a_stub(); 2913 } 2914 2915 2916 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 2917 assert(exceptionOop->as_register() == rax, "must match"); 2918 assert(exceptionPC->as_register() == rdx, "must match"); 2919 2920 // exception object is not added to oop map by LinearScan 2921 // (LinearScan assumes that no oops are in fixed registers) 2922 info->add_register_oop(exceptionOop); 2923 Runtime1::StubID unwind_id; 2924 2925 // get current pc information 2926 // pc is only needed if the method has an exception handler, the unwind code does not need it. 2927 int pc_for_athrow_offset = __ offset(); 2928 InternalAddress pc_for_athrow(__ pc()); 2929 __ lea(exceptionPC->as_register(), pc_for_athrow); 2930 add_call_info(pc_for_athrow_offset, info); // for exception handler 2931 2932 __ verify_not_null_oop(rax); 2933 // search an exception handler (rax: exception oop, rdx: throwing pc) 2934 if (compilation()->has_fpu_code()) { 2935 unwind_id = Runtime1::handle_exception_id; 2936 } else { 2937 unwind_id = Runtime1::handle_exception_nofpu_id; 2938 } 2939 __ call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 2940 2941 // enough room for two byte trap 2942 __ nop(); 2943 } 2944 2945 2946 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 2947 assert(exceptionOop->as_register() == rax, "must match"); 2948 2949 __ jmp(_unwind_handler_entry); 2950 } 2951 2952 2953 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 2954 2955 // optimized version for linear scan: 2956 // * count must be already in ECX (guaranteed by LinearScan) 2957 // * left and dest must be equal 2958 // * tmp must be unused 2959 assert(count->as_register() == SHIFT_count, "count must be in ECX"); 2960 assert(left == dest, "left and dest must be equal"); 2961 assert(tmp->is_illegal(), "wasting a register if tmp is allocated"); 2962 2963 if (left->is_single_cpu()) { 2964 Register value = left->as_register(); 2965 assert(value != SHIFT_count, "left cannot be ECX"); 2966 2967 switch (code) { 2968 case lir_shl: __ shll(value); break; 2969 case lir_shr: __ sarl(value); break; 2970 case lir_ushr: __ shrl(value); break; 2971 default: ShouldNotReachHere(); 2972 } 2973 } else if (left->is_double_cpu()) { 2974 Register lo = left->as_register_lo(); 2975 Register hi = left->as_register_hi(); 2976 assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX"); 2977 #ifdef _LP64 2978 switch (code) { 2979 case lir_shl: __ shlptr(lo); break; 2980 case lir_shr: __ sarptr(lo); break; 2981 case lir_ushr: __ shrptr(lo); break; 2982 default: ShouldNotReachHere(); 2983 } 2984 #else 2985 2986 switch (code) { 2987 case lir_shl: __ lshl(hi, lo); break; 2988 case lir_shr: __ lshr(hi, lo, true); break; 2989 case lir_ushr: __ lshr(hi, lo, false); break; 2990 default: ShouldNotReachHere(); 2991 } 2992 #endif // LP64 2993 } else { 2994 ShouldNotReachHere(); 2995 } 2996 } 2997 2998 2999 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 3000 if (dest->is_single_cpu()) { 3001 // first move left into dest so that left is not destroyed by the shift 3002 Register value = dest->as_register(); 3003 count = count & 0x1F; // Java spec 3004 3005 move_regs(left->as_register(), value); 3006 switch (code) { 3007 case lir_shl: __ shll(value, count); break; 3008 case lir_shr: __ sarl(value, count); break; 3009 case lir_ushr: __ shrl(value, count); break; 3010 default: ShouldNotReachHere(); 3011 } 3012 } else if (dest->is_double_cpu()) { 3013 #ifndef _LP64 3014 Unimplemented(); 3015 #else 3016 // first move left into dest so that left is not destroyed by the shift 3017 Register value = dest->as_register_lo(); 3018 count = count & 0x1F; // Java spec 3019 3020 move_regs(left->as_register_lo(), value); 3021 switch (code) { 3022 case lir_shl: __ shlptr(value, count); break; 3023 case lir_shr: __ sarptr(value, count); break; 3024 case lir_ushr: __ shrptr(value, count); break; 3025 default: ShouldNotReachHere(); 3026 } 3027 #endif // _LP64 3028 } else { 3029 ShouldNotReachHere(); 3030 } 3031 } 3032 3033 3034 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 3035 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 3036 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 3037 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 3038 __ movptr (Address(rsp, offset_from_rsp_in_bytes), r); 3039 } 3040 3041 3042 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 3043 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 3044 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 3045 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 3046 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c); 3047 } 3048 3049 3050 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) { 3051 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 3052 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 3053 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 3054 __ movoop (Address(rsp, offset_from_rsp_in_bytes), o); 3055 } 3056 3057 3058 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) { 3059 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 3060 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 3061 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 3062 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m); 3063 } 3064 3065 3066 // This code replaces a call to arraycopy; no exception may 3067 // be thrown in this code, they must be thrown in the System.arraycopy 3068 // activation frame; we could save some checks if this would not be the case 3069 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) { 3070 ciArrayKlass* default_type = op->expected_type(); 3071 Register src = op->src()->as_register(); 3072 Register dst = op->dst()->as_register(); 3073 Register src_pos = op->src_pos()->as_register(); 3074 Register dst_pos = op->dst_pos()->as_register(); 3075 Register length = op->length()->as_register(); 3076 Register tmp = op->tmp()->as_register(); 3077 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 3078 3079 CodeStub* stub = op->stub(); 3080 int flags = op->flags(); 3081 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL; 3082 if (is_reference_type(basic_type)) basic_type = T_OBJECT; 3083 3084 // if we don't know anything, just go through the generic arraycopy 3085 if (default_type == NULL) { 3086 // save outgoing arguments on stack in case call to System.arraycopy is needed 3087 // HACK ALERT. This code used to push the parameters in a hardwired fashion 3088 // for interpreter calling conventions. Now we have to do it in new style conventions. 3089 // For the moment until C1 gets the new register allocator I just force all the 3090 // args to the right place (except the register args) and then on the back side 3091 // reload the register args properly if we go slow path. Yuck 3092 3093 // These are proper for the calling convention 3094 store_parameter(length, 2); 3095 store_parameter(dst_pos, 1); 3096 store_parameter(dst, 0); 3097 3098 // these are just temporary placements until we need to reload 3099 store_parameter(src_pos, 3); 3100 store_parameter(src, 4); 3101 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");) 3102 3103 address copyfunc_addr = StubRoutines::generic_arraycopy(); 3104 assert(copyfunc_addr != NULL, "generic arraycopy stub required"); 3105 3106 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint 3107 #ifdef _LP64 3108 // The arguments are in java calling convention so we can trivially shift them to C 3109 // convention 3110 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4); 3111 __ mov(c_rarg0, j_rarg0); 3112 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4); 3113 __ mov(c_rarg1, j_rarg1); 3114 assert_different_registers(c_rarg2, j_rarg3, j_rarg4); 3115 __ mov(c_rarg2, j_rarg2); 3116 assert_different_registers(c_rarg3, j_rarg4); 3117 __ mov(c_rarg3, j_rarg3); 3118 #ifdef _WIN64 3119 // Allocate abi space for args but be sure to keep stack aligned 3120 __ subptr(rsp, 6*wordSize); 3121 store_parameter(j_rarg4, 4); 3122 #ifndef PRODUCT 3123 if (PrintC1Statistics) { 3124 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); 3125 } 3126 #endif 3127 __ call(RuntimeAddress(copyfunc_addr)); 3128 __ addptr(rsp, 6*wordSize); 3129 #else 3130 __ mov(c_rarg4, j_rarg4); 3131 #ifndef PRODUCT 3132 if (PrintC1Statistics) { 3133 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); 3134 } 3135 #endif 3136 __ call(RuntimeAddress(copyfunc_addr)); 3137 #endif // _WIN64 3138 #else 3139 __ push(length); 3140 __ push(dst_pos); 3141 __ push(dst); 3142 __ push(src_pos); 3143 __ push(src); 3144 3145 #ifndef PRODUCT 3146 if (PrintC1Statistics) { 3147 __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt)); 3148 } 3149 #endif 3150 __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack 3151 3152 #endif // _LP64 3153 3154 __ cmpl(rax, 0); 3155 __ jcc(Assembler::equal, *stub->continuation()); 3156 3157 __ mov(tmp, rax); 3158 __ xorl(tmp, -1); 3159 3160 // Reload values from the stack so they are where the stub 3161 // expects them. 3162 __ movptr (dst, Address(rsp, 0*BytesPerWord)); 3163 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord)); 3164 __ movptr (length, Address(rsp, 2*BytesPerWord)); 3165 __ movptr (src_pos, Address(rsp, 3*BytesPerWord)); 3166 __ movptr (src, Address(rsp, 4*BytesPerWord)); 3167 3168 __ subl(length, tmp); 3169 __ addl(src_pos, tmp); 3170 __ addl(dst_pos, tmp); 3171 __ jmp(*stub->entry()); 3172 3173 __ bind(*stub->continuation()); 3174 return; 3175 } 3176 3177 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point"); 3178 3179 int elem_size = type2aelembytes(basic_type); 3180 Address::ScaleFactor scale; 3181 3182 switch (elem_size) { 3183 case 1 : 3184 scale = Address::times_1; 3185 break; 3186 case 2 : 3187 scale = Address::times_2; 3188 break; 3189 case 4 : 3190 scale = Address::times_4; 3191 break; 3192 case 8 : 3193 scale = Address::times_8; 3194 break; 3195 default: 3196 scale = Address::no_scale; 3197 ShouldNotReachHere(); 3198 } 3199 3200 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes()); 3201 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes()); 3202 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes()); 3203 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes()); 3204 3205 // length and pos's are all sign extended at this point on 64bit 3206 3207 // test for NULL 3208 if (flags & LIR_OpArrayCopy::src_null_check) { 3209 __ testptr(src, src); 3210 __ jcc(Assembler::zero, *stub->entry()); 3211 } 3212 if (flags & LIR_OpArrayCopy::dst_null_check) { 3213 __ testptr(dst, dst); 3214 __ jcc(Assembler::zero, *stub->entry()); 3215 } 3216 3217 // If the compiler was not able to prove that exact type of the source or the destination 3218 // of the arraycopy is an array type, check at runtime if the source or the destination is 3219 // an instance type. 3220 if (flags & LIR_OpArrayCopy::type_check) { 3221 if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 3222 __ load_klass(tmp, dst, tmp_load_klass); 3223 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value); 3224 __ jcc(Assembler::greaterEqual, *stub->entry()); 3225 } 3226 3227 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 3228 __ load_klass(tmp, src, tmp_load_klass); 3229 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value); 3230 __ jcc(Assembler::greaterEqual, *stub->entry()); 3231 } 3232 } 3233 3234 // check if negative 3235 if (flags & LIR_OpArrayCopy::src_pos_positive_check) { 3236 __ testl(src_pos, src_pos); 3237 __ jcc(Assembler::less, *stub->entry()); 3238 } 3239 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) { 3240 __ testl(dst_pos, dst_pos); 3241 __ jcc(Assembler::less, *stub->entry()); 3242 } 3243 3244 if (flags & LIR_OpArrayCopy::src_range_check) { 3245 __ lea(tmp, Address(src_pos, length, Address::times_1, 0)); 3246 __ cmpl(tmp, src_length_addr); 3247 __ jcc(Assembler::above, *stub->entry()); 3248 } 3249 if (flags & LIR_OpArrayCopy::dst_range_check) { 3250 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0)); 3251 __ cmpl(tmp, dst_length_addr); 3252 __ jcc(Assembler::above, *stub->entry()); 3253 } 3254 3255 if (flags & LIR_OpArrayCopy::length_positive_check) { 3256 __ testl(length, length); 3257 __ jcc(Assembler::less, *stub->entry()); 3258 } 3259 3260 #ifdef _LP64 3261 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null 3262 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null 3263 #endif 3264 3265 if (flags & LIR_OpArrayCopy::type_check) { 3266 // We don't know the array types are compatible 3267 if (basic_type != T_OBJECT) { 3268 // Simple test for basic type arrays 3269 if (UseCompressedClassPointers) { 3270 __ movl(tmp, src_klass_addr); 3271 __ cmpl(tmp, dst_klass_addr); 3272 } else { 3273 __ movptr(tmp, src_klass_addr); 3274 __ cmpptr(tmp, dst_klass_addr); 3275 } 3276 __ jcc(Assembler::notEqual, *stub->entry()); 3277 } else { 3278 // For object arrays, if src is a sub class of dst then we can 3279 // safely do the copy. 3280 Label cont, slow; 3281 3282 __ push(src); 3283 __ push(dst); 3284 3285 __ load_klass(src, src, tmp_load_klass); 3286 __ load_klass(dst, dst, tmp_load_klass); 3287 3288 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL); 3289 3290 __ push(src); 3291 __ push(dst); 3292 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 3293 __ pop(dst); 3294 __ pop(src); 3295 3296 __ cmpl(src, 0); 3297 __ jcc(Assembler::notEqual, cont); 3298 3299 __ bind(slow); 3300 __ pop(dst); 3301 __ pop(src); 3302 3303 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); 3304 if (copyfunc_addr != NULL) { // use stub if available 3305 // src is not a sub class of dst so we have to do a 3306 // per-element check. 3307 3308 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray; 3309 if ((flags & mask) != mask) { 3310 // Check that at least both of them object arrays. 3311 assert(flags & mask, "one of the two should be known to be an object array"); 3312 3313 if (!(flags & LIR_OpArrayCopy::src_objarray)) { 3314 __ load_klass(tmp, src, tmp_load_klass); 3315 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) { 3316 __ load_klass(tmp, dst, tmp_load_klass); 3317 } 3318 int lh_offset = in_bytes(Klass::layout_helper_offset()); 3319 Address klass_lh_addr(tmp, lh_offset); 3320 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 3321 __ cmpl(klass_lh_addr, objArray_lh); 3322 __ jcc(Assembler::notEqual, *stub->entry()); 3323 } 3324 3325 // Spill because stubs can use any register they like and it's 3326 // easier to restore just those that we care about. 3327 store_parameter(dst, 0); 3328 store_parameter(dst_pos, 1); 3329 store_parameter(length, 2); 3330 store_parameter(src_pos, 3); 3331 store_parameter(src, 4); 3332 3333 #ifndef _LP64 3334 __ movptr(tmp, dst_klass_addr); 3335 __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset())); 3336 __ push(tmp); 3337 __ movl(tmp, Address(tmp, Klass::super_check_offset_offset())); 3338 __ push(tmp); 3339 __ push(length); 3340 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3341 __ push(tmp); 3342 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3343 __ push(tmp); 3344 3345 __ call_VM_leaf(copyfunc_addr, 5); 3346 #else 3347 __ movl2ptr(length, length); //higher 32bits must be null 3348 3349 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3350 assert_different_registers(c_rarg0, dst, dst_pos, length); 3351 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3352 assert_different_registers(c_rarg1, dst, length); 3353 3354 __ mov(c_rarg2, length); 3355 assert_different_registers(c_rarg2, dst); 3356 3357 #ifdef _WIN64 3358 // Allocate abi space for args but be sure to keep stack aligned 3359 __ subptr(rsp, 6*wordSize); 3360 __ load_klass(c_rarg3, dst, tmp_load_klass); 3361 __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset())); 3362 store_parameter(c_rarg3, 4); 3363 __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset())); 3364 __ call(RuntimeAddress(copyfunc_addr)); 3365 __ addptr(rsp, 6*wordSize); 3366 #else 3367 __ load_klass(c_rarg4, dst, tmp_load_klass); 3368 __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset())); 3369 __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset())); 3370 __ call(RuntimeAddress(copyfunc_addr)); 3371 #endif 3372 3373 #endif 3374 3375 #ifndef PRODUCT 3376 if (PrintC1Statistics) { 3377 Label failed; 3378 __ testl(rax, rax); 3379 __ jcc(Assembler::notZero, failed); 3380 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt)); 3381 __ bind(failed); 3382 } 3383 #endif 3384 3385 __ testl(rax, rax); 3386 __ jcc(Assembler::zero, *stub->continuation()); 3387 3388 #ifndef PRODUCT 3389 if (PrintC1Statistics) { 3390 __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt)); 3391 } 3392 #endif 3393 3394 __ mov(tmp, rax); 3395 3396 __ xorl(tmp, -1); 3397 3398 // Restore previously spilled arguments 3399 __ movptr (dst, Address(rsp, 0*BytesPerWord)); 3400 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord)); 3401 __ movptr (length, Address(rsp, 2*BytesPerWord)); 3402 __ movptr (src_pos, Address(rsp, 3*BytesPerWord)); 3403 __ movptr (src, Address(rsp, 4*BytesPerWord)); 3404 3405 3406 __ subl(length, tmp); 3407 __ addl(src_pos, tmp); 3408 __ addl(dst_pos, tmp); 3409 } 3410 3411 __ jmp(*stub->entry()); 3412 3413 __ bind(cont); 3414 __ pop(dst); 3415 __ pop(src); 3416 } 3417 } 3418 3419 #ifdef ASSERT 3420 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) { 3421 // Sanity check the known type with the incoming class. For the 3422 // primitive case the types must match exactly with src.klass and 3423 // dst.klass each exactly matching the default type. For the 3424 // object array case, if no type check is needed then either the 3425 // dst type is exactly the expected type and the src type is a 3426 // subtype which we can't check or src is the same array as dst 3427 // but not necessarily exactly of type default_type. 3428 Label known_ok, halt; 3429 __ mov_metadata(tmp, default_type->constant_encoding()); 3430 #ifdef _LP64 3431 if (UseCompressedClassPointers) { 3432 __ encode_klass_not_null(tmp, rscratch1); 3433 } 3434 #endif 3435 3436 if (basic_type != T_OBJECT) { 3437 3438 if (UseCompressedClassPointers) __ cmpl(tmp, dst_klass_addr); 3439 else __ cmpptr(tmp, dst_klass_addr); 3440 __ jcc(Assembler::notEqual, halt); 3441 if (UseCompressedClassPointers) __ cmpl(tmp, src_klass_addr); 3442 else __ cmpptr(tmp, src_klass_addr); 3443 __ jcc(Assembler::equal, known_ok); 3444 } else { 3445 if (UseCompressedClassPointers) __ cmpl(tmp, dst_klass_addr); 3446 else __ cmpptr(tmp, dst_klass_addr); 3447 __ jcc(Assembler::equal, known_ok); 3448 __ cmpptr(src, dst); 3449 __ jcc(Assembler::equal, known_ok); 3450 } 3451 __ bind(halt); 3452 __ stop("incorrect type information in arraycopy"); 3453 __ bind(known_ok); 3454 } 3455 #endif 3456 3457 #ifndef PRODUCT 3458 if (PrintC1Statistics) { 3459 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type))); 3460 } 3461 #endif 3462 3463 #ifdef _LP64 3464 assert_different_registers(c_rarg0, dst, dst_pos, length); 3465 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3466 assert_different_registers(c_rarg1, length); 3467 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3468 __ mov(c_rarg2, length); 3469 3470 #else 3471 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3472 store_parameter(tmp, 0); 3473 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type))); 3474 store_parameter(tmp, 1); 3475 store_parameter(length, 2); 3476 #endif // _LP64 3477 3478 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0; 3479 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0; 3480 const char *name; 3481 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false); 3482 __ call_VM_leaf(entry, 0); 3483 3484 __ bind(*stub->continuation()); 3485 } 3486 3487 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 3488 assert(op->crc()->is_single_cpu(), "crc must be register"); 3489 assert(op->val()->is_single_cpu(), "byte value must be register"); 3490 assert(op->result_opr()->is_single_cpu(), "result must be register"); 3491 Register crc = op->crc()->as_register(); 3492 Register val = op->val()->as_register(); 3493 Register res = op->result_opr()->as_register(); 3494 3495 assert_different_registers(val, crc, res); 3496 3497 __ lea(res, ExternalAddress(StubRoutines::crc_table_addr())); 3498 __ notl(crc); // ~crc 3499 __ update_byte_crc32(crc, val, res); 3500 __ notl(crc); // ~crc 3501 __ mov(res, crc); 3502 } 3503 3504 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 3505 Register obj = op->obj_opr()->as_register(); // may not be an oop 3506 Register hdr = op->hdr_opr()->as_register(); 3507 Register lock = op->lock_opr()->as_register(); 3508 if (UseHeavyMonitors) { 3509 __ jmp(*op->stub()->entry()); 3510 } else if (op->code() == lir_lock) { 3511 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 3512 // add debug info for NullPointerException only if one is possible 3513 int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry()); 3514 if (op->info() != NULL) { 3515 add_debug_info_for_null_check(null_check_offset, op->info()); 3516 } 3517 // done 3518 } else if (op->code() == lir_unlock) { 3519 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 3520 __ unlock_object(hdr, obj, lock, *op->stub()->entry()); 3521 } else { 3522 Unimplemented(); 3523 } 3524 __ bind(*op->stub()->continuation()); 3525 } 3526 3527 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { 3528 Register obj = op->obj()->as_pointer_register(); 3529 Register result = op->result_opr()->as_pointer_register(); 3530 3531 CodeEmitInfo* info = op->info(); 3532 if (info != NULL) { 3533 add_debug_info_for_null_check_here(info); 3534 } 3535 3536 #ifdef _LP64 3537 if (UseCompressedClassPointers) { 3538 __ movl(result, Address(obj, oopDesc::klass_offset_in_bytes())); 3539 __ decode_klass_not_null(result, rscratch1); 3540 } else 3541 #endif 3542 __ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes())); 3543 } 3544 3545 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 3546 ciMethod* method = op->profiled_method(); 3547 int bci = op->profiled_bci(); 3548 ciMethod* callee = op->profiled_callee(); 3549 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 3550 3551 // Update counter for all call types 3552 ciMethodData* md = method->method_data_or_null(); 3553 assert(md != NULL, "Sanity"); 3554 ciProfileData* data = md->bci_to_data(bci); 3555 assert(data != NULL && data->is_CounterData(), "need CounterData for calls"); 3556 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 3557 Register mdo = op->mdo()->as_register(); 3558 __ mov_metadata(mdo, md->constant_encoding()); 3559 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 3560 // Perform additional virtual call profiling for invokevirtual and 3561 // invokeinterface bytecodes 3562 if (op->should_profile_receiver_type()) { 3563 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 3564 Register recv = op->recv()->as_register(); 3565 assert_different_registers(mdo, recv); 3566 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 3567 ciKlass* known_klass = op->known_holder(); 3568 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) { 3569 // We know the type that will be seen at this call site; we can 3570 // statically update the MethodData* rather than needing to do 3571 // dynamic tests on the receiver type 3572 3573 // NOTE: we should probably put a lock around this search to 3574 // avoid collisions by concurrent compilations 3575 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 3576 uint i; 3577 for (i = 0; i < VirtualCallData::row_limit(); i++) { 3578 ciKlass* receiver = vc_data->receiver(i); 3579 if (known_klass->equals(receiver)) { 3580 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 3581 __ addptr(data_addr, DataLayout::counter_increment); 3582 return; 3583 } 3584 } 3585 3586 // Receiver type not found in profile data; select an empty slot 3587 3588 // Note that this is less efficient than it should be because it 3589 // always does a write to the receiver part of the 3590 // VirtualCallData rather than just the first time 3591 for (i = 0; i < VirtualCallData::row_limit(); i++) { 3592 ciKlass* receiver = vc_data->receiver(i); 3593 if (receiver == NULL) { 3594 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); 3595 __ mov_metadata(recv_addr, known_klass->constant_encoding()); 3596 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 3597 __ addptr(data_addr, DataLayout::counter_increment); 3598 return; 3599 } 3600 } 3601 } else { 3602 __ load_klass(recv, recv, tmp_load_klass); 3603 Label update_done; 3604 type_profile_helper(mdo, md, data, recv, &update_done); 3605 // Receiver did not match any saved receiver and there is no empty row for it. 3606 // Increment total counter to indicate polymorphic case. 3607 __ addptr(counter_addr, DataLayout::counter_increment); 3608 3609 __ bind(update_done); 3610 } 3611 } else { 3612 // Static call 3613 __ addptr(counter_addr, DataLayout::counter_increment); 3614 } 3615 } 3616 3617 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 3618 Register obj = op->obj()->as_register(); 3619 Register tmp = op->tmp()->as_pointer_register(); 3620 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg); 3621 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 3622 ciKlass* exact_klass = op->exact_klass(); 3623 intptr_t current_klass = op->current_klass(); 3624 bool not_null = op->not_null(); 3625 bool no_conflict = op->no_conflict(); 3626 3627 Label update, next, none; 3628 3629 bool do_null = !not_null; 3630 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 3631 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 3632 3633 assert(do_null || do_update, "why are we here?"); 3634 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 3635 3636 __ verify_oop(obj); 3637 3638 if (tmp != obj) { 3639 __ mov(tmp, obj); 3640 } 3641 if (do_null) { 3642 __ testptr(tmp, tmp); 3643 __ jccb(Assembler::notZero, update); 3644 if (!TypeEntries::was_null_seen(current_klass)) { 3645 __ orptr(mdo_addr, TypeEntries::null_seen); 3646 } 3647 if (do_update) { 3648 #ifndef ASSERT 3649 __ jmpb(next); 3650 } 3651 #else 3652 __ jmp(next); 3653 } 3654 } else { 3655 __ testptr(tmp, tmp); 3656 __ jcc(Assembler::notZero, update); 3657 __ stop("unexpected null obj"); 3658 #endif 3659 } 3660 3661 __ bind(update); 3662 3663 if (do_update) { 3664 #ifdef ASSERT 3665 if (exact_klass != NULL) { 3666 Label ok; 3667 __ load_klass(tmp, tmp, tmp_load_klass); 3668 __ push(tmp); 3669 __ mov_metadata(tmp, exact_klass->constant_encoding()); 3670 __ cmpptr(tmp, Address(rsp, 0)); 3671 __ jcc(Assembler::equal, ok); 3672 __ stop("exact klass and actual klass differ"); 3673 __ bind(ok); 3674 __ pop(tmp); 3675 } 3676 #endif 3677 if (!no_conflict) { 3678 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) { 3679 if (exact_klass != NULL) { 3680 __ mov_metadata(tmp, exact_klass->constant_encoding()); 3681 } else { 3682 __ load_klass(tmp, tmp, tmp_load_klass); 3683 } 3684 3685 __ xorptr(tmp, mdo_addr); 3686 __ testptr(tmp, TypeEntries::type_klass_mask); 3687 // klass seen before, nothing to do. The unknown bit may have been 3688 // set already but no need to check. 3689 __ jccb(Assembler::zero, next); 3690 3691 __ testptr(tmp, TypeEntries::type_unknown); 3692 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore. 3693 3694 if (TypeEntries::is_type_none(current_klass)) { 3695 __ cmpptr(mdo_addr, 0); 3696 __ jccb(Assembler::equal, none); 3697 __ cmpptr(mdo_addr, TypeEntries::null_seen); 3698 __ jccb(Assembler::equal, none); 3699 // There is a chance that the checks above (re-reading profiling 3700 // data from memory) fail if another thread has just set the 3701 // profiling to this obj's klass 3702 __ xorptr(tmp, mdo_addr); 3703 __ testptr(tmp, TypeEntries::type_klass_mask); 3704 __ jccb(Assembler::zero, next); 3705 } 3706 } else { 3707 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 3708 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 3709 3710 __ movptr(tmp, mdo_addr); 3711 __ testptr(tmp, TypeEntries::type_unknown); 3712 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore. 3713 } 3714 3715 // different than before. Cannot keep accurate profile. 3716 __ orptr(mdo_addr, TypeEntries::type_unknown); 3717 3718 if (TypeEntries::is_type_none(current_klass)) { 3719 __ jmpb(next); 3720 3721 __ bind(none); 3722 // first time here. Set profile type. 3723 __ movptr(mdo_addr, tmp); 3724 } 3725 } else { 3726 // There's a single possible klass at this profile point 3727 assert(exact_klass != NULL, "should be"); 3728 if (TypeEntries::is_type_none(current_klass)) { 3729 __ mov_metadata(tmp, exact_klass->constant_encoding()); 3730 __ xorptr(tmp, mdo_addr); 3731 __ testptr(tmp, TypeEntries::type_klass_mask); 3732 #ifdef ASSERT 3733 __ jcc(Assembler::zero, next); 3734 3735 { 3736 Label ok; 3737 __ push(tmp); 3738 __ cmpptr(mdo_addr, 0); 3739 __ jcc(Assembler::equal, ok); 3740 __ cmpptr(mdo_addr, TypeEntries::null_seen); 3741 __ jcc(Assembler::equal, ok); 3742 // may have been set by another thread 3743 __ mov_metadata(tmp, exact_klass->constant_encoding()); 3744 __ xorptr(tmp, mdo_addr); 3745 __ testptr(tmp, TypeEntries::type_mask); 3746 __ jcc(Assembler::zero, ok); 3747 3748 __ stop("unexpected profiling mismatch"); 3749 __ bind(ok); 3750 __ pop(tmp); 3751 } 3752 #else 3753 __ jccb(Assembler::zero, next); 3754 #endif 3755 // first time here. Set profile type. 3756 __ movptr(mdo_addr, tmp); 3757 } else { 3758 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL && 3759 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 3760 3761 __ movptr(tmp, mdo_addr); 3762 __ testptr(tmp, TypeEntries::type_unknown); 3763 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore. 3764 3765 __ orptr(mdo_addr, TypeEntries::type_unknown); 3766 } 3767 } 3768 3769 __ bind(next); 3770 } 3771 } 3772 3773 void LIR_Assembler::emit_delay(LIR_OpDelay*) { 3774 Unimplemented(); 3775 } 3776 3777 3778 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 3779 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 3780 } 3781 3782 3783 void LIR_Assembler::align_backward_branch_target() { 3784 __ align(BytesPerWord); 3785 } 3786 3787 3788 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) { 3789 if (left->is_single_cpu()) { 3790 __ negl(left->as_register()); 3791 move_regs(left->as_register(), dest->as_register()); 3792 3793 } else if (left->is_double_cpu()) { 3794 Register lo = left->as_register_lo(); 3795 #ifdef _LP64 3796 Register dst = dest->as_register_lo(); 3797 __ movptr(dst, lo); 3798 __ negptr(dst); 3799 #else 3800 Register hi = left->as_register_hi(); 3801 __ lneg(hi, lo); 3802 if (dest->as_register_lo() == hi) { 3803 assert(dest->as_register_hi() != lo, "destroying register"); 3804 move_regs(hi, dest->as_register_hi()); 3805 move_regs(lo, dest->as_register_lo()); 3806 } else { 3807 move_regs(lo, dest->as_register_lo()); 3808 move_regs(hi, dest->as_register_hi()); 3809 } 3810 #endif // _LP64 3811 3812 } else if (dest->is_single_xmm()) { 3813 #ifdef _LP64 3814 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) { 3815 assert(tmp->is_valid(), "need temporary"); 3816 assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg()); 3817 __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2); 3818 } 3819 else 3820 #endif 3821 { 3822 assert(!tmp->is_valid(), "do not need temporary"); 3823 if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) { 3824 __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg()); 3825 } 3826 __ xorps(dest->as_xmm_float_reg(), 3827 ExternalAddress((address)float_signflip_pool)); 3828 } 3829 } else if (dest->is_double_xmm()) { 3830 #ifdef _LP64 3831 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) { 3832 assert(tmp->is_valid(), "need temporary"); 3833 assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg()); 3834 __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2); 3835 } 3836 else 3837 #endif 3838 { 3839 assert(!tmp->is_valid(), "do not need temporary"); 3840 if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) { 3841 __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg()); 3842 } 3843 __ xorpd(dest->as_xmm_double_reg(), 3844 ExternalAddress((address)double_signflip_pool)); 3845 } 3846 #ifndef _LP64 3847 } else if (left->is_single_fpu() || left->is_double_fpu()) { 3848 assert(left->fpu() == 0, "arg must be on TOS"); 3849 assert(dest->fpu() == 0, "dest must be TOS"); 3850 __ fchs(); 3851 #endif // !_LP64 3852 3853 } else { 3854 ShouldNotReachHere(); 3855 } 3856 } 3857 3858 3859 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 3860 assert(src->is_address(), "must be an address"); 3861 assert(dest->is_register(), "must be a register"); 3862 3863 PatchingStub* patch = NULL; 3864 if (patch_code != lir_patch_none) { 3865 patch = new PatchingStub(_masm, PatchingStub::access_field_id); 3866 } 3867 3868 Register reg = dest->as_pointer_register(); 3869 LIR_Address* addr = src->as_address_ptr(); 3870 __ lea(reg, as_Address(addr)); 3871 3872 if (patch != NULL) { 3873 patching_epilog(patch, patch_code, addr->base()->as_register(), info); 3874 } 3875 } 3876 3877 3878 3879 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 3880 assert(!tmp->is_valid(), "don't need temporary"); 3881 __ call(RuntimeAddress(dest)); 3882 if (info != NULL) { 3883 add_call_info_here(info); 3884 } 3885 } 3886 3887 3888 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 3889 assert(type == T_LONG, "only for volatile long fields"); 3890 3891 if (info != NULL) { 3892 add_debug_info_for_null_check_here(info); 3893 } 3894 3895 if (src->is_double_xmm()) { 3896 if (dest->is_double_cpu()) { 3897 #ifdef _LP64 3898 __ movdq(dest->as_register_lo(), src->as_xmm_double_reg()); 3899 #else 3900 __ movdl(dest->as_register_lo(), src->as_xmm_double_reg()); 3901 __ psrlq(src->as_xmm_double_reg(), 32); 3902 __ movdl(dest->as_register_hi(), src->as_xmm_double_reg()); 3903 #endif // _LP64 3904 } else if (dest->is_double_stack()) { 3905 __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg()); 3906 } else if (dest->is_address()) { 3907 __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg()); 3908 } else { 3909 ShouldNotReachHere(); 3910 } 3911 3912 } else if (dest->is_double_xmm()) { 3913 if (src->is_double_stack()) { 3914 __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix())); 3915 } else if (src->is_address()) { 3916 __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr())); 3917 } else { 3918 ShouldNotReachHere(); 3919 } 3920 3921 #ifndef _LP64 3922 } else if (src->is_double_fpu()) { 3923 assert(src->fpu_regnrLo() == 0, "must be TOS"); 3924 if (dest->is_double_stack()) { 3925 __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix())); 3926 } else if (dest->is_address()) { 3927 __ fistp_d(as_Address(dest->as_address_ptr())); 3928 } else { 3929 ShouldNotReachHere(); 3930 } 3931 3932 } else if (dest->is_double_fpu()) { 3933 assert(dest->fpu_regnrLo() == 0, "must be TOS"); 3934 if (src->is_double_stack()) { 3935 __ fild_d(frame_map()->address_for_slot(src->double_stack_ix())); 3936 } else if (src->is_address()) { 3937 __ fild_d(as_Address(src->as_address_ptr())); 3938 } else { 3939 ShouldNotReachHere(); 3940 } 3941 #endif // !_LP64 3942 3943 } else { 3944 ShouldNotReachHere(); 3945 } 3946 } 3947 3948 #ifdef ASSERT 3949 // emit run-time assertion 3950 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 3951 assert(op->code() == lir_assert, "must be"); 3952 3953 if (op->in_opr1()->is_valid()) { 3954 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 3955 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op); 3956 } else { 3957 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 3958 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 3959 } 3960 3961 Label ok; 3962 if (op->condition() != lir_cond_always) { 3963 Assembler::Condition acond = Assembler::zero; 3964 switch (op->condition()) { 3965 case lir_cond_equal: acond = Assembler::equal; break; 3966 case lir_cond_notEqual: acond = Assembler::notEqual; break; 3967 case lir_cond_less: acond = Assembler::less; break; 3968 case lir_cond_lessEqual: acond = Assembler::lessEqual; break; 3969 case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break; 3970 case lir_cond_greater: acond = Assembler::greater; break; 3971 case lir_cond_belowEqual: acond = Assembler::belowEqual; break; 3972 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; break; 3973 default: ShouldNotReachHere(); 3974 } 3975 __ jcc(acond, ok); 3976 } 3977 if (op->halt()) { 3978 const char* str = __ code_string(op->msg()); 3979 __ stop(str); 3980 } else { 3981 breakpoint(); 3982 } 3983 __ bind(ok); 3984 } 3985 #endif 3986 3987 void LIR_Assembler::membar() { 3988 // QQQ sparc TSO uses this, 3989 __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad)); 3990 } 3991 3992 void LIR_Assembler::membar_acquire() { 3993 // No x86 machines currently require load fences 3994 } 3995 3996 void LIR_Assembler::membar_release() { 3997 // No x86 machines currently require store fences 3998 } 3999 4000 void LIR_Assembler::membar_loadload() { 4001 // no-op 4002 //__ membar(Assembler::Membar_mask_bits(Assembler::loadload)); 4003 } 4004 4005 void LIR_Assembler::membar_storestore() { 4006 // no-op 4007 //__ membar(Assembler::Membar_mask_bits(Assembler::storestore)); 4008 } 4009 4010 void LIR_Assembler::membar_loadstore() { 4011 // no-op 4012 //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore)); 4013 } 4014 4015 void LIR_Assembler::membar_storeload() { 4016 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); 4017 } 4018 4019 void LIR_Assembler::on_spin_wait() { 4020 __ pause (); 4021 } 4022 4023 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 4024 assert(result_reg->is_register(), "check"); 4025 #ifdef _LP64 4026 // __ get_thread(result_reg->as_register_lo()); 4027 __ mov(result_reg->as_register(), r15_thread); 4028 #else 4029 __ get_thread(result_reg->as_register()); 4030 #endif // _LP64 4031 } 4032 4033 4034 void LIR_Assembler::peephole(LIR_List*) { 4035 // do nothing for now 4036 } 4037 4038 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) { 4039 assert(data == dest, "xchg/xadd uses only 2 operands"); 4040 4041 if (data->type() == T_INT) { 4042 if (code == lir_xadd) { 4043 __ lock(); 4044 __ xaddl(as_Address(src->as_address_ptr()), data->as_register()); 4045 } else { 4046 __ xchgl(data->as_register(), as_Address(src->as_address_ptr())); 4047 } 4048 } else if (data->is_oop()) { 4049 assert (code == lir_xchg, "xadd for oops"); 4050 Register obj = data->as_register(); 4051 #ifdef _LP64 4052 if (UseCompressedOops) { 4053 __ encode_heap_oop(obj); 4054 __ xchgl(obj, as_Address(src->as_address_ptr())); 4055 __ decode_heap_oop(obj); 4056 } else { 4057 __ xchgptr(obj, as_Address(src->as_address_ptr())); 4058 } 4059 #else 4060 __ xchgl(obj, as_Address(src->as_address_ptr())); 4061 #endif 4062 } else if (data->type() == T_LONG) { 4063 #ifdef _LP64 4064 assert(data->as_register_lo() == data->as_register_hi(), "should be a single register"); 4065 if (code == lir_xadd) { 4066 __ lock(); 4067 __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo()); 4068 } else { 4069 __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr())); 4070 } 4071 #else 4072 ShouldNotReachHere(); 4073 #endif 4074 } else { 4075 ShouldNotReachHere(); 4076 } 4077 } 4078 4079 #undef __