1 /*
   2  * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "c1/c1_CodeStubs.hpp"
  31 #include "c1/c1_Compilation.hpp"
  32 #include "c1/c1_LIRAssembler.hpp"
  33 #include "c1/c1_MacroAssembler.hpp"
  34 #include "c1/c1_Runtime1.hpp"
  35 #include "c1/c1_ValueStack.hpp"
  36 #include "ci/ciArrayKlass.hpp"
  37 #include "ci/ciInstance.hpp"
  38 #include "code/compiledIC.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "nativeInst_riscv.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_riscv.inline.hpp"
  46 
  47 #ifndef PRODUCT
  48 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  49 #else
  50 #define COMMENT(x)
  51 #endif
  52 
  53 NEEDS_CLEANUP // remove this definitions ?
  54 const Register IC_Klass    = t1;    // where the IC klass is cached
  55 const Register SYNC_header = x10;   // synchronization header
  56 const Register SHIFT_count = x10;   // where count for shift operations must be
  57 
  58 #define __ _masm->
  59 
  60 static void select_different_registers(Register preserve,
  61                                        Register extra,
  62                                        Register &tmp1,
  63                                        Register &tmp2) {
  64   if (tmp1 == preserve) {
  65     assert_different_registers(tmp1, tmp2, extra);
  66     tmp1 = extra;
  67   } else if (tmp2 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp2 = extra;
  70   }
  71   assert_different_registers(preserve, tmp1, tmp2);
  72 }
  73 
  74 static void select_different_registers(Register preserve,
  75                                        Register extra,
  76                                        Register &tmp1,
  77                                        Register &tmp2,
  78                                        Register &tmp3) {
  79   if (tmp1 == preserve) {
  80     assert_different_registers(tmp1, tmp2, tmp3, extra);
  81     tmp1 = extra;
  82   } else if (tmp2 == preserve) {
  83     assert_different_registers(tmp1, tmp2, tmp3, extra);
  84     tmp2 = extra;
  85   } else if (tmp3 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp3 = extra;
  88   }
  89   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  90 }
  91 
  92 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  93 
  94 void LIR_Assembler::clinit_barrier(ciMethod* method) {
  95   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  96   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
  97 
  98   Label L_skip_barrier;
  99 
 100   __ mov_metadata(t1, method->holder()->constant_encoding());
 101   __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */);
 102   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 103   __ bind(L_skip_barrier);
 104 }
 105 
 106 LIR_Opr LIR_Assembler::receiverOpr() {
 107   return FrameMap::receiver_opr;
 108 }
 109 
 110 LIR_Opr LIR_Assembler::osrBufferPointer() {
 111   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 112 }
 113 
 114 void LIR_Assembler::breakpoint() { Unimplemented(); }
 115 
 116 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 117 
 118 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 119 
 120 static jlong as_long(LIR_Opr data) {
 121   jlong result;
 122   switch (data->type()) {
 123     case T_INT:
 124       result = (data->as_jint());
 125       break;
 126     case T_LONG:
 127       result = (data->as_jlong());
 128       break;
 129     default:
 130       ShouldNotReachHere();
 131       result = 0;  // unreachable
 132   }
 133   return result;
 134 }
 135 
 136 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 137   ShouldNotReachHere();
 138   return Address();
 139 }
 140 
 141 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 142   return as_Address(addr, t0);
 143 }
 144 
 145 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 146   return as_Address(addr);
 147 }
 148 
 149 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 150 // not encodable as a base + (immediate) offset, generate an explicit address
 151 // calculation to hold the address in t0.
 152 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) {
 153   precond(size == 4 || size == 8);
 154   Address addr = frame_map()->address_for_slot(index, adjust);
 155   precond(addr.getMode() == Address::base_plus_offset);
 156   precond(addr.base() == sp);
 157   precond(addr.offset() > 0);
 158   uint mask = size - 1;
 159   assert((addr.offset() & mask) == 0, "scaled offsets only");
 160 
 161   return addr;
 162 }
 163 
 164 void LIR_Assembler::osr_entry() {
 165   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 166   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 167   guarantee(osr_entry != NULL, "NULL osr_entry!");
 168   ValueStack* entry_state = osr_entry->state();
 169   int number_of_locks = entry_state->locks_size();
 170 
 171   // we jump here if osr happens with the interpreter
 172   // state set up to continue at the beginning of the
 173   // loop that triggered osr - in particular, we have
 174   // the following registers setup:
 175   //
 176   // x12: osr buffer
 177   //
 178 
 179   //build frame
 180   ciMethod* m = compilation()->method();
 181   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 182 
 183   // OSR buffer is
 184   //
 185   // locals[nlocals-1..0]
 186   // monitors[0..number_of_locks]
 187   //
 188   // locals is a direct copy of the interpreter frame so in the osr buffer
 189   // so first slot in the local array is the last local from the interpreter
 190   // and last slot is local[0] (receiver) from the interpreter
 191   //
 192   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 193   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 194   // in the interpreter frame (the method lock if a sync method)
 195 
 196   // Initialize monitors in the compiled activation.
 197   //   x12: pointer to osr buffer
 198   // All other registers are dead at this point and the locals will be
 199   // copied into place by code emitted in the IR.
 200 
 201   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 202   {
 203     assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 204     int monitor_offset = BytesPerWord * method()->max_locals() +
 205       (2 * BytesPerWord) * (number_of_locks - 1);
 206     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 207     // the OSR buffer using 2 word entries: first the lock and then
 208     // the oop.
 209     for (int i = 0; i < number_of_locks; i++) {
 210       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 211 #ifdef ASSERT
 212       // verify the interpreter's monitor has a non-null object
 213       {
 214         Label L;
 215         __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
 216         __ bnez(t0, L);
 217         __ stop("locked object is NULL");
 218         __ bind(L);
 219       }
 220 #endif // ASSERT
 221       __ ld(x9, Address(OSR_buf, slot_offset + 0));
 222       __ sd(x9, frame_map()->address_for_monitor_lock(i));
 223       __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
 224       __ sd(x9, frame_map()->address_for_monitor_object(i));
 225     }
 226   }
 227 }
 228 
 229 // inline cache check; done before the frame is built.
 230 int LIR_Assembler::check_icache() {
 231   Register receiver = FrameMap::receiver_opr->as_register();
 232   Register ic_klass = IC_Klass;
 233   int start_offset = __ offset();
 234   Label dont;
 235   __ inline_cache_check(receiver, ic_klass, dont);
 236 
 237   // if icache check fails, then jump to runtime routine
 238   // Note: RECEIVER must still contain the receiver!
 239   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 240 
 241   // We align the verified entry point unless the method body
 242   // (including its inline cache check) will fit in a single 64-byte
 243   // icache line.
 244   if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 245     // force alignment after the cache check.
 246     __ align(CodeEntryAlignment);
 247   }
 248 
 249   __ bind(dont);
 250   return start_offset;
 251 }
 252 
 253 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 254   if (o == NULL) {
 255     __ mv(reg, zr);
 256   } else {
 257     __ movoop(reg, o, /* immediate */ true);
 258   }
 259 }
 260 
 261 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 262   deoptimize_trap(info);
 263 }
 264 
 265 // This specifies the rsp decrement needed to build the frame
 266 int LIR_Assembler::initial_frame_size_in_bytes() const {
 267   // if rounding, must let FrameMap know!
 268 
 269   return in_bytes(frame_map()->framesize_in_bytes());
 270 }
 271 
 272 int LIR_Assembler::emit_exception_handler() {
 273   // if the last instruction is a call (typically to do a throw which
 274   // is coming at the end after block reordering) the return address
 275   // must still point into the code area in order to avoid assertion
 276   // failures when searching for the corresponding bci ==> add a nop
 277   // (was bug 5/14/1999 -gri)
 278   __ nop();
 279 
 280   // generate code for exception handler
 281   address handler_base = __ start_a_stub(exception_handler_size());
 282   if (handler_base == NULL) {
 283     // not enough space left for the handler
 284     bailout("exception handler overflow");
 285     return -1;
 286   }
 287 
 288   int offset = code_offset();
 289 
 290   // the exception oop and pc are in x10, and x13
 291   // no other registers need to be preserved, so invalidate them
 292   __ invalidate_registers(false, true, true, false, true, true);
 293 
 294   // check that there is really an exception
 295   __ verify_not_null_oop(x10);
 296 
 297   // search an exception handler (x10: exception oop, x13: throwing pc)
 298   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 299   __ should_not_reach_here();
 300   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 301   __ end_a_stub();
 302 
 303   return offset;
 304 }
 305 
 306 // Emit the code to remove the frame from the stack in the exception
 307 // unwind path.
 308 int LIR_Assembler::emit_unwind_handler() {
 309 #ifndef PRODUCT
 310   if (CommentedAssembly) {
 311     _masm->block_comment("Unwind handler");
 312   }
 313 #endif // PRODUCT
 314 
 315   int offset = code_offset();
 316 
 317   // Fetch the exception from TLS and clear out exception related thread state
 318   __ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
 319   __ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
 320   __ sd(zr, Address(xthread, JavaThread::exception_pc_offset()));
 321 
 322   __ bind(_unwind_handler_entry);
 323   __ verify_not_null_oop(x10);
 324   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 325     __ mv(x9, x10);   // Perserve the exception
 326   }
 327 
 328   // Preform needed unlocking
 329   MonitorExitStub* stub = NULL;
 330   if (method()->is_synchronized()) {
 331     monitor_address(0, FrameMap::r10_opr);
 332     stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
 333     __ unlock_object(x15, x14, x10, *stub->entry());
 334     __ bind(*stub->continuation());
 335   }
 336 
 337   if (compilation()->env()->dtrace_method_probes()) {
 338     __ mv(c_rarg0, xthread);
 339     __ mov_metadata(c_rarg1, method()->constant_encoding());
 340     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 341   }
 342 
 343   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 344     __ mv(x10, x9);   // Restore the exception
 345   }
 346 
 347   // remove the activation and dispatch to the unwind handler
 348   __ block_comment("remove_frame and dispatch to the unwind handler");
 349   __ remove_frame(initial_frame_size_in_bytes());
 350   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 351 
 352   // Emit the slow path assembly
 353   if (stub != NULL) {
 354     stub->emit_code(this);
 355   }
 356 
 357   return offset;
 358 }
 359 
 360 int LIR_Assembler::emit_deopt_handler() {
 361   // if the last instruciton is a call (typically to do a throw which
 362   // is coming at the end after block reordering) the return address
 363   // must still point into the code area in order to avoid assertion
 364   // failures when searching for the corresponding bck => add a nop
 365   // (was bug 5/14/1999 - gri)
 366   __ nop();
 367 
 368   // generate code for exception handler
 369   address handler_base = __ start_a_stub(deopt_handler_size());
 370   if (handler_base == NULL) {
 371     // not enough space left for the handler
 372     bailout("deopt handler overflow");
 373     return -1;
 374   }
 375 
 376   int offset = code_offset();
 377 
 378   __ auipc(ra, 0);
 379   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 380   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 381   __ end_a_stub();
 382 
 383   return offset;
 384 }
 385 
 386 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 387   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
 388 
 389   // Pop the stack before the safepoint code
 390   __ remove_frame(initial_frame_size_in_bytes());
 391 
 392   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 393     __ reserved_stack_check();
 394   }
 395 
 396   code_stub->set_safepoint_offset(__ offset());
 397   __ relocate(relocInfo::poll_return_type);
 398   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 399   __ ret();
 400 }
 401 
 402 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 403   guarantee(info != NULL, "Shouldn't be NULL");
 404   __ get_polling_page(t0, relocInfo::poll_type);
 405   add_debug_info_for_branch(info);  // This isn't just debug info:
 406                                     // it's the oop map
 407   __ read_polling_page(t0, 0, relocInfo::poll_type);
 408   return __ offset();
 409 }
 410 
 411 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 412   __ mv(to_reg, from_reg);
 413 }
 414 
 415 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 416 
 417 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 418   assert(src->is_constant(), "should not call otherwise");
 419   assert(dest->is_register(), "should not call otherwise");
 420   LIR_Const* c = src->as_constant_ptr();
 421   address const_addr = NULL;
 422 
 423   switch (c->type()) {
 424     case T_INT:
 425       assert(patch_code == lir_patch_none, "no patching handled here");
 426       __ mvw(dest->as_register(), c->as_jint());
 427       break;
 428 
 429     case T_ADDRESS:
 430       assert(patch_code == lir_patch_none, "no patching handled here");
 431       __ mv(dest->as_register(), c->as_jint());
 432       break;
 433 
 434     case T_LONG:
 435       assert(patch_code == lir_patch_none, "no patching handled here");
 436       __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
 437       break;
 438 
 439     case T_OBJECT:
 440     case T_ARRAY:
 441       if (patch_code == lir_patch_none) {
 442         jobject2reg(c->as_jobject(), dest->as_register());
 443       } else {
 444         jobject2reg_with_patching(dest->as_register(), info);
 445       }
 446       break;
 447 
 448     case T_METADATA:
 449       if (patch_code != lir_patch_none) {
 450         klass2reg_with_patching(dest->as_register(), info);
 451       } else {
 452         __ mov_metadata(dest->as_register(), c->as_metadata());
 453       }
 454       break;
 455 
 456     case T_FLOAT:
 457       const_addr = float_constant(c->as_jfloat());
 458       assert(const_addr != NULL, "must create float constant in the constant table");
 459       __ flw(dest->as_float_reg(), InternalAddress(const_addr));
 460       break;
 461 
 462     case T_DOUBLE:
 463       const_addr = double_constant(c->as_jdouble());
 464       assert(const_addr != NULL, "must create double constant in the constant table");
 465       __ fld(dest->as_double_reg(), InternalAddress(const_addr));
 466       break;
 467 
 468     default:
 469       ShouldNotReachHere();
 470   }
 471 }
 472 
 473 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 474   assert(src->is_constant(), "should not call otherwise");
 475   assert(dest->is_stack(), "should not call otherwise");
 476   LIR_Const* c = src->as_constant_ptr();
 477   switch (c->type()) {
 478     case T_OBJECT:
 479       if (c->as_jobject() == NULL) {
 480         __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 481       } else {
 482         const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
 483         reg2stack(FrameMap::t1_opr, dest, c->type(), false);
 484       }
 485       break;
 486     case T_ADDRESS:   // fall through
 487       const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
 488       reg2stack(FrameMap::t1_opr, dest, c->type(), false);
 489     case T_INT:       // fall through
 490     case T_FLOAT:
 491       if (c->as_jint_bits() == 0) {
 492         __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 493       } else {
 494         __ mvw(t1, c->as_jint_bits());
 495         __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix()));
 496       }
 497       break;
 498     case T_LONG:      // fall through
 499     case T_DOUBLE:
 500       if (c->as_jlong_bits() == 0) {
 501         __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 502                                                 lo_word_offset_in_bytes));
 503       } else {
 504         __ mv(t1, (intptr_t)c->as_jlong_bits());
 505         __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(),
 506                                                 lo_word_offset_in_bytes));
 507       }
 508       break;
 509     default:
 510       ShouldNotReachHere();
 511   }
 512 }
 513 
 514 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 515   assert(src->is_constant(), "should not call otherwise");
 516   assert(dest->is_address(), "should not call otherwise");
 517   LIR_Const* c = src->as_constant_ptr();
 518   LIR_Address* to_addr = dest->as_address_ptr();
 519   void (Assembler::* insn)(Register Rt, const Address &adr, Register temp);
 520   switch (type) {
 521     case T_ADDRESS:
 522       assert(c->as_jint() == 0, "should be");
 523       insn = &Assembler::sd; break;
 524     case T_LONG:
 525       assert(c->as_jlong() == 0, "should be");
 526       insn = &Assembler::sd; break;
 527     case T_DOUBLE:
 528       assert(c->as_jdouble() == 0.0, "should be");
 529       insn = &Assembler::sd; break;
 530     case T_INT:
 531       assert(c->as_jint() == 0, "should be");
 532       insn = &Assembler::sw; break;
 533     case T_FLOAT:
 534       assert(c->as_jfloat() == 0.0f, "should be");
 535       insn = &Assembler::sw; break;
 536     case T_OBJECT:    // fall through
 537     case T_ARRAY:
 538       assert(c->as_jobject() == 0, "should be");
 539       if (UseCompressedOops && !wide) {
 540         insn = &Assembler::sw;
 541       } else {
 542         insn = &Assembler::sd;
 543       }
 544       break;
 545     case T_CHAR:      // fall through
 546     case T_SHORT:
 547       assert(c->as_jint() == 0, "should be");
 548       insn = &Assembler::sh;
 549       break;
 550     case T_BOOLEAN:   // fall through
 551     case T_BYTE:
 552       assert(c->as_jint() == 0, "should be");
 553       insn = &Assembler::sb; break;
 554     default:
 555       ShouldNotReachHere();
 556       insn = &Assembler::sd;  // unreachable
 557   }
 558   if (info != NULL) {
 559     add_debug_info_for_null_check_here(info);
 560   }
 561   (_masm->*insn)(zr, as_Address(to_addr), t0);
 562 }
 563 
 564 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 565   assert(src->is_register(), "should not call otherwise");
 566   assert(dest->is_register(), "should not call otherwise");
 567 
 568   // move between cpu-registers
 569   if (dest->is_single_cpu()) {
 570     if (src->type() == T_LONG) {
 571       // Can do LONG -> OBJECT
 572       move_regs(src->as_register_lo(), dest->as_register());
 573       return;
 574     }
 575     assert(src->is_single_cpu(), "must match");
 576     if (src->type() == T_OBJECT) {
 577       __ verify_oop(src->as_register());
 578     }
 579     move_regs(src->as_register(), dest->as_register());
 580   } else if (dest->is_double_cpu()) {
 581     if (is_reference_type(src->type())) {
 582       __ verify_oop(src->as_register());
 583       move_regs(src->as_register(), dest->as_register_lo());
 584       return;
 585     }
 586     assert(src->is_double_cpu(), "must match");
 587     Register f_lo = src->as_register_lo();
 588     Register f_hi = src->as_register_hi();
 589     Register t_lo = dest->as_register_lo();
 590     Register t_hi = dest->as_register_hi();
 591     assert(f_hi == f_lo, "must be same");
 592     assert(t_hi == t_lo, "must be same");
 593     move_regs(f_lo, t_lo);
 594   } else if (dest->is_single_fpu()) {
 595     assert(src->is_single_fpu(), "expect single fpu");
 596     __ fmv_s(dest->as_float_reg(), src->as_float_reg());
 597   } else if (dest->is_double_fpu()) {
 598     assert(src->is_double_fpu(), "expect double fpu");
 599     __ fmv_d(dest->as_double_reg(), src->as_double_reg());
 600   } else {
 601     ShouldNotReachHere();
 602   }
 603 }
 604 
 605 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 606   precond(src->is_register() && dest->is_stack());
 607 
 608   uint const c_sz32 = sizeof(uint32_t);
 609   uint const c_sz64 = sizeof(uint64_t);
 610 
 611   assert(src->is_register(), "should not call otherwise");
 612   assert(dest->is_stack(), "should not call otherwise");
 613   if (src->is_single_cpu()) {
 614     int index = dest->single_stack_ix();
 615     if (is_reference_type(type)) {
 616       __ sd(src->as_register(), stack_slot_address(index, c_sz64));
 617       __ verify_oop(src->as_register());
 618     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 619       __ sd(src->as_register(), stack_slot_address(index, c_sz64));
 620     } else {
 621       __ sw(src->as_register(), stack_slot_address(index, c_sz32));
 622     }
 623   } else if (src->is_double_cpu()) {
 624     int index = dest->double_stack_ix();
 625     Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
 626     __ sd(src->as_register_lo(), dest_addr_LO);
 627   } else if (src->is_single_fpu()) {
 628     int index = dest->single_stack_ix();
 629     __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32));
 630   } else if (src->is_double_fpu()) {
 631     int index = dest->double_stack_ix();
 632     __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64));
 633   } else {
 634     ShouldNotReachHere();
 635   }
 636 }
 637 
 638 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 639   LIR_Address* to_addr = dest->as_address_ptr();
 640   // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src
 641   Register compressed_src = t1;
 642 
 643   if (patch_code != lir_patch_none) {
 644     deoptimize_trap(info);
 645     return;
 646   }
 647 
 648   if (is_reference_type(type)) {
 649     __ verify_oop(src->as_register());
 650 
 651     if (UseCompressedOops && !wide) {
 652       __ encode_heap_oop(compressed_src, src->as_register());
 653     } else {
 654       compressed_src = src->as_register();
 655     }
 656   }
 657 
 658   int null_check_here = code_offset();
 659 
 660   switch (type) {
 661     case T_FLOAT:
 662       __ fsw(src->as_float_reg(), as_Address(to_addr));
 663       break;
 664 
 665     case T_DOUBLE:
 666       __ fsd(src->as_double_reg(), as_Address(to_addr));
 667       break;
 668 
 669     case T_ARRAY:      // fall through
 670     case T_OBJECT:
 671       if (UseCompressedOops && !wide) {
 672         __ sw(compressed_src, as_Address(to_addr));
 673       } else {
 674         __ sd(compressed_src, as_Address(to_addr));
 675       }
 676       break;
 677     case T_METADATA:
 678       // We get here to store a method pointer to the stack to pass to
 679       // a dtrace runtime call. This can't work on 64 bit with
 680       // compressed klass ptrs: T_METADATA can be compressed klass
 681       // ptr or a 64 bit method pointer.
 682       ShouldNotReachHere();
 683       __ sd(src->as_register(), as_Address(to_addr));
 684       break;
 685     case T_ADDRESS:
 686       __ sd(src->as_register(), as_Address(to_addr));
 687       break;
 688     case T_INT:
 689       __ sw(src->as_register(), as_Address(to_addr));
 690       break;
 691     case T_LONG:
 692       __ sd(src->as_register_lo(), as_Address(to_addr));
 693       break;
 694     case T_BYTE:    // fall through
 695     case T_BOOLEAN:
 696       __ sb(src->as_register(), as_Address(to_addr));
 697       break;
 698     case T_CHAR:    // fall through
 699     case T_SHORT:
 700       __ sh(src->as_register(), as_Address(to_addr));
 701       break;
 702     default:
 703       ShouldNotReachHere();
 704   }
 705 
 706   if (info != NULL) {
 707     add_debug_info_for_null_check(null_check_here, info);
 708   }
 709 }
 710 
 711 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 712   precond(src->is_stack() && dest->is_register());
 713 
 714   uint const c_sz32 = sizeof(uint32_t);
 715   uint const c_sz64 = sizeof(uint64_t);
 716 
 717   if (dest->is_single_cpu()) {
 718     int index = src->single_stack_ix();
 719     if (type == T_INT) {
 720       __ lw(dest->as_register(), stack_slot_address(index, c_sz32));
 721     } else if (is_reference_type(type)) {
 722       __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
 723       __ verify_oop(dest->as_register());
 724     } else if (type == T_METADATA || type == T_ADDRESS) {
 725       __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
 726     } else {
 727       __ lwu(dest->as_register(), stack_slot_address(index, c_sz32));
 728     }
 729   } else if (dest->is_double_cpu()) {
 730     int index = src->double_stack_ix();
 731     Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
 732     __ ld(dest->as_register_lo(), src_addr_LO);
 733   } else if (dest->is_single_fpu()) {
 734     int index = src->single_stack_ix();
 735     __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32));
 736   } else if (dest->is_double_fpu()) {
 737     int index = src->double_stack_ix();
 738     __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64));
 739   } else {
 740     ShouldNotReachHere();
 741   }
 742 }
 743 
 744 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 745   deoptimize_trap(info);
 746 }
 747 
 748 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 749   LIR_Opr temp;
 750   if (type == T_LONG || type == T_DOUBLE) {
 751     temp = FrameMap::t1_long_opr;
 752   } else {
 753     temp = FrameMap::t1_opr;
 754   }
 755 
 756   stack2reg(src, temp, src->type());
 757   reg2stack(temp, dest, dest->type(), false);
 758 }
 759 
 760 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 761   assert(src->is_address(), "should not call otherwise");
 762   assert(dest->is_register(), "should not call otherwise");
 763 
 764   LIR_Address* addr = src->as_address_ptr();
 765   LIR_Address* from_addr = src->as_address_ptr();
 766 
 767   if (addr->base()->type() == T_OBJECT) {
 768     __ verify_oop(addr->base()->as_pointer_register());
 769   }
 770 
 771   if (patch_code != lir_patch_none) {
 772     deoptimize_trap(info);
 773     return;
 774   }
 775 
 776   if (info != NULL) {
 777     add_debug_info_for_null_check_here(info);
 778   }
 779 
 780   int null_check_here = code_offset();
 781   switch (type) {
 782     case T_FLOAT:
 783       __ flw(dest->as_float_reg(), as_Address(from_addr));
 784       break;
 785     case T_DOUBLE:
 786       __ fld(dest->as_double_reg(), as_Address(from_addr));
 787       break;
 788     case T_ARRAY:     // fall through
 789     case T_OBJECT:
 790       if (UseCompressedOops && !wide) {
 791         __ lwu(dest->as_register(), as_Address(from_addr));
 792       } else {
 793         __ ld(dest->as_register(), as_Address(from_addr));
 794       }
 795       break;
 796     case T_METADATA:
 797       // We get here to store a method pointer to the stack to pass to
 798       // a dtrace runtime call. This can't work on 64 bit with
 799       // compressed klass ptrs: T_METADATA can be a compressed klass
 800       // ptr or a 64 bit method pointer.
 801       ShouldNotReachHere();
 802       __ ld(dest->as_register(), as_Address(from_addr));
 803       break;
 804     case T_ADDRESS:
 805       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 806         __ lwu(dest->as_register(), as_Address(from_addr));
 807       } else {
 808         __ ld(dest->as_register(), as_Address(from_addr));
 809       }
 810       break;
 811     case T_INT:
 812       __ lw(dest->as_register(), as_Address(from_addr));
 813       break;
 814     case T_LONG:
 815       __ ld(dest->as_register_lo(), as_Address_lo(from_addr));
 816       break;
 817     case T_BYTE:
 818       __ lb(dest->as_register(), as_Address(from_addr));
 819       break;
 820     case T_BOOLEAN:
 821       __ lbu(dest->as_register(), as_Address(from_addr));
 822       break;
 823     case T_CHAR:
 824       __ lhu(dest->as_register(), as_Address(from_addr));
 825       break;
 826     case T_SHORT:
 827       __ lh(dest->as_register(), as_Address(from_addr));
 828       break;
 829     default:
 830       ShouldNotReachHere();
 831   }
 832 
 833   if (is_reference_type(type)) {
 834     if (UseCompressedOops && !wide) {
 835       __ decode_heap_oop(dest->as_register());
 836     }
 837 
 838     if (!UseZGC) {
 839       // Load barrier has not yet been applied, so ZGC can't verify the oop here
 840       __ verify_oop(dest->as_register());
 841     }
 842   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 843     if (UseCompressedClassPointers) {
 844       __ decode_klass_not_null(dest->as_register());
 845     }
 846   }
 847 }
 848 
 849 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 850   switch (op->code()) {
 851     case lir_idiv: // fall through
 852     case lir_irem:
 853       arithmetic_idiv(op->code(),
 854                       op->in_opr1(),
 855                       op->in_opr2(),
 856                       op->in_opr3(),
 857                       op->result_opr(),
 858                       op->info());
 859       break;
 860     case lir_fmad:
 861       __ fmadd_d(op->result_opr()->as_double_reg(),
 862                  op->in_opr1()->as_double_reg(),
 863                  op->in_opr2()->as_double_reg(),
 864                  op->in_opr3()->as_double_reg());
 865       break;
 866     case lir_fmaf:
 867       __ fmadd_s(op->result_opr()->as_float_reg(),
 868                  op->in_opr1()->as_float_reg(),
 869                  op->in_opr2()->as_float_reg(),
 870                  op->in_opr3()->as_float_reg());
 871       break;
 872     default:
 873       ShouldNotReachHere();
 874   }
 875 }
 876 
 877 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr cmp_opr1, LIR_Opr cmp_opr2, LIR_Opr result, BasicType type) {
 878   Label label;
 879 
 880   emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false,
 881               /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true);
 882 
 883   Label done;
 884   move_op(opr2, result, type, lir_patch_none, NULL,
 885           false,   // pop_fpu_stack
 886           false);  // wide
 887   __ j(done);
 888   __ bind(label);
 889   move_op(opr1, result, type, lir_patch_none, NULL,
 890           false,   // pop_fpu_stack
 891           false);  // wide
 892   __ bind(done);
 893 }
 894 
 895 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 896   LIR_Condition condition = op->cond();
 897   if (condition == lir_cond_always) {
 898     if (op->info() != NULL) {
 899       add_debug_info_for_branch(op->info());
 900     }
 901   } else {
 902     assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands");
 903   }
 904   bool is_unordered = (op->ublock() == op->block());
 905   emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered);
 906 }
 907 
 908 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label,
 909                                 bool is_far, bool is_unordered) {
 910 
 911   if (cmp_flag == lir_cond_always) {
 912     __ j(label);
 913     return;
 914   }
 915 
 916   if (cmp1->is_cpu_register()) {
 917     Register reg1 = as_reg(cmp1);
 918     if (cmp2->is_cpu_register()) {
 919       Register reg2 = as_reg(cmp2);
 920       __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far);
 921     } else if (cmp2->is_constant()) {
 922       const2reg_helper(cmp2);
 923       __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far);
 924     } else {
 925       ShouldNotReachHere();
 926     }
 927   } else if (cmp1->is_single_fpu()) {
 928     assert(cmp2->is_single_fpu(), "expect single float register");
 929     __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered);
 930   } else if (cmp1->is_double_fpu()) {
 931     assert(cmp2->is_double_fpu(), "expect double float register");
 932     __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask,
 933                            cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered);
 934   } else {
 935     ShouldNotReachHere();
 936   }
 937 }
 938 
 939 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 940   LIR_Opr src  = op->in_opr();
 941   LIR_Opr dest = op->result_opr();
 942 
 943   switch (op->bytecode()) {
 944     case Bytecodes::_i2f:
 945       __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break;
 946     case Bytecodes::_i2d:
 947       __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break;
 948     case Bytecodes::_l2d:
 949       __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break;
 950     case Bytecodes::_l2f:
 951       __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break;
 952     case Bytecodes::_f2d:
 953       __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break;
 954     case Bytecodes::_d2f:
 955       __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break;
 956     case Bytecodes::_i2c:
 957       __ zero_ext(dest->as_register(), src->as_register(), registerSize - 16); break; // 16: char size
 958     case Bytecodes::_i2l:
 959       __ addw(dest->as_register_lo(), src->as_register(), zr); break;
 960     case Bytecodes::_i2s:
 961       __ sign_ext(dest->as_register(), src->as_register(), registerSize - 16); break; // 16: short size
 962     case Bytecodes::_i2b:
 963       __ sign_ext(dest->as_register(), src->as_register(), registerSize - 8); break;  // 8: byte size
 964     case Bytecodes::_l2i:
 965       _masm->block_comment("FIXME: This coulde be no-op");
 966       __ addw(dest->as_register(), src->as_register_lo(), zr); break;
 967     case Bytecodes::_d2l:
 968       __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break;
 969     case Bytecodes::_f2i:
 970       __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break;
 971     case Bytecodes::_f2l:
 972       __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break;
 973     case Bytecodes::_d2i:
 974       __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break;
 975     default:
 976       ShouldNotReachHere();
 977   }
 978 }
 979 
 980 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
 981   if (op->init_check()) {
 982     __ lbu(t0, Address(op->klass()->as_register(),
 983                        InstanceKlass::init_state_offset()));
 984     __ mvw(t1, InstanceKlass::fully_initialized);
 985     add_debug_info_for_null_check_here(op->stub()->info());
 986     __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
 987   }
 988 
 989   __ allocate_object(op->obj()->as_register(),
 990                      op->tmp1()->as_register(),
 991                      op->tmp2()->as_register(),
 992                      op->header_size(),
 993                      op->object_size(),
 994                      op->klass()->as_register(),
 995                      *op->stub()->entry());
 996 
 997   __ bind(*op->stub()->continuation());
 998 }
 999 
1000 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1001   Register len = op->len()->as_register();
1002 
1003   if (UseSlowPath ||
1004       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1005       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1006     __ j(*op->stub()->entry());
1007   } else {
1008     Register tmp1 = op->tmp1()->as_register();
1009     Register tmp2 = op->tmp2()->as_register();
1010     Register tmp3 = op->tmp3()->as_register();
1011     if (len == tmp1) {
1012       tmp1 = tmp3;
1013     } else if (len == tmp2) {
1014       tmp2 = tmp3;
1015     } else if (len == tmp3) {
1016       // everything is ok
1017     } else {
1018       __ mv(tmp3, len);
1019     }
1020     __ allocate_array(op->obj()->as_register(),
1021                       len,
1022                       tmp1,
1023                       tmp2,
1024                       arrayOopDesc::header_size(op->type()),
1025                       array_element_size(op->type()),
1026                       op->klass()->as_register(),
1027                       *op->stub()->entry());
1028   }
1029   __ bind(*op->stub()->continuation());
1030 }
1031 
1032 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
1033                                         Register recv, Label* update_done) {
1034   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1035     Label next_test;
1036     // See if the receiver is receiver[n].
1037     __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1038     __ bne(recv, t1, next_test);
1039     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1040     __ add_memory_int64(data_addr, DataLayout::counter_increment);
1041     __ j(*update_done);
1042     __ bind(next_test);
1043   }
1044 
1045   // Didn't find receiver; find next empty slot and fill it in
1046   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1047     Label next_test;
1048     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1049     __ ld(t1, recv_addr);
1050     __ bnez(t1, next_test);
1051     __ sd(recv, recv_addr);
1052     __ li(t1, DataLayout::counter_increment);
1053     __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1054     __ j(*update_done);
1055     __ bind(next_test);
1056   }
1057 }
1058 
1059 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
1060   ciMethod* method = op->profiled_method();
1061   assert(method != NULL, "Should have method");
1062   int bci = op->profiled_bci();
1063   *md = method->method_data_or_null();
1064   guarantee(*md != NULL, "Sanity");
1065   *data = ((*md)->bci_to_data(bci));
1066   assert(*data != NULL, "need data for type check");
1067   assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1068 }
1069 
1070 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1071                                                Register k_RInfo, Register klass_RInfo,
1072                                                Label *failure_target, Label *success_target) {
1073   // get object class
1074   // not a safepoint as obj null check happens earlier
1075   __ load_klass(klass_RInfo, obj);
1076   if (k->is_loaded()) {
1077     // See if we get an immediate positive hit
1078     __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1079     if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1080       __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1081       // successful cast, fall through to profile or jump
1082     } else {
1083       // See if we get an immediate positive hit
1084       __ beq(k_RInfo, t0, *success_target);
1085       // check for self
1086       __ beq(klass_RInfo, k_RInfo, *success_target);
1087 
1088       __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1089       __ sd(k_RInfo, Address(sp, 0));             // sub klass
1090       __ sd(klass_RInfo, Address(sp, wordSize));  // super klass
1091       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1092       // load result to k_RInfo
1093       __ ld(k_RInfo, Address(sp, 0));
1094       __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1095       // result is a boolean
1096       __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1097       // successful cast, fall through to profile or jump
1098     }
1099   } else {
1100     // perform the fast part of the checking logic
1101     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1102     // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1103     __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1104     __ sd(klass_RInfo, Address(sp, wordSize));  // sub klass
1105     __ sd(k_RInfo, Address(sp, 0));             // super klass
1106     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1107     // load result to k_RInfo
1108     __ ld(k_RInfo, Address(sp, 0));
1109     __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1110     // result is a boolean
1111     __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1112     // successful cast, fall thriugh to profile or jump
1113   }
1114 }
1115 
1116 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj,
1117                                    Register klass_RInfo, Label* obj_is_null) {
1118   Label not_null;
1119   __ bnez(obj, not_null);
1120   // Object is null, update MDO and exit
1121   Register mdo = klass_RInfo;
1122   __ mov_metadata(mdo, md->constant_encoding());
1123   Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1124   __ lbu(t0, data_addr);
1125   __ ori(t0, t0, BitData::null_seen_byte_constant());
1126   __ sb(t0, data_addr);
1127   __ j(*obj_is_null);
1128   __ bind(not_null);
1129 }
1130 
1131 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
1132   if (!k->is_loaded()) {
1133     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1134   } else {
1135     __ mov_metadata(k_RInfo, k->constant_encoding());
1136   }
1137 }
1138 
1139 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1140   Register obj = op->object()->as_register();
1141   Register k_RInfo = op->tmp1()->as_register();
1142   Register klass_RInfo = op->tmp2()->as_register();
1143   Register dst = op->result_opr()->as_register();
1144   ciKlass* k = op->klass();
1145   Register Rtmp1 = noreg;
1146 
1147   // check if it needs to be profiled
1148   ciMethodData* md = NULL;
1149   ciProfileData* data = NULL;
1150 
1151   const bool should_profile = op->should_profile();
1152   if (should_profile) {
1153     data_check(op, &md, &data);
1154   }
1155   Label profile_cast_success, profile_cast_failure;
1156   Label *success_target = should_profile ? &profile_cast_success : success;
1157   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1158 
1159   if (obj == k_RInfo) {
1160     k_RInfo = dst;
1161   } else if (obj == klass_RInfo) {
1162     klass_RInfo = dst;
1163   }
1164   if (k->is_loaded() && !UseCompressedClassPointers) {
1165     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1166   } else {
1167     Rtmp1 = op->tmp3()->as_register();
1168     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1169   }
1170 
1171   assert_different_registers(obj, k_RInfo, klass_RInfo);
1172 
1173   if (should_profile) {
1174     profile_object(md, data, obj, klass_RInfo, obj_is_null);
1175   } else {
1176     __ beqz(obj, *obj_is_null);
1177   }
1178 
1179   typecheck_loaded(op, k, k_RInfo);
1180   __ verify_oop(obj);
1181 
1182   if (op->fast_check()) {
1183     // get object class
1184     // not a safepoint as obj null check happens earlier
1185     __ load_klass(t0, obj);
1186     __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1187     // successful cast, fall through to profile or jump
1188   } else {
1189     typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1190   }
1191   if (should_profile) {
1192     type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure);
1193   }
1194   __ j(*success);
1195 }
1196 
1197 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1198   const bool should_profile = op->should_profile();
1199 
1200   LIR_Code code = op->code();
1201   if (code == lir_store_check) {
1202     typecheck_lir_store(op, should_profile);
1203   } else if (code == lir_checkcast) {
1204     Register obj = op->object()->as_register();
1205     Register dst = op->result_opr()->as_register();
1206     Label success;
1207     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1208     __ bind(success);
1209     if (dst != obj) {
1210       __ mv(dst, obj);
1211     }
1212   } else if (code == lir_instanceof) {
1213     Register obj = op->object()->as_register();
1214     Register dst = op->result_opr()->as_register();
1215     Label success, failure, done;
1216     emit_typecheck_helper(op, &success, &failure, &failure);
1217     __ bind(failure);
1218     __ mv(dst, zr);
1219     __ j(done);
1220     __ bind(success);
1221     __ mv(dst, 1);
1222     __ bind(done);
1223   } else {
1224     ShouldNotReachHere();
1225   }
1226 }
1227 
1228 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1229   assert(VM_Version::supports_cx8(), "wrong machine");
1230   Register addr;
1231   if (op->addr()->is_register()) {
1232     addr = as_reg(op->addr());
1233   } else {
1234     assert(op->addr()->is_address(), "what else?");
1235     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1236     assert(addr_ptr->disp() == 0, "need 0 disp");
1237     assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1238     addr = as_reg(addr_ptr->base());
1239   }
1240   Register newval = as_reg(op->new_value());
1241   Register cmpval = as_reg(op->cmp_value());
1242 
1243   if (op->code() == lir_cas_obj) {
1244     if (UseCompressedOops) {
1245       Register tmp1 = op->tmp1()->as_register();
1246       assert(op->tmp1()->is_valid(), "must be");
1247       __ encode_heap_oop(tmp1, cmpval);
1248       cmpval = tmp1;
1249       __ encode_heap_oop(t1, newval);
1250       newval = t1;
1251       caswu(addr, newval, cmpval);
1252     } else {
1253       casl(addr, newval, cmpval);
1254     }
1255   } else if (op->code() == lir_cas_int) {
1256     casw(addr, newval, cmpval);
1257   } else {
1258     casl(addr, newval, cmpval);
1259   }
1260 }
1261 
1262 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1263   switch (code) {
1264     case lir_abs:  __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break;
1265     case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break;
1266     default:       ShouldNotReachHere();
1267   }
1268 }
1269 
1270 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1271   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1272   Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1273   if (dst->is_single_cpu()) {
1274     Register Rdst = dst->as_register();
1275     if (right->is_constant()) {
1276       int right_const = right->as_jint();
1277       if (Assembler::operand_valid_for_add_immediate(right_const)) {
1278         logic_op_imm(Rdst, Rleft, right_const, code);
1279         __ addw(Rdst, Rdst, zr);
1280      } else {
1281         __ mv(t0, right_const);
1282         logic_op_reg32(Rdst, Rleft, t0, code);
1283      }
1284     } else {
1285       Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1286       logic_op_reg32(Rdst, Rleft, Rright, code);
1287     }
1288   } else {
1289     Register Rdst = dst->as_register_lo();
1290     if (right->is_constant()) {
1291       long right_const = right->as_jlong();
1292       if (Assembler::operand_valid_for_add_immediate(right_const)) {
1293         logic_op_imm(Rdst, Rleft, right_const, code);
1294       } else {
1295         __ mv(t0, right_const);
1296         logic_op_reg(Rdst, Rleft, t0, code);
1297       }
1298     } else {
1299       Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1300       logic_op_reg(Rdst, Rleft, Rright, code);
1301     }
1302   }
1303 }
1304 
1305 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) {
1306   ShouldNotCallThis();
1307 }
1308 
1309 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1310   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1311     bool is_unordered_less = (code == lir_ucmp_fd2i);
1312     if (left->is_single_fpu()) {
1313       __ float_cmp(true, is_unordered_less ? -1 : 1,
1314                    left->as_float_reg(), right->as_float_reg(), dst->as_register());
1315     } else if (left->is_double_fpu()) {
1316       __ float_cmp(false, is_unordered_less ? -1 : 1,
1317                    left->as_double_reg(), right->as_double_reg(), dst->as_register());
1318     } else {
1319       ShouldNotReachHere();
1320     }
1321   } else if (code == lir_cmp_l2i) {
1322     __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1323   } else {
1324     ShouldNotReachHere();
1325   }
1326 }
1327 
1328 void LIR_Assembler::align_call(LIR_Code code) {  }
1329 
1330 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1331   address call = __ trampoline_call(Address(op->addr(), rtype));
1332   if (call == NULL) {
1333     bailout("trampoline stub overflow");
1334     return;
1335   }
1336   add_call_info(code_offset(), op->info());
1337 }
1338 
1339 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1340   address call = __ ic_call(op->addr());
1341   if (call == NULL) {
1342     bailout("trampoline stub overflow");
1343     return;
1344   }
1345   add_call_info(code_offset(), op->info());
1346 }
1347 
1348 void LIR_Assembler::emit_static_call_stub() {
1349   address call_pc = __ pc();
1350   address stub = __ start_a_stub(call_stub_size());
1351   if (stub == NULL) {
1352     bailout("static call stub overflow");
1353     return;
1354   }
1355 
1356   int start = __ offset();
1357 
1358   __ relocate(static_stub_Relocation::spec(call_pc));
1359   __ emit_static_call_stub();
1360 
1361   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size() <= call_stub_size(), "stub too big");
1362   __ end_a_stub();
1363 }
1364 
1365 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1366   assert(exceptionOop->as_register() == x10, "must match");
1367   assert(exceptionPC->as_register() == x13, "must match");
1368 
1369   // exception object is not added to oop map by LinearScan
1370   // (LinearScan assumes that no oops are in fixed registers)
1371   info->add_register_oop(exceptionOop);
1372   Runtime1::StubID unwind_id;
1373 
1374   // get current pc information
1375   // pc is only needed if the method has an exception handler, the unwind code does not need it.
1376   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
1377     // As no instructions have been generated yet for this LIR node it's
1378     // possible that an oop map already exists for the current offset.
1379     // In that case insert an dummy NOP here to ensure all oop map PCs
1380     // are unique. See JDK-8237483.
1381     __ nop();
1382   }
1383   int pc_for_athrow_offset = __ offset();
1384   InternalAddress pc_for_athrow(__ pc());
1385   int32_t off = 0;
1386   __ la_patchable(exceptionPC->as_register(), pc_for_athrow, off);
1387   __ addi(exceptionPC->as_register(), exceptionPC->as_register(), off);
1388   add_call_info(pc_for_athrow_offset, info); // for exception handler
1389 
1390   __ verify_not_null_oop(x10);
1391   // search an exception handler (x10: exception oop, x13: throwing pc)
1392   if (compilation()->has_fpu_code()) {
1393     unwind_id = Runtime1::handle_exception_id;
1394   } else {
1395     unwind_id = Runtime1::handle_exception_nofpu_id;
1396   }
1397   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
1398   __ nop();
1399 }
1400 
1401 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1402   assert(exceptionOop->as_register() == x10, "must match");
1403   __ j(_unwind_handler_entry);
1404 }
1405 
1406 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
1407   Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1408   Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1409   Register count_reg = count->as_register();
1410   if (dest->is_single_cpu()) {
1411     assert (dest->type() == T_INT, "unexpected result type");
1412     assert (left->type() == T_INT, "unexpected left type");
1413     __ andi(t0, count_reg, 31); // should not shift more than 31 bits
1414     switch (code) {
1415       case lir_shl:  __ sllw(dest_reg, left_reg, t0); break;
1416       case lir_shr:  __ sraw(dest_reg, left_reg, t0); break;
1417       case lir_ushr: __ srlw(dest_reg, left_reg, t0); break;
1418       default: ShouldNotReachHere();
1419     }
1420   } else if (dest->is_double_cpu()) {
1421     __ andi(t0, count_reg, 63); // should not shift more than 63 bits
1422     switch (code) {
1423       case lir_shl:  __ sll(dest_reg, left_reg, t0); break;
1424       case lir_shr:  __ sra(dest_reg, left_reg, t0); break;
1425       case lir_ushr: __ srl(dest_reg, left_reg, t0); break;
1426       default: ShouldNotReachHere();
1427     }
1428   } else {
1429     ShouldNotReachHere();
1430   }
1431 }
1432 
1433 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
1434   Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1435   Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1436   if (dest->is_single_cpu()) {
1437     assert (dest->type() == T_INT, "unexpected result type");
1438     assert (left->type() == T_INT, "unexpected left type");
1439     count &= 0x1f;
1440     if (count != 0) {
1441       switch (code) {
1442         case lir_shl:  __ slliw(dest_reg, left_reg, count); break;
1443         case lir_shr:  __ sraiw(dest_reg, left_reg, count); break;
1444         case lir_ushr: __ srliw(dest_reg, left_reg, count); break;
1445         default: ShouldNotReachHere();
1446       }
1447     } else {
1448       move_regs(left_reg, dest_reg);
1449     }
1450   } else if (dest->is_double_cpu()) {
1451     count &= 0x3f;
1452     if (count != 0) {
1453       switch (code) {
1454         case lir_shl:  __ slli(dest_reg, left_reg, count); break;
1455         case lir_shr:  __ srai(dest_reg, left_reg, count); break;
1456         case lir_ushr: __ srli(dest_reg, left_reg, count); break;
1457         default: ShouldNotReachHere();
1458       }
1459     } else {
1460       move_regs(left->as_register_lo(), dest->as_register_lo());
1461     }
1462   } else {
1463     ShouldNotReachHere();
1464   }
1465 }
1466 
1467 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
1468   Register obj = op->obj_opr()->as_register();  // may not be an oop
1469   Register hdr = op->hdr_opr()->as_register();
1470   Register lock = op->lock_opr()->as_register();
1471   if (!UseFastLocking) {
1472     __ j(*op->stub()->entry());
1473   } else if (op->code() == lir_lock) {
1474     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1475     // add debug info for NullPointerException only if one is possible
1476     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
1477     if (op->info() != NULL) {
1478       add_debug_info_for_null_check(null_check_offset, op->info());
1479     }
1480   } else if (op->code() == lir_unlock) {
1481     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1482     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
1483   } else {
1484     Unimplemented();
1485   }
1486   __ bind(*op->stub()->continuation());
1487 }
1488 
1489 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
1490   ciMethod* method = op->profiled_method();
1491   int bci          = op->profiled_bci();
1492 
1493   // Update counter for all call types
1494   ciMethodData* md = method->method_data_or_null();
1495   guarantee(md != NULL, "Sanity");
1496   ciProfileData* data = md->bci_to_data(bci);
1497   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
1498   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
1499   Register mdo  = op->mdo()->as_register();
1500   __ mov_metadata(mdo, md->constant_encoding());
1501   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1502   // Perform additional virtual call profiling for invokevirtual and
1503   // invokeinterface bytecodes
1504   if (op->should_profile_receiver_type()) {
1505     assert(op->recv()->is_single_cpu(), "recv must be allocated");
1506     Register recv = op->recv()->as_register();
1507     assert_different_registers(mdo, recv);
1508     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
1509     ciKlass* known_klass = op->known_holder();
1510     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
1511       // We know the type that will be seen at this call site; we can
1512       // statically update the MethodData* rather than needing to do
1513       // dynamic tests on the receiver type
1514       // NOTE: we should probably put a lock around this search to
1515       // avoid collisions by concurrent compilations
1516       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
1517       uint i;
1518       for (i = 0; i < VirtualCallData::row_limit(); i++) {
1519         ciKlass* receiver = vc_data->receiver(i);
1520         if (known_klass->equals(receiver)) {
1521           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1522           __ add_memory_int64(data_addr, DataLayout::counter_increment);
1523           return;
1524         }
1525       }
1526 
1527       // Receiver type not found in profile data; select an empty slot
1528       // Note that this is less efficient than it should be because it
1529       // always does a write to the receiver part of the
1530       // VirtualCallData rather than just the first time
1531       for (i = 0; i < VirtualCallData::row_limit(); i++) {
1532         ciKlass* receiver = vc_data->receiver(i);
1533         if (receiver == NULL) {
1534           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
1535           __ mov_metadata(t1, known_klass->constant_encoding());
1536           __ sd(t1, recv_addr);
1537           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1538           __ add_memory_int64(data_addr, DataLayout::counter_increment);
1539           return;
1540         }
1541       }
1542     } else {
1543       __ load_klass(recv, recv);
1544       Label update_done;
1545       type_profile_helper(mdo, md, data, recv, &update_done);
1546       // Receiver did not match any saved receiver and there is no empty row for it.
1547       // Increment total counter to indicate polymorphic case.
1548       __ add_memory_int64(counter_addr, DataLayout::counter_increment);
1549 
1550       __ bind(update_done);
1551     }
1552   } else {
1553     // Static call
1554     __ add_memory_int64(counter_addr, DataLayout::counter_increment);
1555   }
1556 }
1557 
1558 void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); }
1559 
1560 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
1561   __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
1562 }
1563 
1564 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); }
1565 
1566 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
1567                                    Register tmp, Label &next, Label &none,
1568                                    Address mdo_addr) {
1569   if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
1570     if (exact_klass != NULL) {
1571       __ mov_metadata(tmp, exact_klass->constant_encoding());
1572     } else {
1573       __ load_klass(tmp, tmp);
1574     }
1575 
1576     __ ld(t1, mdo_addr);
1577     __ xorr(tmp, tmp, t1);
1578     __ andi(t0, tmp, TypeEntries::type_klass_mask);
1579     // klass seen before, nothing to do. The unknown bit may have been
1580     // set already but no need to check.
1581     __ beqz(t0, next);
1582 
1583     // already unknown. Nothing to do anymore.
1584     __ andi(t0, tmp, TypeEntries::type_unknown);
1585     __ bnez(t0, next);
1586 
1587     if (TypeEntries::is_type_none(current_klass)) {
1588       __ beqz(t1, none);
1589       __ li(t0, (u1)TypeEntries::null_seen);
1590       __ beq(t0, t1, none);
1591       // There is a chance that the checks above (re-reading profiling
1592       // data from memory) fail if another thread has just set the
1593       // profiling to this obj's klass
1594       __ membar(MacroAssembler::LoadLoad);
1595       __ ld(t1, mdo_addr);
1596       __ xorr(tmp, tmp, t1);
1597       __ andi(t0, tmp, TypeEntries::type_klass_mask);
1598       __ beqz(t0, next);
1599     }
1600   } else {
1601     assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1602            ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
1603 
1604     __ ld(tmp, mdo_addr);
1605     // already unknown. Nothing to do anymore.
1606     __ andi(t0, tmp, TypeEntries::type_unknown);
1607     __ bnez(t0, next);
1608   }
1609 
1610   // different than before. Cannot keep accurate profile.
1611   __ ld(t1, mdo_addr);
1612   __ ori(t1, t1, TypeEntries::type_unknown);
1613   __ sd(t1, mdo_addr);
1614 
1615   if (TypeEntries::is_type_none(current_klass)) {
1616     __ j(next);
1617 
1618     __ bind(none);
1619     // first time here. Set profile type.
1620     __ sd(tmp, mdo_addr);
1621   }
1622 }
1623 
1624 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp,
1625                                       Address mdo_addr, Label &next) {
1626   // There's a single possible klass at this profile point
1627   assert(exact_klass != NULL, "should be");
1628   if (TypeEntries::is_type_none(current_klass)) {
1629     __ mov_metadata(tmp, exact_klass->constant_encoding());
1630     __ ld(t1, mdo_addr);
1631     __ xorr(tmp, tmp, t1);
1632     __ andi(t0, tmp, TypeEntries::type_klass_mask);
1633     __ beqz(t0, next);
1634 #ifdef ASSERT
1635   {
1636     Label ok;
1637     __ ld(t0, mdo_addr);
1638     __ beqz(t0, ok);
1639     __ li(t1, (u1)TypeEntries::null_seen);
1640     __ beq(t0, t1, ok);
1641     // may have been set by another thread
1642     __ membar(MacroAssembler::LoadLoad);
1643     __ mov_metadata(t0, exact_klass->constant_encoding());
1644     __ ld(t1, mdo_addr);
1645     __ xorr(t1, t0, t1);
1646     __ andi(t1, t1, TypeEntries::type_mask);
1647     __ beqz(t1, ok);
1648 
1649     __ stop("unexpected profiling mismatch");
1650     __ bind(ok);
1651   }
1652 #endif
1653     // first time here. Set profile type.
1654     __ sd(tmp, mdo_addr);
1655   } else {
1656     assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1657            ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
1658 
1659     __ ld(tmp, mdo_addr);
1660     // already unknown. Nothing to do anymore.
1661     __ andi(t0, tmp, TypeEntries::type_unknown);
1662     __ bnez(t0, next);
1663 
1664     __ ori(tmp, tmp, TypeEntries::type_unknown);
1665     __ sd(tmp, mdo_addr);
1666   }
1667 }
1668 
1669 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass,
1670                                Address mdo_addr, bool do_update, Label &next) {
1671   __ bnez(tmp, update);
1672   if (!TypeEntries::was_null_seen(current_klass)) {
1673     __ ld(t1, mdo_addr);
1674     __ ori(t1, t1, TypeEntries::null_seen);
1675     __ sd(t1, mdo_addr);
1676   }
1677   if (do_update) {
1678     __ j(next);
1679   }
1680 }
1681 
1682 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
1683   COMMENT("emit_profile_type {");
1684   Register obj = op->obj()->as_register();
1685   Register tmp = op->tmp()->as_pointer_register();
1686   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
1687   ciKlass* exact_klass = op->exact_klass();
1688   intptr_t current_klass = op->current_klass();
1689   bool not_null = op->not_null();
1690   bool no_conflict = op->no_conflict();
1691 
1692   Label update, next, none;
1693 
1694   bool do_null = !not_null;
1695   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
1696   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
1697 
1698   assert(do_null || do_update, "why are we here?");
1699   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
1700   assert_different_registers(tmp, t0, t1, mdo_addr.base());
1701 
1702   __ verify_oop(obj);
1703 
1704   if (tmp != obj) {
1705     __ mv(tmp, obj);
1706   }
1707   if (do_null) {
1708     check_null(tmp, update, current_klass, mdo_addr, do_update, next);
1709 #ifdef ASSERT
1710   } else {
1711     __ bnez(tmp, update);
1712     __ stop("unexpected null obj");
1713 #endif
1714   }
1715 
1716   __ bind(update);
1717 
1718   if (do_update) {
1719 #ifdef ASSERT
1720     if (exact_klass != NULL) {
1721       check_exact_klass(tmp, exact_klass);
1722     }
1723 #endif
1724     if (!no_conflict) {
1725       check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr);
1726     } else {
1727       check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next);
1728     }
1729 
1730     __ bind(next);
1731   }
1732   COMMENT("} emit_profile_type");
1733 }
1734 
1735 void LIR_Assembler::align_backward_branch_target() { }
1736 
1737 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
1738   // tmp must be unused
1739   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
1740 
1741   if (left->is_single_cpu()) {
1742     assert(dest->is_single_cpu(), "expect single result reg");
1743     __ negw(dest->as_register(), left->as_register());
1744   } else if (left->is_double_cpu()) {
1745     assert(dest->is_double_cpu(), "expect double result reg");
1746     __ neg(dest->as_register_lo(), left->as_register_lo());
1747   } else if (left->is_single_fpu()) {
1748     assert(dest->is_single_fpu(), "expect single float result reg");
1749     __ fneg_s(dest->as_float_reg(), left->as_float_reg());
1750   } else {
1751     assert(left->is_double_fpu(), "expect double float operand reg");
1752     assert(dest->is_double_fpu(), "expect double float result reg");
1753     __ fneg_d(dest->as_double_reg(), left->as_double_reg());
1754   }
1755 }
1756 
1757 
1758 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1759   if (patch_code != lir_patch_none) {
1760     deoptimize_trap(info);
1761     return;
1762   }
1763 
1764   LIR_Address* adr = addr->as_address_ptr();
1765   Register dst = dest->as_register_lo();
1766 
1767   assert_different_registers(dst, t0);
1768   if(adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) {
1769 
1770     intptr_t offset = adr->disp();
1771     LIR_Opr index_op = adr->index();
1772     int scale = adr->scale();
1773     if(index_op->is_constant()) {
1774       offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale;
1775     }
1776 
1777     if(!is_imm_in_range(offset, 12, 0)) {
1778       __ la(t0, as_Address(adr));
1779       __ mv(dst, t0);
1780       return;
1781     }
1782   }
1783   __ la(dst, as_Address(adr));
1784 }
1785 
1786 
1787 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
1788   assert(!tmp->is_valid(), "don't need temporary");
1789 
1790   CodeBlob *cb = CodeCache::find_blob(dest);
1791   if (cb != NULL) {
1792     __ far_call(RuntimeAddress(dest));
1793   } else {
1794     int32_t offset = 0;
1795     __ la_patchable(t0, RuntimeAddress(dest), offset);
1796     __ jalr(x1, t0, offset);
1797   }
1798 
1799   if (info != NULL) {
1800     add_call_info_here(info);
1801   }
1802 }
1803 
1804 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
1805   if (dest->is_address() || src->is_address()) {
1806     move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false);
1807   } else {
1808     ShouldNotReachHere();
1809   }
1810 }
1811 
1812 #ifdef ASSERT
1813 // emit run-time assertion
1814 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
1815   assert(op->code() == lir_assert, "must be");
1816 
1817   Label ok;
1818   if (op->in_opr1()->is_valid()) {
1819     assert(op->in_opr2()->is_valid(), "both operands must be valid");
1820     bool is_unordered = false;
1821     LIR_Condition cond = op->condition();
1822     emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false,
1823                 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true);
1824   } else {
1825     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
1826     assert(op->condition() == lir_cond_always, "no other conditions allowed");
1827   }
1828 
1829   if (op->halt()) {
1830     const char* str = __ code_string(op->msg());
1831     __ stop(str);
1832   } else {
1833     breakpoint();
1834   }
1835   __ bind(ok);
1836 }
1837 #endif
1838 
1839 #ifndef PRODUCT
1840 #define COMMENT(x)   do { __ block_comment(x); } while (0)
1841 #else
1842 #define COMMENT(x)
1843 #endif
1844 
1845 void LIR_Assembler::membar() {
1846   COMMENT("membar");
1847   __ membar(MacroAssembler::AnyAny);
1848 }
1849 
1850 void LIR_Assembler::membar_acquire() {
1851   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1852 }
1853 
1854 void LIR_Assembler::membar_release() {
1855   __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1856 }
1857 
1858 void LIR_Assembler::membar_loadload() {
1859   __ membar(MacroAssembler::LoadLoad);
1860 }
1861 
1862 void LIR_Assembler::membar_storestore() {
1863   __ membar(MacroAssembler::StoreStore);
1864 }
1865 
1866 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
1867 
1868 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
1869 
1870 void LIR_Assembler::on_spin_wait() {
1871   Unimplemented();
1872 }
1873 
1874 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
1875   __ mv(result_reg->as_register(), xthread);
1876 }
1877 
1878 void LIR_Assembler::peephole(LIR_List *lir) {}
1879 
1880 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
1881   Address addr = as_Address(src->as_address_ptr());
1882   BasicType type = src->type();
1883   bool is_oop = is_reference_type(type);
1884 
1885   get_op(type);
1886 
1887   switch (code) {
1888     case lir_xadd:
1889       {
1890         RegisterOrConstant inc;
1891         Register tmp = as_reg(tmp_op);
1892         Register dst = as_reg(dest);
1893         if (data->is_constant()) {
1894           inc = RegisterOrConstant(as_long(data));
1895           assert_different_registers(dst, addr.base(), tmp);
1896           assert_different_registers(tmp, t0);
1897         } else {
1898           inc = RegisterOrConstant(as_reg(data));
1899           assert_different_registers(inc.as_register(), dst, addr.base(), tmp);
1900         }
1901         __ la(tmp, addr);
1902         (_masm->*add)(dst, inc, tmp);
1903         break;
1904       }
1905     case lir_xchg:
1906       {
1907         Register tmp = tmp_op->as_register();
1908         Register obj = as_reg(data);
1909         Register dst = as_reg(dest);
1910         if (is_oop && UseCompressedOops) {
1911           __ encode_heap_oop(t0, obj);
1912           obj = t0;
1913         }
1914         assert_different_registers(obj, addr.base(), tmp, dst);
1915         __ la(tmp, addr);
1916         (_masm->*xchg)(dst, obj, tmp);
1917         if (is_oop && UseCompressedOops) {
1918           __ decode_heap_oop(dst);
1919         }
1920       }
1921       break;
1922     default:
1923       ShouldNotReachHere();
1924   }
1925   __ membar(MacroAssembler::AnyAny);
1926 }
1927 
1928 int LIR_Assembler::array_element_size(BasicType type) const {
1929   int elem_size = type2aelembytes(type);
1930   return log2i_exact(elem_size);
1931 }
1932 
1933 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
1934   if (addr->base()->is_illegal()) {
1935     assert(addr->index()->is_illegal(), "must be illegal too");
1936     __ movptr(tmp, addr->disp());
1937     return Address(tmp, 0);
1938   }
1939 
1940   Register base = addr->base()->as_pointer_register();
1941   LIR_Opr index_op = addr->index();
1942   int scale = addr->scale();
1943 
1944   if (index_op->is_illegal()) {
1945     return Address(base, addr->disp());
1946   } else if (index_op->is_cpu_register()) {
1947     Register index;
1948     if (index_op->is_single_cpu()) {
1949       index = index_op->as_register();
1950     } else {
1951       index = index_op->as_register_lo();
1952     }
1953     if (scale != 0) {
1954       __ slli(tmp, index, scale);
1955       __ add(tmp, base, tmp);
1956     } else {
1957       __ add(tmp, base, index);
1958     }
1959     return Address(tmp, addr->disp());
1960   } else if (index_op->is_constant()) {
1961     intptr_t addr_offset = (((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale) + addr->disp();
1962     return Address(base, addr_offset);
1963   }
1964 
1965   Unimplemented();
1966   return Address();
1967 }
1968 
1969 // helper functions which checks for overflow and sets bailout if it
1970 // occurs.  Always returns a valid embeddable pointer but in the
1971 // bailout case the pointer won't be to unique storage.
1972 address LIR_Assembler::float_constant(float f) {
1973   address const_addr = __ float_constant(f);
1974   if (const_addr == NULL) {
1975     bailout("const section overflow");
1976     return __ code()->consts()->start();
1977   } else {
1978     return const_addr;
1979   }
1980 }
1981 
1982 address LIR_Assembler::double_constant(double d) {
1983   address const_addr = __ double_constant(d);
1984   if (const_addr == NULL) {
1985     bailout("const section overflow");
1986     return __ code()->consts()->start();
1987   } else {
1988     return const_addr;
1989   }
1990 }
1991 
1992 address LIR_Assembler::int_constant(jlong n) {
1993   address const_addr = __ long_constant(n);
1994   if (const_addr == NULL) {
1995     bailout("const section overflow");
1996     return __ code()->consts()->start();
1997   } else {
1998     return const_addr;
1999   }
2000 }
2001 
2002 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
2003   __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */,
2004              Assembler::rl /* release */, t0, true /* result as bool */);
2005   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2006   __ membar(MacroAssembler::AnyAny);
2007 }
2008 
2009 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) {
2010   __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */,
2011              Assembler::rl /* release */, t0, true /* result as bool */);
2012   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2013   __ membar(MacroAssembler::AnyAny);
2014 }
2015 
2016 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
2017   __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */,
2018              Assembler::rl /* release */, t0, true /* result as bool */);
2019   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2020   __ membar(MacroAssembler::AnyAny);
2021 }
2022 
2023 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
2024   address target = NULL;
2025 
2026   switch (patching_id(info)) {
2027     case PatchingStub::access_field_id:
2028       target = Runtime1::entry_for(Runtime1::access_field_patching_id);
2029       break;
2030     case PatchingStub::load_klass_id:
2031       target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
2032       break;
2033     case PatchingStub::load_mirror_id:
2034       target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
2035       break;
2036     case PatchingStub::load_appendix_id:
2037       target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
2038       break;
2039     default: ShouldNotReachHere();
2040   }
2041 
2042   __ far_call(RuntimeAddress(target));
2043   add_call_info_here(info);
2044 }
2045 
2046 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) {
2047   Label ok;
2048   __ load_klass(tmp, tmp);
2049   __ mov_metadata(t0, exact_klass->constant_encoding());
2050   __ beq(tmp, t0, ok);
2051   __ stop("exact klass and actual klass differ");
2052   __ bind(ok);
2053 }
2054 
2055 void LIR_Assembler::get_op(BasicType type) {
2056   switch (type) {
2057     case T_INT:
2058       xchg = &MacroAssembler::atomic_xchgalw;
2059       add = &MacroAssembler::atomic_addalw;
2060       break;
2061     case T_LONG:
2062       xchg = &MacroAssembler::atomic_xchgal;
2063       add = &MacroAssembler::atomic_addal;
2064       break;
2065     case T_OBJECT:
2066     case T_ARRAY:
2067       if (UseCompressedOops) {
2068         xchg = &MacroAssembler::atomic_xchgalwu;
2069         add = &MacroAssembler::atomic_addalw;
2070       } else {
2071         xchg = &MacroAssembler::atomic_xchgal;
2072         add = &MacroAssembler::atomic_addal;
2073       }
2074       break;
2075     default:
2076       ShouldNotReachHere();
2077   }
2078 }
2079 
2080 // emit_opTypeCheck sub functions
2081 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) {
2082   Register value = op->object()->as_register();
2083   Register array = op->array()->as_register();
2084   Register k_RInfo = op->tmp1()->as_register();
2085   Register klass_RInfo = op->tmp2()->as_register();
2086   Register Rtmp1 = op->tmp3()->as_register();
2087 
2088   CodeStub* stub = op->stub();
2089 
2090   // check if it needs to be profiled
2091   ciMethodData* md = NULL;
2092   ciProfileData* data = NULL;
2093 
2094   if (should_profile) {
2095     data_check(op, &md, &data);
2096   }
2097   Label profile_cast_success, profile_cast_failure, done;
2098   Label *success_target = should_profile ? &profile_cast_success : &done;
2099   Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
2100 
2101   if (should_profile) {
2102     profile_object(md, data, value, klass_RInfo, &done);
2103   } else {
2104     __ beqz(value, done);
2105   }
2106 
2107   add_debug_info_for_null_check_here(op->info_for_exception());
2108   __ load_klass(k_RInfo, array);
2109   __ load_klass(klass_RInfo, value);
2110 
2111   lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target);
2112 
2113   // fall through to the success case
2114   if (should_profile) {
2115     Register mdo = klass_RInfo;
2116     Register recv = k_RInfo;
2117     __ bind(profile_cast_success);
2118     __ mov_metadata(mdo, md->constant_encoding());
2119     __ load_klass(recv, value);
2120     type_profile_helper(mdo, md, data, recv, &done);
2121     __ j(done);
2122 
2123     __ bind(profile_cast_failure);
2124     __ mov_metadata(mdo, md->constant_encoding());
2125     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2126     __ ld(t1, counter_addr);
2127     __ addi(t1, t1, -DataLayout::counter_increment);
2128     __ sd(t1, counter_addr);
2129     __ j(*stub->entry());
2130   }
2131 
2132   __ bind(done);
2133 }
2134 
2135 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
2136   _masm->code_section()->relocate(adr, relocInfo::poll_type);
2137   int pc_offset = code_offset();
2138   flush_debug_info(pc_offset);
2139   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
2140   if (info->exception_handlers() != NULL) {
2141     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
2142   }
2143 }
2144 
2145 void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo,
2146                                  ciProfileData* data, Label* success, Label* failure,
2147                                  Label& profile_cast_success, Label& profile_cast_failure) {
2148   Register mdo = klass_RInfo;
2149   Register recv = k_RInfo;
2150   __ bind(profile_cast_success);
2151   __ mov_metadata(mdo, md->constant_encoding());
2152   __ load_klass(recv, obj);
2153   Label update_done;
2154   type_profile_helper(mdo, md, data, recv, success);
2155   __ j(*success);
2156 
2157   __ bind(profile_cast_failure);
2158   __ mov_metadata(mdo, md->constant_encoding());
2159   Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2160   __ ld(t0, counter_addr);
2161   __ addi(t0, t0, -DataLayout::counter_increment);
2162   __ sd(t0, counter_addr);
2163   __ j(*failure);
2164 }
2165 
2166 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1,
2167                                         Label* success_target, Label* failure_target) {
2168   // get instance klass (it's already uncompressed)
2169   __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2170   // perform the fast part of the checking logic
2171   __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2172   // call out-of-line instance of __ check_klass_subtype_slow_path(...)
2173   __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
2174   __ sd(klass_RInfo, Address(sp, wordSize));  // sub klass
2175   __ sd(k_RInfo, Address(sp, 0));             // super klass
2176   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2177   // load result to k_RInfo
2178   __ ld(k_RInfo, Address(sp, 0));
2179   __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
2180   // result is a boolean
2181   __ beqz(k_RInfo, *failure_target, /* is_far */ true);
2182 }
2183 
2184 void LIR_Assembler::const2reg_helper(LIR_Opr src) {
2185   switch (src->as_constant_ptr()->type()) {
2186     case T_INT:
2187     case T_ADDRESS:
2188     case T_OBJECT:
2189     case T_ARRAY:
2190     case T_METADATA:
2191         const2reg(src, FrameMap::t0_opr, lir_patch_none, NULL);
2192         break;
2193     case T_LONG:
2194         const2reg(src, FrameMap::t0_long_opr, lir_patch_none, NULL);
2195         break;
2196     case T_FLOAT:
2197     case T_DOUBLE:
2198     default:
2199       ShouldNotReachHere();
2200   }
2201 }
2202 
2203 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) {
2204   switch (code) {
2205     case lir_logic_and: __ andrw(dst, left, right); break;
2206     case lir_logic_or:  __ orrw (dst, left, right); break;
2207     case lir_logic_xor: __ xorrw(dst, left, right); break;
2208     default:            ShouldNotReachHere();
2209   }
2210 }
2211 
2212 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) {
2213   switch (code) {
2214     case lir_logic_and: __ andr(dst, left, right); break;
2215     case lir_logic_or:  __ orr (dst, left, right); break;
2216     case lir_logic_xor: __ xorr(dst, left, right); break;
2217     default:            ShouldNotReachHere();
2218   }
2219 }
2220 
2221 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) {
2222   switch (code) {
2223     case lir_logic_and: __ andi(dst, left, right); break;
2224     case lir_logic_or:  __ ori (dst, left, right); break;
2225     case lir_logic_xor: __ xori(dst, left, right); break;
2226     default:            ShouldNotReachHere();
2227   }
2228 }
2229 
2230 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2231   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2232   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2233   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2234   __ sd(r, Address(sp, offset_from_rsp_in_bytes));
2235 }
2236 
2237 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2238   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2239   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2240   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2241   __ li(t0, c);
2242   __ sd(t0, Address(sp, offset_from_rsp_in_bytes));
2243 }
2244 
2245 #undef __