1 /*
   2  * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "c1/c1_CodeStubs.hpp"
  31 #include "c1/c1_Compilation.hpp"
  32 #include "c1/c1_LIRAssembler.hpp"
  33 #include "c1/c1_MacroAssembler.hpp"
  34 #include "c1/c1_Runtime1.hpp"
  35 #include "c1/c1_ValueStack.hpp"
  36 #include "ci/ciArrayKlass.hpp"
  37 #include "ci/ciInstance.hpp"
  38 #include "code/compiledIC.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "nativeInst_riscv.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_riscv.inline.hpp"
  46 
  47 #ifndef PRODUCT
  48 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  49 #else
  50 #define COMMENT(x)
  51 #endif
  52 
  53 NEEDS_CLEANUP // remove this definitions ?
  54 const Register IC_Klass    = t1;    // where the IC klass is cached
  55 const Register SYNC_header = x10;   // synchronization header
  56 const Register SHIFT_count = x10;   // where count for shift operations must be
  57 
  58 #define __ _masm->
  59 
  60 static void select_different_registers(Register preserve,
  61                                        Register extra,
  62                                        Register &tmp1,
  63                                        Register &tmp2) {
  64   if (tmp1 == preserve) {
  65     assert_different_registers(tmp1, tmp2, extra);
  66     tmp1 = extra;
  67   } else if (tmp2 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp2 = extra;
  70   }
  71   assert_different_registers(preserve, tmp1, tmp2);
  72 }
  73 
  74 static void select_different_registers(Register preserve,
  75                                        Register extra,
  76                                        Register &tmp1,
  77                                        Register &tmp2,
  78                                        Register &tmp3) {
  79   if (tmp1 == preserve) {
  80     assert_different_registers(tmp1, tmp2, tmp3, extra);
  81     tmp1 = extra;
  82   } else if (tmp2 == preserve) {
  83     assert_different_registers(tmp1, tmp2, tmp3, extra);
  84     tmp2 = extra;
  85   } else if (tmp3 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp3 = extra;
  88   }
  89   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  90 }
  91 
  92 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  93 
  94 void LIR_Assembler::clinit_barrier(ciMethod* method) {
  95   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  96   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
  97 
  98   Label L_skip_barrier;
  99 
 100   __ mov_metadata(t1, method->holder()->constant_encoding());
 101   __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */);
 102   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 103   __ bind(L_skip_barrier);
 104 }
 105 
 106 LIR_Opr LIR_Assembler::receiverOpr() {
 107   return FrameMap::receiver_opr;
 108 }
 109 
 110 LIR_Opr LIR_Assembler::osrBufferPointer() {
 111   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 112 }
 113 
 114 void LIR_Assembler::breakpoint() { Unimplemented(); }
 115 
 116 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 117 
 118 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 119 
 120 static jlong as_long(LIR_Opr data) {
 121   jlong result;
 122   switch (data->type()) {
 123     case T_INT:
 124       result = (data->as_jint());
 125       break;
 126     case T_LONG:
 127       result = (data->as_jlong());
 128       break;
 129     default:
 130       ShouldNotReachHere();
 131       result = 0;  // unreachable
 132   }
 133   return result;
 134 }
 135 
 136 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 137   if (addr->base()->is_illegal()) {
 138     assert(addr->index()->is_illegal(), "must be illegal too");
 139     __ movptr(tmp, addr->disp());
 140     return Address(tmp, 0);
 141   }
 142 
 143   Register base = addr->base()->as_pointer_register();
 144   LIR_Opr index_opr = addr->index();
 145 
 146   if (index_opr->is_illegal()) {
 147     return Address(base, addr->disp());
 148   }
 149 
 150   int scale = addr->scale();
 151   if (index_opr->is_cpu_register()) {
 152     Register index;
 153     if (index_opr->is_single_cpu()) {
 154       index = index_opr->as_register();
 155     } else {
 156       index = index_opr->as_register_lo();
 157     }
 158     if (scale != 0) {
 159       __ shadd(tmp, index, base, tmp, scale);
 160     } else {
 161       __ add(tmp, base, index);
 162     }
 163     return Address(tmp, addr->disp());
 164   } else if (index_opr->is_constant()) {
 165     intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp();
 166     return Address(base, addr_offset);
 167   }
 168 
 169   Unimplemented();
 170   return Address();
 171 }
 172 
 173 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 174   ShouldNotReachHere();
 175   return Address();
 176 }
 177 
 178 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 179   return as_Address(addr, t0);
 180 }
 181 
 182 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 183   return as_Address(addr);
 184 }
 185 
 186 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 187 // not encodable as a base + (immediate) offset, generate an explicit address
 188 // calculation to hold the address in t0.
 189 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) {
 190   precond(size == 4 || size == 8);
 191   Address addr = frame_map()->address_for_slot(index, adjust);
 192   precond(addr.getMode() == Address::base_plus_offset);
 193   precond(addr.base() == sp);
 194   precond(addr.offset() > 0);
 195   uint mask = size - 1;
 196   assert((addr.offset() & mask) == 0, "scaled offsets only");
 197 
 198   return addr;
 199 }
 200 
 201 void LIR_Assembler::osr_entry() {
 202   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 203   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 204   guarantee(osr_entry != NULL, "NULL osr_entry!");
 205   ValueStack* entry_state = osr_entry->state();
 206   int number_of_locks = entry_state->locks_size();
 207 
 208   // we jump here if osr happens with the interpreter
 209   // state set up to continue at the beginning of the
 210   // loop that triggered osr - in particular, we have
 211   // the following registers setup:
 212   //
 213   // x12: osr buffer
 214   //
 215 
 216   //build frame
 217   ciMethod* m = compilation()->method();
 218   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 219 
 220   // OSR buffer is
 221   //
 222   // locals[nlocals-1..0]
 223   // monitors[0..number_of_locks]
 224   //
 225   // locals is a direct copy of the interpreter frame so in the osr buffer
 226   // so first slot in the local array is the last local from the interpreter
 227   // and last slot is local[0] (receiver) from the interpreter
 228   //
 229   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 230   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 231   // in the interpreter frame (the method lock if a sync method)
 232 
 233   // Initialize monitors in the compiled activation.
 234   //   x12: pointer to osr buffer
 235   // All other registers are dead at this point and the locals will be
 236   // copied into place by code emitted in the IR.
 237 
 238   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 239   {
 240     assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 241     int monitor_offset = BytesPerWord * method()->max_locals() +
 242       BytesPerWord * (number_of_locks - 1);
 243     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 244     // the OSR buffer using 2 word entries: first the lock and then
 245     // the oop.
 246     for (int i = 0; i < number_of_locks; i++) {
 247       int slot_offset = monitor_offset - (i * BytesPerWord);
 248 #ifdef ASSERT
 249       // verify the interpreter's monitor has a non-null object
 250       {
 251         Label L;
 252         __ ld(t0, Address(OSR_buf, slot_offset));
 253         __ bnez(t0, L);
 254         __ stop("locked object is NULL");
 255         __ bind(L);
 256       }
 257 #endif // ASSERT
 258       __ ld(x9, Address(OSR_buf, slot_offset));
 259       __ sd(x9, frame_map()->address_for_monitor_object(i));
 260     }
 261   }
 262 }
 263 
 264 // inline cache check; done before the frame is built.
 265 int LIR_Assembler::check_icache() {
 266   Register receiver = FrameMap::receiver_opr->as_register();
 267   Register ic_klass = IC_Klass;
 268   int start_offset = __ offset();
 269   Label dont;
 270   __ inline_cache_check(receiver, ic_klass, dont);
 271 
 272   // if icache check fails, then jump to runtime routine
 273   // Note: RECEIVER must still contain the receiver!
 274   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 275 
 276   // We align the verified entry point unless the method body
 277   // (including its inline cache check) will fit in a single 64-byte
 278   // icache line.
 279   if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 280     // force alignment after the cache check.
 281     __ align(CodeEntryAlignment);
 282   }
 283 
 284   __ bind(dont);
 285   return start_offset;
 286 }
 287 
 288 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 289   if (o == NULL) {
 290     __ mv(reg, zr);
 291   } else {
 292     __ movoop(reg, o);
 293   }
 294 }
 295 
 296 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 297   deoptimize_trap(info);
 298 }
 299 
 300 // This specifies the rsp decrement needed to build the frame
 301 int LIR_Assembler::initial_frame_size_in_bytes() const {
 302   // if rounding, must let FrameMap know!
 303 
 304   return in_bytes(frame_map()->framesize_in_bytes());
 305 }
 306 
 307 int LIR_Assembler::emit_exception_handler() {
 308   // generate code for exception handler
 309   address handler_base = __ start_a_stub(exception_handler_size());
 310   if (handler_base == NULL) {
 311     // not enough space left for the handler
 312     bailout("exception handler overflow");
 313     return -1;
 314   }
 315 
 316   int offset = code_offset();
 317 
 318   // the exception oop and pc are in x10, and x13
 319   // no other registers need to be preserved, so invalidate them
 320   __ invalidate_registers(false, true, true, false, true, true);
 321 
 322   // check that there is really an exception
 323   __ verify_not_null_oop(x10);
 324 
 325   // search an exception handler (x10: exception oop, x13: throwing pc)
 326   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 327   __ should_not_reach_here();
 328   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 329   __ end_a_stub();
 330 
 331   return offset;
 332 }
 333 
 334 // Emit the code to remove the frame from the stack in the exception
 335 // unwind path.
 336 int LIR_Assembler::emit_unwind_handler() {
 337 #ifndef PRODUCT
 338   if (CommentedAssembly) {
 339     _masm->block_comment("Unwind handler");
 340   }
 341 #endif // PRODUCT
 342 
 343   int offset = code_offset();
 344 
 345   // Fetch the exception from TLS and clear out exception related thread state
 346   __ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
 347   __ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
 348   __ sd(zr, Address(xthread, JavaThread::exception_pc_offset()));
 349 
 350   __ bind(_unwind_handler_entry);
 351   __ verify_not_null_oop(x10);
 352   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 353     __ mv(x9, x10);   // Preserve the exception
 354   }
 355 
 356   // Perform needed unlocking
 357   MonitorExitStub* stub = NULL;
 358   if (method()->is_synchronized()) {
 359     monitor_address(0, FrameMap::r10_opr);
 360     __ ld(x14, Address(x10, BasicObjectLock::obj_offset_in_bytes()));
 361     stub = new MonitorExitStub(FrameMap::r14_opr);
 362     if (UseHeavyMonitors) {
 363       __ j(*stub->entry());
 364     } else {
 365       __ unlock_object(x15, x14, x10, *stub->entry());
 366     }
 367     __ bind(*stub->continuation());
 368   }
 369 
 370   if (compilation()->env()->dtrace_method_probes()) {
 371     __ mv(c_rarg0, xthread);
 372     __ mov_metadata(c_rarg1, method()->constant_encoding());
 373     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 374   }
 375 
 376   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 377     __ mv(x10, x9);   // Restore the exception
 378   }
 379 
 380   // remove the activation and dispatch to the unwind handler
 381   __ block_comment("remove_frame and dispatch to the unwind handler");
 382   __ remove_frame(initial_frame_size_in_bytes());
 383   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 384 
 385   // Emit the slow path assembly
 386   if (stub != NULL) {
 387     stub->emit_code(this);
 388   }
 389 
 390   return offset;
 391 }
 392 
 393 int LIR_Assembler::emit_deopt_handler() {
 394   // generate code for exception handler
 395   address handler_base = __ start_a_stub(deopt_handler_size());
 396   if (handler_base == NULL) {
 397     // not enough space left for the handler
 398     bailout("deopt handler overflow");
 399     return -1;
 400   }
 401 
 402   int offset = code_offset();
 403 
 404   __ auipc(ra, 0);
 405   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 406   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 407   __ end_a_stub();
 408 
 409   return offset;
 410 }
 411 
 412 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 413   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
 414 
 415   // Pop the stack before the safepoint code
 416   __ remove_frame(initial_frame_size_in_bytes());
 417 
 418   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 419     __ reserved_stack_check();
 420   }
 421 
 422   code_stub->set_safepoint_offset(__ offset());
 423   __ relocate(relocInfo::poll_return_type);
 424   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 425   __ ret();
 426 }
 427 
 428 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 429   guarantee(info != NULL, "Shouldn't be NULL");
 430   __ get_polling_page(t0, relocInfo::poll_type);
 431   add_debug_info_for_branch(info);  // This isn't just debug info:
 432                                     // it's the oop map
 433   __ read_polling_page(t0, 0, relocInfo::poll_type);
 434   return __ offset();
 435 }
 436 
 437 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 438   __ mv(to_reg, from_reg);
 439 }
 440 
 441 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 442 
 443 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 444   assert(src->is_constant(), "should not call otherwise");
 445   assert(dest->is_register(), "should not call otherwise");
 446   LIR_Const* c = src->as_constant_ptr();
 447   address const_addr = NULL;
 448 
 449   switch (c->type()) {
 450     case T_INT:
 451       assert(patch_code == lir_patch_none, "no patching handled here");
 452       __ mvw(dest->as_register(), c->as_jint());
 453       break;
 454 
 455     case T_ADDRESS:
 456       assert(patch_code == lir_patch_none, "no patching handled here");
 457       __ mv(dest->as_register(), c->as_jint());
 458       break;
 459 
 460     case T_LONG:
 461       assert(patch_code == lir_patch_none, "no patching handled here");
 462       __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
 463       break;
 464 
 465     case T_OBJECT:
 466     case T_ARRAY:
 467       if (patch_code == lir_patch_none) {
 468         jobject2reg(c->as_jobject(), dest->as_register());
 469       } else {
 470         jobject2reg_with_patching(dest->as_register(), info);
 471       }
 472       break;
 473 
 474     case T_METADATA:
 475       if (patch_code != lir_patch_none) {
 476         klass2reg_with_patching(dest->as_register(), info);
 477       } else {
 478         __ mov_metadata(dest->as_register(), c->as_metadata());
 479       }
 480       break;
 481 
 482     case T_FLOAT:
 483       const_addr = float_constant(c->as_jfloat());
 484       assert(const_addr != NULL, "must create float constant in the constant table");
 485       __ flw(dest->as_float_reg(), InternalAddress(const_addr));
 486       break;
 487 
 488     case T_DOUBLE:
 489       const_addr = double_constant(c->as_jdouble());
 490       assert(const_addr != NULL, "must create double constant in the constant table");
 491       __ fld(dest->as_double_reg(), InternalAddress(const_addr));
 492       break;
 493 
 494     default:
 495       ShouldNotReachHere();
 496   }
 497 }
 498 
 499 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 500   assert(src->is_constant(), "should not call otherwise");
 501   assert(dest->is_stack(), "should not call otherwise");
 502   LIR_Const* c = src->as_constant_ptr();
 503   switch (c->type()) {
 504     case T_OBJECT:
 505       if (c->as_jobject() == NULL) {
 506         __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 507       } else {
 508         const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
 509         reg2stack(FrameMap::t1_opr, dest, c->type(), false);
 510       }
 511       break;
 512     case T_ADDRESS:   // fall through
 513       const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
 514       reg2stack(FrameMap::t1_opr, dest, c->type(), false);
 515     case T_INT:       // fall through
 516     case T_FLOAT:
 517       if (c->as_jint_bits() == 0) {
 518         __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 519       } else {
 520         __ mvw(t1, c->as_jint_bits());
 521         __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix()));
 522       }
 523       break;
 524     case T_LONG:      // fall through
 525     case T_DOUBLE:
 526       if (c->as_jlong_bits() == 0) {
 527         __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 528                                                 lo_word_offset_in_bytes));
 529       } else {
 530         __ mv(t1, (intptr_t)c->as_jlong_bits());
 531         __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(),
 532                                                 lo_word_offset_in_bytes));
 533       }
 534       break;
 535     default:
 536       ShouldNotReachHere();
 537   }
 538 }
 539 
 540 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 541   assert(src->is_constant(), "should not call otherwise");
 542   assert(dest->is_address(), "should not call otherwise");
 543   LIR_Const* c = src->as_constant_ptr();
 544   LIR_Address* to_addr = dest->as_address_ptr();
 545   void (Assembler::* insn)(Register Rt, const Address &adr, Register temp);
 546   switch (type) {
 547     case T_ADDRESS:
 548       assert(c->as_jint() == 0, "should be");
 549       insn = &Assembler::sd; break;
 550     case T_LONG:
 551       assert(c->as_jlong() == 0, "should be");
 552       insn = &Assembler::sd; break;
 553     case T_DOUBLE:
 554       assert(c->as_jdouble() == 0.0, "should be");
 555       insn = &Assembler::sd; break;
 556     case T_INT:
 557       assert(c->as_jint() == 0, "should be");
 558       insn = &Assembler::sw; break;
 559     case T_FLOAT:
 560       assert(c->as_jfloat() == 0.0f, "should be");
 561       insn = &Assembler::sw; break;
 562     case T_OBJECT:    // fall through
 563     case T_ARRAY:
 564       assert(c->as_jobject() == 0, "should be");
 565       if (UseCompressedOops && !wide) {
 566         insn = &Assembler::sw;
 567       } else {
 568         insn = &Assembler::sd;
 569       }
 570       break;
 571     case T_CHAR:      // fall through
 572     case T_SHORT:
 573       assert(c->as_jint() == 0, "should be");
 574       insn = &Assembler::sh;
 575       break;
 576     case T_BOOLEAN:   // fall through
 577     case T_BYTE:
 578       assert(c->as_jint() == 0, "should be");
 579       insn = &Assembler::sb; break;
 580     default:
 581       ShouldNotReachHere();
 582       insn = &Assembler::sd;  // unreachable
 583   }
 584   if (info != NULL) {
 585     add_debug_info_for_null_check_here(info);
 586   }
 587   (_masm->*insn)(zr, as_Address(to_addr), t0);
 588 }
 589 
 590 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 591   assert(src->is_register(), "should not call otherwise");
 592   assert(dest->is_register(), "should not call otherwise");
 593 
 594   // move between cpu-registers
 595   if (dest->is_single_cpu()) {
 596     if (src->type() == T_LONG) {
 597       // Can do LONG -> OBJECT
 598       move_regs(src->as_register_lo(), dest->as_register());
 599       return;
 600     }
 601     assert(src->is_single_cpu(), "must match");
 602     if (src->type() == T_OBJECT) {
 603       __ verify_oop(src->as_register());
 604     }
 605     move_regs(src->as_register(), dest->as_register());
 606   } else if (dest->is_double_cpu()) {
 607     if (is_reference_type(src->type())) {
 608       __ verify_oop(src->as_register());
 609       move_regs(src->as_register(), dest->as_register_lo());
 610       return;
 611     }
 612     assert(src->is_double_cpu(), "must match");
 613     Register f_lo = src->as_register_lo();
 614     Register f_hi = src->as_register_hi();
 615     Register t_lo = dest->as_register_lo();
 616     Register t_hi = dest->as_register_hi();
 617     assert(f_hi == f_lo, "must be same");
 618     assert(t_hi == t_lo, "must be same");
 619     move_regs(f_lo, t_lo);
 620   } else if (dest->is_single_fpu()) {
 621     assert(src->is_single_fpu(), "expect single fpu");
 622     __ fmv_s(dest->as_float_reg(), src->as_float_reg());
 623   } else if (dest->is_double_fpu()) {
 624     assert(src->is_double_fpu(), "expect double fpu");
 625     __ fmv_d(dest->as_double_reg(), src->as_double_reg());
 626   } else {
 627     ShouldNotReachHere();
 628   }
 629 }
 630 
 631 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 632   precond(src->is_register() && dest->is_stack());
 633 
 634   uint const c_sz32 = sizeof(uint32_t);
 635   uint const c_sz64 = sizeof(uint64_t);
 636 
 637   assert(src->is_register(), "should not call otherwise");
 638   assert(dest->is_stack(), "should not call otherwise");
 639   if (src->is_single_cpu()) {
 640     int index = dest->single_stack_ix();
 641     if (is_reference_type(type)) {
 642       __ sd(src->as_register(), stack_slot_address(index, c_sz64));
 643       __ verify_oop(src->as_register());
 644     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 645       __ sd(src->as_register(), stack_slot_address(index, c_sz64));
 646     } else {
 647       __ sw(src->as_register(), stack_slot_address(index, c_sz32));
 648     }
 649   } else if (src->is_double_cpu()) {
 650     int index = dest->double_stack_ix();
 651     Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
 652     __ sd(src->as_register_lo(), dest_addr_LO);
 653   } else if (src->is_single_fpu()) {
 654     int index = dest->single_stack_ix();
 655     __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32));
 656   } else if (src->is_double_fpu()) {
 657     int index = dest->double_stack_ix();
 658     __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64));
 659   } else {
 660     ShouldNotReachHere();
 661   }
 662 }
 663 
 664 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 665   LIR_Address* to_addr = dest->as_address_ptr();
 666   // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src
 667   Register compressed_src = t1;
 668 
 669   if (patch_code != lir_patch_none) {
 670     deoptimize_trap(info);
 671     return;
 672   }
 673 
 674   if (is_reference_type(type)) {
 675     __ verify_oop(src->as_register());
 676 
 677     if (UseCompressedOops && !wide) {
 678       __ encode_heap_oop(compressed_src, src->as_register());
 679     } else {
 680       compressed_src = src->as_register();
 681     }
 682   }
 683 
 684   int null_check_here = code_offset();
 685 
 686   switch (type) {
 687     case T_FLOAT:
 688       __ fsw(src->as_float_reg(), as_Address(to_addr));
 689       break;
 690 
 691     case T_DOUBLE:
 692       __ fsd(src->as_double_reg(), as_Address(to_addr));
 693       break;
 694 
 695     case T_ARRAY:      // fall through
 696     case T_OBJECT:
 697       if (UseCompressedOops && !wide) {
 698         __ sw(compressed_src, as_Address(to_addr));
 699       } else {
 700         __ sd(compressed_src, as_Address(to_addr));
 701       }
 702       break;
 703     case T_METADATA:
 704       // We get here to store a method pointer to the stack to pass to
 705       // a dtrace runtime call. This can't work on 64 bit with
 706       // compressed klass ptrs: T_METADATA can be compressed klass
 707       // ptr or a 64 bit method pointer.
 708       ShouldNotReachHere();
 709       __ sd(src->as_register(), as_Address(to_addr));
 710       break;
 711     case T_ADDRESS:
 712       __ sd(src->as_register(), as_Address(to_addr));
 713       break;
 714     case T_INT:
 715       __ sw(src->as_register(), as_Address(to_addr));
 716       break;
 717     case T_LONG:
 718       __ sd(src->as_register_lo(), as_Address(to_addr));
 719       break;
 720     case T_BYTE:    // fall through
 721     case T_BOOLEAN:
 722       __ sb(src->as_register(), as_Address(to_addr));
 723       break;
 724     case T_CHAR:    // fall through
 725     case T_SHORT:
 726       __ sh(src->as_register(), as_Address(to_addr));
 727       break;
 728     default:
 729       ShouldNotReachHere();
 730   }
 731 
 732   if (info != NULL) {
 733     add_debug_info_for_null_check(null_check_here, info);
 734   }
 735 }
 736 
 737 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 738   precond(src->is_stack() && dest->is_register());
 739 
 740   uint const c_sz32 = sizeof(uint32_t);
 741   uint const c_sz64 = sizeof(uint64_t);
 742 
 743   if (dest->is_single_cpu()) {
 744     int index = src->single_stack_ix();
 745     if (type == T_INT) {
 746       __ lw(dest->as_register(), stack_slot_address(index, c_sz32));
 747     } else if (is_reference_type(type)) {
 748       __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
 749       __ verify_oop(dest->as_register());
 750     } else if (type == T_METADATA || type == T_ADDRESS) {
 751       __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
 752     } else {
 753       __ lwu(dest->as_register(), stack_slot_address(index, c_sz32));
 754     }
 755   } else if (dest->is_double_cpu()) {
 756     int index = src->double_stack_ix();
 757     Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
 758     __ ld(dest->as_register_lo(), src_addr_LO);
 759   } else if (dest->is_single_fpu()) {
 760     int index = src->single_stack_ix();
 761     __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32));
 762   } else if (dest->is_double_fpu()) {
 763     int index = src->double_stack_ix();
 764     __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64));
 765   } else {
 766     ShouldNotReachHere();
 767   }
 768 }
 769 
 770 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 771   deoptimize_trap(info);
 772 }
 773 
 774 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 775   LIR_Opr temp;
 776   if (type == T_LONG || type == T_DOUBLE) {
 777     temp = FrameMap::t1_long_opr;
 778   } else {
 779     temp = FrameMap::t1_opr;
 780   }
 781 
 782   stack2reg(src, temp, src->type());
 783   reg2stack(temp, dest, dest->type(), false);
 784 }
 785 
 786 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 787   assert(src->is_address(), "should not call otherwise");
 788   assert(dest->is_register(), "should not call otherwise");
 789 
 790   LIR_Address* addr = src->as_address_ptr();
 791   LIR_Address* from_addr = src->as_address_ptr();
 792 
 793   if (addr->base()->type() == T_OBJECT) {
 794     __ verify_oop(addr->base()->as_pointer_register());
 795   }
 796 
 797   if (patch_code != lir_patch_none) {
 798     deoptimize_trap(info);
 799     return;
 800   }
 801 
 802   if (info != NULL) {
 803     add_debug_info_for_null_check_here(info);
 804   }
 805 
 806   int null_check_here = code_offset();
 807   switch (type) {
 808     case T_FLOAT:
 809       __ flw(dest->as_float_reg(), as_Address(from_addr));
 810       break;
 811     case T_DOUBLE:
 812       __ fld(dest->as_double_reg(), as_Address(from_addr));
 813       break;
 814     case T_ARRAY:     // fall through
 815     case T_OBJECT:
 816       if (UseCompressedOops && !wide) {
 817         __ lwu(dest->as_register(), as_Address(from_addr));
 818       } else {
 819         __ ld(dest->as_register(), as_Address(from_addr));
 820       }
 821       break;
 822     case T_METADATA:
 823       // We get here to store a method pointer to the stack to pass to
 824       // a dtrace runtime call. This can't work on 64 bit with
 825       // compressed klass ptrs: T_METADATA can be a compressed klass
 826       // ptr or a 64 bit method pointer.
 827       ShouldNotReachHere();
 828       __ ld(dest->as_register(), as_Address(from_addr));
 829       break;
 830     case T_ADDRESS:
 831       __ ld(dest->as_register(), as_Address(from_addr));
 832       break;
 833     case T_INT:
 834       __ lw(dest->as_register(), as_Address(from_addr));
 835       break;
 836     case T_LONG:
 837       __ ld(dest->as_register_lo(), as_Address_lo(from_addr));
 838       break;
 839     case T_BYTE:
 840       __ lb(dest->as_register(), as_Address(from_addr));
 841       break;
 842     case T_BOOLEAN:
 843       __ lbu(dest->as_register(), as_Address(from_addr));
 844       break;
 845     case T_CHAR:
 846       __ lhu(dest->as_register(), as_Address(from_addr));
 847       break;
 848     case T_SHORT:
 849       __ lh(dest->as_register(), as_Address(from_addr));
 850       break;
 851     default:
 852       ShouldNotReachHere();
 853   }
 854 
 855   if (is_reference_type(type)) {
 856     if (UseCompressedOops && !wide) {
 857       __ decode_heap_oop(dest->as_register());
 858     }
 859 
 860     if (!UseZGC) {
 861       // Load barrier has not yet been applied, so ZGC can't verify the oop here
 862       __ verify_oop(dest->as_register());
 863     }
 864   }
 865 }
 866 
 867 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 868   switch (op->code()) {
 869     case lir_idiv: // fall through
 870     case lir_irem:
 871       arithmetic_idiv(op->code(),
 872                       op->in_opr1(),
 873                       op->in_opr2(),
 874                       op->in_opr3(),
 875                       op->result_opr(),
 876                       op->info());
 877       break;
 878     case lir_fmad:
 879       __ fmadd_d(op->result_opr()->as_double_reg(),
 880                  op->in_opr1()->as_double_reg(),
 881                  op->in_opr2()->as_double_reg(),
 882                  op->in_opr3()->as_double_reg());
 883       break;
 884     case lir_fmaf:
 885       __ fmadd_s(op->result_opr()->as_float_reg(),
 886                  op->in_opr1()->as_float_reg(),
 887                  op->in_opr2()->as_float_reg(),
 888                  op->in_opr3()->as_float_reg());
 889       break;
 890     default:
 891       ShouldNotReachHere();
 892   }
 893 }
 894 
 895 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
 896                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
 897   Label label;
 898 
 899   emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false,
 900               /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true);
 901 
 902   Label done;
 903   move_op(opr2, result, type, lir_patch_none, NULL,
 904           false,   // pop_fpu_stack
 905           false);  // wide
 906   __ j(done);
 907   __ bind(label);
 908   move_op(opr1, result, type, lir_patch_none, NULL,
 909           false,   // pop_fpu_stack
 910           false);  // wide
 911   __ bind(done);
 912 }
 913 
 914 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 915   LIR_Condition condition = op->cond();
 916   if (condition == lir_cond_always) {
 917     if (op->info() != NULL) {
 918       add_debug_info_for_branch(op->info());
 919     }
 920   } else {
 921     assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands");
 922   }
 923   bool is_unordered = (op->ublock() == op->block());
 924   emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered);
 925 }
 926 
 927 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label,
 928                                 bool is_far, bool is_unordered) {
 929 
 930   if (cmp_flag == lir_cond_always) {
 931     __ j(label);
 932     return;
 933   }
 934 
 935   if (cmp1->is_cpu_register()) {
 936     Register reg1 = as_reg(cmp1);
 937     if (cmp2->is_cpu_register()) {
 938       Register reg2 = as_reg(cmp2);
 939       __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far);
 940     } else if (cmp2->is_constant()) {
 941       const2reg_helper(cmp2);
 942       __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far);
 943     } else {
 944       ShouldNotReachHere();
 945     }
 946   } else if (cmp1->is_single_fpu()) {
 947     assert(cmp2->is_single_fpu(), "expect single float register");
 948     __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered);
 949   } else if (cmp1->is_double_fpu()) {
 950     assert(cmp2->is_double_fpu(), "expect double float register");
 951     __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask,
 952                            cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered);
 953   } else {
 954     ShouldNotReachHere();
 955   }
 956 }
 957 
 958 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 959   LIR_Opr src  = op->in_opr();
 960   LIR_Opr dest = op->result_opr();
 961 
 962   switch (op->bytecode()) {
 963     case Bytecodes::_i2f:
 964       __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break;
 965     case Bytecodes::_i2d:
 966       __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break;
 967     case Bytecodes::_l2d:
 968       __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break;
 969     case Bytecodes::_l2f:
 970       __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break;
 971     case Bytecodes::_f2d:
 972       __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break;
 973     case Bytecodes::_d2f:
 974       __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break;
 975     case Bytecodes::_i2c:
 976       __ zero_extend(dest->as_register(), src->as_register(), 16); break;
 977     case Bytecodes::_i2l:
 978       __ addw(dest->as_register_lo(), src->as_register(), zr); break;
 979     case Bytecodes::_i2s:
 980       __ sign_extend(dest->as_register(), src->as_register(), 16); break;
 981     case Bytecodes::_i2b:
 982       __ sign_extend(dest->as_register(), src->as_register(), 8); break;
 983     case Bytecodes::_l2i:
 984       _masm->block_comment("FIXME: This coulde be no-op");
 985       __ addw(dest->as_register(), src->as_register_lo(), zr); break;
 986     case Bytecodes::_d2l:
 987       __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break;
 988     case Bytecodes::_f2i:
 989       __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break;
 990     case Bytecodes::_f2l:
 991       __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break;
 992     case Bytecodes::_d2i:
 993       __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break;
 994     default:
 995       ShouldNotReachHere();
 996   }
 997 }
 998 
 999 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1000   if (op->init_check()) {
1001     __ lbu(t0, Address(op->klass()->as_register(),
1002                        InstanceKlass::init_state_offset()));
1003     __ mvw(t1, InstanceKlass::fully_initialized);
1004     add_debug_info_for_null_check_here(op->stub()->info());
1005     __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
1006   }
1007 
1008   __ allocate_object(op->obj()->as_register(),
1009                      op->tmp1()->as_register(),
1010                      op->tmp2()->as_register(),
1011                      op->header_size(),
1012                      op->object_size(),
1013                      op->klass()->as_register(),
1014                      *op->stub()->entry());
1015 
1016   __ bind(*op->stub()->continuation());
1017 }
1018 
1019 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1020   Register len = op->len()->as_register();
1021 
1022   if (UseSlowPath ||
1023       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1024       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1025     __ j(*op->stub()->entry());
1026   } else {
1027     Register tmp1 = op->tmp1()->as_register();
1028     Register tmp2 = op->tmp2()->as_register();
1029     Register tmp3 = op->tmp3()->as_register();
1030     if (len == tmp1) {
1031       tmp1 = tmp3;
1032     } else if (len == tmp2) {
1033       tmp2 = tmp3;
1034     } else if (len == tmp3) {
1035       // everything is ok
1036     } else {
1037       __ mv(tmp3, len);
1038     }
1039     __ allocate_array(op->obj()->as_register(),
1040                       len,
1041                       tmp1,
1042                       tmp2,
1043                       arrayOopDesc::header_size(op->type()),
1044                       array_element_size(op->type()),
1045                       op->klass()->as_register(),
1046                       *op->stub()->entry());
1047   }
1048   __ bind(*op->stub()->continuation());
1049 }
1050 
1051 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
1052                                         Register recv, Label* update_done) {
1053   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1054     Label next_test;
1055     // See if the receiver is receiver[n].
1056     __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1057     __ bne(recv, t1, next_test);
1058     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1059     __ increment(data_addr, DataLayout::counter_increment);
1060     __ j(*update_done);
1061     __ bind(next_test);
1062   }
1063 
1064   // Didn't find receiver; find next empty slot and fill it in
1065   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1066     Label next_test;
1067     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1068     __ ld(t1, recv_addr);
1069     __ bnez(t1, next_test);
1070     __ sd(recv, recv_addr);
1071     __ mv(t1, DataLayout::counter_increment);
1072     __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1073     __ j(*update_done);
1074     __ bind(next_test);
1075   }
1076 }
1077 
1078 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
1079   ciMethod* method = op->profiled_method();
1080   assert(method != NULL, "Should have method");
1081   int bci = op->profiled_bci();
1082   *md = method->method_data_or_null();
1083   guarantee(*md != NULL, "Sanity");
1084   *data = ((*md)->bci_to_data(bci));
1085   assert(*data != NULL, "need data for type check");
1086   assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1087 }
1088 
1089 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1090                                                Register k_RInfo, Register klass_RInfo,
1091                                                Label *failure_target, Label *success_target) {
1092   // get object class
1093   // not a safepoint as obj null check happens earlier
1094   __ load_klass(klass_RInfo, obj);
1095   if (k->is_loaded()) {
1096     // See if we get an immediate positive hit
1097     __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1098     if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1099       __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1100       // successful cast, fall through to profile or jump
1101     } else {
1102       // See if we get an immediate positive hit
1103       __ beq(k_RInfo, t0, *success_target);
1104       // check for self
1105       __ beq(klass_RInfo, k_RInfo, *success_target);
1106 
1107       __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1108       __ sd(k_RInfo, Address(sp, 0));             // sub klass
1109       __ sd(klass_RInfo, Address(sp, wordSize));  // super klass
1110       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1111       // load result to k_RInfo
1112       __ ld(k_RInfo, Address(sp, 0));
1113       __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1114       // result is a boolean
1115       __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1116       // successful cast, fall through to profile or jump
1117     }
1118   } else {
1119     // perform the fast part of the checking logic
1120     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1121     // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1122     __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1123     __ sd(klass_RInfo, Address(sp, wordSize));  // sub klass
1124     __ sd(k_RInfo, Address(sp, 0));             // super klass
1125     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1126     // load result to k_RInfo
1127     __ ld(k_RInfo, Address(sp, 0));
1128     __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1129     // result is a boolean
1130     __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1131     // successful cast, fall thriugh to profile or jump
1132   }
1133 }
1134 
1135 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj,
1136                                    Register klass_RInfo, Label* obj_is_null) {
1137   Label not_null;
1138   __ bnez(obj, not_null);
1139   // Object is null, update MDO and exit
1140   Register mdo = klass_RInfo;
1141   __ mov_metadata(mdo, md->constant_encoding());
1142   Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1143   __ lbu(t0, data_addr);
1144   __ ori(t0, t0, BitData::null_seen_byte_constant());
1145   __ sb(t0, data_addr);
1146   __ j(*obj_is_null);
1147   __ bind(not_null);
1148 }
1149 
1150 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
1151   if (!k->is_loaded()) {
1152     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1153   } else {
1154     __ mov_metadata(k_RInfo, k->constant_encoding());
1155   }
1156 }
1157 
1158 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1159   Register obj = op->object()->as_register();
1160   Register k_RInfo = op->tmp1()->as_register();
1161   Register klass_RInfo = op->tmp2()->as_register();
1162   Register dst = op->result_opr()->as_register();
1163   ciKlass* k = op->klass();
1164   Register Rtmp1 = noreg;
1165 
1166   // check if it needs to be profiled
1167   ciMethodData* md = NULL;
1168   ciProfileData* data = NULL;
1169 
1170   const bool should_profile = op->should_profile();
1171   if (should_profile) {
1172     data_check(op, &md, &data);
1173   }
1174   Label profile_cast_success, profile_cast_failure;
1175   Label *success_target = should_profile ? &profile_cast_success : success;
1176   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1177 
1178   if (obj == k_RInfo) {
1179     k_RInfo = dst;
1180   } else if (obj == klass_RInfo) {
1181     klass_RInfo = dst;
1182   }
1183   if (k->is_loaded() && !UseCompressedClassPointers) {
1184     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1185   } else {
1186     Rtmp1 = op->tmp3()->as_register();
1187     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1188   }
1189 
1190   assert_different_registers(obj, k_RInfo, klass_RInfo);
1191 
1192   if (should_profile) {
1193     profile_object(md, data, obj, klass_RInfo, obj_is_null);
1194   } else {
1195     __ beqz(obj, *obj_is_null);
1196   }
1197 
1198   typecheck_loaded(op, k, k_RInfo);
1199   __ verify_oop(obj);
1200 
1201   if (op->fast_check()) {
1202     // get object class
1203     // not a safepoint as obj null check happens earlier
1204     __ load_klass(t0, obj);
1205     __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1206     // successful cast, fall through to profile or jump
1207   } else {
1208     typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1209   }
1210   if (should_profile) {
1211     type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure);
1212   }
1213   __ j(*success);
1214 }
1215 
1216 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1217   const bool should_profile = op->should_profile();
1218 
1219   LIR_Code code = op->code();
1220   if (code == lir_store_check) {
1221     typecheck_lir_store(op, should_profile);
1222   } else if (code == lir_checkcast) {
1223     Register obj = op->object()->as_register();
1224     Register dst = op->result_opr()->as_register();
1225     Label success;
1226     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1227     __ bind(success);
1228     if (dst != obj) {
1229       __ mv(dst, obj);
1230     }
1231   } else if (code == lir_instanceof) {
1232     Register obj = op->object()->as_register();
1233     Register dst = op->result_opr()->as_register();
1234     Label success, failure, done;
1235     emit_typecheck_helper(op, &success, &failure, &failure);
1236     __ bind(failure);
1237     __ mv(dst, zr);
1238     __ j(done);
1239     __ bind(success);
1240     __ mv(dst, 1);
1241     __ bind(done);
1242   } else {
1243     ShouldNotReachHere();
1244   }
1245 }
1246 
1247 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1248   assert(VM_Version::supports_cx8(), "wrong machine");
1249   Register addr;
1250   if (op->addr()->is_register()) {
1251     addr = as_reg(op->addr());
1252   } else {
1253     assert(op->addr()->is_address(), "what else?");
1254     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1255     assert(addr_ptr->disp() == 0, "need 0 disp");
1256     assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1257     addr = as_reg(addr_ptr->base());
1258   }
1259   Register newval = as_reg(op->new_value());
1260   Register cmpval = as_reg(op->cmp_value());
1261 
1262   if (op->code() == lir_cas_obj) {
1263     if (UseCompressedOops) {
1264       Register tmp1 = op->tmp1()->as_register();
1265       assert(op->tmp1()->is_valid(), "must be");
1266       __ encode_heap_oop(tmp1, cmpval);
1267       cmpval = tmp1;
1268       __ encode_heap_oop(t1, newval);
1269       newval = t1;
1270       caswu(addr, newval, cmpval);
1271     } else {
1272       casl(addr, newval, cmpval);
1273     }
1274   } else if (op->code() == lir_cas_int) {
1275     casw(addr, newval, cmpval);
1276   } else {
1277     casl(addr, newval, cmpval);
1278   }
1279 }
1280 
1281 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1282   switch (code) {
1283     case lir_abs:  __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break;
1284     case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break;
1285     default:       ShouldNotReachHere();
1286   }
1287 }
1288 
1289 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1290   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1291   Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1292   if (dst->is_single_cpu()) {
1293     Register Rdst = dst->as_register();
1294     if (right->is_constant()) {
1295       int right_const = right->as_jint();
1296       if (Assembler::operand_valid_for_add_immediate(right_const)) {
1297         logic_op_imm(Rdst, Rleft, right_const, code);
1298         __ addw(Rdst, Rdst, zr);
1299      } else {
1300         __ mv(t0, right_const);
1301         logic_op_reg32(Rdst, Rleft, t0, code);
1302      }
1303     } else {
1304       Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1305       logic_op_reg32(Rdst, Rleft, Rright, code);
1306     }
1307   } else {
1308     Register Rdst = dst->as_register_lo();
1309     if (right->is_constant()) {
1310       long right_const = right->as_jlong();
1311       if (Assembler::operand_valid_for_add_immediate(right_const)) {
1312         logic_op_imm(Rdst, Rleft, right_const, code);
1313       } else {
1314         __ mv(t0, right_const);
1315         logic_op_reg(Rdst, Rleft, t0, code);
1316       }
1317     } else {
1318       Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1319       logic_op_reg(Rdst, Rleft, Rright, code);
1320     }
1321   }
1322 }
1323 
1324 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) {
1325   ShouldNotCallThis();
1326 }
1327 
1328 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1329   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1330     bool is_unordered_less = (code == lir_ucmp_fd2i);
1331     if (left->is_single_fpu()) {
1332       __ float_cmp(true, is_unordered_less ? -1 : 1,
1333                    left->as_float_reg(), right->as_float_reg(), dst->as_register());
1334     } else if (left->is_double_fpu()) {
1335       __ float_cmp(false, is_unordered_less ? -1 : 1,
1336                    left->as_double_reg(), right->as_double_reg(), dst->as_register());
1337     } else {
1338       ShouldNotReachHere();
1339     }
1340   } else if (code == lir_cmp_l2i) {
1341     __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1342   } else {
1343     ShouldNotReachHere();
1344   }
1345 }
1346 
1347 void LIR_Assembler::align_call(LIR_Code code) {
1348   // With RVC a call instruction may get 2-byte aligned.
1349   // The address of the call instruction needs to be 4-byte aligned to
1350   // ensure that it does not span a cache line so that it can be patched.
1351   __ align(NativeInstruction::instruction_size);
1352 }
1353 
1354 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1355   address call = __ trampoline_call(Address(op->addr(), rtype));
1356   if (call == NULL) {
1357     bailout("trampoline stub overflow");
1358     return;
1359   }
1360   add_call_info(code_offset(), op->info());
1361 }
1362 
1363 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1364   address call = __ ic_call(op->addr());
1365   if (call == NULL) {
1366     bailout("trampoline stub overflow");
1367     return;
1368   }
1369   add_call_info(code_offset(), op->info());
1370 }
1371 
1372 void LIR_Assembler::emit_static_call_stub() {
1373   address call_pc = __ pc();
1374   MacroAssembler::assert_alignment(call_pc);
1375   address stub = __ start_a_stub(call_stub_size());
1376   if (stub == NULL) {
1377     bailout("static call stub overflow");
1378     return;
1379   }
1380 
1381   int start = __ offset();
1382 
1383   __ relocate(static_stub_Relocation::spec(call_pc));
1384   __ emit_static_call_stub();
1385 
1386   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
1387          <= call_stub_size(), "stub too big");
1388   __ end_a_stub();
1389 }
1390 
1391 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1392   assert(exceptionOop->as_register() == x10, "must match");
1393   assert(exceptionPC->as_register() == x13, "must match");
1394 
1395   // exception object is not added to oop map by LinearScan
1396   // (LinearScan assumes that no oops are in fixed registers)
1397   info->add_register_oop(exceptionOop);
1398   Runtime1::StubID unwind_id;
1399 
1400   // get current pc information
1401   // pc is only needed if the method has an exception handler, the unwind code does not need it.
1402   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
1403     // As no instructions have been generated yet for this LIR node it's
1404     // possible that an oop map already exists for the current offset.
1405     // In that case insert an dummy NOP here to ensure all oop map PCs
1406     // are unique. See JDK-8237483.
1407     __ nop();
1408   }
1409   int pc_for_athrow_offset = __ offset();
1410   InternalAddress pc_for_athrow(__ pc());
1411   int32_t off = 0;
1412   __ la_patchable(exceptionPC->as_register(), pc_for_athrow, off);
1413   __ addi(exceptionPC->as_register(), exceptionPC->as_register(), off);
1414   add_call_info(pc_for_athrow_offset, info); // for exception handler
1415 
1416   __ verify_not_null_oop(x10);
1417   // search an exception handler (x10: exception oop, x13: throwing pc)
1418   if (compilation()->has_fpu_code()) {
1419     unwind_id = Runtime1::handle_exception_id;
1420   } else {
1421     unwind_id = Runtime1::handle_exception_nofpu_id;
1422   }
1423   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
1424   __ nop();
1425 }
1426 
1427 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1428   assert(exceptionOop->as_register() == x10, "must match");
1429   __ j(_unwind_handler_entry);
1430 }
1431 
1432 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
1433   Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1434   Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1435   Register count_reg = count->as_register();
1436   if (dest->is_single_cpu()) {
1437     assert (dest->type() == T_INT, "unexpected result type");
1438     assert (left->type() == T_INT, "unexpected left type");
1439     __ andi(t0, count_reg, 31); // should not shift more than 31 bits
1440     switch (code) {
1441       case lir_shl:  __ sllw(dest_reg, left_reg, t0); break;
1442       case lir_shr:  __ sraw(dest_reg, left_reg, t0); break;
1443       case lir_ushr: __ srlw(dest_reg, left_reg, t0); break;
1444       default: ShouldNotReachHere();
1445     }
1446   } else if (dest->is_double_cpu()) {
1447     __ andi(t0, count_reg, 63); // should not shift more than 63 bits
1448     switch (code) {
1449       case lir_shl:  __ sll(dest_reg, left_reg, t0); break;
1450       case lir_shr:  __ sra(dest_reg, left_reg, t0); break;
1451       case lir_ushr: __ srl(dest_reg, left_reg, t0); break;
1452       default: ShouldNotReachHere();
1453     }
1454   } else {
1455     ShouldNotReachHere();
1456   }
1457 }
1458 
1459 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
1460   Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1461   Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1462   if (dest->is_single_cpu()) {
1463     assert (dest->type() == T_INT, "unexpected result type");
1464     assert (left->type() == T_INT, "unexpected left type");
1465     count &= 0x1f;
1466     if (count != 0) {
1467       switch (code) {
1468         case lir_shl:  __ slliw(dest_reg, left_reg, count); break;
1469         case lir_shr:  __ sraiw(dest_reg, left_reg, count); break;
1470         case lir_ushr: __ srliw(dest_reg, left_reg, count); break;
1471         default: ShouldNotReachHere();
1472       }
1473     } else {
1474       move_regs(left_reg, dest_reg);
1475     }
1476   } else if (dest->is_double_cpu()) {
1477     count &= 0x3f;
1478     if (count != 0) {
1479       switch (code) {
1480         case lir_shl:  __ slli(dest_reg, left_reg, count); break;
1481         case lir_shr:  __ srai(dest_reg, left_reg, count); break;
1482         case lir_ushr: __ srli(dest_reg, left_reg, count); break;
1483         default: ShouldNotReachHere();
1484       }
1485     } else {
1486       move_regs(left->as_register_lo(), dest->as_register_lo());
1487     }
1488   } else {
1489     ShouldNotReachHere();
1490   }
1491 }
1492 
1493 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
1494   Register obj = op->obj_opr()->as_register();  // may not be an oop
1495   Register hdr = op->hdr_opr()->as_register();
1496   Register lock = op->lock_opr()->as_register();
1497   if (UseHeavyMonitors) {
1498     if (op->info() != NULL) {
1499       add_debug_info_for_null_check_here(op->info());
1500       __ null_check(obj);
1501     }
1502     __ j(*op->stub()->entry());
1503   } else if (op->code() == lir_lock) {
1504     // add debug info for NullPointerException only if one is possible
1505     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
1506     if (op->info() != NULL) {
1507       add_debug_info_for_null_check(null_check_offset, op->info());
1508     }
1509   } else if (op->code() == lir_unlock) {
1510     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
1511   } else {
1512     Unimplemented();
1513   }
1514   __ bind(*op->stub()->continuation());
1515 }
1516 
1517 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
1518   Register obj = op->obj()->as_pointer_register();
1519   Register result = op->result_opr()->as_pointer_register();
1520 
1521   CodeEmitInfo* info = op->info();
1522   if (info != NULL) {
1523     add_debug_info_for_null_check_here(info);
1524   }
1525 
1526   if (UseCompressedClassPointers) {
1527     __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes()));
1528     __ decode_klass_not_null(result);
1529   } else {
1530     __ ld(result, Address(obj, oopDesc::klass_offset_in_bytes()));
1531   }
1532 }
1533 
1534 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
1535   ciMethod* method = op->profiled_method();
1536   int bci          = op->profiled_bci();
1537 
1538   // Update counter for all call types
1539   ciMethodData* md = method->method_data_or_null();
1540   guarantee(md != NULL, "Sanity");
1541   ciProfileData* data = md->bci_to_data(bci);
1542   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
1543   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
1544   Register mdo  = op->mdo()->as_register();
1545   __ mov_metadata(mdo, md->constant_encoding());
1546   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1547   // Perform additional virtual call profiling for invokevirtual and
1548   // invokeinterface bytecodes
1549   if (op->should_profile_receiver_type()) {
1550     assert(op->recv()->is_single_cpu(), "recv must be allocated");
1551     Register recv = op->recv()->as_register();
1552     assert_different_registers(mdo, recv);
1553     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
1554     ciKlass* known_klass = op->known_holder();
1555     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
1556       // We know the type that will be seen at this call site; we can
1557       // statically update the MethodData* rather than needing to do
1558       // dynamic tests on the receiver type
1559       // NOTE: we should probably put a lock around this search to
1560       // avoid collisions by concurrent compilations
1561       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
1562       uint i;
1563       for (i = 0; i < VirtualCallData::row_limit(); i++) {
1564         ciKlass* receiver = vc_data->receiver(i);
1565         if (known_klass->equals(receiver)) {
1566           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1567           __ increment(data_addr, DataLayout::counter_increment);
1568           return;
1569         }
1570       }
1571 
1572       // Receiver type not found in profile data; select an empty slot
1573       // Note that this is less efficient than it should be because it
1574       // always does a write to the receiver part of the
1575       // VirtualCallData rather than just the first time
1576       for (i = 0; i < VirtualCallData::row_limit(); i++) {
1577         ciKlass* receiver = vc_data->receiver(i);
1578         if (receiver == NULL) {
1579           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
1580           __ mov_metadata(t1, known_klass->constant_encoding());
1581           __ sd(t1, recv_addr);
1582           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1583           __ increment(data_addr, DataLayout::counter_increment);
1584           return;
1585         }
1586       }
1587     } else {
1588       __ load_klass(recv, recv);
1589       Label update_done;
1590       type_profile_helper(mdo, md, data, recv, &update_done);
1591       // Receiver did not match any saved receiver and there is no empty row for it.
1592       // Increment total counter to indicate polymorphic case.
1593       __ increment(counter_addr, DataLayout::counter_increment);
1594 
1595       __ bind(update_done);
1596     }
1597   } else {
1598     // Static call
1599     __ increment(counter_addr, DataLayout::counter_increment);
1600   }
1601 }
1602 
1603 void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); }
1604 
1605 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
1606   __ la(dst->as_register(), frame_map()->address_for_monitor_object(monitor_no));
1607 }
1608 
1609 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); }
1610 
1611 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
1612                                    Register tmp, Label &next, Label &none,
1613                                    Address mdo_addr) {
1614   if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
1615     if (exact_klass != NULL) {
1616       __ mov_metadata(tmp, exact_klass->constant_encoding());
1617     } else {
1618       __ load_klass(tmp, tmp);
1619     }
1620 
1621     __ ld(t1, mdo_addr);
1622     __ xorr(tmp, tmp, t1);
1623     __ andi(t0, tmp, TypeEntries::type_klass_mask);
1624     // klass seen before, nothing to do. The unknown bit may have been
1625     // set already but no need to check.
1626     __ beqz(t0, next);
1627 
1628     // already unknown. Nothing to do anymore.
1629     __ andi(t0, tmp, TypeEntries::type_unknown);
1630     __ bnez(t0, next);
1631 
1632     if (TypeEntries::is_type_none(current_klass)) {
1633       __ beqz(t1, none);
1634       __ mv(t0, (u1)TypeEntries::null_seen);
1635       __ beq(t0, t1, none);
1636       // There is a chance that the checks above (re-reading profiling
1637       // data from memory) fail if another thread has just set the
1638       // profiling to this obj's klass
1639       __ membar(MacroAssembler::LoadLoad);
1640       __ ld(t1, mdo_addr);
1641       __ xorr(tmp, tmp, t1);
1642       __ andi(t0, tmp, TypeEntries::type_klass_mask);
1643       __ beqz(t0, next);
1644     }
1645   } else {
1646     assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1647            ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
1648 
1649     __ ld(tmp, mdo_addr);
1650     // already unknown. Nothing to do anymore.
1651     __ andi(t0, tmp, TypeEntries::type_unknown);
1652     __ bnez(t0, next);
1653   }
1654 
1655   // different than before. Cannot keep accurate profile.
1656   __ ld(t1, mdo_addr);
1657   __ ori(t1, t1, TypeEntries::type_unknown);
1658   __ sd(t1, mdo_addr);
1659 
1660   if (TypeEntries::is_type_none(current_klass)) {
1661     __ j(next);
1662 
1663     __ bind(none);
1664     // first time here. Set profile type.
1665     __ sd(tmp, mdo_addr);
1666   }
1667 }
1668 
1669 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp,
1670                                       Address mdo_addr, Label &next) {
1671   // There's a single possible klass at this profile point
1672   assert(exact_klass != NULL, "should be");
1673   if (TypeEntries::is_type_none(current_klass)) {
1674     __ mov_metadata(tmp, exact_klass->constant_encoding());
1675     __ ld(t1, mdo_addr);
1676     __ xorr(tmp, tmp, t1);
1677     __ andi(t0, tmp, TypeEntries::type_klass_mask);
1678     __ beqz(t0, next);
1679 #ifdef ASSERT
1680   {
1681     Label ok;
1682     __ ld(t0, mdo_addr);
1683     __ beqz(t0, ok);
1684     __ mv(t1, (u1)TypeEntries::null_seen);
1685     __ beq(t0, t1, ok);
1686     // may have been set by another thread
1687     __ membar(MacroAssembler::LoadLoad);
1688     __ mov_metadata(t0, exact_klass->constant_encoding());
1689     __ ld(t1, mdo_addr);
1690     __ xorr(t1, t0, t1);
1691     __ andi(t1, t1, TypeEntries::type_mask);
1692     __ beqz(t1, ok);
1693 
1694     __ stop("unexpected profiling mismatch");
1695     __ bind(ok);
1696   }
1697 #endif
1698     // first time here. Set profile type.
1699     __ sd(tmp, mdo_addr);
1700   } else {
1701     assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1702            ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
1703 
1704     __ ld(tmp, mdo_addr);
1705     // already unknown. Nothing to do anymore.
1706     __ andi(t0, tmp, TypeEntries::type_unknown);
1707     __ bnez(t0, next);
1708 
1709     __ ori(tmp, tmp, TypeEntries::type_unknown);
1710     __ sd(tmp, mdo_addr);
1711   }
1712 }
1713 
1714 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass,
1715                                Address mdo_addr, bool do_update, Label &next) {
1716   __ bnez(tmp, update);
1717   if (!TypeEntries::was_null_seen(current_klass)) {
1718     __ ld(t1, mdo_addr);
1719     __ ori(t1, t1, TypeEntries::null_seen);
1720     __ sd(t1, mdo_addr);
1721   }
1722   if (do_update) {
1723     __ j(next);
1724   }
1725 }
1726 
1727 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
1728   COMMENT("emit_profile_type {");
1729   Register obj = op->obj()->as_register();
1730   Register tmp = op->tmp()->as_pointer_register();
1731   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
1732   ciKlass* exact_klass = op->exact_klass();
1733   intptr_t current_klass = op->current_klass();
1734   bool not_null = op->not_null();
1735   bool no_conflict = op->no_conflict();
1736 
1737   Label update, next, none;
1738 
1739   bool do_null = !not_null;
1740   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
1741   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
1742 
1743   assert(do_null || do_update, "why are we here?");
1744   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
1745   assert_different_registers(tmp, t0, t1, mdo_addr.base());
1746 
1747   __ verify_oop(obj);
1748 
1749   if (tmp != obj) {
1750     __ mv(tmp, obj);
1751   }
1752   if (do_null) {
1753     check_null(tmp, update, current_klass, mdo_addr, do_update, next);
1754 #ifdef ASSERT
1755   } else {
1756     __ bnez(tmp, update);
1757     __ stop("unexpected null obj");
1758 #endif
1759   }
1760 
1761   __ bind(update);
1762 
1763   if (do_update) {
1764 #ifdef ASSERT
1765     if (exact_klass != NULL) {
1766       check_exact_klass(tmp, exact_klass);
1767     }
1768 #endif
1769     if (!no_conflict) {
1770       check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr);
1771     } else {
1772       check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next);
1773     }
1774 
1775     __ bind(next);
1776   }
1777   COMMENT("} emit_profile_type");
1778 }
1779 
1780 void LIR_Assembler::align_backward_branch_target() { }
1781 
1782 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
1783   // tmp must be unused
1784   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
1785 
1786   if (left->is_single_cpu()) {
1787     assert(dest->is_single_cpu(), "expect single result reg");
1788     __ negw(dest->as_register(), left->as_register());
1789   } else if (left->is_double_cpu()) {
1790     assert(dest->is_double_cpu(), "expect double result reg");
1791     __ neg(dest->as_register_lo(), left->as_register_lo());
1792   } else if (left->is_single_fpu()) {
1793     assert(dest->is_single_fpu(), "expect single float result reg");
1794     __ fneg_s(dest->as_float_reg(), left->as_float_reg());
1795   } else {
1796     assert(left->is_double_fpu(), "expect double float operand reg");
1797     assert(dest->is_double_fpu(), "expect double float result reg");
1798     __ fneg_d(dest->as_double_reg(), left->as_double_reg());
1799   }
1800 }
1801 
1802 
1803 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1804   if (patch_code != lir_patch_none) {
1805     deoptimize_trap(info);
1806     return;
1807   }
1808 
1809   LIR_Address* adr = addr->as_address_ptr();
1810   Register dst = dest->as_register_lo();
1811 
1812   assert_different_registers(dst, t0);
1813   if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) {
1814     int scale = adr->scale();
1815     intptr_t offset = adr->disp();
1816     LIR_Opr index_op = adr->index();
1817     if (index_op->is_constant()) {
1818       offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale;
1819     }
1820 
1821     if (!is_imm_in_range(offset, 12, 0)) {
1822       __ la(t0, as_Address(adr));
1823       __ mv(dst, t0);
1824       return;
1825     }
1826   }
1827 
1828   __ la(dst, as_Address(adr));
1829 }
1830 
1831 
1832 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
1833   assert(!tmp->is_valid(), "don't need temporary");
1834 
1835   CodeBlob *cb = CodeCache::find_blob(dest);
1836   if (cb != NULL) {
1837     __ far_call(RuntimeAddress(dest));
1838   } else {
1839     int32_t offset = 0;
1840     __ la_patchable(t0, RuntimeAddress(dest), offset);
1841     __ jalr(x1, t0, offset);
1842   }
1843 
1844   if (info != NULL) {
1845     add_call_info_here(info);
1846   }
1847 }
1848 
1849 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
1850   if (dest->is_address() || src->is_address()) {
1851     move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false);
1852   } else {
1853     ShouldNotReachHere();
1854   }
1855 }
1856 
1857 #ifdef ASSERT
1858 // emit run-time assertion
1859 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
1860   assert(op->code() == lir_assert, "must be");
1861 
1862   Label ok;
1863   if (op->in_opr1()->is_valid()) {
1864     assert(op->in_opr2()->is_valid(), "both operands must be valid");
1865     bool is_unordered = false;
1866     LIR_Condition cond = op->condition();
1867     emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false,
1868                 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true);
1869   } else {
1870     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
1871     assert(op->condition() == lir_cond_always, "no other conditions allowed");
1872   }
1873 
1874   if (op->halt()) {
1875     const char* str = __ code_string(op->msg());
1876     __ stop(str);
1877   } else {
1878     breakpoint();
1879   }
1880   __ bind(ok);
1881 }
1882 #endif
1883 
1884 #ifndef PRODUCT
1885 #define COMMENT(x)   do { __ block_comment(x); } while (0)
1886 #else
1887 #define COMMENT(x)
1888 #endif
1889 
1890 void LIR_Assembler::membar() {
1891   COMMENT("membar");
1892   __ membar(MacroAssembler::AnyAny);
1893 }
1894 
1895 void LIR_Assembler::membar_acquire() {
1896   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1897 }
1898 
1899 void LIR_Assembler::membar_release() {
1900   __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1901 }
1902 
1903 void LIR_Assembler::membar_loadload() {
1904   __ membar(MacroAssembler::LoadLoad);
1905 }
1906 
1907 void LIR_Assembler::membar_storestore() {
1908   __ membar(MacroAssembler::StoreStore);
1909 }
1910 
1911 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
1912 
1913 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
1914 
1915 void LIR_Assembler::on_spin_wait() {
1916   Unimplemented();
1917 }
1918 
1919 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
1920   __ mv(result_reg->as_register(), xthread);
1921 }
1922 
1923 void LIR_Assembler::peephole(LIR_List *lir) {}
1924 
1925 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
1926   Address addr = as_Address(src->as_address_ptr());
1927   BasicType type = src->type();
1928   bool is_oop = is_reference_type(type);
1929 
1930   get_op(type);
1931 
1932   switch (code) {
1933     case lir_xadd:
1934       {
1935         RegisterOrConstant inc;
1936         Register tmp = as_reg(tmp_op);
1937         Register dst = as_reg(dest);
1938         if (data->is_constant()) {
1939           inc = RegisterOrConstant(as_long(data));
1940           assert_different_registers(dst, addr.base(), tmp);
1941           assert_different_registers(tmp, t0);
1942         } else {
1943           inc = RegisterOrConstant(as_reg(data));
1944           assert_different_registers(inc.as_register(), dst, addr.base(), tmp);
1945         }
1946         __ la(tmp, addr);
1947         (_masm->*add)(dst, inc, tmp);
1948         break;
1949       }
1950     case lir_xchg:
1951       {
1952         Register tmp = tmp_op->as_register();
1953         Register obj = as_reg(data);
1954         Register dst = as_reg(dest);
1955         if (is_oop && UseCompressedOops) {
1956           __ encode_heap_oop(t0, obj);
1957           obj = t0;
1958         }
1959         assert_different_registers(obj, addr.base(), tmp, dst);
1960         __ la(tmp, addr);
1961         (_masm->*xchg)(dst, obj, tmp);
1962         if (is_oop && UseCompressedOops) {
1963           __ decode_heap_oop(dst);
1964         }
1965       }
1966       break;
1967     default:
1968       ShouldNotReachHere();
1969   }
1970   __ membar(MacroAssembler::AnyAny);
1971 }
1972 
1973 int LIR_Assembler::array_element_size(BasicType type) const {
1974   int elem_size = type2aelembytes(type);
1975   return exact_log2(elem_size);
1976 }
1977 
1978 // helper functions which checks for overflow and sets bailout if it
1979 // occurs.  Always returns a valid embeddable pointer but in the
1980 // bailout case the pointer won't be to unique storage.
1981 address LIR_Assembler::float_constant(float f) {
1982   address const_addr = __ float_constant(f);
1983   if (const_addr == NULL) {
1984     bailout("const section overflow");
1985     return __ code()->consts()->start();
1986   } else {
1987     return const_addr;
1988   }
1989 }
1990 
1991 address LIR_Assembler::double_constant(double d) {
1992   address const_addr = __ double_constant(d);
1993   if (const_addr == NULL) {
1994     bailout("const section overflow");
1995     return __ code()->consts()->start();
1996   } else {
1997     return const_addr;
1998   }
1999 }
2000 
2001 address LIR_Assembler::int_constant(jlong n) {
2002   address const_addr = __ long_constant(n);
2003   if (const_addr == NULL) {
2004     bailout("const section overflow");
2005     return __ code()->consts()->start();
2006   } else {
2007     return const_addr;
2008   }
2009 }
2010 
2011 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
2012   __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */,
2013              Assembler::rl /* release */, t0, true /* result as bool */);
2014   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2015   __ membar(MacroAssembler::AnyAny);
2016 }
2017 
2018 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) {
2019   __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */,
2020              Assembler::rl /* release */, t0, true /* result as bool */);
2021   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2022   __ membar(MacroAssembler::AnyAny);
2023 }
2024 
2025 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
2026   __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */,
2027              Assembler::rl /* release */, t0, true /* result as bool */);
2028   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2029   __ membar(MacroAssembler::AnyAny);
2030 }
2031 
2032 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
2033   address target = NULL;
2034 
2035   switch (patching_id(info)) {
2036     case PatchingStub::access_field_id:
2037       target = Runtime1::entry_for(Runtime1::access_field_patching_id);
2038       break;
2039     case PatchingStub::load_klass_id:
2040       target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
2041       break;
2042     case PatchingStub::load_mirror_id:
2043       target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
2044       break;
2045     case PatchingStub::load_appendix_id:
2046       target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
2047       break;
2048     default: ShouldNotReachHere();
2049   }
2050 
2051   __ far_call(RuntimeAddress(target));
2052   add_call_info_here(info);
2053 }
2054 
2055 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) {
2056   Label ok;
2057   __ load_klass(tmp, tmp);
2058   __ mov_metadata(t0, exact_klass->constant_encoding());
2059   __ beq(tmp, t0, ok);
2060   __ stop("exact klass and actual klass differ");
2061   __ bind(ok);
2062 }
2063 
2064 void LIR_Assembler::get_op(BasicType type) {
2065   switch (type) {
2066     case T_INT:
2067       xchg = &MacroAssembler::atomic_xchgalw;
2068       add = &MacroAssembler::atomic_addalw;
2069       break;
2070     case T_LONG:
2071       xchg = &MacroAssembler::atomic_xchgal;
2072       add = &MacroAssembler::atomic_addal;
2073       break;
2074     case T_OBJECT:
2075     case T_ARRAY:
2076       if (UseCompressedOops) {
2077         xchg = &MacroAssembler::atomic_xchgalwu;
2078         add = &MacroAssembler::atomic_addalw;
2079       } else {
2080         xchg = &MacroAssembler::atomic_xchgal;
2081         add = &MacroAssembler::atomic_addal;
2082       }
2083       break;
2084     default:
2085       ShouldNotReachHere();
2086   }
2087 }
2088 
2089 // emit_opTypeCheck sub functions
2090 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) {
2091   Register value = op->object()->as_register();
2092   Register array = op->array()->as_register();
2093   Register k_RInfo = op->tmp1()->as_register();
2094   Register klass_RInfo = op->tmp2()->as_register();
2095   Register Rtmp1 = op->tmp3()->as_register();
2096 
2097   CodeStub* stub = op->stub();
2098 
2099   // check if it needs to be profiled
2100   ciMethodData* md = NULL;
2101   ciProfileData* data = NULL;
2102 
2103   if (should_profile) {
2104     data_check(op, &md, &data);
2105   }
2106   Label profile_cast_success, profile_cast_failure, done;
2107   Label *success_target = should_profile ? &profile_cast_success : &done;
2108   Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
2109 
2110   if (should_profile) {
2111     profile_object(md, data, value, klass_RInfo, &done);
2112   } else {
2113     __ beqz(value, done);
2114   }
2115 
2116   add_debug_info_for_null_check_here(op->info_for_exception());
2117   __ load_klass(k_RInfo, array);
2118   __ load_klass(klass_RInfo, value);
2119 
2120   lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target);
2121 
2122   // fall through to the success case
2123   if (should_profile) {
2124     Register mdo = klass_RInfo;
2125     Register recv = k_RInfo;
2126     __ bind(profile_cast_success);
2127     __ mov_metadata(mdo, md->constant_encoding());
2128     __ load_klass(recv, value);
2129     type_profile_helper(mdo, md, data, recv, &done);
2130     __ j(done);
2131 
2132     __ bind(profile_cast_failure);
2133     __ mov_metadata(mdo, md->constant_encoding());
2134     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2135     __ ld(t1, counter_addr);
2136     __ addi(t1, t1, -DataLayout::counter_increment);
2137     __ sd(t1, counter_addr);
2138     __ j(*stub->entry());
2139   }
2140 
2141   __ bind(done);
2142 }
2143 
2144 void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo,
2145                                  ciProfileData* data, Label* success, Label* failure,
2146                                  Label& profile_cast_success, Label& profile_cast_failure) {
2147   Register mdo = klass_RInfo;
2148   Register recv = k_RInfo;
2149   __ bind(profile_cast_success);
2150   __ mov_metadata(mdo, md->constant_encoding());
2151   __ load_klass(recv, obj);
2152   Label update_done;
2153   type_profile_helper(mdo, md, data, recv, success);
2154   __ j(*success);
2155 
2156   __ bind(profile_cast_failure);
2157   __ mov_metadata(mdo, md->constant_encoding());
2158   Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2159   __ ld(t0, counter_addr);
2160   __ addi(t0, t0, -DataLayout::counter_increment);
2161   __ sd(t0, counter_addr);
2162   __ j(*failure);
2163 }
2164 
2165 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1,
2166                                         Label* success_target, Label* failure_target) {
2167   // get instance klass (it's already uncompressed)
2168   __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2169   // perform the fast part of the checking logic
2170   __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2171   // call out-of-line instance of __ check_klass_subtype_slow_path(...)
2172   __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
2173   __ sd(klass_RInfo, Address(sp, wordSize));  // sub klass
2174   __ sd(k_RInfo, Address(sp, 0));             // super klass
2175   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2176   // load result to k_RInfo
2177   __ ld(k_RInfo, Address(sp, 0));
2178   __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
2179   // result is a boolean
2180   __ beqz(k_RInfo, *failure_target, /* is_far */ true);
2181 }
2182 
2183 void LIR_Assembler::const2reg_helper(LIR_Opr src) {
2184   switch (src->as_constant_ptr()->type()) {
2185     case T_INT:
2186     case T_ADDRESS:
2187     case T_OBJECT:
2188     case T_ARRAY:
2189     case T_METADATA:
2190         const2reg(src, FrameMap::t0_opr, lir_patch_none, NULL);
2191         break;
2192     case T_LONG:
2193         const2reg(src, FrameMap::t0_long_opr, lir_patch_none, NULL);
2194         break;
2195     case T_FLOAT:
2196     case T_DOUBLE:
2197     default:
2198       ShouldNotReachHere();
2199   }
2200 }
2201 
2202 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) {
2203   switch (code) {
2204     case lir_logic_and: __ andrw(dst, left, right); break;
2205     case lir_logic_or:  __ orrw (dst, left, right); break;
2206     case lir_logic_xor: __ xorrw(dst, left, right); break;
2207     default:            ShouldNotReachHere();
2208   }
2209 }
2210 
2211 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) {
2212   switch (code) {
2213     case lir_logic_and: __ andr(dst, left, right); break;
2214     case lir_logic_or:  __ orr (dst, left, right); break;
2215     case lir_logic_xor: __ xorr(dst, left, right); break;
2216     default:            ShouldNotReachHere();
2217   }
2218 }
2219 
2220 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) {
2221   switch (code) {
2222     case lir_logic_and: __ andi(dst, left, right); break;
2223     case lir_logic_or:  __ ori (dst, left, right); break;
2224     case lir_logic_xor: __ xori(dst, left, right); break;
2225     default:            ShouldNotReachHere();
2226   }
2227 }
2228 
2229 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2230   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2231   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2232   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2233   __ sd(r, Address(sp, offset_from_rsp_in_bytes));
2234 }
2235 
2236 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2237   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2238   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2239   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2240   __ mv(t0, c);
2241   __ sd(t0, Address(sp, offset_from_rsp_in_bytes));
2242 }
2243 
2244 #undef __