1 /*
   2  * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "c1/c1_CodeStubs.hpp"
  31 #include "c1/c1_Compilation.hpp"
  32 #include "c1/c1_LIRAssembler.hpp"
  33 #include "c1/c1_MacroAssembler.hpp"
  34 #include "c1/c1_Runtime1.hpp"
  35 #include "c1/c1_ValueStack.hpp"
  36 #include "ci/ciArrayKlass.hpp"
  37 #include "ci/ciInstance.hpp"
  38 #include "code/compiledIC.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "nativeInst_riscv.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_riscv.inline.hpp"
  46 
  47 #ifndef PRODUCT
  48 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  49 #else
  50 #define COMMENT(x)
  51 #endif
  52 
  53 NEEDS_CLEANUP // remove this definitions ?
  54 const Register IC_Klass    = t1;    // where the IC klass is cached
  55 const Register SYNC_header = x10;   // synchronization header
  56 const Register SHIFT_count = x10;   // where count for shift operations must be
  57 
  58 #define __ _masm->
  59 
  60 static void select_different_registers(Register preserve,
  61                                        Register extra,
  62                                        Register &tmp1,
  63                                        Register &tmp2) {
  64   if (tmp1 == preserve) {
  65     assert_different_registers(tmp1, tmp2, extra);
  66     tmp1 = extra;
  67   } else if (tmp2 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp2 = extra;
  70   }
  71   assert_different_registers(preserve, tmp1, tmp2);
  72 }
  73 
  74 static void select_different_registers(Register preserve,
  75                                        Register extra,
  76                                        Register &tmp1,
  77                                        Register &tmp2,
  78                                        Register &tmp3) {
  79   if (tmp1 == preserve) {
  80     assert_different_registers(tmp1, tmp2, tmp3, extra);
  81     tmp1 = extra;
  82   } else if (tmp2 == preserve) {
  83     assert_different_registers(tmp1, tmp2, tmp3, extra);
  84     tmp2 = extra;
  85   } else if (tmp3 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp3 = extra;
  88   }
  89   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  90 }
  91 
  92 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  93 
  94 void LIR_Assembler::clinit_barrier(ciMethod* method) {
  95   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  96   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
  97 
  98   Label L_skip_barrier;
  99 
 100   __ mov_metadata(t1, method->holder()->constant_encoding());
 101   __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */);
 102   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 103   __ bind(L_skip_barrier);
 104 }
 105 
 106 LIR_Opr LIR_Assembler::receiverOpr() {
 107   return FrameMap::receiver_opr;
 108 }
 109 
 110 LIR_Opr LIR_Assembler::osrBufferPointer() {
 111   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 112 }
 113 
 114 void LIR_Assembler::breakpoint() { Unimplemented(); }
 115 
 116 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 117 
 118 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 119 
 120 static jlong as_long(LIR_Opr data) {
 121   jlong result;
 122   switch (data->type()) {
 123     case T_INT:
 124       result = (data->as_jint());
 125       break;
 126     case T_LONG:
 127       result = (data->as_jlong());
 128       break;
 129     default:
 130       ShouldNotReachHere();
 131       result = 0;  // unreachable
 132   }
 133   return result;
 134 }
 135 
 136 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 137   if (addr->base()->is_illegal()) {
 138     assert(addr->index()->is_illegal(), "must be illegal too");
 139     __ movptr(tmp, addr->disp());
 140     return Address(tmp, 0);
 141   }
 142 
 143   Register base = addr->base()->as_pointer_register();
 144   LIR_Opr index_opr = addr->index();
 145 
 146   if (index_opr->is_illegal()) {
 147     return Address(base, addr->disp());
 148   }
 149 
 150   int scale = addr->scale();
 151   if (index_opr->is_cpu_register()) {
 152     Register index;
 153     if (index_opr->is_single_cpu()) {
 154       index = index_opr->as_register();
 155     } else {
 156       index = index_opr->as_register_lo();
 157     }
 158     if (scale != 0) {
 159       __ shadd(tmp, index, base, tmp, scale);
 160     } else {
 161       __ add(tmp, base, index);
 162     }
 163     return Address(tmp, addr->disp());
 164   } else if (index_opr->is_constant()) {
 165     intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp();
 166     return Address(base, addr_offset);
 167   }
 168 
 169   Unimplemented();
 170   return Address();
 171 }
 172 
 173 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 174   ShouldNotReachHere();
 175   return Address();
 176 }
 177 
 178 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 179   return as_Address(addr, t0);
 180 }
 181 
 182 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 183   return as_Address(addr);
 184 }
 185 
 186 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 187 // not encodable as a base + (immediate) offset, generate an explicit address
 188 // calculation to hold the address in t0.
 189 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) {
 190   precond(size == 4 || size == 8);
 191   Address addr = frame_map()->address_for_slot(index, adjust);
 192   precond(addr.getMode() == Address::base_plus_offset);
 193   precond(addr.base() == sp);
 194   precond(addr.offset() > 0);
 195   uint mask = size - 1;
 196   assert((addr.offset() & mask) == 0, "scaled offsets only");
 197 
 198   return addr;
 199 }
 200 
 201 void LIR_Assembler::osr_entry() {
 202   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 203   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 204   guarantee(osr_entry != NULL, "NULL osr_entry!");
 205   ValueStack* entry_state = osr_entry->state();
 206   int number_of_locks = entry_state->locks_size();
 207 
 208   // we jump here if osr happens with the interpreter
 209   // state set up to continue at the beginning of the
 210   // loop that triggered osr - in particular, we have
 211   // the following registers setup:
 212   //
 213   // x12: osr buffer
 214   //
 215 
 216   //build frame
 217   ciMethod* m = compilation()->method();
 218   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 219 
 220   // OSR buffer is
 221   //
 222   // locals[nlocals-1..0]
 223   // monitors[0..number_of_locks]
 224   //
 225   // locals is a direct copy of the interpreter frame so in the osr buffer
 226   // so first slot in the local array is the last local from the interpreter
 227   // and last slot is local[0] (receiver) from the interpreter
 228   //
 229   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 230   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 231   // in the interpreter frame (the method lock if a sync method)
 232 
 233   // Initialize monitors in the compiled activation.
 234   //   x12: pointer to osr buffer
 235   // All other registers are dead at this point and the locals will be
 236   // copied into place by code emitted in the IR.
 237 
 238   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 239   {
 240     assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 241     int monitor_offset = BytesPerWord * method()->max_locals() +
 242       (2 * BytesPerWord) * (number_of_locks - 1);
 243     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 244     // the OSR buffer using 2 word entries: first the lock and then
 245     // the oop.
 246     for (int i = 0; i < number_of_locks; i++) {
 247       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 248 #ifdef ASSERT
 249       // verify the interpreter's monitor has a non-null object
 250       {
 251         Label L;
 252         __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
 253         __ bnez(t0, L);
 254         __ stop("locked object is NULL");
 255         __ bind(L);
 256       }
 257 #endif // ASSERT
 258       __ ld(x9, Address(OSR_buf, slot_offset + 0));
 259       __ sd(x9, frame_map()->address_for_monitor_lock(i));
 260       __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
 261       __ sd(x9, frame_map()->address_for_monitor_object(i));
 262     }
 263   }
 264 }
 265 
 266 // inline cache check; done before the frame is built.
 267 int LIR_Assembler::check_icache() {
 268   Register receiver = FrameMap::receiver_opr->as_register();
 269   Register ic_klass = IC_Klass;
 270   int start_offset = __ offset();
 271   Label dont;
 272   __ inline_cache_check(receiver, ic_klass, dont);
 273 
 274   // if icache check fails, then jump to runtime routine
 275   // Note: RECEIVER must still contain the receiver!
 276   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 277 
 278   // We align the verified entry point unless the method body
 279   // (including its inline cache check) will fit in a single 64-byte
 280   // icache line.
 281   if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 282     // force alignment after the cache check.
 283     __ align(CodeEntryAlignment);
 284   }
 285 
 286   __ bind(dont);
 287   return start_offset;
 288 }
 289 
 290 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 291   if (o == NULL) {
 292     __ mv(reg, zr);
 293   } else {
 294     __ movoop(reg, o);
 295   }
 296 }
 297 
 298 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 299   deoptimize_trap(info);
 300 }
 301 
 302 // This specifies the rsp decrement needed to build the frame
 303 int LIR_Assembler::initial_frame_size_in_bytes() const {
 304   // if rounding, must let FrameMap know!
 305 
 306   return in_bytes(frame_map()->framesize_in_bytes());
 307 }
 308 
 309 int LIR_Assembler::emit_exception_handler() {
 310   // generate code for exception handler
 311   address handler_base = __ start_a_stub(exception_handler_size());
 312   if (handler_base == NULL) {
 313     // not enough space left for the handler
 314     bailout("exception handler overflow");
 315     return -1;
 316   }
 317 
 318   int offset = code_offset();
 319 
 320   // the exception oop and pc are in x10, and x13
 321   // no other registers need to be preserved, so invalidate them
 322   __ invalidate_registers(false, true, true, false, true, true);
 323 
 324   // check that there is really an exception
 325   __ verify_not_null_oop(x10);
 326 
 327   // search an exception handler (x10: exception oop, x13: throwing pc)
 328   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 329   __ should_not_reach_here();
 330   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 331   __ end_a_stub();
 332 
 333   return offset;
 334 }
 335 
 336 // Emit the code to remove the frame from the stack in the exception
 337 // unwind path.
 338 int LIR_Assembler::emit_unwind_handler() {
 339 #ifndef PRODUCT
 340   if (CommentedAssembly) {
 341     _masm->block_comment("Unwind handler");
 342   }
 343 #endif // PRODUCT
 344 
 345   int offset = code_offset();
 346 
 347   // Fetch the exception from TLS and clear out exception related thread state
 348   __ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
 349   __ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
 350   __ sd(zr, Address(xthread, JavaThread::exception_pc_offset()));
 351 
 352   __ bind(_unwind_handler_entry);
 353   __ verify_not_null_oop(x10);
 354   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 355     __ mv(x9, x10);   // Preserve the exception
 356   }
 357 
 358   // Perform needed unlocking
 359   MonitorExitStub* stub = NULL;
 360   if (method()->is_synchronized()) {
 361     monitor_address(0, FrameMap::r10_opr);
 362     stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
 363     if (UseHeavyMonitors) {
 364       __ j(*stub->entry());
 365     } else {
 366       __ unlock_object(x15, x14, x10, *stub->entry());
 367     }
 368     __ bind(*stub->continuation());
 369   }
 370 
 371   if (compilation()->env()->dtrace_method_probes()) {
 372     __ mv(c_rarg0, xthread);
 373     __ mov_metadata(c_rarg1, method()->constant_encoding());
 374     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 375   }
 376 
 377   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 378     __ mv(x10, x9);   // Restore the exception
 379   }
 380 
 381   // remove the activation and dispatch to the unwind handler
 382   __ block_comment("remove_frame and dispatch to the unwind handler");
 383   __ remove_frame(initial_frame_size_in_bytes());
 384   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 385 
 386   // Emit the slow path assembly
 387   if (stub != NULL) {
 388     stub->emit_code(this);
 389   }
 390 
 391   return offset;
 392 }
 393 
 394 int LIR_Assembler::emit_deopt_handler() {
 395   // generate code for exception handler
 396   address handler_base = __ start_a_stub(deopt_handler_size());
 397   if (handler_base == NULL) {
 398     // not enough space left for the handler
 399     bailout("deopt handler overflow");
 400     return -1;
 401   }
 402 
 403   int offset = code_offset();
 404 
 405   __ auipc(ra, 0);
 406   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 407   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 408   __ end_a_stub();
 409 
 410   return offset;
 411 }
 412 
 413 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 414   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
 415 
 416   // Pop the stack before the safepoint code
 417   __ remove_frame(initial_frame_size_in_bytes());
 418 
 419   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 420     __ reserved_stack_check();
 421   }
 422 
 423   code_stub->set_safepoint_offset(__ offset());
 424   __ relocate(relocInfo::poll_return_type);
 425   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 426   __ ret();
 427 }
 428 
 429 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 430   guarantee(info != NULL, "Shouldn't be NULL");
 431   __ get_polling_page(t0, relocInfo::poll_type);
 432   add_debug_info_for_branch(info);  // This isn't just debug info:
 433                                     // it's the oop map
 434   __ read_polling_page(t0, 0, relocInfo::poll_type);
 435   return __ offset();
 436 }
 437 
 438 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 439   __ mv(to_reg, from_reg);
 440 }
 441 
 442 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 443 
 444 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 445   assert(src->is_constant(), "should not call otherwise");
 446   assert(dest->is_register(), "should not call otherwise");
 447   LIR_Const* c = src->as_constant_ptr();
 448   address const_addr = NULL;
 449 
 450   switch (c->type()) {
 451     case T_INT:
 452       assert(patch_code == lir_patch_none, "no patching handled here");
 453       __ mvw(dest->as_register(), c->as_jint());
 454       break;
 455 
 456     case T_ADDRESS:
 457       assert(patch_code == lir_patch_none, "no patching handled here");
 458       __ mv(dest->as_register(), c->as_jint());
 459       break;
 460 
 461     case T_LONG:
 462       assert(patch_code == lir_patch_none, "no patching handled here");
 463       __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
 464       break;
 465 
 466     case T_OBJECT:
 467     case T_ARRAY:
 468       if (patch_code == lir_patch_none) {
 469         jobject2reg(c->as_jobject(), dest->as_register());
 470       } else {
 471         jobject2reg_with_patching(dest->as_register(), info);
 472       }
 473       break;
 474 
 475     case T_METADATA:
 476       if (patch_code != lir_patch_none) {
 477         klass2reg_with_patching(dest->as_register(), info);
 478       } else {
 479         __ mov_metadata(dest->as_register(), c->as_metadata());
 480       }
 481       break;
 482 
 483     case T_FLOAT:
 484       const_addr = float_constant(c->as_jfloat());
 485       assert(const_addr != NULL, "must create float constant in the constant table");
 486       __ flw(dest->as_float_reg(), InternalAddress(const_addr));
 487       break;
 488 
 489     case T_DOUBLE:
 490       const_addr = double_constant(c->as_jdouble());
 491       assert(const_addr != NULL, "must create double constant in the constant table");
 492       __ fld(dest->as_double_reg(), InternalAddress(const_addr));
 493       break;
 494 
 495     default:
 496       ShouldNotReachHere();
 497   }
 498 }
 499 
 500 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 501   assert(src->is_constant(), "should not call otherwise");
 502   assert(dest->is_stack(), "should not call otherwise");
 503   LIR_Const* c = src->as_constant_ptr();
 504   switch (c->type()) {
 505     case T_OBJECT:
 506       if (c->as_jobject() == NULL) {
 507         __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 508       } else {
 509         const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
 510         reg2stack(FrameMap::t1_opr, dest, c->type(), false);
 511       }
 512       break;
 513     case T_ADDRESS:   // fall through
 514       const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
 515       reg2stack(FrameMap::t1_opr, dest, c->type(), false);
 516     case T_INT:       // fall through
 517     case T_FLOAT:
 518       if (c->as_jint_bits() == 0) {
 519         __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 520       } else {
 521         __ mvw(t1, c->as_jint_bits());
 522         __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix()));
 523       }
 524       break;
 525     case T_LONG:      // fall through
 526     case T_DOUBLE:
 527       if (c->as_jlong_bits() == 0) {
 528         __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 529                                                 lo_word_offset_in_bytes));
 530       } else {
 531         __ mv(t1, (intptr_t)c->as_jlong_bits());
 532         __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(),
 533                                                 lo_word_offset_in_bytes));
 534       }
 535       break;
 536     default:
 537       ShouldNotReachHere();
 538   }
 539 }
 540 
 541 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 542   assert(src->is_constant(), "should not call otherwise");
 543   assert(dest->is_address(), "should not call otherwise");
 544   LIR_Const* c = src->as_constant_ptr();
 545   LIR_Address* to_addr = dest->as_address_ptr();
 546   void (Assembler::* insn)(Register Rt, const Address &adr, Register temp);
 547   switch (type) {
 548     case T_ADDRESS:
 549       assert(c->as_jint() == 0, "should be");
 550       insn = &Assembler::sd; break;
 551     case T_LONG:
 552       assert(c->as_jlong() == 0, "should be");
 553       insn = &Assembler::sd; break;
 554     case T_DOUBLE:
 555       assert(c->as_jdouble() == 0.0, "should be");
 556       insn = &Assembler::sd; break;
 557     case T_INT:
 558       assert(c->as_jint() == 0, "should be");
 559       insn = &Assembler::sw; break;
 560     case T_FLOAT:
 561       assert(c->as_jfloat() == 0.0f, "should be");
 562       insn = &Assembler::sw; break;
 563     case T_OBJECT:    // fall through
 564     case T_ARRAY:
 565       assert(c->as_jobject() == 0, "should be");
 566       if (UseCompressedOops && !wide) {
 567         insn = &Assembler::sw;
 568       } else {
 569         insn = &Assembler::sd;
 570       }
 571       break;
 572     case T_CHAR:      // fall through
 573     case T_SHORT:
 574       assert(c->as_jint() == 0, "should be");
 575       insn = &Assembler::sh;
 576       break;
 577     case T_BOOLEAN:   // fall through
 578     case T_BYTE:
 579       assert(c->as_jint() == 0, "should be");
 580       insn = &Assembler::sb; break;
 581     default:
 582       ShouldNotReachHere();
 583       insn = &Assembler::sd;  // unreachable
 584   }
 585   if (info != NULL) {
 586     add_debug_info_for_null_check_here(info);
 587   }
 588   (_masm->*insn)(zr, as_Address(to_addr), t0);
 589 }
 590 
 591 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 592   assert(src->is_register(), "should not call otherwise");
 593   assert(dest->is_register(), "should not call otherwise");
 594 
 595   // move between cpu-registers
 596   if (dest->is_single_cpu()) {
 597     if (src->type() == T_LONG) {
 598       // Can do LONG -> OBJECT
 599       move_regs(src->as_register_lo(), dest->as_register());
 600       return;
 601     }
 602     assert(src->is_single_cpu(), "must match");
 603     if (src->type() == T_OBJECT) {
 604       __ verify_oop(src->as_register());
 605     }
 606     move_regs(src->as_register(), dest->as_register());
 607   } else if (dest->is_double_cpu()) {
 608     if (is_reference_type(src->type())) {
 609       __ verify_oop(src->as_register());
 610       move_regs(src->as_register(), dest->as_register_lo());
 611       return;
 612     }
 613     assert(src->is_double_cpu(), "must match");
 614     Register f_lo = src->as_register_lo();
 615     Register f_hi = src->as_register_hi();
 616     Register t_lo = dest->as_register_lo();
 617     Register t_hi = dest->as_register_hi();
 618     assert(f_hi == f_lo, "must be same");
 619     assert(t_hi == t_lo, "must be same");
 620     move_regs(f_lo, t_lo);
 621   } else if (dest->is_single_fpu()) {
 622     assert(src->is_single_fpu(), "expect single fpu");
 623     __ fmv_s(dest->as_float_reg(), src->as_float_reg());
 624   } else if (dest->is_double_fpu()) {
 625     assert(src->is_double_fpu(), "expect double fpu");
 626     __ fmv_d(dest->as_double_reg(), src->as_double_reg());
 627   } else {
 628     ShouldNotReachHere();
 629   }
 630 }
 631 
 632 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 633   precond(src->is_register() && dest->is_stack());
 634 
 635   uint const c_sz32 = sizeof(uint32_t);
 636   uint const c_sz64 = sizeof(uint64_t);
 637 
 638   assert(src->is_register(), "should not call otherwise");
 639   assert(dest->is_stack(), "should not call otherwise");
 640   if (src->is_single_cpu()) {
 641     int index = dest->single_stack_ix();
 642     if (is_reference_type(type)) {
 643       __ sd(src->as_register(), stack_slot_address(index, c_sz64));
 644       __ verify_oop(src->as_register());
 645     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 646       __ sd(src->as_register(), stack_slot_address(index, c_sz64));
 647     } else {
 648       __ sw(src->as_register(), stack_slot_address(index, c_sz32));
 649     }
 650   } else if (src->is_double_cpu()) {
 651     int index = dest->double_stack_ix();
 652     Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
 653     __ sd(src->as_register_lo(), dest_addr_LO);
 654   } else if (src->is_single_fpu()) {
 655     int index = dest->single_stack_ix();
 656     __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32));
 657   } else if (src->is_double_fpu()) {
 658     int index = dest->double_stack_ix();
 659     __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64));
 660   } else {
 661     ShouldNotReachHere();
 662   }
 663 }
 664 
 665 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 666   LIR_Address* to_addr = dest->as_address_ptr();
 667   // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src
 668   Register compressed_src = t1;
 669 
 670   if (patch_code != lir_patch_none) {
 671     deoptimize_trap(info);
 672     return;
 673   }
 674 
 675   if (is_reference_type(type)) {
 676     __ verify_oop(src->as_register());
 677 
 678     if (UseCompressedOops && !wide) {
 679       __ encode_heap_oop(compressed_src, src->as_register());
 680     } else {
 681       compressed_src = src->as_register();
 682     }
 683   }
 684 
 685   int null_check_here = code_offset();
 686 
 687   switch (type) {
 688     case T_FLOAT:
 689       __ fsw(src->as_float_reg(), as_Address(to_addr));
 690       break;
 691 
 692     case T_DOUBLE:
 693       __ fsd(src->as_double_reg(), as_Address(to_addr));
 694       break;
 695 
 696     case T_ARRAY:      // fall through
 697     case T_OBJECT:
 698       if (UseCompressedOops && !wide) {
 699         __ sw(compressed_src, as_Address(to_addr));
 700       } else {
 701         __ sd(compressed_src, as_Address(to_addr));
 702       }
 703       break;
 704     case T_METADATA:
 705       // We get here to store a method pointer to the stack to pass to
 706       // a dtrace runtime call. This can't work on 64 bit with
 707       // compressed klass ptrs: T_METADATA can be compressed klass
 708       // ptr or a 64 bit method pointer.
 709       ShouldNotReachHere();
 710       __ sd(src->as_register(), as_Address(to_addr));
 711       break;
 712     case T_ADDRESS:
 713       __ sd(src->as_register(), as_Address(to_addr));
 714       break;
 715     case T_INT:
 716       __ sw(src->as_register(), as_Address(to_addr));
 717       break;
 718     case T_LONG:
 719       __ sd(src->as_register_lo(), as_Address(to_addr));
 720       break;
 721     case T_BYTE:    // fall through
 722     case T_BOOLEAN:
 723       __ sb(src->as_register(), as_Address(to_addr));
 724       break;
 725     case T_CHAR:    // fall through
 726     case T_SHORT:
 727       __ sh(src->as_register(), as_Address(to_addr));
 728       break;
 729     default:
 730       ShouldNotReachHere();
 731   }
 732 
 733   if (info != NULL) {
 734     add_debug_info_for_null_check(null_check_here, info);
 735   }
 736 }
 737 
 738 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 739   precond(src->is_stack() && dest->is_register());
 740 
 741   uint const c_sz32 = sizeof(uint32_t);
 742   uint const c_sz64 = sizeof(uint64_t);
 743 
 744   if (dest->is_single_cpu()) {
 745     int index = src->single_stack_ix();
 746     if (type == T_INT) {
 747       __ lw(dest->as_register(), stack_slot_address(index, c_sz32));
 748     } else if (is_reference_type(type)) {
 749       __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
 750       __ verify_oop(dest->as_register());
 751     } else if (type == T_METADATA || type == T_ADDRESS) {
 752       __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
 753     } else {
 754       __ lwu(dest->as_register(), stack_slot_address(index, c_sz32));
 755     }
 756   } else if (dest->is_double_cpu()) {
 757     int index = src->double_stack_ix();
 758     Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
 759     __ ld(dest->as_register_lo(), src_addr_LO);
 760   } else if (dest->is_single_fpu()) {
 761     int index = src->single_stack_ix();
 762     __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32));
 763   } else if (dest->is_double_fpu()) {
 764     int index = src->double_stack_ix();
 765     __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64));
 766   } else {
 767     ShouldNotReachHere();
 768   }
 769 }
 770 
 771 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 772   deoptimize_trap(info);
 773 }
 774 
 775 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 776   LIR_Opr temp;
 777   if (type == T_LONG || type == T_DOUBLE) {
 778     temp = FrameMap::t1_long_opr;
 779   } else {
 780     temp = FrameMap::t1_opr;
 781   }
 782 
 783   stack2reg(src, temp, src->type());
 784   reg2stack(temp, dest, dest->type(), false);
 785 }
 786 
 787 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 788   assert(src->is_address(), "should not call otherwise");
 789   assert(dest->is_register(), "should not call otherwise");
 790 
 791   LIR_Address* addr = src->as_address_ptr();
 792   LIR_Address* from_addr = src->as_address_ptr();
 793 
 794   if (addr->base()->type() == T_OBJECT) {
 795     __ verify_oop(addr->base()->as_pointer_register());
 796   }
 797 
 798   if (patch_code != lir_patch_none) {
 799     deoptimize_trap(info);
 800     return;
 801   }
 802 
 803   if (info != NULL) {
 804     add_debug_info_for_null_check_here(info);
 805   }
 806 
 807   int null_check_here = code_offset();
 808   switch (type) {
 809     case T_FLOAT:
 810       __ flw(dest->as_float_reg(), as_Address(from_addr));
 811       break;
 812     case T_DOUBLE:
 813       __ fld(dest->as_double_reg(), as_Address(from_addr));
 814       break;
 815     case T_ARRAY:     // fall through
 816     case T_OBJECT:
 817       if (UseCompressedOops && !wide) {
 818         __ lwu(dest->as_register(), as_Address(from_addr));
 819       } else {
 820         __ ld(dest->as_register(), as_Address(from_addr));
 821       }
 822       break;
 823     case T_METADATA:
 824       // We get here to store a method pointer to the stack to pass to
 825       // a dtrace runtime call. This can't work on 64 bit with
 826       // compressed klass ptrs: T_METADATA can be a compressed klass
 827       // ptr or a 64 bit method pointer.
 828       ShouldNotReachHere();
 829       __ ld(dest->as_register(), as_Address(from_addr));
 830       break;
 831     case T_ADDRESS:
 832       __ ld(dest->as_register(), as_Address(from_addr));
 833       break;
 834     case T_INT:
 835       __ lw(dest->as_register(), as_Address(from_addr));
 836       break;
 837     case T_LONG:
 838       __ ld(dest->as_register_lo(), as_Address_lo(from_addr));
 839       break;
 840     case T_BYTE:
 841       __ lb(dest->as_register(), as_Address(from_addr));
 842       break;
 843     case T_BOOLEAN:
 844       __ lbu(dest->as_register(), as_Address(from_addr));
 845       break;
 846     case T_CHAR:
 847       __ lhu(dest->as_register(), as_Address(from_addr));
 848       break;
 849     case T_SHORT:
 850       __ lh(dest->as_register(), as_Address(from_addr));
 851       break;
 852     default:
 853       ShouldNotReachHere();
 854   }
 855 
 856   if (is_reference_type(type)) {
 857     if (UseCompressedOops && !wide) {
 858       __ decode_heap_oop(dest->as_register());
 859     }
 860 
 861     if (!UseZGC) {
 862       // Load barrier has not yet been applied, so ZGC can't verify the oop here
 863       __ verify_oop(dest->as_register());
 864     }
 865   }
 866 }
 867 
 868 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 869   switch (op->code()) {
 870     case lir_idiv: // fall through
 871     case lir_irem:
 872       arithmetic_idiv(op->code(),
 873                       op->in_opr1(),
 874                       op->in_opr2(),
 875                       op->in_opr3(),
 876                       op->result_opr(),
 877                       op->info());
 878       break;
 879     case lir_fmad:
 880       __ fmadd_d(op->result_opr()->as_double_reg(),
 881                  op->in_opr1()->as_double_reg(),
 882                  op->in_opr2()->as_double_reg(),
 883                  op->in_opr3()->as_double_reg());
 884       break;
 885     case lir_fmaf:
 886       __ fmadd_s(op->result_opr()->as_float_reg(),
 887                  op->in_opr1()->as_float_reg(),
 888                  op->in_opr2()->as_float_reg(),
 889                  op->in_opr3()->as_float_reg());
 890       break;
 891     default:
 892       ShouldNotReachHere();
 893   }
 894 }
 895 
 896 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
 897                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
 898   Label label;
 899 
 900   emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false,
 901               /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true);
 902 
 903   Label done;
 904   move_op(opr2, result, type, lir_patch_none, NULL,
 905           false,   // pop_fpu_stack
 906           false);  // wide
 907   __ j(done);
 908   __ bind(label);
 909   move_op(opr1, result, type, lir_patch_none, NULL,
 910           false,   // pop_fpu_stack
 911           false);  // wide
 912   __ bind(done);
 913 }
 914 
 915 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 916   LIR_Condition condition = op->cond();
 917   if (condition == lir_cond_always) {
 918     if (op->info() != NULL) {
 919       add_debug_info_for_branch(op->info());
 920     }
 921   } else {
 922     assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands");
 923   }
 924   bool is_unordered = (op->ublock() == op->block());
 925   emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered);
 926 }
 927 
 928 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label,
 929                                 bool is_far, bool is_unordered) {
 930 
 931   if (cmp_flag == lir_cond_always) {
 932     __ j(label);
 933     return;
 934   }
 935 
 936   if (cmp1->is_cpu_register()) {
 937     Register reg1 = as_reg(cmp1);
 938     if (cmp2->is_cpu_register()) {
 939       Register reg2 = as_reg(cmp2);
 940       __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far);
 941     } else if (cmp2->is_constant()) {
 942       const2reg_helper(cmp2);
 943       __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far);
 944     } else {
 945       ShouldNotReachHere();
 946     }
 947   } else if (cmp1->is_single_fpu()) {
 948     assert(cmp2->is_single_fpu(), "expect single float register");
 949     __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered);
 950   } else if (cmp1->is_double_fpu()) {
 951     assert(cmp2->is_double_fpu(), "expect double float register");
 952     __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask,
 953                            cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered);
 954   } else {
 955     ShouldNotReachHere();
 956   }
 957 }
 958 
 959 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 960   LIR_Opr src  = op->in_opr();
 961   LIR_Opr dest = op->result_opr();
 962 
 963   switch (op->bytecode()) {
 964     case Bytecodes::_i2f:
 965       __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break;
 966     case Bytecodes::_i2d:
 967       __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break;
 968     case Bytecodes::_l2d:
 969       __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break;
 970     case Bytecodes::_l2f:
 971       __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break;
 972     case Bytecodes::_f2d:
 973       __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break;
 974     case Bytecodes::_d2f:
 975       __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break;
 976     case Bytecodes::_i2c:
 977       __ zero_extend(dest->as_register(), src->as_register(), 16); break;
 978     case Bytecodes::_i2l:
 979       __ addw(dest->as_register_lo(), src->as_register(), zr); break;
 980     case Bytecodes::_i2s:
 981       __ sign_extend(dest->as_register(), src->as_register(), 16); break;
 982     case Bytecodes::_i2b:
 983       __ sign_extend(dest->as_register(), src->as_register(), 8); break;
 984     case Bytecodes::_l2i:
 985       _masm->block_comment("FIXME: This coulde be no-op");
 986       __ addw(dest->as_register(), src->as_register_lo(), zr); break;
 987     case Bytecodes::_d2l:
 988       __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break;
 989     case Bytecodes::_f2i:
 990       __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break;
 991     case Bytecodes::_f2l:
 992       __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break;
 993     case Bytecodes::_d2i:
 994       __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break;
 995     default:
 996       ShouldNotReachHere();
 997   }
 998 }
 999 
1000 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1001   if (op->init_check()) {
1002     __ lbu(t0, Address(op->klass()->as_register(),
1003                        InstanceKlass::init_state_offset()));
1004     __ mvw(t1, InstanceKlass::fully_initialized);
1005     add_debug_info_for_null_check_here(op->stub()->info());
1006     __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
1007   }
1008 
1009   __ allocate_object(op->obj()->as_register(),
1010                      op->tmp1()->as_register(),
1011                      op->tmp2()->as_register(),
1012                      op->header_size(),
1013                      op->object_size(),
1014                      op->klass()->as_register(),
1015                      *op->stub()->entry());
1016 
1017   __ bind(*op->stub()->continuation());
1018 }
1019 
1020 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1021   Register len = op->len()->as_register();
1022 
1023   if (UseSlowPath ||
1024       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1025       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1026     __ j(*op->stub()->entry());
1027   } else {
1028     Register tmp1 = op->tmp1()->as_register();
1029     Register tmp2 = op->tmp2()->as_register();
1030     Register tmp3 = op->tmp3()->as_register();
1031     if (len == tmp1) {
1032       tmp1 = tmp3;
1033     } else if (len == tmp2) {
1034       tmp2 = tmp3;
1035     } else if (len == tmp3) {
1036       // everything is ok
1037     } else {
1038       __ mv(tmp3, len);
1039     }
1040     __ allocate_array(op->obj()->as_register(),
1041                       len,
1042                       tmp1,
1043                       tmp2,
1044                       arrayOopDesc::header_size(op->type()),
1045                       array_element_size(op->type()),
1046                       op->klass()->as_register(),
1047                       *op->stub()->entry());
1048   }
1049   __ bind(*op->stub()->continuation());
1050 }
1051 
1052 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
1053                                         Register recv, Label* update_done) {
1054   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1055     Label next_test;
1056     // See if the receiver is receiver[n].
1057     __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1058     __ bne(recv, t1, next_test);
1059     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1060     __ increment(data_addr, DataLayout::counter_increment);
1061     __ j(*update_done);
1062     __ bind(next_test);
1063   }
1064 
1065   // Didn't find receiver; find next empty slot and fill it in
1066   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1067     Label next_test;
1068     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1069     __ ld(t1, recv_addr);
1070     __ bnez(t1, next_test);
1071     __ sd(recv, recv_addr);
1072     __ mv(t1, DataLayout::counter_increment);
1073     __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1074     __ j(*update_done);
1075     __ bind(next_test);
1076   }
1077 }
1078 
1079 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
1080   ciMethod* method = op->profiled_method();
1081   assert(method != NULL, "Should have method");
1082   int bci = op->profiled_bci();
1083   *md = method->method_data_or_null();
1084   guarantee(*md != NULL, "Sanity");
1085   *data = ((*md)->bci_to_data(bci));
1086   assert(*data != NULL, "need data for type check");
1087   assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1088 }
1089 
1090 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1091                                                Register k_RInfo, Register klass_RInfo,
1092                                                Label *failure_target, Label *success_target) {
1093   // get object class
1094   // not a safepoint as obj null check happens earlier
1095   __ load_klass(klass_RInfo, obj);
1096   if (k->is_loaded()) {
1097     // See if we get an immediate positive hit
1098     __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1099     if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1100       __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1101       // successful cast, fall through to profile or jump
1102     } else {
1103       // See if we get an immediate positive hit
1104       __ beq(k_RInfo, t0, *success_target);
1105       // check for self
1106       __ beq(klass_RInfo, k_RInfo, *success_target);
1107 
1108       __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1109       __ sd(k_RInfo, Address(sp, 0));             // sub klass
1110       __ sd(klass_RInfo, Address(sp, wordSize));  // super klass
1111       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1112       // load result to k_RInfo
1113       __ ld(k_RInfo, Address(sp, 0));
1114       __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1115       // result is a boolean
1116       __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1117       // successful cast, fall through to profile or jump
1118     }
1119   } else {
1120     // perform the fast part of the checking logic
1121     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1122     // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1123     __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1124     __ sd(klass_RInfo, Address(sp, wordSize));  // sub klass
1125     __ sd(k_RInfo, Address(sp, 0));             // super klass
1126     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1127     // load result to k_RInfo
1128     __ ld(k_RInfo, Address(sp, 0));
1129     __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1130     // result is a boolean
1131     __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1132     // successful cast, fall thriugh to profile or jump
1133   }
1134 }
1135 
1136 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj,
1137                                    Register klass_RInfo, Label* obj_is_null) {
1138   Label not_null;
1139   __ bnez(obj, not_null);
1140   // Object is null, update MDO and exit
1141   Register mdo = klass_RInfo;
1142   __ mov_metadata(mdo, md->constant_encoding());
1143   Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1144   __ lbu(t0, data_addr);
1145   __ ori(t0, t0, BitData::null_seen_byte_constant());
1146   __ sb(t0, data_addr);
1147   __ j(*obj_is_null);
1148   __ bind(not_null);
1149 }
1150 
1151 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
1152   if (!k->is_loaded()) {
1153     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1154   } else {
1155     __ mov_metadata(k_RInfo, k->constant_encoding());
1156   }
1157 }
1158 
1159 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1160   Register obj = op->object()->as_register();
1161   Register k_RInfo = op->tmp1()->as_register();
1162   Register klass_RInfo = op->tmp2()->as_register();
1163   Register dst = op->result_opr()->as_register();
1164   ciKlass* k = op->klass();
1165   Register Rtmp1 = noreg;
1166 
1167   // check if it needs to be profiled
1168   ciMethodData* md = NULL;
1169   ciProfileData* data = NULL;
1170 
1171   const bool should_profile = op->should_profile();
1172   if (should_profile) {
1173     data_check(op, &md, &data);
1174   }
1175   Label profile_cast_success, profile_cast_failure;
1176   Label *success_target = should_profile ? &profile_cast_success : success;
1177   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1178 
1179   if (obj == k_RInfo) {
1180     k_RInfo = dst;
1181   } else if (obj == klass_RInfo) {
1182     klass_RInfo = dst;
1183   }
1184   if (k->is_loaded() && !UseCompressedClassPointers) {
1185     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1186   } else {
1187     Rtmp1 = op->tmp3()->as_register();
1188     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1189   }
1190 
1191   assert_different_registers(obj, k_RInfo, klass_RInfo);
1192 
1193   if (should_profile) {
1194     profile_object(md, data, obj, klass_RInfo, obj_is_null);
1195   } else {
1196     __ beqz(obj, *obj_is_null);
1197   }
1198 
1199   typecheck_loaded(op, k, k_RInfo);
1200   __ verify_oop(obj);
1201 
1202   if (op->fast_check()) {
1203     // get object class
1204     // not a safepoint as obj null check happens earlier
1205     __ load_klass(t0, obj);
1206     __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1207     // successful cast, fall through to profile or jump
1208   } else {
1209     typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1210   }
1211   if (should_profile) {
1212     type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure);
1213   }
1214   __ j(*success);
1215 }
1216 
1217 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1218   const bool should_profile = op->should_profile();
1219 
1220   LIR_Code code = op->code();
1221   if (code == lir_store_check) {
1222     typecheck_lir_store(op, should_profile);
1223   } else if (code == lir_checkcast) {
1224     Register obj = op->object()->as_register();
1225     Register dst = op->result_opr()->as_register();
1226     Label success;
1227     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1228     __ bind(success);
1229     if (dst != obj) {
1230       __ mv(dst, obj);
1231     }
1232   } else if (code == lir_instanceof) {
1233     Register obj = op->object()->as_register();
1234     Register dst = op->result_opr()->as_register();
1235     Label success, failure, done;
1236     emit_typecheck_helper(op, &success, &failure, &failure);
1237     __ bind(failure);
1238     __ mv(dst, zr);
1239     __ j(done);
1240     __ bind(success);
1241     __ mv(dst, 1);
1242     __ bind(done);
1243   } else {
1244     ShouldNotReachHere();
1245   }
1246 }
1247 
1248 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1249   assert(VM_Version::supports_cx8(), "wrong machine");
1250   Register addr;
1251   if (op->addr()->is_register()) {
1252     addr = as_reg(op->addr());
1253   } else {
1254     assert(op->addr()->is_address(), "what else?");
1255     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1256     assert(addr_ptr->disp() == 0, "need 0 disp");
1257     assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1258     addr = as_reg(addr_ptr->base());
1259   }
1260   Register newval = as_reg(op->new_value());
1261   Register cmpval = as_reg(op->cmp_value());
1262 
1263   if (op->code() == lir_cas_obj) {
1264     if (UseCompressedOops) {
1265       Register tmp1 = op->tmp1()->as_register();
1266       assert(op->tmp1()->is_valid(), "must be");
1267       __ encode_heap_oop(tmp1, cmpval);
1268       cmpval = tmp1;
1269       __ encode_heap_oop(t1, newval);
1270       newval = t1;
1271       caswu(addr, newval, cmpval);
1272     } else {
1273       casl(addr, newval, cmpval);
1274     }
1275   } else if (op->code() == lir_cas_int) {
1276     casw(addr, newval, cmpval);
1277   } else {
1278     casl(addr, newval, cmpval);
1279   }
1280 }
1281 
1282 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1283   switch (code) {
1284     case lir_abs:  __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break;
1285     case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break;
1286     default:       ShouldNotReachHere();
1287   }
1288 }
1289 
1290 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1291   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1292   Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1293   if (dst->is_single_cpu()) {
1294     Register Rdst = dst->as_register();
1295     if (right->is_constant()) {
1296       int right_const = right->as_jint();
1297       if (Assembler::operand_valid_for_add_immediate(right_const)) {
1298         logic_op_imm(Rdst, Rleft, right_const, code);
1299         __ addw(Rdst, Rdst, zr);
1300      } else {
1301         __ mv(t0, right_const);
1302         logic_op_reg32(Rdst, Rleft, t0, code);
1303      }
1304     } else {
1305       Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1306       logic_op_reg32(Rdst, Rleft, Rright, code);
1307     }
1308   } else {
1309     Register Rdst = dst->as_register_lo();
1310     if (right->is_constant()) {
1311       long right_const = right->as_jlong();
1312       if (Assembler::operand_valid_for_add_immediate(right_const)) {
1313         logic_op_imm(Rdst, Rleft, right_const, code);
1314       } else {
1315         __ mv(t0, right_const);
1316         logic_op_reg(Rdst, Rleft, t0, code);
1317       }
1318     } else {
1319       Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1320       logic_op_reg(Rdst, Rleft, Rright, code);
1321     }
1322   }
1323 }
1324 
1325 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) {
1326   ShouldNotCallThis();
1327 }
1328 
1329 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1330   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1331     bool is_unordered_less = (code == lir_ucmp_fd2i);
1332     if (left->is_single_fpu()) {
1333       __ float_cmp(true, is_unordered_less ? -1 : 1,
1334                    left->as_float_reg(), right->as_float_reg(), dst->as_register());
1335     } else if (left->is_double_fpu()) {
1336       __ float_cmp(false, is_unordered_less ? -1 : 1,
1337                    left->as_double_reg(), right->as_double_reg(), dst->as_register());
1338     } else {
1339       ShouldNotReachHere();
1340     }
1341   } else if (code == lir_cmp_l2i) {
1342     __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1343   } else {
1344     ShouldNotReachHere();
1345   }
1346 }
1347 
1348 void LIR_Assembler::align_call(LIR_Code code) {
1349   // With RVC a call instruction may get 2-byte aligned.
1350   // The address of the call instruction needs to be 4-byte aligned to
1351   // ensure that it does not span a cache line so that it can be patched.
1352   __ align(NativeInstruction::instruction_size);
1353 }
1354 
1355 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1356   address call = __ trampoline_call(Address(op->addr(), rtype));
1357   if (call == NULL) {
1358     bailout("trampoline stub overflow");
1359     return;
1360   }
1361   add_call_info(code_offset(), op->info());
1362 }
1363 
1364 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1365   address call = __ ic_call(op->addr());
1366   if (call == NULL) {
1367     bailout("trampoline stub overflow");
1368     return;
1369   }
1370   add_call_info(code_offset(), op->info());
1371 }
1372 
1373 void LIR_Assembler::emit_static_call_stub() {
1374   address call_pc = __ pc();
1375   MacroAssembler::assert_alignment(call_pc);
1376   address stub = __ start_a_stub(call_stub_size());
1377   if (stub == NULL) {
1378     bailout("static call stub overflow");
1379     return;
1380   }
1381 
1382   int start = __ offset();
1383 
1384   __ relocate(static_stub_Relocation::spec(call_pc));
1385   __ emit_static_call_stub();
1386 
1387   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
1388          <= call_stub_size(), "stub too big");
1389   __ end_a_stub();
1390 }
1391 
1392 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1393   assert(exceptionOop->as_register() == x10, "must match");
1394   assert(exceptionPC->as_register() == x13, "must match");
1395 
1396   // exception object is not added to oop map by LinearScan
1397   // (LinearScan assumes that no oops are in fixed registers)
1398   info->add_register_oop(exceptionOop);
1399   Runtime1::StubID unwind_id;
1400 
1401   // get current pc information
1402   // pc is only needed if the method has an exception handler, the unwind code does not need it.
1403   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
1404     // As no instructions have been generated yet for this LIR node it's
1405     // possible that an oop map already exists for the current offset.
1406     // In that case insert an dummy NOP here to ensure all oop map PCs
1407     // are unique. See JDK-8237483.
1408     __ nop();
1409   }
1410   int pc_for_athrow_offset = __ offset();
1411   InternalAddress pc_for_athrow(__ pc());
1412   int32_t off = 0;
1413   __ la_patchable(exceptionPC->as_register(), pc_for_athrow, off);
1414   __ addi(exceptionPC->as_register(), exceptionPC->as_register(), off);
1415   add_call_info(pc_for_athrow_offset, info); // for exception handler
1416 
1417   __ verify_not_null_oop(x10);
1418   // search an exception handler (x10: exception oop, x13: throwing pc)
1419   if (compilation()->has_fpu_code()) {
1420     unwind_id = Runtime1::handle_exception_id;
1421   } else {
1422     unwind_id = Runtime1::handle_exception_nofpu_id;
1423   }
1424   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
1425   __ nop();
1426 }
1427 
1428 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1429   assert(exceptionOop->as_register() == x10, "must match");
1430   __ j(_unwind_handler_entry);
1431 }
1432 
1433 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
1434   Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1435   Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1436   Register count_reg = count->as_register();
1437   if (dest->is_single_cpu()) {
1438     assert (dest->type() == T_INT, "unexpected result type");
1439     assert (left->type() == T_INT, "unexpected left type");
1440     __ andi(t0, count_reg, 31); // should not shift more than 31 bits
1441     switch (code) {
1442       case lir_shl:  __ sllw(dest_reg, left_reg, t0); break;
1443       case lir_shr:  __ sraw(dest_reg, left_reg, t0); break;
1444       case lir_ushr: __ srlw(dest_reg, left_reg, t0); break;
1445       default: ShouldNotReachHere();
1446     }
1447   } else if (dest->is_double_cpu()) {
1448     __ andi(t0, count_reg, 63); // should not shift more than 63 bits
1449     switch (code) {
1450       case lir_shl:  __ sll(dest_reg, left_reg, t0); break;
1451       case lir_shr:  __ sra(dest_reg, left_reg, t0); break;
1452       case lir_ushr: __ srl(dest_reg, left_reg, t0); break;
1453       default: ShouldNotReachHere();
1454     }
1455   } else {
1456     ShouldNotReachHere();
1457   }
1458 }
1459 
1460 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
1461   Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1462   Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1463   if (dest->is_single_cpu()) {
1464     assert (dest->type() == T_INT, "unexpected result type");
1465     assert (left->type() == T_INT, "unexpected left type");
1466     count &= 0x1f;
1467     if (count != 0) {
1468       switch (code) {
1469         case lir_shl:  __ slliw(dest_reg, left_reg, count); break;
1470         case lir_shr:  __ sraiw(dest_reg, left_reg, count); break;
1471         case lir_ushr: __ srliw(dest_reg, left_reg, count); break;
1472         default: ShouldNotReachHere();
1473       }
1474     } else {
1475       move_regs(left_reg, dest_reg);
1476     }
1477   } else if (dest->is_double_cpu()) {
1478     count &= 0x3f;
1479     if (count != 0) {
1480       switch (code) {
1481         case lir_shl:  __ slli(dest_reg, left_reg, count); break;
1482         case lir_shr:  __ srai(dest_reg, left_reg, count); break;
1483         case lir_ushr: __ srli(dest_reg, left_reg, count); break;
1484         default: ShouldNotReachHere();
1485       }
1486     } else {
1487       move_regs(left->as_register_lo(), dest->as_register_lo());
1488     }
1489   } else {
1490     ShouldNotReachHere();
1491   }
1492 }
1493 
1494 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
1495   Register obj = op->obj_opr()->as_register();  // may not be an oop
1496   Register hdr = op->hdr_opr()->as_register();
1497   Register lock = op->lock_opr()->as_register();
1498   if (UseHeavyMonitors) {
1499     if (op->info() != NULL) {
1500       add_debug_info_for_null_check_here(op->info());
1501       __ null_check(obj);
1502     }
1503     __ j(*op->stub()->entry());
1504   } else if (op->code() == lir_lock) {
1505     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1506     // add debug info for NullPointerException only if one is possible
1507     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
1508     if (op->info() != NULL) {
1509       add_debug_info_for_null_check(null_check_offset, op->info());
1510     }
1511   } else if (op->code() == lir_unlock) {
1512     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1513     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
1514   } else {
1515     Unimplemented();
1516   }
1517   __ bind(*op->stub()->continuation());
1518 }
1519 
1520 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
1521   Register obj = op->obj()->as_pointer_register();
1522   Register result = op->result_opr()->as_pointer_register();
1523 
1524   CodeEmitInfo* info = op->info();
1525   if (info != NULL) {
1526     add_debug_info_for_null_check_here(info);
1527   }
1528 
1529   if (UseCompressedClassPointers) {
1530     __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes()));
1531     __ decode_klass_not_null(result);
1532   } else {
1533     __ ld(result, Address(obj, oopDesc::klass_offset_in_bytes()));
1534   }
1535 }
1536 
1537 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
1538   ciMethod* method = op->profiled_method();
1539   int bci          = op->profiled_bci();
1540 
1541   // Update counter for all call types
1542   ciMethodData* md = method->method_data_or_null();
1543   guarantee(md != NULL, "Sanity");
1544   ciProfileData* data = md->bci_to_data(bci);
1545   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
1546   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
1547   Register mdo  = op->mdo()->as_register();
1548   __ mov_metadata(mdo, md->constant_encoding());
1549   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1550   // Perform additional virtual call profiling for invokevirtual and
1551   // invokeinterface bytecodes
1552   if (op->should_profile_receiver_type()) {
1553     assert(op->recv()->is_single_cpu(), "recv must be allocated");
1554     Register recv = op->recv()->as_register();
1555     assert_different_registers(mdo, recv);
1556     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
1557     ciKlass* known_klass = op->known_holder();
1558     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
1559       // We know the type that will be seen at this call site; we can
1560       // statically update the MethodData* rather than needing to do
1561       // dynamic tests on the receiver type
1562       // NOTE: we should probably put a lock around this search to
1563       // avoid collisions by concurrent compilations
1564       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
1565       uint i;
1566       for (i = 0; i < VirtualCallData::row_limit(); i++) {
1567         ciKlass* receiver = vc_data->receiver(i);
1568         if (known_klass->equals(receiver)) {
1569           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1570           __ increment(data_addr, DataLayout::counter_increment);
1571           return;
1572         }
1573       }
1574 
1575       // Receiver type not found in profile data; select an empty slot
1576       // Note that this is less efficient than it should be because it
1577       // always does a write to the receiver part of the
1578       // VirtualCallData rather than just the first time
1579       for (i = 0; i < VirtualCallData::row_limit(); i++) {
1580         ciKlass* receiver = vc_data->receiver(i);
1581         if (receiver == NULL) {
1582           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
1583           __ mov_metadata(t1, known_klass->constant_encoding());
1584           __ sd(t1, recv_addr);
1585           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1586           __ increment(data_addr, DataLayout::counter_increment);
1587           return;
1588         }
1589       }
1590     } else {
1591       __ load_klass(recv, recv);
1592       Label update_done;
1593       type_profile_helper(mdo, md, data, recv, &update_done);
1594       // Receiver did not match any saved receiver and there is no empty row for it.
1595       // Increment total counter to indicate polymorphic case.
1596       __ increment(counter_addr, DataLayout::counter_increment);
1597 
1598       __ bind(update_done);
1599     }
1600   } else {
1601     // Static call
1602     __ increment(counter_addr, DataLayout::counter_increment);
1603   }
1604 }
1605 
1606 void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); }
1607 
1608 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
1609   __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
1610 }
1611 
1612 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); }
1613 
1614 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
1615                                    Register tmp, Label &next, Label &none,
1616                                    Address mdo_addr) {
1617   if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
1618     if (exact_klass != NULL) {
1619       __ mov_metadata(tmp, exact_klass->constant_encoding());
1620     } else {
1621       __ load_klass(tmp, tmp);
1622     }
1623 
1624     __ ld(t1, mdo_addr);
1625     __ xorr(tmp, tmp, t1);
1626     __ andi(t0, tmp, TypeEntries::type_klass_mask);
1627     // klass seen before, nothing to do. The unknown bit may have been
1628     // set already but no need to check.
1629     __ beqz(t0, next);
1630 
1631     // already unknown. Nothing to do anymore.
1632     __ andi(t0, tmp, TypeEntries::type_unknown);
1633     __ bnez(t0, next);
1634 
1635     if (TypeEntries::is_type_none(current_klass)) {
1636       __ beqz(t1, none);
1637       __ mv(t0, (u1)TypeEntries::null_seen);
1638       __ beq(t0, t1, none);
1639       // There is a chance that the checks above (re-reading profiling
1640       // data from memory) fail if another thread has just set the
1641       // profiling to this obj's klass
1642       __ membar(MacroAssembler::LoadLoad);
1643       __ ld(t1, mdo_addr);
1644       __ xorr(tmp, tmp, t1);
1645       __ andi(t0, tmp, TypeEntries::type_klass_mask);
1646       __ beqz(t0, next);
1647     }
1648   } else {
1649     assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1650            ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
1651 
1652     __ ld(tmp, mdo_addr);
1653     // already unknown. Nothing to do anymore.
1654     __ andi(t0, tmp, TypeEntries::type_unknown);
1655     __ bnez(t0, next);
1656   }
1657 
1658   // different than before. Cannot keep accurate profile.
1659   __ ld(t1, mdo_addr);
1660   __ ori(t1, t1, TypeEntries::type_unknown);
1661   __ sd(t1, mdo_addr);
1662 
1663   if (TypeEntries::is_type_none(current_klass)) {
1664     __ j(next);
1665 
1666     __ bind(none);
1667     // first time here. Set profile type.
1668     __ sd(tmp, mdo_addr);
1669   }
1670 }
1671 
1672 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp,
1673                                       Address mdo_addr, Label &next) {
1674   // There's a single possible klass at this profile point
1675   assert(exact_klass != NULL, "should be");
1676   if (TypeEntries::is_type_none(current_klass)) {
1677     __ mov_metadata(tmp, exact_klass->constant_encoding());
1678     __ ld(t1, mdo_addr);
1679     __ xorr(tmp, tmp, t1);
1680     __ andi(t0, tmp, TypeEntries::type_klass_mask);
1681     __ beqz(t0, next);
1682 #ifdef ASSERT
1683   {
1684     Label ok;
1685     __ ld(t0, mdo_addr);
1686     __ beqz(t0, ok);
1687     __ mv(t1, (u1)TypeEntries::null_seen);
1688     __ beq(t0, t1, ok);
1689     // may have been set by another thread
1690     __ membar(MacroAssembler::LoadLoad);
1691     __ mov_metadata(t0, exact_klass->constant_encoding());
1692     __ ld(t1, mdo_addr);
1693     __ xorr(t1, t0, t1);
1694     __ andi(t1, t1, TypeEntries::type_mask);
1695     __ beqz(t1, ok);
1696 
1697     __ stop("unexpected profiling mismatch");
1698     __ bind(ok);
1699   }
1700 #endif
1701     // first time here. Set profile type.
1702     __ sd(tmp, mdo_addr);
1703   } else {
1704     assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1705            ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
1706 
1707     __ ld(tmp, mdo_addr);
1708     // already unknown. Nothing to do anymore.
1709     __ andi(t0, tmp, TypeEntries::type_unknown);
1710     __ bnez(t0, next);
1711 
1712     __ ori(tmp, tmp, TypeEntries::type_unknown);
1713     __ sd(tmp, mdo_addr);
1714   }
1715 }
1716 
1717 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass,
1718                                Address mdo_addr, bool do_update, Label &next) {
1719   __ bnez(tmp, update);
1720   if (!TypeEntries::was_null_seen(current_klass)) {
1721     __ ld(t1, mdo_addr);
1722     __ ori(t1, t1, TypeEntries::null_seen);
1723     __ sd(t1, mdo_addr);
1724   }
1725   if (do_update) {
1726     __ j(next);
1727   }
1728 }
1729 
1730 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
1731   COMMENT("emit_profile_type {");
1732   Register obj = op->obj()->as_register();
1733   Register tmp = op->tmp()->as_pointer_register();
1734   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
1735   ciKlass* exact_klass = op->exact_klass();
1736   intptr_t current_klass = op->current_klass();
1737   bool not_null = op->not_null();
1738   bool no_conflict = op->no_conflict();
1739 
1740   Label update, next, none;
1741 
1742   bool do_null = !not_null;
1743   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
1744   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
1745 
1746   assert(do_null || do_update, "why are we here?");
1747   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
1748   assert_different_registers(tmp, t0, t1, mdo_addr.base());
1749 
1750   __ verify_oop(obj);
1751 
1752   if (tmp != obj) {
1753     __ mv(tmp, obj);
1754   }
1755   if (do_null) {
1756     check_null(tmp, update, current_klass, mdo_addr, do_update, next);
1757 #ifdef ASSERT
1758   } else {
1759     __ bnez(tmp, update);
1760     __ stop("unexpected null obj");
1761 #endif
1762   }
1763 
1764   __ bind(update);
1765 
1766   if (do_update) {
1767 #ifdef ASSERT
1768     if (exact_klass != NULL) {
1769       check_exact_klass(tmp, exact_klass);
1770     }
1771 #endif
1772     if (!no_conflict) {
1773       check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr);
1774     } else {
1775       check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next);
1776     }
1777 
1778     __ bind(next);
1779   }
1780   COMMENT("} emit_profile_type");
1781 }
1782 
1783 void LIR_Assembler::align_backward_branch_target() { }
1784 
1785 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
1786   // tmp must be unused
1787   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
1788 
1789   if (left->is_single_cpu()) {
1790     assert(dest->is_single_cpu(), "expect single result reg");
1791     __ negw(dest->as_register(), left->as_register());
1792   } else if (left->is_double_cpu()) {
1793     assert(dest->is_double_cpu(), "expect double result reg");
1794     __ neg(dest->as_register_lo(), left->as_register_lo());
1795   } else if (left->is_single_fpu()) {
1796     assert(dest->is_single_fpu(), "expect single float result reg");
1797     __ fneg_s(dest->as_float_reg(), left->as_float_reg());
1798   } else {
1799     assert(left->is_double_fpu(), "expect double float operand reg");
1800     assert(dest->is_double_fpu(), "expect double float result reg");
1801     __ fneg_d(dest->as_double_reg(), left->as_double_reg());
1802   }
1803 }
1804 
1805 
1806 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1807   if (patch_code != lir_patch_none) {
1808     deoptimize_trap(info);
1809     return;
1810   }
1811 
1812   LIR_Address* adr = addr->as_address_ptr();
1813   Register dst = dest->as_register_lo();
1814 
1815   assert_different_registers(dst, t0);
1816   if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) {
1817     int scale = adr->scale();
1818     intptr_t offset = adr->disp();
1819     LIR_Opr index_op = adr->index();
1820     if (index_op->is_constant()) {
1821       offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale;
1822     }
1823 
1824     if (!is_imm_in_range(offset, 12, 0)) {
1825       __ la(t0, as_Address(adr));
1826       __ mv(dst, t0);
1827       return;
1828     }
1829   }
1830 
1831   __ la(dst, as_Address(adr));
1832 }
1833 
1834 
1835 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
1836   assert(!tmp->is_valid(), "don't need temporary");
1837 
1838   CodeBlob *cb = CodeCache::find_blob(dest);
1839   if (cb != NULL) {
1840     __ far_call(RuntimeAddress(dest));
1841   } else {
1842     int32_t offset = 0;
1843     __ la_patchable(t0, RuntimeAddress(dest), offset);
1844     __ jalr(x1, t0, offset);
1845   }
1846 
1847   if (info != NULL) {
1848     add_call_info_here(info);
1849   }
1850 }
1851 
1852 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
1853   if (dest->is_address() || src->is_address()) {
1854     move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false);
1855   } else {
1856     ShouldNotReachHere();
1857   }
1858 }
1859 
1860 #ifdef ASSERT
1861 // emit run-time assertion
1862 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
1863   assert(op->code() == lir_assert, "must be");
1864 
1865   Label ok;
1866   if (op->in_opr1()->is_valid()) {
1867     assert(op->in_opr2()->is_valid(), "both operands must be valid");
1868     bool is_unordered = false;
1869     LIR_Condition cond = op->condition();
1870     emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false,
1871                 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true);
1872   } else {
1873     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
1874     assert(op->condition() == lir_cond_always, "no other conditions allowed");
1875   }
1876 
1877   if (op->halt()) {
1878     const char* str = __ code_string(op->msg());
1879     __ stop(str);
1880   } else {
1881     breakpoint();
1882   }
1883   __ bind(ok);
1884 }
1885 #endif
1886 
1887 #ifndef PRODUCT
1888 #define COMMENT(x)   do { __ block_comment(x); } while (0)
1889 #else
1890 #define COMMENT(x)
1891 #endif
1892 
1893 void LIR_Assembler::membar() {
1894   COMMENT("membar");
1895   __ membar(MacroAssembler::AnyAny);
1896 }
1897 
1898 void LIR_Assembler::membar_acquire() {
1899   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1900 }
1901 
1902 void LIR_Assembler::membar_release() {
1903   __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1904 }
1905 
1906 void LIR_Assembler::membar_loadload() {
1907   __ membar(MacroAssembler::LoadLoad);
1908 }
1909 
1910 void LIR_Assembler::membar_storestore() {
1911   __ membar(MacroAssembler::StoreStore);
1912 }
1913 
1914 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
1915 
1916 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
1917 
1918 void LIR_Assembler::on_spin_wait() {
1919   Unimplemented();
1920 }
1921 
1922 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
1923   __ mv(result_reg->as_register(), xthread);
1924 }
1925 
1926 void LIR_Assembler::peephole(LIR_List *lir) {}
1927 
1928 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
1929   Address addr = as_Address(src->as_address_ptr());
1930   BasicType type = src->type();
1931   bool is_oop = is_reference_type(type);
1932 
1933   get_op(type);
1934 
1935   switch (code) {
1936     case lir_xadd:
1937       {
1938         RegisterOrConstant inc;
1939         Register tmp = as_reg(tmp_op);
1940         Register dst = as_reg(dest);
1941         if (data->is_constant()) {
1942           inc = RegisterOrConstant(as_long(data));
1943           assert_different_registers(dst, addr.base(), tmp);
1944           assert_different_registers(tmp, t0);
1945         } else {
1946           inc = RegisterOrConstant(as_reg(data));
1947           assert_different_registers(inc.as_register(), dst, addr.base(), tmp);
1948         }
1949         __ la(tmp, addr);
1950         (_masm->*add)(dst, inc, tmp);
1951         break;
1952       }
1953     case lir_xchg:
1954       {
1955         Register tmp = tmp_op->as_register();
1956         Register obj = as_reg(data);
1957         Register dst = as_reg(dest);
1958         if (is_oop && UseCompressedOops) {
1959           __ encode_heap_oop(t0, obj);
1960           obj = t0;
1961         }
1962         assert_different_registers(obj, addr.base(), tmp, dst);
1963         __ la(tmp, addr);
1964         (_masm->*xchg)(dst, obj, tmp);
1965         if (is_oop && UseCompressedOops) {
1966           __ decode_heap_oop(dst);
1967         }
1968       }
1969       break;
1970     default:
1971       ShouldNotReachHere();
1972   }
1973   __ membar(MacroAssembler::AnyAny);
1974 }
1975 
1976 int LIR_Assembler::array_element_size(BasicType type) const {
1977   int elem_size = type2aelembytes(type);
1978   return exact_log2(elem_size);
1979 }
1980 
1981 // helper functions which checks for overflow and sets bailout if it
1982 // occurs.  Always returns a valid embeddable pointer but in the
1983 // bailout case the pointer won't be to unique storage.
1984 address LIR_Assembler::float_constant(float f) {
1985   address const_addr = __ float_constant(f);
1986   if (const_addr == NULL) {
1987     bailout("const section overflow");
1988     return __ code()->consts()->start();
1989   } else {
1990     return const_addr;
1991   }
1992 }
1993 
1994 address LIR_Assembler::double_constant(double d) {
1995   address const_addr = __ double_constant(d);
1996   if (const_addr == NULL) {
1997     bailout("const section overflow");
1998     return __ code()->consts()->start();
1999   } else {
2000     return const_addr;
2001   }
2002 }
2003 
2004 address LIR_Assembler::int_constant(jlong n) {
2005   address const_addr = __ long_constant(n);
2006   if (const_addr == NULL) {
2007     bailout("const section overflow");
2008     return __ code()->consts()->start();
2009   } else {
2010     return const_addr;
2011   }
2012 }
2013 
2014 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
2015   __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */,
2016              Assembler::rl /* release */, t0, true /* result as bool */);
2017   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2018   __ membar(MacroAssembler::AnyAny);
2019 }
2020 
2021 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) {
2022   __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */,
2023              Assembler::rl /* release */, t0, true /* result as bool */);
2024   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2025   __ membar(MacroAssembler::AnyAny);
2026 }
2027 
2028 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
2029   __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */,
2030              Assembler::rl /* release */, t0, true /* result as bool */);
2031   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2032   __ membar(MacroAssembler::AnyAny);
2033 }
2034 
2035 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
2036   address target = NULL;
2037 
2038   switch (patching_id(info)) {
2039     case PatchingStub::access_field_id:
2040       target = Runtime1::entry_for(Runtime1::access_field_patching_id);
2041       break;
2042     case PatchingStub::load_klass_id:
2043       target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
2044       break;
2045     case PatchingStub::load_mirror_id:
2046       target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
2047       break;
2048     case PatchingStub::load_appendix_id:
2049       target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
2050       break;
2051     default: ShouldNotReachHere();
2052   }
2053 
2054   __ far_call(RuntimeAddress(target));
2055   add_call_info_here(info);
2056 }
2057 
2058 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) {
2059   Label ok;
2060   __ load_klass(tmp, tmp);
2061   __ mov_metadata(t0, exact_klass->constant_encoding());
2062   __ beq(tmp, t0, ok);
2063   __ stop("exact klass and actual klass differ");
2064   __ bind(ok);
2065 }
2066 
2067 void LIR_Assembler::get_op(BasicType type) {
2068   switch (type) {
2069     case T_INT:
2070       xchg = &MacroAssembler::atomic_xchgalw;
2071       add = &MacroAssembler::atomic_addalw;
2072       break;
2073     case T_LONG:
2074       xchg = &MacroAssembler::atomic_xchgal;
2075       add = &MacroAssembler::atomic_addal;
2076       break;
2077     case T_OBJECT:
2078     case T_ARRAY:
2079       if (UseCompressedOops) {
2080         xchg = &MacroAssembler::atomic_xchgalwu;
2081         add = &MacroAssembler::atomic_addalw;
2082       } else {
2083         xchg = &MacroAssembler::atomic_xchgal;
2084         add = &MacroAssembler::atomic_addal;
2085       }
2086       break;
2087     default:
2088       ShouldNotReachHere();
2089   }
2090 }
2091 
2092 // emit_opTypeCheck sub functions
2093 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) {
2094   Register value = op->object()->as_register();
2095   Register array = op->array()->as_register();
2096   Register k_RInfo = op->tmp1()->as_register();
2097   Register klass_RInfo = op->tmp2()->as_register();
2098   Register Rtmp1 = op->tmp3()->as_register();
2099 
2100   CodeStub* stub = op->stub();
2101 
2102   // check if it needs to be profiled
2103   ciMethodData* md = NULL;
2104   ciProfileData* data = NULL;
2105 
2106   if (should_profile) {
2107     data_check(op, &md, &data);
2108   }
2109   Label profile_cast_success, profile_cast_failure, done;
2110   Label *success_target = should_profile ? &profile_cast_success : &done;
2111   Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
2112 
2113   if (should_profile) {
2114     profile_object(md, data, value, klass_RInfo, &done);
2115   } else {
2116     __ beqz(value, done);
2117   }
2118 
2119   add_debug_info_for_null_check_here(op->info_for_exception());
2120   __ load_klass(k_RInfo, array);
2121   __ load_klass(klass_RInfo, value);
2122 
2123   lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target);
2124 
2125   // fall through to the success case
2126   if (should_profile) {
2127     Register mdo = klass_RInfo;
2128     Register recv = k_RInfo;
2129     __ bind(profile_cast_success);
2130     __ mov_metadata(mdo, md->constant_encoding());
2131     __ load_klass(recv, value);
2132     type_profile_helper(mdo, md, data, recv, &done);
2133     __ j(done);
2134 
2135     __ bind(profile_cast_failure);
2136     __ mov_metadata(mdo, md->constant_encoding());
2137     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2138     __ ld(t1, counter_addr);
2139     __ addi(t1, t1, -DataLayout::counter_increment);
2140     __ sd(t1, counter_addr);
2141     __ j(*stub->entry());
2142   }
2143 
2144   __ bind(done);
2145 }
2146 
2147 void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo,
2148                                  ciProfileData* data, Label* success, Label* failure,
2149                                  Label& profile_cast_success, Label& profile_cast_failure) {
2150   Register mdo = klass_RInfo;
2151   Register recv = k_RInfo;
2152   __ bind(profile_cast_success);
2153   __ mov_metadata(mdo, md->constant_encoding());
2154   __ load_klass(recv, obj);
2155   Label update_done;
2156   type_profile_helper(mdo, md, data, recv, success);
2157   __ j(*success);
2158 
2159   __ bind(profile_cast_failure);
2160   __ mov_metadata(mdo, md->constant_encoding());
2161   Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2162   __ ld(t0, counter_addr);
2163   __ addi(t0, t0, -DataLayout::counter_increment);
2164   __ sd(t0, counter_addr);
2165   __ j(*failure);
2166 }
2167 
2168 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1,
2169                                         Label* success_target, Label* failure_target) {
2170   // get instance klass (it's already uncompressed)
2171   __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2172   // perform the fast part of the checking logic
2173   __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2174   // call out-of-line instance of __ check_klass_subtype_slow_path(...)
2175   __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
2176   __ sd(klass_RInfo, Address(sp, wordSize));  // sub klass
2177   __ sd(k_RInfo, Address(sp, 0));             // super klass
2178   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2179   // load result to k_RInfo
2180   __ ld(k_RInfo, Address(sp, 0));
2181   __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
2182   // result is a boolean
2183   __ beqz(k_RInfo, *failure_target, /* is_far */ true);
2184 }
2185 
2186 void LIR_Assembler::const2reg_helper(LIR_Opr src) {
2187   switch (src->as_constant_ptr()->type()) {
2188     case T_INT:
2189     case T_ADDRESS:
2190     case T_OBJECT:
2191     case T_ARRAY:
2192     case T_METADATA:
2193         const2reg(src, FrameMap::t0_opr, lir_patch_none, NULL);
2194         break;
2195     case T_LONG:
2196         const2reg(src, FrameMap::t0_long_opr, lir_patch_none, NULL);
2197         break;
2198     case T_FLOAT:
2199     case T_DOUBLE:
2200     default:
2201       ShouldNotReachHere();
2202   }
2203 }
2204 
2205 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) {
2206   switch (code) {
2207     case lir_logic_and: __ andrw(dst, left, right); break;
2208     case lir_logic_or:  __ orrw (dst, left, right); break;
2209     case lir_logic_xor: __ xorrw(dst, left, right); break;
2210     default:            ShouldNotReachHere();
2211   }
2212 }
2213 
2214 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) {
2215   switch (code) {
2216     case lir_logic_and: __ andr(dst, left, right); break;
2217     case lir_logic_or:  __ orr (dst, left, right); break;
2218     case lir_logic_xor: __ xorr(dst, left, right); break;
2219     default:            ShouldNotReachHere();
2220   }
2221 }
2222 
2223 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) {
2224   switch (code) {
2225     case lir_logic_and: __ andi(dst, left, right); break;
2226     case lir_logic_or:  __ ori (dst, left, right); break;
2227     case lir_logic_xor: __ xori(dst, left, right); break;
2228     default:            ShouldNotReachHere();
2229   }
2230 }
2231 
2232 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2233   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2234   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2235   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2236   __ sd(r, Address(sp, offset_from_rsp_in_bytes));
2237 }
2238 
2239 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2240   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2241   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2242   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2243   __ mv(t0, c);
2244   __ sd(t0, Address(sp, offset_from_rsp_in_bytes));
2245 }
2246 
2247 #undef __