1 /*
   2  * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "c1/c1_CodeStubs.hpp"
  31 #include "c1/c1_Compilation.hpp"
  32 #include "c1/c1_LIRAssembler.hpp"
  33 #include "c1/c1_MacroAssembler.hpp"
  34 #include "c1/c1_Runtime1.hpp"
  35 #include "c1/c1_ValueStack.hpp"
  36 #include "ci/ciArrayKlass.hpp"
  37 #include "ci/ciInstance.hpp"
  38 #include "code/compiledIC.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "nativeInst_riscv.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_riscv.inline.hpp"
  46 
  47 #ifndef PRODUCT
  48 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  49 #else
  50 #define COMMENT(x)
  51 #endif
  52 
  53 NEEDS_CLEANUP // remove this definitions ?
  54 const Register IC_Klass    = t1;    // where the IC klass is cached
  55 const Register SYNC_header = x10;   // synchronization header
  56 const Register SHIFT_count = x10;   // where count for shift operations must be
  57 
  58 #define __ _masm->
  59 
  60 static void select_different_registers(Register preserve,
  61                                        Register extra,
  62                                        Register &tmp1,
  63                                        Register &tmp2) {
  64   if (tmp1 == preserve) {
  65     assert_different_registers(tmp1, tmp2, extra);
  66     tmp1 = extra;
  67   } else if (tmp2 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp2 = extra;
  70   }
  71   assert_different_registers(preserve, tmp1, tmp2);
  72 }
  73 
  74 static void select_different_registers(Register preserve,
  75                                        Register extra,
  76                                        Register &tmp1,
  77                                        Register &tmp2,
  78                                        Register &tmp3) {
  79   if (tmp1 == preserve) {
  80     assert_different_registers(tmp1, tmp2, tmp3, extra);
  81     tmp1 = extra;
  82   } else if (tmp2 == preserve) {
  83     assert_different_registers(tmp1, tmp2, tmp3, extra);
  84     tmp2 = extra;
  85   } else if (tmp3 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp3 = extra;
  88   }
  89   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  90 }
  91 
  92 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  93 
  94 void LIR_Assembler::clinit_barrier(ciMethod* method) {
  95   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  96   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
  97 
  98   Label L_skip_barrier;
  99 
 100   __ mov_metadata(t1, method->holder()->constant_encoding());
 101   __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */);
 102   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 103   __ bind(L_skip_barrier);
 104 }
 105 
 106 LIR_Opr LIR_Assembler::receiverOpr() {
 107   return FrameMap::receiver_opr;
 108 }
 109 
 110 LIR_Opr LIR_Assembler::osrBufferPointer() {
 111   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 112 }
 113 
 114 void LIR_Assembler::breakpoint() { Unimplemented(); }
 115 
 116 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 117 
 118 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 119 
 120 static jlong as_long(LIR_Opr data) {
 121   jlong result;
 122   switch (data->type()) {
 123     case T_INT:
 124       result = (data->as_jint());
 125       break;
 126     case T_LONG:
 127       result = (data->as_jlong());
 128       break;
 129     default:
 130       ShouldNotReachHere();
 131       result = 0;  // unreachable
 132   }
 133   return result;
 134 }
 135 
 136 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 137   if (addr->base()->is_illegal()) {
 138     assert(addr->index()->is_illegal(), "must be illegal too");
 139     __ movptr(tmp, addr->disp());
 140     return Address(tmp, 0);
 141   }
 142 
 143   Register base = addr->base()->as_pointer_register();
 144   LIR_Opr index_opr = addr->index();
 145 
 146   if (index_opr->is_illegal()) {
 147     return Address(base, addr->disp());
 148   }
 149 
 150   int scale = addr->scale();
 151   if (index_opr->is_cpu_register()) {
 152     Register index;
 153     if (index_opr->is_single_cpu()) {
 154       index = index_opr->as_register();
 155     } else {
 156       index = index_opr->as_register_lo();
 157     }
 158     if (scale != 0) {
 159       __ shadd(tmp, index, base, tmp, scale);
 160     } else {
 161       __ add(tmp, base, index);
 162     }
 163     return Address(tmp, addr->disp());
 164   } else if (index_opr->is_constant()) {
 165     intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp();
 166     return Address(base, addr_offset);
 167   }
 168 
 169   Unimplemented();
 170   return Address();
 171 }
 172 
 173 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 174   ShouldNotReachHere();
 175   return Address();
 176 }
 177 
 178 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 179   return as_Address(addr, t0);
 180 }
 181 
 182 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 183   return as_Address(addr);
 184 }
 185 
 186 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 187 // not encodable as a base + (immediate) offset, generate an explicit address
 188 // calculation to hold the address in t0.
 189 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) {
 190   precond(size == 4 || size == 8);
 191   Address addr = frame_map()->address_for_slot(index, adjust);
 192   precond(addr.getMode() == Address::base_plus_offset);
 193   precond(addr.base() == sp);
 194   precond(addr.offset() > 0);
 195   uint mask = size - 1;
 196   assert((addr.offset() & mask) == 0, "scaled offsets only");
 197 
 198   return addr;
 199 }
 200 
 201 void LIR_Assembler::osr_entry() {
 202   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 203   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 204   guarantee(osr_entry != NULL, "NULL osr_entry!");
 205   ValueStack* entry_state = osr_entry->state();
 206   int number_of_locks = entry_state->locks_size();
 207 
 208   // we jump here if osr happens with the interpreter
 209   // state set up to continue at the beginning of the
 210   // loop that triggered osr - in particular, we have
 211   // the following registers setup:
 212   //
 213   // x12: osr buffer
 214   //
 215 
 216   //build frame
 217   ciMethod* m = compilation()->method();
 218   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 219 
 220   // OSR buffer is
 221   //
 222   // locals[nlocals-1..0]
 223   // monitors[0..number_of_locks]
 224   //
 225   // locals is a direct copy of the interpreter frame so in the osr buffer
 226   // so first slot in the local array is the last local from the interpreter
 227   // and last slot is local[0] (receiver) from the interpreter
 228   //
 229   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 230   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 231   // in the interpreter frame (the method lock if a sync method)
 232 
 233   // Initialize monitors in the compiled activation.
 234   //   x12: pointer to osr buffer
 235   // All other registers are dead at this point and the locals will be
 236   // copied into place by code emitted in the IR.
 237 
 238   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 239   {
 240     assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 241     int monitor_offset = BytesPerWord * method()->max_locals() +
 242       (2 * BytesPerWord) * (number_of_locks - 1);
 243     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 244     // the OSR buffer using 2 word entries: first the lock and then
 245     // the oop.
 246     for (int i = 0; i < number_of_locks; i++) {
 247       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 248 #ifdef ASSERT
 249       // verify the interpreter's monitor has a non-null object
 250       {
 251         Label L;
 252         __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
 253         __ bnez(t0, L);
 254         __ stop("locked object is NULL");
 255         __ bind(L);
 256       }
 257 #endif // ASSERT
 258       __ ld(x9, Address(OSR_buf, slot_offset + 0));
 259       __ sd(x9, frame_map()->address_for_monitor_lock(i));
 260       __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord));
 261       __ sd(x9, frame_map()->address_for_monitor_object(i));
 262     }
 263   }
 264 }
 265 
 266 // inline cache check; done before the frame is built.
 267 int LIR_Assembler::check_icache() {
 268   Register receiver = FrameMap::receiver_opr->as_register();
 269   Register ic_klass = IC_Klass;
 270   int start_offset = __ offset();
 271   Label dont;
 272   __ inline_cache_check(receiver, ic_klass, dont);
 273 
 274   // if icache check fails, then jump to runtime routine
 275   // Note: RECEIVER must still contain the receiver!
 276   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 277 
 278   // We align the verified entry point unless the method body
 279   // (including its inline cache check) will fit in a single 64-byte
 280   // icache line.
 281   if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 282     // force alignment after the cache check.
 283     __ align(CodeEntryAlignment);
 284   }
 285 
 286   __ bind(dont);
 287   return start_offset;
 288 }
 289 
 290 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 291   if (o == NULL) {
 292     __ mv(reg, zr);
 293   } else {
 294     __ movoop(reg, o, /* immediate */ true);
 295   }
 296 }
 297 
 298 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 299   deoptimize_trap(info);
 300 }
 301 
 302 // This specifies the rsp decrement needed to build the frame
 303 int LIR_Assembler::initial_frame_size_in_bytes() const {
 304   // if rounding, must let FrameMap know!
 305 
 306   return in_bytes(frame_map()->framesize_in_bytes());
 307 }
 308 
 309 int LIR_Assembler::emit_exception_handler() {
 310   // if the last instruction is a call (typically to do a throw which
 311   // is coming at the end after block reordering) the return address
 312   // must still point into the code area in order to avoid assertion
 313   // failures when searching for the corresponding bci ==> add a nop
 314   // (was bug 5/14/1999 -gri)
 315   __ nop();
 316 
 317   // generate code for exception handler
 318   address handler_base = __ start_a_stub(exception_handler_size());
 319   if (handler_base == NULL) {
 320     // not enough space left for the handler
 321     bailout("exception handler overflow");
 322     return -1;
 323   }
 324 
 325   int offset = code_offset();
 326 
 327   // the exception oop and pc are in x10, and x13
 328   // no other registers need to be preserved, so invalidate them
 329   __ invalidate_registers(false, true, true, false, true, true);
 330 
 331   // check that there is really an exception
 332   __ verify_not_null_oop(x10);
 333 
 334   // search an exception handler (x10: exception oop, x13: throwing pc)
 335   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 336   __ should_not_reach_here();
 337   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 338   __ end_a_stub();
 339 
 340   return offset;
 341 }
 342 
 343 // Emit the code to remove the frame from the stack in the exception
 344 // unwind path.
 345 int LIR_Assembler::emit_unwind_handler() {
 346 #ifndef PRODUCT
 347   if (CommentedAssembly) {
 348     _masm->block_comment("Unwind handler");
 349   }
 350 #endif // PRODUCT
 351 
 352   int offset = code_offset();
 353 
 354   // Fetch the exception from TLS and clear out exception related thread state
 355   __ ld(x10, Address(xthread, JavaThread::exception_oop_offset()));
 356   __ sd(zr, Address(xthread, JavaThread::exception_oop_offset()));
 357   __ sd(zr, Address(xthread, JavaThread::exception_pc_offset()));
 358 
 359   __ bind(_unwind_handler_entry);
 360   __ verify_not_null_oop(x10);
 361   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 362     __ mv(x9, x10);   // Perserve the exception
 363   }
 364 
 365   // Preform needed unlocking
 366   MonitorExitStub* stub = NULL;
 367   if (method()->is_synchronized()) {
 368     monitor_address(0, FrameMap::r10_opr);
 369     stub = new MonitorExitStub(FrameMap::r10_opr, true, 0);
 370     __ unlock_object(x15, x14, x10, *stub->entry());
 371     __ bind(*stub->continuation());
 372   }
 373 
 374   if (compilation()->env()->dtrace_method_probes()) {
 375     __ mv(c_rarg0, xthread);
 376     __ mov_metadata(c_rarg1, method()->constant_encoding());
 377     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 378   }
 379 
 380   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 381     __ mv(x10, x9);   // Restore the exception
 382   }
 383 
 384   // remove the activation and dispatch to the unwind handler
 385   __ block_comment("remove_frame and dispatch to the unwind handler");
 386   __ remove_frame(initial_frame_size_in_bytes());
 387   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 388 
 389   // Emit the slow path assembly
 390   if (stub != NULL) {
 391     stub->emit_code(this);
 392   }
 393 
 394   return offset;
 395 }
 396 
 397 int LIR_Assembler::emit_deopt_handler() {
 398   // if the last instruciton is a call (typically to do a throw which
 399   // is coming at the end after block reordering) the return address
 400   // must still point into the code area in order to avoid assertion
 401   // failures when searching for the corresponding bck => add a nop
 402   // (was bug 5/14/1999 - gri)
 403   __ nop();
 404 
 405   // generate code for exception handler
 406   address handler_base = __ start_a_stub(deopt_handler_size());
 407   if (handler_base == NULL) {
 408     // not enough space left for the handler
 409     bailout("deopt handler overflow");
 410     return -1;
 411   }
 412 
 413   int offset = code_offset();
 414 
 415   __ auipc(ra, 0);
 416   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 417   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 418   __ end_a_stub();
 419 
 420   return offset;
 421 }
 422 
 423 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 424   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10");
 425 
 426   // Pop the stack before the safepoint code
 427   __ remove_frame(initial_frame_size_in_bytes());
 428 
 429   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 430     __ reserved_stack_check();
 431   }
 432 
 433   code_stub->set_safepoint_offset(__ offset());
 434   __ relocate(relocInfo::poll_return_type);
 435   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 436   __ ret();
 437 }
 438 
 439 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 440   guarantee(info != NULL, "Shouldn't be NULL");
 441   __ get_polling_page(t0, relocInfo::poll_type);
 442   add_debug_info_for_branch(info);  // This isn't just debug info:
 443                                     // it's the oop map
 444   __ read_polling_page(t0, 0, relocInfo::poll_type);
 445   return __ offset();
 446 }
 447 
 448 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 449   __ mv(to_reg, from_reg);
 450 }
 451 
 452 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 453 
 454 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 455   assert(src->is_constant(), "should not call otherwise");
 456   assert(dest->is_register(), "should not call otherwise");
 457   LIR_Const* c = src->as_constant_ptr();
 458   address const_addr = NULL;
 459 
 460   switch (c->type()) {
 461     case T_INT:
 462       assert(patch_code == lir_patch_none, "no patching handled here");
 463       __ mvw(dest->as_register(), c->as_jint());
 464       break;
 465 
 466     case T_ADDRESS:
 467       assert(patch_code == lir_patch_none, "no patching handled here");
 468       __ mv(dest->as_register(), c->as_jint());
 469       break;
 470 
 471     case T_LONG:
 472       assert(patch_code == lir_patch_none, "no patching handled here");
 473       __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong());
 474       break;
 475 
 476     case T_OBJECT:
 477     case T_ARRAY:
 478       if (patch_code == lir_patch_none) {
 479         jobject2reg(c->as_jobject(), dest->as_register());
 480       } else {
 481         jobject2reg_with_patching(dest->as_register(), info);
 482       }
 483       break;
 484 
 485     case T_METADATA:
 486       if (patch_code != lir_patch_none) {
 487         klass2reg_with_patching(dest->as_register(), info);
 488       } else {
 489         __ mov_metadata(dest->as_register(), c->as_metadata());
 490       }
 491       break;
 492 
 493     case T_FLOAT:
 494       const_addr = float_constant(c->as_jfloat());
 495       assert(const_addr != NULL, "must create float constant in the constant table");
 496       __ flw(dest->as_float_reg(), InternalAddress(const_addr));
 497       break;
 498 
 499     case T_DOUBLE:
 500       const_addr = double_constant(c->as_jdouble());
 501       assert(const_addr != NULL, "must create double constant in the constant table");
 502       __ fld(dest->as_double_reg(), InternalAddress(const_addr));
 503       break;
 504 
 505     default:
 506       ShouldNotReachHere();
 507   }
 508 }
 509 
 510 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 511   assert(src->is_constant(), "should not call otherwise");
 512   assert(dest->is_stack(), "should not call otherwise");
 513   LIR_Const* c = src->as_constant_ptr();
 514   switch (c->type()) {
 515     case T_OBJECT:
 516       if (c->as_jobject() == NULL) {
 517         __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 518       } else {
 519         const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
 520         reg2stack(FrameMap::t1_opr, dest, c->type(), false);
 521       }
 522       break;
 523     case T_ADDRESS:   // fall through
 524       const2reg(src, FrameMap::t1_opr, lir_patch_none, NULL);
 525       reg2stack(FrameMap::t1_opr, dest, c->type(), false);
 526     case T_INT:       // fall through
 527     case T_FLOAT:
 528       if (c->as_jint_bits() == 0) {
 529         __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 530       } else {
 531         __ mvw(t1, c->as_jint_bits());
 532         __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix()));
 533       }
 534       break;
 535     case T_LONG:      // fall through
 536     case T_DOUBLE:
 537       if (c->as_jlong_bits() == 0) {
 538         __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 539                                                 lo_word_offset_in_bytes));
 540       } else {
 541         __ mv(t1, (intptr_t)c->as_jlong_bits());
 542         __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(),
 543                                                 lo_word_offset_in_bytes));
 544       }
 545       break;
 546     default:
 547       ShouldNotReachHere();
 548   }
 549 }
 550 
 551 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 552   assert(src->is_constant(), "should not call otherwise");
 553   assert(dest->is_address(), "should not call otherwise");
 554   LIR_Const* c = src->as_constant_ptr();
 555   LIR_Address* to_addr = dest->as_address_ptr();
 556   void (Assembler::* insn)(Register Rt, const Address &adr, Register temp);
 557   switch (type) {
 558     case T_ADDRESS:
 559       assert(c->as_jint() == 0, "should be");
 560       insn = &Assembler::sd; break;
 561     case T_LONG:
 562       assert(c->as_jlong() == 0, "should be");
 563       insn = &Assembler::sd; break;
 564     case T_DOUBLE:
 565       assert(c->as_jdouble() == 0.0, "should be");
 566       insn = &Assembler::sd; break;
 567     case T_INT:
 568       assert(c->as_jint() == 0, "should be");
 569       insn = &Assembler::sw; break;
 570     case T_FLOAT:
 571       assert(c->as_jfloat() == 0.0f, "should be");
 572       insn = &Assembler::sw; break;
 573     case T_OBJECT:    // fall through
 574     case T_ARRAY:
 575       assert(c->as_jobject() == 0, "should be");
 576       if (UseCompressedOops && !wide) {
 577         insn = &Assembler::sw;
 578       } else {
 579         insn = &Assembler::sd;
 580       }
 581       break;
 582     case T_CHAR:      // fall through
 583     case T_SHORT:
 584       assert(c->as_jint() == 0, "should be");
 585       insn = &Assembler::sh;
 586       break;
 587     case T_BOOLEAN:   // fall through
 588     case T_BYTE:
 589       assert(c->as_jint() == 0, "should be");
 590       insn = &Assembler::sb; break;
 591     default:
 592       ShouldNotReachHere();
 593       insn = &Assembler::sd;  // unreachable
 594   }
 595   if (info != NULL) {
 596     add_debug_info_for_null_check_here(info);
 597   }
 598   (_masm->*insn)(zr, as_Address(to_addr), t0);
 599 }
 600 
 601 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 602   assert(src->is_register(), "should not call otherwise");
 603   assert(dest->is_register(), "should not call otherwise");
 604 
 605   // move between cpu-registers
 606   if (dest->is_single_cpu()) {
 607     if (src->type() == T_LONG) {
 608       // Can do LONG -> OBJECT
 609       move_regs(src->as_register_lo(), dest->as_register());
 610       return;
 611     }
 612     assert(src->is_single_cpu(), "must match");
 613     if (src->type() == T_OBJECT) {
 614       __ verify_oop(src->as_register());
 615     }
 616     move_regs(src->as_register(), dest->as_register());
 617   } else if (dest->is_double_cpu()) {
 618     if (is_reference_type(src->type())) {
 619       __ verify_oop(src->as_register());
 620       move_regs(src->as_register(), dest->as_register_lo());
 621       return;
 622     }
 623     assert(src->is_double_cpu(), "must match");
 624     Register f_lo = src->as_register_lo();
 625     Register f_hi = src->as_register_hi();
 626     Register t_lo = dest->as_register_lo();
 627     Register t_hi = dest->as_register_hi();
 628     assert(f_hi == f_lo, "must be same");
 629     assert(t_hi == t_lo, "must be same");
 630     move_regs(f_lo, t_lo);
 631   } else if (dest->is_single_fpu()) {
 632     assert(src->is_single_fpu(), "expect single fpu");
 633     __ fmv_s(dest->as_float_reg(), src->as_float_reg());
 634   } else if (dest->is_double_fpu()) {
 635     assert(src->is_double_fpu(), "expect double fpu");
 636     __ fmv_d(dest->as_double_reg(), src->as_double_reg());
 637   } else {
 638     ShouldNotReachHere();
 639   }
 640 }
 641 
 642 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 643   precond(src->is_register() && dest->is_stack());
 644 
 645   uint const c_sz32 = sizeof(uint32_t);
 646   uint const c_sz64 = sizeof(uint64_t);
 647 
 648   assert(src->is_register(), "should not call otherwise");
 649   assert(dest->is_stack(), "should not call otherwise");
 650   if (src->is_single_cpu()) {
 651     int index = dest->single_stack_ix();
 652     if (is_reference_type(type)) {
 653       __ sd(src->as_register(), stack_slot_address(index, c_sz64));
 654       __ verify_oop(src->as_register());
 655     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 656       __ sd(src->as_register(), stack_slot_address(index, c_sz64));
 657     } else {
 658       __ sw(src->as_register(), stack_slot_address(index, c_sz32));
 659     }
 660   } else if (src->is_double_cpu()) {
 661     int index = dest->double_stack_ix();
 662     Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
 663     __ sd(src->as_register_lo(), dest_addr_LO);
 664   } else if (src->is_single_fpu()) {
 665     int index = dest->single_stack_ix();
 666     __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32));
 667   } else if (src->is_double_fpu()) {
 668     int index = dest->double_stack_ix();
 669     __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64));
 670   } else {
 671     ShouldNotReachHere();
 672   }
 673 }
 674 
 675 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 676   LIR_Address* to_addr = dest->as_address_ptr();
 677   // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src
 678   Register compressed_src = t1;
 679 
 680   if (patch_code != lir_patch_none) {
 681     deoptimize_trap(info);
 682     return;
 683   }
 684 
 685   if (is_reference_type(type)) {
 686     __ verify_oop(src->as_register());
 687 
 688     if (UseCompressedOops && !wide) {
 689       __ encode_heap_oop(compressed_src, src->as_register());
 690     } else {
 691       compressed_src = src->as_register();
 692     }
 693   }
 694 
 695   int null_check_here = code_offset();
 696 
 697   switch (type) {
 698     case T_FLOAT:
 699       __ fsw(src->as_float_reg(), as_Address(to_addr));
 700       break;
 701 
 702     case T_DOUBLE:
 703       __ fsd(src->as_double_reg(), as_Address(to_addr));
 704       break;
 705 
 706     case T_ARRAY:      // fall through
 707     case T_OBJECT:
 708       if (UseCompressedOops && !wide) {
 709         __ sw(compressed_src, as_Address(to_addr));
 710       } else {
 711         __ sd(compressed_src, as_Address(to_addr));
 712       }
 713       break;
 714     case T_METADATA:
 715       // We get here to store a method pointer to the stack to pass to
 716       // a dtrace runtime call. This can't work on 64 bit with
 717       // compressed klass ptrs: T_METADATA can be compressed klass
 718       // ptr or a 64 bit method pointer.
 719       ShouldNotReachHere();
 720       __ sd(src->as_register(), as_Address(to_addr));
 721       break;
 722     case T_ADDRESS:
 723       __ sd(src->as_register(), as_Address(to_addr));
 724       break;
 725     case T_INT:
 726       __ sw(src->as_register(), as_Address(to_addr));
 727       break;
 728     case T_LONG:
 729       __ sd(src->as_register_lo(), as_Address(to_addr));
 730       break;
 731     case T_BYTE:    // fall through
 732     case T_BOOLEAN:
 733       __ sb(src->as_register(), as_Address(to_addr));
 734       break;
 735     case T_CHAR:    // fall through
 736     case T_SHORT:
 737       __ sh(src->as_register(), as_Address(to_addr));
 738       break;
 739     default:
 740       ShouldNotReachHere();
 741   }
 742 
 743   if (info != NULL) {
 744     add_debug_info_for_null_check(null_check_here, info);
 745   }
 746 }
 747 
 748 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 749   precond(src->is_stack() && dest->is_register());
 750 
 751   uint const c_sz32 = sizeof(uint32_t);
 752   uint const c_sz64 = sizeof(uint64_t);
 753 
 754   if (dest->is_single_cpu()) {
 755     int index = src->single_stack_ix();
 756     if (type == T_INT) {
 757       __ lw(dest->as_register(), stack_slot_address(index, c_sz32));
 758     } else if (is_reference_type(type)) {
 759       __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
 760       __ verify_oop(dest->as_register());
 761     } else if (type == T_METADATA || type == T_ADDRESS) {
 762       __ ld(dest->as_register(), stack_slot_address(index, c_sz64));
 763     } else {
 764       __ lwu(dest->as_register(), stack_slot_address(index, c_sz32));
 765     }
 766   } else if (dest->is_double_cpu()) {
 767     int index = src->double_stack_ix();
 768     Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes);
 769     __ ld(dest->as_register_lo(), src_addr_LO);
 770   } else if (dest->is_single_fpu()) {
 771     int index = src->single_stack_ix();
 772     __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32));
 773   } else if (dest->is_double_fpu()) {
 774     int index = src->double_stack_ix();
 775     __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64));
 776   } else {
 777     ShouldNotReachHere();
 778   }
 779 }
 780 
 781 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 782   deoptimize_trap(info);
 783 }
 784 
 785 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 786   LIR_Opr temp;
 787   if (type == T_LONG || type == T_DOUBLE) {
 788     temp = FrameMap::t1_long_opr;
 789   } else {
 790     temp = FrameMap::t1_opr;
 791   }
 792 
 793   stack2reg(src, temp, src->type());
 794   reg2stack(temp, dest, dest->type(), false);
 795 }
 796 
 797 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 798   assert(src->is_address(), "should not call otherwise");
 799   assert(dest->is_register(), "should not call otherwise");
 800 
 801   LIR_Address* addr = src->as_address_ptr();
 802   LIR_Address* from_addr = src->as_address_ptr();
 803 
 804   if (addr->base()->type() == T_OBJECT) {
 805     __ verify_oop(addr->base()->as_pointer_register());
 806   }
 807 
 808   if (patch_code != lir_patch_none) {
 809     deoptimize_trap(info);
 810     return;
 811   }
 812 
 813   if (info != NULL) {
 814     add_debug_info_for_null_check_here(info);
 815   }
 816 
 817   int null_check_here = code_offset();
 818   switch (type) {
 819     case T_FLOAT:
 820       __ flw(dest->as_float_reg(), as_Address(from_addr));
 821       break;
 822     case T_DOUBLE:
 823       __ fld(dest->as_double_reg(), as_Address(from_addr));
 824       break;
 825     case T_ARRAY:     // fall through
 826     case T_OBJECT:
 827       if (UseCompressedOops && !wide) {
 828         __ lwu(dest->as_register(), as_Address(from_addr));
 829       } else {
 830         __ ld(dest->as_register(), as_Address(from_addr));
 831       }
 832       break;
 833     case T_METADATA:
 834       // We get here to store a method pointer to the stack to pass to
 835       // a dtrace runtime call. This can't work on 64 bit with
 836       // compressed klass ptrs: T_METADATA can be a compressed klass
 837       // ptr or a 64 bit method pointer.
 838       ShouldNotReachHere();
 839       __ ld(dest->as_register(), as_Address(from_addr));
 840       break;
 841     case T_ADDRESS:
 842       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 843         __ lwu(dest->as_register(), as_Address(from_addr));
 844       } else {
 845         __ ld(dest->as_register(), as_Address(from_addr));
 846       }
 847       break;
 848     case T_INT:
 849       __ lw(dest->as_register(), as_Address(from_addr));
 850       break;
 851     case T_LONG:
 852       __ ld(dest->as_register_lo(), as_Address_lo(from_addr));
 853       break;
 854     case T_BYTE:
 855       __ lb(dest->as_register(), as_Address(from_addr));
 856       break;
 857     case T_BOOLEAN:
 858       __ lbu(dest->as_register(), as_Address(from_addr));
 859       break;
 860     case T_CHAR:
 861       __ lhu(dest->as_register(), as_Address(from_addr));
 862       break;
 863     case T_SHORT:
 864       __ lh(dest->as_register(), as_Address(from_addr));
 865       break;
 866     default:
 867       ShouldNotReachHere();
 868   }
 869 
 870   if (is_reference_type(type)) {
 871     if (UseCompressedOops && !wide) {
 872       __ decode_heap_oop(dest->as_register());
 873     }
 874 
 875     if (!UseZGC) {
 876       // Load barrier has not yet been applied, so ZGC can't verify the oop here
 877       __ verify_oop(dest->as_register());
 878     }
 879   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 880     if (UseCompressedClassPointers) {
 881       __ decode_klass_not_null(dest->as_register());
 882     }
 883   }
 884 }
 885 
 886 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 887   switch (op->code()) {
 888     case lir_idiv: // fall through
 889     case lir_irem:
 890       arithmetic_idiv(op->code(),
 891                       op->in_opr1(),
 892                       op->in_opr2(),
 893                       op->in_opr3(),
 894                       op->result_opr(),
 895                       op->info());
 896       break;
 897     case lir_fmad:
 898       __ fmadd_d(op->result_opr()->as_double_reg(),
 899                  op->in_opr1()->as_double_reg(),
 900                  op->in_opr2()->as_double_reg(),
 901                  op->in_opr3()->as_double_reg());
 902       break;
 903     case lir_fmaf:
 904       __ fmadd_s(op->result_opr()->as_float_reg(),
 905                  op->in_opr1()->as_float_reg(),
 906                  op->in_opr2()->as_float_reg(),
 907                  op->in_opr3()->as_float_reg());
 908       break;
 909     default:
 910       ShouldNotReachHere();
 911   }
 912 }
 913 
 914 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
 915                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
 916   Label label;
 917 
 918   emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false,
 919               /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true);
 920 
 921   Label done;
 922   move_op(opr2, result, type, lir_patch_none, NULL,
 923           false,   // pop_fpu_stack
 924           false,   // wide
 925           false);  // unaligned
 926   __ j(done);
 927   __ bind(label);
 928   move_op(opr1, result, type, lir_patch_none, NULL,
 929           false,   // pop_fpu_stack
 930           false,   // wide
 931           false);  // unaligned
 932   __ bind(done);
 933 }
 934 
 935 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 936   LIR_Condition condition = op->cond();
 937   if (condition == lir_cond_always) {
 938     if (op->info() != NULL) {
 939       add_debug_info_for_branch(op->info());
 940     }
 941   } else {
 942     assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands");
 943   }
 944   bool is_unordered = (op->ublock() == op->block());
 945   emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered);
 946 }
 947 
 948 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label,
 949                                 bool is_far, bool is_unordered) {
 950 
 951   if (cmp_flag == lir_cond_always) {
 952     __ j(label);
 953     return;
 954   }
 955 
 956   if (cmp1->is_cpu_register()) {
 957     Register reg1 = as_reg(cmp1);
 958     if (cmp2->is_cpu_register()) {
 959       Register reg2 = as_reg(cmp2);
 960       __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far);
 961     } else if (cmp2->is_constant()) {
 962       const2reg_helper(cmp2);
 963       __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far);
 964     } else {
 965       ShouldNotReachHere();
 966     }
 967   } else if (cmp1->is_single_fpu()) {
 968     assert(cmp2->is_single_fpu(), "expect single float register");
 969     __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered);
 970   } else if (cmp1->is_double_fpu()) {
 971     assert(cmp2->is_double_fpu(), "expect double float register");
 972     __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask,
 973                            cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered);
 974   } else {
 975     ShouldNotReachHere();
 976   }
 977 }
 978 
 979 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 980   LIR_Opr src  = op->in_opr();
 981   LIR_Opr dest = op->result_opr();
 982 
 983   switch (op->bytecode()) {
 984     case Bytecodes::_i2f:
 985       __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break;
 986     case Bytecodes::_i2d:
 987       __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break;
 988     case Bytecodes::_l2d:
 989       __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break;
 990     case Bytecodes::_l2f:
 991       __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break;
 992     case Bytecodes::_f2d:
 993       __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break;
 994     case Bytecodes::_d2f:
 995       __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break;
 996     case Bytecodes::_i2c:
 997       __ zero_extend(dest->as_register(), src->as_register(), 16); break;
 998     case Bytecodes::_i2l:
 999       __ addw(dest->as_register_lo(), src->as_register(), zr); break;
1000     case Bytecodes::_i2s:
1001       __ sign_extend(dest->as_register(), src->as_register(), 16); break;
1002     case Bytecodes::_i2b:
1003       __ sign_extend(dest->as_register(), src->as_register(), 8); break;
1004     case Bytecodes::_l2i:
1005       _masm->block_comment("FIXME: This coulde be no-op");
1006       __ addw(dest->as_register(), src->as_register_lo(), zr); break;
1007     case Bytecodes::_d2l:
1008       __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break;
1009     case Bytecodes::_f2i:
1010       __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break;
1011     case Bytecodes::_f2l:
1012       __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break;
1013     case Bytecodes::_d2i:
1014       __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break;
1015     default:
1016       ShouldNotReachHere();
1017   }
1018 }
1019 
1020 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1021   if (op->init_check()) {
1022     __ lbu(t0, Address(op->klass()->as_register(),
1023                        InstanceKlass::init_state_offset()));
1024     __ mvw(t1, InstanceKlass::fully_initialized);
1025     add_debug_info_for_null_check_here(op->stub()->info());
1026     __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true);
1027   }
1028 
1029   __ allocate_object(op->obj()->as_register(),
1030                      op->tmp1()->as_register(),
1031                      op->tmp2()->as_register(),
1032                      op->header_size(),
1033                      op->object_size(),
1034                      op->klass()->as_register(),
1035                      *op->stub()->entry());
1036 
1037   __ bind(*op->stub()->continuation());
1038 }
1039 
1040 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1041   Register len = op->len()->as_register();
1042 
1043   if (UseSlowPath ||
1044       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1045       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1046     __ j(*op->stub()->entry());
1047   } else {
1048     Register tmp1 = op->tmp1()->as_register();
1049     Register tmp2 = op->tmp2()->as_register();
1050     Register tmp3 = op->tmp3()->as_register();
1051     if (len == tmp1) {
1052       tmp1 = tmp3;
1053     } else if (len == tmp2) {
1054       tmp2 = tmp3;
1055     } else if (len == tmp3) {
1056       // everything is ok
1057     } else {
1058       __ mv(tmp3, len);
1059     }
1060     __ allocate_array(op->obj()->as_register(),
1061                       len,
1062                       tmp1,
1063                       tmp2,
1064                       arrayOopDesc::header_size(op->type()),
1065                       array_element_size(op->type()),
1066                       op->klass()->as_register(),
1067                       *op->stub()->entry());
1068   }
1069   __ bind(*op->stub()->continuation());
1070 }
1071 
1072 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
1073                                         Register recv, Label* update_done) {
1074   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1075     Label next_test;
1076     // See if the receiver is receiver[n].
1077     __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1078     __ bne(recv, t1, next_test);
1079     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1080     __ add_memory_int64(data_addr, DataLayout::counter_increment);
1081     __ j(*update_done);
1082     __ bind(next_test);
1083   }
1084 
1085   // Didn't find receiver; find next empty slot and fill it in
1086   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1087     Label next_test;
1088     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1089     __ ld(t1, recv_addr);
1090     __ bnez(t1, next_test);
1091     __ sd(recv, recv_addr);
1092     __ li(t1, DataLayout::counter_increment);
1093     __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1094     __ j(*update_done);
1095     __ bind(next_test);
1096   }
1097 }
1098 
1099 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) {
1100   ciMethod* method = op->profiled_method();
1101   assert(method != NULL, "Should have method");
1102   int bci = op->profiled_bci();
1103   *md = method->method_data_or_null();
1104   guarantee(*md != NULL, "Sanity");
1105   *data = ((*md)->bci_to_data(bci));
1106   assert(*data != NULL, "need data for type check");
1107   assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1108 }
1109 
1110 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1,
1111                                                Register k_RInfo, Register klass_RInfo,
1112                                                Label *failure_target, Label *success_target) {
1113   // get object class
1114   // not a safepoint as obj null check happens earlier
1115   __ load_klass(klass_RInfo, obj);
1116   if (k->is_loaded()) {
1117     // See if we get an immediate positive hit
1118     __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset())));
1119     if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1120       __ bne(k_RInfo, t0, *failure_target, /* is_far */ true);
1121       // successful cast, fall through to profile or jump
1122     } else {
1123       // See if we get an immediate positive hit
1124       __ beq(k_RInfo, t0, *success_target);
1125       // check for self
1126       __ beq(klass_RInfo, k_RInfo, *success_target);
1127 
1128       __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1129       __ sd(k_RInfo, Address(sp, 0));             // sub klass
1130       __ sd(klass_RInfo, Address(sp, wordSize));  // super klass
1131       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1132       // load result to k_RInfo
1133       __ ld(k_RInfo, Address(sp, 0));
1134       __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1135       // result is a boolean
1136       __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1137       // successful cast, fall through to profile or jump
1138     }
1139   } else {
1140     // perform the fast part of the checking logic
1141     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1142     // call out-of-line instance of __ check_klass_subtytpe_slow_path(...)
1143     __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
1144     __ sd(klass_RInfo, Address(sp, wordSize));  // sub klass
1145     __ sd(k_RInfo, Address(sp, 0));             // super klass
1146     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1147     // load result to k_RInfo
1148     __ ld(k_RInfo, Address(sp, 0));
1149     __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
1150     // result is a boolean
1151     __ beqz(k_RInfo, *failure_target, /* is_far */ true);
1152     // successful cast, fall thriugh to profile or jump
1153   }
1154 }
1155 
1156 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj,
1157                                    Register klass_RInfo, Label* obj_is_null) {
1158   Label not_null;
1159   __ bnez(obj, not_null);
1160   // Object is null, update MDO and exit
1161   Register mdo = klass_RInfo;
1162   __ mov_metadata(mdo, md->constant_encoding());
1163   Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1164   __ lbu(t0, data_addr);
1165   __ ori(t0, t0, BitData::null_seen_byte_constant());
1166   __ sb(t0, data_addr);
1167   __ j(*obj_is_null);
1168   __ bind(not_null);
1169 }
1170 
1171 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) {
1172   if (!k->is_loaded()) {
1173     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1174   } else {
1175     __ mov_metadata(k_RInfo, k->constant_encoding());
1176   }
1177 }
1178 
1179 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1180   Register obj = op->object()->as_register();
1181   Register k_RInfo = op->tmp1()->as_register();
1182   Register klass_RInfo = op->tmp2()->as_register();
1183   Register dst = op->result_opr()->as_register();
1184   ciKlass* k = op->klass();
1185   Register Rtmp1 = noreg;
1186 
1187   // check if it needs to be profiled
1188   ciMethodData* md = NULL;
1189   ciProfileData* data = NULL;
1190 
1191   const bool should_profile = op->should_profile();
1192   if (should_profile) {
1193     data_check(op, &md, &data);
1194   }
1195   Label profile_cast_success, profile_cast_failure;
1196   Label *success_target = should_profile ? &profile_cast_success : success;
1197   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1198 
1199   if (obj == k_RInfo) {
1200     k_RInfo = dst;
1201   } else if (obj == klass_RInfo) {
1202     klass_RInfo = dst;
1203   }
1204   if (k->is_loaded() && !UseCompressedClassPointers) {
1205     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1206   } else {
1207     Rtmp1 = op->tmp3()->as_register();
1208     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1209   }
1210 
1211   assert_different_registers(obj, k_RInfo, klass_RInfo);
1212 
1213   if (should_profile) {
1214     profile_object(md, data, obj, klass_RInfo, obj_is_null);
1215   } else {
1216     __ beqz(obj, *obj_is_null);
1217   }
1218 
1219   typecheck_loaded(op, k, k_RInfo);
1220   __ verify_oop(obj);
1221 
1222   if (op->fast_check()) {
1223     // get object class
1224     // not a safepoint as obj null check happens earlier
1225     __ load_klass(t0, obj, t1);
1226     __ bne(t0, k_RInfo, *failure_target, /* is_far */ true);
1227     // successful cast, fall through to profile or jump
1228   } else {
1229     typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target);
1230   }
1231   if (should_profile) {
1232     type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure);
1233   }
1234   __ j(*success);
1235 }
1236 
1237 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1238   const bool should_profile = op->should_profile();
1239 
1240   LIR_Code code = op->code();
1241   if (code == lir_store_check) {
1242     typecheck_lir_store(op, should_profile);
1243   } else if (code == lir_checkcast) {
1244     Register obj = op->object()->as_register();
1245     Register dst = op->result_opr()->as_register();
1246     Label success;
1247     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1248     __ bind(success);
1249     if (dst != obj) {
1250       __ mv(dst, obj);
1251     }
1252   } else if (code == lir_instanceof) {
1253     Register obj = op->object()->as_register();
1254     Register dst = op->result_opr()->as_register();
1255     Label success, failure, done;
1256     emit_typecheck_helper(op, &success, &failure, &failure);
1257     __ bind(failure);
1258     __ mv(dst, zr);
1259     __ j(done);
1260     __ bind(success);
1261     __ mv(dst, 1);
1262     __ bind(done);
1263   } else {
1264     ShouldNotReachHere();
1265   }
1266 }
1267 
1268 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1269   assert(VM_Version::supports_cx8(), "wrong machine");
1270   Register addr;
1271   if (op->addr()->is_register()) {
1272     addr = as_reg(op->addr());
1273   } else {
1274     assert(op->addr()->is_address(), "what else?");
1275     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1276     assert(addr_ptr->disp() == 0, "need 0 disp");
1277     assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1278     addr = as_reg(addr_ptr->base());
1279   }
1280   Register newval = as_reg(op->new_value());
1281   Register cmpval = as_reg(op->cmp_value());
1282 
1283   if (op->code() == lir_cas_obj) {
1284     if (UseCompressedOops) {
1285       Register tmp1 = op->tmp1()->as_register();
1286       assert(op->tmp1()->is_valid(), "must be");
1287       __ encode_heap_oop(tmp1, cmpval);
1288       cmpval = tmp1;
1289       __ encode_heap_oop(t1, newval);
1290       newval = t1;
1291       caswu(addr, newval, cmpval);
1292     } else {
1293       casl(addr, newval, cmpval);
1294     }
1295   } else if (op->code() == lir_cas_int) {
1296     casw(addr, newval, cmpval);
1297   } else {
1298     casl(addr, newval, cmpval);
1299   }
1300 }
1301 
1302 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1303   switch (code) {
1304     case lir_abs:  __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break;
1305     case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break;
1306     default:       ShouldNotReachHere();
1307   }
1308 }
1309 
1310 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1311   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1312   Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1313   if (dst->is_single_cpu()) {
1314     Register Rdst = dst->as_register();
1315     if (right->is_constant()) {
1316       int right_const = right->as_jint();
1317       if (Assembler::operand_valid_for_add_immediate(right_const)) {
1318         logic_op_imm(Rdst, Rleft, right_const, code);
1319         __ addw(Rdst, Rdst, zr);
1320      } else {
1321         __ mv(t0, right_const);
1322         logic_op_reg32(Rdst, Rleft, t0, code);
1323      }
1324     } else {
1325       Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1326       logic_op_reg32(Rdst, Rleft, Rright, code);
1327     }
1328   } else {
1329     Register Rdst = dst->as_register_lo();
1330     if (right->is_constant()) {
1331       long right_const = right->as_jlong();
1332       if (Assembler::operand_valid_for_add_immediate(right_const)) {
1333         logic_op_imm(Rdst, Rleft, right_const, code);
1334       } else {
1335         __ mv(t0, right_const);
1336         logic_op_reg(Rdst, Rleft, t0, code);
1337       }
1338     } else {
1339       Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo();
1340       logic_op_reg(Rdst, Rleft, Rright, code);
1341     }
1342   }
1343 }
1344 
1345 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) {
1346   ShouldNotCallThis();
1347 }
1348 
1349 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1350   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1351     bool is_unordered_less = (code == lir_ucmp_fd2i);
1352     if (left->is_single_fpu()) {
1353       __ float_cmp(true, is_unordered_less ? -1 : 1,
1354                    left->as_float_reg(), right->as_float_reg(), dst->as_register());
1355     } else if (left->is_double_fpu()) {
1356       __ float_cmp(false, is_unordered_less ? -1 : 1,
1357                    left->as_double_reg(), right->as_double_reg(), dst->as_register());
1358     } else {
1359       ShouldNotReachHere();
1360     }
1361   } else if (code == lir_cmp_l2i) {
1362     __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo());
1363   } else {
1364     ShouldNotReachHere();
1365   }
1366 }
1367 
1368 void LIR_Assembler::align_call(LIR_Code code) {
1369   // With RVC a call instruction may get 2-byte aligned.
1370   // The address of the call instruction needs to be 4-byte aligned to
1371   // ensure that it does not span a cache line so that it can be patched.
1372   __ align(NativeInstruction::instruction_size);
1373 }
1374 
1375 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1376   address call = __ trampoline_call(Address(op->addr(), rtype));
1377   if (call == NULL) {
1378     bailout("trampoline stub overflow");
1379     return;
1380   }
1381   add_call_info(code_offset(), op->info());
1382 }
1383 
1384 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1385   address call = __ ic_call(op->addr());
1386   if (call == NULL) {
1387     bailout("trampoline stub overflow");
1388     return;
1389   }
1390   add_call_info(code_offset(), op->info());
1391 }
1392 
1393 void LIR_Assembler::emit_static_call_stub() {
1394   address call_pc = __ pc();
1395   MacroAssembler::assert_alignment(call_pc);
1396   address stub = __ start_a_stub(call_stub_size());
1397   if (stub == NULL) {
1398     bailout("static call stub overflow");
1399     return;
1400   }
1401 
1402   int start = __ offset();
1403 
1404   __ relocate(static_stub_Relocation::spec(call_pc));
1405   __ emit_static_call_stub();
1406 
1407   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
1408          <= call_stub_size(), "stub too big");
1409   __ end_a_stub();
1410 }
1411 
1412 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1413   assert(exceptionOop->as_register() == x10, "must match");
1414   assert(exceptionPC->as_register() == x13, "must match");
1415 
1416   // exception object is not added to oop map by LinearScan
1417   // (LinearScan assumes that no oops are in fixed registers)
1418   info->add_register_oop(exceptionOop);
1419   Runtime1::StubID unwind_id;
1420 
1421   // get current pc information
1422   // pc is only needed if the method has an exception handler, the unwind code does not need it.
1423   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
1424     // As no instructions have been generated yet for this LIR node it's
1425     // possible that an oop map already exists for the current offset.
1426     // In that case insert an dummy NOP here to ensure all oop map PCs
1427     // are unique. See JDK-8237483.
1428     __ nop();
1429   }
1430   int pc_for_athrow_offset = __ offset();
1431   InternalAddress pc_for_athrow(__ pc());
1432   int32_t off = 0;
1433   __ la_patchable(exceptionPC->as_register(), pc_for_athrow, off);
1434   __ addi(exceptionPC->as_register(), exceptionPC->as_register(), off);
1435   add_call_info(pc_for_athrow_offset, info); // for exception handler
1436 
1437   __ verify_not_null_oop(x10);
1438   // search an exception handler (x10: exception oop, x13: throwing pc)
1439   if (compilation()->has_fpu_code()) {
1440     unwind_id = Runtime1::handle_exception_id;
1441   } else {
1442     unwind_id = Runtime1::handle_exception_nofpu_id;
1443   }
1444   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
1445   __ nop();
1446 }
1447 
1448 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1449   assert(exceptionOop->as_register() == x10, "must match");
1450   __ j(_unwind_handler_entry);
1451 }
1452 
1453 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
1454   Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1455   Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1456   Register count_reg = count->as_register();
1457   if (dest->is_single_cpu()) {
1458     assert (dest->type() == T_INT, "unexpected result type");
1459     assert (left->type() == T_INT, "unexpected left type");
1460     __ andi(t0, count_reg, 31); // should not shift more than 31 bits
1461     switch (code) {
1462       case lir_shl:  __ sllw(dest_reg, left_reg, t0); break;
1463       case lir_shr:  __ sraw(dest_reg, left_reg, t0); break;
1464       case lir_ushr: __ srlw(dest_reg, left_reg, t0); break;
1465       default: ShouldNotReachHere();
1466     }
1467   } else if (dest->is_double_cpu()) {
1468     __ andi(t0, count_reg, 63); // should not shift more than 63 bits
1469     switch (code) {
1470       case lir_shl:  __ sll(dest_reg, left_reg, t0); break;
1471       case lir_shr:  __ sra(dest_reg, left_reg, t0); break;
1472       case lir_ushr: __ srl(dest_reg, left_reg, t0); break;
1473       default: ShouldNotReachHere();
1474     }
1475   } else {
1476     ShouldNotReachHere();
1477   }
1478 }
1479 
1480 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
1481   Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
1482   Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
1483   if (dest->is_single_cpu()) {
1484     assert (dest->type() == T_INT, "unexpected result type");
1485     assert (left->type() == T_INT, "unexpected left type");
1486     count &= 0x1f;
1487     if (count != 0) {
1488       switch (code) {
1489         case lir_shl:  __ slliw(dest_reg, left_reg, count); break;
1490         case lir_shr:  __ sraiw(dest_reg, left_reg, count); break;
1491         case lir_ushr: __ srliw(dest_reg, left_reg, count); break;
1492         default: ShouldNotReachHere();
1493       }
1494     } else {
1495       move_regs(left_reg, dest_reg);
1496     }
1497   } else if (dest->is_double_cpu()) {
1498     count &= 0x3f;
1499     if (count != 0) {
1500       switch (code) {
1501         case lir_shl:  __ slli(dest_reg, left_reg, count); break;
1502         case lir_shr:  __ srai(dest_reg, left_reg, count); break;
1503         case lir_ushr: __ srli(dest_reg, left_reg, count); break;
1504         default: ShouldNotReachHere();
1505       }
1506     } else {
1507       move_regs(left->as_register_lo(), dest->as_register_lo());
1508     }
1509   } else {
1510     ShouldNotReachHere();
1511   }
1512 }
1513 
1514 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
1515   Register obj = op->obj_opr()->as_register();  // may not be an oop
1516   Register hdr = op->hdr_opr()->as_register();
1517   Register lock = op->lock_opr()->as_register();
1518   if (!UseFastLocking) {
1519     __ j(*op->stub()->entry());
1520   } else if (op->code() == lir_lock) {
1521     Register tmp = noreg;
1522     if (UseBiasedLocking) {
1523       tmp = op->scratch_opr()->as_register();
1524     }
1525     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1526     // add debug info for NullPointerException only if one is possible
1527     int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
1528     if (op->info() != NULL) {
1529       add_debug_info_for_null_check(null_check_offset, op->info());
1530     }
1531   } else if (op->code() == lir_unlock) {
1532     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
1533     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
1534   } else {
1535     Unimplemented();
1536   }
1537   __ bind(*op->stub()->continuation());
1538 }
1539 
1540 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
1541   ciMethod* method = op->profiled_method();
1542   int bci          = op->profiled_bci();
1543 
1544   // Update counter for all call types
1545   ciMethodData* md = method->method_data_or_null();
1546   guarantee(md != NULL, "Sanity");
1547   ciProfileData* data = md->bci_to_data(bci);
1548   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
1549   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
1550   Register mdo  = op->mdo()->as_register();
1551   __ mov_metadata(mdo, md->constant_encoding());
1552   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1553   // Perform additional virtual call profiling for invokevirtual and
1554   // invokeinterface bytecodes
1555   if (op->should_profile_receiver_type()) {
1556     assert(op->recv()->is_single_cpu(), "recv must be allocated");
1557     Register recv = op->recv()->as_register();
1558     assert_different_registers(mdo, recv);
1559     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
1560     ciKlass* known_klass = op->known_holder();
1561     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
1562       // We know the type that will be seen at this call site; we can
1563       // statically update the MethodData* rather than needing to do
1564       // dynamic tests on the receiver type
1565       // NOTE: we should probably put a lock around this search to
1566       // avoid collisions by concurrent compilations
1567       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
1568       uint i;
1569       for (i = 0; i < VirtualCallData::row_limit(); i++) {
1570         ciKlass* receiver = vc_data->receiver(i);
1571         if (known_klass->equals(receiver)) {
1572           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1573           __ add_memory_int64(data_addr, DataLayout::counter_increment);
1574           return;
1575         }
1576       }
1577 
1578       // Receiver type not found in profile data; select an empty slot
1579       // Note that this is less efficient than it should be because it
1580       // always does a write to the receiver part of the
1581       // VirtualCallData rather than just the first time
1582       for (i = 0; i < VirtualCallData::row_limit(); i++) {
1583         ciKlass* receiver = vc_data->receiver(i);
1584         if (receiver == NULL) {
1585           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
1586           __ mov_metadata(t1, known_klass->constant_encoding());
1587           __ sd(t1, recv_addr);
1588           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
1589           __ add_memory_int64(data_addr, DataLayout::counter_increment);
1590           return;
1591         }
1592       }
1593     } else {
1594       __ load_klass(recv, recv);
1595       Label update_done;
1596       type_profile_helper(mdo, md, data, recv, &update_done);
1597       // Receiver did not match any saved receiver and there is no empty row for it.
1598       // Increment total counter to indicate polymorphic case.
1599       __ add_memory_int64(counter_addr, DataLayout::counter_increment);
1600 
1601       __ bind(update_done);
1602     }
1603   } else {
1604     // Static call
1605     __ add_memory_int64(counter_addr, DataLayout::counter_increment);
1606   }
1607 }
1608 
1609 void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); }
1610 
1611 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
1612   __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
1613 }
1614 
1615 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); }
1616 
1617 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
1618                                    Register tmp, Label &next, Label &none,
1619                                    Address mdo_addr) {
1620   if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
1621     if (exact_klass != NULL) {
1622       __ mov_metadata(tmp, exact_klass->constant_encoding());
1623     } else {
1624       __ load_klass(tmp, tmp);
1625     }
1626 
1627     __ ld(t1, mdo_addr);
1628     __ xorr(tmp, tmp, t1);
1629     __ andi(t0, tmp, TypeEntries::type_klass_mask);
1630     // klass seen before, nothing to do. The unknown bit may have been
1631     // set already but no need to check.
1632     __ beqz(t0, next);
1633 
1634     // already unknown. Nothing to do anymore.
1635     __ andi(t0, tmp, TypeEntries::type_unknown);
1636     __ bnez(t0, next);
1637 
1638     if (TypeEntries::is_type_none(current_klass)) {
1639       __ beqz(t1, none);
1640       __ li(t0, (u1)TypeEntries::null_seen);
1641       __ beq(t0, t1, none);
1642       // There is a chance that the checks above (re-reading profiling
1643       // data from memory) fail if another thread has just set the
1644       // profiling to this obj's klass
1645       __ membar(MacroAssembler::LoadLoad);
1646       __ ld(t1, mdo_addr);
1647       __ xorr(tmp, tmp, t1);
1648       __ andi(t0, tmp, TypeEntries::type_klass_mask);
1649       __ beqz(t0, next);
1650     }
1651   } else {
1652     assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1653            ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
1654 
1655     __ ld(tmp, mdo_addr);
1656     // already unknown. Nothing to do anymore.
1657     __ andi(t0, tmp, TypeEntries::type_unknown);
1658     __ bnez(t0, next);
1659   }
1660 
1661   // different than before. Cannot keep accurate profile.
1662   __ ld(t1, mdo_addr);
1663   __ ori(t1, t1, TypeEntries::type_unknown);
1664   __ sd(t1, mdo_addr);
1665 
1666   if (TypeEntries::is_type_none(current_klass)) {
1667     __ j(next);
1668 
1669     __ bind(none);
1670     // first time here. Set profile type.
1671     __ sd(tmp, mdo_addr);
1672   }
1673 }
1674 
1675 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp,
1676                                       Address mdo_addr, Label &next) {
1677   // There's a single possible klass at this profile point
1678   assert(exact_klass != NULL, "should be");
1679   if (TypeEntries::is_type_none(current_klass)) {
1680     __ mov_metadata(tmp, exact_klass->constant_encoding());
1681     __ ld(t1, mdo_addr);
1682     __ xorr(tmp, tmp, t1);
1683     __ andi(t0, tmp, TypeEntries::type_klass_mask);
1684     __ beqz(t0, next);
1685 #ifdef ASSERT
1686   {
1687     Label ok;
1688     __ ld(t0, mdo_addr);
1689     __ beqz(t0, ok);
1690     __ li(t1, (u1)TypeEntries::null_seen);
1691     __ beq(t0, t1, ok);
1692     // may have been set by another thread
1693     __ membar(MacroAssembler::LoadLoad);
1694     __ mov_metadata(t0, exact_klass->constant_encoding());
1695     __ ld(t1, mdo_addr);
1696     __ xorr(t1, t0, t1);
1697     __ andi(t1, t1, TypeEntries::type_mask);
1698     __ beqz(t1, ok);
1699 
1700     __ stop("unexpected profiling mismatch");
1701     __ bind(ok);
1702   }
1703 #endif
1704     // first time here. Set profile type.
1705     __ sd(tmp, mdo_addr);
1706   } else {
1707     assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
1708            ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
1709 
1710     __ ld(tmp, mdo_addr);
1711     // already unknown. Nothing to do anymore.
1712     __ andi(t0, tmp, TypeEntries::type_unknown);
1713     __ bnez(t0, next);
1714 
1715     __ ori(tmp, tmp, TypeEntries::type_unknown);
1716     __ sd(tmp, mdo_addr);
1717   }
1718 }
1719 
1720 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass,
1721                                Address mdo_addr, bool do_update, Label &next) {
1722   __ bnez(tmp, update);
1723   if (!TypeEntries::was_null_seen(current_klass)) {
1724     __ ld(t1, mdo_addr);
1725     __ ori(t1, t1, TypeEntries::null_seen);
1726     __ sd(t1, mdo_addr);
1727   }
1728   if (do_update) {
1729     __ j(next);
1730   }
1731 }
1732 
1733 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
1734   COMMENT("emit_profile_type {");
1735   Register obj = op->obj()->as_register();
1736   Register tmp = op->tmp()->as_pointer_register();
1737   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
1738   ciKlass* exact_klass = op->exact_klass();
1739   intptr_t current_klass = op->current_klass();
1740   bool not_null = op->not_null();
1741   bool no_conflict = op->no_conflict();
1742 
1743   Label update, next, none;
1744 
1745   bool do_null = !not_null;
1746   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
1747   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
1748 
1749   assert(do_null || do_update, "why are we here?");
1750   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
1751   assert_different_registers(tmp, t0, t1, mdo_addr.base());
1752 
1753   __ verify_oop(obj);
1754 
1755   if (tmp != obj) {
1756     __ mv(tmp, obj);
1757   }
1758   if (do_null) {
1759     check_null(tmp, update, current_klass, mdo_addr, do_update, next);
1760 #ifdef ASSERT
1761   } else {
1762     __ bnez(tmp, update);
1763     __ stop("unexpected null obj");
1764 #endif
1765   }
1766 
1767   __ bind(update);
1768 
1769   if (do_update) {
1770 #ifdef ASSERT
1771     if (exact_klass != NULL) {
1772       check_exact_klass(tmp, exact_klass);
1773     }
1774 #endif
1775     if (!no_conflict) {
1776       check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr);
1777     } else {
1778       check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next);
1779     }
1780 
1781     __ bind(next);
1782   }
1783   COMMENT("} emit_profile_type");
1784 }
1785 
1786 void LIR_Assembler::align_backward_branch_target() { }
1787 
1788 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
1789   // tmp must be unused
1790   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
1791 
1792   if (left->is_single_cpu()) {
1793     assert(dest->is_single_cpu(), "expect single result reg");
1794     __ negw(dest->as_register(), left->as_register());
1795   } else if (left->is_double_cpu()) {
1796     assert(dest->is_double_cpu(), "expect double result reg");
1797     __ neg(dest->as_register_lo(), left->as_register_lo());
1798   } else if (left->is_single_fpu()) {
1799     assert(dest->is_single_fpu(), "expect single float result reg");
1800     __ fneg_s(dest->as_float_reg(), left->as_float_reg());
1801   } else {
1802     assert(left->is_double_fpu(), "expect double float operand reg");
1803     assert(dest->is_double_fpu(), "expect double float result reg");
1804     __ fneg_d(dest->as_double_reg(), left->as_double_reg());
1805   }
1806 }
1807 
1808 
1809 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1810   if (patch_code != lir_patch_none) {
1811     deoptimize_trap(info);
1812     return;
1813   }
1814 
1815   LIR_Address* adr = addr->as_address_ptr();
1816   Register dst = dest->as_register_lo();
1817 
1818   assert_different_registers(dst, t0);
1819   if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) {
1820     int scale = adr->scale();
1821     intptr_t offset = adr->disp();
1822     LIR_Opr index_op = adr->index();
1823     if (index_op->is_constant()) {
1824       offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale;
1825     }
1826 
1827     if (!is_imm_in_range(offset, 12, 0)) {
1828       __ la(t0, as_Address(adr));
1829       __ mv(dst, t0);
1830       return;
1831     }
1832   }
1833 
1834   __ la(dst, as_Address(adr));
1835 }
1836 
1837 
1838 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
1839   assert(!tmp->is_valid(), "don't need temporary");
1840 
1841   CodeBlob *cb = CodeCache::find_blob(dest);
1842   if (cb != NULL) {
1843     __ far_call(RuntimeAddress(dest));
1844   } else {
1845     int32_t offset = 0;
1846     __ la_patchable(t0, RuntimeAddress(dest), offset);
1847     __ jalr(x1, t0, offset);
1848   }
1849 
1850   if (info != NULL) {
1851     add_call_info_here(info);
1852   }
1853 }
1854 
1855 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
1856   if (dest->is_address() || src->is_address()) {
1857     move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false, /* unaligned */ false);
1858   } else {
1859     ShouldNotReachHere();
1860   }
1861 }
1862 
1863 #ifdef ASSERT
1864 // emit run-time assertion
1865 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
1866   assert(op->code() == lir_assert, "must be");
1867 
1868   Label ok;
1869   if (op->in_opr1()->is_valid()) {
1870     assert(op->in_opr2()->is_valid(), "both operands must be valid");
1871     bool is_unordered = false;
1872     LIR_Condition cond = op->condition();
1873     emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false,
1874                 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true);
1875   } else {
1876     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
1877     assert(op->condition() == lir_cond_always, "no other conditions allowed");
1878   }
1879 
1880   if (op->halt()) {
1881     const char* str = __ code_string(op->msg());
1882     __ stop(str);
1883   } else {
1884     breakpoint();
1885   }
1886   __ bind(ok);
1887 }
1888 #endif
1889 
1890 #ifndef PRODUCT
1891 #define COMMENT(x)   do { __ block_comment(x); } while (0)
1892 #else
1893 #define COMMENT(x)
1894 #endif
1895 
1896 void LIR_Assembler::membar() {
1897   COMMENT("membar");
1898   __ membar(MacroAssembler::AnyAny);
1899 }
1900 
1901 void LIR_Assembler::membar_acquire() {
1902   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
1903 }
1904 
1905 void LIR_Assembler::membar_release() {
1906   __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
1907 }
1908 
1909 void LIR_Assembler::membar_loadload() {
1910   __ membar(MacroAssembler::LoadLoad);
1911 }
1912 
1913 void LIR_Assembler::membar_storestore() {
1914   __ membar(MacroAssembler::StoreStore);
1915 }
1916 
1917 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
1918 
1919 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
1920 
1921 void LIR_Assembler::on_spin_wait() {
1922   Unimplemented();
1923 }
1924 
1925 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
1926   __ mv(result_reg->as_register(), xthread);
1927 }
1928 
1929 void LIR_Assembler::peephole(LIR_List *lir) {}
1930 
1931 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
1932   Address addr = as_Address(src->as_address_ptr());
1933   BasicType type = src->type();
1934   bool is_oop = is_reference_type(type);
1935 
1936   get_op(type);
1937 
1938   switch (code) {
1939     case lir_xadd:
1940       {
1941         RegisterOrConstant inc;
1942         Register tmp = as_reg(tmp_op);
1943         Register dst = as_reg(dest);
1944         if (data->is_constant()) {
1945           inc = RegisterOrConstant(as_long(data));
1946           assert_different_registers(dst, addr.base(), tmp);
1947           assert_different_registers(tmp, t0);
1948         } else {
1949           inc = RegisterOrConstant(as_reg(data));
1950           assert_different_registers(inc.as_register(), dst, addr.base(), tmp);
1951         }
1952         __ la(tmp, addr);
1953         (_masm->*add)(dst, inc, tmp);
1954         break;
1955       }
1956     case lir_xchg:
1957       {
1958         Register tmp = tmp_op->as_register();
1959         Register obj = as_reg(data);
1960         Register dst = as_reg(dest);
1961         if (is_oop && UseCompressedOops) {
1962           __ encode_heap_oop(t0, obj);
1963           obj = t0;
1964         }
1965         assert_different_registers(obj, addr.base(), tmp, dst);
1966         __ la(tmp, addr);
1967         (_masm->*xchg)(dst, obj, tmp);
1968         if (is_oop && UseCompressedOops) {
1969           __ decode_heap_oop(dst);
1970         }
1971       }
1972       break;
1973     default:
1974       ShouldNotReachHere();
1975   }
1976   __ membar(MacroAssembler::AnyAny);
1977 }
1978 
1979 int LIR_Assembler::array_element_size(BasicType type) const {
1980   int elem_size = type2aelembytes(type);
1981   return exact_log2(elem_size);
1982 }
1983 
1984 // helper functions which checks for overflow and sets bailout if it
1985 // occurs.  Always returns a valid embeddable pointer but in the
1986 // bailout case the pointer won't be to unique storage.
1987 address LIR_Assembler::float_constant(float f) {
1988   address const_addr = __ float_constant(f);
1989   if (const_addr == NULL) {
1990     bailout("const section overflow");
1991     return __ code()->consts()->start();
1992   } else {
1993     return const_addr;
1994   }
1995 }
1996 
1997 address LIR_Assembler::double_constant(double d) {
1998   address const_addr = __ double_constant(d);
1999   if (const_addr == NULL) {
2000     bailout("const section overflow");
2001     return __ code()->consts()->start();
2002   } else {
2003     return const_addr;
2004   }
2005 }
2006 
2007 address LIR_Assembler::int_constant(jlong n) {
2008   address const_addr = __ long_constant(n);
2009   if (const_addr == NULL) {
2010     bailout("const section overflow");
2011     return __ code()->consts()->start();
2012   } else {
2013     return const_addr;
2014   }
2015 }
2016 
2017 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
2018   __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */,
2019              Assembler::rl /* release */, t0, true /* result as bool */);
2020   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2021   __ membar(MacroAssembler::AnyAny);
2022 }
2023 
2024 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) {
2025   __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */,
2026              Assembler::rl /* release */, t0, true /* result as bool */);
2027   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2028   __ membar(MacroAssembler::AnyAny);
2029 }
2030 
2031 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
2032   __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */,
2033              Assembler::rl /* release */, t0, true /* result as bool */);
2034   __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1
2035   __ membar(MacroAssembler::AnyAny);
2036 }
2037 
2038 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
2039   address target = NULL;
2040 
2041   switch (patching_id(info)) {
2042     case PatchingStub::access_field_id:
2043       target = Runtime1::entry_for(Runtime1::access_field_patching_id);
2044       break;
2045     case PatchingStub::load_klass_id:
2046       target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
2047       break;
2048     case PatchingStub::load_mirror_id:
2049       target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
2050       break;
2051     case PatchingStub::load_appendix_id:
2052       target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
2053       break;
2054     default: ShouldNotReachHere();
2055   }
2056 
2057   __ far_call(RuntimeAddress(target));
2058   add_call_info_here(info);
2059 }
2060 
2061 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) {
2062   Label ok;
2063   __ load_klass(tmp, tmp);
2064   __ mov_metadata(t0, exact_klass->constant_encoding());
2065   __ beq(tmp, t0, ok);
2066   __ stop("exact klass and actual klass differ");
2067   __ bind(ok);
2068 }
2069 
2070 void LIR_Assembler::get_op(BasicType type) {
2071   switch (type) {
2072     case T_INT:
2073       xchg = &MacroAssembler::atomic_xchgalw;
2074       add = &MacroAssembler::atomic_addalw;
2075       break;
2076     case T_LONG:
2077       xchg = &MacroAssembler::atomic_xchgal;
2078       add = &MacroAssembler::atomic_addal;
2079       break;
2080     case T_OBJECT:
2081     case T_ARRAY:
2082       if (UseCompressedOops) {
2083         xchg = &MacroAssembler::atomic_xchgalwu;
2084         add = &MacroAssembler::atomic_addalw;
2085       } else {
2086         xchg = &MacroAssembler::atomic_xchgal;
2087         add = &MacroAssembler::atomic_addal;
2088       }
2089       break;
2090     default:
2091       ShouldNotReachHere();
2092   }
2093 }
2094 
2095 // emit_opTypeCheck sub functions
2096 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) {
2097   Register value = op->object()->as_register();
2098   Register array = op->array()->as_register();
2099   Register k_RInfo = op->tmp1()->as_register();
2100   Register klass_RInfo = op->tmp2()->as_register();
2101   Register Rtmp1 = op->tmp3()->as_register();
2102 
2103   CodeStub* stub = op->stub();
2104 
2105   // check if it needs to be profiled
2106   ciMethodData* md = NULL;
2107   ciProfileData* data = NULL;
2108 
2109   if (should_profile) {
2110     data_check(op, &md, &data);
2111   }
2112   Label profile_cast_success, profile_cast_failure, done;
2113   Label *success_target = should_profile ? &profile_cast_success : &done;
2114   Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
2115 
2116   if (should_profile) {
2117     profile_object(md, data, value, klass_RInfo, &done);
2118   } else {
2119     __ beqz(value, done);
2120   }
2121 
2122   add_debug_info_for_null_check_here(op->info_for_exception());
2123   __ load_klass(k_RInfo, array);
2124   __ load_klass(klass_RInfo, value);
2125 
2126   lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target);
2127 
2128   // fall through to the success case
2129   if (should_profile) {
2130     Register mdo = klass_RInfo;
2131     Register recv = k_RInfo;
2132     __ bind(profile_cast_success);
2133     __ mov_metadata(mdo, md->constant_encoding());
2134     __ load_klass(recv, value);
2135     type_profile_helper(mdo, md, data, recv, &done);
2136     __ j(done);
2137 
2138     __ bind(profile_cast_failure);
2139     __ mov_metadata(mdo, md->constant_encoding());
2140     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2141     __ ld(t1, counter_addr);
2142     __ addi(t1, t1, -DataLayout::counter_increment);
2143     __ sd(t1, counter_addr);
2144     __ j(*stub->entry());
2145   }
2146 
2147   __ bind(done);
2148 }
2149 
2150 void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo,
2151                                  ciProfileData* data, Label* success, Label* failure,
2152                                  Label& profile_cast_success, Label& profile_cast_failure) {
2153   Register mdo = klass_RInfo;
2154   Register recv = k_RInfo;
2155   __ bind(profile_cast_success);
2156   __ mov_metadata(mdo, md->constant_encoding());
2157   __ load_klass(recv, obj);
2158   Label update_done;
2159   type_profile_helper(mdo, md, data, recv, success);
2160   __ j(*success);
2161 
2162   __ bind(profile_cast_failure);
2163   __ mov_metadata(mdo, md->constant_encoding());
2164   Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2165   __ ld(t0, counter_addr);
2166   __ addi(t0, t0, -DataLayout::counter_increment);
2167   __ sd(t0, counter_addr);
2168   __ j(*failure);
2169 }
2170 
2171 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1,
2172                                         Label* success_target, Label* failure_target) {
2173   // get instance klass (it's already uncompressed)
2174   __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2175   // perform the fast part of the checking logic
2176   __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2177   // call out-of-line instance of __ check_klass_subtype_slow_path(...)
2178   __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo
2179   __ sd(klass_RInfo, Address(sp, wordSize));  // sub klass
2180   __ sd(k_RInfo, Address(sp, 0));             // super klass
2181   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2182   // load result to k_RInfo
2183   __ ld(k_RInfo, Address(sp, 0));
2184   __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo
2185   // result is a boolean
2186   __ beqz(k_RInfo, *failure_target, /* is_far */ true);
2187 }
2188 
2189 void LIR_Assembler::const2reg_helper(LIR_Opr src) {
2190   switch (src->as_constant_ptr()->type()) {
2191     case T_INT:
2192     case T_ADDRESS:
2193     case T_OBJECT:
2194     case T_ARRAY:
2195     case T_METADATA:
2196         const2reg(src, FrameMap::t0_opr, lir_patch_none, NULL);
2197         break;
2198     case T_LONG:
2199         const2reg(src, FrameMap::t0_long_opr, lir_patch_none, NULL);
2200         break;
2201     case T_FLOAT:
2202     case T_DOUBLE:
2203     default:
2204       ShouldNotReachHere();
2205   }
2206 }
2207 
2208 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) {
2209   switch (code) {
2210     case lir_logic_and: __ andrw(dst, left, right); break;
2211     case lir_logic_or:  __ orrw (dst, left, right); break;
2212     case lir_logic_xor: __ xorrw(dst, left, right); break;
2213     default:            ShouldNotReachHere();
2214   }
2215 }
2216 
2217 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) {
2218   switch (code) {
2219     case lir_logic_and: __ andr(dst, left, right); break;
2220     case lir_logic_or:  __ orr (dst, left, right); break;
2221     case lir_logic_xor: __ xorr(dst, left, right); break;
2222     default:            ShouldNotReachHere();
2223   }
2224 }
2225 
2226 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) {
2227   switch (code) {
2228     case lir_logic_and: __ andi(dst, left, right); break;
2229     case lir_logic_or:  __ ori (dst, left, right); break;
2230     case lir_logic_xor: __ xori(dst, left, right); break;
2231     default:            ShouldNotReachHere();
2232   }
2233 }
2234 
2235 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2236   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2237   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2238   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2239   __ sd(r, Address(sp, offset_from_rsp_in_bytes));
2240 }
2241 
2242 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2243   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2244   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2245   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2246   __ li(t0, c);
2247   __ sd(t0, Address(sp, offset_from_rsp_in_bytes));
2248 }
2249 
2250 #undef __