1 /*
   2  * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_FrameMap.hpp"
  30 #include "c1/c1_Instruction.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_LIRGenerator.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArray.hpp"
  36 #include "ci/ciObjArrayKlass.hpp"
  37 #include "ci/ciTypeArrayKlass.hpp"
  38 #include "compiler/compilerDefinitions.inline.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "runtime/stubRoutines.hpp"
  41 #include "utilities/powerOfTwo.hpp"
  42 #include "vmreg_aarch64.inline.hpp"
  43 
  44 #ifdef ASSERT
  45 #define __ gen()->lir(__FILE__, __LINE__)->
  46 #else
  47 #define __ gen()->lir()->
  48 #endif
  49 
  50 // Item will be loaded into a byte register; Intel only
  51 void LIRItem::load_byte_item() {
  52   load_item();
  53 }
  54 
  55 
  56 void LIRItem::load_nonconstant() {
  57   LIR_Opr r = value()->operand();
  58   if (r->is_constant()) {
  59     _result = r;
  60   } else {
  61     load_item();
  62   }
  63 }
  64 
  65 //--------------------------------------------------------------
  66 //               LIRGenerator
  67 //--------------------------------------------------------------
  68 
  69 
  70 LIR_Opr LIRGenerator::exceptionOopOpr() { return FrameMap::r0_oop_opr; }
  71 LIR_Opr LIRGenerator::exceptionPcOpr()  { return FrameMap::r3_opr; }
  72 LIR_Opr LIRGenerator::divInOpr()        { Unimplemented(); return LIR_OprFact::illegalOpr; }
  73 LIR_Opr LIRGenerator::divOutOpr()       { Unimplemented(); return LIR_OprFact::illegalOpr; }
  74 LIR_Opr LIRGenerator::remOutOpr()       { Unimplemented(); return LIR_OprFact::illegalOpr; }
  75 LIR_Opr LIRGenerator::shiftCountOpr()   { Unimplemented(); return LIR_OprFact::illegalOpr; }
  76 LIR_Opr LIRGenerator::syncLockOpr()     { return new_register(T_INT); }
  77 LIR_Opr LIRGenerator::syncTempOpr()     { return FrameMap::r0_opr; }
  78 LIR_Opr LIRGenerator::getThreadTemp()   { return LIR_OprFact::illegalOpr; }
  79 
  80 
  81 LIR_Opr LIRGenerator::result_register_for(ValueType* type, bool callee) {
  82   LIR_Opr opr;
  83   switch (type->tag()) {
  84     case intTag:     opr = FrameMap::r0_opr;          break;
  85     case objectTag:  opr = FrameMap::r0_oop_opr;      break;
  86     case longTag:    opr = FrameMap::long0_opr;        break;
  87     case floatTag:   opr = FrameMap::fpu0_float_opr;  break;
  88     case doubleTag:  opr = FrameMap::fpu0_double_opr;  break;
  89 
  90     case addressTag:
  91     default: ShouldNotReachHere(); return LIR_OprFact::illegalOpr;
  92   }
  93 
  94   assert(opr->type_field() == as_OprType(as_BasicType(type)), "type mismatch");
  95   return opr;
  96 }
  97 
  98 
  99 LIR_Opr LIRGenerator::rlock_byte(BasicType type) {
 100   LIR_Opr reg = new_register(T_INT);
 101   set_vreg_flag(reg, LIRGenerator::byte_reg);
 102   return reg;
 103 }
 104 
 105 
 106 //--------- loading items into registers --------------------------------
 107 
 108 
 109 bool LIRGenerator::can_store_as_constant(Value v, BasicType type) const {
 110   if (v->type()->as_IntConstant() != nullptr) {
 111     return v->type()->as_IntConstant()->value() == 0L;
 112   } else if (v->type()->as_LongConstant() != nullptr) {
 113     return v->type()->as_LongConstant()->value() == 0L;
 114   } else if (v->type()->as_ObjectConstant() != nullptr) {
 115     return v->type()->as_ObjectConstant()->value()->is_null_object();
 116   } else {
 117     return false;
 118   }
 119 }
 120 
 121 bool LIRGenerator::can_inline_as_constant(Value v) const {
 122   // FIXME: Just a guess
 123   if (v->type()->as_IntConstant() != nullptr) {
 124     return Assembler::operand_valid_for_add_sub_immediate(v->type()->as_IntConstant()->value());
 125   } else if (v->type()->as_LongConstant() != nullptr) {
 126     return v->type()->as_LongConstant()->value() == 0L;
 127   } else if (v->type()->as_ObjectConstant() != nullptr) {
 128     return v->type()->as_ObjectConstant()->value()->is_null_object();
 129   } else {
 130     return false;
 131   }
 132 }
 133 
 134 
 135 bool LIRGenerator::can_inline_as_constant(LIR_Const* c) const { return false; }
 136 
 137 
 138 LIR_Opr LIRGenerator::safepoint_poll_register() {
 139   return LIR_OprFact::illegalOpr;
 140 }
 141 
 142 
 143 LIR_Address* LIRGenerator::generate_address(LIR_Opr base, LIR_Opr index,
 144                                             int shift, int disp, BasicType type) {
 145   assert(base->is_register(), "must be");
 146   intx large_disp = disp;
 147 
 148   // accumulate fixed displacements
 149   if (index->is_constant()) {
 150     LIR_Const *constant = index->as_constant_ptr();
 151     if (constant->type() == T_INT) {
 152       large_disp += ((intx)index->as_jint()) << shift;
 153     } else {
 154       assert(constant->type() == T_LONG, "should be");
 155       jlong c = index->as_jlong() << shift;
 156       if ((jlong)((jint)c) == c) {
 157         large_disp += c;
 158         index = LIR_OprFact::illegalOpr;
 159       } else {
 160         LIR_Opr tmp = new_register(T_LONG);
 161         __ move(index, tmp);
 162         index = tmp;
 163         // apply shift and displacement below
 164       }
 165     }
 166   }
 167 
 168   if (index->is_register()) {
 169     // apply the shift and accumulate the displacement
 170     if (shift > 0) {
 171       // Use long register to avoid overflow when shifting large index values left.
 172       LIR_Opr tmp = new_register(T_LONG);
 173       __ convert(Bytecodes::_i2l, index, tmp);
 174       __ shift_left(tmp, shift, tmp);
 175       index = tmp;
 176     }
 177     if (large_disp != 0) {
 178       LIR_Opr tmp = new_pointer_register();
 179       if (Assembler::operand_valid_for_add_sub_immediate(large_disp)) {
 180         __ add(index, LIR_OprFact::intptrConst(large_disp), tmp);
 181         index = tmp;
 182       } else {
 183         __ move(LIR_OprFact::intptrConst(large_disp), tmp);
 184         __ add(tmp, index, tmp);
 185         index = tmp;
 186       }
 187       large_disp = 0;
 188     }
 189   } else if (large_disp != 0 && !Address::offset_ok_for_immed(large_disp, shift)) {
 190     // index is illegal so replace it with the displacement loaded into a register
 191     index = new_pointer_register();
 192     __ move(LIR_OprFact::intptrConst(large_disp), index);
 193     large_disp = 0;
 194   }
 195 
 196   // at this point we either have base + index or base + displacement
 197   if (large_disp == 0 && index->is_register()) {
 198     return new LIR_Address(base, index, type);
 199   } else {
 200     assert(Address::offset_ok_for_immed(large_disp, shift), "failed for large_disp: " INTPTR_FORMAT " and shift %d", large_disp, shift);
 201     return new LIR_Address(base, large_disp, type);
 202   }
 203 }
 204 
 205 LIR_Address* LIRGenerator::emit_array_address(LIR_Opr array_opr, LIR_Opr index_opr,
 206                                               BasicType type) {
 207   int offset_in_bytes = arrayOopDesc::base_offset_in_bytes(type);
 208   int elem_size = type2aelembytes(type);
 209   int shift = exact_log2(elem_size);
 210   return generate_address(array_opr, index_opr, shift, offset_in_bytes, type);
 211 }
 212 
 213 LIR_Opr LIRGenerator::load_immediate(jlong x, BasicType type) {
 214   LIR_Opr r;
 215   if (type == T_LONG) {
 216     r = LIR_OprFact::longConst(x);
 217     if (!Assembler::operand_valid_for_logical_immediate(false, x)) {
 218       LIR_Opr tmp = new_register(type);
 219       __ move(r, tmp);
 220       return tmp;
 221     }
 222   } else if (type == T_INT) {
 223     r = LIR_OprFact::intConst(checked_cast<jint>(x));
 224     if (!Assembler::operand_valid_for_logical_immediate(true, x)) {
 225       // This is all rather nasty.  We don't know whether our constant
 226       // is required for a logical or an arithmetic operation, wo we
 227       // don't know what the range of valid values is!!
 228       LIR_Opr tmp = new_register(type);
 229       __ move(r, tmp);
 230       return tmp;
 231     }
 232   } else {
 233     ShouldNotReachHere();
 234   }
 235   return r;
 236 }
 237 
 238 
 239 
 240 void LIRGenerator::increment_counter(address counter, BasicType type, int step) {
 241   LIR_Opr pointer = new_pointer_register();
 242   __ move(LIR_OprFact::intptrConst(counter), pointer);
 243   LIR_Address* addr = new LIR_Address(pointer, type);
 244   increment_counter(addr, step);
 245 }
 246 
 247 
 248 void LIRGenerator::increment_counter(LIR_Address* addr, int step) {
 249   LIR_Opr imm;
 250   switch(addr->type()) {
 251   case T_INT:
 252     imm = LIR_OprFact::intConst(step);
 253     break;
 254   case T_LONG:
 255     imm = LIR_OprFact::longConst(step);
 256     break;
 257   default:
 258     ShouldNotReachHere();
 259   }
 260   LIR_Opr reg = new_register(addr->type());
 261   __ load(addr, reg);
 262   __ add(reg, imm, reg);
 263   __ store(reg, addr);
 264 }
 265 
 266 void LIRGenerator::cmp_mem_int(LIR_Condition condition, LIR_Opr base, int disp, int c, CodeEmitInfo* info) {
 267   LIR_Opr reg = new_register(T_INT);
 268   __ load(generate_address(base, disp, T_INT), reg, info);
 269   __ cmp(condition, reg, LIR_OprFact::intConst(c));
 270 }
 271 
 272 void LIRGenerator::cmp_reg_mem(LIR_Condition condition, LIR_Opr reg, LIR_Opr base, int disp, BasicType type, CodeEmitInfo* info) {
 273   LIR_Opr reg1 = new_register(T_INT);
 274   __ load(generate_address(base, disp, type), reg1, info);
 275   __ cmp(condition, reg, reg1);
 276 }
 277 
 278 
 279 bool LIRGenerator::strength_reduce_multiply(LIR_Opr left, jint c, LIR_Opr result, LIR_Opr tmp) {
 280 
 281   if (is_power_of_2(c - 1)) {
 282     __ shift_left(left, exact_log2(c - 1), tmp);
 283     __ add(tmp, left, result);
 284     return true;
 285   } else if (is_power_of_2(c + 1)) {
 286     __ shift_left(left, exact_log2(c + 1), tmp);
 287     __ sub(tmp, left, result);
 288     return true;
 289   } else {
 290     return false;
 291   }
 292 }
 293 
 294 void LIRGenerator::store_stack_parameter (LIR_Opr item, ByteSize offset_from_sp) {
 295   BasicType type = item->type();
 296   __ store(item, new LIR_Address(FrameMap::sp_opr, in_bytes(offset_from_sp), type));
 297 }
 298 
 299 void LIRGenerator::array_store_check(LIR_Opr value, LIR_Opr array, CodeEmitInfo* store_check_info, ciMethod* profiled_method, int profiled_bci) {
 300     LIR_Opr tmp1 = new_register(objectType);
 301     LIR_Opr tmp2 = new_register(objectType);
 302     LIR_Opr tmp3 = new_register(objectType);
 303     __ store_check(value, array, tmp1, tmp2, tmp3, store_check_info, profiled_method, profiled_bci);
 304 }
 305 
 306 //----------------------------------------------------------------------
 307 //             visitor functions
 308 //----------------------------------------------------------------------
 309 
 310 void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
 311   assert(x->is_pinned(),"");
 312   LIRItem obj(x->obj(), this);
 313   obj.load_item();
 314 
 315   set_no_result(x);
 316 
 317   // "lock" stores the address of the monitor stack slot, so this is not an oop
 318   LIR_Opr lock = new_register(T_INT);
 319   LIR_Opr scratch = new_register(T_INT);
 320 
 321   CodeEmitInfo* info_for_exception = nullptr;
 322   if (x->needs_null_check()) {
 323     info_for_exception = state_for(x);
 324   }
 325   // this CodeEmitInfo must not have the xhandlers because here the
 326   // object is already locked (xhandlers expect object to be unlocked)
 327   CodeEmitInfo* info = state_for(x, x->state(), true);
 328   monitor_enter(obj.result(), lock, syncTempOpr(), scratch,
 329                         x->monitor_no(), info_for_exception, info);
 330 }
 331 
 332 
 333 void LIRGenerator::do_MonitorExit(MonitorExit* x) {
 334   assert(x->is_pinned(),"");
 335 
 336   LIRItem obj(x->obj(), this);
 337   obj.dont_load_item();
 338 
 339   LIR_Opr lock = new_register(T_INT);
 340   LIR_Opr obj_temp = new_register(T_INT);
 341   LIR_Opr scratch = new_register(T_INT);
 342   set_no_result(x);
 343   monitor_exit(obj_temp, lock, syncTempOpr(), scratch, x->monitor_no());
 344 }
 345 
 346 void LIRGenerator::do_NegateOp(NegateOp* x) {
 347 
 348   LIRItem from(x->x(), this);
 349   from.load_item();
 350   LIR_Opr result = rlock_result(x);
 351   __ negate (from.result(), result);
 352 
 353 }
 354 
 355 // for  _fadd, _fmul, _fsub, _fdiv, _frem
 356 //      _dadd, _dmul, _dsub, _ddiv, _drem
 357 void LIRGenerator::do_ArithmeticOp_FPU(ArithmeticOp* x) {
 358 
 359   if (x->op() == Bytecodes::_frem || x->op() == Bytecodes::_drem) {
 360     // float remainder is implemented as a direct call into the runtime
 361     LIRItem right(x->x(), this);
 362     LIRItem left(x->y(), this);
 363 
 364     BasicTypeList signature(2);
 365     if (x->op() == Bytecodes::_frem) {
 366       signature.append(T_FLOAT);
 367       signature.append(T_FLOAT);
 368     } else {
 369       signature.append(T_DOUBLE);
 370       signature.append(T_DOUBLE);
 371     }
 372     CallingConvention* cc = frame_map()->c_calling_convention(&signature);
 373 
 374     const LIR_Opr result_reg = result_register_for(x->type());
 375     left.load_item_force(cc->at(1));
 376     right.load_item();
 377 
 378     __ move(right.result(), cc->at(0));
 379 
 380     address entry;
 381     if (x->op() == Bytecodes::_frem) {
 382       entry = CAST_FROM_FN_PTR(address, SharedRuntime::frem);
 383     } else {
 384       entry = CAST_FROM_FN_PTR(address, SharedRuntime::drem);
 385     }
 386 
 387     LIR_Opr result = rlock_result(x);
 388     __ call_runtime_leaf(entry, getThreadTemp(), result_reg, cc->args());
 389     __ move(result_reg, result);
 390 
 391     return;
 392   }
 393 
 394   LIRItem left(x->x(),  this);
 395   LIRItem right(x->y(), this);
 396   LIRItem* left_arg  = &left;
 397   LIRItem* right_arg = &right;
 398 
 399   // Always load right hand side.
 400   right.load_item();
 401 
 402   if (!left.is_register())
 403     left.load_item();
 404 
 405   LIR_Opr reg = rlock(x);
 406 
 407   arithmetic_op_fpu(x->op(), reg, left.result(), right.result());
 408 
 409   set_result(x, round_item(reg));
 410 }
 411 
 412 // for  _ladd, _lmul, _lsub, _ldiv, _lrem
 413 void LIRGenerator::do_ArithmeticOp_Long(ArithmeticOp* x) {
 414 
 415   // missing test if instr is commutative and if we should swap
 416   LIRItem left(x->x(), this);
 417   LIRItem right(x->y(), this);
 418 
 419   if (x->op() == Bytecodes::_ldiv || x->op() == Bytecodes::_lrem) {
 420 
 421     left.load_item();
 422     bool need_zero_check = true;
 423     if (right.is_constant()) {
 424       jlong c = right.get_jlong_constant();
 425       // no need to do div-by-zero check if the divisor is a non-zero constant
 426       if (c != 0) need_zero_check = false;
 427       // do not load right if the divisor is a power-of-2 constant
 428       if (c > 0 && is_power_of_2(c)) {
 429         right.dont_load_item();
 430       } else {
 431         right.load_item();
 432       }
 433     } else {
 434       right.load_item();
 435     }
 436     if (need_zero_check) {
 437       CodeEmitInfo* info = state_for(x);
 438       __ cmp(lir_cond_equal, right.result(), LIR_OprFact::longConst(0));
 439       __ branch(lir_cond_equal, new DivByZeroStub(info));
 440     }
 441 
 442     rlock_result(x);
 443     switch (x->op()) {
 444     case Bytecodes::_lrem:
 445       __ rem (left.result(), right.result(), x->operand());
 446       break;
 447     case Bytecodes::_ldiv:
 448       __ div (left.result(), right.result(), x->operand());
 449       break;
 450     default:
 451       ShouldNotReachHere();
 452       break;
 453     }
 454 
 455 
 456   } else {
 457     assert (x->op() == Bytecodes::_lmul || x->op() == Bytecodes::_ladd || x->op() == Bytecodes::_lsub,
 458             "expect lmul, ladd or lsub");
 459     // add, sub, mul
 460     left.load_item();
 461     if (! right.is_register()) {
 462       if (x->op() == Bytecodes::_lmul
 463           || ! right.is_constant()
 464           || ! Assembler::operand_valid_for_add_sub_immediate(right.get_jlong_constant())) {
 465         right.load_item();
 466       } else { // add, sub
 467         assert (x->op() == Bytecodes::_ladd || x->op() == Bytecodes::_lsub, "expect ladd or lsub");
 468         // don't load constants to save register
 469         right.load_nonconstant();
 470       }
 471     }
 472     rlock_result(x);
 473     arithmetic_op_long(x->op(), x->operand(), left.result(), right.result(), nullptr);
 474   }
 475 }
 476 
 477 // for: _iadd, _imul, _isub, _idiv, _irem
 478 void LIRGenerator::do_ArithmeticOp_Int(ArithmeticOp* x) {
 479 
 480   // Test if instr is commutative and if we should swap
 481   LIRItem left(x->x(),  this);
 482   LIRItem right(x->y(), this);
 483   LIRItem* left_arg = &left;
 484   LIRItem* right_arg = &right;
 485   if (x->is_commutative() && left.is_stack() && right.is_register()) {
 486     // swap them if left is real stack (or cached) and right is real register(not cached)
 487     left_arg = &right;
 488     right_arg = &left;
 489   }
 490 
 491   left_arg->load_item();
 492 
 493   // do not need to load right, as we can handle stack and constants
 494   if (x->op() == Bytecodes::_idiv || x->op() == Bytecodes::_irem) {
 495 
 496     rlock_result(x);
 497     bool need_zero_check = true;
 498     if (right.is_constant()) {
 499       jint c = right.get_jint_constant();
 500       // no need to do div-by-zero check if the divisor is a non-zero constant
 501       if (c != 0) need_zero_check = false;
 502       // do not load right if the divisor is a power-of-2 constant
 503       if (c > 0 && is_power_of_2(c)) {
 504         right_arg->dont_load_item();
 505       } else {
 506         right_arg->load_item();
 507       }
 508     } else {
 509       right_arg->load_item();
 510     }
 511     if (need_zero_check) {
 512       CodeEmitInfo* info = state_for(x);
 513       __ cmp(lir_cond_equal, right_arg->result(), LIR_OprFact::longConst(0));
 514       __ branch(lir_cond_equal, new DivByZeroStub(info));
 515     }
 516 
 517     LIR_Opr ill = LIR_OprFact::illegalOpr;
 518     if (x->op() == Bytecodes::_irem) {
 519       __ irem(left_arg->result(), right_arg->result(), x->operand(), ill, nullptr);
 520     } else if (x->op() == Bytecodes::_idiv) {
 521       __ idiv(left_arg->result(), right_arg->result(), x->operand(), ill, nullptr);
 522     }
 523 
 524   } else if (x->op() == Bytecodes::_iadd || x->op() == Bytecodes::_isub) {
 525     if (right.is_constant()
 526         && Assembler::operand_valid_for_add_sub_immediate(right.get_jint_constant())) {
 527       right.load_nonconstant();
 528     } else {
 529       right.load_item();
 530     }
 531     rlock_result(x);
 532     arithmetic_op_int(x->op(), x->operand(), left_arg->result(), right_arg->result(), LIR_OprFact::illegalOpr);
 533   } else {
 534     assert (x->op() == Bytecodes::_imul, "expect imul");
 535     if (right.is_constant()) {
 536       jint c = right.get_jint_constant();
 537       if (c > 0 && c < max_jint && (is_power_of_2(c) || is_power_of_2(c - 1) || is_power_of_2(c + 1))) {
 538         right_arg->dont_load_item();
 539       } else {
 540         // Cannot use constant op.
 541         right_arg->load_item();
 542       }
 543     } else {
 544       right.load_item();
 545     }
 546     rlock_result(x);
 547     arithmetic_op_int(x->op(), x->operand(), left_arg->result(), right_arg->result(), new_register(T_INT));
 548   }
 549 }
 550 
 551 void LIRGenerator::do_ArithmeticOp(ArithmeticOp* x) {
 552   // when an operand with use count 1 is the left operand, then it is
 553   // likely that no move for 2-operand-LIR-form is necessary
 554   if (x->is_commutative() && x->y()->as_Constant() == nullptr && x->x()->use_count() > x->y()->use_count()) {
 555     x->swap_operands();
 556   }
 557 
 558   ValueTag tag = x->type()->tag();
 559   assert(x->x()->type()->tag() == tag && x->y()->type()->tag() == tag, "wrong parameters");
 560   switch (tag) {
 561     case floatTag:
 562     case doubleTag:  do_ArithmeticOp_FPU(x);  return;
 563     case longTag:    do_ArithmeticOp_Long(x); return;
 564     case intTag:     do_ArithmeticOp_Int(x);  return;
 565     default:         ShouldNotReachHere();    return;
 566   }
 567 }
 568 
 569 // _ishl, _lshl, _ishr, _lshr, _iushr, _lushr
 570 void LIRGenerator::do_ShiftOp(ShiftOp* x) {
 571 
 572   LIRItem left(x->x(),  this);
 573   LIRItem right(x->y(), this);
 574 
 575   left.load_item();
 576 
 577   rlock_result(x);
 578   if (right.is_constant()) {
 579     right.dont_load_item();
 580 
 581     switch (x->op()) {
 582     case Bytecodes::_ishl: {
 583       int c = right.get_jint_constant() & 0x1f;
 584       __ shift_left(left.result(), c, x->operand());
 585       break;
 586     }
 587     case Bytecodes::_ishr: {
 588       int c = right.get_jint_constant() & 0x1f;
 589       __ shift_right(left.result(), c, x->operand());
 590       break;
 591     }
 592     case Bytecodes::_iushr: {
 593       int c = right.get_jint_constant() & 0x1f;
 594       __ unsigned_shift_right(left.result(), c, x->operand());
 595       break;
 596     }
 597     case Bytecodes::_lshl: {
 598       int c = right.get_jint_constant() & 0x3f;
 599       __ shift_left(left.result(), c, x->operand());
 600       break;
 601     }
 602     case Bytecodes::_lshr: {
 603       int c = right.get_jint_constant() & 0x3f;
 604       __ shift_right(left.result(), c, x->operand());
 605       break;
 606     }
 607     case Bytecodes::_lushr: {
 608       int c = right.get_jint_constant() & 0x3f;
 609       __ unsigned_shift_right(left.result(), c, x->operand());
 610       break;
 611     }
 612     default:
 613       ShouldNotReachHere();
 614     }
 615   } else {
 616     right.load_item();
 617     LIR_Opr tmp = new_register(T_INT);
 618     switch (x->op()) {
 619     case Bytecodes::_ishl: {
 620       __ logical_and(right.result(), LIR_OprFact::intConst(0x1f), tmp);
 621       __ shift_left(left.result(), tmp, x->operand(), tmp);
 622       break;
 623     }
 624     case Bytecodes::_ishr: {
 625       __ logical_and(right.result(), LIR_OprFact::intConst(0x1f), tmp);
 626       __ shift_right(left.result(), tmp, x->operand(), tmp);
 627       break;
 628     }
 629     case Bytecodes::_iushr: {
 630       __ logical_and(right.result(), LIR_OprFact::intConst(0x1f), tmp);
 631       __ unsigned_shift_right(left.result(), tmp, x->operand(), tmp);
 632       break;
 633     }
 634     case Bytecodes::_lshl: {
 635       __ logical_and(right.result(), LIR_OprFact::intConst(0x3f), tmp);
 636       __ shift_left(left.result(), tmp, x->operand(), tmp);
 637       break;
 638     }
 639     case Bytecodes::_lshr: {
 640       __ logical_and(right.result(), LIR_OprFact::intConst(0x3f), tmp);
 641       __ shift_right(left.result(), tmp, x->operand(), tmp);
 642       break;
 643     }
 644     case Bytecodes::_lushr: {
 645       __ logical_and(right.result(), LIR_OprFact::intConst(0x3f), tmp);
 646       __ unsigned_shift_right(left.result(), tmp, x->operand(), tmp);
 647       break;
 648     }
 649     default:
 650       ShouldNotReachHere();
 651     }
 652   }
 653 }
 654 
 655 // _iand, _land, _ior, _lor, _ixor, _lxor
 656 void LIRGenerator::do_LogicOp(LogicOp* x) {
 657 
 658   LIRItem left(x->x(),  this);
 659   LIRItem right(x->y(), this);
 660 
 661   left.load_item();
 662 
 663   rlock_result(x);
 664   if (right.is_constant()
 665       && ((right.type()->tag() == intTag
 666            && Assembler::operand_valid_for_logical_immediate(true, right.get_jint_constant()))
 667           || (right.type()->tag() == longTag
 668               && Assembler::operand_valid_for_logical_immediate(false, right.get_jlong_constant()))))  {
 669     right.dont_load_item();
 670   } else {
 671     right.load_item();
 672   }
 673   switch (x->op()) {
 674   case Bytecodes::_iand:
 675   case Bytecodes::_land:
 676     __ logical_and(left.result(), right.result(), x->operand()); break;
 677   case Bytecodes::_ior:
 678   case Bytecodes::_lor:
 679     __ logical_or (left.result(), right.result(), x->operand()); break;
 680   case Bytecodes::_ixor:
 681   case Bytecodes::_lxor:
 682     __ logical_xor(left.result(), right.result(), x->operand()); break;
 683   default: Unimplemented();
 684   }
 685 }
 686 
 687 // _lcmp, _fcmpl, _fcmpg, _dcmpl, _dcmpg
 688 void LIRGenerator::do_CompareOp(CompareOp* x) {
 689   LIRItem left(x->x(), this);
 690   LIRItem right(x->y(), this);
 691   ValueTag tag = x->x()->type()->tag();
 692   if (tag == longTag) {
 693     left.set_destroys_register();
 694   }
 695   left.load_item();
 696   right.load_item();
 697   LIR_Opr reg = rlock_result(x);
 698 
 699   if (x->x()->type()->is_float_kind()) {
 700     Bytecodes::Code code = x->op();
 701     __ fcmp2int(left.result(), right.result(), reg, (code == Bytecodes::_fcmpl || code == Bytecodes::_dcmpl));
 702   } else if (x->x()->type()->tag() == longTag) {
 703     __ lcmp2int(left.result(), right.result(), reg);
 704   } else {
 705     Unimplemented();
 706   }
 707 }
 708 
 709 LIR_Opr LIRGenerator::atomic_cmpxchg(BasicType type, LIR_Opr addr, LIRItem& cmp_value, LIRItem& new_value) {
 710   LIR_Opr ill = LIR_OprFact::illegalOpr;  // for convenience
 711   new_value.load_item();
 712   cmp_value.load_item();
 713   LIR_Opr result = new_register(T_INT);
 714   if (is_reference_type(type)) {
 715     __ cas_obj(addr, cmp_value.result(), new_value.result(), new_register(T_INT), new_register(T_INT), result);
 716   } else if (type == T_INT) {
 717     __ cas_int(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill);
 718   } else if (type == T_LONG) {
 719     __ cas_long(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill);
 720   } else {
 721     ShouldNotReachHere();
 722     Unimplemented();
 723   }
 724   __ logical_xor(FrameMap::r8_opr, LIR_OprFact::intConst(1), result);
 725   return result;
 726 }
 727 
 728 LIR_Opr LIRGenerator::atomic_xchg(BasicType type, LIR_Opr addr, LIRItem& value) {
 729   bool is_oop = is_reference_type(type);
 730   LIR_Opr result = new_register(type);
 731   value.load_item();
 732   assert(type == T_INT || is_oop LP64_ONLY( || type == T_LONG ), "unexpected type");
 733   LIR_Opr tmp = new_register(T_INT);
 734   __ xchg(addr, value.result(), result, tmp);
 735   return result;
 736 }
 737 
 738 LIR_Opr LIRGenerator::atomic_add(BasicType type, LIR_Opr addr, LIRItem& value) {
 739   LIR_Opr result = new_register(type);
 740   value.load_item();
 741   assert(type == T_INT LP64_ONLY( || type == T_LONG ), "unexpected type");
 742   LIR_Opr tmp = new_register(T_INT);
 743   __ xadd(addr, value.result(), result, tmp);
 744   return result;
 745 }
 746 
 747 void LIRGenerator::do_MathIntrinsic(Intrinsic* x) {
 748   assert(x->number_of_arguments() == 1 || (x->number_of_arguments() == 2 && x->id() == vmIntrinsics::_dpow), "wrong type");
 749   if (x->id() == vmIntrinsics::_dexp || x->id() == vmIntrinsics::_dlog ||
 750       x->id() == vmIntrinsics::_dpow || x->id() == vmIntrinsics::_dcos ||
 751       x->id() == vmIntrinsics::_dsin || x->id() == vmIntrinsics::_dtan ||
 752       x->id() == vmIntrinsics::_dlog10) {
 753     do_LibmIntrinsic(x);
 754     return;
 755   }
 756   switch (x->id()) {
 757     case vmIntrinsics::_dabs:
 758     case vmIntrinsics::_dsqrt:
 759     case vmIntrinsics::_dsqrt_strict:
 760     case vmIntrinsics::_floatToFloat16:
 761     case vmIntrinsics::_float16ToFloat: {
 762       assert(x->number_of_arguments() == 1, "wrong type");
 763       LIRItem value(x->argument_at(0), this);
 764       value.load_item();
 765       LIR_Opr src = value.result();
 766       LIR_Opr dst = rlock_result(x);
 767 
 768       switch (x->id()) {
 769         case vmIntrinsics::_dsqrt:
 770         case vmIntrinsics::_dsqrt_strict: {
 771           __ sqrt(src, dst, LIR_OprFact::illegalOpr);
 772           break;
 773         }
 774         case vmIntrinsics::_dabs: {
 775           __ abs(src, dst, LIR_OprFact::illegalOpr);
 776           break;
 777         }
 778         case vmIntrinsics::_floatToFloat16: {
 779           LIR_Opr tmp = new_register(T_FLOAT);
 780           __ move(LIR_OprFact::floatConst(-0.0), tmp);
 781           __ f2hf(src, dst, tmp);
 782           break;
 783         }
 784         case vmIntrinsics::_float16ToFloat: {
 785           LIR_Opr tmp = new_register(T_FLOAT);
 786           __ move(LIR_OprFact::floatConst(-0.0), tmp);
 787           __ hf2f(src, dst, tmp);
 788           break;
 789         }
 790         default:
 791           ShouldNotReachHere();
 792       }
 793       break;
 794     }
 795     default:
 796       ShouldNotReachHere();
 797   }
 798 }
 799 
 800 void LIRGenerator::do_LibmIntrinsic(Intrinsic* x) {
 801   LIRItem value(x->argument_at(0), this);
 802   value.set_destroys_register();
 803 
 804   LIR_Opr calc_result = rlock_result(x);
 805   LIR_Opr result_reg = result_register_for(x->type());
 806 
 807   CallingConvention* cc = nullptr;
 808 
 809   if (x->id() == vmIntrinsics::_dpow) {
 810     LIRItem value1(x->argument_at(1), this);
 811 
 812     value1.set_destroys_register();
 813 
 814     BasicTypeList signature(2);
 815     signature.append(T_DOUBLE);
 816     signature.append(T_DOUBLE);
 817     cc = frame_map()->c_calling_convention(&signature);
 818     value.load_item_force(cc->at(0));
 819     value1.load_item_force(cc->at(1));
 820   } else {
 821     BasicTypeList signature(1);
 822     signature.append(T_DOUBLE);
 823     cc = frame_map()->c_calling_convention(&signature);
 824     value.load_item_force(cc->at(0));
 825   }
 826 
 827   switch (x->id()) {
 828     case vmIntrinsics::_dexp:
 829       if (StubRoutines::dexp() != nullptr) {
 830         __ call_runtime_leaf(StubRoutines::dexp(), getThreadTemp(), result_reg, cc->args());
 831       } else {
 832         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dexp), getThreadTemp(), result_reg, cc->args());
 833       }
 834       break;
 835     case vmIntrinsics::_dlog:
 836       // Math.log intrinsic is not implemented on AArch64 (see JDK-8210858),
 837       // but we can still call the shared runtime.
 838       __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog), getThreadTemp(), result_reg, cc->args());
 839       break;
 840     case vmIntrinsics::_dlog10:
 841       __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), getThreadTemp(), result_reg, cc->args());
 842       break;
 843     case vmIntrinsics::_dpow:
 844       if (StubRoutines::dpow() != nullptr) {
 845         __ call_runtime_leaf(StubRoutines::dpow(), getThreadTemp(), result_reg, cc->args());
 846       } else {
 847         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dpow), getThreadTemp(), result_reg, cc->args());
 848       }
 849       break;
 850     case vmIntrinsics::_dsin:
 851       if (StubRoutines::dsin() != nullptr) {
 852         __ call_runtime_leaf(StubRoutines::dsin(), getThreadTemp(), result_reg, cc->args());
 853       } else {
 854         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dsin), getThreadTemp(), result_reg, cc->args());
 855       }
 856       break;
 857     case vmIntrinsics::_dcos:
 858       if (StubRoutines::dcos() != nullptr) {
 859         __ call_runtime_leaf(StubRoutines::dcos(), getThreadTemp(), result_reg, cc->args());
 860       } else {
 861         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dcos), getThreadTemp(), result_reg, cc->args());
 862       }
 863       break;
 864     case vmIntrinsics::_dtan:
 865       if (StubRoutines::dtan() != nullptr) {
 866         __ call_runtime_leaf(StubRoutines::dtan(), getThreadTemp(), result_reg, cc->args());
 867       } else {
 868         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtan), getThreadTemp(), result_reg, cc->args());
 869       }
 870       break;
 871     default:  ShouldNotReachHere();
 872   }
 873   __ move(result_reg, calc_result);
 874 }
 875 
 876 
 877 void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
 878   assert(x->number_of_arguments() == 5, "wrong type");
 879 
 880   // Make all state_for calls early since they can emit code
 881   CodeEmitInfo* info = nullptr;
 882   if (x->state_before() != nullptr && x->state_before()->force_reexecute()) {
 883     info = state_for(x, x->state_before());
 884     info->set_force_reexecute();
 885   } else {
 886     info = state_for(x, x->state());
 887   }
 888 
 889   LIRItem src(x->argument_at(0), this);
 890   LIRItem src_pos(x->argument_at(1), this);
 891   LIRItem dst(x->argument_at(2), this);
 892   LIRItem dst_pos(x->argument_at(3), this);
 893   LIRItem length(x->argument_at(4), this);
 894 
 895   // operands for arraycopy must use fixed registers, otherwise
 896   // LinearScan will fail allocation (because arraycopy always needs a
 897   // call)
 898 
 899   // The java calling convention will give us enough registers
 900   // so that on the stub side the args will be perfect already.
 901   // On the other slow/special case side we call C and the arg
 902   // positions are not similar enough to pick one as the best.
 903   // Also because the java calling convention is a "shifted" version
 904   // of the C convention we can process the java args trivially into C
 905   // args without worry of overwriting during the xfer
 906 
 907   src.load_item_force     (FrameMap::as_oop_opr(j_rarg0));
 908   src_pos.load_item_force (FrameMap::as_opr(j_rarg1));
 909   dst.load_item_force     (FrameMap::as_oop_opr(j_rarg2));
 910   dst_pos.load_item_force (FrameMap::as_opr(j_rarg3));
 911   length.load_item_force  (FrameMap::as_opr(j_rarg4));
 912 
 913   LIR_Opr tmp =           FrameMap::as_opr(j_rarg5);
 914 
 915   set_no_result(x);
 916 
 917   int flags;
 918   ciArrayKlass* expected_type;
 919   arraycopy_helper(x, &flags, &expected_type);
 920   if (x->check_flag(Instruction::OmitChecksFlag)) {
 921     flags = 0;
 922   }
 923 
 924   __ arraycopy(src.result(), src_pos.result(), dst.result(), dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint
 925 }
 926 
 927 void LIRGenerator::do_update_CRC32(Intrinsic* x) {
 928   assert(UseCRC32Intrinsics, "why are we here?");
 929   // Make all state_for calls early since they can emit code
 930   LIR_Opr result = rlock_result(x);
 931   switch (x->id()) {
 932     case vmIntrinsics::_updateCRC32: {
 933       LIRItem crc(x->argument_at(0), this);
 934       LIRItem val(x->argument_at(1), this);
 935       // val is destroyed by update_crc32
 936       val.set_destroys_register();
 937       crc.load_item();
 938       val.load_item();
 939       __ update_crc32(crc.result(), val.result(), result);
 940       break;
 941     }
 942     case vmIntrinsics::_updateBytesCRC32:
 943     case vmIntrinsics::_updateByteBufferCRC32: {
 944       bool is_updateBytes = (x->id() == vmIntrinsics::_updateBytesCRC32);
 945 
 946       LIRItem crc(x->argument_at(0), this);
 947       LIRItem buf(x->argument_at(1), this);
 948       LIRItem off(x->argument_at(2), this);
 949       LIRItem len(x->argument_at(3), this);
 950       buf.load_item();
 951       off.load_nonconstant();
 952 
 953       LIR_Opr index = off.result();
 954       int offset = is_updateBytes ? arrayOopDesc::base_offset_in_bytes(T_BYTE) : 0;
 955       if (off.result()->is_constant()) {
 956         index = LIR_OprFact::illegalOpr;
 957         offset += off.result()->as_jint();
 958       }
 959       LIR_Opr base_op = buf.result();
 960 
 961       if (index->is_valid()) {
 962         LIR_Opr tmp = new_register(T_LONG);
 963         __ convert(Bytecodes::_i2l, index, tmp);
 964         index = tmp;
 965       }
 966 
 967       if (offset) {
 968         LIR_Opr tmp = new_pointer_register();
 969         __ add(base_op, LIR_OprFact::intConst(offset), tmp);
 970         base_op = tmp;
 971         offset = 0;
 972       }
 973 
 974       LIR_Address* a = new LIR_Address(base_op,
 975                                        index,
 976                                        offset,
 977                                        T_BYTE);
 978       BasicTypeList signature(3);
 979       signature.append(T_INT);
 980       signature.append(T_ADDRESS);
 981       signature.append(T_INT);
 982       CallingConvention* cc = frame_map()->c_calling_convention(&signature);
 983       const LIR_Opr result_reg = result_register_for(x->type());
 984 
 985       LIR_Opr addr = new_pointer_register();
 986       __ leal(LIR_OprFact::address(a), addr);
 987 
 988       crc.load_item_force(cc->at(0));
 989       __ move(addr, cc->at(1));
 990       len.load_item_force(cc->at(2));
 991 
 992       __ call_runtime_leaf(StubRoutines::updateBytesCRC32(), getThreadTemp(), result_reg, cc->args());
 993       __ move(result_reg, result);
 994 
 995       break;
 996     }
 997     default: {
 998       ShouldNotReachHere();
 999     }
1000   }
1001 }
1002 
1003 void LIRGenerator::do_update_CRC32C(Intrinsic* x) {
1004   assert(UseCRC32CIntrinsics, "why are we here?");
1005   // Make all state_for calls early since they can emit code
1006   LIR_Opr result = rlock_result(x);
1007   switch (x->id()) {
1008     case vmIntrinsics::_updateBytesCRC32C:
1009     case vmIntrinsics::_updateDirectByteBufferCRC32C: {
1010       bool is_updateBytes = (x->id() == vmIntrinsics::_updateBytesCRC32C);
1011       int offset = is_updateBytes ? arrayOopDesc::base_offset_in_bytes(T_BYTE) : 0;
1012 
1013       LIRItem crc(x->argument_at(0), this);
1014       LIRItem buf(x->argument_at(1), this);
1015       LIRItem off(x->argument_at(2), this);
1016       LIRItem end(x->argument_at(3), this);
1017 
1018       buf.load_item();
1019       off.load_nonconstant();
1020       end.load_nonconstant();
1021 
1022       // len = end - off
1023       LIR_Opr len  = end.result();
1024       LIR_Opr tmpA = new_register(T_INT);
1025       LIR_Opr tmpB = new_register(T_INT);
1026       __ move(end.result(), tmpA);
1027       __ move(off.result(), tmpB);
1028       __ sub(tmpA, tmpB, tmpA);
1029       len = tmpA;
1030 
1031       LIR_Opr index = off.result();
1032       if(off.result()->is_constant()) {
1033         index = LIR_OprFact::illegalOpr;
1034         offset += off.result()->as_jint();
1035       }
1036       LIR_Opr base_op = buf.result();
1037 
1038       if (index->is_valid()) {
1039         LIR_Opr tmp = new_register(T_LONG);
1040         __ convert(Bytecodes::_i2l, index, tmp);
1041         index = tmp;
1042       }
1043 
1044       if (offset) {
1045         LIR_Opr tmp = new_pointer_register();
1046         __ add(base_op, LIR_OprFact::intConst(offset), tmp);
1047         base_op = tmp;
1048         offset = 0;
1049       }
1050 
1051       LIR_Address* a = new LIR_Address(base_op,
1052                                        index,
1053                                        offset,
1054                                        T_BYTE);
1055       BasicTypeList signature(3);
1056       signature.append(T_INT);
1057       signature.append(T_ADDRESS);
1058       signature.append(T_INT);
1059       CallingConvention* cc = frame_map()->c_calling_convention(&signature);
1060       const LIR_Opr result_reg = result_register_for(x->type());
1061 
1062       LIR_Opr addr = new_pointer_register();
1063       __ leal(LIR_OprFact::address(a), addr);
1064 
1065       crc.load_item_force(cc->at(0));
1066       __ move(addr, cc->at(1));
1067       __ move(len, cc->at(2));
1068 
1069       __ call_runtime_leaf(StubRoutines::updateBytesCRC32C(), getThreadTemp(), result_reg, cc->args());
1070       __ move(result_reg, result);
1071 
1072       break;
1073     }
1074     default: {
1075       ShouldNotReachHere();
1076     }
1077   }
1078 }
1079 
1080 void LIRGenerator::do_FmaIntrinsic(Intrinsic* x) {
1081   assert(x->number_of_arguments() == 3, "wrong type");
1082   assert(UseFMA, "Needs FMA instructions support.");
1083   LIRItem value(x->argument_at(0), this);
1084   LIRItem value1(x->argument_at(1), this);
1085   LIRItem value2(x->argument_at(2), this);
1086 
1087   value.load_item();
1088   value1.load_item();
1089   value2.load_item();
1090 
1091   LIR_Opr calc_input = value.result();
1092   LIR_Opr calc_input1 = value1.result();
1093   LIR_Opr calc_input2 = value2.result();
1094   LIR_Opr calc_result = rlock_result(x);
1095 
1096   switch (x->id()) {
1097   case vmIntrinsics::_fmaD:   __ fmad(calc_input, calc_input1, calc_input2, calc_result); break;
1098   case vmIntrinsics::_fmaF:   __ fmaf(calc_input, calc_input1, calc_input2, calc_result); break;
1099   default:                    ShouldNotReachHere();
1100   }
1101 }
1102 
1103 void LIRGenerator::do_vectorizedMismatch(Intrinsic* x) {
1104   fatal("vectorizedMismatch intrinsic is not implemented on this platform");
1105 }
1106 
1107 // _i2l, _i2f, _i2d, _l2i, _l2f, _l2d, _f2i, _f2l, _f2d, _d2i, _d2l, _d2f
1108 // _i2b, _i2c, _i2s
1109 void LIRGenerator::do_Convert(Convert* x) {
1110   LIRItem value(x->value(), this);
1111   value.load_item();
1112   LIR_Opr input = value.result();
1113   LIR_Opr result = rlock(x);
1114 
1115   // arguments of lir_convert
1116   LIR_Opr conv_input = input;
1117   LIR_Opr conv_result = result;
1118 
1119   __ convert(x->op(), conv_input, conv_result);
1120 
1121   assert(result->is_virtual(), "result must be virtual register");
1122   set_result(x, result);
1123 }
1124 
1125 void LIRGenerator::do_NewInstance(NewInstance* x) {
1126 #ifndef PRODUCT
1127   if (PrintNotLoaded && !x->klass()->is_loaded()) {
1128     tty->print_cr("   ###class not loaded at new bci %d", x->printable_bci());
1129   }
1130 #endif
1131   CodeEmitInfo* info = state_for(x, x->state());
1132   LIR_Opr reg = result_register_for(x->type());
1133   new_instance(reg, x->klass(), x->is_unresolved(),
1134                        FrameMap::r10_oop_opr,
1135                        FrameMap::r11_oop_opr,
1136                        FrameMap::r4_oop_opr,
1137                        LIR_OprFact::illegalOpr,
1138                        FrameMap::r3_metadata_opr, info);
1139   LIR_Opr result = rlock_result(x);
1140   __ move(reg, result);
1141 }
1142 
1143 void LIRGenerator::do_NewTypeArray(NewTypeArray* x) {
1144   CodeEmitInfo* info = nullptr;
1145   if (x->state_before() != nullptr && x->state_before()->force_reexecute()) {
1146     info = state_for(x, x->state_before());
1147     info->set_force_reexecute();
1148   } else {
1149     info = state_for(x, x->state());
1150   }
1151 
1152   LIRItem length(x->length(), this);
1153   length.load_item_force(FrameMap::r19_opr);
1154 
1155   LIR_Opr reg = result_register_for(x->type());
1156   LIR_Opr tmp1 = FrameMap::r10_oop_opr;
1157   LIR_Opr tmp2 = FrameMap::r11_oop_opr;
1158   LIR_Opr tmp3 = FrameMap::r5_oop_opr;
1159   LIR_Opr tmp4 = reg;
1160   LIR_Opr klass_reg = FrameMap::r3_metadata_opr;
1161   LIR_Opr len = length.result();
1162   BasicType elem_type = x->elt_type();
1163 
1164   __ metadata2reg(ciTypeArrayKlass::make(elem_type)->constant_encoding(), klass_reg);
1165 
1166   CodeStub* slow_path = new NewTypeArrayStub(klass_reg, len, reg, info);
1167   __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path, x->zero_array());
1168 
1169   LIR_Opr result = rlock_result(x);
1170   __ move(reg, result);
1171 }
1172 
1173 void LIRGenerator::do_NewObjectArray(NewObjectArray* x) {
1174   LIRItem length(x->length(), this);
1175   // in case of patching (i.e., object class is not yet loaded), we need to reexecute the instruction
1176   // and therefore provide the state before the parameters have been consumed
1177   CodeEmitInfo* patching_info = nullptr;
1178   if (!x->klass()->is_loaded() || PatchALot) {
1179     patching_info =  state_for(x, x->state_before());
1180   }
1181 
1182   CodeEmitInfo* info = state_for(x, x->state());
1183 
1184   LIR_Opr reg = result_register_for(x->type());
1185   LIR_Opr tmp1 = FrameMap::r10_oop_opr;
1186   LIR_Opr tmp2 = FrameMap::r11_oop_opr;
1187   LIR_Opr tmp3 = FrameMap::r5_oop_opr;
1188   LIR_Opr tmp4 = reg;
1189   LIR_Opr klass_reg = FrameMap::r3_metadata_opr;
1190 
1191   length.load_item_force(FrameMap::r19_opr);
1192   LIR_Opr len = length.result();
1193 
1194   CodeStub* slow_path = new NewObjectArrayStub(klass_reg, len, reg, info);
1195   ciKlass* obj = (ciKlass*) ciObjArrayKlass::make(x->klass());
1196   if (obj == ciEnv::unloaded_ciobjarrayklass()) {
1197     BAILOUT("encountered unloaded_ciobjarrayklass due to out of memory error");
1198   }
1199   klass2reg_with_patching(klass_reg, obj, patching_info);
1200   __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, T_OBJECT, klass_reg, slow_path);
1201 
1202   LIR_Opr result = rlock_result(x);
1203   __ move(reg, result);
1204 }
1205 
1206 
1207 void LIRGenerator::do_NewMultiArray(NewMultiArray* x) {
1208   Values* dims = x->dims();
1209   int i = dims->length();
1210   LIRItemList* items = new LIRItemList(i, i, nullptr);
1211   while (i-- > 0) {
1212     LIRItem* size = new LIRItem(dims->at(i), this);
1213     items->at_put(i, size);
1214   }
1215 
1216   // Evaluate state_for early since it may emit code.
1217   CodeEmitInfo* patching_info = nullptr;
1218   if (!x->klass()->is_loaded() || PatchALot) {
1219     patching_info = state_for(x, x->state_before());
1220 
1221     // Cannot re-use same xhandlers for multiple CodeEmitInfos, so
1222     // clone all handlers (NOTE: Usually this is handled transparently
1223     // by the CodeEmitInfo cloning logic in CodeStub constructors but
1224     // is done explicitly here because a stub isn't being used).
1225     x->set_exception_handlers(new XHandlers(x->exception_handlers()));
1226   }
1227   CodeEmitInfo* info = state_for(x, x->state());
1228 
1229   i = dims->length();
1230   while (i-- > 0) {
1231     LIRItem* size = items->at(i);
1232     size->load_item();
1233 
1234     store_stack_parameter(size->result(), in_ByteSize(i*4));
1235   }
1236 
1237   LIR_Opr klass_reg = FrameMap::r0_metadata_opr;
1238   klass2reg_with_patching(klass_reg, x->klass(), patching_info);
1239 
1240   LIR_Opr rank = FrameMap::r19_opr;
1241   __ move(LIR_OprFact::intConst(x->rank()), rank);
1242   LIR_Opr varargs = FrameMap::r2_opr;
1243   __ move(FrameMap::sp_opr, varargs);
1244   LIR_OprList* args = new LIR_OprList(3);
1245   args->append(klass_reg);
1246   args->append(rank);
1247   args->append(varargs);
1248   LIR_Opr reg = result_register_for(x->type());
1249   __ call_runtime(Runtime1::entry_for(Runtime1::new_multi_array_id),
1250                   LIR_OprFact::illegalOpr,
1251                   reg, args, info);
1252 
1253   LIR_Opr result = rlock_result(x);
1254   __ move(reg, result);
1255 }
1256 
1257 void LIRGenerator::do_BlockBegin(BlockBegin* x) {
1258   // nothing to do for now
1259 }
1260 
1261 void LIRGenerator::do_CheckCast(CheckCast* x) {
1262   LIRItem obj(x->obj(), this);
1263 
1264   CodeEmitInfo* patching_info = nullptr;
1265   if (!x->klass()->is_loaded() || (PatchALot && !x->is_incompatible_class_change_check() && !x->is_invokespecial_receiver_check())) {
1266     // must do this before locking the destination register as an oop register,
1267     // and before the obj is loaded (the latter is for deoptimization)
1268     patching_info = state_for(x, x->state_before());
1269   }
1270   obj.load_item();
1271 
1272   // info for exceptions
1273   CodeEmitInfo* info_for_exception =
1274       (x->needs_exception_state() ? state_for(x) :
1275                                     state_for(x, x->state_before(), true /*ignore_xhandler*/));
1276 
1277   CodeStub* stub;
1278   if (x->is_incompatible_class_change_check()) {
1279     assert(patching_info == nullptr, "can't patch this");
1280     stub = new SimpleExceptionStub(Runtime1::throw_incompatible_class_change_error_id, LIR_OprFact::illegalOpr, info_for_exception);
1281   } else if (x->is_invokespecial_receiver_check()) {
1282     assert(patching_info == nullptr, "can't patch this");
1283     stub = new DeoptimizeStub(info_for_exception,
1284                               Deoptimization::Reason_class_check,
1285                               Deoptimization::Action_none);
1286   } else {
1287     stub = new SimpleExceptionStub(Runtime1::throw_class_cast_exception_id, obj.result(), info_for_exception);
1288   }
1289   LIR_Opr reg = rlock_result(x);
1290   LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
1291   if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
1292     tmp3 = new_register(objectType);
1293   }
1294   __ checkcast(reg, obj.result(), x->klass(),
1295                new_register(objectType), new_register(objectType), tmp3,
1296                x->direct_compare(), info_for_exception, patching_info, stub,
1297                x->profiled_method(), x->profiled_bci());
1298 }
1299 
1300 void LIRGenerator::do_InstanceOf(InstanceOf* x) {
1301   LIRItem obj(x->obj(), this);
1302 
1303   // result and test object may not be in same register
1304   LIR_Opr reg = rlock_result(x);
1305   CodeEmitInfo* patching_info = nullptr;
1306   if ((!x->klass()->is_loaded() || PatchALot)) {
1307     // must do this before locking the destination register as an oop register
1308     patching_info = state_for(x, x->state_before());
1309   }
1310   obj.load_item();
1311   LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
1312   if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
1313     tmp3 = new_register(objectType);
1314   }
1315   __ instanceof(reg, obj.result(), x->klass(),
1316                 new_register(objectType), new_register(objectType), tmp3,
1317                 x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
1318 }
1319 
1320 void LIRGenerator::do_If(If* x) {
1321   assert(x->number_of_sux() == 2, "inconsistency");
1322   ValueTag tag = x->x()->type()->tag();
1323   bool is_safepoint = x->is_safepoint();
1324 
1325   If::Condition cond = x->cond();
1326 
1327   LIRItem xitem(x->x(), this);
1328   LIRItem yitem(x->y(), this);
1329   LIRItem* xin = &xitem;
1330   LIRItem* yin = &yitem;
1331 
1332   if (tag == longTag) {
1333     // for longs, only conditions "eql", "neq", "lss", "geq" are valid;
1334     // mirror for other conditions
1335     if (cond == If::gtr || cond == If::leq) {
1336       cond = Instruction::mirror(cond);
1337       xin = &yitem;
1338       yin = &xitem;
1339     }
1340     xin->set_destroys_register();
1341   }
1342   xin->load_item();
1343 
1344   if (tag == longTag) {
1345     if (yin->is_constant()
1346         && Assembler::operand_valid_for_add_sub_immediate(yin->get_jlong_constant())) {
1347       yin->dont_load_item();
1348     } else {
1349       yin->load_item();
1350     }
1351   } else if (tag == intTag) {
1352     if (yin->is_constant()
1353         && Assembler::operand_valid_for_add_sub_immediate(yin->get_jint_constant()))  {
1354       yin->dont_load_item();
1355     } else {
1356       yin->load_item();
1357     }
1358   } else {
1359     yin->load_item();
1360   }
1361 
1362   set_no_result(x);
1363 
1364   LIR_Opr left = xin->result();
1365   LIR_Opr right = yin->result();
1366 
1367   // add safepoint before generating condition code so it can be recomputed
1368   if (x->is_safepoint()) {
1369     // increment backedge counter if needed
1370     increment_backedge_counter_conditionally(lir_cond(cond), left, right, state_for(x, x->state_before()),
1371         x->tsux()->bci(), x->fsux()->bci(), x->profiled_bci());
1372     __ safepoint(LIR_OprFact::illegalOpr, state_for(x, x->state_before()));
1373   }
1374 
1375   __ cmp(lir_cond(cond), left, right);
1376   // Generate branch profiling. Profiling code doesn't kill flags.
1377   profile_branch(x, cond);
1378   move_to_phi(x->state());
1379   if (x->x()->type()->is_float_kind()) {
1380     __ branch(lir_cond(cond), x->tsux(), x->usux());
1381   } else {
1382     __ branch(lir_cond(cond), x->tsux());
1383   }
1384   assert(x->default_sux() == x->fsux(), "wrong destination above");
1385   __ jump(x->default_sux());
1386 }
1387 
1388 LIR_Opr LIRGenerator::getThreadPointer() {
1389    return FrameMap::as_pointer_opr(rthread);
1390 }
1391 
1392 void LIRGenerator::trace_block_entry(BlockBegin* block) { Unimplemented(); }
1393 
1394 void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address,
1395                                         CodeEmitInfo* info) {
1396   __ volatile_store_mem_reg(value, address, info);
1397 }
1398 
1399 void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
1400                                        CodeEmitInfo* info) {
1401   // 8179954: We need to make sure that the code generated for
1402   // volatile accesses forms a sequentially-consistent set of
1403   // operations when combined with STLR and LDAR.  Without a leading
1404   // membar it's possible for a simple Dekker test to fail if loads
1405   // use LD;DMB but stores use STLR.  This can happen if C2 compiles
1406   // the stores in one method and C1 compiles the loads in another.
1407   if (!CompilerConfig::is_c1_only_no_jvmci()) {
1408     __ membar();
1409   }
1410   __ volatile_load_mem_reg(address, result, info);
1411 }