1 /*
   2  * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_FrameMap.hpp"
  30 #include "c1/c1_Instruction.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_LIRGenerator.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArray.hpp"
  36 #include "ci/ciObjArrayKlass.hpp"
  37 #include "ci/ciTypeArrayKlass.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "utilities/powerOfTwo.hpp"
  41 #include "vmreg_aarch64.inline.hpp"
  42 
  43 #ifdef ASSERT
  44 #define __ gen()->lir(__FILE__, __LINE__)->
  45 #else
  46 #define __ gen()->lir()->
  47 #endif
  48 
  49 // Item will be loaded into a byte register; Intel only
  50 void LIRItem::load_byte_item() {
  51   load_item();
  52 }
  53 
  54 
  55 void LIRItem::load_nonconstant() {
  56   LIR_Opr r = value()->operand();
  57   if (r->is_constant()) {
  58     _result = r;
  59   } else {
  60     load_item();
  61   }
  62 }
  63 
  64 //--------------------------------------------------------------
  65 //               LIRGenerator
  66 //--------------------------------------------------------------
  67 
  68 
  69 LIR_Opr LIRGenerator::exceptionOopOpr() { return FrameMap::r0_oop_opr; }
  70 LIR_Opr LIRGenerator::exceptionPcOpr()  { return FrameMap::r3_opr; }
  71 LIR_Opr LIRGenerator::divInOpr()        { Unimplemented(); return LIR_OprFact::illegalOpr; }
  72 LIR_Opr LIRGenerator::divOutOpr()       { Unimplemented(); return LIR_OprFact::illegalOpr; }
  73 LIR_Opr LIRGenerator::remOutOpr()       { Unimplemented(); return LIR_OprFact::illegalOpr; }
  74 LIR_Opr LIRGenerator::shiftCountOpr()   { Unimplemented(); return LIR_OprFact::illegalOpr; }
  75 LIR_Opr LIRGenerator::syncLockOpr()     { return new_register(T_INT); }
  76 LIR_Opr LIRGenerator::syncTempOpr()     { return FrameMap::r0_opr; }
  77 LIR_Opr LIRGenerator::getThreadTemp()   { return LIR_OprFact::illegalOpr; }
  78 
  79 
  80 LIR_Opr LIRGenerator::result_register_for(ValueType* type, bool callee) {
  81   LIR_Opr opr;
  82   switch (type->tag()) {
  83     case intTag:     opr = FrameMap::r0_opr;          break;
  84     case objectTag:  opr = FrameMap::r0_oop_opr;      break;
  85     case longTag:    opr = FrameMap::long0_opr;        break;
  86     case floatTag:   opr = FrameMap::fpu0_float_opr;  break;
  87     case doubleTag:  opr = FrameMap::fpu0_double_opr;  break;
  88 
  89     case addressTag:
  90     default: ShouldNotReachHere(); return LIR_OprFact::illegalOpr;
  91   }
  92 
  93   assert(opr->type_field() == as_OprType(as_BasicType(type)), "type mismatch");
  94   return opr;
  95 }
  96 
  97 
  98 LIR_Opr LIRGenerator::rlock_byte(BasicType type) {
  99   LIR_Opr reg = new_register(T_INT);
 100   set_vreg_flag(reg, LIRGenerator::byte_reg);
 101   return reg;
 102 }
 103 
 104 
 105 //--------- loading items into registers --------------------------------
 106 
 107 
 108 bool LIRGenerator::can_store_as_constant(Value v, BasicType type) const {
 109   if (v->type()->as_IntConstant() != NULL) {
 110     return v->type()->as_IntConstant()->value() == 0L;
 111   } else if (v->type()->as_LongConstant() != NULL) {
 112     return v->type()->as_LongConstant()->value() == 0L;
 113   } else if (v->type()->as_ObjectConstant() != NULL) {
 114     return v->type()->as_ObjectConstant()->value()->is_null_object();
 115   } else {
 116     return false;
 117   }
 118 }
 119 
 120 bool LIRGenerator::can_inline_as_constant(Value v) const {
 121   // FIXME: Just a guess
 122   if (v->type()->as_IntConstant() != NULL) {
 123     return Assembler::operand_valid_for_add_sub_immediate(v->type()->as_IntConstant()->value());
 124   } else if (v->type()->as_LongConstant() != NULL) {
 125     return v->type()->as_LongConstant()->value() == 0L;
 126   } else if (v->type()->as_ObjectConstant() != NULL) {
 127     return v->type()->as_ObjectConstant()->value()->is_null_object();
 128   } else {
 129     return false;
 130   }
 131 }
 132 
 133 
 134 bool LIRGenerator::can_inline_as_constant(LIR_Const* c) const { return false; }
 135 
 136 
 137 LIR_Opr LIRGenerator::safepoint_poll_register() {
 138   return LIR_OprFact::illegalOpr;
 139 }
 140 
 141 
 142 LIR_Address* LIRGenerator::generate_address(LIR_Opr base, LIR_Opr index,
 143                                             int shift, int disp, BasicType type) {
 144   assert(base->is_register(), "must be");
 145   intx large_disp = disp;
 146 
 147   // accumulate fixed displacements
 148   if (index->is_constant()) {
 149     LIR_Const *constant = index->as_constant_ptr();
 150     if (constant->type() == T_INT) {
 151       large_disp += index->as_jint() << shift;
 152     } else {
 153       assert(constant->type() == T_LONG, "should be");
 154       jlong c = index->as_jlong() << shift;
 155       if ((jlong)((jint)c) == c) {
 156         large_disp += c;
 157         index = LIR_OprFact::illegalOpr;
 158       } else {
 159         LIR_Opr tmp = new_register(T_LONG);
 160         __ move(index, tmp);
 161         index = tmp;
 162         // apply shift and displacement below
 163       }
 164     }
 165   }
 166 
 167   if (index->is_register()) {
 168     // apply the shift and accumulate the displacement
 169     if (shift > 0) {
 170       LIR_Opr tmp = new_pointer_register();
 171       __ shift_left(index, shift, tmp);
 172       index = tmp;
 173     }
 174     if (large_disp != 0) {
 175       LIR_Opr tmp = new_pointer_register();
 176       if (Assembler::operand_valid_for_add_sub_immediate(large_disp)) {
 177         __ add(index, LIR_OprFact::intptrConst(large_disp), tmp);
 178         index = tmp;
 179       } else {
 180         __ move(LIR_OprFact::intptrConst(large_disp), tmp);
 181         __ add(tmp, index, tmp);
 182         index = tmp;
 183       }
 184       large_disp = 0;
 185     }
 186   } else if (large_disp != 0 && !Address::offset_ok_for_immed(large_disp, shift)) {
 187     // index is illegal so replace it with the displacement loaded into a register
 188     index = new_pointer_register();
 189     __ move(LIR_OprFact::intptrConst(large_disp), index);
 190     large_disp = 0;
 191   }
 192 
 193   // at this point we either have base + index or base + displacement
 194   if (large_disp == 0 && index->is_register()) {
 195     return new LIR_Address(base, index, type);
 196   } else {
 197     assert(Address::offset_ok_for_immed(large_disp, 0), "must be");
 198     return new LIR_Address(base, large_disp, type);
 199   }
 200 }
 201 
 202 LIR_Address* LIRGenerator::emit_array_address(LIR_Opr array_opr, LIR_Opr index_opr,
 203                                               BasicType type) {
 204   int offset_in_bytes = arrayOopDesc::base_offset_in_bytes(type);
 205   int elem_size = type2aelembytes(type);
 206   int shift = exact_log2(elem_size);
 207 
 208   LIR_Address* addr;
 209   if (index_opr->is_constant()) {
 210     addr = new LIR_Address(array_opr,
 211                            offset_in_bytes + (intx)(index_opr->as_jint()) * elem_size, type);
 212   } else {
 213     if (offset_in_bytes) {
 214       LIR_Opr tmp = new_pointer_register();
 215       __ add(array_opr, LIR_OprFact::intConst(offset_in_bytes), tmp);
 216       array_opr = tmp;
 217       offset_in_bytes = 0;
 218     }
 219     addr =  new LIR_Address(array_opr,
 220                             index_opr,
 221                             LIR_Address::scale(type),
 222                             offset_in_bytes, type);
 223   }
 224   return addr;
 225 }
 226 
 227 LIR_Opr LIRGenerator::load_immediate(int x, BasicType type) {
 228   LIR_Opr r;
 229   if (type == T_LONG) {
 230     r = LIR_OprFact::longConst(x);
 231     if (!Assembler::operand_valid_for_logical_immediate(false, x)) {
 232       LIR_Opr tmp = new_register(type);
 233       __ move(r, tmp);
 234       return tmp;
 235     }
 236   } else if (type == T_INT) {
 237     r = LIR_OprFact::intConst(x);
 238     if (!Assembler::operand_valid_for_logical_immediate(true, x)) {
 239       // This is all rather nasty.  We don't know whether our constant
 240       // is required for a logical or an arithmetic operation, wo we
 241       // don't know what the range of valid values is!!
 242       LIR_Opr tmp = new_register(type);
 243       __ move(r, tmp);
 244       return tmp;
 245     }
 246   } else {
 247     ShouldNotReachHere();
 248     r = NULL;  // unreachable
 249   }
 250   return r;
 251 }
 252 
 253 
 254 
 255 void LIRGenerator::increment_counter(address counter, BasicType type, int step) {
 256   LIR_Opr pointer = new_pointer_register();
 257   __ move(LIR_OprFact::intptrConst(counter), pointer);
 258   LIR_Address* addr = new LIR_Address(pointer, type);
 259   increment_counter(addr, step);
 260 }
 261 
 262 
 263 void LIRGenerator::increment_counter(LIR_Address* addr, int step) {
 264   LIR_Opr imm = NULL;
 265   switch(addr->type()) {
 266   case T_INT:
 267     imm = LIR_OprFact::intConst(step);
 268     break;
 269   case T_LONG:
 270     imm = LIR_OprFact::longConst(step);
 271     break;
 272   default:
 273     ShouldNotReachHere();
 274   }
 275   LIR_Opr reg = new_register(addr->type());
 276   __ load(addr, reg);
 277   __ add(reg, imm, reg);
 278   __ store(reg, addr);
 279 }
 280 
 281 void LIRGenerator::cmp_mem_int(LIR_Condition condition, LIR_Opr base, int disp, int c, CodeEmitInfo* info) {
 282   LIR_Opr reg = new_register(T_INT);
 283   __ load(generate_address(base, disp, T_INT), reg, info);
 284   __ cmp(condition, reg, LIR_OprFact::intConst(c));
 285 }
 286 
 287 void LIRGenerator::cmp_reg_mem(LIR_Condition condition, LIR_Opr reg, LIR_Opr base, int disp, BasicType type, CodeEmitInfo* info) {
 288   LIR_Opr reg1 = new_register(T_INT);
 289   __ load(generate_address(base, disp, type), reg1, info);
 290   __ cmp(condition, reg, reg1);
 291 }
 292 
 293 
 294 bool LIRGenerator::strength_reduce_multiply(LIR_Opr left, jint c, LIR_Opr result, LIR_Opr tmp) {
 295 
 296   if (is_power_of_2(c - 1)) {
 297     __ shift_left(left, exact_log2(c - 1), tmp);
 298     __ add(tmp, left, result);
 299     return true;
 300   } else if (is_power_of_2(c + 1)) {
 301     __ shift_left(left, exact_log2(c + 1), tmp);
 302     __ sub(tmp, left, result);
 303     return true;
 304   } else {
 305     return false;
 306   }
 307 }
 308 
 309 void LIRGenerator::store_stack_parameter (LIR_Opr item, ByteSize offset_from_sp) {
 310   BasicType type = item->type();
 311   __ store(item, new LIR_Address(FrameMap::sp_opr, in_bytes(offset_from_sp), type));
 312 }
 313 
 314 void LIRGenerator::array_store_check(LIR_Opr value, LIR_Opr array, CodeEmitInfo* store_check_info, ciMethod* profiled_method, int profiled_bci) {
 315     LIR_Opr tmp1 = new_register(objectType);
 316     LIR_Opr tmp2 = new_register(objectType);
 317     LIR_Opr tmp3 = new_register(objectType);
 318     __ store_check(value, array, tmp1, tmp2, tmp3, store_check_info, profiled_method, profiled_bci);
 319 }
 320 
 321 //----------------------------------------------------------------------
 322 //             visitor functions
 323 //----------------------------------------------------------------------
 324 
 325 void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
 326   assert(x->is_pinned(),"");
 327   LIRItem obj(x->obj(), this);
 328   obj.load_item();
 329 
 330   set_no_result(x);
 331 
 332   // "lock" stores the address of the monitor stack slot, so this is not an oop
 333   LIR_Opr lock = new_register(T_INT);
 334 
 335   CodeEmitInfo* info_for_exception = NULL;
 336   if (x->needs_null_check()) {
 337     info_for_exception = state_for(x);
 338   }
 339   // this CodeEmitInfo must not have the xhandlers because here the
 340   // object is already locked (xhandlers expect object to be unlocked)
 341   CodeEmitInfo* info = state_for(x, x->state(), true);
 342   monitor_enter(obj.result(), lock, syncTempOpr(), LIR_OprFact::illegalOpr,
 343                         x->monitor_no(), info_for_exception, info);
 344 }
 345 
 346 
 347 void LIRGenerator::do_MonitorExit(MonitorExit* x) {
 348   assert(x->is_pinned(),"");
 349 
 350   LIRItem obj(x->obj(), this);
 351   obj.dont_load_item();
 352 
 353   LIR_Opr lock = new_register(T_INT);
 354   LIR_Opr obj_temp = new_register(T_INT);
 355   set_no_result(x);
 356   monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no());
 357 }
 358 
 359 void LIRGenerator::do_continuation_doYield(Intrinsic* x) {
 360   BasicTypeList signature(0);
 361   CallingConvention* cc = frame_map()->java_calling_convention(&signature, true);
 362 
 363   const LIR_Opr result_reg = result_register_for(x->type());
 364   address entry = StubRoutines::cont_doYield();
 365   LIR_Opr result = rlock_result(x);
 366   CodeEmitInfo* info = state_for(x, x->state());
 367   __ call_runtime(entry, LIR_OprFact::illegalOpr, result_reg, cc->args(), info);
 368   __ move(result_reg, result);
 369 }
 370 
 371 void LIRGenerator::do_NegateOp(NegateOp* x) {
 372 
 373   LIRItem from(x->x(), this);
 374   from.load_item();
 375   LIR_Opr result = rlock_result(x);
 376   __ negate (from.result(), result);
 377 
 378 }
 379 
 380 // for  _fadd, _fmul, _fsub, _fdiv, _frem
 381 //      _dadd, _dmul, _dsub, _ddiv, _drem
 382 void LIRGenerator::do_ArithmeticOp_FPU(ArithmeticOp* x) {
 383 
 384   if (x->op() == Bytecodes::_frem || x->op() == Bytecodes::_drem) {
 385     // float remainder is implemented as a direct call into the runtime
 386     LIRItem right(x->x(), this);
 387     LIRItem left(x->y(), this);
 388 
 389     BasicTypeList signature(2);
 390     if (x->op() == Bytecodes::_frem) {
 391       signature.append(T_FLOAT);
 392       signature.append(T_FLOAT);
 393     } else {
 394       signature.append(T_DOUBLE);
 395       signature.append(T_DOUBLE);
 396     }
 397     CallingConvention* cc = frame_map()->c_calling_convention(&signature);
 398 
 399     const LIR_Opr result_reg = result_register_for(x->type());
 400     left.load_item_force(cc->at(1));
 401     right.load_item();
 402 
 403     __ move(right.result(), cc->at(0));
 404 
 405     address entry;
 406     if (x->op() == Bytecodes::_frem) {
 407       entry = CAST_FROM_FN_PTR(address, SharedRuntime::frem);
 408     } else {
 409       entry = CAST_FROM_FN_PTR(address, SharedRuntime::drem);
 410     }
 411 
 412     LIR_Opr result = rlock_result(x);
 413     __ call_runtime_leaf(entry, getThreadTemp(), result_reg, cc->args());
 414     __ move(result_reg, result);
 415 
 416     return;
 417   }
 418 
 419   LIRItem left(x->x(),  this);
 420   LIRItem right(x->y(), this);
 421   LIRItem* left_arg  = &left;
 422   LIRItem* right_arg = &right;
 423 
 424   // Always load right hand side.
 425   right.load_item();
 426 
 427   if (!left.is_register())
 428     left.load_item();
 429 
 430   LIR_Opr reg = rlock(x);
 431 
 432   arithmetic_op_fpu(x->op(), reg, left.result(), right.result());
 433 
 434   set_result(x, round_item(reg));
 435 }
 436 
 437 // for  _ladd, _lmul, _lsub, _ldiv, _lrem
 438 void LIRGenerator::do_ArithmeticOp_Long(ArithmeticOp* x) {
 439 
 440   // missing test if instr is commutative and if we should swap
 441   LIRItem left(x->x(), this);
 442   LIRItem right(x->y(), this);
 443 
 444   if (x->op() == Bytecodes::_ldiv || x->op() == Bytecodes::_lrem) {
 445 
 446     left.load_item();
 447     bool need_zero_check = true;
 448     if (right.is_constant()) {
 449       jlong c = right.get_jlong_constant();
 450       // no need to do div-by-zero check if the divisor is a non-zero constant
 451       if (c != 0) need_zero_check = false;
 452       // do not load right if the divisor is a power-of-2 constant
 453       if (c > 0 && is_power_of_2(c)) {
 454         right.dont_load_item();
 455       } else {
 456         right.load_item();
 457       }
 458     } else {
 459       right.load_item();
 460     }
 461     if (need_zero_check) {
 462       CodeEmitInfo* info = state_for(x);
 463       __ cmp(lir_cond_equal, right.result(), LIR_OprFact::longConst(0));
 464       __ branch(lir_cond_equal, new DivByZeroStub(info));
 465     }
 466 
 467     rlock_result(x);
 468     switch (x->op()) {
 469     case Bytecodes::_lrem:
 470       __ rem (left.result(), right.result(), x->operand());
 471       break;
 472     case Bytecodes::_ldiv:
 473       __ div (left.result(), right.result(), x->operand());
 474       break;
 475     default:
 476       ShouldNotReachHere();
 477       break;
 478     }
 479 
 480 
 481   } else {
 482     assert (x->op() == Bytecodes::_lmul || x->op() == Bytecodes::_ladd || x->op() == Bytecodes::_lsub,
 483             "expect lmul, ladd or lsub");
 484     // add, sub, mul
 485     left.load_item();
 486     if (! right.is_register()) {
 487       if (x->op() == Bytecodes::_lmul
 488           || ! right.is_constant()
 489           || ! Assembler::operand_valid_for_add_sub_immediate(right.get_jlong_constant())) {
 490         right.load_item();
 491       } else { // add, sub
 492         assert (x->op() == Bytecodes::_ladd || x->op() == Bytecodes::_lsub, "expect ladd or lsub");
 493         // don't load constants to save register
 494         right.load_nonconstant();
 495       }
 496     }
 497     rlock_result(x);
 498     arithmetic_op_long(x->op(), x->operand(), left.result(), right.result(), NULL);
 499   }
 500 }
 501 
 502 // for: _iadd, _imul, _isub, _idiv, _irem
 503 void LIRGenerator::do_ArithmeticOp_Int(ArithmeticOp* x) {
 504 
 505   // Test if instr is commutative and if we should swap
 506   LIRItem left(x->x(),  this);
 507   LIRItem right(x->y(), this);
 508   LIRItem* left_arg = &left;
 509   LIRItem* right_arg = &right;
 510   if (x->is_commutative() && left.is_stack() && right.is_register()) {
 511     // swap them if left is real stack (or cached) and right is real register(not cached)
 512     left_arg = &right;
 513     right_arg = &left;
 514   }
 515 
 516   left_arg->load_item();
 517 
 518   // do not need to load right, as we can handle stack and constants
 519   if (x->op() == Bytecodes::_idiv || x->op() == Bytecodes::_irem) {
 520 
 521     rlock_result(x);
 522     bool need_zero_check = true;
 523     if (right.is_constant()) {
 524       jint c = right.get_jint_constant();
 525       // no need to do div-by-zero check if the divisor is a non-zero constant
 526       if (c != 0) need_zero_check = false;
 527       // do not load right if the divisor is a power-of-2 constant
 528       if (c > 0 && is_power_of_2(c)) {
 529         right_arg->dont_load_item();
 530       } else {
 531         right_arg->load_item();
 532       }
 533     } else {
 534       right_arg->load_item();
 535     }
 536     if (need_zero_check) {
 537       CodeEmitInfo* info = state_for(x);
 538       __ cmp(lir_cond_equal, right_arg->result(), LIR_OprFact::longConst(0));
 539       __ branch(lir_cond_equal, new DivByZeroStub(info));
 540     }
 541 
 542     LIR_Opr ill = LIR_OprFact::illegalOpr;
 543     if (x->op() == Bytecodes::_irem) {
 544       __ irem(left_arg->result(), right_arg->result(), x->operand(), ill, NULL);
 545     } else if (x->op() == Bytecodes::_idiv) {
 546       __ idiv(left_arg->result(), right_arg->result(), x->operand(), ill, NULL);
 547     }
 548 
 549   } else if (x->op() == Bytecodes::_iadd || x->op() == Bytecodes::_isub) {
 550     if (right.is_constant()
 551         && Assembler::operand_valid_for_add_sub_immediate(right.get_jint_constant())) {
 552       right.load_nonconstant();
 553     } else {
 554       right.load_item();
 555     }
 556     rlock_result(x);
 557     arithmetic_op_int(x->op(), x->operand(), left_arg->result(), right_arg->result(), LIR_OprFact::illegalOpr);
 558   } else {
 559     assert (x->op() == Bytecodes::_imul, "expect imul");
 560     if (right.is_constant()) {
 561       jint c = right.get_jint_constant();
 562       if (c > 0 && c < max_jint && (is_power_of_2(c) || is_power_of_2(c - 1) || is_power_of_2(c + 1))) {
 563         right_arg->dont_load_item();
 564       } else {
 565         // Cannot use constant op.
 566         right_arg->load_item();
 567       }
 568     } else {
 569       right.load_item();
 570     }
 571     rlock_result(x);
 572     arithmetic_op_int(x->op(), x->operand(), left_arg->result(), right_arg->result(), new_register(T_INT));
 573   }
 574 }
 575 
 576 void LIRGenerator::do_ArithmeticOp(ArithmeticOp* x) {
 577   // when an operand with use count 1 is the left operand, then it is
 578   // likely that no move for 2-operand-LIR-form is necessary
 579   if (x->is_commutative() && x->y()->as_Constant() == NULL && x->x()->use_count() > x->y()->use_count()) {
 580     x->swap_operands();
 581   }
 582 
 583   ValueTag tag = x->type()->tag();
 584   assert(x->x()->type()->tag() == tag && x->y()->type()->tag() == tag, "wrong parameters");
 585   switch (tag) {
 586     case floatTag:
 587     case doubleTag:  do_ArithmeticOp_FPU(x);  return;
 588     case longTag:    do_ArithmeticOp_Long(x); return;
 589     case intTag:     do_ArithmeticOp_Int(x);  return;
 590     default:         ShouldNotReachHere();    return;
 591   }
 592 }
 593 
 594 // _ishl, _lshl, _ishr, _lshr, _iushr, _lushr
 595 void LIRGenerator::do_ShiftOp(ShiftOp* x) {
 596 
 597   LIRItem left(x->x(),  this);
 598   LIRItem right(x->y(), this);
 599 
 600   left.load_item();
 601 
 602   rlock_result(x);
 603   if (right.is_constant()) {
 604     right.dont_load_item();
 605 
 606     switch (x->op()) {
 607     case Bytecodes::_ishl: {
 608       int c = right.get_jint_constant() & 0x1f;
 609       __ shift_left(left.result(), c, x->operand());
 610       break;
 611     }
 612     case Bytecodes::_ishr: {
 613       int c = right.get_jint_constant() & 0x1f;
 614       __ shift_right(left.result(), c, x->operand());
 615       break;
 616     }
 617     case Bytecodes::_iushr: {
 618       int c = right.get_jint_constant() & 0x1f;
 619       __ unsigned_shift_right(left.result(), c, x->operand());
 620       break;
 621     }
 622     case Bytecodes::_lshl: {
 623       int c = right.get_jint_constant() & 0x3f;
 624       __ shift_left(left.result(), c, x->operand());
 625       break;
 626     }
 627     case Bytecodes::_lshr: {
 628       int c = right.get_jint_constant() & 0x3f;
 629       __ shift_right(left.result(), c, x->operand());
 630       break;
 631     }
 632     case Bytecodes::_lushr: {
 633       int c = right.get_jint_constant() & 0x3f;
 634       __ unsigned_shift_right(left.result(), c, x->operand());
 635       break;
 636     }
 637     default:
 638       ShouldNotReachHere();
 639     }
 640   } else {
 641     right.load_item();
 642     LIR_Opr tmp = new_register(T_INT);
 643     switch (x->op()) {
 644     case Bytecodes::_ishl: {
 645       __ logical_and(right.result(), LIR_OprFact::intConst(0x1f), tmp);
 646       __ shift_left(left.result(), tmp, x->operand(), tmp);
 647       break;
 648     }
 649     case Bytecodes::_ishr: {
 650       __ logical_and(right.result(), LIR_OprFact::intConst(0x1f), tmp);
 651       __ shift_right(left.result(), tmp, x->operand(), tmp);
 652       break;
 653     }
 654     case Bytecodes::_iushr: {
 655       __ logical_and(right.result(), LIR_OprFact::intConst(0x1f), tmp);
 656       __ unsigned_shift_right(left.result(), tmp, x->operand(), tmp);
 657       break;
 658     }
 659     case Bytecodes::_lshl: {
 660       __ logical_and(right.result(), LIR_OprFact::intConst(0x3f), tmp);
 661       __ shift_left(left.result(), tmp, x->operand(), tmp);
 662       break;
 663     }
 664     case Bytecodes::_lshr: {
 665       __ logical_and(right.result(), LIR_OprFact::intConst(0x3f), tmp);
 666       __ shift_right(left.result(), tmp, x->operand(), tmp);
 667       break;
 668     }
 669     case Bytecodes::_lushr: {
 670       __ logical_and(right.result(), LIR_OprFact::intConst(0x3f), tmp);
 671       __ unsigned_shift_right(left.result(), tmp, x->operand(), tmp);
 672       break;
 673     }
 674     default:
 675       ShouldNotReachHere();
 676     }
 677   }
 678 }
 679 
 680 // _iand, _land, _ior, _lor, _ixor, _lxor
 681 void LIRGenerator::do_LogicOp(LogicOp* x) {
 682 
 683   LIRItem left(x->x(),  this);
 684   LIRItem right(x->y(), this);
 685 
 686   left.load_item();
 687 
 688   rlock_result(x);
 689   if (right.is_constant()
 690       && ((right.type()->tag() == intTag
 691            && Assembler::operand_valid_for_logical_immediate(true, right.get_jint_constant()))
 692           || (right.type()->tag() == longTag
 693               && Assembler::operand_valid_for_logical_immediate(false, right.get_jlong_constant()))))  {
 694     right.dont_load_item();
 695   } else {
 696     right.load_item();
 697   }
 698   switch (x->op()) {
 699   case Bytecodes::_iand:
 700   case Bytecodes::_land:
 701     __ logical_and(left.result(), right.result(), x->operand()); break;
 702   case Bytecodes::_ior:
 703   case Bytecodes::_lor:
 704     __ logical_or (left.result(), right.result(), x->operand()); break;
 705   case Bytecodes::_ixor:
 706   case Bytecodes::_lxor:
 707     __ logical_xor(left.result(), right.result(), x->operand()); break;
 708   default: Unimplemented();
 709   }
 710 }
 711 
 712 // _lcmp, _fcmpl, _fcmpg, _dcmpl, _dcmpg
 713 void LIRGenerator::do_CompareOp(CompareOp* x) {
 714   LIRItem left(x->x(), this);
 715   LIRItem right(x->y(), this);
 716   ValueTag tag = x->x()->type()->tag();
 717   if (tag == longTag) {
 718     left.set_destroys_register();
 719   }
 720   left.load_item();
 721   right.load_item();
 722   LIR_Opr reg = rlock_result(x);
 723 
 724   if (x->x()->type()->is_float_kind()) {
 725     Bytecodes::Code code = x->op();
 726     __ fcmp2int(left.result(), right.result(), reg, (code == Bytecodes::_fcmpl || code == Bytecodes::_dcmpl));
 727   } else if (x->x()->type()->tag() == longTag) {
 728     __ lcmp2int(left.result(), right.result(), reg);
 729   } else {
 730     Unimplemented();
 731   }
 732 }
 733 
 734 LIR_Opr LIRGenerator::atomic_cmpxchg(BasicType type, LIR_Opr addr, LIRItem& cmp_value, LIRItem& new_value) {
 735   LIR_Opr ill = LIR_OprFact::illegalOpr;  // for convenience
 736   new_value.load_item();
 737   cmp_value.load_item();
 738   LIR_Opr result = new_register(T_INT);
 739   if (is_reference_type(type)) {
 740     __ cas_obj(addr, cmp_value.result(), new_value.result(), new_register(T_INT), new_register(T_INT), result);
 741   } else if (type == T_INT) {
 742     __ cas_int(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill);
 743   } else if (type == T_LONG) {
 744     __ cas_long(addr->as_address_ptr()->base(), cmp_value.result(), new_value.result(), ill, ill);
 745   } else {
 746     ShouldNotReachHere();
 747     Unimplemented();
 748   }
 749   __ logical_xor(FrameMap::r8_opr, LIR_OprFact::intConst(1), result);
 750   return result;
 751 }
 752 
 753 LIR_Opr LIRGenerator::atomic_xchg(BasicType type, LIR_Opr addr, LIRItem& value) {
 754   bool is_oop = is_reference_type(type);
 755   LIR_Opr result = new_register(type);
 756   value.load_item();
 757   assert(type == T_INT || is_oop LP64_ONLY( || type == T_LONG ), "unexpected type");
 758   LIR_Opr tmp = new_register(T_INT);
 759   __ xchg(addr, value.result(), result, tmp);
 760   return result;
 761 }
 762 
 763 LIR_Opr LIRGenerator::atomic_add(BasicType type, LIR_Opr addr, LIRItem& value) {
 764   LIR_Opr result = new_register(type);
 765   value.load_item();
 766   assert(type == T_INT LP64_ONLY( || type == T_LONG ), "unexpected type");
 767   LIR_Opr tmp = new_register(T_INT);
 768   __ xadd(addr, value.result(), result, tmp);
 769   return result;
 770 }
 771 
 772 void LIRGenerator::do_MathIntrinsic(Intrinsic* x) {
 773   assert(x->number_of_arguments() == 1 || (x->number_of_arguments() == 2 && x->id() == vmIntrinsics::_dpow), "wrong type");
 774   if (x->id() == vmIntrinsics::_dexp || x->id() == vmIntrinsics::_dlog ||
 775       x->id() == vmIntrinsics::_dpow || x->id() == vmIntrinsics::_dcos ||
 776       x->id() == vmIntrinsics::_dsin || x->id() == vmIntrinsics::_dtan ||
 777       x->id() == vmIntrinsics::_dlog10) {
 778     do_LibmIntrinsic(x);
 779     return;
 780   }
 781   switch (x->id()) {
 782     case vmIntrinsics::_dabs:
 783     case vmIntrinsics::_dsqrt: {
 784       assert(x->number_of_arguments() == 1, "wrong type");
 785       LIRItem value(x->argument_at(0), this);
 786       value.load_item();
 787       LIR_Opr dst = rlock_result(x);
 788 
 789       switch (x->id()) {
 790         case vmIntrinsics::_dsqrt: {
 791           __ sqrt(value.result(), dst, LIR_OprFact::illegalOpr);
 792           break;
 793         }
 794         case vmIntrinsics::_dabs: {
 795           __ abs(value.result(), dst, LIR_OprFact::illegalOpr);
 796           break;
 797         }
 798         default:
 799           ShouldNotReachHere();
 800       }
 801       break;
 802     }
 803     default:
 804       ShouldNotReachHere();
 805   }
 806 }
 807 
 808 void LIRGenerator::do_LibmIntrinsic(Intrinsic* x) {
 809   LIRItem value(x->argument_at(0), this);
 810   value.set_destroys_register();
 811 
 812   LIR_Opr calc_result = rlock_result(x);
 813   LIR_Opr result_reg = result_register_for(x->type());
 814 
 815   CallingConvention* cc = NULL;
 816 
 817   if (x->id() == vmIntrinsics::_dpow) {
 818     LIRItem value1(x->argument_at(1), this);
 819 
 820     value1.set_destroys_register();
 821 
 822     BasicTypeList signature(2);
 823     signature.append(T_DOUBLE);
 824     signature.append(T_DOUBLE);
 825     cc = frame_map()->c_calling_convention(&signature);
 826     value.load_item_force(cc->at(0));
 827     value1.load_item_force(cc->at(1));
 828   } else {
 829     BasicTypeList signature(1);
 830     signature.append(T_DOUBLE);
 831     cc = frame_map()->c_calling_convention(&signature);
 832     value.load_item_force(cc->at(0));
 833   }
 834 
 835   switch (x->id()) {
 836     case vmIntrinsics::_dexp:
 837       if (StubRoutines::dexp() != NULL) {
 838         __ call_runtime_leaf(StubRoutines::dexp(), getThreadTemp(), result_reg, cc->args());
 839       } else {
 840         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dexp), getThreadTemp(), result_reg, cc->args());
 841       }
 842       break;
 843     case vmIntrinsics::_dlog:
 844       if (StubRoutines::dlog() != NULL) {
 845         __ call_runtime_leaf(StubRoutines::dlog(), getThreadTemp(), result_reg, cc->args());
 846       } else {
 847         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog), getThreadTemp(), result_reg, cc->args());
 848       }
 849       break;
 850     case vmIntrinsics::_dlog10:
 851       if (StubRoutines::dlog10() != NULL) {
 852         __ call_runtime_leaf(StubRoutines::dlog10(), getThreadTemp(), result_reg, cc->args());
 853       } else {
 854         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), getThreadTemp(), result_reg, cc->args());
 855       }
 856       break;
 857     case vmIntrinsics::_dpow:
 858       if (StubRoutines::dpow() != NULL) {
 859         __ call_runtime_leaf(StubRoutines::dpow(), getThreadTemp(), result_reg, cc->args());
 860       } else {
 861         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dpow), getThreadTemp(), result_reg, cc->args());
 862       }
 863       break;
 864     case vmIntrinsics::_dsin:
 865       if (StubRoutines::dsin() != NULL) {
 866         __ call_runtime_leaf(StubRoutines::dsin(), getThreadTemp(), result_reg, cc->args());
 867       } else {
 868         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dsin), getThreadTemp(), result_reg, cc->args());
 869       }
 870       break;
 871     case vmIntrinsics::_dcos:
 872       if (StubRoutines::dcos() != NULL) {
 873         __ call_runtime_leaf(StubRoutines::dcos(), getThreadTemp(), result_reg, cc->args());
 874       } else {
 875         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dcos), getThreadTemp(), result_reg, cc->args());
 876       }
 877       break;
 878     case vmIntrinsics::_dtan:
 879       if (StubRoutines::dtan() != NULL) {
 880         __ call_runtime_leaf(StubRoutines::dtan(), getThreadTemp(), result_reg, cc->args());
 881       } else {
 882         __ call_runtime_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtan), getThreadTemp(), result_reg, cc->args());
 883       }
 884       break;
 885     default:  ShouldNotReachHere();
 886   }
 887   __ move(result_reg, calc_result);
 888 }
 889 
 890 
 891 void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
 892   assert(x->number_of_arguments() == 5, "wrong type");
 893 
 894   // Make all state_for calls early since they can emit code
 895   CodeEmitInfo* info = state_for(x, x->state());
 896 
 897   LIRItem src(x->argument_at(0), this);
 898   LIRItem src_pos(x->argument_at(1), this);
 899   LIRItem dst(x->argument_at(2), this);
 900   LIRItem dst_pos(x->argument_at(3), this);
 901   LIRItem length(x->argument_at(4), this);
 902 
 903   // operands for arraycopy must use fixed registers, otherwise
 904   // LinearScan will fail allocation (because arraycopy always needs a
 905   // call)
 906 
 907   // The java calling convention will give us enough registers
 908   // so that on the stub side the args will be perfect already.
 909   // On the other slow/special case side we call C and the arg
 910   // positions are not similar enough to pick one as the best.
 911   // Also because the java calling convention is a "shifted" version
 912   // of the C convention we can process the java args trivially into C
 913   // args without worry of overwriting during the xfer
 914 
 915   src.load_item_force     (FrameMap::as_oop_opr(j_rarg0));
 916   src_pos.load_item_force (FrameMap::as_opr(j_rarg1));
 917   dst.load_item_force     (FrameMap::as_oop_opr(j_rarg2));
 918   dst_pos.load_item_force (FrameMap::as_opr(j_rarg3));
 919   length.load_item_force  (FrameMap::as_opr(j_rarg4));
 920 
 921   LIR_Opr tmp =           FrameMap::as_opr(j_rarg5);
 922 
 923   set_no_result(x);
 924 
 925   int flags;
 926   ciArrayKlass* expected_type;
 927   arraycopy_helper(x, &flags, &expected_type);
 928 
 929   __ arraycopy(src.result(), src_pos.result(), dst.result(), dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint
 930 }
 931 
 932 void LIRGenerator::do_update_CRC32(Intrinsic* x) {
 933   assert(UseCRC32Intrinsics, "why are we here?");
 934   // Make all state_for calls early since they can emit code
 935   LIR_Opr result = rlock_result(x);
 936   int flags = 0;
 937   switch (x->id()) {
 938     case vmIntrinsics::_updateCRC32: {
 939       LIRItem crc(x->argument_at(0), this);
 940       LIRItem val(x->argument_at(1), this);
 941       // val is destroyed by update_crc32
 942       val.set_destroys_register();
 943       crc.load_item();
 944       val.load_item();
 945       __ update_crc32(crc.result(), val.result(), result);
 946       break;
 947     }
 948     case vmIntrinsics::_updateBytesCRC32:
 949     case vmIntrinsics::_updateByteBufferCRC32: {
 950       bool is_updateBytes = (x->id() == vmIntrinsics::_updateBytesCRC32);
 951 
 952       LIRItem crc(x->argument_at(0), this);
 953       LIRItem buf(x->argument_at(1), this);
 954       LIRItem off(x->argument_at(2), this);
 955       LIRItem len(x->argument_at(3), this);
 956       buf.load_item();
 957       off.load_nonconstant();
 958 
 959       LIR_Opr index = off.result();
 960       int offset = is_updateBytes ? arrayOopDesc::base_offset_in_bytes(T_BYTE) : 0;
 961       if(off.result()->is_constant()) {
 962         index = LIR_OprFact::illegalOpr;
 963        offset += off.result()->as_jint();
 964       }
 965       LIR_Opr base_op = buf.result();
 966 
 967       if (index->is_valid()) {
 968         LIR_Opr tmp = new_register(T_LONG);
 969         __ convert(Bytecodes::_i2l, index, tmp);
 970         index = tmp;
 971       }
 972 
 973       if (offset) {
 974         LIR_Opr tmp = new_pointer_register();
 975         __ add(base_op, LIR_OprFact::intConst(offset), tmp);
 976         base_op = tmp;
 977         offset = 0;
 978       }
 979 
 980       LIR_Address* a = new LIR_Address(base_op,
 981                                        index,
 982                                        offset,
 983                                        T_BYTE);
 984       BasicTypeList signature(3);
 985       signature.append(T_INT);
 986       signature.append(T_ADDRESS);
 987       signature.append(T_INT);
 988       CallingConvention* cc = frame_map()->c_calling_convention(&signature);
 989       const LIR_Opr result_reg = result_register_for(x->type());
 990 
 991       LIR_Opr addr = new_pointer_register();
 992       __ leal(LIR_OprFact::address(a), addr);
 993 
 994       crc.load_item_force(cc->at(0));
 995       __ move(addr, cc->at(1));
 996       len.load_item_force(cc->at(2));
 997 
 998       __ call_runtime_leaf(StubRoutines::updateBytesCRC32(), getThreadTemp(), result_reg, cc->args());
 999       __ move(result_reg, result);
1000 
1001       break;
1002     }
1003     default: {
1004       ShouldNotReachHere();
1005     }
1006   }
1007 }
1008 
1009 void LIRGenerator::do_update_CRC32C(Intrinsic* x) {
1010   assert(UseCRC32CIntrinsics, "why are we here?");
1011   // Make all state_for calls early since they can emit code
1012   LIR_Opr result = rlock_result(x);
1013   int flags = 0;
1014   switch (x->id()) {
1015     case vmIntrinsics::_updateBytesCRC32C:
1016     case vmIntrinsics::_updateDirectByteBufferCRC32C: {
1017       bool is_updateBytes = (x->id() == vmIntrinsics::_updateBytesCRC32C);
1018       int offset = is_updateBytes ? arrayOopDesc::base_offset_in_bytes(T_BYTE) : 0;
1019 
1020       LIRItem crc(x->argument_at(0), this);
1021       LIRItem buf(x->argument_at(1), this);
1022       LIRItem off(x->argument_at(2), this);
1023       LIRItem end(x->argument_at(3), this);
1024 
1025       buf.load_item();
1026       off.load_nonconstant();
1027       end.load_nonconstant();
1028 
1029       // len = end - off
1030       LIR_Opr len  = end.result();
1031       LIR_Opr tmpA = new_register(T_INT);
1032       LIR_Opr tmpB = new_register(T_INT);
1033       __ move(end.result(), tmpA);
1034       __ move(off.result(), tmpB);
1035       __ sub(tmpA, tmpB, tmpA);
1036       len = tmpA;
1037 
1038       LIR_Opr index = off.result();
1039       if(off.result()->is_constant()) {
1040         index = LIR_OprFact::illegalOpr;
1041         offset += off.result()->as_jint();
1042       }
1043       LIR_Opr base_op = buf.result();
1044 
1045       if (index->is_valid()) {
1046         LIR_Opr tmp = new_register(T_LONG);
1047         __ convert(Bytecodes::_i2l, index, tmp);
1048         index = tmp;
1049       }
1050 
1051       if (offset) {
1052         LIR_Opr tmp = new_pointer_register();
1053         __ add(base_op, LIR_OprFact::intConst(offset), tmp);
1054         base_op = tmp;
1055         offset = 0;
1056       }
1057 
1058       LIR_Address* a = new LIR_Address(base_op,
1059                                        index,
1060                                        offset,
1061                                        T_BYTE);
1062       BasicTypeList signature(3);
1063       signature.append(T_INT);
1064       signature.append(T_ADDRESS);
1065       signature.append(T_INT);
1066       CallingConvention* cc = frame_map()->c_calling_convention(&signature);
1067       const LIR_Opr result_reg = result_register_for(x->type());
1068 
1069       LIR_Opr addr = new_pointer_register();
1070       __ leal(LIR_OprFact::address(a), addr);
1071 
1072       crc.load_item_force(cc->at(0));
1073       __ move(addr, cc->at(1));
1074       __ move(len, cc->at(2));
1075 
1076       __ call_runtime_leaf(StubRoutines::updateBytesCRC32C(), getThreadTemp(), result_reg, cc->args());
1077       __ move(result_reg, result);
1078 
1079       break;
1080     }
1081     default: {
1082       ShouldNotReachHere();
1083     }
1084   }
1085 }
1086 
1087 void LIRGenerator::do_FmaIntrinsic(Intrinsic* x) {
1088   assert(x->number_of_arguments() == 3, "wrong type");
1089   assert(UseFMA, "Needs FMA instructions support.");
1090   LIRItem value(x->argument_at(0), this);
1091   LIRItem value1(x->argument_at(1), this);
1092   LIRItem value2(x->argument_at(2), this);
1093 
1094   value.load_item();
1095   value1.load_item();
1096   value2.load_item();
1097 
1098   LIR_Opr calc_input = value.result();
1099   LIR_Opr calc_input1 = value1.result();
1100   LIR_Opr calc_input2 = value2.result();
1101   LIR_Opr calc_result = rlock_result(x);
1102 
1103   switch (x->id()) {
1104   case vmIntrinsics::_fmaD:   __ fmad(calc_input, calc_input1, calc_input2, calc_result); break;
1105   case vmIntrinsics::_fmaF:   __ fmaf(calc_input, calc_input1, calc_input2, calc_result); break;
1106   default:                    ShouldNotReachHere();
1107   }
1108 }
1109 
1110 void LIRGenerator::do_vectorizedMismatch(Intrinsic* x) {
1111   fatal("vectorizedMismatch intrinsic is not implemented on this platform");
1112 }
1113 
1114 // _i2l, _i2f, _i2d, _l2i, _l2f, _l2d, _f2i, _f2l, _f2d, _d2i, _d2l, _d2f
1115 // _i2b, _i2c, _i2s
1116 void LIRGenerator::do_Convert(Convert* x) {
1117   LIRItem value(x->value(), this);
1118   value.load_item();
1119   LIR_Opr input = value.result();
1120   LIR_Opr result = rlock(x);
1121 
1122   // arguments of lir_convert
1123   LIR_Opr conv_input = input;
1124   LIR_Opr conv_result = result;
1125 
1126   __ convert(x->op(), conv_input, conv_result);
1127 
1128   assert(result->is_virtual(), "result must be virtual register");
1129   set_result(x, result);
1130 }
1131 
1132 void LIRGenerator::do_NewInstance(NewInstance* x) {
1133 #ifndef PRODUCT
1134   if (PrintNotLoaded && !x->klass()->is_loaded()) {
1135     tty->print_cr("   ###class not loaded at new bci %d", x->printable_bci());
1136   }
1137 #endif
1138   CodeEmitInfo* info = state_for(x, x->state());
1139   LIR_Opr reg = result_register_for(x->type());
1140   new_instance(reg, x->klass(), x->is_unresolved(),
1141                        FrameMap::r10_oop_opr,
1142                        FrameMap::r11_oop_opr,
1143                        FrameMap::r4_oop_opr,
1144                        LIR_OprFact::illegalOpr,
1145                        FrameMap::r3_metadata_opr, info);
1146   LIR_Opr result = rlock_result(x);
1147   __ move(reg, result);
1148 }
1149 
1150 void LIRGenerator::do_NewTypeArray(NewTypeArray* x) {
1151   CodeEmitInfo* info = state_for(x, x->state());
1152 
1153   LIRItem length(x->length(), this);
1154   length.load_item_force(FrameMap::r19_opr);
1155 
1156   LIR_Opr reg = result_register_for(x->type());
1157   LIR_Opr tmp1 = FrameMap::r10_oop_opr;
1158   LIR_Opr tmp2 = FrameMap::r11_oop_opr;
1159   LIR_Opr tmp3 = FrameMap::r5_oop_opr;
1160   LIR_Opr tmp4 = reg;
1161   LIR_Opr klass_reg = FrameMap::r3_metadata_opr;
1162   LIR_Opr len = length.result();
1163   BasicType elem_type = x->elt_type();
1164 
1165   __ metadata2reg(ciTypeArrayKlass::make(elem_type)->constant_encoding(), klass_reg);
1166 
1167   CodeStub* slow_path = new NewTypeArrayStub(klass_reg, len, reg, info);
1168   __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path);
1169 
1170   LIR_Opr result = rlock_result(x);
1171   __ move(reg, result);
1172 }
1173 
1174 void LIRGenerator::do_NewObjectArray(NewObjectArray* x) {
1175   LIRItem length(x->length(), this);
1176   // in case of patching (i.e., object class is not yet loaded), we need to reexecute the instruction
1177   // and therefore provide the state before the parameters have been consumed
1178   CodeEmitInfo* patching_info = NULL;
1179   if (!x->klass()->is_loaded() || PatchALot) {
1180     patching_info =  state_for(x, x->state_before());
1181   }
1182 
1183   CodeEmitInfo* info = state_for(x, x->state());
1184 
1185   LIR_Opr reg = result_register_for(x->type());
1186   LIR_Opr tmp1 = FrameMap::r10_oop_opr;
1187   LIR_Opr tmp2 = FrameMap::r11_oop_opr;
1188   LIR_Opr tmp3 = FrameMap::r5_oop_opr;
1189   LIR_Opr tmp4 = reg;
1190   LIR_Opr klass_reg = FrameMap::r3_metadata_opr;
1191 
1192   length.load_item_force(FrameMap::r19_opr);
1193   LIR_Opr len = length.result();
1194 
1195   CodeStub* slow_path = new NewObjectArrayStub(klass_reg, len, reg, info);
1196   ciKlass* obj = (ciKlass*) ciObjArrayKlass::make(x->klass());
1197   if (obj == ciEnv::unloaded_ciobjarrayklass()) {
1198     BAILOUT("encountered unloaded_ciobjarrayklass due to out of memory error");
1199   }
1200   klass2reg_with_patching(klass_reg, obj, patching_info);
1201   __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, T_OBJECT, klass_reg, slow_path);
1202 
1203   LIR_Opr result = rlock_result(x);
1204   __ move(reg, result);
1205 }
1206 
1207 
1208 void LIRGenerator::do_NewMultiArray(NewMultiArray* x) {
1209   Values* dims = x->dims();
1210   int i = dims->length();
1211   LIRItemList* items = new LIRItemList(i, i, NULL);
1212   while (i-- > 0) {
1213     LIRItem* size = new LIRItem(dims->at(i), this);
1214     items->at_put(i, size);
1215   }
1216 
1217   // Evaluate state_for early since it may emit code.
1218   CodeEmitInfo* patching_info = NULL;
1219   if (!x->klass()->is_loaded() || PatchALot) {
1220     patching_info = state_for(x, x->state_before());
1221 
1222     // Cannot re-use same xhandlers for multiple CodeEmitInfos, so
1223     // clone all handlers (NOTE: Usually this is handled transparently
1224     // by the CodeEmitInfo cloning logic in CodeStub constructors but
1225     // is done explicitly here because a stub isn't being used).
1226     x->set_exception_handlers(new XHandlers(x->exception_handlers()));
1227   }
1228   CodeEmitInfo* info = state_for(x, x->state());
1229 
1230   i = dims->length();
1231   while (i-- > 0) {
1232     LIRItem* size = items->at(i);
1233     size->load_item();
1234 
1235     store_stack_parameter(size->result(), in_ByteSize(i*4));
1236   }
1237 
1238   LIR_Opr klass_reg = FrameMap::r0_metadata_opr;
1239   klass2reg_with_patching(klass_reg, x->klass(), patching_info);
1240 
1241   LIR_Opr rank = FrameMap::r19_opr;
1242   __ move(LIR_OprFact::intConst(x->rank()), rank);
1243   LIR_Opr varargs = FrameMap::r2_opr;
1244   __ move(FrameMap::sp_opr, varargs);
1245   LIR_OprList* args = new LIR_OprList(3);
1246   args->append(klass_reg);
1247   args->append(rank);
1248   args->append(varargs);
1249   LIR_Opr reg = result_register_for(x->type());
1250   __ call_runtime(Runtime1::entry_for(Runtime1::new_multi_array_id),
1251                   LIR_OprFact::illegalOpr,
1252                   reg, args, info);
1253 
1254   LIR_Opr result = rlock_result(x);
1255   __ move(reg, result);
1256 }
1257 
1258 void LIRGenerator::do_BlockBegin(BlockBegin* x) {
1259   // nothing to do for now
1260 }
1261 
1262 void LIRGenerator::do_CheckCast(CheckCast* x) {
1263   LIRItem obj(x->obj(), this);
1264 
1265   CodeEmitInfo* patching_info = NULL;
1266   if (!x->klass()->is_loaded() || (PatchALot && !x->is_incompatible_class_change_check() && !x->is_invokespecial_receiver_check())) {
1267     // must do this before locking the destination register as an oop register,
1268     // and before the obj is loaded (the latter is for deoptimization)
1269     patching_info = state_for(x, x->state_before());
1270   }
1271   obj.load_item();
1272 
1273   // info for exceptions
1274   CodeEmitInfo* info_for_exception =
1275       (x->needs_exception_state() ? state_for(x) :
1276                                     state_for(x, x->state_before(), true /*ignore_xhandler*/));
1277 
1278   CodeStub* stub;
1279   if (x->is_incompatible_class_change_check()) {
1280     assert(patching_info == NULL, "can't patch this");
1281     stub = new SimpleExceptionStub(Runtime1::throw_incompatible_class_change_error_id, LIR_OprFact::illegalOpr, info_for_exception);
1282   } else if (x->is_invokespecial_receiver_check()) {
1283     assert(patching_info == NULL, "can't patch this");
1284     stub = new DeoptimizeStub(info_for_exception,
1285                               Deoptimization::Reason_class_check,
1286                               Deoptimization::Action_none);
1287   } else {
1288     stub = new SimpleExceptionStub(Runtime1::throw_class_cast_exception_id, obj.result(), info_for_exception);
1289   }
1290   LIR_Opr reg = rlock_result(x);
1291   LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
1292   if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
1293     tmp3 = new_register(objectType);
1294   }
1295   __ checkcast(reg, obj.result(), x->klass(),
1296                new_register(objectType), new_register(objectType), tmp3,
1297                x->direct_compare(), info_for_exception, patching_info, stub,
1298                x->profiled_method(), x->profiled_bci());
1299 }
1300 
1301 void LIRGenerator::do_InstanceOf(InstanceOf* x) {
1302   LIRItem obj(x->obj(), this);
1303 
1304   // result and test object may not be in same register
1305   LIR_Opr reg = rlock_result(x);
1306   CodeEmitInfo* patching_info = NULL;
1307   if ((!x->klass()->is_loaded() || PatchALot)) {
1308     // must do this before locking the destination register as an oop register
1309     patching_info = state_for(x, x->state_before());
1310   }
1311   obj.load_item();
1312   LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
1313   if (!x->klass()->is_loaded() || UseCompressedClassPointers) {
1314     tmp3 = new_register(objectType);
1315   }
1316   __ instanceof(reg, obj.result(), x->klass(),
1317                 new_register(objectType), new_register(objectType), tmp3,
1318                 x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
1319 }
1320 
1321 void LIRGenerator::do_If(If* x) {
1322   assert(x->number_of_sux() == 2, "inconsistency");
1323   ValueTag tag = x->x()->type()->tag();
1324   bool is_safepoint = x->is_safepoint();
1325 
1326   If::Condition cond = x->cond();
1327 
1328   LIRItem xitem(x->x(), this);
1329   LIRItem yitem(x->y(), this);
1330   LIRItem* xin = &xitem;
1331   LIRItem* yin = &yitem;
1332 
1333   if (tag == longTag) {
1334     // for longs, only conditions "eql", "neq", "lss", "geq" are valid;
1335     // mirror for other conditions
1336     if (cond == If::gtr || cond == If::leq) {
1337       cond = Instruction::mirror(cond);
1338       xin = &yitem;
1339       yin = &xitem;
1340     }
1341     xin->set_destroys_register();
1342   }
1343   xin->load_item();
1344 
1345   if (tag == longTag) {
1346     if (yin->is_constant()
1347         && Assembler::operand_valid_for_add_sub_immediate(yin->get_jlong_constant())) {
1348       yin->dont_load_item();
1349     } else {
1350       yin->load_item();
1351     }
1352   } else if (tag == intTag) {
1353     if (yin->is_constant()
1354         && Assembler::operand_valid_for_add_sub_immediate(yin->get_jint_constant()))  {
1355       yin->dont_load_item();
1356     } else {
1357       yin->load_item();
1358     }
1359   } else {
1360     yin->load_item();
1361   }
1362 
1363   set_no_result(x);
1364 
1365   LIR_Opr left = xin->result();
1366   LIR_Opr right = yin->result();
1367 
1368   // add safepoint before generating condition code so it can be recomputed
1369   if (x->is_safepoint()) {
1370     // increment backedge counter if needed
1371     increment_backedge_counter_conditionally(lir_cond(cond), left, right, state_for(x, x->state_before()),
1372         x->tsux()->bci(), x->fsux()->bci(), x->profiled_bci());
1373     __ safepoint(LIR_OprFact::illegalOpr, state_for(x, x->state_before()));
1374   }
1375 
1376   __ cmp(lir_cond(cond), left, right);
1377   // Generate branch profiling. Profiling code doesn't kill flags.
1378   profile_branch(x, cond);
1379   move_to_phi(x->state());
1380   if (x->x()->type()->is_float_kind()) {
1381     __ branch(lir_cond(cond), x->tsux(), x->usux());
1382   } else {
1383     __ branch(lir_cond(cond), x->tsux());
1384   }
1385   assert(x->default_sux() == x->fsux(), "wrong destination above");
1386   __ jump(x->default_sux());
1387 }
1388 
1389 LIR_Opr LIRGenerator::getThreadPointer() {
1390    return FrameMap::as_pointer_opr(rthread);
1391 }
1392 
1393 void LIRGenerator::trace_block_entry(BlockBegin* block) { Unimplemented(); }
1394 
1395 void LIRGenerator::volatile_field_store(LIR_Opr value, LIR_Address* address,
1396                                         CodeEmitInfo* info) {
1397   __ volatile_store_mem_reg(value, address, info);
1398 }
1399 
1400 void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result,
1401                                        CodeEmitInfo* info) {
1402   // 8179954: We need to make sure that the code generated for
1403   // volatile accesses forms a sequentially-consistent set of
1404   // operations when combined with STLR and LDAR.  Without a leading
1405   // membar it's possible for a simple Dekker test to fail if loads
1406   // use LD;DMB but stores use STLR.  This can happen if C2 compiles
1407   // the stores in one method and C1 compiles the loads in another.
1408   if (!CompilerConfig::is_c1_only_no_jvmci()) {
1409     __ membar();
1410   }
1411   __ volatile_load_mem_reg(address, result, info);
1412 }