1 /*
   2  * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2016, 2019 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInstance.hpp"
  35 #include "gc/shared/collectedHeap.hpp"
  36 #include "memory/universe.hpp"
  37 #include "nativeInst_s390.hpp"
  38 #include "oops/objArrayKlass.hpp"
  39 #include "runtime/frame.inline.hpp"
  40 #include "runtime/safepointMechanism.inline.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 #include "runtime/stubRoutines.hpp"
  43 #include "utilities/macros.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_s390.inline.hpp"
  46 
  47 #define __ _masm->
  48 
  49 #ifndef PRODUCT
  50 #undef __
  51 #define __ (Verbose ? (_masm->block_comment(FILE_AND_LINE),_masm) : _masm)->
  52 #endif
  53 
  54 //------------------------------------------------------------
  55 
  56 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
  57   // Not used on ZARCH_64
  58   ShouldNotCallThis();
  59   return false;
  60 }
  61 
  62 LIR_Opr LIR_Assembler::receiverOpr() {
  63   return FrameMap::Z_R2_oop_opr;
  64 }
  65 
  66 LIR_Opr LIR_Assembler::osrBufferPointer() {
  67   return FrameMap::Z_R2_opr;
  68 }
  69 
  70 int LIR_Assembler::initial_frame_size_in_bytes() const {
  71   return in_bytes(frame_map()->framesize_in_bytes());
  72 }
  73 
  74 // Inline cache check: done before the frame is built.
  75 // The inline cached class is in Z_inline_cache(Z_R9).
  76 // We fetch the class of the receiver and compare it with the cached class.
  77 // If they do not match we jump to the slow case.
  78 int LIR_Assembler::check_icache() {
  79   Register receiver = receiverOpr()->as_register();
  80   int offset = __ offset();
  81   __ inline_cache_check(receiver, Z_inline_cache);
  82   return offset;
  83 }
  84 
  85 void LIR_Assembler::clinit_barrier(ciMethod* method) {
  86   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
  87 
  88   Label L_skip_barrier;
  89   Register klass = Z_R1_scratch;
  90 
  91   metadata2reg(method->holder()->constant_encoding(), klass);
  92   __ clinit_barrier(klass, Z_thread, &L_skip_barrier /*L_fast_path*/);
  93 
  94   __ load_const_optimized(klass, SharedRuntime::get_handle_wrong_method_stub());
  95   __ z_br(klass);
  96 
  97   __ bind(L_skip_barrier);
  98 }
  99 
 100 void LIR_Assembler::osr_entry() {
 101   // On-stack-replacement entry sequence (interpreter frame layout described in frame_s390.hpp):
 102   //
 103   //   1. Create a new compiled activation.
 104   //   2. Initialize local variables in the compiled activation. The expression stack must be empty
 105   //      at the osr_bci; it is not initialized.
 106   //   3. Jump to the continuation address in compiled code to resume execution.
 107 
 108   // OSR entry point
 109   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 110   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 111   ValueStack* entry_state = osr_entry->end()->state();
 112   int number_of_locks = entry_state->locks_size();
 113 
 114   // Create a frame for the compiled activation.
 115   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 116 
 117   // OSR buffer is
 118   //
 119   // locals[nlocals-1..0]
 120   // monitors[number_of_locks-1..0]
 121   //
 122   // Locals is a direct copy of the interpreter frame so in the osr buffer
 123   // the first slot in the local array is the last local from the interpreter
 124   // and the last slot is local[0] (receiver) from the interpreter
 125   //
 126   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 127   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 128   // in the interpreter frame (the method lock if a sync method)
 129 
 130   // Initialize monitors in the compiled activation.
 131   //   I0: pointer to osr buffer
 132   //
 133   // All other registers are dead at this point and the locals will be
 134   // copied into place by code emitted in the IR.
 135 
 136   Register OSR_buf = osrBufferPointer()->as_register();
 137   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 138     int monitor_offset = BytesPerWord * method()->max_locals() +
 139       BytesPerWord * (number_of_locks - 1);
 140     for (int i = 0; i < number_of_locks; i++) {
 141       int slot_offset = monitor_offset - (i * BytesPerWord);
 142       // Verify the interpreter's monitor has a non-null object.
 143       __ asm_assert_mem8_isnot_zero(slot_offset + 1*BytesPerWord, OSR_buf, "locked object is NULL", __LINE__);
 144       // Copy the lock field into the compiled activation.
 145       __ z_lg(Z_R1_scratch, slot_offset, OSR_buf);
 146       __ z_stg(Z_R1_scratch, frame_map()->address_for_monitor_object(i));
 147     }
 148   }
 149 }
 150 
 151 // --------------------------------------------------------------------------------------------
 152 
 153 address LIR_Assembler::emit_call_c(address a) {
 154   __ align_call_far_patchable(__ pc());
 155   address call_addr = __ call_c_opt(a);
 156   if (call_addr == NULL) {
 157     bailout("const section overflow");
 158   }
 159   return call_addr;
 160 }
 161 
 162 int LIR_Assembler::emit_exception_handler() {
 163   // Generate code for exception handler.
 164   address handler_base = __ start_a_stub(exception_handler_size());
 165   if (handler_base == NULL) {
 166     // Not enough space left for the handler.
 167     bailout("exception handler overflow");
 168     return -1;
 169   }
 170 
 171   int offset = code_offset();
 172 
 173   address a = Runtime1::entry_for (Runtime1::handle_exception_from_callee_id);
 174   address call_addr = emit_call_c(a);
 175   CHECK_BAILOUT_(-1);
 176   __ should_not_reach_here();
 177   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 178   __ end_a_stub();
 179 
 180   return offset;
 181 }
 182 
 183 // Emit the code to remove the frame from the stack in the exception
 184 // unwind path.
 185 int LIR_Assembler::emit_unwind_handler() {
 186 #ifndef PRODUCT
 187   if (CommentedAssembly) {
 188     _masm->block_comment("Unwind handler");
 189   }
 190 #endif
 191 
 192   int offset = code_offset();
 193   Register exception_oop_callee_saved = Z_R10; // Z_R10 is callee-saved.
 194   Register Rtmp1                      = Z_R11;
 195   Register Rtmp2                      = Z_R12;
 196 
 197   // Fetch the exception from TLS and clear out exception related thread state.
 198   Address exc_oop_addr = Address(Z_thread, JavaThread::exception_oop_offset());
 199   Address exc_pc_addr  = Address(Z_thread, JavaThread::exception_pc_offset());
 200   __ z_lg(Z_EXC_OOP, exc_oop_addr);
 201   __ clear_mem(exc_oop_addr, sizeof(oop));
 202   __ clear_mem(exc_pc_addr, sizeof(intptr_t));
 203 
 204   __ bind(_unwind_handler_entry);
 205   __ verify_not_null_oop(Z_EXC_OOP);
 206   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 207     __ lgr_if_needed(exception_oop_callee_saved, Z_EXC_OOP); // Preserve the exception.
 208   }
 209 
 210   // Perform needed unlocking.
 211   MonitorExitStub* stub = NULL;
 212   if (method()->is_synchronized()) {
 213     // Runtime1::monitorexit_id expects lock address in Z_R1_scratch.
 214     LIR_Opr lock = FrameMap::as_opr(Z_R1_scratch);
 215     monitor_address(0, lock);
 216     __ z_lg(Z_R1_scratch, Address(Z_R1_scratch, BasicObjectLock::obj_offset_in_bytes()));
 217     stub = new MonitorExitStub(lock);
 218     __ branch_optimized(Assembler::bcondAlways, *stub->entry());
 219     __ bind(*stub->continuation());
 220   }
 221 
 222   if (compilation()->env()->dtrace_method_probes()) {
 223     ShouldNotReachHere(); // Not supported.
 224 #if 0
 225     __ mov(rdi, r15_thread);
 226     __ mov_metadata(rsi, method()->constant_encoding());
 227     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 228 #endif
 229   }
 230 
 231   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 232     __ lgr_if_needed(Z_EXC_OOP, exception_oop_callee_saved);  // Restore the exception.
 233   }
 234 
 235   // Remove the activation and dispatch to the unwind handler.
 236   __ pop_frame();
 237   __ z_lg(Z_EXC_PC, _z_abi16(return_pc), Z_SP);
 238 
 239   // Z_EXC_OOP: exception oop
 240   // Z_EXC_PC: exception pc
 241 
 242   // Dispatch to the unwind logic.
 243   __ load_const_optimized(Z_R5, Runtime1::entry_for (Runtime1::unwind_exception_id));
 244   __ z_br(Z_R5);
 245 
 246   // Emit the slow path assembly.
 247   if (stub != NULL) {
 248     stub->emit_code(this);
 249   }
 250 
 251   return offset;
 252 }
 253 
 254 int LIR_Assembler::emit_deopt_handler() {
 255   // Generate code for exception handler.
 256   address handler_base = __ start_a_stub(deopt_handler_size());
 257   if (handler_base == NULL) {
 258     // Not enough space left for the handler.
 259     bailout("deopt handler overflow");
 260     return -1;
 261   }  int offset = code_offset();
 262   // Size must be constant (see HandlerImpl::emit_deopt_handler).
 263   __ load_const(Z_R1_scratch, SharedRuntime::deopt_blob()->unpack());
 264   __ call(Z_R1_scratch);
 265   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 266   __ end_a_stub();
 267 
 268   return offset;
 269 }
 270 
 271 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 272   if (o == NULL) {
 273     __ clear_reg(reg, true/*64bit*/, false/*set cc*/); // Must not kill cc set by cmove.
 274   } else {
 275     AddressLiteral a = __ allocate_oop_address(o);
 276     bool success = __ load_oop_from_toc(reg, a, reg);
 277     if (!success) {
 278       bailout("const section overflow");
 279     }
 280   }
 281 }
 282 
 283 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 284   // Allocate a new index in table to hold the object once it's been patched.
 285   int oop_index = __ oop_recorder()->allocate_oop_index(NULL);
 286   PatchingStub* patch = new PatchingStub(_masm, patching_id(info), oop_index);
 287 
 288   AddressLiteral addrlit((intptr_t)0, oop_Relocation::spec(oop_index));
 289   assert(addrlit.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
 290   // The NULL will be dynamically patched later so the sequence to
 291   // load the address literal must not be optimized.
 292   __ load_const(reg, addrlit);
 293 
 294   patching_epilog(patch, lir_patch_normal, reg, info);
 295 }
 296 
 297 void LIR_Assembler::metadata2reg(Metadata* md, Register reg) {
 298   bool success = __ set_metadata_constant(md, reg);
 299   if (!success) {
 300     bailout("const section overflow");
 301     return;
 302   }
 303 }
 304 
 305 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo *info) {
 306   // Allocate a new index in table to hold the klass once it's been patched.
 307   int index = __ oop_recorder()->allocate_metadata_index(NULL);
 308   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
 309   AddressLiteral addrlit((intptr_t)0, metadata_Relocation::spec(index));
 310   assert(addrlit.rspec().type() == relocInfo::metadata_type, "must be an metadata reloc");
 311   // The NULL will be dynamically patched later so the sequence to
 312   // load the address literal must not be optimized.
 313   __ load_const(reg, addrlit);
 314 
 315   patching_epilog(patch, lir_patch_normal, reg, info);
 316 }
 317 
 318 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 319   switch (op->code()) {
 320     case lir_idiv:
 321     case lir_irem:
 322       arithmetic_idiv(op->code(),
 323                       op->in_opr1(),
 324                       op->in_opr2(),
 325                       op->in_opr3(),
 326                       op->result_opr(),
 327                       op->info());
 328       break;
 329     case lir_fmad: {
 330       const FloatRegister opr1 = op->in_opr1()->as_double_reg(),
 331                           opr2 = op->in_opr2()->as_double_reg(),
 332                           opr3 = op->in_opr3()->as_double_reg(),
 333                           res  = op->result_opr()->as_double_reg();
 334       __ z_madbr(opr3, opr1, opr2);
 335       if (res != opr3) { __ z_ldr(res, opr3); }
 336     } break;
 337     case lir_fmaf: {
 338       const FloatRegister opr1 = op->in_opr1()->as_float_reg(),
 339                           opr2 = op->in_opr2()->as_float_reg(),
 340                           opr3 = op->in_opr3()->as_float_reg(),
 341                           res  = op->result_opr()->as_float_reg();
 342       __ z_maebr(opr3, opr1, opr2);
 343       if (res != opr3) { __ z_ler(res, opr3); }
 344     } break;
 345     default: ShouldNotReachHere(); break;
 346   }
 347 }
 348 
 349 
 350 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 351 #ifdef ASSERT
 352   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
 353   if (op->block() != NULL)  { _branch_target_blocks.append(op->block()); }
 354   if (op->ublock() != NULL) { _branch_target_blocks.append(op->ublock()); }
 355 #endif
 356 
 357   if (op->cond() == lir_cond_always) {
 358     if (op->info() != NULL) { add_debug_info_for_branch(op->info()); }
 359     __ branch_optimized(Assembler::bcondAlways, *(op->label()));
 360   } else {
 361     Assembler::branch_condition acond = Assembler::bcondZero;
 362     if (op->code() == lir_cond_float_branch) {
 363       assert(op->ublock() != NULL, "must have unordered successor");
 364       __ branch_optimized(Assembler::bcondNotOrdered, *(op->ublock()->label()));
 365     }
 366     switch (op->cond()) {
 367       case lir_cond_equal:        acond = Assembler::bcondEqual;     break;
 368       case lir_cond_notEqual:     acond = Assembler::bcondNotEqual;  break;
 369       case lir_cond_less:         acond = Assembler::bcondLow;       break;
 370       case lir_cond_lessEqual:    acond = Assembler::bcondNotHigh;   break;
 371       case lir_cond_greaterEqual: acond = Assembler::bcondNotLow;    break;
 372       case lir_cond_greater:      acond = Assembler::bcondHigh;      break;
 373       case lir_cond_belowEqual:   acond = Assembler::bcondNotHigh;   break;
 374       case lir_cond_aboveEqual:   acond = Assembler::bcondNotLow;    break;
 375       default:                         ShouldNotReachHere();
 376     }
 377     __ branch_optimized(acond,*(op->label()));
 378   }
 379 }
 380 
 381 
 382 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 383   LIR_Opr src  = op->in_opr();
 384   LIR_Opr dest = op->result_opr();
 385 
 386   switch (op->bytecode()) {
 387     case Bytecodes::_i2l:
 388       __ move_reg_if_needed(dest->as_register_lo(), T_LONG, src->as_register(), T_INT);
 389       break;
 390 
 391     case Bytecodes::_l2i:
 392       __ move_reg_if_needed(dest->as_register(), T_INT, src->as_register_lo(), T_LONG);
 393       break;
 394 
 395     case Bytecodes::_i2b:
 396       __ move_reg_if_needed(dest->as_register(), T_BYTE, src->as_register(), T_INT);
 397       break;
 398 
 399     case Bytecodes::_i2c:
 400       __ move_reg_if_needed(dest->as_register(), T_CHAR, src->as_register(), T_INT);
 401       break;
 402 
 403     case Bytecodes::_i2s:
 404       __ move_reg_if_needed(dest->as_register(), T_SHORT, src->as_register(), T_INT);
 405       break;
 406 
 407     case Bytecodes::_f2d:
 408       assert(dest->is_double_fpu(), "check");
 409       __ move_freg_if_needed(dest->as_double_reg(), T_DOUBLE, src->as_float_reg(), T_FLOAT);
 410       break;
 411 
 412     case Bytecodes::_d2f:
 413       assert(dest->is_single_fpu(), "check");
 414       __ move_freg_if_needed(dest->as_float_reg(), T_FLOAT, src->as_double_reg(), T_DOUBLE);
 415       break;
 416 
 417     case Bytecodes::_i2f:
 418       __ z_cefbr(dest->as_float_reg(), src->as_register());
 419       break;
 420 
 421     case Bytecodes::_i2d:
 422       __ z_cdfbr(dest->as_double_reg(), src->as_register());
 423       break;
 424 
 425     case Bytecodes::_l2f:
 426       __ z_cegbr(dest->as_float_reg(), src->as_register_lo());
 427       break;
 428     case Bytecodes::_l2d:
 429       __ z_cdgbr(dest->as_double_reg(), src->as_register_lo());
 430       break;
 431 
 432     case Bytecodes::_f2i:
 433     case Bytecodes::_f2l: {
 434       Label done;
 435       FloatRegister Rsrc = src->as_float_reg();
 436       Register Rdst = (op->bytecode() == Bytecodes::_f2i ? dest->as_register() : dest->as_register_lo());
 437       __ clear_reg(Rdst, true, false);
 438       __ z_cebr(Rsrc, Rsrc);
 439       __ z_brno(done); // NaN -> 0
 440       if (op->bytecode() == Bytecodes::_f2i) {
 441         __ z_cfebr(Rdst, Rsrc, Assembler::to_zero);
 442       } else { // op->bytecode() == Bytecodes::_f2l
 443         __ z_cgebr(Rdst, Rsrc, Assembler::to_zero);
 444       }
 445       __ bind(done);
 446     }
 447     break;
 448 
 449     case Bytecodes::_d2i:
 450     case Bytecodes::_d2l: {
 451       Label done;
 452       FloatRegister Rsrc = src->as_double_reg();
 453       Register Rdst = (op->bytecode() == Bytecodes::_d2i ? dest->as_register() : dest->as_register_lo());
 454       __ clear_reg(Rdst, true, false);  // Don't set CC.
 455       __ z_cdbr(Rsrc, Rsrc);
 456       __ z_brno(done); // NaN -> 0
 457       if (op->bytecode() == Bytecodes::_d2i) {
 458         __ z_cfdbr(Rdst, Rsrc, Assembler::to_zero);
 459       } else { // Bytecodes::_d2l
 460         __ z_cgdbr(Rdst, Rsrc, Assembler::to_zero);
 461       }
 462       __ bind(done);
 463     }
 464     break;
 465 
 466     default: ShouldNotReachHere();
 467   }
 468 }
 469 
 470 void LIR_Assembler::align_call(LIR_Code code) {
 471   // End of call instruction must be 4 byte aligned.
 472   int offset = __ offset();
 473   switch (code) {
 474     case lir_icvirtual_call:
 475       offset += MacroAssembler::load_const_from_toc_size();
 476       // no break
 477     case lir_static_call:
 478     case lir_optvirtual_call:
 479     case lir_dynamic_call:
 480       offset += NativeCall::call_far_pcrelative_displacement_offset;
 481       break;
 482     default: ShouldNotReachHere();
 483   }
 484   if ((offset & (NativeCall::call_far_pcrelative_displacement_alignment-1)) != 0) {
 485     __ nop();
 486   }
 487 }
 488 
 489 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
 490   assert((__ offset() + NativeCall::call_far_pcrelative_displacement_offset) % NativeCall::call_far_pcrelative_displacement_alignment == 0,
 491          "must be aligned (offset=%d)", __ offset());
 492   assert(rtype == relocInfo::none ||
 493          rtype == relocInfo::opt_virtual_call_type ||
 494          rtype == relocInfo::static_call_type, "unexpected rtype");
 495   // Prepend each BRASL with a nop.
 496   __ relocate(rtype);
 497   __ z_nop();
 498   __ z_brasl(Z_R14, op->addr());
 499   add_call_info(code_offset(), op->info());
 500 }
 501 
 502 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
 503   address virtual_call_oop_addr = NULL;
 504   AddressLiteral empty_ic((address) Universe::non_oop_word());
 505   virtual_call_oop_addr = __ pc();
 506   bool success = __ load_const_from_toc(Z_inline_cache, empty_ic);
 507   if (!success) {
 508     bailout("const section overflow");
 509     return;
 510   }
 511 
 512   // CALL to fixup routine. Fixup routine uses ScopeDesc info
 513   // to determine who we intended to call.
 514   __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr));
 515   call(op, relocInfo::none);
 516 }
 517 
 518 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 519   if (from_reg != to_reg) __ z_lgr(to_reg, from_reg);
 520 }
 521 
 522 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 523   assert(src->is_constant(), "should not call otherwise");
 524   assert(dest->is_stack(), "should not call otherwise");
 525   LIR_Const* c = src->as_constant_ptr();
 526 
 527   unsigned int lmem = 0;
 528   unsigned int lcon = 0;
 529   int64_t cbits = 0;
 530   Address dest_addr;
 531   switch (c->type()) {
 532     case T_INT:  // fall through
 533     case T_FLOAT:
 534       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 535       lmem = 4; lcon = 4; cbits = c->as_jint_bits();
 536       break;
 537 
 538     case T_ADDRESS:
 539       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 540       lmem = 8; lcon = 4; cbits = c->as_jint_bits();
 541       break;
 542 
 543     case T_OBJECT:
 544       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 545       if (c->as_jobject() == NULL) {
 546         __ store_const(dest_addr, (int64_t)NULL_WORD, 8, 8);
 547       } else {
 548         jobject2reg(c->as_jobject(), Z_R1_scratch);
 549         __ reg2mem_opt(Z_R1_scratch, dest_addr, true);
 550       }
 551       return;
 552 
 553     case T_LONG:  // fall through
 554     case T_DOUBLE:
 555       dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 556       lmem = 8; lcon = 8; cbits = (int64_t)(c->as_jlong_bits());
 557       break;
 558 
 559     default:
 560       ShouldNotReachHere();
 561   }
 562 
 563   __ store_const(dest_addr, cbits, lmem, lcon);
 564 }
 565 
 566 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 567   assert(src->is_constant(), "should not call otherwise");
 568   assert(dest->is_address(), "should not call otherwise");
 569 
 570   LIR_Const* c = src->as_constant_ptr();
 571   Address addr = as_Address(dest->as_address_ptr());
 572 
 573   int store_offset = -1;
 574 
 575   if (dest->as_address_ptr()->index()->is_valid()) {
 576     switch (type) {
 577       case T_INT:    // fall through
 578       case T_FLOAT:
 579         __ load_const_optimized(Z_R0_scratch, c->as_jint_bits());
 580         store_offset = __ offset();
 581         if (Immediate::is_uimm12(addr.disp())) {
 582           __ z_st(Z_R0_scratch, addr);
 583         } else {
 584           __ z_sty(Z_R0_scratch, addr);
 585         }
 586         break;
 587 
 588       case T_ADDRESS:
 589         __ load_const_optimized(Z_R1_scratch, c->as_jint_bits());
 590         store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 591         break;
 592 
 593       case T_OBJECT:  // fall through
 594       case T_ARRAY:
 595         if (c->as_jobject() == NULL) {
 596           if (UseCompressedOops && !wide) {
 597             __ clear_reg(Z_R1_scratch, false);
 598             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
 599           } else {
 600             __ clear_reg(Z_R1_scratch, true);
 601             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 602           }
 603         } else {
 604           jobject2reg(c->as_jobject(), Z_R1_scratch);
 605           if (UseCompressedOops && !wide) {
 606             __ encode_heap_oop(Z_R1_scratch);
 607             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
 608           } else {
 609             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 610           }
 611         }
 612         assert(store_offset >= 0, "check");
 613         break;
 614 
 615       case T_LONG:    // fall through
 616       case T_DOUBLE:
 617         __ load_const_optimized(Z_R1_scratch, (int64_t)(c->as_jlong_bits()));
 618         store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 619         break;
 620 
 621       case T_BOOLEAN: // fall through
 622       case T_BYTE:
 623         __ load_const_optimized(Z_R0_scratch, (int8_t)(c->as_jint()));
 624         store_offset = __ offset();
 625         if (Immediate::is_uimm12(addr.disp())) {
 626           __ z_stc(Z_R0_scratch, addr);
 627         } else {
 628           __ z_stcy(Z_R0_scratch, addr);
 629         }
 630         break;
 631 
 632       case T_CHAR:    // fall through
 633       case T_SHORT:
 634         __ load_const_optimized(Z_R0_scratch, (int16_t)(c->as_jint()));
 635         store_offset = __ offset();
 636         if (Immediate::is_uimm12(addr.disp())) {
 637           __ z_sth(Z_R0_scratch, addr);
 638         } else {
 639           __ z_sthy(Z_R0_scratch, addr);
 640         }
 641         break;
 642 
 643       default:
 644         ShouldNotReachHere();
 645     }
 646 
 647   } else { // no index
 648 
 649     unsigned int lmem = 0;
 650     unsigned int lcon = 0;
 651     int64_t cbits = 0;
 652 
 653     switch (type) {
 654       case T_INT:    // fall through
 655       case T_FLOAT:
 656         lmem = 4; lcon = 4; cbits = c->as_jint_bits();
 657         break;
 658 
 659       case T_ADDRESS:
 660         lmem = 8; lcon = 4; cbits = c->as_jint_bits();
 661         break;
 662 
 663       case T_OBJECT:  // fall through
 664       case T_ARRAY:
 665         if (c->as_jobject() == NULL) {
 666           if (UseCompressedOops && !wide) {
 667             store_offset = __ store_const(addr, (int32_t)NULL_WORD, 4, 4);
 668           } else {
 669             store_offset = __ store_const(addr, (int64_t)NULL_WORD, 8, 8);
 670           }
 671         } else {
 672           jobject2reg(c->as_jobject(), Z_R1_scratch);
 673           if (UseCompressedOops && !wide) {
 674             __ encode_heap_oop(Z_R1_scratch);
 675             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
 676           } else {
 677             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 678           }
 679         }
 680         assert(store_offset >= 0, "check");
 681         break;
 682 
 683       case T_LONG:    // fall through
 684       case T_DOUBLE:
 685         lmem = 8; lcon = 8; cbits = (int64_t)(c->as_jlong_bits());
 686         break;
 687 
 688       case T_BOOLEAN: // fall through
 689       case T_BYTE:
 690         lmem = 1; lcon = 1; cbits = (int8_t)(c->as_jint());
 691         break;
 692 
 693       case T_CHAR:    // fall through
 694       case T_SHORT:
 695         lmem = 2; lcon = 2; cbits = (int16_t)(c->as_jint());
 696         break;
 697 
 698       default:
 699         ShouldNotReachHere();
 700     }
 701 
 702     if (store_offset == -1) {
 703       store_offset = __ store_const(addr, cbits, lmem, lcon);
 704       assert(store_offset >= 0, "check");
 705     }
 706   }
 707 
 708   if (info != NULL) {
 709     add_debug_info_for_null_check(store_offset, info);
 710   }
 711 }
 712 
 713 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 714   assert(src->is_constant(), "should not call otherwise");
 715   assert(dest->is_register(), "should not call otherwise");
 716   LIR_Const* c = src->as_constant_ptr();
 717 
 718   switch (c->type()) {
 719     case T_INT: {
 720       assert(patch_code == lir_patch_none, "no patching handled here");
 721       __ load_const_optimized(dest->as_register(), c->as_jint());
 722       break;
 723     }
 724 
 725     case T_ADDRESS: {
 726       assert(patch_code == lir_patch_none, "no patching handled here");
 727       __ load_const_optimized(dest->as_register(), c->as_jint());
 728       break;
 729     }
 730 
 731     case T_LONG: {
 732       assert(patch_code == lir_patch_none, "no patching handled here");
 733       __ load_const_optimized(dest->as_register_lo(), (intptr_t)c->as_jlong());
 734       break;
 735     }
 736 
 737     case T_OBJECT: {
 738       if (patch_code != lir_patch_none) {
 739         jobject2reg_with_patching(dest->as_register(), info);
 740       } else {
 741         jobject2reg(c->as_jobject(), dest->as_register());
 742       }
 743       break;
 744     }
 745 
 746     case T_METADATA: {
 747       if (patch_code != lir_patch_none) {
 748         klass2reg_with_patching(dest->as_register(), info);
 749       } else {
 750         metadata2reg(c->as_metadata(), dest->as_register());
 751       }
 752       break;
 753     }
 754 
 755     case T_FLOAT: {
 756       Register toc_reg = Z_R1_scratch;
 757       __ load_toc(toc_reg);
 758       address const_addr = __ float_constant(c->as_jfloat());
 759       if (const_addr == NULL) {
 760         bailout("const section overflow");
 761         break;
 762       }
 763       int displ = const_addr - _masm->code()->consts()->start();
 764       if (dest->is_single_fpu()) {
 765         __ z_ley(dest->as_float_reg(), displ, toc_reg);
 766       } else {
 767         assert(dest->is_single_cpu(), "Must be a cpu register.");
 768         __ z_ly(dest->as_register(), displ, toc_reg);
 769       }
 770     }
 771     break;
 772 
 773     case T_DOUBLE: {
 774       Register toc_reg = Z_R1_scratch;
 775       __ load_toc(toc_reg);
 776       address const_addr = __ double_constant(c->as_jdouble());
 777       if (const_addr == NULL) {
 778         bailout("const section overflow");
 779         break;
 780       }
 781       int displ = const_addr - _masm->code()->consts()->start();
 782       if (dest->is_double_fpu()) {
 783         __ z_ldy(dest->as_double_reg(), displ, toc_reg);
 784       } else {
 785         assert(dest->is_double_cpu(), "Must be a long register.");
 786         __ z_lg(dest->as_register_lo(), displ, toc_reg);
 787       }
 788     }
 789     break;
 790 
 791     default:
 792       ShouldNotReachHere();
 793   }
 794 }
 795 
 796 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 797   if (addr->base()->is_illegal()) {
 798     Unimplemented();
 799   }
 800 
 801   Register base = addr->base()->as_pointer_register();
 802 
 803   if (addr->index()->is_illegal()) {
 804     return Address(base, addr->disp());
 805   } else if (addr->index()->is_cpu_register()) {
 806     Register index = addr->index()->as_pointer_register();
 807     return Address(base, index, addr->disp());
 808   } else if (addr->index()->is_constant()) {
 809     intptr_t addr_offset = addr->index()->as_constant_ptr()->as_jint() + addr->disp();
 810     return Address(base, addr_offset);
 811   } else {
 812     ShouldNotReachHere();
 813     return Address();
 814   }
 815 }
 816 
 817 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 818   switch (type) {
 819     case T_INT:
 820     case T_FLOAT: {
 821       Register tmp = Z_R1_scratch;
 822       Address from = frame_map()->address_for_slot(src->single_stack_ix());
 823       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
 824       __ mem2reg_opt(tmp, from, false);
 825       __ reg2mem_opt(tmp, to, false);
 826       break;
 827     }
 828     case T_ADDRESS:
 829     case T_OBJECT: {
 830       Register tmp = Z_R1_scratch;
 831       Address from = frame_map()->address_for_slot(src->single_stack_ix());
 832       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
 833       __ mem2reg_opt(tmp, from, true);
 834       __ reg2mem_opt(tmp, to, true);
 835       break;
 836     }
 837     case T_LONG:
 838     case T_DOUBLE: {
 839       Register tmp = Z_R1_scratch;
 840       Address from = frame_map()->address_for_double_slot(src->double_stack_ix());
 841       Address to   = frame_map()->address_for_double_slot(dest->double_stack_ix());
 842       __ mem2reg_opt(tmp, from, true);
 843       __ reg2mem_opt(tmp, to, true);
 844       break;
 845     }
 846 
 847     default:
 848       ShouldNotReachHere();
 849   }
 850 }
 851 
 852 // 4-byte accesses only! Don't use it to access 8 bytes!
 853 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 854   ShouldNotCallThis();
 855   return 0; // unused
 856 }
 857 
 858 // 4-byte accesses only! Don't use it to access 8 bytes!
 859 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 860   ShouldNotCallThis();
 861   return 0; // unused
 862 }
 863 
 864 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code,
 865                             CodeEmitInfo* info, bool wide) {
 866 
 867   assert(type != T_METADATA, "load of metadata ptr not supported");
 868   LIR_Address* addr = src_opr->as_address_ptr();
 869   LIR_Opr to_reg = dest;
 870 
 871   Register src = addr->base()->as_pointer_register();
 872   Register disp_reg = Z_R0;
 873   int disp_value = addr->disp();
 874   bool needs_patching = (patch_code != lir_patch_none);
 875 
 876   if (addr->base()->type() == T_OBJECT) {
 877     __ verify_oop(src, FILE_AND_LINE);
 878   }
 879 
 880   PatchingStub* patch = NULL;
 881   if (needs_patching) {
 882     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 883     assert(!to_reg->is_double_cpu() ||
 884            patch_code == lir_patch_none ||
 885            patch_code == lir_patch_normal, "patching doesn't match register");
 886   }
 887 
 888   if (addr->index()->is_illegal()) {
 889     if (!Immediate::is_simm20(disp_value)) {
 890       if (needs_patching) {
 891         __ load_const(Z_R1_scratch, (intptr_t)0);
 892       } else {
 893         __ load_const_optimized(Z_R1_scratch, disp_value);
 894       }
 895       disp_reg = Z_R1_scratch;
 896       disp_value = 0;
 897     }
 898   } else {
 899     if (!Immediate::is_simm20(disp_value)) {
 900       __ load_const_optimized(Z_R1_scratch, disp_value);
 901       __ z_la(Z_R1_scratch, 0, Z_R1_scratch, addr->index()->as_register());
 902       disp_reg = Z_R1_scratch;
 903       disp_value = 0;
 904     }
 905     disp_reg = addr->index()->as_pointer_register();
 906   }
 907 
 908   // Remember the offset of the load. The patching_epilog must be done
 909   // before the call to add_debug_info, otherwise the PcDescs don't get
 910   // entered in increasing order.
 911   int offset = code_offset();
 912 
 913   assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
 914 
 915   bool short_disp = Immediate::is_uimm12(disp_value);
 916 
 917   switch (type) {
 918     case T_BOOLEAN: // fall through
 919     case T_BYTE  :  __ z_lb(dest->as_register(),   disp_value, disp_reg, src); break;
 920     case T_CHAR  :  __ z_llgh(dest->as_register(), disp_value, disp_reg, src); break;
 921     case T_SHORT :
 922       if (short_disp) {
 923                     __ z_lh(dest->as_register(),   disp_value, disp_reg, src);
 924       } else {
 925                     __ z_lhy(dest->as_register(),  disp_value, disp_reg, src);
 926       }
 927       break;
 928     case T_INT   :
 929       if (short_disp) {
 930                     __ z_l(dest->as_register(),    disp_value, disp_reg, src);
 931       } else {
 932                     __ z_ly(dest->as_register(),   disp_value, disp_reg, src);
 933       }
 934       break;
 935     case T_ADDRESS:
 936       __ z_lg(dest->as_register(), disp_value, disp_reg, src);
 937       break;
 938     case T_ARRAY : // fall through
 939     case T_OBJECT:
 940     {
 941       if (UseCompressedOops && !wide) {
 942         __ z_llgf(dest->as_register(), disp_value, disp_reg, src);
 943         __ oop_decoder(dest->as_register(), dest->as_register(), true);
 944       } else {
 945         __ z_lg(dest->as_register(), disp_value, disp_reg, src);
 946       }
 947       __ verify_oop(dest->as_register(), FILE_AND_LINE);
 948       break;
 949     }
 950     case T_FLOAT:
 951       if (short_disp) {
 952                     __ z_le(dest->as_float_reg(),  disp_value, disp_reg, src);
 953       } else {
 954                     __ z_ley(dest->as_float_reg(), disp_value, disp_reg, src);
 955       }
 956       break;
 957     case T_DOUBLE:
 958       if (short_disp) {
 959                     __ z_ld(dest->as_double_reg(),  disp_value, disp_reg, src);
 960       } else {
 961                     __ z_ldy(dest->as_double_reg(), disp_value, disp_reg, src);
 962       }
 963       break;
 964     case T_LONG  :  __ z_lg(dest->as_register_lo(), disp_value, disp_reg, src); break;
 965     default      : ShouldNotReachHere();
 966   }
 967 
 968   if (patch != NULL) {
 969     patching_epilog(patch, patch_code, src, info);
 970   }
 971   if (info != NULL) add_debug_info_for_null_check(offset, info);
 972 }
 973 
 974 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 975   assert(src->is_stack(), "should not call otherwise");
 976   assert(dest->is_register(), "should not call otherwise");
 977 
 978   if (dest->is_single_cpu()) {
 979     if (is_reference_type(type)) {
 980       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
 981       __ verify_oop(dest->as_register(), FILE_AND_LINE);
 982     } else if (type == T_METADATA || type == T_ADDRESS) {
 983       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
 984     } else {
 985       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), false);
 986     }
 987   } else if (dest->is_double_cpu()) {
 988     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix());
 989     __ mem2reg_opt(dest->as_register_lo(), src_addr_LO, true);
 990   } else if (dest->is_single_fpu()) {
 991     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 992     __ mem2freg_opt(dest->as_float_reg(), src_addr, false);
 993   } else if (dest->is_double_fpu()) {
 994     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 995     __ mem2freg_opt(dest->as_double_reg(), src_addr, true);
 996   } else {
 997     ShouldNotReachHere();
 998   }
 999 }
1000 
1001 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
1002   assert(src->is_register(), "should not call otherwise");
1003   assert(dest->is_stack(), "should not call otherwise");
1004 
1005   if (src->is_single_cpu()) {
1006     const Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
1007     if (is_reference_type(type)) {
1008       __ verify_oop(src->as_register(), FILE_AND_LINE);
1009       __ reg2mem_opt(src->as_register(), dst, true);
1010     } else if (type == T_METADATA || type == T_ADDRESS) {
1011       __ reg2mem_opt(src->as_register(), dst, true);
1012     } else {
1013       __ reg2mem_opt(src->as_register(), dst, false);
1014     }
1015   } else if (src->is_double_cpu()) {
1016     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix());
1017     __ reg2mem_opt(src->as_register_lo(), dstLO, true);
1018   } else if (src->is_single_fpu()) {
1019     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
1020     __ freg2mem_opt(src->as_float_reg(), dst_addr, false);
1021   } else if (src->is_double_fpu()) {
1022     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
1023     __ freg2mem_opt(src->as_double_reg(), dst_addr, true);
1024   } else {
1025     ShouldNotReachHere();
1026   }
1027 }
1028 
1029 void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
1030   if (from_reg->is_float_kind() && to_reg->is_float_kind()) {
1031     if (from_reg->is_double_fpu()) {
1032       // double to double moves
1033       assert(to_reg->is_double_fpu(), "should match");
1034       __ z_ldr(to_reg->as_double_reg(), from_reg->as_double_reg());
1035     } else {
1036       // float to float moves
1037       assert(to_reg->is_single_fpu(), "should match");
1038       __ z_ler(to_reg->as_float_reg(), from_reg->as_float_reg());
1039     }
1040   } else if (!from_reg->is_float_kind() && !to_reg->is_float_kind()) {
1041     if (from_reg->is_double_cpu()) {
1042       __ z_lgr(to_reg->as_pointer_register(), from_reg->as_pointer_register());
1043     } else if (to_reg->is_double_cpu()) {
1044       // int to int moves
1045       __ z_lgr(to_reg->as_register_lo(), from_reg->as_register());
1046     } else {
1047       // int to int moves
1048       __ z_lgr(to_reg->as_register(), from_reg->as_register());
1049     }
1050   } else {
1051     ShouldNotReachHere();
1052   }
1053   if (is_reference_type(to_reg->type())) {
1054     __ verify_oop(to_reg->as_register(), FILE_AND_LINE);
1055   }
1056 }
1057 
1058 void LIR_Assembler::reg2mem(LIR_Opr from, LIR_Opr dest_opr, BasicType type,
1059                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
1060                             bool wide) {
1061   assert(type != T_METADATA, "store of metadata ptr not supported");
1062   LIR_Address* addr = dest_opr->as_address_ptr();
1063 
1064   Register dest = addr->base()->as_pointer_register();
1065   Register disp_reg = Z_R0;
1066   int disp_value = addr->disp();
1067   bool needs_patching = (patch_code != lir_patch_none);
1068 
1069   if (addr->base()->is_oop_register()) {
1070     __ verify_oop(dest, FILE_AND_LINE);
1071   }
1072 
1073   PatchingStub* patch = NULL;
1074   if (needs_patching) {
1075     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1076     assert(!from->is_double_cpu() ||
1077            patch_code == lir_patch_none ||
1078            patch_code == lir_patch_normal, "patching doesn't match register");
1079   }
1080 
1081   assert(!needs_patching || (!Immediate::is_simm20(disp_value) && addr->index()->is_illegal()), "assumption");
1082   if (addr->index()->is_illegal()) {
1083     if (!Immediate::is_simm20(disp_value)) {
1084       if (needs_patching) {
1085         __ load_const(Z_R1_scratch, (intptr_t)0);
1086       } else {
1087         __ load_const_optimized(Z_R1_scratch, disp_value);
1088       }
1089       disp_reg = Z_R1_scratch;
1090       disp_value = 0;
1091     }
1092   } else {
1093     if (!Immediate::is_simm20(disp_value)) {
1094       __ load_const_optimized(Z_R1_scratch, disp_value);
1095       __ z_la(Z_R1_scratch, 0, Z_R1_scratch, addr->index()->as_register());
1096       disp_reg = Z_R1_scratch;
1097       disp_value = 0;
1098     }
1099     disp_reg = addr->index()->as_pointer_register();
1100   }
1101 
1102   assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
1103 
1104   if (is_reference_type(type)) {
1105     __ verify_oop(from->as_register(), FILE_AND_LINE);
1106   }
1107 
1108   bool short_disp = Immediate::is_uimm12(disp_value);
1109 
1110   // Remember the offset of the store. The patching_epilog must be done
1111   // before the call to add_debug_info_for_null_check, otherwise the PcDescs don't get
1112   // entered in increasing order.
1113   int offset = code_offset();
1114   switch (type) {
1115     case T_BOOLEAN: // fall through
1116     case T_BYTE  :
1117       if (short_disp) {
1118                     __ z_stc(from->as_register(),  disp_value, disp_reg, dest);
1119       } else {
1120                     __ z_stcy(from->as_register(), disp_value, disp_reg, dest);
1121       }
1122       break;
1123     case T_CHAR  : // fall through
1124     case T_SHORT :
1125       if (short_disp) {
1126                     __ z_sth(from->as_register(),  disp_value, disp_reg, dest);
1127       } else {
1128                     __ z_sthy(from->as_register(), disp_value, disp_reg, dest);
1129       }
1130       break;
1131     case T_INT   :
1132       if (short_disp) {
1133                     __ z_st(from->as_register(),  disp_value, disp_reg, dest);
1134       } else {
1135                     __ z_sty(from->as_register(), disp_value, disp_reg, dest);
1136       }
1137       break;
1138     case T_LONG  :  __ z_stg(from->as_register_lo(), disp_value, disp_reg, dest); break;
1139     case T_ADDRESS: __ z_stg(from->as_register(),    disp_value, disp_reg, dest); break;
1140       break;
1141     case T_ARRAY : // fall through
1142     case T_OBJECT:
1143       {
1144         if (UseCompressedOops && !wide) {
1145           Register compressed_src = Z_R14;
1146           __ oop_encoder(compressed_src, from->as_register(), true, (disp_reg != Z_R1) ? Z_R1 : Z_R0, -1, true);
1147           offset = code_offset();
1148           if (short_disp) {
1149             __ z_st(compressed_src,  disp_value, disp_reg, dest);
1150           } else {
1151             __ z_sty(compressed_src, disp_value, disp_reg, dest);
1152           }
1153         } else {
1154           __ z_stg(from->as_register(), disp_value, disp_reg, dest);
1155         }
1156         break;
1157       }
1158     case T_FLOAT :
1159       if (short_disp) {
1160         __ z_ste(from->as_float_reg(),  disp_value, disp_reg, dest);
1161       } else {
1162         __ z_stey(from->as_float_reg(), disp_value, disp_reg, dest);
1163       }
1164       break;
1165     case T_DOUBLE:
1166       if (short_disp) {
1167         __ z_std(from->as_double_reg(),  disp_value, disp_reg, dest);
1168       } else {
1169         __ z_stdy(from->as_double_reg(), disp_value, disp_reg, dest);
1170       }
1171       break;
1172     default: ShouldNotReachHere();
1173   }
1174 
1175   if (patch != NULL) {
1176     patching_epilog(patch, patch_code, dest, info);
1177   }
1178 
1179   if (info != NULL) add_debug_info_for_null_check(offset, info);
1180 }
1181 
1182 
1183 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
1184   assert(result->is_illegal() ||
1185          (result->is_single_cpu() && result->as_register() == Z_R2) ||
1186          (result->is_double_cpu() && result->as_register_lo() == Z_R2) ||
1187          (result->is_single_fpu() && result->as_float_reg() == Z_F0) ||
1188          (result->is_double_fpu() && result->as_double_reg() == Z_F0), "convention");
1189 
1190   __ z_lg(Z_R1_scratch, Address(Z_thread, JavaThread::polling_page_offset()));
1191 
1192   // Pop the frame before the safepoint code.
1193   __ pop_frame_restore_retPC(initial_frame_size_in_bytes());
1194 
1195   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
1196     __ reserved_stack_check(Z_R14);
1197   }
1198 
1199   // We need to mark the code position where the load from the safepoint
1200   // polling page was emitted as relocInfo::poll_return_type here.
1201   __ relocate(relocInfo::poll_return_type);
1202   __ load_from_polling_page(Z_R1_scratch);
1203 
1204   __ z_br(Z_R14); // Return to caller.
1205 }
1206 
1207 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
1208   const Register poll_addr = tmp->as_register_lo();
1209   __ z_lg(poll_addr, Address(Z_thread, JavaThread::polling_page_offset()));
1210   guarantee(info != NULL, "Shouldn't be NULL");
1211   add_debug_info_for_branch(info);
1212   int offset = __ offset();
1213   __ relocate(relocInfo::poll_type);
1214   __ load_from_polling_page(poll_addr);
1215   return offset;
1216 }
1217 
1218 void LIR_Assembler::emit_static_call_stub() {
1219 
1220   // Stub is fixed up when the corresponding call is converted from calling
1221   // compiled code to calling interpreted code.
1222 
1223   address call_pc = __ pc();
1224   address stub = __ start_a_stub(call_stub_size());
1225   if (stub == NULL) {
1226     bailout("static call stub overflow");
1227     return;
1228   }
1229 
1230   int start = __ offset();
1231 
1232   __ relocate(static_stub_Relocation::spec(call_pc));
1233 
1234   // See also Matcher::interpreter_method_reg().
1235   AddressLiteral meta = __ allocate_metadata_address(NULL);
1236   bool success = __ load_const_from_toc(Z_method, meta);
1237 
1238   __ set_inst_mark();
1239   AddressLiteral a((address)-1);
1240   success = success && __ load_const_from_toc(Z_R1, a);
1241   if (!success) {
1242     bailout("const section overflow");
1243     return;
1244   }
1245 
1246   __ z_br(Z_R1);
1247   assert(__ offset() - start <= call_stub_size(), "stub too big");
1248   __ end_a_stub(); // Update current stubs pointer and restore insts_end.
1249 }
1250 
1251 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1252   bool unsigned_comp = condition == lir_cond_belowEqual || condition == lir_cond_aboveEqual;
1253   if (opr1->is_single_cpu()) {
1254     Register reg1 = opr1->as_register();
1255     if (opr2->is_single_cpu()) {
1256       // cpu register - cpu register
1257       if (is_reference_type(opr1->type())) {
1258         __ z_clgr(reg1, opr2->as_register());
1259       } else {
1260         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1261         if (unsigned_comp) {
1262           __ z_clr(reg1, opr2->as_register());
1263         } else {
1264           __ z_cr(reg1, opr2->as_register());
1265         }
1266       }
1267     } else if (opr2->is_stack()) {
1268       // cpu register - stack
1269       if (is_reference_type(opr1->type())) {
1270         __ z_cg(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1271       } else {
1272         if (unsigned_comp) {
1273           __ z_cly(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1274         } else {
1275           __ z_cy(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1276         }
1277       }
1278     } else if (opr2->is_constant()) {
1279       // cpu register - constant
1280       LIR_Const* c = opr2->as_constant_ptr();
1281       if (c->type() == T_INT) {
1282         if (unsigned_comp) {
1283           __ z_clfi(reg1, c->as_jint());
1284         } else {
1285           __ z_cfi(reg1, c->as_jint());
1286         }
1287       } else if (c->type() == T_METADATA) {
1288         // We only need, for now, comparison with NULL for metadata.
1289         assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
1290         Metadata* m = c->as_metadata();
1291         if (m == NULL) {
1292           __ z_cghi(reg1, 0);
1293         } else {
1294           ShouldNotReachHere();
1295         }
1296       } else if (is_reference_type(c->type())) {
1297         // In 64bit oops are single register.
1298         jobject o = c->as_jobject();
1299         if (o == NULL) {
1300           __ z_ltgr(reg1, reg1);
1301         } else {
1302           jobject2reg(o, Z_R1_scratch);
1303           __ z_cgr(reg1, Z_R1_scratch);
1304         }
1305       } else {
1306         fatal("unexpected type: %s", basictype_to_str(c->type()));
1307       }
1308       // cpu register - address
1309     } else if (opr2->is_address()) {
1310       if (op->info() != NULL) {
1311         add_debug_info_for_null_check_here(op->info());
1312       }
1313       if (unsigned_comp) {
1314         __ z_cly(reg1, as_Address(opr2->as_address_ptr()));
1315       } else {
1316         __ z_cy(reg1, as_Address(opr2->as_address_ptr()));
1317       }
1318     } else {
1319       ShouldNotReachHere();
1320     }
1321 
1322   } else if (opr1->is_double_cpu()) {
1323     assert(!unsigned_comp, "unexpected");
1324     Register xlo = opr1->as_register_lo();
1325     Register xhi = opr1->as_register_hi();
1326     if (opr2->is_double_cpu()) {
1327       __ z_cgr(xlo, opr2->as_register_lo());
1328     } else if (opr2->is_constant()) {
1329       // cpu register - constant 0
1330       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
1331       __ z_ltgr(xlo, xlo);
1332     } else {
1333       ShouldNotReachHere();
1334     }
1335 
1336   } else if (opr1->is_single_fpu()) {
1337     if (opr2->is_single_fpu()) {
1338       __ z_cebr(opr1->as_float_reg(), opr2->as_float_reg());
1339     } else {
1340       // stack slot
1341       Address addr = frame_map()->address_for_slot(opr2->single_stack_ix());
1342       if (Immediate::is_uimm12(addr.disp())) {
1343         __ z_ceb(opr1->as_float_reg(), addr);
1344       } else {
1345         __ z_ley(Z_fscratch_1, addr);
1346         __ z_cebr(opr1->as_float_reg(), Z_fscratch_1);
1347       }
1348     }
1349   } else if (opr1->is_double_fpu()) {
1350     if (opr2->is_double_fpu()) {
1351     __ z_cdbr(opr1->as_double_reg(), opr2->as_double_reg());
1352     } else {
1353       // stack slot
1354       Address addr = frame_map()->address_for_slot(opr2->double_stack_ix());
1355       if (Immediate::is_uimm12(addr.disp())) {
1356         __ z_cdb(opr1->as_double_reg(), addr);
1357       } else {
1358         __ z_ldy(Z_fscratch_1, addr);
1359         __ z_cdbr(opr1->as_double_reg(), Z_fscratch_1);
1360       }
1361     }
1362   } else {
1363     ShouldNotReachHere();
1364   }
1365 }
1366 
1367 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1368   Label    done;
1369   Register dreg = dst->as_register();
1370 
1371   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1372     assert((left->is_single_fpu() && right->is_single_fpu()) ||
1373            (left->is_double_fpu() && right->is_double_fpu()), "unexpected operand types");
1374     bool is_single = left->is_single_fpu();
1375     bool is_unordered_less = (code == lir_ucmp_fd2i);
1376     FloatRegister lreg = is_single ? left->as_float_reg() : left->as_double_reg();
1377     FloatRegister rreg = is_single ? right->as_float_reg() : right->as_double_reg();
1378     if (is_single) {
1379       __ z_cebr(lreg, rreg);
1380     } else {
1381       __ z_cdbr(lreg, rreg);
1382     }
1383     if (VM_Version::has_LoadStoreConditional()) {
1384       Register one       = Z_R0_scratch;
1385       Register minus_one = Z_R1_scratch;
1386       __ z_lghi(minus_one, -1);
1387       __ z_lghi(one,  1);
1388       __ z_lghi(dreg, 0);
1389       __ z_locgr(dreg, one,       is_unordered_less ? Assembler::bcondHigh            : Assembler::bcondHighOrNotOrdered);
1390       __ z_locgr(dreg, minus_one, is_unordered_less ? Assembler::bcondLowOrNotOrdered : Assembler::bcondLow);
1391     } else {
1392       __ clear_reg(dreg, true, false);
1393       __ z_bre(done); // if (left == right) dst = 0
1394 
1395       // if (left > right || ((code ~= cmpg) && (left <> right)) dst := 1
1396       __ z_lhi(dreg, 1);
1397       __ z_brc(is_unordered_less ? Assembler::bcondHigh : Assembler::bcondHighOrNotOrdered, done);
1398 
1399       // if (left < right || ((code ~= cmpl) && (left <> right)) dst := -1
1400       __ z_lhi(dreg, -1);
1401     }
1402   } else {
1403     assert(code == lir_cmp_l2i, "check");
1404     if (VM_Version::has_LoadStoreConditional()) {
1405       Register one       = Z_R0_scratch;
1406       Register minus_one = Z_R1_scratch;
1407       __ z_cgr(left->as_register_lo(), right->as_register_lo());
1408       __ z_lghi(minus_one, -1);
1409       __ z_lghi(one,  1);
1410       __ z_lghi(dreg, 0);
1411       __ z_locgr(dreg, one, Assembler::bcondHigh);
1412       __ z_locgr(dreg, minus_one, Assembler::bcondLow);
1413     } else {
1414       __ z_cgr(left->as_register_lo(), right->as_register_lo());
1415       __ z_lghi(dreg,  0);     // eq value
1416       __ z_bre(done);
1417       __ z_lghi(dreg,  1);     // gt value
1418       __ z_brh(done);
1419       __ z_lghi(dreg, -1);     // lt value
1420     }
1421   }
1422   __ bind(done);
1423 }
1424 
1425 // result = condition ? opr1 : opr2
1426 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1427                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1428   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on s390");
1429 
1430   Assembler::branch_condition acond = Assembler::bcondEqual, ncond = Assembler::bcondNotEqual;
1431   switch (condition) {
1432     case lir_cond_equal:        acond = Assembler::bcondEqual;    ncond = Assembler::bcondNotEqual; break;
1433     case lir_cond_notEqual:     acond = Assembler::bcondNotEqual; ncond = Assembler::bcondEqual;    break;
1434     case lir_cond_less:         acond = Assembler::bcondLow;      ncond = Assembler::bcondNotLow;   break;
1435     case lir_cond_lessEqual:    acond = Assembler::bcondNotHigh;  ncond = Assembler::bcondHigh;     break;
1436     case lir_cond_greaterEqual: acond = Assembler::bcondNotLow;   ncond = Assembler::bcondLow;      break;
1437     case lir_cond_greater:      acond = Assembler::bcondHigh;     ncond = Assembler::bcondNotHigh;  break;
1438     case lir_cond_belowEqual:   acond = Assembler::bcondNotHigh;  ncond = Assembler::bcondHigh;     break;
1439     case lir_cond_aboveEqual:   acond = Assembler::bcondNotLow;   ncond = Assembler::bcondLow;      break;
1440     default:                    ShouldNotReachHere();
1441   }
1442 
1443   if (opr1->is_cpu_register()) {
1444     reg2reg(opr1, result);
1445   } else if (opr1->is_stack()) {
1446     stack2reg(opr1, result, result->type());
1447   } else if (opr1->is_constant()) {
1448     const2reg(opr1, result, lir_patch_none, NULL);
1449   } else {
1450     ShouldNotReachHere();
1451   }
1452 
1453   if (VM_Version::has_LoadStoreConditional() && !opr2->is_constant()) {
1454     // Optimized version that does not require a branch.
1455     if (opr2->is_single_cpu()) {
1456       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1457       __ z_locgr(result->as_register(), opr2->as_register(), ncond);
1458     } else if (opr2->is_double_cpu()) {
1459       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1460       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1461       __ z_locgr(result->as_register_lo(), opr2->as_register_lo(), ncond);
1462     } else if (opr2->is_single_stack()) {
1463       __ z_loc(result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()), ncond);
1464     } else if (opr2->is_double_stack()) {
1465       __ z_locg(result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix()), ncond);
1466     } else {
1467       ShouldNotReachHere();
1468     }
1469   } else {
1470     Label skip;
1471     __ z_brc(acond, skip);
1472     if (opr2->is_cpu_register()) {
1473       reg2reg(opr2, result);
1474     } else if (opr2->is_stack()) {
1475       stack2reg(opr2, result, result->type());
1476     } else if (opr2->is_constant()) {
1477       const2reg(opr2, result, lir_patch_none, NULL);
1478     } else {
1479       ShouldNotReachHere();
1480     }
1481     __ bind(skip);
1482   }
1483 }
1484 
1485 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest,
1486                              CodeEmitInfo* info, bool pop_fpu_stack) {
1487   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1488 
1489   if (left->is_single_cpu()) {
1490     assert(left == dest, "left and dest must be equal");
1491     Register lreg = left->as_register();
1492 
1493     if (right->is_single_cpu()) {
1494       // cpu register - cpu register
1495       Register rreg = right->as_register();
1496       switch (code) {
1497         case lir_add: __ z_ar (lreg, rreg); break;
1498         case lir_sub: __ z_sr (lreg, rreg); break;
1499         case lir_mul: __ z_msr(lreg, rreg); break;
1500         default: ShouldNotReachHere();
1501       }
1502 
1503     } else if (right->is_stack()) {
1504       // cpu register - stack
1505       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1506       switch (code) {
1507         case lir_add: __ z_ay(lreg, raddr); break;
1508         case lir_sub: __ z_sy(lreg, raddr); break;
1509         default: ShouldNotReachHere();
1510       }
1511 
1512     } else if (right->is_constant()) {
1513       // cpu register - constant
1514       jint c = right->as_constant_ptr()->as_jint();
1515       switch (code) {
1516         case lir_add: __ z_agfi(lreg, c);  break;
1517         case lir_sub: __ z_agfi(lreg, -c); break; // note: -min_jint == min_jint
1518         case lir_mul: __ z_msfi(lreg, c);  break;
1519         default: ShouldNotReachHere();
1520       }
1521 
1522     } else {
1523       ShouldNotReachHere();
1524     }
1525 
1526   } else if (left->is_double_cpu()) {
1527     assert(left == dest, "left and dest must be equal");
1528     Register lreg_lo = left->as_register_lo();
1529     Register lreg_hi = left->as_register_hi();
1530 
1531     if (right->is_double_cpu()) {
1532       // cpu register - cpu register
1533       Register rreg_lo = right->as_register_lo();
1534       Register rreg_hi = right->as_register_hi();
1535       assert_different_registers(lreg_lo, rreg_lo);
1536       switch (code) {
1537         case lir_add:
1538           __ z_agr(lreg_lo, rreg_lo);
1539           break;
1540         case lir_sub:
1541           __ z_sgr(lreg_lo, rreg_lo);
1542           break;
1543         case lir_mul:
1544           __ z_msgr(lreg_lo, rreg_lo);
1545           break;
1546         default:
1547           ShouldNotReachHere();
1548       }
1549 
1550     } else if (right->is_constant()) {
1551       // cpu register - constant
1552       jlong c = right->as_constant_ptr()->as_jlong_bits();
1553       switch (code) {
1554         case lir_add: __ z_agfi(lreg_lo, c); break;
1555         case lir_sub:
1556           if (c != min_jint) {
1557                       __ z_agfi(lreg_lo, -c);
1558           } else {
1559             // -min_jint cannot be represented as simm32 in z_agfi
1560             // min_jint sign extended:      0xffffffff80000000
1561             // -min_jint as 64 bit integer: 0x0000000080000000
1562             // 0x80000000 can be represented as uimm32 in z_algfi
1563             // lreg_lo := lreg_lo + -min_jint == lreg_lo + 0x80000000
1564                       __ z_algfi(lreg_lo, UCONST64(0x80000000));
1565           }
1566           break;
1567         case lir_mul: __ z_msgfi(lreg_lo, c); break;
1568         default:
1569           ShouldNotReachHere();
1570       }
1571 
1572     } else {
1573       ShouldNotReachHere();
1574     }
1575 
1576   } else if (left->is_single_fpu()) {
1577     assert(left == dest, "left and dest must be equal");
1578     FloatRegister lreg = left->as_float_reg();
1579     FloatRegister rreg = right->is_single_fpu() ? right->as_float_reg() : fnoreg;
1580     Address raddr;
1581 
1582     if (rreg == fnoreg) {
1583       assert(right->is_single_stack(), "constants should be loaded into register");
1584       raddr = frame_map()->address_for_slot(right->single_stack_ix());
1585       if (!Immediate::is_uimm12(raddr.disp())) {
1586         __ mem2freg_opt(rreg = Z_fscratch_1, raddr, false);
1587       }
1588     }
1589 
1590     if (rreg != fnoreg) {
1591       switch (code) {
1592         case lir_add: __ z_aebr(lreg, rreg);  break;
1593         case lir_sub: __ z_sebr(lreg, rreg);  break;
1594         case lir_mul: __ z_meebr(lreg, rreg); break;
1595         case lir_div: __ z_debr(lreg, rreg);  break;
1596         default: ShouldNotReachHere();
1597       }
1598     } else {
1599       switch (code) {
1600         case lir_add: __ z_aeb(lreg, raddr);  break;
1601         case lir_sub: __ z_seb(lreg, raddr);  break;
1602         case lir_mul: __ z_meeb(lreg, raddr);  break;
1603         case lir_div: __ z_deb(lreg, raddr);  break;
1604         default: ShouldNotReachHere();
1605       }
1606     }
1607   } else if (left->is_double_fpu()) {
1608     assert(left == dest, "left and dest must be equal");
1609     FloatRegister lreg = left->as_double_reg();
1610     FloatRegister rreg = right->is_double_fpu() ? right->as_double_reg() : fnoreg;
1611     Address raddr;
1612 
1613     if (rreg == fnoreg) {
1614       assert(right->is_double_stack(), "constants should be loaded into register");
1615       raddr = frame_map()->address_for_slot(right->double_stack_ix());
1616       if (!Immediate::is_uimm12(raddr.disp())) {
1617         __ mem2freg_opt(rreg = Z_fscratch_1, raddr, true);
1618       }
1619     }
1620 
1621     if (rreg != fnoreg) {
1622       switch (code) {
1623         case lir_add: __ z_adbr(lreg, rreg); break;
1624         case lir_sub: __ z_sdbr(lreg, rreg); break;
1625         case lir_mul: __ z_mdbr(lreg, rreg); break;
1626         case lir_div: __ z_ddbr(lreg, rreg); break;
1627         default: ShouldNotReachHere();
1628       }
1629     } else {
1630       switch (code) {
1631         case lir_add: __ z_adb(lreg, raddr); break;
1632         case lir_sub: __ z_sdb(lreg, raddr); break;
1633         case lir_mul: __ z_mdb(lreg, raddr); break;
1634         case lir_div: __ z_ddb(lreg, raddr); break;
1635         default: ShouldNotReachHere();
1636       }
1637     }
1638   } else if (left->is_address()) {
1639     assert(left == dest, "left and dest must be equal");
1640     assert(code == lir_add, "unsupported operation");
1641     assert(right->is_constant(), "unsupported operand");
1642     jint c = right->as_constant_ptr()->as_jint();
1643     LIR_Address* lir_addr = left->as_address_ptr();
1644     Address addr = as_Address(lir_addr);
1645     switch (lir_addr->type()) {
1646       case T_INT:
1647         __ add2mem_32(addr, c, Z_R1_scratch);
1648         break;
1649       case T_LONG:
1650         __ add2mem_64(addr, c, Z_R1_scratch);
1651         break;
1652       default:
1653         ShouldNotReachHere();
1654     }
1655   } else {
1656     ShouldNotReachHere();
1657   }
1658 }
1659 
1660 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, LIR_Opr dest, LIR_Op* op) {
1661   switch (code) {
1662     case lir_sqrt: {
1663       assert(!thread->is_valid(), "there is no need for a thread_reg for dsqrt");
1664       FloatRegister src_reg = value->as_double_reg();
1665       FloatRegister dst_reg = dest->as_double_reg();
1666       __ z_sqdbr(dst_reg, src_reg);
1667       break;
1668     }
1669     case lir_abs: {
1670       assert(!thread->is_valid(), "there is no need for a thread_reg for fabs");
1671       FloatRegister src_reg = value->as_double_reg();
1672       FloatRegister dst_reg = dest->as_double_reg();
1673       __ z_lpdbr(dst_reg, src_reg);
1674       break;
1675     }
1676     default: {
1677       ShouldNotReachHere();
1678       break;
1679     }
1680   }
1681 }
1682 
1683 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1684   if (left->is_single_cpu()) {
1685     Register reg = left->as_register();
1686     if (right->is_constant()) {
1687       int val = right->as_constant_ptr()->as_jint();
1688       switch (code) {
1689         case lir_logic_and: __ z_nilf(reg, val); break;
1690         case lir_logic_or:  __ z_oilf(reg, val); break;
1691         case lir_logic_xor: __ z_xilf(reg, val); break;
1692         default: ShouldNotReachHere();
1693       }
1694     } else if (right->is_stack()) {
1695       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1696       switch (code) {
1697         case lir_logic_and: __ z_ny(reg, raddr); break;
1698         case lir_logic_or:  __ z_oy(reg, raddr); break;
1699         case lir_logic_xor: __ z_xy(reg, raddr); break;
1700         default: ShouldNotReachHere();
1701       }
1702     } else {
1703       Register rright = right->as_register();
1704       switch (code) {
1705         case lir_logic_and: __ z_nr(reg, rright); break;
1706         case lir_logic_or : __ z_or(reg, rright); break;
1707         case lir_logic_xor: __ z_xr(reg, rright); break;
1708         default: ShouldNotReachHere();
1709       }
1710     }
1711     move_regs(reg, dst->as_register());
1712   } else {
1713     Register l_lo = left->as_register_lo();
1714     if (right->is_constant()) {
1715       __ load_const_optimized(Z_R1_scratch, right->as_constant_ptr()->as_jlong());
1716       switch (code) {
1717         case lir_logic_and:
1718           __ z_ngr(l_lo, Z_R1_scratch);
1719           break;
1720         case lir_logic_or:
1721           __ z_ogr(l_lo, Z_R1_scratch);
1722           break;
1723         case lir_logic_xor:
1724           __ z_xgr(l_lo, Z_R1_scratch);
1725           break;
1726         default: ShouldNotReachHere();
1727       }
1728     } else {
1729       Register r_lo;
1730       if (is_reference_type(right->type())) {
1731         r_lo = right->as_register();
1732       } else {
1733         r_lo = right->as_register_lo();
1734       }
1735       switch (code) {
1736         case lir_logic_and:
1737           __ z_ngr(l_lo, r_lo);
1738           break;
1739         case lir_logic_or:
1740           __ z_ogr(l_lo, r_lo);
1741           break;
1742         case lir_logic_xor:
1743           __ z_xgr(l_lo, r_lo);
1744           break;
1745         default: ShouldNotReachHere();
1746       }
1747     }
1748 
1749     Register dst_lo = dst->as_register_lo();
1750 
1751     move_regs(l_lo, dst_lo);
1752   }
1753 }
1754 
1755 // See operand selection in LIRGenerator::do_ArithmeticOp_Int().
1756 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1757   if (left->is_double_cpu()) {
1758     // 64 bit integer case
1759     assert(left->is_double_cpu(), "left must be register");
1760     assert(right->is_double_cpu() || is_power_of_2(right->as_jlong()),
1761            "right must be register or power of 2 constant");
1762     assert(result->is_double_cpu(), "result must be register");
1763 
1764     Register lreg = left->as_register_lo();
1765     Register dreg = result->as_register_lo();
1766 
1767     if (right->is_constant()) {
1768       // Convert division by a power of two into some shifts and logical operations.
1769       Register treg1 = Z_R0_scratch;
1770       Register treg2 = Z_R1_scratch;
1771       jlong divisor = right->as_jlong();
1772       jlong log_divisor = log2i_exact(right->as_jlong());
1773 
1774       if (divisor == min_jlong) {
1775         // Min_jlong is special. Result is '0' except for min_jlong/min_jlong = 1.
1776         if (dreg == lreg) {
1777           NearLabel done;
1778           __ load_const_optimized(treg2, min_jlong);
1779           __ z_cgr(lreg, treg2);
1780           __ z_lghi(dreg, 0);           // Preserves condition code.
1781           __ z_brne(done);
1782           __ z_lghi(dreg, 1);           // min_jlong / min_jlong = 1
1783           __ bind(done);
1784         } else {
1785           assert_different_registers(dreg, lreg);
1786           NearLabel done;
1787           __ z_lghi(dreg, 0);
1788           __ compare64_and_branch(lreg, min_jlong, Assembler::bcondNotEqual, done);
1789           __ z_lghi(dreg, 1);
1790           __ bind(done);
1791         }
1792         return;
1793       }
1794       __ move_reg_if_needed(dreg, T_LONG, lreg, T_LONG);
1795       if (divisor == 2) {
1796         __ z_srlg(treg2, dreg, 63);     // dividend < 0 ? 1 : 0
1797       } else {
1798         __ z_srag(treg2, dreg, 63);     // dividend < 0 ? -1 : 0
1799         __ and_imm(treg2, divisor - 1, treg1, true);
1800       }
1801       if (code == lir_idiv) {
1802         __ z_agr(dreg, treg2);
1803         __ z_srag(dreg, dreg, log_divisor);
1804       } else {
1805         assert(code == lir_irem, "check");
1806         __ z_agr(treg2, dreg);
1807         __ and_imm(treg2, ~(divisor - 1), treg1, true);
1808         __ z_sgr(dreg, treg2);
1809       }
1810       return;
1811     }
1812 
1813     // Divisor is not a power of 2 constant.
1814     Register rreg = right->as_register_lo();
1815     Register treg = temp->as_register_lo();
1816     assert(right->is_double_cpu(), "right must be register");
1817     assert(lreg == Z_R11, "see ldivInOpr()");
1818     assert(rreg != lreg, "right register must not be same as left register");
1819     assert((code == lir_idiv && dreg == Z_R11 && treg == Z_R10) ||
1820            (code == lir_irem && dreg == Z_R10 && treg == Z_R11), "see ldivInOpr(), ldivOutOpr(), lremOutOpr()");
1821 
1822     Register R1 = lreg->predecessor();
1823     Register R2 = rreg;
1824     assert(code != lir_idiv || lreg==dreg, "see code below");
1825     if (code == lir_idiv) {
1826       __ z_lcgr(lreg, lreg);
1827     } else {
1828       __ clear_reg(dreg, true, false);
1829     }
1830     NearLabel done;
1831     __ compare64_and_branch(R2, -1, Assembler::bcondEqual, done);
1832     if (code == lir_idiv) {
1833       __ z_lcgr(lreg, lreg); // Revert lcgr above.
1834     }
1835     if (ImplicitDiv0Checks) {
1836       // No debug info because the idiv won't trap.
1837       // Add_debug_info_for_div0 would instantiate another DivByZeroStub,
1838       // which is unnecessary, too.
1839       add_debug_info_for_div0(__ offset(), info);
1840     }
1841     __ z_dsgr(R1, R2);
1842     __ bind(done);
1843     return;
1844   }
1845 
1846   // 32 bit integer case
1847 
1848   assert(left->is_single_cpu(), "left must be register");
1849   assert(right->is_single_cpu() || is_power_of_2(right->as_jint()), "right must be register or power of 2 constant");
1850   assert(result->is_single_cpu(), "result must be register");
1851 
1852   Register lreg = left->as_register();
1853   Register dreg = result->as_register();
1854 
1855   if (right->is_constant()) {
1856     // Convert division by a power of two into some shifts and logical operations.
1857     Register treg1 = Z_R0_scratch;
1858     Register treg2 = Z_R1_scratch;
1859     jlong divisor = right->as_jint();
1860     jlong log_divisor = log2i_exact(right->as_jint());
1861     __ move_reg_if_needed(dreg, T_LONG, lreg, T_INT); // sign extend
1862     if (divisor == 2) {
1863       __ z_srlg(treg2, dreg, 63);     // dividend < 0 ?  1 : 0
1864     } else {
1865       __ z_srag(treg2, dreg, 63);     // dividend < 0 ? -1 : 0
1866       __ and_imm(treg2, divisor - 1, treg1, true);
1867     }
1868     if (code == lir_idiv) {
1869       __ z_agr(dreg, treg2);
1870       __ z_srag(dreg, dreg, log_divisor);
1871     } else {
1872       assert(code == lir_irem, "check");
1873       __ z_agr(treg2, dreg);
1874       __ and_imm(treg2, ~(divisor - 1), treg1, true);
1875       __ z_sgr(dreg, treg2);
1876     }
1877     return;
1878   }
1879 
1880   // Divisor is not a power of 2 constant.
1881   Register rreg = right->as_register();
1882   Register treg = temp->as_register();
1883   assert(right->is_single_cpu(), "right must be register");
1884   assert(lreg == Z_R11, "left register must be rax,");
1885   assert(rreg != lreg, "right register must not be same as left register");
1886   assert((code == lir_idiv && dreg == Z_R11 && treg == Z_R10)
1887       || (code == lir_irem && dreg == Z_R10 && treg == Z_R11), "see divInOpr(), divOutOpr(), remOutOpr()");
1888 
1889   Register R1 = lreg->predecessor();
1890   Register R2 = rreg;
1891   __ move_reg_if_needed(lreg, T_LONG, lreg, T_INT); // sign extend
1892   if (ImplicitDiv0Checks) {
1893     // No debug info because the idiv won't trap.
1894     // Add_debug_info_for_div0 would instantiate another DivByZeroStub,
1895     // which is unnecessary, too.
1896     add_debug_info_for_div0(__ offset(), info);
1897   }
1898   __ z_dsgfr(R1, R2);
1899 }
1900 
1901 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1902   assert(exceptionOop->as_register() == Z_EXC_OOP, "should match");
1903   assert(exceptionPC->as_register() == Z_EXC_PC, "should match");
1904 
1905   // Exception object is not added to oop map by LinearScan
1906   // (LinearScan assumes that no oops are in fixed registers).
1907   info->add_register_oop(exceptionOop);
1908 
1909   // Reuse the debug info from the safepoint poll for the throw op itself.
1910   __ get_PC(Z_EXC_PC);
1911   add_call_info(__ offset(), info); // for exception handler
1912   address stub = Runtime1::entry_for (compilation()->has_fpu_code() ? Runtime1::handle_exception_id
1913                                                                     : Runtime1::handle_exception_nofpu_id);
1914   emit_call_c(stub);
1915 }
1916 
1917 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1918   assert(exceptionOop->as_register() == Z_EXC_OOP, "should match");
1919 
1920   __ branch_optimized(Assembler::bcondAlways, _unwind_handler_entry);
1921 }
1922 
1923 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
1924   ciArrayKlass* default_type = op->expected_type();
1925   Register src = op->src()->as_register();
1926   Register dst = op->dst()->as_register();
1927   Register src_pos = op->src_pos()->as_register();
1928   Register dst_pos = op->dst_pos()->as_register();
1929   Register length  = op->length()->as_register();
1930   Register tmp = op->tmp()->as_register();
1931 
1932   CodeStub* stub = op->stub();
1933   int flags = op->flags();
1934   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
1935   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
1936 
1937   // If we don't know anything, just go through the generic arraycopy.
1938   if (default_type == NULL) {
1939     address copyfunc_addr = StubRoutines::generic_arraycopy();
1940 
1941     if (copyfunc_addr == NULL) {
1942       // Take a slow path for generic arraycopy.
1943       __ branch_optimized(Assembler::bcondAlways, *stub->entry());
1944       __ bind(*stub->continuation());
1945       return;
1946     }
1947 
1948     // Save outgoing arguments in callee saved registers (C convention) in case
1949     // a call to System.arraycopy is needed.
1950     Register callee_saved_src     = Z_R10;
1951     Register callee_saved_src_pos = Z_R11;
1952     Register callee_saved_dst     = Z_R12;
1953     Register callee_saved_dst_pos = Z_R13;
1954     Register callee_saved_length  = Z_ARG5; // Z_ARG5 == Z_R6 is callee saved.
1955 
1956     __ lgr_if_needed(callee_saved_src, src);
1957     __ lgr_if_needed(callee_saved_src_pos, src_pos);
1958     __ lgr_if_needed(callee_saved_dst, dst);
1959     __ lgr_if_needed(callee_saved_dst_pos, dst_pos);
1960     __ lgr_if_needed(callee_saved_length, length);
1961 
1962     // C function requires 64 bit values.
1963     __ z_lgfr(src_pos, src_pos);
1964     __ z_lgfr(dst_pos, dst_pos);
1965     __ z_lgfr(length, length);
1966 
1967     // Pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint.
1968 
1969     // The arguments are in the corresponding registers.
1970     assert(Z_ARG1 == src,     "assumption");
1971     assert(Z_ARG2 == src_pos, "assumption");
1972     assert(Z_ARG3 == dst,     "assumption");
1973     assert(Z_ARG4 == dst_pos, "assumption");
1974     assert(Z_ARG5 == length,  "assumption");
1975 #ifndef PRODUCT
1976     if (PrintC1Statistics) {
1977       __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_generic_arraycopystub_cnt);
1978       __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
1979     }
1980 #endif
1981     emit_call_c(copyfunc_addr);
1982     CHECK_BAILOUT();
1983 
1984     __ compare32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondEqual, *stub->continuation());
1985 
1986     __ z_lgr(tmp, Z_RET);
1987     __ z_xilf(tmp, -1);
1988 
1989     // Restore values from callee saved registers so they are where the stub
1990     // expects them.
1991     __ lgr_if_needed(src, callee_saved_src);
1992     __ lgr_if_needed(src_pos, callee_saved_src_pos);
1993     __ lgr_if_needed(dst, callee_saved_dst);
1994     __ lgr_if_needed(dst_pos, callee_saved_dst_pos);
1995     __ lgr_if_needed(length, callee_saved_length);
1996 
1997     __ z_sr(length, tmp);
1998     __ z_ar(src_pos, tmp);
1999     __ z_ar(dst_pos, tmp);
2000     __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2001 
2002     __ bind(*stub->continuation());
2003     return;
2004   }
2005 
2006   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2007 
2008   int elem_size = type2aelembytes(basic_type);
2009   int shift_amount;
2010 
2011   switch (elem_size) {
2012     case 1 :
2013       shift_amount = 0;
2014       break;
2015     case 2 :
2016       shift_amount = 1;
2017       break;
2018     case 4 :
2019       shift_amount = 2;
2020       break;
2021     case 8 :
2022       shift_amount = 3;
2023       break;
2024     default:
2025       shift_amount = -1;
2026       ShouldNotReachHere();
2027   }
2028 
2029   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2030   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2031   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2032   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2033 
2034   // Length and pos's are all sign extended at this point on 64bit.
2035 
2036   // test for NULL
2037   if (flags & LIR_OpArrayCopy::src_null_check) {
2038     __ compareU64_and_branch(src, (intptr_t)0, Assembler::bcondZero, *stub->entry());
2039   }
2040   if (flags & LIR_OpArrayCopy::dst_null_check) {
2041     __ compareU64_and_branch(dst, (intptr_t)0, Assembler::bcondZero, *stub->entry());
2042   }
2043 
2044   // Check if negative.
2045   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2046     __ compare32_and_branch(src_pos, (intptr_t)0, Assembler::bcondLow, *stub->entry());
2047   }
2048   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2049     __ compare32_and_branch(dst_pos, (intptr_t)0, Assembler::bcondLow, *stub->entry());
2050   }
2051 
2052   // If the compiler was not able to prove that exact type of the source or the destination
2053   // of the arraycopy is an array type, check at runtime if the source or the destination is
2054   // an instance type.
2055   if (flags & LIR_OpArrayCopy::type_check) {
2056     assert(Klass::_lh_neutral_value == 0, "or replace z_lt instructions");
2057 
2058     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2059       __ load_klass(tmp, dst);
2060       __ z_lt(tmp, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2061       __ branch_optimized(Assembler::bcondNotLow, *stub->entry());
2062     }
2063 
2064     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2065       __ load_klass(tmp, src);
2066       __ z_lt(tmp, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2067       __ branch_optimized(Assembler::bcondNotLow, *stub->entry());
2068     }
2069   }
2070 
2071   if (flags & LIR_OpArrayCopy::src_range_check) {
2072     __ z_la(tmp, Address(src_pos, length));
2073     __ z_cl(tmp, src_length_addr);
2074     __ branch_optimized(Assembler::bcondHigh, *stub->entry());
2075   }
2076   if (flags & LIR_OpArrayCopy::dst_range_check) {
2077     __ z_la(tmp, Address(dst_pos, length));
2078     __ z_cl(tmp, dst_length_addr);
2079     __ branch_optimized(Assembler::bcondHigh, *stub->entry());
2080   }
2081 
2082   if (flags & LIR_OpArrayCopy::length_positive_check) {
2083     __ z_ltr(length, length);
2084     __ branch_optimized(Assembler::bcondNegative, *stub->entry());
2085   }
2086 
2087   // Stubs require 64 bit values.
2088   __ z_lgfr(src_pos, src_pos); // int -> long
2089   __ z_lgfr(dst_pos, dst_pos); // int -> long
2090   __ z_lgfr(length, length);   // int -> long
2091 
2092   if (flags & LIR_OpArrayCopy::type_check) {
2093     // We don't know the array types are compatible.
2094     if (basic_type != T_OBJECT) {
2095       // Simple test for basic type arrays.
2096       if (UseCompressedClassPointers) {
2097         __ z_l(tmp, src_klass_addr);
2098         __ z_c(tmp, dst_klass_addr);
2099       } else {
2100         __ z_lg(tmp, src_klass_addr);
2101         __ z_cg(tmp, dst_klass_addr);
2102       }
2103       __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
2104     } else {
2105       // For object arrays, if src is a sub class of dst then we can
2106       // safely do the copy.
2107       NearLabel cont, slow;
2108       Register src_klass = Z_R1_scratch;
2109       Register dst_klass = Z_R10;
2110 
2111       __ load_klass(src_klass, src);
2112       __ load_klass(dst_klass, dst);
2113 
2114       __ check_klass_subtype_fast_path(src_klass, dst_klass, tmp, &cont, &slow, NULL);
2115 
2116       store_parameter(src_klass, 0); // sub
2117       store_parameter(dst_klass, 1); // super
2118       emit_call_c(Runtime1::entry_for (Runtime1::slow_subtype_check_id));
2119       CHECK_BAILOUT2(cont, slow);
2120       // Sets condition code 0 for match (2 otherwise).
2121       __ branch_optimized(Assembler::bcondEqual, cont);
2122 
2123       __ bind(slow);
2124 
2125       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2126       if (copyfunc_addr != NULL) { // use stub if available
2127         // Src is not a sub class of dst so we have to do a
2128         // per-element check.
2129 
2130         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2131         if ((flags & mask) != mask) {
2132           // Check that at least both of them object arrays.
2133           assert(flags & mask, "one of the two should be known to be an object array");
2134 
2135           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2136             __ load_klass(tmp, src);
2137           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2138             __ load_klass(tmp, dst);
2139           }
2140           Address klass_lh_addr(tmp, Klass::layout_helper_offset());
2141           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2142           __ load_const_optimized(Z_R1_scratch, objArray_lh);
2143           __ z_c(Z_R1_scratch, klass_lh_addr);
2144           __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
2145         }
2146 
2147         // Save outgoing arguments in callee saved registers (C convention) in case
2148         // a call to System.arraycopy is needed.
2149         Register callee_saved_src     = Z_R10;
2150         Register callee_saved_src_pos = Z_R11;
2151         Register callee_saved_dst     = Z_R12;
2152         Register callee_saved_dst_pos = Z_R13;
2153         Register callee_saved_length  = Z_ARG5; // Z_ARG5 == Z_R6 is callee saved.
2154 
2155         __ lgr_if_needed(callee_saved_src, src);
2156         __ lgr_if_needed(callee_saved_src_pos, src_pos);
2157         __ lgr_if_needed(callee_saved_dst, dst);
2158         __ lgr_if_needed(callee_saved_dst_pos, dst_pos);
2159         __ lgr_if_needed(callee_saved_length, length);
2160 
2161         __ z_llgfr(length, length); // Higher 32bits must be null.
2162 
2163         __ z_sllg(Z_ARG1, src_pos, shift_amount); // index -> byte offset
2164         __ z_sllg(Z_ARG2, dst_pos, shift_amount); // index -> byte offset
2165 
2166         __ z_la(Z_ARG1, Address(src, Z_ARG1, arrayOopDesc::base_offset_in_bytes(basic_type)));
2167         assert_different_registers(Z_ARG1, dst, dst_pos, length);
2168         __ z_la(Z_ARG2, Address(dst, Z_ARG2, arrayOopDesc::base_offset_in_bytes(basic_type)));
2169         assert_different_registers(Z_ARG2, dst, length);
2170 
2171         __ z_lgr(Z_ARG3, length);
2172         assert_different_registers(Z_ARG3, dst);
2173 
2174         __ load_klass(Z_ARG5, dst);
2175         __ z_lg(Z_ARG5, Address(Z_ARG5, ObjArrayKlass::element_klass_offset()));
2176         __ z_lg(Z_ARG4, Address(Z_ARG5, Klass::super_check_offset_offset()));
2177         emit_call_c(copyfunc_addr);
2178         CHECK_BAILOUT2(cont, slow);
2179 
2180 #ifndef PRODUCT
2181         if (PrintC1Statistics) {
2182           NearLabel failed;
2183           __ compareU32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondNotEqual, failed);
2184           __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_checkcast_cnt);
2185           __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2186           __ bind(failed);
2187         }
2188 #endif
2189 
2190         __ compareU32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondEqual, *stub->continuation());
2191 
2192 #ifndef PRODUCT
2193         if (PrintC1Statistics) {
2194           __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_checkcast_attempt_cnt);
2195           __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2196         }
2197 #endif
2198 
2199         __ z_lgr(tmp, Z_RET);
2200         __ z_xilf(tmp, -1);
2201 
2202         // Restore previously spilled arguments
2203         __ lgr_if_needed(src, callee_saved_src);
2204         __ lgr_if_needed(src_pos, callee_saved_src_pos);
2205         __ lgr_if_needed(dst, callee_saved_dst);
2206         __ lgr_if_needed(dst_pos, callee_saved_dst_pos);
2207         __ lgr_if_needed(length, callee_saved_length);
2208 
2209         __ z_sr(length, tmp);
2210         __ z_ar(src_pos, tmp);
2211         __ z_ar(dst_pos, tmp);
2212       }
2213 
2214       __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2215 
2216       __ bind(cont);
2217     }
2218   }
2219 
2220 #ifdef ASSERT
2221   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2222     // Sanity check the known type with the incoming class. For the
2223     // primitive case the types must match exactly with src.klass and
2224     // dst.klass each exactly matching the default type. For the
2225     // object array case, if no type check is needed then either the
2226     // dst type is exactly the expected type and the src type is a
2227     // subtype which we can't check or src is the same array as dst
2228     // but not necessarily exactly of type default_type.
2229     NearLabel known_ok, halt;
2230     metadata2reg(default_type->constant_encoding(), tmp);
2231     if (UseCompressedClassPointers) {
2232       __ encode_klass_not_null(tmp);
2233     }
2234 
2235     if (basic_type != T_OBJECT) {
2236       if (UseCompressedClassPointers)         { __ z_c (tmp, dst_klass_addr); }
2237       else                                    { __ z_cg(tmp, dst_klass_addr); }
2238       __ branch_optimized(Assembler::bcondNotEqual, halt);
2239       if (UseCompressedClassPointers)         { __ z_c (tmp, src_klass_addr); }
2240       else                                    { __ z_cg(tmp, src_klass_addr); }
2241       __ branch_optimized(Assembler::bcondEqual, known_ok);
2242     } else {
2243       if (UseCompressedClassPointers)         { __ z_c (tmp, dst_klass_addr); }
2244       else                                    { __ z_cg(tmp, dst_klass_addr); }
2245       __ branch_optimized(Assembler::bcondEqual, known_ok);
2246       __ compareU64_and_branch(src, dst, Assembler::bcondEqual, known_ok);
2247     }
2248     __ bind(halt);
2249     __ stop("incorrect type information in arraycopy");
2250     __ bind(known_ok);
2251   }
2252 #endif
2253 
2254 #ifndef PRODUCT
2255   if (PrintC1Statistics) {
2256     __ load_const_optimized(Z_R1_scratch, Runtime1::arraycopy_count_address(basic_type));
2257     __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2258   }
2259 #endif
2260 
2261   __ z_sllg(tmp, src_pos, shift_amount); // index -> byte offset
2262   __ z_sllg(Z_R1_scratch, dst_pos, shift_amount); // index -> byte offset
2263 
2264   assert_different_registers(Z_ARG1, dst, dst_pos, length);
2265   __ z_la(Z_ARG1, Address(src, tmp, arrayOopDesc::base_offset_in_bytes(basic_type)));
2266   assert_different_registers(Z_ARG2, length);
2267   __ z_la(Z_ARG2, Address(dst, Z_R1_scratch, arrayOopDesc::base_offset_in_bytes(basic_type)));
2268   __ lgr_if_needed(Z_ARG3, length);
2269 
2270   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2271   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2272   const char *name;
2273   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2274   __ call_VM_leaf(entry);
2275 
2276   __ bind(*stub->continuation());
2277 }
2278 
2279 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2280   if (dest->is_single_cpu()) {
2281     if (left->type() == T_OBJECT) {
2282       switch (code) {
2283         case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2284         case lir_shr:  __ z_srag (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2285         case lir_ushr: __ z_srlg (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2286         default: ShouldNotReachHere();
2287       }
2288     } else {
2289       assert(code == lir_shl || left == dest, "left and dest must be equal for 2 operand form right shifts");
2290       Register masked_count = Z_R1_scratch;
2291       __ z_lr(masked_count, count->as_register());
2292       __ z_nill(masked_count, 31);
2293       switch (code) {
2294         case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), 0, masked_count); break;
2295         case lir_shr:  __ z_sra  (dest->as_register(), 0, masked_count); break;
2296         case lir_ushr: __ z_srl  (dest->as_register(), 0, masked_count); break;
2297         default: ShouldNotReachHere();
2298       }
2299     }
2300   } else {
2301     switch (code) {
2302       case lir_shl:  __ z_sllg (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2303       case lir_shr:  __ z_srag (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2304       case lir_ushr: __ z_srlg (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2305       default: ShouldNotReachHere();
2306     }
2307   }
2308 }
2309 
2310 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2311   if (left->type() == T_OBJECT) {
2312     count = count & 63;  // Shouldn't shift by more than sizeof(intptr_t).
2313     Register l = left->as_register();
2314     Register d = dest->as_register_lo();
2315     switch (code) {
2316       case lir_shl:  __ z_sllg (d, l, count); break;
2317       case lir_shr:  __ z_srag (d, l, count); break;
2318       case lir_ushr: __ z_srlg (d, l, count); break;
2319       default: ShouldNotReachHere();
2320     }
2321     return;
2322   }
2323   if (dest->is_single_cpu()) {
2324     assert(code == lir_shl || left == dest, "left and dest must be equal for 2 operand form right shifts");
2325     count = count & 0x1F; // Java spec
2326     switch (code) {
2327       case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), count); break;
2328       case lir_shr:  __ z_sra  (dest->as_register(), count); break;
2329       case lir_ushr: __ z_srl  (dest->as_register(), count); break;
2330       default: ShouldNotReachHere();
2331     }
2332   } else if (dest->is_double_cpu()) {
2333     count = count & 63; // Java spec
2334     Register l = left->as_pointer_register();
2335     Register d = dest->as_pointer_register();
2336     switch (code) {
2337       case lir_shl:  __ z_sllg (d, l, count); break;
2338       case lir_shr:  __ z_srag (d, l, count); break;
2339       case lir_ushr: __ z_srlg (d, l, count); break;
2340       default: ShouldNotReachHere();
2341     }
2342   } else {
2343     ShouldNotReachHere();
2344   }
2345 }
2346 
2347 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
2348   if (op->init_check()) {
2349     // Make sure klass is initialized & doesn't have finalizer.
2350     const int state_offset = in_bytes(InstanceKlass::init_state_offset());
2351     Register iklass = op->klass()->as_register();
2352     add_debug_info_for_null_check_here(op->stub()->info());
2353     if (Immediate::is_uimm12(state_offset)) {
2354       __ z_cli(state_offset, iklass, InstanceKlass::fully_initialized);
2355     } else {
2356       __ z_cliy(state_offset, iklass, InstanceKlass::fully_initialized);
2357     }
2358     __ branch_optimized(Assembler::bcondNotEqual, *op->stub()->entry()); // Use long branch, because slow_case might be far.
2359   }
2360   __ allocate_object(op->obj()->as_register(),
2361                      op->tmp1()->as_register(),
2362                      op->tmp2()->as_register(),
2363                      op->header_size(),
2364                      op->object_size(),
2365                      op->klass()->as_register(),
2366                      *op->stub()->entry());
2367   __ bind(*op->stub()->continuation());
2368   __ verify_oop(op->obj()->as_register(), FILE_AND_LINE);
2369 }
2370 
2371 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
2372   Register len = op->len()->as_register();
2373   __ move_reg_if_needed(len, T_LONG, len, T_INT); // sign extend
2374 
2375   if (UseSlowPath ||
2376       (!UseFastNewObjectArray && (is_reference_type(op->type()))) ||
2377       (!UseFastNewTypeArray   && (!is_reference_type(op->type())))) {
2378     __ z_brul(*op->stub()->entry());
2379   } else {
2380     __ allocate_array(op->obj()->as_register(),
2381                       op->len()->as_register(),
2382                       op->tmp1()->as_register(),
2383                       op->tmp2()->as_register(),
2384                       arrayOopDesc::header_size(op->type()),
2385                       type2aelembytes(op->type()),
2386                       op->klass()->as_register(),
2387                       *op->stub()->entry());
2388   }
2389   __ bind(*op->stub()->continuation());
2390 }
2391 
2392 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
2393                                         Register recv, Register tmp1, Label* update_done) {
2394   uint i;
2395   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2396     Label next_test;
2397     // See if the receiver is receiver[n].
2398     Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
2399     __ z_cg(recv, receiver_addr);
2400     __ z_brne(next_test);
2401     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
2402     __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2403     __ branch_optimized(Assembler::bcondAlways, *update_done);
2404     __ bind(next_test);
2405   }
2406 
2407   // Didn't find receiver; find next empty slot and fill it in.
2408   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2409     Label next_test;
2410     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
2411     __ z_ltg(Z_R0_scratch, recv_addr);
2412     __ z_brne(next_test);
2413     __ z_stg(recv, recv_addr);
2414     __ load_const_optimized(tmp1, DataLayout::counter_increment);
2415     __ z_stg(tmp1, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)), mdo);
2416     __ branch_optimized(Assembler::bcondAlways, *update_done);
2417     __ bind(next_test);
2418   }
2419 }
2420 
2421 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
2422                                     ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
2423   Unimplemented();
2424 }
2425 
2426 void LIR_Assembler::store_parameter(Register r, int param_num) {
2427   assert(param_num >= 0, "invalid num");
2428   int offset_in_bytes = param_num * BytesPerWord + FrameMap::first_available_sp_in_frame;
2429   assert(offset_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2430   __ z_stg(r, offset_in_bytes, Z_SP);
2431 }
2432 
2433 void LIR_Assembler::store_parameter(jint c, int param_num) {
2434   assert(param_num >= 0, "invalid num");
2435   int offset_in_bytes = param_num * BytesPerWord + FrameMap::first_available_sp_in_frame;
2436   assert(offset_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2437   __ store_const(Address(Z_SP, offset_in_bytes), c, Z_R1_scratch, true);
2438 }
2439 
2440 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
2441   // We always need a stub for the failure case.
2442   CodeStub* stub = op->stub();
2443   Register obj = op->object()->as_register();
2444   Register k_RInfo = op->tmp1()->as_register();
2445   Register klass_RInfo = op->tmp2()->as_register();
2446   Register dst = op->result_opr()->as_register();
2447   Register Rtmp1 = Z_R1_scratch;
2448   ciKlass* k = op->klass();
2449 
2450   assert(!op->tmp3()->is_valid(), "tmp3's not needed");
2451 
2452   // Check if it needs to be profiled.
2453   ciMethodData* md = NULL;
2454   ciProfileData* data = NULL;
2455 
2456   if (op->should_profile()) {
2457     ciMethod* method = op->profiled_method();
2458     assert(method != NULL, "Should have method");
2459     int bci = op->profiled_bci();
2460     md = method->method_data_or_null();
2461     assert(md != NULL, "Sanity");
2462     data = md->bci_to_data(bci);
2463     assert(data != NULL,                "need data for type check");
2464     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2465   }
2466 
2467   // Temp operands do not overlap with inputs, if this is their last
2468   // use (end of range is exclusive), so a register conflict is possible.
2469   if (obj == k_RInfo) {
2470     k_RInfo = dst;
2471   } else if (obj == klass_RInfo) {
2472     klass_RInfo = dst;
2473   }
2474   assert_different_registers(obj, k_RInfo, klass_RInfo);
2475 
2476   if (op->should_profile()) {
2477     NearLabel not_null;
2478     __ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
2479     // Object is null; update MDO and exit.
2480     Register mdo = klass_RInfo;
2481     metadata2reg(md->constant_encoding(), mdo);
2482     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
2483     int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
2484     __ or2mem_8(data_addr, header_bits);
2485     __ branch_optimized(Assembler::bcondAlways, *obj_is_null);
2486     __ bind(not_null);
2487   } else {
2488     __ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondEqual, *obj_is_null);
2489   }
2490 
2491   NearLabel profile_cast_failure, profile_cast_success;
2492   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
2493   Label *success_target = op->should_profile() ? &profile_cast_success : success;
2494 
2495   // Patching may screw with our temporaries,
2496   // so let's do it before loading the class.
2497   if (k->is_loaded()) {
2498     metadata2reg(k->constant_encoding(), k_RInfo);
2499   } else {
2500     klass2reg_with_patching(k_RInfo, op->info_for_patch());
2501   }
2502   assert(obj != k_RInfo, "must be different");
2503 
2504   __ verify_oop(obj, FILE_AND_LINE);
2505 
2506   // Get object class.
2507   // Not a safepoint as obj null check happens earlier.
2508   if (op->fast_check()) {
2509     if (UseCompressedClassPointers) {
2510       __ load_klass(klass_RInfo, obj);
2511       __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
2512     } else {
2513       __ z_cg(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
2514       __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2515     }
2516     // Successful cast, fall through to profile or jump.
2517   } else {
2518     bool need_slow_path = !k->is_loaded() ||
2519                           ((int) k->super_check_offset() == in_bytes(Klass::secondary_super_cache_offset()));
2520     intptr_t super_check_offset = k->is_loaded() ? k->super_check_offset() : -1L;
2521     __ load_klass(klass_RInfo, obj);
2522     // Perform the fast part of the checking logic.
2523     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1,
2524                                      (need_slow_path ? success_target : NULL),
2525                                      failure_target, NULL,
2526                                      RegisterOrConstant(super_check_offset));
2527     if (need_slow_path) {
2528       // Call out-of-line instance of __ check_klass_subtype_slow_path(...):
2529       address a = Runtime1::entry_for (Runtime1::slow_subtype_check_id);
2530       store_parameter(klass_RInfo, 0); // sub
2531       store_parameter(k_RInfo, 1);     // super
2532       emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
2533       CHECK_BAILOUT2(profile_cast_failure, profile_cast_success);
2534       __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2535       // Fall through to success case.
2536     }
2537   }
2538 
2539   if (op->should_profile()) {
2540     Register mdo = klass_RInfo, recv = k_RInfo;
2541     assert_different_registers(obj, mdo, recv);
2542     __ bind(profile_cast_success);
2543     metadata2reg(md->constant_encoding(), mdo);
2544     __ load_klass(recv, obj);
2545     type_profile_helper(mdo, md, data, recv, Rtmp1, success);
2546     __ branch_optimized(Assembler::bcondAlways, *success);
2547 
2548     __ bind(profile_cast_failure);
2549     metadata2reg(md->constant_encoding(), mdo);
2550     __ add2mem_64(Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())), -(int)DataLayout::counter_increment, Rtmp1);
2551     __ branch_optimized(Assembler::bcondAlways, *failure);
2552   } else {
2553     __ branch_optimized(Assembler::bcondAlways, *success);
2554   }
2555 }
2556 
2557 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
2558   LIR_Code code = op->code();
2559   if (code == lir_store_check) {
2560     Register value = op->object()->as_register();
2561     Register array = op->array()->as_register();
2562     Register k_RInfo = op->tmp1()->as_register();
2563     Register klass_RInfo = op->tmp2()->as_register();
2564     Register Rtmp1 = Z_R1_scratch;
2565 
2566     CodeStub* stub = op->stub();
2567 
2568     // Check if it needs to be profiled.
2569     ciMethodData* md = NULL;
2570     ciProfileData* data = NULL;
2571 
2572     assert_different_registers(value, k_RInfo, klass_RInfo);
2573 
2574     if (op->should_profile()) {
2575       ciMethod* method = op->profiled_method();
2576       assert(method != NULL, "Should have method");
2577       int bci = op->profiled_bci();
2578       md = method->method_data_or_null();
2579       assert(md != NULL, "Sanity");
2580       data = md->bci_to_data(bci);
2581       assert(data != NULL,                "need data for type check");
2582       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2583     }
2584     NearLabel profile_cast_success, profile_cast_failure, done;
2585     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
2586     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
2587 
2588     if (op->should_profile()) {
2589       NearLabel not_null;
2590       __ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
2591       // Object is null; update MDO and exit.
2592       Register mdo = klass_RInfo;
2593       metadata2reg(md->constant_encoding(), mdo);
2594       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
2595       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
2596       __ or2mem_8(data_addr, header_bits);
2597       __ branch_optimized(Assembler::bcondAlways, done);
2598       __ bind(not_null);
2599     } else {
2600       __ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondEqual, done);
2601     }
2602 
2603     add_debug_info_for_null_check_here(op->info_for_exception());
2604     __ load_klass(k_RInfo, array);
2605     __ load_klass(klass_RInfo, value);
2606 
2607     // Get instance klass (it's already uncompressed).
2608     __ z_lg(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2609     // Perform the fast part of the checking logic.
2610     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2611     // Call out-of-line instance of __ check_klass_subtype_slow_path(...):
2612     address a = Runtime1::entry_for (Runtime1::slow_subtype_check_id);
2613     store_parameter(klass_RInfo, 0); // sub
2614     store_parameter(k_RInfo, 1);     // super
2615     emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
2616     CHECK_BAILOUT3(profile_cast_success, profile_cast_failure, done);
2617     __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2618     // Fall through to success case.
2619 
2620     if (op->should_profile()) {
2621       Register mdo = klass_RInfo, recv = k_RInfo;
2622       assert_different_registers(value, mdo, recv);
2623       __ bind(profile_cast_success);
2624       metadata2reg(md->constant_encoding(), mdo);
2625       __ load_klass(recv, value);
2626       type_profile_helper(mdo, md, data, recv, Rtmp1, &done);
2627       __ branch_optimized(Assembler::bcondAlways, done);
2628 
2629       __ bind(profile_cast_failure);
2630       metadata2reg(md->constant_encoding(), mdo);
2631       __ add2mem_64(Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())), -(int)DataLayout::counter_increment, Rtmp1);
2632       __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2633     }
2634 
2635     __ bind(done);
2636   } else {
2637     if (code == lir_checkcast) {
2638       Register obj = op->object()->as_register();
2639       Register dst = op->result_opr()->as_register();
2640       NearLabel success;
2641       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
2642       __ bind(success);
2643       __ lgr_if_needed(dst, obj);
2644     } else {
2645       if (code == lir_instanceof) {
2646         Register obj = op->object()->as_register();
2647         Register dst = op->result_opr()->as_register();
2648         NearLabel success, failure, done;
2649         emit_typecheck_helper(op, &success, &failure, &failure);
2650         __ bind(failure);
2651         __ clear_reg(dst);
2652         __ branch_optimized(Assembler::bcondAlways, done);
2653         __ bind(success);
2654         __ load_const_optimized(dst, 1);
2655         __ bind(done);
2656       } else {
2657         ShouldNotReachHere();
2658       }
2659     }
2660   }
2661 }
2662 
2663 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2664   Register addr = op->addr()->as_pointer_register();
2665   Register t1_cmp = Z_R1_scratch;
2666   if (op->code() == lir_cas_long) {
2667     assert(VM_Version::supports_cx8(), "wrong machine");
2668     Register cmp_value_lo = op->cmp_value()->as_register_lo();
2669     Register new_value_lo = op->new_value()->as_register_lo();
2670     __ z_lgr(t1_cmp, cmp_value_lo);
2671     // Perform the compare and swap operation.
2672     __ z_csg(t1_cmp, new_value_lo, 0, addr);
2673   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
2674     Register cmp_value = op->cmp_value()->as_register();
2675     Register new_value = op->new_value()->as_register();
2676     if (op->code() == lir_cas_obj) {
2677       if (UseCompressedOops) {
2678                  t1_cmp = op->tmp1()->as_register();
2679         Register t2_new = op->tmp2()->as_register();
2680         assert_different_registers(cmp_value, new_value, addr, t1_cmp, t2_new);
2681         __ oop_encoder(t1_cmp, cmp_value, true /*maybe null*/);
2682         __ oop_encoder(t2_new, new_value, true /*maybe null*/);
2683         __ z_cs(t1_cmp, t2_new, 0, addr);
2684       } else {
2685         __ z_lgr(t1_cmp, cmp_value);
2686         __ z_csg(t1_cmp, new_value, 0, addr);
2687       }
2688     } else {
2689       __ z_lr(t1_cmp, cmp_value);
2690       __ z_cs(t1_cmp, new_value, 0, addr);
2691     }
2692   } else {
2693     ShouldNotReachHere(); // new lir_cas_??
2694   }
2695 }
2696 
2697 void LIR_Assembler::breakpoint() {
2698   Unimplemented();
2699   //  __ breakpoint_trap();
2700 }
2701 
2702 void LIR_Assembler::push(LIR_Opr opr) {
2703   ShouldNotCallThis(); // unused
2704 }
2705 
2706 void LIR_Assembler::pop(LIR_Opr opr) {
2707   ShouldNotCallThis(); // unused
2708 }
2709 
2710 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst_opr) {
2711   Address addr = frame_map()->address_for_monitor_object(monitor_no);
2712   __ add2reg(dst_opr->as_register(), addr.disp(), addr.base());
2713 }
2714 
2715 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2716   Register obj = op->obj_opr()->as_register();  // May not be an oop.
2717   Register hdr = op->hdr_opr()->as_register();
2718   Register lock = op->lock_opr()->as_register();
2719   if (op->info() != NULL) {
2720     add_debug_info_for_null_check_here(op->info());
2721     __ null_check(obj);
2722   }
2723   __ branch_optimized(Assembler::bcondAlways, *op->stub()->entry());
2724   __ bind(*op->stub()->continuation());
2725 }
2726 
2727 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2728   Register obj = op->obj()->as_pointer_register();
2729   Register result = op->result_opr()->as_pointer_register();
2730 
2731   CodeEmitInfo* info = op->info();
2732   if (info != NULL) {
2733     add_debug_info_for_null_check_here(info);
2734   }
2735 
2736   if (UseCompressedClassPointers) {
2737     __ z_llgf(result, Address(obj, oopDesc::klass_offset_in_bytes()));
2738     __ decode_klass_not_null(result);
2739   } else {
2740     __ z_lg(result, Address(obj, oopDesc::klass_offset_in_bytes()));
2741   }
2742 }
2743 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2744   ciMethod* method = op->profiled_method();
2745   int bci          = op->profiled_bci();
2746   ciMethod* callee = op->profiled_callee();
2747 
2748   // Update counter for all call types.
2749   ciMethodData* md = method->method_data_or_null();
2750   assert(md != NULL, "Sanity");
2751   ciProfileData* data = md->bci_to_data(bci);
2752   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2753   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2754   Register mdo  = op->mdo()->as_register();
2755   assert(op->tmp1()->is_double_cpu(), "tmp1 must be allocated");
2756   Register tmp1 = op->tmp1()->as_register_lo();
2757   metadata2reg(md->constant_encoding(), mdo);
2758 
2759   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2760   // Perform additional virtual call profiling for invokevirtual and
2761   // invokeinterface bytecodes
2762   if (op->should_profile_receiver_type()) {
2763     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2764     Register recv = op->recv()->as_register();
2765     assert_different_registers(mdo, tmp1, recv);
2766     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2767     ciKlass* known_klass = op->known_holder();
2768     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2769       // We know the type that will be seen at this call site; we can
2770       // statically update the MethodData* rather than needing to do
2771       // dynamic tests on the receiver type.
2772 
2773       // NOTE: we should probably put a lock around this search to
2774       // avoid collisions by concurrent compilations.
2775       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2776       uint i;
2777       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2778         ciKlass* receiver = vc_data->receiver(i);
2779         if (known_klass->equals(receiver)) {
2780           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2781           __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2782           return;
2783         }
2784       }
2785 
2786       // Receiver type not found in profile data. Select an empty slot.
2787 
2788       // Note that this is less efficient than it should be because it
2789       // always does a write to the receiver part of the
2790       // VirtualCallData rather than just the first time.
2791       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2792         ciKlass* receiver = vc_data->receiver(i);
2793         if (receiver == NULL) {
2794           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2795           metadata2reg(known_klass->constant_encoding(), tmp1);
2796           __ z_stg(tmp1, recv_addr);
2797           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2798           __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2799           return;
2800         }
2801       }
2802     } else {
2803       __ load_klass(recv, recv);
2804       NearLabel update_done;
2805       type_profile_helper(mdo, md, data, recv, tmp1, &update_done);
2806       // Receiver did not match any saved receiver and there is no empty row for it.
2807       // Increment total counter to indicate polymorphic case.
2808       __ add2mem_64(counter_addr, DataLayout::counter_increment, tmp1);
2809       __ bind(update_done);
2810     }
2811   } else {
2812     // static call
2813     __ add2mem_64(counter_addr, DataLayout::counter_increment, tmp1);
2814   }
2815 }
2816 
2817 void LIR_Assembler::align_backward_branch_target() {
2818   __ align(OptoLoopAlignment);
2819 }
2820 
2821 void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
2822   ShouldNotCallThis(); // There are no delay slots on ZARCH_64.
2823 }
2824 
2825 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2826   // tmp must be unused
2827   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2828   assert(left->is_register(), "can only handle registers");
2829 
2830   if (left->is_single_cpu()) {
2831     __ z_lcr(dest->as_register(), left->as_register());
2832   } else if (left->is_single_fpu()) {
2833     __ z_lcebr(dest->as_float_reg(), left->as_float_reg());
2834   } else if (left->is_double_fpu()) {
2835     __ z_lcdbr(dest->as_double_reg(), left->as_double_reg());
2836   } else {
2837     assert(left->is_double_cpu(), "Must be a long");
2838     __ z_lcgr(dest->as_register_lo(), left->as_register_lo());
2839   }
2840 }
2841 
2842 void LIR_Assembler::rt_call(LIR_Opr result, address dest,
2843                             const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2844   assert(!tmp->is_valid(), "don't need temporary");
2845   emit_call_c(dest);
2846   CHECK_BAILOUT();
2847   if (info != NULL) {
2848     add_call_info_here(info);
2849   }
2850 }
2851 
2852 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2853   ShouldNotCallThis(); // not needed on ZARCH_64
2854 }
2855 
2856 void LIR_Assembler::membar() {
2857   __ z_fence();
2858 }
2859 
2860 void LIR_Assembler::membar_acquire() {
2861   __ z_acquire();
2862 }
2863 
2864 void LIR_Assembler::membar_release() {
2865   __ z_release();
2866 }
2867 
2868 void LIR_Assembler::membar_loadload() {
2869   __ z_acquire();
2870 }
2871 
2872 void LIR_Assembler::membar_storestore() {
2873   __ z_release();
2874 }
2875 
2876 void LIR_Assembler::membar_loadstore() {
2877   __ z_acquire();
2878 }
2879 
2880 void LIR_Assembler::membar_storeload() {
2881   __ z_fence();
2882 }
2883 
2884 void LIR_Assembler::on_spin_wait() {
2885   Unimplemented();
2886 }
2887 
2888 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2889   assert(patch_code == lir_patch_none, "Patch code not supported");
2890   LIR_Address* addr = addr_opr->as_address_ptr();
2891   assert(addr->scale() == LIR_Address::times_1, "scaling unsupported");
2892   __ load_address(dest->as_pointer_register(), as_Address(addr));
2893 }
2894 
2895 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2896   ShouldNotCallThis(); // unused
2897 }
2898 
2899 #ifdef ASSERT
2900 // Emit run-time assertion.
2901 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2902   Unimplemented();
2903 }
2904 #endif
2905 
2906 void LIR_Assembler::peephole(LIR_List*) {
2907   // Do nothing for now.
2908 }
2909 
2910 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
2911   assert(code == lir_xadd, "lir_xchg not supported");
2912   Address src_addr = as_Address(src->as_address_ptr());
2913   Register base = src_addr.base();
2914   intptr_t disp = src_addr.disp();
2915   if (src_addr.index()->is_valid()) {
2916     // LAA and LAAG do not support index register.
2917     __ load_address(Z_R1_scratch, src_addr);
2918     base = Z_R1_scratch;
2919     disp = 0;
2920   }
2921   if (data->type() == T_INT) {
2922     __ z_laa(dest->as_register(), data->as_register(), disp, base);
2923   } else if (data->type() == T_LONG) {
2924     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
2925     __ z_laag(dest->as_register_lo(), data->as_register_lo(), disp, base);
2926   } else {
2927     ShouldNotReachHere();
2928   }
2929 }
2930 
2931 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2932   Register obj = op->obj()->as_register();
2933   Register tmp1 = op->tmp()->as_pointer_register();
2934   Register tmp2 = Z_R1_scratch;
2935   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2936   ciKlass* exact_klass = op->exact_klass();
2937   intptr_t current_klass = op->current_klass();
2938   bool not_null = op->not_null();
2939   bool no_conflict = op->no_conflict();
2940 
2941   Label update, next, none, null_seen, init_klass;
2942 
2943   bool do_null = !not_null;
2944   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2945   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2946 
2947   assert(do_null || do_update, "why are we here?");
2948   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2949 
2950   __ verify_oop(obj, FILE_AND_LINE);
2951 
2952   if (do_null || tmp1 != obj DEBUG_ONLY(|| true)) {
2953     __ z_ltgr(tmp1, obj);
2954   }
2955   if (do_null) {
2956     __ z_brnz(update);
2957     if (!TypeEntries::was_null_seen(current_klass)) {
2958       __ z_lg(tmp1, mdo_addr);
2959       __ z_oill(tmp1, TypeEntries::null_seen);
2960       __ z_stg(tmp1, mdo_addr);
2961     }
2962     if (do_update) {
2963       __ z_bru(next);
2964     }
2965   } else {
2966     __ asm_assert_ne("unexpected null obj", __LINE__);
2967   }
2968 
2969   __ bind(update);
2970 
2971   if (do_update) {
2972 #ifdef ASSERT
2973     if (exact_klass != NULL) {
2974       __ load_klass(tmp1, tmp1);
2975       metadata2reg(exact_klass->constant_encoding(), tmp2);
2976       __ z_cgr(tmp1, tmp2);
2977       __ asm_assert_eq("exact klass and actual klass differ", __LINE__);
2978     }
2979 #endif
2980 
2981     Label do_update;
2982     __ z_lg(tmp2, mdo_addr);
2983 
2984     if (!no_conflict) {
2985       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2986         if (exact_klass != NULL) {
2987           metadata2reg(exact_klass->constant_encoding(), tmp1);
2988         } else {
2989           __ load_klass(tmp1, tmp1);
2990         }
2991 
2992         // Klass seen before: nothing to do (regardless of unknown bit).
2993         __ z_lgr(Z_R0_scratch, tmp2);
2994         assert(Immediate::is_uimm(~TypeEntries::type_klass_mask, 16), "or change following instruction");
2995         __ z_nill(Z_R0_scratch, TypeEntries::type_klass_mask & 0xFFFF);
2996         __ compareU64_and_branch(Z_R0_scratch, tmp1, Assembler::bcondEqual, next);
2997 
2998         // Already unknown: Nothing to do anymore.
2999         __ z_tmll(tmp2, TypeEntries::type_unknown);
3000         __ z_brc(Assembler::bcondAllOne, next);
3001 
3002         if (TypeEntries::is_type_none(current_klass)) {
3003           __ z_lgr(Z_R0_scratch, tmp2);
3004           assert(Immediate::is_uimm(~TypeEntries::type_mask, 16), "or change following instruction");
3005           __ z_nill(Z_R0_scratch, TypeEntries::type_mask & 0xFFFF);
3006           __ compareU64_and_branch(Z_R0_scratch, (intptr_t)0, Assembler::bcondEqual, init_klass);
3007         }
3008       } else {
3009         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3010                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3011 
3012         // Already unknown: Nothing to do anymore.
3013         __ z_tmll(tmp2, TypeEntries::type_unknown);
3014         __ z_brc(Assembler::bcondAllOne, next);
3015       }
3016 
3017       // Different than before. Cannot keep accurate profile.
3018       __ z_oill(tmp2, TypeEntries::type_unknown);
3019       __ z_bru(do_update);
3020     } else {
3021       // There's a single possible klass at this profile point.
3022       assert(exact_klass != NULL, "should be");
3023       if (TypeEntries::is_type_none(current_klass)) {
3024         metadata2reg(exact_klass->constant_encoding(), tmp1);
3025         __ z_lgr(Z_R0_scratch, tmp2);
3026         assert(Immediate::is_uimm(~TypeEntries::type_klass_mask, 16), "or change following instruction");
3027         __ z_nill(Z_R0_scratch, TypeEntries::type_klass_mask & 0xFFFF);
3028         __ compareU64_and_branch(Z_R0_scratch, tmp1, Assembler::bcondEqual, next);
3029 #ifdef ASSERT
3030         {
3031           Label ok;
3032           __ z_lgr(Z_R0_scratch, tmp2);
3033           assert(Immediate::is_uimm(~TypeEntries::type_mask, 16), "or change following instruction");
3034           __ z_nill(Z_R0_scratch, TypeEntries::type_mask & 0xFFFF);
3035           __ compareU64_and_branch(Z_R0_scratch, (intptr_t)0, Assembler::bcondEqual, ok);
3036           __ stop("unexpected profiling mismatch");
3037           __ bind(ok);
3038         }
3039 #endif
3040 
3041       } else {
3042         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3043                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3044 
3045         // Already unknown: Nothing to do anymore.
3046         __ z_tmll(tmp2, TypeEntries::type_unknown);
3047         __ z_brc(Assembler::bcondAllOne, next);
3048         __ z_oill(tmp2, TypeEntries::type_unknown);
3049         __ z_bru(do_update);
3050       }
3051     }
3052 
3053     __ bind(init_klass);
3054     // Combine klass and null_seen bit (only used if (tmp & type_mask)==0).
3055     __ z_ogr(tmp2, tmp1);
3056 
3057     __ bind(do_update);
3058     __ z_stg(tmp2, mdo_addr);
3059 
3060     __ bind(next);
3061   }
3062 }
3063 
3064 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3065   assert(op->crc()->is_single_cpu(), "crc must be register");
3066   assert(op->val()->is_single_cpu(), "byte value must be register");
3067   assert(op->result_opr()->is_single_cpu(), "result must be register");
3068   Register crc = op->crc()->as_register();
3069   Register val = op->val()->as_register();
3070   Register res = op->result_opr()->as_register();
3071 
3072   assert_different_registers(val, crc, res);
3073 
3074   __ load_const_optimized(res, StubRoutines::crc_table_addr());
3075   __ kernel_crc32_singleByteReg(crc, val, res, true);
3076   __ z_lgfr(res, crc);
3077 }
3078 
3079 #undef __