1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer/long registers
 173 reg_class all_reg(RAX, RAX_H,
 174                   RDX, RDX_H,
 175                   RBP, RBP_H,
 176                   RDI, RDI_H,
 177                   RSI, RSI_H,
 178                   RCX, RCX_H,
 179                   RBX, RBX_H,
 180                   RSP, RSP_H,
 181                   R8,  R8_H,
 182                   R9,  R9_H,
 183                   R10, R10_H,
 184                   R11, R11_H,
 185                   R12, R12_H,
 186                   R13, R13_H,
 187                   R14, R14_H,
 188                   R15, R15_H);
 189 
 190 // Class for all int registers
 191 reg_class all_int_reg(RAX
 192                       RDX,
 193                       RBP,
 194                       RDI,
 195                       RSI,
 196                       RCX,
 197                       RBX,
 198                       R8,
 199                       R9,
 200                       R10,
 201                       R11,
 202                       R12,
 203                       R13,
 204                       R14);
 205 
 206 // Class for all pointer registers
 207 reg_class any_reg %{
 208   return _ANY_REG_mask;
 209 %}
 210 
 211 // Class for all pointer registers (excluding RSP)
 212 reg_class ptr_reg %{
 213   return _PTR_REG_mask;
 214 %}
 215 
 216 // Class for all pointer registers (excluding RSP and RBP)
 217 reg_class ptr_reg_no_rbp %{
 218   return _PTR_REG_NO_RBP_mask;
 219 %}
 220 
 221 // Class for all pointer registers (excluding RAX and RSP)
 222 reg_class ptr_no_rax_reg %{
 223   return _PTR_NO_RAX_REG_mask;
 224 %}
 225 
 226 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 227 reg_class ptr_no_rax_rbx_reg %{
 228   return _PTR_NO_RAX_RBX_REG_mask;
 229 %}
 230 
 231 // Class for all long registers (excluding RSP)
 232 reg_class long_reg %{
 233   return _LONG_REG_mask;
 234 %}
 235 
 236 // Class for all long registers (excluding RAX, RDX and RSP)
 237 reg_class long_no_rax_rdx_reg %{
 238   return _LONG_NO_RAX_RDX_REG_mask;
 239 %}
 240 
 241 // Class for all long registers (excluding RCX and RSP)
 242 reg_class long_no_rcx_reg %{
 243   return _LONG_NO_RCX_REG_mask;
 244 %}
 245 
 246 // Class for all int registers (excluding RSP)
 247 reg_class int_reg %{
 248   return _INT_REG_mask;
 249 %}
 250 
 251 // Class for all int registers (excluding RAX, RDX, and RSP)
 252 reg_class int_no_rax_rdx_reg %{
 253   return _INT_NO_RAX_RDX_REG_mask;
 254 %}
 255 
 256 // Class for all int registers (excluding RCX and RSP)
 257 reg_class int_no_rcx_reg %{
 258   return _INT_NO_RCX_REG_mask;
 259 %}
 260 
 261 // Singleton class for RAX pointer register
 262 reg_class ptr_rax_reg(RAX, RAX_H);
 263 
 264 // Singleton class for RBX pointer register
 265 reg_class ptr_rbx_reg(RBX, RBX_H);
 266 
 267 // Singleton class for RSI pointer register
 268 reg_class ptr_rsi_reg(RSI, RSI_H);
 269 
 270 // Singleton class for RDI pointer register
 271 reg_class ptr_rdi_reg(RDI, RDI_H);
 272 
 273 // Singleton class for stack pointer
 274 reg_class ptr_rsp_reg(RSP, RSP_H);
 275 
 276 // Singleton class for TLS pointer
 277 reg_class ptr_r15_reg(R15, R15_H);
 278 
 279 // Singleton class for RAX long register
 280 reg_class long_rax_reg(RAX, RAX_H);
 281 
 282 // Singleton class for RCX long register
 283 reg_class long_rcx_reg(RCX, RCX_H);
 284 
 285 // Singleton class for RDX long register
 286 reg_class long_rdx_reg(RDX, RDX_H);
 287 
 288 // Singleton class for RAX int register
 289 reg_class int_rax_reg(RAX);
 290 
 291 // Singleton class for RBX int register
 292 reg_class int_rbx_reg(RBX);
 293 
 294 // Singleton class for RCX int register
 295 reg_class int_rcx_reg(RCX);
 296 
 297 // Singleton class for RCX int register
 298 reg_class int_rdx_reg(RDX);
 299 
 300 // Singleton class for RCX int register
 301 reg_class int_rdi_reg(RDI);
 302 
 303 // Singleton class for instruction pointer
 304 // reg_class ip_reg(RIP);
 305 
 306 %}
 307 
 308 //----------SOURCE BLOCK-------------------------------------------------------
 309 // This is a block of C++ code which provides values, functions, and
 310 // definitions necessary in the rest of the architecture description
 311 source_hpp %{
 312 
 313 extern RegMask _ANY_REG_mask;
 314 extern RegMask _PTR_REG_mask;
 315 extern RegMask _PTR_REG_NO_RBP_mask;
 316 extern RegMask _PTR_NO_RAX_REG_mask;
 317 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
 318 extern RegMask _LONG_REG_mask;
 319 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
 320 extern RegMask _LONG_NO_RCX_REG_mask;
 321 extern RegMask _INT_REG_mask;
 322 extern RegMask _INT_NO_RAX_RDX_REG_mask;
 323 extern RegMask _INT_NO_RCX_REG_mask;
 324 
 325 extern RegMask _STACK_OR_PTR_REG_mask;
 326 extern RegMask _STACK_OR_LONG_REG_mask;
 327 extern RegMask _STACK_OR_INT_REG_mask;
 328 
 329 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
 330 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
 331 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
 332 
 333 %}
 334 
 335 source %{
 336 #define   RELOC_IMM64    Assembler::imm_operand
 337 #define   RELOC_DISP32   Assembler::disp32_operand
 338 
 339 #define __ _masm.
 340 
 341 RegMask _ANY_REG_mask;
 342 RegMask _PTR_REG_mask;
 343 RegMask _PTR_REG_NO_RBP_mask;
 344 RegMask _PTR_NO_RAX_REG_mask;
 345 RegMask _PTR_NO_RAX_RBX_REG_mask;
 346 RegMask _LONG_REG_mask;
 347 RegMask _LONG_NO_RAX_RDX_REG_mask;
 348 RegMask _LONG_NO_RCX_REG_mask;
 349 RegMask _INT_REG_mask;
 350 RegMask _INT_NO_RAX_RDX_REG_mask;
 351 RegMask _INT_NO_RCX_REG_mask;
 352 RegMask _STACK_OR_PTR_REG_mask;
 353 RegMask _STACK_OR_LONG_REG_mask;
 354 RegMask _STACK_OR_INT_REG_mask;
 355 
 356 static bool need_r12_heapbase() {
 357   return UseCompressedOops || UseCompressedClassPointers;
 358 }
 359 
 360 void reg_mask_init() {
 361   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
 362   // We derive a number of subsets from it.
 363   _ANY_REG_mask = _ALL_REG_mask;
 364 
 365   if (PreserveFramePointer) {
 366     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 367     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 368   }
 369   if (need_r12_heapbase()) {
 370     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 371     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
 372   }
 373 
 374   _PTR_REG_mask = _ANY_REG_mask;
 375   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
 376   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
 377   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
 378   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
 379 
 380   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
 381   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 382 
 383   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
 384   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 385   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 386 
 387   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
 388   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 389   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 390 
 391   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
 392   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
 393   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
 394 
 395   _LONG_REG_mask = _PTR_REG_mask;
 396   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
 397   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 398 
 399   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
 400   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 401   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 402   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 403   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
 404 
 405   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
 406   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 407   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
 408 
 409   _INT_REG_mask = _ALL_INT_REG_mask;
 410   if (PreserveFramePointer) {
 411     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 412   }
 413   if (need_r12_heapbase()) {
 414     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 415   }
 416 
 417   _STACK_OR_INT_REG_mask = _INT_REG_mask;
 418   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 419 
 420   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
 421   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 422   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 423 
 424   _INT_NO_RCX_REG_mask = _INT_REG_mask;
 425   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 426 }
 427 
 428 static bool generate_vzeroupper(Compile* C) {
 429   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 430 }
 431 
 432 static int clear_avx_size() {
 433   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 434 }
 435 
 436 // !!!!! Special hack to get all types of calls to specify the byte offset
 437 //       from the start of the call to the point where the return address
 438 //       will point.
 439 int MachCallStaticJavaNode::ret_addr_offset()
 440 {
 441   int offset = 5; // 5 bytes from start of call to where return address points
 442   offset += clear_avx_size();
 443   return offset;
 444 }
 445 
 446 int MachCallDynamicJavaNode::ret_addr_offset()
 447 {
 448   int offset = 15; // 15 bytes from start of call to where return address points
 449   offset += clear_avx_size();
 450   return offset;
 451 }
 452 
 453 int MachCallRuntimeNode::ret_addr_offset() {
 454   int offset = 13; // movq r10,#addr; callq (r10)
 455   offset += clear_avx_size();
 456   return offset;
 457 }
 458 
 459 // Indicate if the safepoint node needs the polling page as an input,
 460 // it does if the polling page is more than disp32 away.
 461 bool SafePointNode::needs_polling_address_input()
 462 {
 463   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 464 }
 465 
 466 //
 467 // Compute padding required for nodes which need alignment
 468 //
 469 
 470 // The address of the call instruction needs to be 4-byte aligned to
 471 // ensure that it does not span a cache line so that it can be patched.
 472 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 473 {
 474   current_offset += clear_avx_size(); // skip vzeroupper
 475   current_offset += 1; // skip call opcode byte
 476   return align_up(current_offset, alignment_required()) - current_offset;
 477 }
 478 
 479 // The address of the call instruction needs to be 4-byte aligned to
 480 // ensure that it does not span a cache line so that it can be patched.
 481 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 482 {
 483   current_offset += clear_avx_size(); // skip vzeroupper
 484   current_offset += 11; // skip movq instruction + call opcode byte
 485   return align_up(current_offset, alignment_required()) - current_offset;
 486 }
 487 
 488 // EMIT_RM()
 489 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 490   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 491   cbuf.insts()->emit_int8(c);
 492 }
 493 
 494 // EMIT_CC()
 495 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 496   unsigned char c = (unsigned char) (f1 | f2);
 497   cbuf.insts()->emit_int8(c);
 498 }
 499 
 500 // EMIT_OPCODE()
 501 void emit_opcode(CodeBuffer &cbuf, int code) {
 502   cbuf.insts()->emit_int8((unsigned char) code);
 503 }
 504 
 505 // EMIT_OPCODE() w/ relocation information
 506 void emit_opcode(CodeBuffer &cbuf,
 507                  int code, relocInfo::relocType reloc, int offset, int format)
 508 {
 509   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 510   emit_opcode(cbuf, code);
 511 }
 512 
 513 // EMIT_D8()
 514 void emit_d8(CodeBuffer &cbuf, int d8) {
 515   cbuf.insts()->emit_int8((unsigned char) d8);
 516 }
 517 
 518 // EMIT_D16()
 519 void emit_d16(CodeBuffer &cbuf, int d16) {
 520   cbuf.insts()->emit_int16(d16);
 521 }
 522 
 523 // EMIT_D32()
 524 void emit_d32(CodeBuffer &cbuf, int d32) {
 525   cbuf.insts()->emit_int32(d32);
 526 }
 527 
 528 // EMIT_D64()
 529 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 530   cbuf.insts()->emit_int64(d64);
 531 }
 532 
 533 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 534 void emit_d32_reloc(CodeBuffer& cbuf,
 535                     int d32,
 536                     relocInfo::relocType reloc,
 537                     int format)
 538 {
 539   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 540   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 541   cbuf.insts()->emit_int32(d32);
 542 }
 543 
 544 // emit 32 bit value and construct relocation entry from RelocationHolder
 545 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 546 #ifdef ASSERT
 547   if (rspec.reloc()->type() == relocInfo::oop_type &&
 548       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 549     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 550     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
 551   }
 552 #endif
 553   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 554   cbuf.insts()->emit_int32(d32);
 555 }
 556 
 557 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 558   address next_ip = cbuf.insts_end() + 4;
 559   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 560                  external_word_Relocation::spec(addr),
 561                  RELOC_DISP32);
 562 }
 563 
 564 
 565 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 566 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 567   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 568   cbuf.insts()->emit_int64(d64);
 569 }
 570 
 571 // emit 64 bit value and construct relocation entry from RelocationHolder
 572 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 573 #ifdef ASSERT
 574   if (rspec.reloc()->type() == relocInfo::oop_type &&
 575       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 576     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 577     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
 578   }
 579 #endif
 580   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 581   cbuf.insts()->emit_int64(d64);
 582 }
 583 
 584 // Access stack slot for load or store
 585 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 586 {
 587   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 588   if (-0x80 <= disp && disp < 0x80) {
 589     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 590     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 591     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 592   } else {
 593     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 594     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 595     emit_d32(cbuf, disp);     // Displacement // R/M byte
 596   }
 597 }
 598 
 599    // rRegI ereg, memory mem) %{    // emit_reg_mem
 600 void encode_RegMem(CodeBuffer &cbuf,
 601                    int reg,
 602                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 603 {
 604   assert(disp_reloc == relocInfo::none, "cannot have disp");
 605   int regenc = reg & 7;
 606   int baseenc = base & 7;
 607   int indexenc = index & 7;
 608 
 609   // There is no index & no scale, use form without SIB byte
 610   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 611     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 612     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 613       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 614     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 615       // If 8-bit displacement, mode 0x1
 616       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 617       emit_d8(cbuf, disp);
 618     } else {
 619       // If 32-bit displacement
 620       if (base == -1) { // Special flag for absolute address
 621         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 622         if (disp_reloc != relocInfo::none) {
 623           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 624         } else {
 625           emit_d32(cbuf, disp);
 626         }
 627       } else {
 628         // Normal base + offset
 629         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 630         if (disp_reloc != relocInfo::none) {
 631           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 632         } else {
 633           emit_d32(cbuf, disp);
 634         }
 635       }
 636     }
 637   } else {
 638     // Else, encode with the SIB byte
 639     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 640     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 641       // If no displacement
 642       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 643       emit_rm(cbuf, scale, indexenc, baseenc);
 644     } else {
 645       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 646         // If 8-bit displacement, mode 0x1
 647         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 648         emit_rm(cbuf, scale, indexenc, baseenc);
 649         emit_d8(cbuf, disp);
 650       } else {
 651         // If 32-bit displacement
 652         if (base == 0x04 ) {
 653           emit_rm(cbuf, 0x2, regenc, 0x4);
 654           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 655         } else {
 656           emit_rm(cbuf, 0x2, regenc, 0x4);
 657           emit_rm(cbuf, scale, indexenc, baseenc); // *
 658         }
 659         if (disp_reloc != relocInfo::none) {
 660           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 661         } else {
 662           emit_d32(cbuf, disp);
 663         }
 664       }
 665     }
 666   }
 667 }
 668 
 669 // This could be in MacroAssembler but it's fairly C2 specific
 670 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 671   Label exit;
 672   __ jccb(Assembler::noParity, exit);
 673   __ pushf();
 674   //
 675   // comiss/ucomiss instructions set ZF,PF,CF flags and
 676   // zero OF,AF,SF for NaN values.
 677   // Fixup flags by zeroing ZF,PF so that compare of NaN
 678   // values returns 'less than' result (CF is set).
 679   // Leave the rest of flags unchanged.
 680   //
 681   //    7 6 5 4 3 2 1 0
 682   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 683   //    0 0 1 0 1 0 1 1   (0x2B)
 684   //
 685   __ andq(Address(rsp, 0), 0xffffff2b);
 686   __ popf();
 687   __ bind(exit);
 688 }
 689 
 690 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 691   Label done;
 692   __ movl(dst, -1);
 693   __ jcc(Assembler::parity, done);
 694   __ jcc(Assembler::below, done);
 695   __ setb(Assembler::notEqual, dst);
 696   __ movzbl(dst, dst);
 697   __ bind(done);
 698 }
 699 
 700 // Math.min()    # Math.max()
 701 // --------------------------
 702 // ucomis[s/d]   #
 703 // ja   -> b     # a
 704 // jp   -> NaN   # NaN
 705 // jb   -> a     # b
 706 // je            #
 707 // |-jz -> a | b # a & b
 708 // |    -> a     #
 709 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
 710                      XMMRegister a, XMMRegister b,
 711                      XMMRegister xmmt, Register rt,
 712                      bool min, bool single) {
 713 
 714   Label nan, zero, below, above, done;
 715 
 716   if (single)
 717     __ ucomiss(a, b);
 718   else
 719     __ ucomisd(a, b);
 720 
 721   if (dst->encoding() != (min ? b : a)->encoding())
 722     __ jccb(Assembler::above, above); // CF=0 & ZF=0
 723   else
 724     __ jccb(Assembler::above, done);
 725 
 726   __ jccb(Assembler::parity, nan);  // PF=1
 727   __ jccb(Assembler::below, below); // CF=1
 728 
 729   // equal
 730   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
 731   if (single) {
 732     __ ucomiss(a, xmmt);
 733     __ jccb(Assembler::equal, zero);
 734 
 735     __ movflt(dst, a);
 736     __ jmp(done);
 737   }
 738   else {
 739     __ ucomisd(a, xmmt);
 740     __ jccb(Assembler::equal, zero);
 741 
 742     __ movdbl(dst, a);
 743     __ jmp(done);
 744   }
 745 
 746   __ bind(zero);
 747   if (min)
 748     __ vpor(dst, a, b, Assembler::AVX_128bit);
 749   else
 750     __ vpand(dst, a, b, Assembler::AVX_128bit);
 751 
 752   __ jmp(done);
 753 
 754   __ bind(above);
 755   if (single)
 756     __ movflt(dst, min ? b : a);
 757   else
 758     __ movdbl(dst, min ? b : a);
 759 
 760   __ jmp(done);
 761 
 762   __ bind(nan);
 763   if (single) {
 764     __ movl(rt, 0x7fc00000); // Float.NaN
 765     __ movdl(dst, rt);
 766   }
 767   else {
 768     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
 769     __ movdq(dst, rt);
 770   }
 771   __ jmp(done);
 772 
 773   __ bind(below);
 774   if (single)
 775     __ movflt(dst, min ? a : b);
 776   else
 777     __ movdbl(dst, min ? a : b);
 778 
 779   __ bind(done);
 780 }
 781 
 782 //=============================================================================
 783 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 784 
 785 int Compile::ConstantTable::calculate_table_base_offset() const {
 786   return 0;  // absolute addressing, no offset
 787 }
 788 
 789 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 790 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 791   ShouldNotReachHere();
 792 }
 793 
 794 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 795   // Empty encoding
 796 }
 797 
 798 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 799   return 0;
 800 }
 801 
 802 #ifndef PRODUCT
 803 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 804   st->print("# MachConstantBaseNode (empty encoding)");
 805 }
 806 #endif
 807 
 808 
 809 //=============================================================================
 810 #ifndef PRODUCT
 811 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 812   Compile* C = ra_->C;
 813 
 814   int framesize = C->frame_size_in_bytes();
 815   int bangsize = C->bang_size_in_bytes();
 816   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 817   // Remove wordSize for return addr which is already pushed.
 818   framesize -= wordSize;
 819 
 820   if (C->need_stack_bang(bangsize)) {
 821     framesize -= wordSize;
 822     st->print("# stack bang (%d bytes)", bangsize);
 823     st->print("\n\t");
 824     st->print("pushq   rbp\t# Save rbp");
 825     if (PreserveFramePointer) {
 826         st->print("\n\t");
 827         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 828     }
 829     if (framesize) {
 830       st->print("\n\t");
 831       st->print("subq    rsp, #%d\t# Create frame",framesize);
 832     }
 833   } else {
 834     st->print("subq    rsp, #%d\t# Create frame",framesize);
 835     st->print("\n\t");
 836     framesize -= wordSize;
 837     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 838     if (PreserveFramePointer) {
 839       st->print("\n\t");
 840       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 841       if (framesize > 0) {
 842         st->print("\n\t");
 843         st->print("addq    rbp, #%d", framesize);
 844       }
 845     }
 846   }
 847 
 848   if (VerifyStackAtCalls) {
 849     st->print("\n\t");
 850     framesize -= wordSize;
 851     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 852 #ifdef ASSERT
 853     st->print("\n\t");
 854     st->print("# stack alignment check");
 855 #endif
 856   }
 857   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 858     st->print("\n\t");
 859     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
 860     st->print("\n\t");
 861     st->print("je      fast_entry\t");
 862     st->print("\n\t");
 863     st->print("call    #nmethod_entry_barrier_stub\t");
 864     st->print("\n\tfast_entry:");
 865   }
 866   st->cr();
 867 }
 868 #endif
 869 
 870 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 871   Compile* C = ra_->C;
 872   MacroAssembler _masm(&cbuf);
 873 
 874   int framesize = C->frame_size_in_bytes();
 875   int bangsize = C->bang_size_in_bytes();
 876 
 877   if (C->clinit_barrier_on_entry()) {
 878     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 879     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 880 
 881     Label L_skip_barrier;
 882     Register klass = rscratch1;
 883 
 884     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
 885     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
 886 
 887     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
 888 
 889     __ bind(L_skip_barrier);
 890   }
 891 
 892   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
 893 
 894   C->set_frame_complete(cbuf.insts_size());
 895 
 896   if (C->has_mach_constant_base_node()) {
 897     // NOTE: We set the table base offset here because users might be
 898     // emitted before MachConstantBaseNode.
 899     Compile::ConstantTable& constant_table = C->constant_table();
 900     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 901   }
 902 }
 903 
 904 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 905 {
 906   return MachNode::size(ra_); // too many variables; just compute it
 907                               // the hard way
 908 }
 909 
 910 int MachPrologNode::reloc() const
 911 {
 912   return 0; // a large enough number
 913 }
 914 
 915 //=============================================================================
 916 #ifndef PRODUCT
 917 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 918 {
 919   Compile* C = ra_->C;
 920   if (generate_vzeroupper(C)) {
 921     st->print("vzeroupper");
 922     st->cr(); st->print("\t");
 923   }
 924 
 925   int framesize = C->frame_size_in_bytes();
 926   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 927   // Remove word for return adr already pushed
 928   // and RBP
 929   framesize -= 2*wordSize;
 930 
 931   if (framesize) {
 932     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 933     st->print("\t");
 934   }
 935 
 936   st->print_cr("popq    rbp");
 937   if (do_polling() && C->is_method_compilation()) {
 938     st->print("\t");
 939     if (SafepointMechanism::uses_thread_local_poll()) {
 940       st->print_cr("movq    rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 941                    "testl   rax, [rscratch1]\t"
 942                    "# Safepoint: poll for GC");
 943     } else if (Assembler::is_polling_page_far()) {
 944       st->print_cr("movq    rscratch1, #polling_page_address\n\t"
 945                    "testl   rax, [rscratch1]\t"
 946                    "# Safepoint: poll for GC");
 947     } else {
 948       st->print_cr("testl   rax, [rip + #offset_to_poll_page]\t"
 949                    "# Safepoint: poll for GC");
 950     }
 951   }
 952 }
 953 #endif
 954 
 955 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 956 {
 957   Compile* C = ra_->C;
 958   MacroAssembler _masm(&cbuf);
 959 
 960   if (generate_vzeroupper(C)) {
 961     // Clear upper bits of YMM registers when current compiled code uses
 962     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 963     __ vzeroupper();
 964   }
 965 
 966   int framesize = C->frame_size_in_bytes();
 967   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 968   // Remove word for return adr already pushed
 969   // and RBP
 970   framesize -= 2*wordSize;
 971 
 972   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 973 
 974   if (framesize) {
 975     emit_opcode(cbuf, Assembler::REX_W);
 976     if (framesize < 0x80) {
 977       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 978       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 979       emit_d8(cbuf, framesize);
 980     } else {
 981       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 982       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 983       emit_d32(cbuf, framesize);
 984     }
 985   }
 986 
 987   // popq rbp
 988   emit_opcode(cbuf, 0x58 | RBP_enc);
 989 
 990   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 991     __ reserved_stack_check();
 992   }
 993 
 994   if (do_polling() && C->is_method_compilation()) {
 995     MacroAssembler _masm(&cbuf);
 996     if (SafepointMechanism::uses_thread_local_poll()) {
 997       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
 998       __ relocate(relocInfo::poll_return_type);
 999       __ testl(rax, Address(rscratch1, 0));
1000     } else {
1001       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1002       if (Assembler::is_polling_page_far()) {
1003         __ lea(rscratch1, polling_page);
1004         __ relocate(relocInfo::poll_return_type);
1005         __ testl(rax, Address(rscratch1, 0));
1006       } else {
1007         __ testl(rax, polling_page);
1008       }
1009     }
1010   }
1011 }
1012 
1013 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1014 {
1015   return MachNode::size(ra_); // too many variables; just compute it
1016                               // the hard way
1017 }
1018 
1019 int MachEpilogNode::reloc() const
1020 {
1021   return 2; // a large enough number
1022 }
1023 
1024 const Pipeline* MachEpilogNode::pipeline() const
1025 {
1026   return MachNode::pipeline_class();
1027 }
1028 
1029 int MachEpilogNode::safepoint_offset() const
1030 {
1031   return 0;
1032 }
1033 
1034 //=============================================================================
1035 
1036 enum RC {
1037   rc_bad,
1038   rc_int,
1039   rc_float,
1040   rc_stack
1041 };
1042 
1043 static enum RC rc_class(OptoReg::Name reg)
1044 {
1045   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1046 
1047   if (OptoReg::is_stack(reg)) return rc_stack;
1048 
1049   VMReg r = OptoReg::as_VMReg(reg);
1050 
1051   if (r->is_Register()) return rc_int;
1052 
1053   assert(r->is_XMMRegister(), "must be");
1054   return rc_float;
1055 }
1056 
1057 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1058 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1059                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1060 
1061 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1062                             int stack_offset, int reg, uint ireg, outputStream* st);
1063 
1064 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1065                                       int dst_offset, uint ireg, outputStream* st) {
1066   if (cbuf) {
1067     MacroAssembler _masm(cbuf);
1068     switch (ireg) {
1069     case Op_VecS:
1070       __ movq(Address(rsp, -8), rax);
1071       __ movl(rax, Address(rsp, src_offset));
1072       __ movl(Address(rsp, dst_offset), rax);
1073       __ movq(rax, Address(rsp, -8));
1074       break;
1075     case Op_VecD:
1076       __ pushq(Address(rsp, src_offset));
1077       __ popq (Address(rsp, dst_offset));
1078       break;
1079     case Op_VecX:
1080       __ pushq(Address(rsp, src_offset));
1081       __ popq (Address(rsp, dst_offset));
1082       __ pushq(Address(rsp, src_offset+8));
1083       __ popq (Address(rsp, dst_offset+8));
1084       break;
1085     case Op_VecY:
1086       __ vmovdqu(Address(rsp, -32), xmm0);
1087       __ vmovdqu(xmm0, Address(rsp, src_offset));
1088       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1089       __ vmovdqu(xmm0, Address(rsp, -32));
1090       break;
1091     case Op_VecZ:
1092       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1093       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1094       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1095       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1096       break;
1097     default:
1098       ShouldNotReachHere();
1099     }
1100 #ifndef PRODUCT
1101   } else {
1102     switch (ireg) {
1103     case Op_VecS:
1104       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1105                 "movl    rax, [rsp + #%d]\n\t"
1106                 "movl    [rsp + #%d], rax\n\t"
1107                 "movq    rax, [rsp - #8]",
1108                 src_offset, dst_offset);
1109       break;
1110     case Op_VecD:
1111       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1112                 "popq    [rsp + #%d]",
1113                 src_offset, dst_offset);
1114       break;
1115      case Op_VecX:
1116       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1117                 "popq    [rsp + #%d]\n\t"
1118                 "pushq   [rsp + #%d]\n\t"
1119                 "popq    [rsp + #%d]",
1120                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1121       break;
1122     case Op_VecY:
1123       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1124                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1125                 "vmovdqu [rsp + #%d], xmm0\n\t"
1126                 "vmovdqu xmm0, [rsp - #32]",
1127                 src_offset, dst_offset);
1128       break;
1129     case Op_VecZ:
1130       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1131                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1132                 "vmovdqu [rsp + #%d], xmm0\n\t"
1133                 "vmovdqu xmm0, [rsp - #64]",
1134                 src_offset, dst_offset);
1135       break;
1136     default:
1137       ShouldNotReachHere();
1138     }
1139 #endif
1140   }
1141 }
1142 
1143 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1144                                        PhaseRegAlloc* ra_,
1145                                        bool do_size,
1146                                        outputStream* st) const {
1147   assert(cbuf != NULL || st  != NULL, "sanity");
1148   // Get registers to move
1149   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1150   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1151   OptoReg::Name dst_second = ra_->get_reg_second(this);
1152   OptoReg::Name dst_first = ra_->get_reg_first(this);
1153 
1154   enum RC src_second_rc = rc_class(src_second);
1155   enum RC src_first_rc = rc_class(src_first);
1156   enum RC dst_second_rc = rc_class(dst_second);
1157   enum RC dst_first_rc = rc_class(dst_first);
1158 
1159   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1160          "must move at least 1 register" );
1161 
1162   if (src_first == dst_first && src_second == dst_second) {
1163     // Self copy, no move
1164     return 0;
1165   }
1166   if (bottom_type()->isa_vect() != NULL) {
1167     uint ireg = ideal_reg();
1168     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1169     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1170     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1171       // mem -> mem
1172       int src_offset = ra_->reg2offset(src_first);
1173       int dst_offset = ra_->reg2offset(dst_first);
1174       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1175     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1176       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1177     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1178       int stack_offset = ra_->reg2offset(dst_first);
1179       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1180     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1181       int stack_offset = ra_->reg2offset(src_first);
1182       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1183     } else {
1184       ShouldNotReachHere();
1185     }
1186     return 0;
1187   }
1188   if (src_first_rc == rc_stack) {
1189     // mem ->
1190     if (dst_first_rc == rc_stack) {
1191       // mem -> mem
1192       assert(src_second != dst_first, "overlap");
1193       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1194           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1195         // 64-bit
1196         int src_offset = ra_->reg2offset(src_first);
1197         int dst_offset = ra_->reg2offset(dst_first);
1198         if (cbuf) {
1199           MacroAssembler _masm(cbuf);
1200           __ pushq(Address(rsp, src_offset));
1201           __ popq (Address(rsp, dst_offset));
1202 #ifndef PRODUCT
1203         } else {
1204           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1205                     "popq    [rsp + #%d]",
1206                      src_offset, dst_offset);
1207 #endif
1208         }
1209       } else {
1210         // 32-bit
1211         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1212         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1213         // No pushl/popl, so:
1214         int src_offset = ra_->reg2offset(src_first);
1215         int dst_offset = ra_->reg2offset(dst_first);
1216         if (cbuf) {
1217           MacroAssembler _masm(cbuf);
1218           __ movq(Address(rsp, -8), rax);
1219           __ movl(rax, Address(rsp, src_offset));
1220           __ movl(Address(rsp, dst_offset), rax);
1221           __ movq(rax, Address(rsp, -8));
1222 #ifndef PRODUCT
1223         } else {
1224           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1225                     "movl    rax, [rsp + #%d]\n\t"
1226                     "movl    [rsp + #%d], rax\n\t"
1227                     "movq    rax, [rsp - #8]",
1228                      src_offset, dst_offset);
1229 #endif
1230         }
1231       }
1232       return 0;
1233     } else if (dst_first_rc == rc_int) {
1234       // mem -> gpr
1235       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1236           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1237         // 64-bit
1238         int offset = ra_->reg2offset(src_first);
1239         if (cbuf) {
1240           MacroAssembler _masm(cbuf);
1241           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1242 #ifndef PRODUCT
1243         } else {
1244           st->print("movq    %s, [rsp + #%d]\t# spill",
1245                      Matcher::regName[dst_first],
1246                      offset);
1247 #endif
1248         }
1249       } else {
1250         // 32-bit
1251         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1252         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1253         int offset = ra_->reg2offset(src_first);
1254         if (cbuf) {
1255           MacroAssembler _masm(cbuf);
1256           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1257 #ifndef PRODUCT
1258         } else {
1259           st->print("movl    %s, [rsp + #%d]\t# spill",
1260                      Matcher::regName[dst_first],
1261                      offset);
1262 #endif
1263         }
1264       }
1265       return 0;
1266     } else if (dst_first_rc == rc_float) {
1267       // mem-> xmm
1268       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1269           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1270         // 64-bit
1271         int offset = ra_->reg2offset(src_first);
1272         if (cbuf) {
1273           MacroAssembler _masm(cbuf);
1274           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1275 #ifndef PRODUCT
1276         } else {
1277           st->print("%s  %s, [rsp + #%d]\t# spill",
1278                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1279                      Matcher::regName[dst_first],
1280                      offset);
1281 #endif
1282         }
1283       } else {
1284         // 32-bit
1285         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1286         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1287         int offset = ra_->reg2offset(src_first);
1288         if (cbuf) {
1289           MacroAssembler _masm(cbuf);
1290           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1291 #ifndef PRODUCT
1292         } else {
1293           st->print("movss   %s, [rsp + #%d]\t# spill",
1294                      Matcher::regName[dst_first],
1295                      offset);
1296 #endif
1297         }
1298       }
1299       return 0;
1300     }
1301   } else if (src_first_rc == rc_int) {
1302     // gpr ->
1303     if (dst_first_rc == rc_stack) {
1304       // gpr -> mem
1305       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1306           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1307         // 64-bit
1308         int offset = ra_->reg2offset(dst_first);
1309         if (cbuf) {
1310           MacroAssembler _masm(cbuf);
1311           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1312 #ifndef PRODUCT
1313         } else {
1314           st->print("movq    [rsp + #%d], %s\t# spill",
1315                      offset,
1316                      Matcher::regName[src_first]);
1317 #endif
1318         }
1319       } else {
1320         // 32-bit
1321         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1322         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1323         int offset = ra_->reg2offset(dst_first);
1324         if (cbuf) {
1325           MacroAssembler _masm(cbuf);
1326           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1327 #ifndef PRODUCT
1328         } else {
1329           st->print("movl    [rsp + #%d], %s\t# spill",
1330                      offset,
1331                      Matcher::regName[src_first]);
1332 #endif
1333         }
1334       }
1335       return 0;
1336     } else if (dst_first_rc == rc_int) {
1337       // gpr -> gpr
1338       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1339           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1340         // 64-bit
1341         if (cbuf) {
1342           MacroAssembler _masm(cbuf);
1343           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1344                   as_Register(Matcher::_regEncode[src_first]));
1345 #ifndef PRODUCT
1346         } else {
1347           st->print("movq    %s, %s\t# spill",
1348                      Matcher::regName[dst_first],
1349                      Matcher::regName[src_first]);
1350 #endif
1351         }
1352         return 0;
1353       } else {
1354         // 32-bit
1355         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1356         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1357         if (cbuf) {
1358           MacroAssembler _masm(cbuf);
1359           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1360                   as_Register(Matcher::_regEncode[src_first]));
1361 #ifndef PRODUCT
1362         } else {
1363           st->print("movl    %s, %s\t# spill",
1364                      Matcher::regName[dst_first],
1365                      Matcher::regName[src_first]);
1366 #endif
1367         }
1368         return 0;
1369       }
1370     } else if (dst_first_rc == rc_float) {
1371       // gpr -> xmm
1372       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1373           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1374         // 64-bit
1375         if (cbuf) {
1376           MacroAssembler _masm(cbuf);
1377           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1378 #ifndef PRODUCT
1379         } else {
1380           st->print("movdq   %s, %s\t# spill",
1381                      Matcher::regName[dst_first],
1382                      Matcher::regName[src_first]);
1383 #endif
1384         }
1385       } else {
1386         // 32-bit
1387         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1388         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1389         if (cbuf) {
1390           MacroAssembler _masm(cbuf);
1391           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1392 #ifndef PRODUCT
1393         } else {
1394           st->print("movdl   %s, %s\t# spill",
1395                      Matcher::regName[dst_first],
1396                      Matcher::regName[src_first]);
1397 #endif
1398         }
1399       }
1400       return 0;
1401     }
1402   } else if (src_first_rc == rc_float) {
1403     // xmm ->
1404     if (dst_first_rc == rc_stack) {
1405       // xmm -> mem
1406       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1407           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1408         // 64-bit
1409         int offset = ra_->reg2offset(dst_first);
1410         if (cbuf) {
1411           MacroAssembler _masm(cbuf);
1412           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1413 #ifndef PRODUCT
1414         } else {
1415           st->print("movsd   [rsp + #%d], %s\t# spill",
1416                      offset,
1417                      Matcher::regName[src_first]);
1418 #endif
1419         }
1420       } else {
1421         // 32-bit
1422         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1423         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1424         int offset = ra_->reg2offset(dst_first);
1425         if (cbuf) {
1426           MacroAssembler _masm(cbuf);
1427           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1428 #ifndef PRODUCT
1429         } else {
1430           st->print("movss   [rsp + #%d], %s\t# spill",
1431                      offset,
1432                      Matcher::regName[src_first]);
1433 #endif
1434         }
1435       }
1436       return 0;
1437     } else if (dst_first_rc == rc_int) {
1438       // xmm -> gpr
1439       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1440           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1441         // 64-bit
1442         if (cbuf) {
1443           MacroAssembler _masm(cbuf);
1444           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1445 #ifndef PRODUCT
1446         } else {
1447           st->print("movdq   %s, %s\t# spill",
1448                      Matcher::regName[dst_first],
1449                      Matcher::regName[src_first]);
1450 #endif
1451         }
1452       } else {
1453         // 32-bit
1454         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1455         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1456         if (cbuf) {
1457           MacroAssembler _masm(cbuf);
1458           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1459 #ifndef PRODUCT
1460         } else {
1461           st->print("movdl   %s, %s\t# spill",
1462                      Matcher::regName[dst_first],
1463                      Matcher::regName[src_first]);
1464 #endif
1465         }
1466       }
1467       return 0;
1468     } else if (dst_first_rc == rc_float) {
1469       // xmm -> xmm
1470       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1471           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1472         // 64-bit
1473         if (cbuf) {
1474           MacroAssembler _masm(cbuf);
1475           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1476 #ifndef PRODUCT
1477         } else {
1478           st->print("%s  %s, %s\t# spill",
1479                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1480                      Matcher::regName[dst_first],
1481                      Matcher::regName[src_first]);
1482 #endif
1483         }
1484       } else {
1485         // 32-bit
1486         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1487         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1488         if (cbuf) {
1489           MacroAssembler _masm(cbuf);
1490           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1491 #ifndef PRODUCT
1492         } else {
1493           st->print("%s  %s, %s\t# spill",
1494                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1495                      Matcher::regName[dst_first],
1496                      Matcher::regName[src_first]);
1497 #endif
1498         }
1499       }
1500       return 0;
1501     }
1502   }
1503 
1504   assert(0," foo ");
1505   Unimplemented();
1506   return 0;
1507 }
1508 
1509 #ifndef PRODUCT
1510 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1511   implementation(NULL, ra_, false, st);
1512 }
1513 #endif
1514 
1515 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1516   implementation(&cbuf, ra_, false, NULL);
1517 }
1518 
1519 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1520   return MachNode::size(ra_);
1521 }
1522 
1523 //=============================================================================
1524 #ifndef PRODUCT
1525 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1526 {
1527   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1528   int reg = ra_->get_reg_first(this);
1529   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1530             Matcher::regName[reg], offset);
1531 }
1532 #endif
1533 
1534 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1535 {
1536   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1537   int reg = ra_->get_encode(this);
1538   if (offset >= 0x80) {
1539     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1540     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1541     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1542     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1543     emit_d32(cbuf, offset);
1544   } else {
1545     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1546     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1547     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1548     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1549     emit_d8(cbuf, offset);
1550   }
1551 }
1552 
1553 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1554 {
1555   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1556   return (offset < 0x80) ? 5 : 8; // REX
1557 }
1558 
1559 //=============================================================================
1560 #ifndef PRODUCT
1561 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1562 {
1563   if (UseCompressedClassPointers) {
1564     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1565     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1566     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1567   } else {
1568     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1569                  "# Inline cache check");
1570   }
1571   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1572   st->print_cr("\tnop\t# nops to align entry point");
1573 }
1574 #endif
1575 
1576 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1577 {
1578   MacroAssembler masm(&cbuf);
1579   uint insts_size = cbuf.insts_size();
1580   if (UseCompressedClassPointers) {
1581     masm.load_klass(rscratch1, j_rarg0);
1582     masm.cmpptr(rax, rscratch1);
1583   } else {
1584     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1585   }
1586 
1587   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1588 
1589   /* WARNING these NOPs are critical so that verified entry point is properly
1590      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1591   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1592   if (OptoBreakpoint) {
1593     // Leave space for int3
1594     nops_cnt -= 1;
1595   }
1596   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1597   if (nops_cnt > 0)
1598     masm.nop(nops_cnt);
1599 }
1600 
1601 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1602 {
1603   return MachNode::size(ra_); // too many variables; just compute it
1604                               // the hard way
1605 }
1606 
1607 
1608 //=============================================================================
1609 
1610 int Matcher::regnum_to_fpu_offset(int regnum)
1611 {
1612   return regnum - 32; // The FP registers are in the second chunk
1613 }
1614 
1615 // This is UltraSparc specific, true just means we have fast l2f conversion
1616 const bool Matcher::convL2FSupported(void) {
1617   return true;
1618 }
1619 
1620 // Is this branch offset short enough that a short branch can be used?
1621 //
1622 // NOTE: If the platform does not provide any short branch variants, then
1623 //       this method should return false for offset 0.
1624 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1625   // The passed offset is relative to address of the branch.
1626   // On 86 a branch displacement is calculated relative to address
1627   // of a next instruction.
1628   offset -= br_size;
1629 
1630   // the short version of jmpConUCF2 contains multiple branches,
1631   // making the reach slightly less
1632   if (rule == jmpConUCF2_rule)
1633     return (-126 <= offset && offset <= 125);
1634   return (-128 <= offset && offset <= 127);
1635 }
1636 
1637 const bool Matcher::isSimpleConstant64(jlong value) {
1638   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1639   //return value == (int) value;  // Cf. storeImmL and immL32.
1640 
1641   // Probably always true, even if a temp register is required.
1642   return true;
1643 }
1644 
1645 // The ecx parameter to rep stosq for the ClearArray node is in words.
1646 const bool Matcher::init_array_count_is_in_bytes = false;
1647 
1648 // No additional cost for CMOVL.
1649 const int Matcher::long_cmove_cost() { return 0; }
1650 
1651 // No CMOVF/CMOVD with SSE2
1652 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1653 
1654 // Does the CPU require late expand (see block.cpp for description of late expand)?
1655 const bool Matcher::require_postalloc_expand = false;
1656 
1657 // Do we need to mask the count passed to shift instructions or does
1658 // the cpu only look at the lower 5/6 bits anyway?
1659 const bool Matcher::need_masked_shift_count = false;
1660 
1661 bool Matcher::narrow_oop_use_complex_address() {
1662   assert(UseCompressedOops, "only for compressed oops code");
1663   return (LogMinObjAlignmentInBytes <= 3);
1664 }
1665 
1666 bool Matcher::narrow_klass_use_complex_address() {
1667   assert(UseCompressedClassPointers, "only for compressed klass code");
1668   return (LogKlassAlignmentInBytes <= 3);
1669 }
1670 
1671 bool Matcher::const_oop_prefer_decode() {
1672   // Prefer ConN+DecodeN over ConP.
1673   return true;
1674 }
1675 
1676 bool Matcher::const_klass_prefer_decode() {
1677   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1678   //       or condisider the following:
1679   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1680   //return CompressedKlassPointers::base() == NULL;
1681   return true;
1682 }
1683 
1684 // Is it better to copy float constants, or load them directly from
1685 // memory?  Intel can load a float constant from a direct address,
1686 // requiring no extra registers.  Most RISCs will have to materialize
1687 // an address into a register first, so they would do better to copy
1688 // the constant from stack.
1689 const bool Matcher::rematerialize_float_constants = true; // XXX
1690 
1691 // If CPU can load and store mis-aligned doubles directly then no
1692 // fixup is needed.  Else we split the double into 2 integer pieces
1693 // and move it piece-by-piece.  Only happens when passing doubles into
1694 // C code as the Java calling convention forces doubles to be aligned.
1695 const bool Matcher::misaligned_doubles_ok = true;
1696 
1697 // No-op on amd64
1698 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1699 
1700 // Advertise here if the CPU requires explicit rounding operations to
1701 // implement the UseStrictFP mode.
1702 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1703 
1704 // Are floats conerted to double when stored to stack during deoptimization?
1705 // On x64 it is stored without convertion so we can use normal access.
1706 bool Matcher::float_in_double() { return false; }
1707 
1708 // Do ints take an entire long register or just half?
1709 const bool Matcher::int_in_long = true;
1710 
1711 // Return whether or not this register is ever used as an argument.
1712 // This function is used on startup to build the trampoline stubs in
1713 // generateOptoStub.  Registers not mentioned will be killed by the VM
1714 // call in the trampoline, and arguments in those registers not be
1715 // available to the callee.
1716 bool Matcher::can_be_java_arg(int reg)
1717 {
1718   return
1719     reg ==  RDI_num || reg == RDI_H_num ||
1720     reg ==  RSI_num || reg == RSI_H_num ||
1721     reg ==  RDX_num || reg == RDX_H_num ||
1722     reg ==  RCX_num || reg == RCX_H_num ||
1723     reg ==   R8_num || reg ==  R8_H_num ||
1724     reg ==   R9_num || reg ==  R9_H_num ||
1725     reg ==  R12_num || reg == R12_H_num ||
1726     reg == XMM0_num || reg == XMM0b_num ||
1727     reg == XMM1_num || reg == XMM1b_num ||
1728     reg == XMM2_num || reg == XMM2b_num ||
1729     reg == XMM3_num || reg == XMM3b_num ||
1730     reg == XMM4_num || reg == XMM4b_num ||
1731     reg == XMM5_num || reg == XMM5b_num ||
1732     reg == XMM6_num || reg == XMM6b_num ||
1733     reg == XMM7_num || reg == XMM7b_num;
1734 }
1735 
1736 bool Matcher::is_spillable_arg(int reg)
1737 {
1738   return can_be_java_arg(reg);
1739 }
1740 
1741 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1742   // In 64 bit mode a code which use multiply when
1743   // devisor is constant is faster than hardware
1744   // DIV instruction (it uses MulHiL).
1745   return false;
1746 }
1747 
1748 // Register for DIVI projection of divmodI
1749 RegMask Matcher::divI_proj_mask() {
1750   return INT_RAX_REG_mask();
1751 }
1752 
1753 // Register for MODI projection of divmodI
1754 RegMask Matcher::modI_proj_mask() {
1755   return INT_RDX_REG_mask();
1756 }
1757 
1758 // Register for DIVL projection of divmodL
1759 RegMask Matcher::divL_proj_mask() {
1760   return LONG_RAX_REG_mask();
1761 }
1762 
1763 // Register for MODL projection of divmodL
1764 RegMask Matcher::modL_proj_mask() {
1765   return LONG_RDX_REG_mask();
1766 }
1767 
1768 // Register for saving SP into on method handle invokes. Not used on x86_64.
1769 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1770     return NO_REG_mask();
1771 }
1772 
1773 %}
1774 
1775 //----------ENCODING BLOCK-----------------------------------------------------
1776 // This block specifies the encoding classes used by the compiler to
1777 // output byte streams.  Encoding classes are parameterized macros
1778 // used by Machine Instruction Nodes in order to generate the bit
1779 // encoding of the instruction.  Operands specify their base encoding
1780 // interface with the interface keyword.  There are currently
1781 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1782 // COND_INTER.  REG_INTER causes an operand to generate a function
1783 // which returns its register number when queried.  CONST_INTER causes
1784 // an operand to generate a function which returns the value of the
1785 // constant when queried.  MEMORY_INTER causes an operand to generate
1786 // four functions which return the Base Register, the Index Register,
1787 // the Scale Value, and the Offset Value of the operand when queried.
1788 // COND_INTER causes an operand to generate six functions which return
1789 // the encoding code (ie - encoding bits for the instruction)
1790 // associated with each basic boolean condition for a conditional
1791 // instruction.
1792 //
1793 // Instructions specify two basic values for encoding.  Again, a
1794 // function is available to check if the constant displacement is an
1795 // oop. They use the ins_encode keyword to specify their encoding
1796 // classes (which must be a sequence of enc_class names, and their
1797 // parameters, specified in the encoding block), and they use the
1798 // opcode keyword to specify, in order, their primary, secondary, and
1799 // tertiary opcode.  Only the opcode sections which a particular
1800 // instruction needs for encoding need to be specified.
1801 encode %{
1802   // Build emit functions for each basic byte or larger field in the
1803   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1804   // from C++ code in the enc_class source block.  Emit functions will
1805   // live in the main source block for now.  In future, we can
1806   // generalize this by adding a syntax that specifies the sizes of
1807   // fields in an order, so that the adlc can build the emit functions
1808   // automagically
1809 
1810   // Emit primary opcode
1811   enc_class OpcP
1812   %{
1813     emit_opcode(cbuf, $primary);
1814   %}
1815 
1816   // Emit secondary opcode
1817   enc_class OpcS
1818   %{
1819     emit_opcode(cbuf, $secondary);
1820   %}
1821 
1822   // Emit tertiary opcode
1823   enc_class OpcT
1824   %{
1825     emit_opcode(cbuf, $tertiary);
1826   %}
1827 
1828   // Emit opcode directly
1829   enc_class Opcode(immI d8)
1830   %{
1831     emit_opcode(cbuf, $d8$$constant);
1832   %}
1833 
1834   // Emit size prefix
1835   enc_class SizePrefix
1836   %{
1837     emit_opcode(cbuf, 0x66);
1838   %}
1839 
1840   enc_class reg(rRegI reg)
1841   %{
1842     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1843   %}
1844 
1845   enc_class reg_reg(rRegI dst, rRegI src)
1846   %{
1847     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1848   %}
1849 
1850   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1851   %{
1852     emit_opcode(cbuf, $opcode$$constant);
1853     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1854   %}
1855 
1856   enc_class cdql_enc(no_rax_rdx_RegI div)
1857   %{
1858     // Full implementation of Java idiv and irem; checks for
1859     // special case as described in JVM spec., p.243 & p.271.
1860     //
1861     //         normal case                           special case
1862     //
1863     // input : rax: dividend                         min_int
1864     //         reg: divisor                          -1
1865     //
1866     // output: rax: quotient  (= rax idiv reg)       min_int
1867     //         rdx: remainder (= rax irem reg)       0
1868     //
1869     //  Code sequnce:
1870     //
1871     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1872     //    5:   75 07/08                jne    e <normal>
1873     //    7:   33 d2                   xor    %edx,%edx
1874     //  [div >= 8 -> offset + 1]
1875     //  [REX_B]
1876     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1877     //    c:   74 03/04                je     11 <done>
1878     // 000000000000000e <normal>:
1879     //    e:   99                      cltd
1880     //  [div >= 8 -> offset + 1]
1881     //  [REX_B]
1882     //    f:   f7 f9                   idiv   $div
1883     // 0000000000000011 <done>:
1884 
1885     // cmp    $0x80000000,%eax
1886     emit_opcode(cbuf, 0x3d);
1887     emit_d8(cbuf, 0x00);
1888     emit_d8(cbuf, 0x00);
1889     emit_d8(cbuf, 0x00);
1890     emit_d8(cbuf, 0x80);
1891 
1892     // jne    e <normal>
1893     emit_opcode(cbuf, 0x75);
1894     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1895 
1896     // xor    %edx,%edx
1897     emit_opcode(cbuf, 0x33);
1898     emit_d8(cbuf, 0xD2);
1899 
1900     // cmp    $0xffffffffffffffff,%ecx
1901     if ($div$$reg >= 8) {
1902       emit_opcode(cbuf, Assembler::REX_B);
1903     }
1904     emit_opcode(cbuf, 0x83);
1905     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1906     emit_d8(cbuf, 0xFF);
1907 
1908     // je     11 <done>
1909     emit_opcode(cbuf, 0x74);
1910     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1911 
1912     // <normal>
1913     // cltd
1914     emit_opcode(cbuf, 0x99);
1915 
1916     // idivl (note: must be emitted by the user of this rule)
1917     // <done>
1918   %}
1919 
1920   enc_class cdqq_enc(no_rax_rdx_RegL div)
1921   %{
1922     // Full implementation of Java ldiv and lrem; checks for
1923     // special case as described in JVM spec., p.243 & p.271.
1924     //
1925     //         normal case                           special case
1926     //
1927     // input : rax: dividend                         min_long
1928     //         reg: divisor                          -1
1929     //
1930     // output: rax: quotient  (= rax idiv reg)       min_long
1931     //         rdx: remainder (= rax irem reg)       0
1932     //
1933     //  Code sequnce:
1934     //
1935     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1936     //    7:   00 00 80
1937     //    a:   48 39 d0                cmp    %rdx,%rax
1938     //    d:   75 08                   jne    17 <normal>
1939     //    f:   33 d2                   xor    %edx,%edx
1940     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1941     //   15:   74 05                   je     1c <done>
1942     // 0000000000000017 <normal>:
1943     //   17:   48 99                   cqto
1944     //   19:   48 f7 f9                idiv   $div
1945     // 000000000000001c <done>:
1946 
1947     // mov    $0x8000000000000000,%rdx
1948     emit_opcode(cbuf, Assembler::REX_W);
1949     emit_opcode(cbuf, 0xBA);
1950     emit_d8(cbuf, 0x00);
1951     emit_d8(cbuf, 0x00);
1952     emit_d8(cbuf, 0x00);
1953     emit_d8(cbuf, 0x00);
1954     emit_d8(cbuf, 0x00);
1955     emit_d8(cbuf, 0x00);
1956     emit_d8(cbuf, 0x00);
1957     emit_d8(cbuf, 0x80);
1958 
1959     // cmp    %rdx,%rax
1960     emit_opcode(cbuf, Assembler::REX_W);
1961     emit_opcode(cbuf, 0x39);
1962     emit_d8(cbuf, 0xD0);
1963 
1964     // jne    17 <normal>
1965     emit_opcode(cbuf, 0x75);
1966     emit_d8(cbuf, 0x08);
1967 
1968     // xor    %edx,%edx
1969     emit_opcode(cbuf, 0x33);
1970     emit_d8(cbuf, 0xD2);
1971 
1972     // cmp    $0xffffffffffffffff,$div
1973     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1974     emit_opcode(cbuf, 0x83);
1975     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1976     emit_d8(cbuf, 0xFF);
1977 
1978     // je     1e <done>
1979     emit_opcode(cbuf, 0x74);
1980     emit_d8(cbuf, 0x05);
1981 
1982     // <normal>
1983     // cqto
1984     emit_opcode(cbuf, Assembler::REX_W);
1985     emit_opcode(cbuf, 0x99);
1986 
1987     // idivq (note: must be emitted by the user of this rule)
1988     // <done>
1989   %}
1990 
1991   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1992   enc_class OpcSE(immI imm)
1993   %{
1994     // Emit primary opcode and set sign-extend bit
1995     // Check for 8-bit immediate, and set sign extend bit in opcode
1996     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1997       emit_opcode(cbuf, $primary | 0x02);
1998     } else {
1999       // 32-bit immediate
2000       emit_opcode(cbuf, $primary);
2001     }
2002   %}
2003 
2004   enc_class OpcSErm(rRegI dst, immI imm)
2005   %{
2006     // OpcSEr/m
2007     int dstenc = $dst$$reg;
2008     if (dstenc >= 8) {
2009       emit_opcode(cbuf, Assembler::REX_B);
2010       dstenc -= 8;
2011     }
2012     // Emit primary opcode and set sign-extend bit
2013     // Check for 8-bit immediate, and set sign extend bit in opcode
2014     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2015       emit_opcode(cbuf, $primary | 0x02);
2016     } else {
2017       // 32-bit immediate
2018       emit_opcode(cbuf, $primary);
2019     }
2020     // Emit r/m byte with secondary opcode, after primary opcode.
2021     emit_rm(cbuf, 0x3, $secondary, dstenc);
2022   %}
2023 
2024   enc_class OpcSErm_wide(rRegL dst, immI imm)
2025   %{
2026     // OpcSEr/m
2027     int dstenc = $dst$$reg;
2028     if (dstenc < 8) {
2029       emit_opcode(cbuf, Assembler::REX_W);
2030     } else {
2031       emit_opcode(cbuf, Assembler::REX_WB);
2032       dstenc -= 8;
2033     }
2034     // Emit primary opcode and set sign-extend bit
2035     // Check for 8-bit immediate, and set sign extend bit in opcode
2036     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2037       emit_opcode(cbuf, $primary | 0x02);
2038     } else {
2039       // 32-bit immediate
2040       emit_opcode(cbuf, $primary);
2041     }
2042     // Emit r/m byte with secondary opcode, after primary opcode.
2043     emit_rm(cbuf, 0x3, $secondary, dstenc);
2044   %}
2045 
2046   enc_class Con8or32(immI imm)
2047   %{
2048     // Check for 8-bit immediate, and set sign extend bit in opcode
2049     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2050       $$$emit8$imm$$constant;
2051     } else {
2052       // 32-bit immediate
2053       $$$emit32$imm$$constant;
2054     }
2055   %}
2056 
2057   enc_class opc2_reg(rRegI dst)
2058   %{
2059     // BSWAP
2060     emit_cc(cbuf, $secondary, $dst$$reg);
2061   %}
2062 
2063   enc_class opc3_reg(rRegI dst)
2064   %{
2065     // BSWAP
2066     emit_cc(cbuf, $tertiary, $dst$$reg);
2067   %}
2068 
2069   enc_class reg_opc(rRegI div)
2070   %{
2071     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2072     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2073   %}
2074 
2075   enc_class enc_cmov(cmpOp cop)
2076   %{
2077     // CMOV
2078     $$$emit8$primary;
2079     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2080   %}
2081 
2082   enc_class enc_PartialSubtypeCheck()
2083   %{
2084     Register Rrdi = as_Register(RDI_enc); // result register
2085     Register Rrax = as_Register(RAX_enc); // super class
2086     Register Rrcx = as_Register(RCX_enc); // killed
2087     Register Rrsi = as_Register(RSI_enc); // sub class
2088     Label miss;
2089     const bool set_cond_codes = true;
2090 
2091     MacroAssembler _masm(&cbuf);
2092     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2093                                      NULL, &miss,
2094                                      /*set_cond_codes:*/ true);
2095     if ($primary) {
2096       __ xorptr(Rrdi, Rrdi);
2097     }
2098     __ bind(miss);
2099   %}
2100 
2101   enc_class clear_avx %{
2102     debug_only(int off0 = cbuf.insts_size());
2103     if (generate_vzeroupper(Compile::current())) {
2104       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2105       // Clear upper bits of YMM registers when current compiled code uses
2106       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2107       MacroAssembler _masm(&cbuf);
2108       __ vzeroupper();
2109     }
2110     debug_only(int off1 = cbuf.insts_size());
2111     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2112   %}
2113 
2114   enc_class Java_To_Runtime(method meth) %{
2115     // No relocation needed
2116     MacroAssembler _masm(&cbuf);
2117     __ mov64(r10, (int64_t) $meth$$method);
2118     __ call(r10);
2119     __ post_call_nop();
2120   %}
2121 
2122   enc_class Java_To_Interpreter(method meth)
2123   %{
2124     // CALL Java_To_Interpreter
2125     // This is the instruction starting address for relocation info.
2126     cbuf.set_insts_mark();
2127     $$$emit8$primary;
2128     // CALL directly to the runtime
2129     emit_d32_reloc(cbuf,
2130                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2131                    runtime_call_Relocation::spec(),
2132                    RELOC_DISP32);
2133     __ post_call_nop();
2134   %}
2135 
2136   enc_class Java_Static_Call(method meth)
2137   %{
2138     // JAVA STATIC CALL
2139     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2140     // determine who we intended to call.
2141     MacroAssembler _masm(&cbuf);
2142     cbuf.set_insts_mark();
2143     $$$emit8$primary;
2144 
2145     if (!_method) {
2146       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2147                      runtime_call_Relocation::spec(),
2148                      RELOC_DISP32);
2149     } else {
2150       int method_index = resolved_method_index(cbuf);
2151       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2152                                                   : static_call_Relocation::spec(method_index);
2153       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2154                      rspec, RELOC_DISP32);
2155       // Emit stubs for static call.
2156       address mark = cbuf.insts_mark();
2157       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2158       if (stub == NULL) {
2159         ciEnv::current()->record_failure("CodeCache is full");
2160         return;
2161       }
2162 #if INCLUDE_AOT
2163       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2164 #endif
2165     }
2166     __ post_call_nop();
2167   %}
2168 
2169   enc_class Java_Dynamic_Call(method meth) %{
2170     MacroAssembler _masm(&cbuf);
2171     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2172     __ post_call_nop();
2173   %}
2174 
2175   enc_class Java_Compiled_Call(method meth)
2176   %{
2177     // JAVA COMPILED CALL
2178     int disp = in_bytes(Method:: from_compiled_offset());
2179 
2180     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2181     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2182 
2183     // callq *disp(%rax)
2184     MacroAssembler _masm(&cbuf);
2185     cbuf.set_insts_mark();
2186     $$$emit8$primary;
2187     if (disp < 0x80) {
2188       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2189       emit_d8(cbuf, disp); // Displacement
2190     } else {
2191       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2192       emit_d32(cbuf, disp); // Displacement
2193     }
2194     __ post_call_nop();
2195   %}
2196 
2197   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2198   %{
2199     // SAL, SAR, SHR
2200     int dstenc = $dst$$reg;
2201     if (dstenc >= 8) {
2202       emit_opcode(cbuf, Assembler::REX_B);
2203       dstenc -= 8;
2204     }
2205     $$$emit8$primary;
2206     emit_rm(cbuf, 0x3, $secondary, dstenc);
2207     $$$emit8$shift$$constant;
2208   %}
2209 
2210   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2211   %{
2212     // SAL, SAR, SHR
2213     int dstenc = $dst$$reg;
2214     if (dstenc < 8) {
2215       emit_opcode(cbuf, Assembler::REX_W);
2216     } else {
2217       emit_opcode(cbuf, Assembler::REX_WB);
2218       dstenc -= 8;
2219     }
2220     $$$emit8$primary;
2221     emit_rm(cbuf, 0x3, $secondary, dstenc);
2222     $$$emit8$shift$$constant;
2223   %}
2224 
2225   enc_class load_immI(rRegI dst, immI src)
2226   %{
2227     int dstenc = $dst$$reg;
2228     if (dstenc >= 8) {
2229       emit_opcode(cbuf, Assembler::REX_B);
2230       dstenc -= 8;
2231     }
2232     emit_opcode(cbuf, 0xB8 | dstenc);
2233     $$$emit32$src$$constant;
2234   %}
2235 
2236   enc_class load_immL(rRegL dst, immL src)
2237   %{
2238     int dstenc = $dst$$reg;
2239     if (dstenc < 8) {
2240       emit_opcode(cbuf, Assembler::REX_W);
2241     } else {
2242       emit_opcode(cbuf, Assembler::REX_WB);
2243       dstenc -= 8;
2244     }
2245     emit_opcode(cbuf, 0xB8 | dstenc);
2246     emit_d64(cbuf, $src$$constant);
2247   %}
2248 
2249   enc_class load_immUL32(rRegL dst, immUL32 src)
2250   %{
2251     // same as load_immI, but this time we care about zeroes in the high word
2252     int dstenc = $dst$$reg;
2253     if (dstenc >= 8) {
2254       emit_opcode(cbuf, Assembler::REX_B);
2255       dstenc -= 8;
2256     }
2257     emit_opcode(cbuf, 0xB8 | dstenc);
2258     $$$emit32$src$$constant;
2259   %}
2260 
2261   enc_class load_immL32(rRegL dst, immL32 src)
2262   %{
2263     int dstenc = $dst$$reg;
2264     if (dstenc < 8) {
2265       emit_opcode(cbuf, Assembler::REX_W);
2266     } else {
2267       emit_opcode(cbuf, Assembler::REX_WB);
2268       dstenc -= 8;
2269     }
2270     emit_opcode(cbuf, 0xC7);
2271     emit_rm(cbuf, 0x03, 0x00, dstenc);
2272     $$$emit32$src$$constant;
2273   %}
2274 
2275   enc_class load_immP31(rRegP dst, immP32 src)
2276   %{
2277     // same as load_immI, but this time we care about zeroes in the high word
2278     int dstenc = $dst$$reg;
2279     if (dstenc >= 8) {
2280       emit_opcode(cbuf, Assembler::REX_B);
2281       dstenc -= 8;
2282     }
2283     emit_opcode(cbuf, 0xB8 | dstenc);
2284     $$$emit32$src$$constant;
2285   %}
2286 
2287   enc_class load_immP(rRegP dst, immP src)
2288   %{
2289     int dstenc = $dst$$reg;
2290     if (dstenc < 8) {
2291       emit_opcode(cbuf, Assembler::REX_W);
2292     } else {
2293       emit_opcode(cbuf, Assembler::REX_WB);
2294       dstenc -= 8;
2295     }
2296     emit_opcode(cbuf, 0xB8 | dstenc);
2297     // This next line should be generated from ADLC
2298     if ($src->constant_reloc() != relocInfo::none) {
2299       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2300     } else {
2301       emit_d64(cbuf, $src$$constant);
2302     }
2303   %}
2304 
2305   enc_class Con32(immI src)
2306   %{
2307     // Output immediate
2308     $$$emit32$src$$constant;
2309   %}
2310 
2311   enc_class Con32F_as_bits(immF src)
2312   %{
2313     // Output Float immediate bits
2314     jfloat jf = $src$$constant;
2315     jint jf_as_bits = jint_cast(jf);
2316     emit_d32(cbuf, jf_as_bits);
2317   %}
2318 
2319   enc_class Con16(immI src)
2320   %{
2321     // Output immediate
2322     $$$emit16$src$$constant;
2323   %}
2324 
2325   // How is this different from Con32??? XXX
2326   enc_class Con_d32(immI src)
2327   %{
2328     emit_d32(cbuf,$src$$constant);
2329   %}
2330 
2331   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2332     // Output immediate memory reference
2333     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2334     emit_d32(cbuf, 0x00);
2335   %}
2336 
2337   enc_class lock_prefix()
2338   %{
2339     emit_opcode(cbuf, 0xF0); // lock
2340   %}
2341 
2342   enc_class REX_mem(memory mem)
2343   %{
2344     if ($mem$$base >= 8) {
2345       if ($mem$$index < 8) {
2346         emit_opcode(cbuf, Assembler::REX_B);
2347       } else {
2348         emit_opcode(cbuf, Assembler::REX_XB);
2349       }
2350     } else {
2351       if ($mem$$index >= 8) {
2352         emit_opcode(cbuf, Assembler::REX_X);
2353       }
2354     }
2355   %}
2356 
2357   enc_class REX_mem_wide(memory mem)
2358   %{
2359     if ($mem$$base >= 8) {
2360       if ($mem$$index < 8) {
2361         emit_opcode(cbuf, Assembler::REX_WB);
2362       } else {
2363         emit_opcode(cbuf, Assembler::REX_WXB);
2364       }
2365     } else {
2366       if ($mem$$index < 8) {
2367         emit_opcode(cbuf, Assembler::REX_W);
2368       } else {
2369         emit_opcode(cbuf, Assembler::REX_WX);
2370       }
2371     }
2372   %}
2373 
2374   // for byte regs
2375   enc_class REX_breg(rRegI reg)
2376   %{
2377     if ($reg$$reg >= 4) {
2378       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2379     }
2380   %}
2381 
2382   // for byte regs
2383   enc_class REX_reg_breg(rRegI dst, rRegI src)
2384   %{
2385     if ($dst$$reg < 8) {
2386       if ($src$$reg >= 4) {
2387         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2388       }
2389     } else {
2390       if ($src$$reg < 8) {
2391         emit_opcode(cbuf, Assembler::REX_R);
2392       } else {
2393         emit_opcode(cbuf, Assembler::REX_RB);
2394       }
2395     }
2396   %}
2397 
2398   // for byte regs
2399   enc_class REX_breg_mem(rRegI reg, memory mem)
2400   %{
2401     if ($reg$$reg < 8) {
2402       if ($mem$$base < 8) {
2403         if ($mem$$index >= 8) {
2404           emit_opcode(cbuf, Assembler::REX_X);
2405         } else if ($reg$$reg >= 4) {
2406           emit_opcode(cbuf, Assembler::REX);
2407         }
2408       } else {
2409         if ($mem$$index < 8) {
2410           emit_opcode(cbuf, Assembler::REX_B);
2411         } else {
2412           emit_opcode(cbuf, Assembler::REX_XB);
2413         }
2414       }
2415     } else {
2416       if ($mem$$base < 8) {
2417         if ($mem$$index < 8) {
2418           emit_opcode(cbuf, Assembler::REX_R);
2419         } else {
2420           emit_opcode(cbuf, Assembler::REX_RX);
2421         }
2422       } else {
2423         if ($mem$$index < 8) {
2424           emit_opcode(cbuf, Assembler::REX_RB);
2425         } else {
2426           emit_opcode(cbuf, Assembler::REX_RXB);
2427         }
2428       }
2429     }
2430   %}
2431 
2432   enc_class REX_reg(rRegI reg)
2433   %{
2434     if ($reg$$reg >= 8) {
2435       emit_opcode(cbuf, Assembler::REX_B);
2436     }
2437   %}
2438 
2439   enc_class REX_reg_wide(rRegI reg)
2440   %{
2441     if ($reg$$reg < 8) {
2442       emit_opcode(cbuf, Assembler::REX_W);
2443     } else {
2444       emit_opcode(cbuf, Assembler::REX_WB);
2445     }
2446   %}
2447 
2448   enc_class REX_reg_reg(rRegI dst, rRegI src)
2449   %{
2450     if ($dst$$reg < 8) {
2451       if ($src$$reg >= 8) {
2452         emit_opcode(cbuf, Assembler::REX_B);
2453       }
2454     } else {
2455       if ($src$$reg < 8) {
2456         emit_opcode(cbuf, Assembler::REX_R);
2457       } else {
2458         emit_opcode(cbuf, Assembler::REX_RB);
2459       }
2460     }
2461   %}
2462 
2463   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2464   %{
2465     if ($dst$$reg < 8) {
2466       if ($src$$reg < 8) {
2467         emit_opcode(cbuf, Assembler::REX_W);
2468       } else {
2469         emit_opcode(cbuf, Assembler::REX_WB);
2470       }
2471     } else {
2472       if ($src$$reg < 8) {
2473         emit_opcode(cbuf, Assembler::REX_WR);
2474       } else {
2475         emit_opcode(cbuf, Assembler::REX_WRB);
2476       }
2477     }
2478   %}
2479 
2480   enc_class REX_reg_mem(rRegI reg, memory mem)
2481   %{
2482     if ($reg$$reg < 8) {
2483       if ($mem$$base < 8) {
2484         if ($mem$$index >= 8) {
2485           emit_opcode(cbuf, Assembler::REX_X);
2486         }
2487       } else {
2488         if ($mem$$index < 8) {
2489           emit_opcode(cbuf, Assembler::REX_B);
2490         } else {
2491           emit_opcode(cbuf, Assembler::REX_XB);
2492         }
2493       }
2494     } else {
2495       if ($mem$$base < 8) {
2496         if ($mem$$index < 8) {
2497           emit_opcode(cbuf, Assembler::REX_R);
2498         } else {
2499           emit_opcode(cbuf, Assembler::REX_RX);
2500         }
2501       } else {
2502         if ($mem$$index < 8) {
2503           emit_opcode(cbuf, Assembler::REX_RB);
2504         } else {
2505           emit_opcode(cbuf, Assembler::REX_RXB);
2506         }
2507       }
2508     }
2509   %}
2510 
2511   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2512   %{
2513     if ($reg$$reg < 8) {
2514       if ($mem$$base < 8) {
2515         if ($mem$$index < 8) {
2516           emit_opcode(cbuf, Assembler::REX_W);
2517         } else {
2518           emit_opcode(cbuf, Assembler::REX_WX);
2519         }
2520       } else {
2521         if ($mem$$index < 8) {
2522           emit_opcode(cbuf, Assembler::REX_WB);
2523         } else {
2524           emit_opcode(cbuf, Assembler::REX_WXB);
2525         }
2526       }
2527     } else {
2528       if ($mem$$base < 8) {
2529         if ($mem$$index < 8) {
2530           emit_opcode(cbuf, Assembler::REX_WR);
2531         } else {
2532           emit_opcode(cbuf, Assembler::REX_WRX);
2533         }
2534       } else {
2535         if ($mem$$index < 8) {
2536           emit_opcode(cbuf, Assembler::REX_WRB);
2537         } else {
2538           emit_opcode(cbuf, Assembler::REX_WRXB);
2539         }
2540       }
2541     }
2542   %}
2543 
2544   enc_class reg_mem(rRegI ereg, memory mem)
2545   %{
2546     // High registers handle in encode_RegMem
2547     int reg = $ereg$$reg;
2548     int base = $mem$$base;
2549     int index = $mem$$index;
2550     int scale = $mem$$scale;
2551     int disp = $mem$$disp;
2552     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2553 
2554     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2555   %}
2556 
2557   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2558   %{
2559     int rm_byte_opcode = $rm_opcode$$constant;
2560 
2561     // High registers handle in encode_RegMem
2562     int base = $mem$$base;
2563     int index = $mem$$index;
2564     int scale = $mem$$scale;
2565     int displace = $mem$$disp;
2566 
2567     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2568                                             // working with static
2569                                             // globals
2570     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2571                   disp_reloc);
2572   %}
2573 
2574   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2575   %{
2576     int reg_encoding = $dst$$reg;
2577     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2578     int index        = 0x04;            // 0x04 indicates no index
2579     int scale        = 0x00;            // 0x00 indicates no scale
2580     int displace     = $src1$$constant; // 0x00 indicates no displacement
2581     relocInfo::relocType disp_reloc = relocInfo::none;
2582     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2583                   disp_reloc);
2584   %}
2585 
2586   enc_class neg_reg(rRegI dst)
2587   %{
2588     int dstenc = $dst$$reg;
2589     if (dstenc >= 8) {
2590       emit_opcode(cbuf, Assembler::REX_B);
2591       dstenc -= 8;
2592     }
2593     // NEG $dst
2594     emit_opcode(cbuf, 0xF7);
2595     emit_rm(cbuf, 0x3, 0x03, dstenc);
2596   %}
2597 
2598   enc_class neg_reg_wide(rRegI dst)
2599   %{
2600     int dstenc = $dst$$reg;
2601     if (dstenc < 8) {
2602       emit_opcode(cbuf, Assembler::REX_W);
2603     } else {
2604       emit_opcode(cbuf, Assembler::REX_WB);
2605       dstenc -= 8;
2606     }
2607     // NEG $dst
2608     emit_opcode(cbuf, 0xF7);
2609     emit_rm(cbuf, 0x3, 0x03, dstenc);
2610   %}
2611 
2612   enc_class setLT_reg(rRegI dst)
2613   %{
2614     int dstenc = $dst$$reg;
2615     if (dstenc >= 8) {
2616       emit_opcode(cbuf, Assembler::REX_B);
2617       dstenc -= 8;
2618     } else if (dstenc >= 4) {
2619       emit_opcode(cbuf, Assembler::REX);
2620     }
2621     // SETLT $dst
2622     emit_opcode(cbuf, 0x0F);
2623     emit_opcode(cbuf, 0x9C);
2624     emit_rm(cbuf, 0x3, 0x0, dstenc);
2625   %}
2626 
2627   enc_class setNZ_reg(rRegI dst)
2628   %{
2629     int dstenc = $dst$$reg;
2630     if (dstenc >= 8) {
2631       emit_opcode(cbuf, Assembler::REX_B);
2632       dstenc -= 8;
2633     } else if (dstenc >= 4) {
2634       emit_opcode(cbuf, Assembler::REX);
2635     }
2636     // SETNZ $dst
2637     emit_opcode(cbuf, 0x0F);
2638     emit_opcode(cbuf, 0x95);
2639     emit_rm(cbuf, 0x3, 0x0, dstenc);
2640   %}
2641 
2642 
2643   // Compare the lonogs and set -1, 0, or 1 into dst
2644   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2645   %{
2646     int src1enc = $src1$$reg;
2647     int src2enc = $src2$$reg;
2648     int dstenc = $dst$$reg;
2649 
2650     // cmpq $src1, $src2
2651     if (src1enc < 8) {
2652       if (src2enc < 8) {
2653         emit_opcode(cbuf, Assembler::REX_W);
2654       } else {
2655         emit_opcode(cbuf, Assembler::REX_WB);
2656       }
2657     } else {
2658       if (src2enc < 8) {
2659         emit_opcode(cbuf, Assembler::REX_WR);
2660       } else {
2661         emit_opcode(cbuf, Assembler::REX_WRB);
2662       }
2663     }
2664     emit_opcode(cbuf, 0x3B);
2665     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2666 
2667     // movl $dst, -1
2668     if (dstenc >= 8) {
2669       emit_opcode(cbuf, Assembler::REX_B);
2670     }
2671     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2672     emit_d32(cbuf, -1);
2673 
2674     // jl,s done
2675     emit_opcode(cbuf, 0x7C);
2676     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2677 
2678     // setne $dst
2679     if (dstenc >= 4) {
2680       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2681     }
2682     emit_opcode(cbuf, 0x0F);
2683     emit_opcode(cbuf, 0x95);
2684     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2685 
2686     // movzbl $dst, $dst
2687     if (dstenc >= 4) {
2688       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2689     }
2690     emit_opcode(cbuf, 0x0F);
2691     emit_opcode(cbuf, 0xB6);
2692     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2693   %}
2694 
2695   enc_class Push_ResultXD(regD dst) %{
2696     MacroAssembler _masm(&cbuf);
2697     __ fstp_d(Address(rsp, 0));
2698     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2699     __ addptr(rsp, 8);
2700   %}
2701 
2702   enc_class Push_SrcXD(regD src) %{
2703     MacroAssembler _masm(&cbuf);
2704     __ subptr(rsp, 8);
2705     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2706     __ fld_d(Address(rsp, 0));
2707   %}
2708 
2709 
2710   enc_class enc_rethrow()
2711   %{
2712     cbuf.set_insts_mark();
2713     emit_opcode(cbuf, 0xE9); // jmp entry
2714     emit_d32_reloc(cbuf,
2715                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2716                    runtime_call_Relocation::spec(),
2717                    RELOC_DISP32);
2718   %}
2719 
2720 %}
2721 
2722 
2723 
2724 //----------FRAME--------------------------------------------------------------
2725 // Definition of frame structure and management information.
2726 //
2727 //  S T A C K   L A Y O U T    Allocators stack-slot number
2728 //                             |   (to get allocators register number
2729 //  G  Owned by    |        |  v    add OptoReg::stack0())
2730 //  r   CALLER     |        |
2731 //  o     |        +--------+      pad to even-align allocators stack-slot
2732 //  w     V        |  pad0  |        numbers; owned by CALLER
2733 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2734 //  h     ^        |   in   |  5
2735 //        |        |  args  |  4   Holes in incoming args owned by SELF
2736 //  |     |        |        |  3
2737 //  |     |        +--------+
2738 //  V     |        | old out|      Empty on Intel, window on Sparc
2739 //        |    old |preserve|      Must be even aligned.
2740 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2741 //        |        |   in   |  3   area for Intel ret address
2742 //     Owned by    |preserve|      Empty on Sparc.
2743 //       SELF      +--------+
2744 //        |        |  pad2  |  2   pad to align old SP
2745 //        |        +--------+  1
2746 //        |        | locks  |  0
2747 //        |        +--------+----> OptoReg::stack0(), even aligned
2748 //        |        |  pad1  | 11   pad to align new SP
2749 //        |        +--------+
2750 //        |        |        | 10
2751 //        |        | spills |  9   spills
2752 //        V        |        |  8   (pad0 slot for callee)
2753 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2754 //        ^        |  out   |  7
2755 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2756 //     Owned by    +--------+
2757 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2758 //        |    new |preserve|      Must be even-aligned.
2759 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2760 //        |        |        |
2761 //
2762 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2763 //         known from SELF's arguments and the Java calling convention.
2764 //         Region 6-7 is determined per call site.
2765 // Note 2: If the calling convention leaves holes in the incoming argument
2766 //         area, those holes are owned by SELF.  Holes in the outgoing area
2767 //         are owned by the CALLEE.  Holes should not be nessecary in the
2768 //         incoming area, as the Java calling convention is completely under
2769 //         the control of the AD file.  Doubles can be sorted and packed to
2770 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2771 //         varargs C calling conventions.
2772 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2773 //         even aligned with pad0 as needed.
2774 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2775 //         region 6-11 is even aligned; it may be padded out more so that
2776 //         the region from SP to FP meets the minimum stack alignment.
2777 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2778 //         alignment.  Region 11, pad1, may be dynamically extended so that
2779 //         SP meets the minimum alignment.
2780 
2781 frame
2782 %{
2783   // What direction does stack grow in (assumed to be same for C & Java)
2784   stack_direction(TOWARDS_LOW);
2785 
2786   // These three registers define part of the calling convention
2787   // between compiled code and the interpreter.
2788   inline_cache_reg(RAX);                // Inline Cache Register
2789   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2790                                         // calling interpreter
2791 
2792   // Optional: name the operand used by cisc-spilling to access
2793   // [stack_pointer + offset]
2794   cisc_spilling_operand_name(indOffset32);
2795 
2796   // Number of stack slots consumed by locking an object
2797   sync_stack_slots(2);
2798 
2799   // Compiled code's Frame Pointer
2800   frame_pointer(RSP);
2801 
2802   // Interpreter stores its frame pointer in a register which is
2803   // stored to the stack by I2CAdaptors.
2804   // I2CAdaptors convert from interpreted java to compiled java.
2805   interpreter_frame_pointer(RBP);
2806 
2807   // Stack alignment requirement
2808   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2809 
2810   // Number of stack slots between incoming argument block and the start of
2811   // a new frame.  The PROLOG must add this many slots to the stack.  The
2812   // EPILOG must remove this many slots.  amd64 needs two slots for
2813   // return address.
2814   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2815 
2816   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2817   // for calls to C.  Supports the var-args backing area for register parms.
2818   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2819 
2820   // The after-PROLOG location of the return address.  Location of
2821   // return address specifies a type (REG or STACK) and a number
2822   // representing the register number (i.e. - use a register name) or
2823   // stack slot.
2824   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2825   // Otherwise, it is above the locks and verification slot and alignment word
2826   return_addr(STACK - 2 +
2827               align_up((Compile::current()->in_preserve_stack_slots() +
2828                         Compile::current()->fixed_slots()),
2829                        stack_alignment_in_slots()));
2830 
2831   // Body of function which returns an integer array locating
2832   // arguments either in registers or in stack slots.  Passed an array
2833   // of ideal registers called "sig" and a "length" count.  Stack-slot
2834   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2835   // arguments for a CALLEE.  Incoming stack arguments are
2836   // automatically biased by the preserve_stack_slots field above.
2837 
2838   calling_convention
2839   %{
2840     // No difference between ingoing/outgoing just pass false
2841     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2842   %}
2843 
2844   c_calling_convention
2845   %{
2846     // This is obviously always outgoing
2847     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2848   %}
2849 
2850   // Location of compiled Java return values.  Same as C for now.
2851   return_value
2852   %{
2853     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2854            "only return normal values");
2855 
2856     static const int lo[Op_RegL + 1] = {
2857       0,
2858       0,
2859       RAX_num,  // Op_RegN
2860       RAX_num,  // Op_RegI
2861       RAX_num,  // Op_RegP
2862       XMM0_num, // Op_RegF
2863       XMM0_num, // Op_RegD
2864       RAX_num   // Op_RegL
2865     };
2866     static const int hi[Op_RegL + 1] = {
2867       0,
2868       0,
2869       OptoReg::Bad, // Op_RegN
2870       OptoReg::Bad, // Op_RegI
2871       RAX_H_num,    // Op_RegP
2872       OptoReg::Bad, // Op_RegF
2873       XMM0b_num,    // Op_RegD
2874       RAX_H_num     // Op_RegL
2875     };
2876     // Excluded flags and vector registers.
2877     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2878     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2879   %}
2880 %}
2881 
2882 //----------ATTRIBUTES---------------------------------------------------------
2883 //----------Operand Attributes-------------------------------------------------
2884 op_attrib op_cost(0);        // Required cost attribute
2885 
2886 //----------Instruction Attributes---------------------------------------------
2887 ins_attrib ins_cost(100);       // Required cost attribute
2888 ins_attrib ins_size(8);         // Required size attribute (in bits)
2889 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2890                                 // a non-matching short branch variant
2891                                 // of some long branch?
2892 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2893                                 // be a power of 2) specifies the
2894                                 // alignment that some part of the
2895                                 // instruction (not necessarily the
2896                                 // start) requires.  If > 1, a
2897                                 // compute_padding() function must be
2898                                 // provided for the instruction
2899 
2900 //----------OPERANDS-----------------------------------------------------------
2901 // Operand definitions must precede instruction definitions for correct parsing
2902 // in the ADLC because operands constitute user defined types which are used in
2903 // instruction definitions.
2904 
2905 //----------Simple Operands----------------------------------------------------
2906 // Immediate Operands
2907 // Integer Immediate
2908 operand immI()
2909 %{
2910   match(ConI);
2911 
2912   op_cost(10);
2913   format %{ %}
2914   interface(CONST_INTER);
2915 %}
2916 
2917 // Constant for test vs zero
2918 operand immI0()
2919 %{
2920   predicate(n->get_int() == 0);
2921   match(ConI);
2922 
2923   op_cost(0);
2924   format %{ %}
2925   interface(CONST_INTER);
2926 %}
2927 
2928 // Constant for increment
2929 operand immI1()
2930 %{
2931   predicate(n->get_int() == 1);
2932   match(ConI);
2933 
2934   op_cost(0);
2935   format %{ %}
2936   interface(CONST_INTER);
2937 %}
2938 
2939 // Constant for decrement
2940 operand immI_M1()
2941 %{
2942   predicate(n->get_int() == -1);
2943   match(ConI);
2944 
2945   op_cost(0);
2946   format %{ %}
2947   interface(CONST_INTER);
2948 %}
2949 
2950 // Valid scale values for addressing modes
2951 operand immI2()
2952 %{
2953   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2954   match(ConI);
2955 
2956   format %{ %}
2957   interface(CONST_INTER);
2958 %}
2959 
2960 operand immI8()
2961 %{
2962   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2963   match(ConI);
2964 
2965   op_cost(5);
2966   format %{ %}
2967   interface(CONST_INTER);
2968 %}
2969 
2970 operand immU8()
2971 %{
2972   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2973   match(ConI);
2974 
2975   op_cost(5);
2976   format %{ %}
2977   interface(CONST_INTER);
2978 %}
2979 
2980 operand immI16()
2981 %{
2982   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2983   match(ConI);
2984 
2985   op_cost(10);
2986   format %{ %}
2987   interface(CONST_INTER);
2988 %}
2989 
2990 // Int Immediate non-negative
2991 operand immU31()
2992 %{
2993   predicate(n->get_int() >= 0);
2994   match(ConI);
2995 
2996   op_cost(0);
2997   format %{ %}
2998   interface(CONST_INTER);
2999 %}
3000 
3001 // Constant for long shifts
3002 operand immI_32()
3003 %{
3004   predicate( n->get_int() == 32 );
3005   match(ConI);
3006 
3007   op_cost(0);
3008   format %{ %}
3009   interface(CONST_INTER);
3010 %}
3011 
3012 // Constant for long shifts
3013 operand immI_64()
3014 %{
3015   predicate( n->get_int() == 64 );
3016   match(ConI);
3017 
3018   op_cost(0);
3019   format %{ %}
3020   interface(CONST_INTER);
3021 %}
3022 
3023 // Pointer Immediate
3024 operand immP()
3025 %{
3026   match(ConP);
3027 
3028   op_cost(10);
3029   format %{ %}
3030   interface(CONST_INTER);
3031 %}
3032 
3033 // NULL Pointer Immediate
3034 operand immP0()
3035 %{
3036   predicate(n->get_ptr() == 0);
3037   match(ConP);
3038 
3039   op_cost(5);
3040   format %{ %}
3041   interface(CONST_INTER);
3042 %}
3043 
3044 // Pointer Immediate
3045 operand immN() %{
3046   match(ConN);
3047 
3048   op_cost(10);
3049   format %{ %}
3050   interface(CONST_INTER);
3051 %}
3052 
3053 operand immNKlass() %{
3054   match(ConNKlass);
3055 
3056   op_cost(10);
3057   format %{ %}
3058   interface(CONST_INTER);
3059 %}
3060 
3061 // NULL Pointer Immediate
3062 operand immN0() %{
3063   predicate(n->get_narrowcon() == 0);
3064   match(ConN);
3065 
3066   op_cost(5);
3067   format %{ %}
3068   interface(CONST_INTER);
3069 %}
3070 
3071 operand immP31()
3072 %{
3073   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3074             && (n->get_ptr() >> 31) == 0);
3075   match(ConP);
3076 
3077   op_cost(5);
3078   format %{ %}
3079   interface(CONST_INTER);
3080 %}
3081 
3082 
3083 // Long Immediate
3084 operand immL()
3085 %{
3086   match(ConL);
3087 
3088   op_cost(20);
3089   format %{ %}
3090   interface(CONST_INTER);
3091 %}
3092 
3093 // Long Immediate 8-bit
3094 operand immL8()
3095 %{
3096   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3097   match(ConL);
3098 
3099   op_cost(5);
3100   format %{ %}
3101   interface(CONST_INTER);
3102 %}
3103 
3104 // Long Immediate 32-bit unsigned
3105 operand immUL32()
3106 %{
3107   predicate(n->get_long() == (unsigned int) (n->get_long()));
3108   match(ConL);
3109 
3110   op_cost(10);
3111   format %{ %}
3112   interface(CONST_INTER);
3113 %}
3114 
3115 // Long Immediate 32-bit signed
3116 operand immL32()
3117 %{
3118   predicate(n->get_long() == (int) (n->get_long()));
3119   match(ConL);
3120 
3121   op_cost(15);
3122   format %{ %}
3123   interface(CONST_INTER);
3124 %}
3125 
3126 // Long Immediate zero
3127 operand immL0()
3128 %{
3129   predicate(n->get_long() == 0L);
3130   match(ConL);
3131 
3132   op_cost(10);
3133   format %{ %}
3134   interface(CONST_INTER);
3135 %}
3136 
3137 // Constant for increment
3138 operand immL1()
3139 %{
3140   predicate(n->get_long() == 1);
3141   match(ConL);
3142 
3143   format %{ %}
3144   interface(CONST_INTER);
3145 %}
3146 
3147 // Constant for decrement
3148 operand immL_M1()
3149 %{
3150   predicate(n->get_long() == -1);
3151   match(ConL);
3152 
3153   format %{ %}
3154   interface(CONST_INTER);
3155 %}
3156 
3157 // Long Immediate: the value 10
3158 operand immL10()
3159 %{
3160   predicate(n->get_long() == 10);
3161   match(ConL);
3162 
3163   format %{ %}
3164   interface(CONST_INTER);
3165 %}
3166 
3167 // Long immediate from 0 to 127.
3168 // Used for a shorter form of long mul by 10.
3169 operand immL_127()
3170 %{
3171   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3172   match(ConL);
3173 
3174   op_cost(10);
3175   format %{ %}
3176   interface(CONST_INTER);
3177 %}
3178 
3179 // Long Immediate: low 32-bit mask
3180 operand immL_32bits()
3181 %{
3182   predicate(n->get_long() == 0xFFFFFFFFL);
3183   match(ConL);
3184   op_cost(20);
3185 
3186   format %{ %}
3187   interface(CONST_INTER);
3188 %}
3189 
3190 // Float Immediate zero
3191 operand immF0()
3192 %{
3193   predicate(jint_cast(n->getf()) == 0);
3194   match(ConF);
3195 
3196   op_cost(5);
3197   format %{ %}
3198   interface(CONST_INTER);
3199 %}
3200 
3201 // Float Immediate
3202 operand immF()
3203 %{
3204   match(ConF);
3205 
3206   op_cost(15);
3207   format %{ %}
3208   interface(CONST_INTER);
3209 %}
3210 
3211 // Double Immediate zero
3212 operand immD0()
3213 %{
3214   predicate(jlong_cast(n->getd()) == 0);
3215   match(ConD);
3216 
3217   op_cost(5);
3218   format %{ %}
3219   interface(CONST_INTER);
3220 %}
3221 
3222 // Double Immediate
3223 operand immD()
3224 %{
3225   match(ConD);
3226 
3227   op_cost(15);
3228   format %{ %}
3229   interface(CONST_INTER);
3230 %}
3231 
3232 // Immediates for special shifts (sign extend)
3233 
3234 // Constants for increment
3235 operand immI_16()
3236 %{
3237   predicate(n->get_int() == 16);
3238   match(ConI);
3239 
3240   format %{ %}
3241   interface(CONST_INTER);
3242 %}
3243 
3244 operand immI_24()
3245 %{
3246   predicate(n->get_int() == 24);
3247   match(ConI);
3248 
3249   format %{ %}
3250   interface(CONST_INTER);
3251 %}
3252 
3253 // Constant for byte-wide masking
3254 operand immI_255()
3255 %{
3256   predicate(n->get_int() == 255);
3257   match(ConI);
3258 
3259   format %{ %}
3260   interface(CONST_INTER);
3261 %}
3262 
3263 // Constant for short-wide masking
3264 operand immI_65535()
3265 %{
3266   predicate(n->get_int() == 65535);
3267   match(ConI);
3268 
3269   format %{ %}
3270   interface(CONST_INTER);
3271 %}
3272 
3273 // Constant for byte-wide masking
3274 operand immL_255()
3275 %{
3276   predicate(n->get_long() == 255);
3277   match(ConL);
3278 
3279   format %{ %}
3280   interface(CONST_INTER);
3281 %}
3282 
3283 // Constant for short-wide masking
3284 operand immL_65535()
3285 %{
3286   predicate(n->get_long() == 65535);
3287   match(ConL);
3288 
3289   format %{ %}
3290   interface(CONST_INTER);
3291 %}
3292 
3293 // Register Operands
3294 // Integer Register
3295 operand rRegI()
3296 %{
3297   constraint(ALLOC_IN_RC(int_reg));
3298   match(RegI);
3299 
3300   match(rax_RegI);
3301   match(rbx_RegI);
3302   match(rcx_RegI);
3303   match(rdx_RegI);
3304   match(rdi_RegI);
3305 
3306   format %{ %}
3307   interface(REG_INTER);
3308 %}
3309 
3310 // Special Registers
3311 operand rax_RegI()
3312 %{
3313   constraint(ALLOC_IN_RC(int_rax_reg));
3314   match(RegI);
3315   match(rRegI);
3316 
3317   format %{ "RAX" %}
3318   interface(REG_INTER);
3319 %}
3320 
3321 // Special Registers
3322 operand rbx_RegI()
3323 %{
3324   constraint(ALLOC_IN_RC(int_rbx_reg));
3325   match(RegI);
3326   match(rRegI);
3327 
3328   format %{ "RBX" %}
3329   interface(REG_INTER);
3330 %}
3331 
3332 operand rcx_RegI()
3333 %{
3334   constraint(ALLOC_IN_RC(int_rcx_reg));
3335   match(RegI);
3336   match(rRegI);
3337 
3338   format %{ "RCX" %}
3339   interface(REG_INTER);
3340 %}
3341 
3342 operand rdx_RegI()
3343 %{
3344   constraint(ALLOC_IN_RC(int_rdx_reg));
3345   match(RegI);
3346   match(rRegI);
3347 
3348   format %{ "RDX" %}
3349   interface(REG_INTER);
3350 %}
3351 
3352 operand rdi_RegI()
3353 %{
3354   constraint(ALLOC_IN_RC(int_rdi_reg));
3355   match(RegI);
3356   match(rRegI);
3357 
3358   format %{ "RDI" %}
3359   interface(REG_INTER);
3360 %}
3361 
3362 operand no_rcx_RegI()
3363 %{
3364   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3365   match(RegI);
3366   match(rax_RegI);
3367   match(rbx_RegI);
3368   match(rdx_RegI);
3369   match(rdi_RegI);
3370 
3371   format %{ %}
3372   interface(REG_INTER);
3373 %}
3374 
3375 operand no_rax_rdx_RegI()
3376 %{
3377   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3378   match(RegI);
3379   match(rbx_RegI);
3380   match(rcx_RegI);
3381   match(rdi_RegI);
3382 
3383   format %{ %}
3384   interface(REG_INTER);
3385 %}
3386 
3387 // Pointer Register
3388 operand any_RegP()
3389 %{
3390   constraint(ALLOC_IN_RC(any_reg));
3391   match(RegP);
3392   match(rax_RegP);
3393   match(rbx_RegP);
3394   match(rdi_RegP);
3395   match(rsi_RegP);
3396   match(rbp_RegP);
3397   match(r15_RegP);
3398   match(rRegP);
3399 
3400   format %{ %}
3401   interface(REG_INTER);
3402 %}
3403 
3404 operand rRegP()
3405 %{
3406   constraint(ALLOC_IN_RC(ptr_reg));
3407   match(RegP);
3408   match(rax_RegP);
3409   match(rbx_RegP);
3410   match(rdi_RegP);
3411   match(rsi_RegP);
3412   match(rbp_RegP);  // See Q&A below about
3413   match(r15_RegP);  // r15_RegP and rbp_RegP.
3414 
3415   format %{ %}
3416   interface(REG_INTER);
3417 %}
3418 
3419 operand rRegN() %{
3420   constraint(ALLOC_IN_RC(int_reg));
3421   match(RegN);
3422 
3423   format %{ %}
3424   interface(REG_INTER);
3425 %}
3426 
3427 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3428 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3429 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3430 // The output of an instruction is controlled by the allocator, which respects
3431 // register class masks, not match rules.  Unless an instruction mentions
3432 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3433 // by the allocator as an input.
3434 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3435 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3436 // result, RBP is not included in the output of the instruction either.
3437 
3438 operand no_rax_RegP()
3439 %{
3440   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3441   match(RegP);
3442   match(rbx_RegP);
3443   match(rsi_RegP);
3444   match(rdi_RegP);
3445 
3446   format %{ %}
3447   interface(REG_INTER);
3448 %}
3449 
3450 // This operand is not allowed to use RBP even if
3451 // RBP is not used to hold the frame pointer.
3452 operand no_rbp_RegP()
3453 %{
3454   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3455   match(RegP);
3456   match(rbx_RegP);
3457   match(rsi_RegP);
3458   match(rdi_RegP);
3459 
3460   format %{ %}
3461   interface(REG_INTER);
3462 %}
3463 
3464 operand no_rax_rbx_RegP()
3465 %{
3466   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3467   match(RegP);
3468   match(rsi_RegP);
3469   match(rdi_RegP);
3470 
3471   format %{ %}
3472   interface(REG_INTER);
3473 %}
3474 
3475 // Special Registers
3476 // Return a pointer value
3477 operand rax_RegP()
3478 %{
3479   constraint(ALLOC_IN_RC(ptr_rax_reg));
3480   match(RegP);
3481   match(rRegP);
3482 
3483   format %{ %}
3484   interface(REG_INTER);
3485 %}
3486 
3487 // Special Registers
3488 // Return a compressed pointer value
3489 operand rax_RegN()
3490 %{
3491   constraint(ALLOC_IN_RC(int_rax_reg));
3492   match(RegN);
3493   match(rRegN);
3494 
3495   format %{ %}
3496   interface(REG_INTER);
3497 %}
3498 
3499 // Used in AtomicAdd
3500 operand rbx_RegP()
3501 %{
3502   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3503   match(RegP);
3504   match(rRegP);
3505 
3506   format %{ %}
3507   interface(REG_INTER);
3508 %}
3509 
3510 operand rsi_RegP()
3511 %{
3512   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3513   match(RegP);
3514   match(rRegP);
3515 
3516   format %{ %}
3517   interface(REG_INTER);
3518 %}
3519 
3520 // Used in rep stosq
3521 operand rdi_RegP()
3522 %{
3523   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3524   match(RegP);
3525   match(rRegP);
3526 
3527   format %{ %}
3528   interface(REG_INTER);
3529 %}
3530 
3531 operand r15_RegP()
3532 %{
3533   constraint(ALLOC_IN_RC(ptr_r15_reg));
3534   match(RegP);
3535   match(rRegP);
3536 
3537   format %{ %}
3538   interface(REG_INTER);
3539 %}
3540 
3541 operand rRegL()
3542 %{
3543   constraint(ALLOC_IN_RC(long_reg));
3544   match(RegL);
3545   match(rax_RegL);
3546   match(rdx_RegL);
3547 
3548   format %{ %}
3549   interface(REG_INTER);
3550 %}
3551 
3552 // Special Registers
3553 operand no_rax_rdx_RegL()
3554 %{
3555   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3556   match(RegL);
3557   match(rRegL);
3558 
3559   format %{ %}
3560   interface(REG_INTER);
3561 %}
3562 
3563 operand no_rax_RegL()
3564 %{
3565   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3566   match(RegL);
3567   match(rRegL);
3568   match(rdx_RegL);
3569 
3570   format %{ %}
3571   interface(REG_INTER);
3572 %}
3573 
3574 operand no_rcx_RegL()
3575 %{
3576   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3577   match(RegL);
3578   match(rRegL);
3579 
3580   format %{ %}
3581   interface(REG_INTER);
3582 %}
3583 
3584 operand rax_RegL()
3585 %{
3586   constraint(ALLOC_IN_RC(long_rax_reg));
3587   match(RegL);
3588   match(rRegL);
3589 
3590   format %{ "RAX" %}
3591   interface(REG_INTER);
3592 %}
3593 
3594 operand rcx_RegL()
3595 %{
3596   constraint(ALLOC_IN_RC(long_rcx_reg));
3597   match(RegL);
3598   match(rRegL);
3599 
3600   format %{ %}
3601   interface(REG_INTER);
3602 %}
3603 
3604 operand rdx_RegL()
3605 %{
3606   constraint(ALLOC_IN_RC(long_rdx_reg));
3607   match(RegL);
3608   match(rRegL);
3609 
3610   format %{ %}
3611   interface(REG_INTER);
3612 %}
3613 
3614 // Flags register, used as output of compare instructions
3615 operand rFlagsReg()
3616 %{
3617   constraint(ALLOC_IN_RC(int_flags));
3618   match(RegFlags);
3619 
3620   format %{ "RFLAGS" %}
3621   interface(REG_INTER);
3622 %}
3623 
3624 // Flags register, used as output of FLOATING POINT compare instructions
3625 operand rFlagsRegU()
3626 %{
3627   constraint(ALLOC_IN_RC(int_flags));
3628   match(RegFlags);
3629 
3630   format %{ "RFLAGS_U" %}
3631   interface(REG_INTER);
3632 %}
3633 
3634 operand rFlagsRegUCF() %{
3635   constraint(ALLOC_IN_RC(int_flags));
3636   match(RegFlags);
3637   predicate(false);
3638 
3639   format %{ "RFLAGS_U_CF" %}
3640   interface(REG_INTER);
3641 %}
3642 
3643 // Float register operands
3644 operand regF() %{
3645    constraint(ALLOC_IN_RC(float_reg));
3646    match(RegF);
3647 
3648    format %{ %}
3649    interface(REG_INTER);
3650 %}
3651 
3652 // Float register operands
3653 operand legRegF() %{
3654    constraint(ALLOC_IN_RC(float_reg_legacy));
3655    match(RegF);
3656 
3657    format %{ %}
3658    interface(REG_INTER);
3659 %}
3660 
3661 // Float register operands
3662 operand vlRegF() %{
3663    constraint(ALLOC_IN_RC(float_reg_vl));
3664    match(RegF);
3665 
3666    format %{ %}
3667    interface(REG_INTER);
3668 %}
3669 
3670 // Double register operands
3671 operand regD() %{
3672    constraint(ALLOC_IN_RC(double_reg));
3673    match(RegD);
3674 
3675    format %{ %}
3676    interface(REG_INTER);
3677 %}
3678 
3679 // Double register operands
3680 operand legRegD() %{
3681    constraint(ALLOC_IN_RC(double_reg_legacy));
3682    match(RegD);
3683 
3684    format %{ %}
3685    interface(REG_INTER);
3686 %}
3687 
3688 // Double register operands
3689 operand vlRegD() %{
3690    constraint(ALLOC_IN_RC(double_reg_vl));
3691    match(RegD);
3692 
3693    format %{ %}
3694    interface(REG_INTER);
3695 %}
3696 
3697 // Vectors
3698 operand vecS() %{
3699   constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
3700   match(VecS);
3701 
3702   format %{ %}
3703   interface(REG_INTER);
3704 %}
3705 
3706 // Vectors
3707 operand legVecS() %{
3708   constraint(ALLOC_IN_RC(vectors_reg_legacy));
3709   match(VecS);
3710 
3711   format %{ %}
3712   interface(REG_INTER);
3713 %}
3714 
3715 operand vecD() %{
3716   constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
3717   match(VecD);
3718 
3719   format %{ %}
3720   interface(REG_INTER);
3721 %}
3722 
3723 operand legVecD() %{
3724   constraint(ALLOC_IN_RC(vectord_reg_legacy));
3725   match(VecD);
3726 
3727   format %{ %}
3728   interface(REG_INTER);
3729 %}
3730 
3731 operand vecX() %{
3732   constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
3733   match(VecX);
3734 
3735   format %{ %}
3736   interface(REG_INTER);
3737 %}
3738 
3739 operand legVecX() %{
3740   constraint(ALLOC_IN_RC(vectorx_reg_legacy));
3741   match(VecX);
3742 
3743   format %{ %}
3744   interface(REG_INTER);
3745 %}
3746 
3747 operand vecY() %{
3748   constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
3749   match(VecY);
3750 
3751   format %{ %}
3752   interface(REG_INTER);
3753 %}
3754 
3755 operand legVecY() %{
3756   constraint(ALLOC_IN_RC(vectory_reg_legacy));
3757   match(VecY);
3758 
3759   format %{ %}
3760   interface(REG_INTER);
3761 %}
3762 
3763 //----------Memory Operands----------------------------------------------------
3764 // Direct Memory Operand
3765 // operand direct(immP addr)
3766 // %{
3767 //   match(addr);
3768 
3769 //   format %{ "[$addr]" %}
3770 //   interface(MEMORY_INTER) %{
3771 //     base(0xFFFFFFFF);
3772 //     index(0x4);
3773 //     scale(0x0);
3774 //     disp($addr);
3775 //   %}
3776 // %}
3777 
3778 // Indirect Memory Operand
3779 operand indirect(any_RegP reg)
3780 %{
3781   constraint(ALLOC_IN_RC(ptr_reg));
3782   match(reg);
3783 
3784   format %{ "[$reg]" %}
3785   interface(MEMORY_INTER) %{
3786     base($reg);
3787     index(0x4);
3788     scale(0x0);
3789     disp(0x0);
3790   %}
3791 %}
3792 
3793 // Indirect Memory Plus Short Offset Operand
3794 operand indOffset8(any_RegP reg, immL8 off)
3795 %{
3796   constraint(ALLOC_IN_RC(ptr_reg));
3797   match(AddP reg off);
3798 
3799   format %{ "[$reg + $off (8-bit)]" %}
3800   interface(MEMORY_INTER) %{
3801     base($reg);
3802     index(0x4);
3803     scale(0x0);
3804     disp($off);
3805   %}
3806 %}
3807 
3808 // Indirect Memory Plus Long Offset Operand
3809 operand indOffset32(any_RegP reg, immL32 off)
3810 %{
3811   constraint(ALLOC_IN_RC(ptr_reg));
3812   match(AddP reg off);
3813 
3814   format %{ "[$reg + $off (32-bit)]" %}
3815   interface(MEMORY_INTER) %{
3816     base($reg);
3817     index(0x4);
3818     scale(0x0);
3819     disp($off);
3820   %}
3821 %}
3822 
3823 // Indirect Memory Plus Index Register Plus Offset Operand
3824 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3825 %{
3826   constraint(ALLOC_IN_RC(ptr_reg));
3827   match(AddP (AddP reg lreg) off);
3828 
3829   op_cost(10);
3830   format %{"[$reg + $off + $lreg]" %}
3831   interface(MEMORY_INTER) %{
3832     base($reg);
3833     index($lreg);
3834     scale(0x0);
3835     disp($off);
3836   %}
3837 %}
3838 
3839 // Indirect Memory Plus Index Register Plus Offset Operand
3840 operand indIndex(any_RegP reg, rRegL lreg)
3841 %{
3842   constraint(ALLOC_IN_RC(ptr_reg));
3843   match(AddP reg lreg);
3844 
3845   op_cost(10);
3846   format %{"[$reg + $lreg]" %}
3847   interface(MEMORY_INTER) %{
3848     base($reg);
3849     index($lreg);
3850     scale(0x0);
3851     disp(0x0);
3852   %}
3853 %}
3854 
3855 // Indirect Memory Times Scale Plus Index Register
3856 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3857 %{
3858   constraint(ALLOC_IN_RC(ptr_reg));
3859   match(AddP reg (LShiftL lreg scale));
3860 
3861   op_cost(10);
3862   format %{"[$reg + $lreg << $scale]" %}
3863   interface(MEMORY_INTER) %{
3864     base($reg);
3865     index($lreg);
3866     scale($scale);
3867     disp(0x0);
3868   %}
3869 %}
3870 
3871 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3872 %{
3873   constraint(ALLOC_IN_RC(ptr_reg));
3874   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3875   match(AddP reg (LShiftL (ConvI2L idx) scale));
3876 
3877   op_cost(10);
3878   format %{"[$reg + pos $idx << $scale]" %}
3879   interface(MEMORY_INTER) %{
3880     base($reg);
3881     index($idx);
3882     scale($scale);
3883     disp(0x0);
3884   %}
3885 %}
3886 
3887 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3888 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3889 %{
3890   constraint(ALLOC_IN_RC(ptr_reg));
3891   match(AddP (AddP reg (LShiftL lreg scale)) off);
3892 
3893   op_cost(10);
3894   format %{"[$reg + $off + $lreg << $scale]" %}
3895   interface(MEMORY_INTER) %{
3896     base($reg);
3897     index($lreg);
3898     scale($scale);
3899     disp($off);
3900   %}
3901 %}
3902 
3903 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3904 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3905 %{
3906   constraint(ALLOC_IN_RC(ptr_reg));
3907   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3908   match(AddP (AddP reg (ConvI2L idx)) off);
3909 
3910   op_cost(10);
3911   format %{"[$reg + $off + $idx]" %}
3912   interface(MEMORY_INTER) %{
3913     base($reg);
3914     index($idx);
3915     scale(0x0);
3916     disp($off);
3917   %}
3918 %}
3919 
3920 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3921 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3922 %{
3923   constraint(ALLOC_IN_RC(ptr_reg));
3924   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3925   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3926 
3927   op_cost(10);
3928   format %{"[$reg + $off + $idx << $scale]" %}
3929   interface(MEMORY_INTER) %{
3930     base($reg);
3931     index($idx);
3932     scale($scale);
3933     disp($off);
3934   %}
3935 %}
3936 
3937 // Indirect Narrow Oop Plus Offset Operand
3938 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3939 // we can't free r12 even with CompressedOops::base() == NULL.
3940 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3941   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
3942   constraint(ALLOC_IN_RC(ptr_reg));
3943   match(AddP (DecodeN reg) off);
3944 
3945   op_cost(10);
3946   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3947   interface(MEMORY_INTER) %{
3948     base(0xc); // R12
3949     index($reg);
3950     scale(0x3);
3951     disp($off);
3952   %}
3953 %}
3954 
3955 // Indirect Memory Operand
3956 operand indirectNarrow(rRegN reg)
3957 %{
3958   predicate(CompressedOops::shift() == 0);
3959   constraint(ALLOC_IN_RC(ptr_reg));
3960   match(DecodeN reg);
3961 
3962   format %{ "[$reg]" %}
3963   interface(MEMORY_INTER) %{
3964     base($reg);
3965     index(0x4);
3966     scale(0x0);
3967     disp(0x0);
3968   %}
3969 %}
3970 
3971 // Indirect Memory Plus Short Offset Operand
3972 operand indOffset8Narrow(rRegN reg, immL8 off)
3973 %{
3974   predicate(CompressedOops::shift() == 0);
3975   constraint(ALLOC_IN_RC(ptr_reg));
3976   match(AddP (DecodeN reg) off);
3977 
3978   format %{ "[$reg + $off (8-bit)]" %}
3979   interface(MEMORY_INTER) %{
3980     base($reg);
3981     index(0x4);
3982     scale(0x0);
3983     disp($off);
3984   %}
3985 %}
3986 
3987 // Indirect Memory Plus Long Offset Operand
3988 operand indOffset32Narrow(rRegN reg, immL32 off)
3989 %{
3990   predicate(CompressedOops::shift() == 0);
3991   constraint(ALLOC_IN_RC(ptr_reg));
3992   match(AddP (DecodeN reg) off);
3993 
3994   format %{ "[$reg + $off (32-bit)]" %}
3995   interface(MEMORY_INTER) %{
3996     base($reg);
3997     index(0x4);
3998     scale(0x0);
3999     disp($off);
4000   %}
4001 %}
4002 
4003 // Indirect Memory Plus Index Register Plus Offset Operand
4004 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
4005 %{
4006   predicate(CompressedOops::shift() == 0);
4007   constraint(ALLOC_IN_RC(ptr_reg));
4008   match(AddP (AddP (DecodeN reg) lreg) off);
4009 
4010   op_cost(10);
4011   format %{"[$reg + $off + $lreg]" %}
4012   interface(MEMORY_INTER) %{
4013     base($reg);
4014     index($lreg);
4015     scale(0x0);
4016     disp($off);
4017   %}
4018 %}
4019 
4020 // Indirect Memory Plus Index Register Plus Offset Operand
4021 operand indIndexNarrow(rRegN reg, rRegL lreg)
4022 %{
4023   predicate(CompressedOops::shift() == 0);
4024   constraint(ALLOC_IN_RC(ptr_reg));
4025   match(AddP (DecodeN reg) lreg);
4026 
4027   op_cost(10);
4028   format %{"[$reg + $lreg]" %}
4029   interface(MEMORY_INTER) %{
4030     base($reg);
4031     index($lreg);
4032     scale(0x0);
4033     disp(0x0);
4034   %}
4035 %}
4036 
4037 // Indirect Memory Times Scale Plus Index Register
4038 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
4039 %{
4040   predicate(CompressedOops::shift() == 0);
4041   constraint(ALLOC_IN_RC(ptr_reg));
4042   match(AddP (DecodeN reg) (LShiftL lreg scale));
4043 
4044   op_cost(10);
4045   format %{"[$reg + $lreg << $scale]" %}
4046   interface(MEMORY_INTER) %{
4047     base($reg);
4048     index($lreg);
4049     scale($scale);
4050     disp(0x0);
4051   %}
4052 %}
4053 
4054 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4055 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4056 %{
4057   predicate(CompressedOops::shift() == 0);
4058   constraint(ALLOC_IN_RC(ptr_reg));
4059   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4060 
4061   op_cost(10);
4062   format %{"[$reg + $off + $lreg << $scale]" %}
4063   interface(MEMORY_INTER) %{
4064     base($reg);
4065     index($lreg);
4066     scale($scale);
4067     disp($off);
4068   %}
4069 %}
4070 
4071 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4072 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4073 %{
4074   constraint(ALLOC_IN_RC(ptr_reg));
4075   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4076   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4077 
4078   op_cost(10);
4079   format %{"[$reg + $off + $idx]" %}
4080   interface(MEMORY_INTER) %{
4081     base($reg);
4082     index($idx);
4083     scale(0x0);
4084     disp($off);
4085   %}
4086 %}
4087 
4088 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4089 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4090 %{
4091   constraint(ALLOC_IN_RC(ptr_reg));
4092   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4093   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4094 
4095   op_cost(10);
4096   format %{"[$reg + $off + $idx << $scale]" %}
4097   interface(MEMORY_INTER) %{
4098     base($reg);
4099     index($idx);
4100     scale($scale);
4101     disp($off);
4102   %}
4103 %}
4104 
4105 //----------Special Memory Operands--------------------------------------------
4106 // Stack Slot Operand - This operand is used for loading and storing temporary
4107 //                      values on the stack where a match requires a value to
4108 //                      flow through memory.
4109 operand stackSlotP(sRegP reg)
4110 %{
4111   constraint(ALLOC_IN_RC(stack_slots));
4112   // No match rule because this operand is only generated in matching
4113 
4114   format %{ "[$reg]" %}
4115   interface(MEMORY_INTER) %{
4116     base(0x4);   // RSP
4117     index(0x4);  // No Index
4118     scale(0x0);  // No Scale
4119     disp($reg);  // Stack Offset
4120   %}
4121 %}
4122 
4123 operand stackSlotI(sRegI reg)
4124 %{
4125   constraint(ALLOC_IN_RC(stack_slots));
4126   // No match rule because this operand is only generated in matching
4127 
4128   format %{ "[$reg]" %}
4129   interface(MEMORY_INTER) %{
4130     base(0x4);   // RSP
4131     index(0x4);  // No Index
4132     scale(0x0);  // No Scale
4133     disp($reg);  // Stack Offset
4134   %}
4135 %}
4136 
4137 operand stackSlotF(sRegF reg)
4138 %{
4139   constraint(ALLOC_IN_RC(stack_slots));
4140   // No match rule because this operand is only generated in matching
4141 
4142   format %{ "[$reg]" %}
4143   interface(MEMORY_INTER) %{
4144     base(0x4);   // RSP
4145     index(0x4);  // No Index
4146     scale(0x0);  // No Scale
4147     disp($reg);  // Stack Offset
4148   %}
4149 %}
4150 
4151 operand stackSlotD(sRegD reg)
4152 %{
4153   constraint(ALLOC_IN_RC(stack_slots));
4154   // No match rule because this operand is only generated in matching
4155 
4156   format %{ "[$reg]" %}
4157   interface(MEMORY_INTER) %{
4158     base(0x4);   // RSP
4159     index(0x4);  // No Index
4160     scale(0x0);  // No Scale
4161     disp($reg);  // Stack Offset
4162   %}
4163 %}
4164 operand stackSlotL(sRegL reg)
4165 %{
4166   constraint(ALLOC_IN_RC(stack_slots));
4167   // No match rule because this operand is only generated in matching
4168 
4169   format %{ "[$reg]" %}
4170   interface(MEMORY_INTER) %{
4171     base(0x4);   // RSP
4172     index(0x4);  // No Index
4173     scale(0x0);  // No Scale
4174     disp($reg);  // Stack Offset
4175   %}
4176 %}
4177 
4178 //----------Conditional Branch Operands----------------------------------------
4179 // Comparison Op  - This is the operation of the comparison, and is limited to
4180 //                  the following set of codes:
4181 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4182 //
4183 // Other attributes of the comparison, such as unsignedness, are specified
4184 // by the comparison instruction that sets a condition code flags register.
4185 // That result is represented by a flags operand whose subtype is appropriate
4186 // to the unsignedness (etc.) of the comparison.
4187 //
4188 // Later, the instruction which matches both the Comparison Op (a Bool) and
4189 // the flags (produced by the Cmp) specifies the coding of the comparison op
4190 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4191 
4192 // Comparision Code
4193 operand cmpOp()
4194 %{
4195   match(Bool);
4196 
4197   format %{ "" %}
4198   interface(COND_INTER) %{
4199     equal(0x4, "e");
4200     not_equal(0x5, "ne");
4201     less(0xC, "l");
4202     greater_equal(0xD, "ge");
4203     less_equal(0xE, "le");
4204     greater(0xF, "g");
4205     overflow(0x0, "o");
4206     no_overflow(0x1, "no");
4207   %}
4208 %}
4209 
4210 // Comparison Code, unsigned compare.  Used by FP also, with
4211 // C2 (unordered) turned into GT or LT already.  The other bits
4212 // C0 and C3 are turned into Carry & Zero flags.
4213 operand cmpOpU()
4214 %{
4215   match(Bool);
4216 
4217   format %{ "" %}
4218   interface(COND_INTER) %{
4219     equal(0x4, "e");
4220     not_equal(0x5, "ne");
4221     less(0x2, "b");
4222     greater_equal(0x3, "nb");
4223     less_equal(0x6, "be");
4224     greater(0x7, "nbe");
4225     overflow(0x0, "o");
4226     no_overflow(0x1, "no");
4227   %}
4228 %}
4229 
4230 
4231 // Floating comparisons that don't require any fixup for the unordered case
4232 operand cmpOpUCF() %{
4233   match(Bool);
4234   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4235             n->as_Bool()->_test._test == BoolTest::ge ||
4236             n->as_Bool()->_test._test == BoolTest::le ||
4237             n->as_Bool()->_test._test == BoolTest::gt);
4238   format %{ "" %}
4239   interface(COND_INTER) %{
4240     equal(0x4, "e");
4241     not_equal(0x5, "ne");
4242     less(0x2, "b");
4243     greater_equal(0x3, "nb");
4244     less_equal(0x6, "be");
4245     greater(0x7, "nbe");
4246     overflow(0x0, "o");
4247     no_overflow(0x1, "no");
4248   %}
4249 %}
4250 
4251 
4252 // Floating comparisons that can be fixed up with extra conditional jumps
4253 operand cmpOpUCF2() %{
4254   match(Bool);
4255   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4256             n->as_Bool()->_test._test == BoolTest::eq);
4257   format %{ "" %}
4258   interface(COND_INTER) %{
4259     equal(0x4, "e");
4260     not_equal(0x5, "ne");
4261     less(0x2, "b");
4262     greater_equal(0x3, "nb");
4263     less_equal(0x6, "be");
4264     greater(0x7, "nbe");
4265     overflow(0x0, "o");
4266     no_overflow(0x1, "no");
4267   %}
4268 %}
4269 
4270 // Operands for bound floating pointer register arguments
4271 operand rxmm0() %{
4272   constraint(ALLOC_IN_RC(xmm0_reg));
4273   match(VecX);
4274   format%{%}
4275   interface(REG_INTER);
4276 %}
4277 operand rxmm1() %{
4278   constraint(ALLOC_IN_RC(xmm1_reg));
4279   match(VecX);
4280   format%{%}
4281   interface(REG_INTER);
4282 %}
4283 operand rxmm2() %{
4284   constraint(ALLOC_IN_RC(xmm2_reg));
4285   match(VecX);
4286   format%{%}
4287   interface(REG_INTER);
4288 %}
4289 operand rxmm3() %{
4290   constraint(ALLOC_IN_RC(xmm3_reg));
4291   match(VecX);
4292   format%{%}
4293   interface(REG_INTER);
4294 %}
4295 operand rxmm4() %{
4296   constraint(ALLOC_IN_RC(xmm4_reg));
4297   match(VecX);
4298   format%{%}
4299   interface(REG_INTER);
4300 %}
4301 operand rxmm5() %{
4302   constraint(ALLOC_IN_RC(xmm5_reg));
4303   match(VecX);
4304   format%{%}
4305   interface(REG_INTER);
4306 %}
4307 operand rxmm6() %{
4308   constraint(ALLOC_IN_RC(xmm6_reg));
4309   match(VecX);
4310   format%{%}
4311   interface(REG_INTER);
4312 %}
4313 operand rxmm7() %{
4314   constraint(ALLOC_IN_RC(xmm7_reg));
4315   match(VecX);
4316   format%{%}
4317   interface(REG_INTER);
4318 %}
4319 operand rxmm8() %{
4320   constraint(ALLOC_IN_RC(xmm8_reg));
4321   match(VecX);
4322   format%{%}
4323   interface(REG_INTER);
4324 %}
4325 operand rxmm9() %{
4326   constraint(ALLOC_IN_RC(xmm9_reg));
4327   match(VecX);
4328   format%{%}
4329   interface(REG_INTER);
4330 %}
4331 operand rxmm10() %{
4332   constraint(ALLOC_IN_RC(xmm10_reg));
4333   match(VecX);
4334   format%{%}
4335   interface(REG_INTER);
4336 %}
4337 operand rxmm11() %{
4338   constraint(ALLOC_IN_RC(xmm11_reg));
4339   match(VecX);
4340   format%{%}
4341   interface(REG_INTER);
4342 %}
4343 operand rxmm12() %{
4344   constraint(ALLOC_IN_RC(xmm12_reg));
4345   match(VecX);
4346   format%{%}
4347   interface(REG_INTER);
4348 %}
4349 operand rxmm13() %{
4350   constraint(ALLOC_IN_RC(xmm13_reg));
4351   match(VecX);
4352   format%{%}
4353   interface(REG_INTER);
4354 %}
4355 operand rxmm14() %{
4356   constraint(ALLOC_IN_RC(xmm14_reg));
4357   match(VecX);
4358   format%{%}
4359   interface(REG_INTER);
4360 %}
4361 operand rxmm15() %{
4362   constraint(ALLOC_IN_RC(xmm15_reg));
4363   match(VecX);
4364   format%{%}
4365   interface(REG_INTER);
4366 %}
4367 operand rxmm16() %{
4368   constraint(ALLOC_IN_RC(xmm16_reg));
4369   match(VecX);
4370   format%{%}
4371   interface(REG_INTER);
4372 %}
4373 operand rxmm17() %{
4374   constraint(ALLOC_IN_RC(xmm17_reg));
4375   match(VecX);
4376   format%{%}
4377   interface(REG_INTER);
4378 %}
4379 operand rxmm18() %{
4380   constraint(ALLOC_IN_RC(xmm18_reg));
4381   match(VecX);
4382   format%{%}
4383   interface(REG_INTER);
4384 %}
4385 operand rxmm19() %{
4386   constraint(ALLOC_IN_RC(xmm19_reg));
4387   match(VecX);
4388   format%{%}
4389   interface(REG_INTER);
4390 %}
4391 operand rxmm20() %{
4392   constraint(ALLOC_IN_RC(xmm20_reg));
4393   match(VecX);
4394   format%{%}
4395   interface(REG_INTER);
4396 %}
4397 operand rxmm21() %{
4398   constraint(ALLOC_IN_RC(xmm21_reg));
4399   match(VecX);
4400   format%{%}
4401   interface(REG_INTER);
4402 %}
4403 operand rxmm22() %{
4404   constraint(ALLOC_IN_RC(xmm22_reg));
4405   match(VecX);
4406   format%{%}
4407   interface(REG_INTER);
4408 %}
4409 operand rxmm23() %{
4410   constraint(ALLOC_IN_RC(xmm23_reg));
4411   match(VecX);
4412   format%{%}
4413   interface(REG_INTER);
4414 %}
4415 operand rxmm24() %{
4416   constraint(ALLOC_IN_RC(xmm24_reg));
4417   match(VecX);
4418   format%{%}
4419   interface(REG_INTER);
4420 %}
4421 operand rxmm25() %{
4422   constraint(ALLOC_IN_RC(xmm25_reg));
4423   match(VecX);
4424   format%{%}
4425   interface(REG_INTER);
4426 %}
4427 operand rxmm26() %{
4428   constraint(ALLOC_IN_RC(xmm26_reg));
4429   match(VecX);
4430   format%{%}
4431   interface(REG_INTER);
4432 %}
4433 operand rxmm27() %{
4434   constraint(ALLOC_IN_RC(xmm27_reg));
4435   match(VecX);
4436   format%{%}
4437   interface(REG_INTER);
4438 %}
4439 operand rxmm28() %{
4440   constraint(ALLOC_IN_RC(xmm28_reg));
4441   match(VecX);
4442   format%{%}
4443   interface(REG_INTER);
4444 %}
4445 operand rxmm29() %{
4446   constraint(ALLOC_IN_RC(xmm29_reg));
4447   match(VecX);
4448   format%{%}
4449   interface(REG_INTER);
4450 %}
4451 operand rxmm30() %{
4452   constraint(ALLOC_IN_RC(xmm30_reg));
4453   match(VecX);
4454   format%{%}
4455   interface(REG_INTER);
4456 %}
4457 operand rxmm31() %{
4458   constraint(ALLOC_IN_RC(xmm31_reg));
4459   match(VecX);
4460   format%{%}
4461   interface(REG_INTER);
4462 %}
4463 
4464 //----------OPERAND CLASSES----------------------------------------------------
4465 // Operand Classes are groups of operands that are used as to simplify
4466 // instruction definitions by not requiring the AD writer to specify separate
4467 // instructions for every form of operand when the instruction accepts
4468 // multiple operand types with the same basic encoding and format.  The classic
4469 // case of this is memory operands.
4470 
4471 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4472                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4473                indCompressedOopOffset,
4474                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4475                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4476                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4477 
4478 //----------PIPELINE-----------------------------------------------------------
4479 // Rules which define the behavior of the target architectures pipeline.
4480 pipeline %{
4481 
4482 //----------ATTRIBUTES---------------------------------------------------------
4483 attributes %{
4484   variable_size_instructions;        // Fixed size instructions
4485   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4486   instruction_unit_size = 1;         // An instruction is 1 bytes long
4487   instruction_fetch_unit_size = 16;  // The processor fetches one line
4488   instruction_fetch_units = 1;       // of 16 bytes
4489 
4490   // List of nop instructions
4491   nops( MachNop );
4492 %}
4493 
4494 //----------RESOURCES----------------------------------------------------------
4495 // Resources are the functional units available to the machine
4496 
4497 // Generic P2/P3 pipeline
4498 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4499 // 3 instructions decoded per cycle.
4500 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4501 // 3 ALU op, only ALU0 handles mul instructions.
4502 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4503            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4504            BR, FPU,
4505            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4506 
4507 //----------PIPELINE DESCRIPTION-----------------------------------------------
4508 // Pipeline Description specifies the stages in the machine's pipeline
4509 
4510 // Generic P2/P3 pipeline
4511 pipe_desc(S0, S1, S2, S3, S4, S5);
4512 
4513 //----------PIPELINE CLASSES---------------------------------------------------
4514 // Pipeline Classes describe the stages in which input and output are
4515 // referenced by the hardware pipeline.
4516 
4517 // Naming convention: ialu or fpu
4518 // Then: _reg
4519 // Then: _reg if there is a 2nd register
4520 // Then: _long if it's a pair of instructions implementing a long
4521 // Then: _fat if it requires the big decoder
4522 //   Or: _mem if it requires the big decoder and a memory unit.
4523 
4524 // Integer ALU reg operation
4525 pipe_class ialu_reg(rRegI dst)
4526 %{
4527     single_instruction;
4528     dst    : S4(write);
4529     dst    : S3(read);
4530     DECODE : S0;        // any decoder
4531     ALU    : S3;        // any alu
4532 %}
4533 
4534 // Long ALU reg operation
4535 pipe_class ialu_reg_long(rRegL dst)
4536 %{
4537     instruction_count(2);
4538     dst    : S4(write);
4539     dst    : S3(read);
4540     DECODE : S0(2);     // any 2 decoders
4541     ALU    : S3(2);     // both alus
4542 %}
4543 
4544 // Integer ALU reg operation using big decoder
4545 pipe_class ialu_reg_fat(rRegI dst)
4546 %{
4547     single_instruction;
4548     dst    : S4(write);
4549     dst    : S3(read);
4550     D0     : S0;        // big decoder only
4551     ALU    : S3;        // any alu
4552 %}
4553 
4554 // Long ALU reg operation using big decoder
4555 pipe_class ialu_reg_long_fat(rRegL dst)
4556 %{
4557     instruction_count(2);
4558     dst    : S4(write);
4559     dst    : S3(read);
4560     D0     : S0(2);     // big decoder only; twice
4561     ALU    : S3(2);     // any 2 alus
4562 %}
4563 
4564 // Integer ALU reg-reg operation
4565 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4566 %{
4567     single_instruction;
4568     dst    : S4(write);
4569     src    : S3(read);
4570     DECODE : S0;        // any decoder
4571     ALU    : S3;        // any alu
4572 %}
4573 
4574 // Long ALU reg-reg operation
4575 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4576 %{
4577     instruction_count(2);
4578     dst    : S4(write);
4579     src    : S3(read);
4580     DECODE : S0(2);     // any 2 decoders
4581     ALU    : S3(2);     // both alus
4582 %}
4583 
4584 // Integer ALU reg-reg operation
4585 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4586 %{
4587     single_instruction;
4588     dst    : S4(write);
4589     src    : S3(read);
4590     D0     : S0;        // big decoder only
4591     ALU    : S3;        // any alu
4592 %}
4593 
4594 // Long ALU reg-reg operation
4595 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4596 %{
4597     instruction_count(2);
4598     dst    : S4(write);
4599     src    : S3(read);
4600     D0     : S0(2);     // big decoder only; twice
4601     ALU    : S3(2);     // both alus
4602 %}
4603 
4604 // Integer ALU reg-mem operation
4605 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4606 %{
4607     single_instruction;
4608     dst    : S5(write);
4609     mem    : S3(read);
4610     D0     : S0;        // big decoder only
4611     ALU    : S4;        // any alu
4612     MEM    : S3;        // any mem
4613 %}
4614 
4615 // Integer mem operation (prefetch)
4616 pipe_class ialu_mem(memory mem)
4617 %{
4618     single_instruction;
4619     mem    : S3(read);
4620     D0     : S0;        // big decoder only
4621     MEM    : S3;        // any mem
4622 %}
4623 
4624 // Integer Store to Memory
4625 pipe_class ialu_mem_reg(memory mem, rRegI src)
4626 %{
4627     single_instruction;
4628     mem    : S3(read);
4629     src    : S5(read);
4630     D0     : S0;        // big decoder only
4631     ALU    : S4;        // any alu
4632     MEM    : S3;
4633 %}
4634 
4635 // // Long Store to Memory
4636 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4637 // %{
4638 //     instruction_count(2);
4639 //     mem    : S3(read);
4640 //     src    : S5(read);
4641 //     D0     : S0(2);          // big decoder only; twice
4642 //     ALU    : S4(2);     // any 2 alus
4643 //     MEM    : S3(2);  // Both mems
4644 // %}
4645 
4646 // Integer Store to Memory
4647 pipe_class ialu_mem_imm(memory mem)
4648 %{
4649     single_instruction;
4650     mem    : S3(read);
4651     D0     : S0;        // big decoder only
4652     ALU    : S4;        // any alu
4653     MEM    : S3;
4654 %}
4655 
4656 // Integer ALU0 reg-reg operation
4657 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4658 %{
4659     single_instruction;
4660     dst    : S4(write);
4661     src    : S3(read);
4662     D0     : S0;        // Big decoder only
4663     ALU0   : S3;        // only alu0
4664 %}
4665 
4666 // Integer ALU0 reg-mem operation
4667 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4668 %{
4669     single_instruction;
4670     dst    : S5(write);
4671     mem    : S3(read);
4672     D0     : S0;        // big decoder only
4673     ALU0   : S4;        // ALU0 only
4674     MEM    : S3;        // any mem
4675 %}
4676 
4677 // Integer ALU reg-reg operation
4678 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4679 %{
4680     single_instruction;
4681     cr     : S4(write);
4682     src1   : S3(read);
4683     src2   : S3(read);
4684     DECODE : S0;        // any decoder
4685     ALU    : S3;        // any alu
4686 %}
4687 
4688 // Integer ALU reg-imm operation
4689 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4690 %{
4691     single_instruction;
4692     cr     : S4(write);
4693     src1   : S3(read);
4694     DECODE : S0;        // any decoder
4695     ALU    : S3;        // any alu
4696 %}
4697 
4698 // Integer ALU reg-mem operation
4699 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4700 %{
4701     single_instruction;
4702     cr     : S4(write);
4703     src1   : S3(read);
4704     src2   : S3(read);
4705     D0     : S0;        // big decoder only
4706     ALU    : S4;        // any alu
4707     MEM    : S3;
4708 %}
4709 
4710 // Conditional move reg-reg
4711 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4712 %{
4713     instruction_count(4);
4714     y      : S4(read);
4715     q      : S3(read);
4716     p      : S3(read);
4717     DECODE : S0(4);     // any decoder
4718 %}
4719 
4720 // Conditional move reg-reg
4721 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4722 %{
4723     single_instruction;
4724     dst    : S4(write);
4725     src    : S3(read);
4726     cr     : S3(read);
4727     DECODE : S0;        // any decoder
4728 %}
4729 
4730 // Conditional move reg-mem
4731 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4732 %{
4733     single_instruction;
4734     dst    : S4(write);
4735     src    : S3(read);
4736     cr     : S3(read);
4737     DECODE : S0;        // any decoder
4738     MEM    : S3;
4739 %}
4740 
4741 // Conditional move reg-reg long
4742 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4743 %{
4744     single_instruction;
4745     dst    : S4(write);
4746     src    : S3(read);
4747     cr     : S3(read);
4748     DECODE : S0(2);     // any 2 decoders
4749 %}
4750 
4751 // XXX
4752 // // Conditional move double reg-reg
4753 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4754 // %{
4755 //     single_instruction;
4756 //     dst    : S4(write);
4757 //     src    : S3(read);
4758 //     cr     : S3(read);
4759 //     DECODE : S0;     // any decoder
4760 // %}
4761 
4762 // Float reg-reg operation
4763 pipe_class fpu_reg(regD dst)
4764 %{
4765     instruction_count(2);
4766     dst    : S3(read);
4767     DECODE : S0(2);     // any 2 decoders
4768     FPU    : S3;
4769 %}
4770 
4771 // Float reg-reg operation
4772 pipe_class fpu_reg_reg(regD dst, regD src)
4773 %{
4774     instruction_count(2);
4775     dst    : S4(write);
4776     src    : S3(read);
4777     DECODE : S0(2);     // any 2 decoders
4778     FPU    : S3;
4779 %}
4780 
4781 // Float reg-reg operation
4782 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4783 %{
4784     instruction_count(3);
4785     dst    : S4(write);
4786     src1   : S3(read);
4787     src2   : S3(read);
4788     DECODE : S0(3);     // any 3 decoders
4789     FPU    : S3(2);
4790 %}
4791 
4792 // Float reg-reg operation
4793 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4794 %{
4795     instruction_count(4);
4796     dst    : S4(write);
4797     src1   : S3(read);
4798     src2   : S3(read);
4799     src3   : S3(read);
4800     DECODE : S0(4);     // any 3 decoders
4801     FPU    : S3(2);
4802 %}
4803 
4804 // Float reg-reg operation
4805 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4806 %{
4807     instruction_count(4);
4808     dst    : S4(write);
4809     src1   : S3(read);
4810     src2   : S3(read);
4811     src3   : S3(read);
4812     DECODE : S1(3);     // any 3 decoders
4813     D0     : S0;        // Big decoder only
4814     FPU    : S3(2);
4815     MEM    : S3;
4816 %}
4817 
4818 // Float reg-mem operation
4819 pipe_class fpu_reg_mem(regD dst, memory mem)
4820 %{
4821     instruction_count(2);
4822     dst    : S5(write);
4823     mem    : S3(read);
4824     D0     : S0;        // big decoder only
4825     DECODE : S1;        // any decoder for FPU POP
4826     FPU    : S4;
4827     MEM    : S3;        // any mem
4828 %}
4829 
4830 // Float reg-mem operation
4831 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4832 %{
4833     instruction_count(3);
4834     dst    : S5(write);
4835     src1   : S3(read);
4836     mem    : S3(read);
4837     D0     : S0;        // big decoder only
4838     DECODE : S1(2);     // any decoder for FPU POP
4839     FPU    : S4;
4840     MEM    : S3;        // any mem
4841 %}
4842 
4843 // Float mem-reg operation
4844 pipe_class fpu_mem_reg(memory mem, regD src)
4845 %{
4846     instruction_count(2);
4847     src    : S5(read);
4848     mem    : S3(read);
4849     DECODE : S0;        // any decoder for FPU PUSH
4850     D0     : S1;        // big decoder only
4851     FPU    : S4;
4852     MEM    : S3;        // any mem
4853 %}
4854 
4855 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4856 %{
4857     instruction_count(3);
4858     src1   : S3(read);
4859     src2   : S3(read);
4860     mem    : S3(read);
4861     DECODE : S0(2);     // any decoder for FPU PUSH
4862     D0     : S1;        // big decoder only
4863     FPU    : S4;
4864     MEM    : S3;        // any mem
4865 %}
4866 
4867 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4868 %{
4869     instruction_count(3);
4870     src1   : S3(read);
4871     src2   : S3(read);
4872     mem    : S4(read);
4873     DECODE : S0;        // any decoder for FPU PUSH
4874     D0     : S0(2);     // big decoder only
4875     FPU    : S4;
4876     MEM    : S3(2);     // any mem
4877 %}
4878 
4879 pipe_class fpu_mem_mem(memory dst, memory src1)
4880 %{
4881     instruction_count(2);
4882     src1   : S3(read);
4883     dst    : S4(read);
4884     D0     : S0(2);     // big decoder only
4885     MEM    : S3(2);     // any mem
4886 %}
4887 
4888 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4889 %{
4890     instruction_count(3);
4891     src1   : S3(read);
4892     src2   : S3(read);
4893     dst    : S4(read);
4894     D0     : S0(3);     // big decoder only
4895     FPU    : S4;
4896     MEM    : S3(3);     // any mem
4897 %}
4898 
4899 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4900 %{
4901     instruction_count(3);
4902     src1   : S4(read);
4903     mem    : S4(read);
4904     DECODE : S0;        // any decoder for FPU PUSH
4905     D0     : S0(2);     // big decoder only
4906     FPU    : S4;
4907     MEM    : S3(2);     // any mem
4908 %}
4909 
4910 // Float load constant
4911 pipe_class fpu_reg_con(regD dst)
4912 %{
4913     instruction_count(2);
4914     dst    : S5(write);
4915     D0     : S0;        // big decoder only for the load
4916     DECODE : S1;        // any decoder for FPU POP
4917     FPU    : S4;
4918     MEM    : S3;        // any mem
4919 %}
4920 
4921 // Float load constant
4922 pipe_class fpu_reg_reg_con(regD dst, regD src)
4923 %{
4924     instruction_count(3);
4925     dst    : S5(write);
4926     src    : S3(read);
4927     D0     : S0;        // big decoder only for the load
4928     DECODE : S1(2);     // any decoder for FPU POP
4929     FPU    : S4;
4930     MEM    : S3;        // any mem
4931 %}
4932 
4933 // UnConditional branch
4934 pipe_class pipe_jmp(label labl)
4935 %{
4936     single_instruction;
4937     BR   : S3;
4938 %}
4939 
4940 // Conditional branch
4941 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4942 %{
4943     single_instruction;
4944     cr    : S1(read);
4945     BR    : S3;
4946 %}
4947 
4948 // Allocation idiom
4949 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4950 %{
4951     instruction_count(1); force_serialization;
4952     fixed_latency(6);
4953     heap_ptr : S3(read);
4954     DECODE   : S0(3);
4955     D0       : S2;
4956     MEM      : S3;
4957     ALU      : S3(2);
4958     dst      : S5(write);
4959     BR       : S5;
4960 %}
4961 
4962 // Generic big/slow expanded idiom
4963 pipe_class pipe_slow()
4964 %{
4965     instruction_count(10); multiple_bundles; force_serialization;
4966     fixed_latency(100);
4967     D0  : S0(2);
4968     MEM : S3(2);
4969 %}
4970 
4971 // The real do-nothing guy
4972 pipe_class empty()
4973 %{
4974     instruction_count(0);
4975 %}
4976 
4977 // Define the class for the Nop node
4978 define
4979 %{
4980    MachNop = empty;
4981 %}
4982 
4983 %}
4984 
4985 //----------INSTRUCTIONS-------------------------------------------------------
4986 //
4987 // match      -- States which machine-independent subtree may be replaced
4988 //               by this instruction.
4989 // ins_cost   -- The estimated cost of this instruction is used by instruction
4990 //               selection to identify a minimum cost tree of machine
4991 //               instructions that matches a tree of machine-independent
4992 //               instructions.
4993 // format     -- A string providing the disassembly for this instruction.
4994 //               The value of an instruction's operand may be inserted
4995 //               by referring to it with a '$' prefix.
4996 // opcode     -- Three instruction opcodes may be provided.  These are referred
4997 //               to within an encode class as $primary, $secondary, and $tertiary
4998 //               rrspectively.  The primary opcode is commonly used to
4999 //               indicate the type of machine instruction, while secondary
5000 //               and tertiary are often used for prefix options or addressing
5001 //               modes.
5002 // ins_encode -- A list of encode classes with parameters. The encode class
5003 //               name must have been defined in an 'enc_class' specification
5004 //               in the encode section of the architecture description.
5005 
5006 
5007 //----------Load/Store/Move Instructions---------------------------------------
5008 //----------Load Instructions--------------------------------------------------
5009 
5010 // Load Byte (8 bit signed)
5011 instruct loadB(rRegI dst, memory mem)
5012 %{
5013   match(Set dst (LoadB mem));
5014 
5015   ins_cost(125);
5016   format %{ "movsbl  $dst, $mem\t# byte" %}
5017 
5018   ins_encode %{
5019     __ movsbl($dst$$Register, $mem$$Address);
5020   %}
5021 
5022   ins_pipe(ialu_reg_mem);
5023 %}
5024 
5025 // Load Byte (8 bit signed) into Long Register
5026 instruct loadB2L(rRegL dst, memory mem)
5027 %{
5028   match(Set dst (ConvI2L (LoadB mem)));
5029 
5030   ins_cost(125);
5031   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
5032 
5033   ins_encode %{
5034     __ movsbq($dst$$Register, $mem$$Address);
5035   %}
5036 
5037   ins_pipe(ialu_reg_mem);
5038 %}
5039 
5040 // Load Unsigned Byte (8 bit UNsigned)
5041 instruct loadUB(rRegI dst, memory mem)
5042 %{
5043   match(Set dst (LoadUB mem));
5044 
5045   ins_cost(125);
5046   format %{ "movzbl  $dst, $mem\t# ubyte" %}
5047 
5048   ins_encode %{
5049     __ movzbl($dst$$Register, $mem$$Address);
5050   %}
5051 
5052   ins_pipe(ialu_reg_mem);
5053 %}
5054 
5055 // Load Unsigned Byte (8 bit UNsigned) into Long Register
5056 instruct loadUB2L(rRegL dst, memory mem)
5057 %{
5058   match(Set dst (ConvI2L (LoadUB mem)));
5059 
5060   ins_cost(125);
5061   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
5062 
5063   ins_encode %{
5064     __ movzbq($dst$$Register, $mem$$Address);
5065   %}
5066 
5067   ins_pipe(ialu_reg_mem);
5068 %}
5069 
5070 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
5071 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5072   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5073   effect(KILL cr);
5074 
5075   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
5076             "andl    $dst, right_n_bits($mask, 8)" %}
5077   ins_encode %{
5078     Register Rdst = $dst$$Register;
5079     __ movzbq(Rdst, $mem$$Address);
5080     __ andl(Rdst, $mask$$constant & right_n_bits(8));
5081   %}
5082   ins_pipe(ialu_reg_mem);
5083 %}
5084 
5085 // Load Short (16 bit signed)
5086 instruct loadS(rRegI dst, memory mem)
5087 %{
5088   match(Set dst (LoadS mem));
5089 
5090   ins_cost(125);
5091   format %{ "movswl $dst, $mem\t# short" %}
5092 
5093   ins_encode %{
5094     __ movswl($dst$$Register, $mem$$Address);
5095   %}
5096 
5097   ins_pipe(ialu_reg_mem);
5098 %}
5099 
5100 // Load Short (16 bit signed) to Byte (8 bit signed)
5101 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5102   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5103 
5104   ins_cost(125);
5105   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5106   ins_encode %{
5107     __ movsbl($dst$$Register, $mem$$Address);
5108   %}
5109   ins_pipe(ialu_reg_mem);
5110 %}
5111 
5112 // Load Short (16 bit signed) into Long Register
5113 instruct loadS2L(rRegL dst, memory mem)
5114 %{
5115   match(Set dst (ConvI2L (LoadS mem)));
5116 
5117   ins_cost(125);
5118   format %{ "movswq $dst, $mem\t# short -> long" %}
5119 
5120   ins_encode %{
5121     __ movswq($dst$$Register, $mem$$Address);
5122   %}
5123 
5124   ins_pipe(ialu_reg_mem);
5125 %}
5126 
5127 // Load Unsigned Short/Char (16 bit UNsigned)
5128 instruct loadUS(rRegI dst, memory mem)
5129 %{
5130   match(Set dst (LoadUS mem));
5131 
5132   ins_cost(125);
5133   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5134 
5135   ins_encode %{
5136     __ movzwl($dst$$Register, $mem$$Address);
5137   %}
5138 
5139   ins_pipe(ialu_reg_mem);
5140 %}
5141 
5142 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5143 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5144   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5145 
5146   ins_cost(125);
5147   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5148   ins_encode %{
5149     __ movsbl($dst$$Register, $mem$$Address);
5150   %}
5151   ins_pipe(ialu_reg_mem);
5152 %}
5153 
5154 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5155 instruct loadUS2L(rRegL dst, memory mem)
5156 %{
5157   match(Set dst (ConvI2L (LoadUS mem)));
5158 
5159   ins_cost(125);
5160   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5161 
5162   ins_encode %{
5163     __ movzwq($dst$$Register, $mem$$Address);
5164   %}
5165 
5166   ins_pipe(ialu_reg_mem);
5167 %}
5168 
5169 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5170 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5171   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5172 
5173   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5174   ins_encode %{
5175     __ movzbq($dst$$Register, $mem$$Address);
5176   %}
5177   ins_pipe(ialu_reg_mem);
5178 %}
5179 
5180 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5181 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5182   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5183   effect(KILL cr);
5184 
5185   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5186             "andl    $dst, right_n_bits($mask, 16)" %}
5187   ins_encode %{
5188     Register Rdst = $dst$$Register;
5189     __ movzwq(Rdst, $mem$$Address);
5190     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5191   %}
5192   ins_pipe(ialu_reg_mem);
5193 %}
5194 
5195 // Load Integer
5196 instruct loadI(rRegI dst, memory mem)
5197 %{
5198   match(Set dst (LoadI mem));
5199 
5200   ins_cost(125);
5201   format %{ "movl    $dst, $mem\t# int" %}
5202 
5203   ins_encode %{
5204     __ movl($dst$$Register, $mem$$Address);
5205   %}
5206 
5207   ins_pipe(ialu_reg_mem);
5208 %}
5209 
5210 // Load Integer (32 bit signed) to Byte (8 bit signed)
5211 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5212   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5213 
5214   ins_cost(125);
5215   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5216   ins_encode %{
5217     __ movsbl($dst$$Register, $mem$$Address);
5218   %}
5219   ins_pipe(ialu_reg_mem);
5220 %}
5221 
5222 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5223 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5224   match(Set dst (AndI (LoadI mem) mask));
5225 
5226   ins_cost(125);
5227   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5228   ins_encode %{
5229     __ movzbl($dst$$Register, $mem$$Address);
5230   %}
5231   ins_pipe(ialu_reg_mem);
5232 %}
5233 
5234 // Load Integer (32 bit signed) to Short (16 bit signed)
5235 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5236   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5237 
5238   ins_cost(125);
5239   format %{ "movswl  $dst, $mem\t# int -> short" %}
5240   ins_encode %{
5241     __ movswl($dst$$Register, $mem$$Address);
5242   %}
5243   ins_pipe(ialu_reg_mem);
5244 %}
5245 
5246 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5247 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5248   match(Set dst (AndI (LoadI mem) mask));
5249 
5250   ins_cost(125);
5251   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5252   ins_encode %{
5253     __ movzwl($dst$$Register, $mem$$Address);
5254   %}
5255   ins_pipe(ialu_reg_mem);
5256 %}
5257 
5258 // Load Integer into Long Register
5259 instruct loadI2L(rRegL dst, memory mem)
5260 %{
5261   match(Set dst (ConvI2L (LoadI mem)));
5262 
5263   ins_cost(125);
5264   format %{ "movslq  $dst, $mem\t# int -> long" %}
5265 
5266   ins_encode %{
5267     __ movslq($dst$$Register, $mem$$Address);
5268   %}
5269 
5270   ins_pipe(ialu_reg_mem);
5271 %}
5272 
5273 // Load Integer with mask 0xFF into Long Register
5274 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5275   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5276 
5277   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5278   ins_encode %{
5279     __ movzbq($dst$$Register, $mem$$Address);
5280   %}
5281   ins_pipe(ialu_reg_mem);
5282 %}
5283 
5284 // Load Integer with mask 0xFFFF into Long Register
5285 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5286   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5287 
5288   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5289   ins_encode %{
5290     __ movzwq($dst$$Register, $mem$$Address);
5291   %}
5292   ins_pipe(ialu_reg_mem);
5293 %}
5294 
5295 // Load Integer with a 31-bit mask into Long Register
5296 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5297   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5298   effect(KILL cr);
5299 
5300   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5301             "andl    $dst, $mask" %}
5302   ins_encode %{
5303     Register Rdst = $dst$$Register;
5304     __ movl(Rdst, $mem$$Address);
5305     __ andl(Rdst, $mask$$constant);
5306   %}
5307   ins_pipe(ialu_reg_mem);
5308 %}
5309 
5310 // Load Unsigned Integer into Long Register
5311 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5312 %{
5313   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5314 
5315   ins_cost(125);
5316   format %{ "movl    $dst, $mem\t# uint -> long" %}
5317 
5318   ins_encode %{
5319     __ movl($dst$$Register, $mem$$Address);
5320   %}
5321 
5322   ins_pipe(ialu_reg_mem);
5323 %}
5324 
5325 // Load Long
5326 instruct loadL(rRegL dst, memory mem)
5327 %{
5328   match(Set dst (LoadL mem));
5329 
5330   ins_cost(125);
5331   format %{ "movq    $dst, $mem\t# long" %}
5332 
5333   ins_encode %{
5334     __ movq($dst$$Register, $mem$$Address);
5335   %}
5336 
5337   ins_pipe(ialu_reg_mem); // XXX
5338 %}
5339 
5340 // Load Range
5341 instruct loadRange(rRegI dst, memory mem)
5342 %{
5343   match(Set dst (LoadRange mem));
5344 
5345   ins_cost(125); // XXX
5346   format %{ "movl    $dst, $mem\t# range" %}
5347   opcode(0x8B);
5348   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5349   ins_pipe(ialu_reg_mem);
5350 %}
5351 
5352 // Load Pointer
5353 instruct loadP(rRegP dst, memory mem)
5354 %{
5355   match(Set dst (LoadP mem));
5356 
5357   ins_cost(125); // XXX
5358   format %{ "movq    $dst, $mem\t# ptr" %}
5359   opcode(0x8B);
5360   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5361   ins_pipe(ialu_reg_mem); // XXX
5362 %}
5363 
5364 // Load Compressed Pointer
5365 instruct loadN(rRegN dst, memory mem)
5366 %{
5367    match(Set dst (LoadN mem));
5368 
5369    ins_cost(125); // XXX
5370    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5371    ins_encode %{
5372      __ movl($dst$$Register, $mem$$Address);
5373    %}
5374    ins_pipe(ialu_reg_mem); // XXX
5375 %}
5376 
5377 
5378 // Load Klass Pointer
5379 instruct loadKlass(rRegP dst, memory mem)
5380 %{
5381   match(Set dst (LoadKlass mem));
5382 
5383   ins_cost(125); // XXX
5384   format %{ "movq    $dst, $mem\t# class" %}
5385   opcode(0x8B);
5386   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5387   ins_pipe(ialu_reg_mem); // XXX
5388 %}
5389 
5390 // Load narrow Klass Pointer
5391 instruct loadNKlass(rRegN dst, memory mem)
5392 %{
5393   match(Set dst (LoadNKlass mem));
5394 
5395   ins_cost(125); // XXX
5396   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5397   ins_encode %{
5398     __ movl($dst$$Register, $mem$$Address);
5399   %}
5400   ins_pipe(ialu_reg_mem); // XXX
5401 %}
5402 
5403 // Load Float
5404 instruct loadF(regF dst, memory mem)
5405 %{
5406   match(Set dst (LoadF mem));
5407 
5408   ins_cost(145); // XXX
5409   format %{ "movss   $dst, $mem\t# float" %}
5410   ins_encode %{
5411     __ movflt($dst$$XMMRegister, $mem$$Address);
5412   %}
5413   ins_pipe(pipe_slow); // XXX
5414 %}
5415 
5416 // Load Float
5417 instruct MoveF2VL(vlRegF dst, regF src) %{
5418   match(Set dst src);
5419   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5420   ins_encode %{
5421     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5422   %}
5423   ins_pipe( fpu_reg_reg );
5424 %}
5425 
5426 // Load Float
5427 instruct MoveF2LEG(legRegF dst, regF src) %{
5428   match(Set dst src);
5429   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
5430   ins_encode %{
5431     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5432   %}
5433   ins_pipe( fpu_reg_reg );
5434 %}
5435 
5436 // Load Float
5437 instruct MoveVL2F(regF dst, vlRegF src) %{
5438   match(Set dst src);
5439   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5440   ins_encode %{
5441     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5442   %}
5443   ins_pipe( fpu_reg_reg );
5444 %}
5445 
5446 // Load Float
5447 instruct MoveLEG2F(regF dst, legRegF src) %{
5448   match(Set dst src);
5449   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
5450   ins_encode %{
5451     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5452   %}
5453   ins_pipe( fpu_reg_reg );
5454 %}
5455 
5456 // Load Double
5457 instruct loadD_partial(regD dst, memory mem)
5458 %{
5459   predicate(!UseXmmLoadAndClearUpper);
5460   match(Set dst (LoadD mem));
5461 
5462   ins_cost(145); // XXX
5463   format %{ "movlpd  $dst, $mem\t# double" %}
5464   ins_encode %{
5465     __ movdbl($dst$$XMMRegister, $mem$$Address);
5466   %}
5467   ins_pipe(pipe_slow); // XXX
5468 %}
5469 
5470 instruct loadD(regD dst, memory mem)
5471 %{
5472   predicate(UseXmmLoadAndClearUpper);
5473   match(Set dst (LoadD mem));
5474 
5475   ins_cost(145); // XXX
5476   format %{ "movsd   $dst, $mem\t# double" %}
5477   ins_encode %{
5478     __ movdbl($dst$$XMMRegister, $mem$$Address);
5479   %}
5480   ins_pipe(pipe_slow); // XXX
5481 %}
5482 
5483 // Load Double
5484 instruct MoveD2VL(vlRegD dst, regD src) %{
5485   match(Set dst src);
5486   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5487   ins_encode %{
5488     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5489   %}
5490   ins_pipe( fpu_reg_reg );
5491 %}
5492 
5493 // Load Double
5494 instruct MoveD2LEG(legRegD dst, regD src) %{
5495   match(Set dst src);
5496   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
5497   ins_encode %{
5498     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5499   %}
5500   ins_pipe( fpu_reg_reg );
5501 %}
5502 
5503 // Load Double
5504 instruct MoveVL2D(regD dst, vlRegD src) %{
5505   match(Set dst src);
5506   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5507   ins_encode %{
5508     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5509   %}
5510   ins_pipe( fpu_reg_reg );
5511 %}
5512 
5513 // Load Double
5514 instruct MoveLEG2D(regD dst, legRegD src) %{
5515   match(Set dst src);
5516   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
5517   ins_encode %{
5518     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5519   %}
5520   ins_pipe( fpu_reg_reg );
5521 %}
5522 
5523 // Following pseudo code describes the algorithm for max[FD]:
5524 // Min algorithm is on similar lines
5525 //  btmp = (b < +0.0) ? a : b
5526 //  atmp = (b < +0.0) ? b : a
5527 //  Tmp  = Max_Float(atmp , btmp)
5528 //  Res  = (atmp == NaN) ? atmp : Tmp
5529 
5530 // max = java.lang.Math.max(float a, float b)
5531 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
5532   predicate(UseAVX > 0 && !n->is_reduction());
5533   match(Set dst (MaxF a b));
5534   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5535   format %{
5536      "blendvps         $btmp,$b,$a,$b           \n\t"
5537      "blendvps         $atmp,$a,$b,$b           \n\t"
5538      "vmaxss           $tmp,$atmp,$btmp         \n\t"
5539      "cmpps.unordered  $btmp,$atmp,$atmp        \n\t"
5540      "blendvps         $dst,$tmp,$atmp,$btmp    \n\t"
5541   %}
5542   ins_encode %{
5543     int vector_len = Assembler::AVX_128bit;
5544     __ blendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
5545     __ blendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
5546     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5547     __ cmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5548     __ blendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5549  %}
5550   ins_pipe( pipe_slow );
5551 %}
5552 
5553 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
5554   predicate(UseAVX > 0 && n->is_reduction());
5555   match(Set dst (MaxF a b));
5556   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5557 
5558   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
5559   ins_encode %{
5560     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5561                     false /*min*/, true /*single*/);
5562   %}
5563   ins_pipe( pipe_slow );
5564 %}
5565 
5566 // max = java.lang.Math.max(double a, double b)
5567 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
5568   predicate(UseAVX > 0 && !n->is_reduction());
5569   match(Set dst (MaxD a b));
5570   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
5571   format %{
5572      "blendvpd         $btmp,$b,$a,$b            \n\t"
5573      "blendvpd         $atmp,$a,$b,$b            \n\t"
5574      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
5575      "cmppd.unordered  $btmp,$atmp,$atmp         \n\t"
5576      "blendvpd         $dst,$tmp,$atmp,$btmp     \n\t"
5577   %}
5578   ins_encode %{
5579     int vector_len = Assembler::AVX_128bit;
5580     __ blendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
5581     __ blendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
5582     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5583     __ cmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5584     __ blendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5585   %}
5586   ins_pipe( pipe_slow );
5587 %}
5588 
5589 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
5590   predicate(UseAVX > 0 && n->is_reduction());
5591   match(Set dst (MaxD a b));
5592   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5593 
5594   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
5595   ins_encode %{
5596     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5597                     false /*min*/, false /*single*/);
5598   %}
5599   ins_pipe( pipe_slow );
5600 %}
5601 
5602 // min = java.lang.Math.min(float a, float b)
5603 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
5604   predicate(UseAVX > 0 && !n->is_reduction());
5605   match(Set dst (MinF a b));
5606   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5607   format %{
5608      "blendvps         $atmp,$a,$b,$a             \n\t"
5609      "blendvps         $btmp,$b,$a,$a             \n\t"
5610      "vminss           $tmp,$atmp,$btmp           \n\t"
5611      "cmpps.unordered  $btmp,$atmp,$atmp          \n\t"
5612      "blendvps         $dst,$tmp,$atmp,$btmp      \n\t"
5613   %}
5614   ins_encode %{
5615     int vector_len = Assembler::AVX_128bit;
5616     __ blendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
5617     __ blendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
5618     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5619     __ cmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5620     __ blendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5621   %}
5622   ins_pipe( pipe_slow );
5623 %}
5624 
5625 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
5626   predicate(UseAVX > 0 && n->is_reduction());
5627   match(Set dst (MinF a b));
5628   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5629 
5630   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
5631   ins_encode %{
5632     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5633                     true /*min*/, true /*single*/);
5634   %}
5635   ins_pipe( pipe_slow );
5636 %}
5637 
5638 // min = java.lang.Math.min(double a, double b)
5639 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
5640   predicate(UseAVX > 0 && !n->is_reduction());
5641   match(Set dst (MinD a b));
5642   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5643   format %{
5644      "blendvpd         $atmp,$a,$b,$a           \n\t"
5645      "blendvpd         $btmp,$b,$a,$a           \n\t"
5646      "vminsd           $tmp,$atmp,$btmp         \n\t"
5647      "cmppd.unordered  $btmp,$atmp,$atmp        \n\t"
5648      "blendvpd         $dst,$tmp,$atmp,$btmp    \n\t"
5649   %}
5650   ins_encode %{
5651     int vector_len = Assembler::AVX_128bit;
5652     __ blendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
5653     __ blendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
5654     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5655     __ cmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5656     __ blendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5657   %}
5658   ins_pipe( pipe_slow );
5659 %}
5660 
5661 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
5662   predicate(UseAVX > 0 && n->is_reduction());
5663   match(Set dst (MinD a b));
5664   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5665 
5666   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
5667   ins_encode %{
5668     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5669                     true /*min*/, false /*single*/);
5670   %}
5671   ins_pipe( pipe_slow );
5672 %}
5673 
5674 // Load Effective Address
5675 instruct leaP8(rRegP dst, indOffset8 mem)
5676 %{
5677   match(Set dst mem);
5678 
5679   ins_cost(110); // XXX
5680   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5681   opcode(0x8D);
5682   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5683   ins_pipe(ialu_reg_reg_fat);
5684 %}
5685 
5686 instruct leaP32(rRegP dst, indOffset32 mem)
5687 %{
5688   match(Set dst mem);
5689 
5690   ins_cost(110);
5691   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5692   opcode(0x8D);
5693   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5694   ins_pipe(ialu_reg_reg_fat);
5695 %}
5696 
5697 // instruct leaPIdx(rRegP dst, indIndex mem)
5698 // %{
5699 //   match(Set dst mem);
5700 
5701 //   ins_cost(110);
5702 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5703 //   opcode(0x8D);
5704 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5705 //   ins_pipe(ialu_reg_reg_fat);
5706 // %}
5707 
5708 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5709 %{
5710   match(Set dst mem);
5711 
5712   ins_cost(110);
5713   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5714   opcode(0x8D);
5715   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5716   ins_pipe(ialu_reg_reg_fat);
5717 %}
5718 
5719 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5720 %{
5721   match(Set dst mem);
5722 
5723   ins_cost(110);
5724   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5725   opcode(0x8D);
5726   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5727   ins_pipe(ialu_reg_reg_fat);
5728 %}
5729 
5730 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5731 %{
5732   match(Set dst mem);
5733 
5734   ins_cost(110);
5735   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5736   opcode(0x8D);
5737   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5738   ins_pipe(ialu_reg_reg_fat);
5739 %}
5740 
5741 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5742 %{
5743   match(Set dst mem);
5744 
5745   ins_cost(110);
5746   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5747   opcode(0x8D);
5748   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5749   ins_pipe(ialu_reg_reg_fat);
5750 %}
5751 
5752 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5753 %{
5754   match(Set dst mem);
5755 
5756   ins_cost(110);
5757   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5758   opcode(0x8D);
5759   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5760   ins_pipe(ialu_reg_reg_fat);
5761 %}
5762 
5763 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5764 %{
5765   match(Set dst mem);
5766 
5767   ins_cost(110);
5768   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5769   opcode(0x8D);
5770   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5771   ins_pipe(ialu_reg_reg_fat);
5772 %}
5773 
5774 // Load Effective Address which uses Narrow (32-bits) oop
5775 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5776 %{
5777   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
5778   match(Set dst mem);
5779 
5780   ins_cost(110);
5781   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5782   opcode(0x8D);
5783   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5784   ins_pipe(ialu_reg_reg_fat);
5785 %}
5786 
5787 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5788 %{
5789   predicate(CompressedOops::shift() == 0);
5790   match(Set dst mem);
5791 
5792   ins_cost(110); // XXX
5793   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5794   opcode(0x8D);
5795   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5796   ins_pipe(ialu_reg_reg_fat);
5797 %}
5798 
5799 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5800 %{
5801   predicate(CompressedOops::shift() == 0);
5802   match(Set dst mem);
5803 
5804   ins_cost(110);
5805   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5806   opcode(0x8D);
5807   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5808   ins_pipe(ialu_reg_reg_fat);
5809 %}
5810 
5811 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5812 %{
5813   predicate(CompressedOops::shift() == 0);
5814   match(Set dst mem);
5815 
5816   ins_cost(110);
5817   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5818   opcode(0x8D);
5819   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5820   ins_pipe(ialu_reg_reg_fat);
5821 %}
5822 
5823 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5824 %{
5825   predicate(CompressedOops::shift() == 0);
5826   match(Set dst mem);
5827 
5828   ins_cost(110);
5829   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5830   opcode(0x8D);
5831   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5832   ins_pipe(ialu_reg_reg_fat);
5833 %}
5834 
5835 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5836 %{
5837   predicate(CompressedOops::shift() == 0);
5838   match(Set dst mem);
5839 
5840   ins_cost(110);
5841   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5842   opcode(0x8D);
5843   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5844   ins_pipe(ialu_reg_reg_fat);
5845 %}
5846 
5847 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5848 %{
5849   predicate(CompressedOops::shift() == 0);
5850   match(Set dst mem);
5851 
5852   ins_cost(110);
5853   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5854   opcode(0x8D);
5855   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5856   ins_pipe(ialu_reg_reg_fat);
5857 %}
5858 
5859 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5860 %{
5861   predicate(CompressedOops::shift() == 0);
5862   match(Set dst mem);
5863 
5864   ins_cost(110);
5865   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5866   opcode(0x8D);
5867   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5868   ins_pipe(ialu_reg_reg_fat);
5869 %}
5870 
5871 instruct loadConI(rRegI dst, immI src)
5872 %{
5873   match(Set dst src);
5874 
5875   format %{ "movl    $dst, $src\t# int" %}
5876   ins_encode(load_immI(dst, src));
5877   ins_pipe(ialu_reg_fat); // XXX
5878 %}
5879 
5880 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5881 %{
5882   match(Set dst src);
5883   effect(KILL cr);
5884 
5885   ins_cost(50);
5886   format %{ "xorl    $dst, $dst\t# int" %}
5887   opcode(0x33); /* + rd */
5888   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5889   ins_pipe(ialu_reg);
5890 %}
5891 
5892 instruct loadConL(rRegL dst, immL src)
5893 %{
5894   match(Set dst src);
5895 
5896   ins_cost(150);
5897   format %{ "movq    $dst, $src\t# long" %}
5898   ins_encode(load_immL(dst, src));
5899   ins_pipe(ialu_reg);
5900 %}
5901 
5902 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5903 %{
5904   match(Set dst src);
5905   effect(KILL cr);
5906 
5907   ins_cost(50);
5908   format %{ "xorl    $dst, $dst\t# long" %}
5909   opcode(0x33); /* + rd */
5910   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5911   ins_pipe(ialu_reg); // XXX
5912 %}
5913 
5914 instruct loadConUL32(rRegL dst, immUL32 src)
5915 %{
5916   match(Set dst src);
5917 
5918   ins_cost(60);
5919   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5920   ins_encode(load_immUL32(dst, src));
5921   ins_pipe(ialu_reg);
5922 %}
5923 
5924 instruct loadConL32(rRegL dst, immL32 src)
5925 %{
5926   match(Set dst src);
5927 
5928   ins_cost(70);
5929   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5930   ins_encode(load_immL32(dst, src));
5931   ins_pipe(ialu_reg);
5932 %}
5933 
5934 instruct loadConP(rRegP dst, immP con) %{
5935   match(Set dst con);
5936 
5937   format %{ "movq    $dst, $con\t# ptr" %}
5938   ins_encode(load_immP(dst, con));
5939   ins_pipe(ialu_reg_fat); // XXX
5940 %}
5941 
5942 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5943 %{
5944   match(Set dst src);
5945   effect(KILL cr);
5946 
5947   ins_cost(50);
5948   format %{ "xorl    $dst, $dst\t# ptr" %}
5949   opcode(0x33); /* + rd */
5950   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5951   ins_pipe(ialu_reg);
5952 %}
5953 
5954 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5955 %{
5956   match(Set dst src);
5957   effect(KILL cr);
5958 
5959   ins_cost(60);
5960   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5961   ins_encode(load_immP31(dst, src));
5962   ins_pipe(ialu_reg);
5963 %}
5964 
5965 instruct loadConF(regF dst, immF con) %{
5966   match(Set dst con);
5967   ins_cost(125);
5968   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5969   ins_encode %{
5970     __ movflt($dst$$XMMRegister, $constantaddress($con));
5971   %}
5972   ins_pipe(pipe_slow);
5973 %}
5974 
5975 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5976   match(Set dst src);
5977   effect(KILL cr);
5978   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5979   ins_encode %{
5980     __ xorq($dst$$Register, $dst$$Register);
5981   %}
5982   ins_pipe(ialu_reg);
5983 %}
5984 
5985 instruct loadConN(rRegN dst, immN src) %{
5986   match(Set dst src);
5987 
5988   ins_cost(125);
5989   format %{ "movl    $dst, $src\t# compressed ptr" %}
5990   ins_encode %{
5991     address con = (address)$src$$constant;
5992     if (con == NULL) {
5993       ShouldNotReachHere();
5994     } else {
5995       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5996     }
5997   %}
5998   ins_pipe(ialu_reg_fat); // XXX
5999 %}
6000 
6001 instruct loadConNKlass(rRegN dst, immNKlass src) %{
6002   match(Set dst src);
6003 
6004   ins_cost(125);
6005   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
6006   ins_encode %{
6007     address con = (address)$src$$constant;
6008     if (con == NULL) {
6009       ShouldNotReachHere();
6010     } else {
6011       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
6012     }
6013   %}
6014   ins_pipe(ialu_reg_fat); // XXX
6015 %}
6016 
6017 instruct loadConF0(regF dst, immF0 src)
6018 %{
6019   match(Set dst src);
6020   ins_cost(100);
6021 
6022   format %{ "xorps   $dst, $dst\t# float 0.0" %}
6023   ins_encode %{
6024     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
6025   %}
6026   ins_pipe(pipe_slow);
6027 %}
6028 
6029 // Use the same format since predicate() can not be used here.
6030 instruct loadConD(regD dst, immD con) %{
6031   match(Set dst con);
6032   ins_cost(125);
6033   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
6034   ins_encode %{
6035     __ movdbl($dst$$XMMRegister, $constantaddress($con));
6036   %}
6037   ins_pipe(pipe_slow);
6038 %}
6039 
6040 instruct loadConD0(regD dst, immD0 src)
6041 %{
6042   match(Set dst src);
6043   ins_cost(100);
6044 
6045   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
6046   ins_encode %{
6047     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
6048   %}
6049   ins_pipe(pipe_slow);
6050 %}
6051 
6052 instruct loadSSI(rRegI dst, stackSlotI src)
6053 %{
6054   match(Set dst src);
6055 
6056   ins_cost(125);
6057   format %{ "movl    $dst, $src\t# int stk" %}
6058   opcode(0x8B);
6059   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6060   ins_pipe(ialu_reg_mem);
6061 %}
6062 
6063 instruct loadSSL(rRegL dst, stackSlotL src)
6064 %{
6065   match(Set dst src);
6066 
6067   ins_cost(125);
6068   format %{ "movq    $dst, $src\t# long stk" %}
6069   opcode(0x8B);
6070   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6071   ins_pipe(ialu_reg_mem);
6072 %}
6073 
6074 instruct loadSSP(rRegP dst, stackSlotP src)
6075 %{
6076   match(Set dst src);
6077 
6078   ins_cost(125);
6079   format %{ "movq    $dst, $src\t# ptr stk" %}
6080   opcode(0x8B);
6081   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6082   ins_pipe(ialu_reg_mem);
6083 %}
6084 
6085 instruct loadSSF(regF dst, stackSlotF src)
6086 %{
6087   match(Set dst src);
6088 
6089   ins_cost(125);
6090   format %{ "movss   $dst, $src\t# float stk" %}
6091   ins_encode %{
6092     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
6093   %}
6094   ins_pipe(pipe_slow); // XXX
6095 %}
6096 
6097 // Use the same format since predicate() can not be used here.
6098 instruct loadSSD(regD dst, stackSlotD src)
6099 %{
6100   match(Set dst src);
6101 
6102   ins_cost(125);
6103   format %{ "movsd   $dst, $src\t# double stk" %}
6104   ins_encode  %{
6105     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
6106   %}
6107   ins_pipe(pipe_slow); // XXX
6108 %}
6109 
6110 // Prefetch instructions for allocation.
6111 // Must be safe to execute with invalid address (cannot fault).
6112 
6113 instruct prefetchAlloc( memory mem ) %{
6114   predicate(AllocatePrefetchInstr==3);
6115   match(PrefetchAllocation mem);
6116   ins_cost(125);
6117 
6118   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
6119   ins_encode %{
6120     __ prefetchw($mem$$Address);
6121   %}
6122   ins_pipe(ialu_mem);
6123 %}
6124 
6125 instruct prefetchAllocNTA( memory mem ) %{
6126   predicate(AllocatePrefetchInstr==0);
6127   match(PrefetchAllocation mem);
6128   ins_cost(125);
6129 
6130   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
6131   ins_encode %{
6132     __ prefetchnta($mem$$Address);
6133   %}
6134   ins_pipe(ialu_mem);
6135 %}
6136 
6137 instruct prefetchAllocT0( memory mem ) %{
6138   predicate(AllocatePrefetchInstr==1);
6139   match(PrefetchAllocation mem);
6140   ins_cost(125);
6141 
6142   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
6143   ins_encode %{
6144     __ prefetcht0($mem$$Address);
6145   %}
6146   ins_pipe(ialu_mem);
6147 %}
6148 
6149 instruct prefetchAllocT2( memory mem ) %{
6150   predicate(AllocatePrefetchInstr==2);
6151   match(PrefetchAllocation mem);
6152   ins_cost(125);
6153 
6154   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
6155   ins_encode %{
6156     __ prefetcht2($mem$$Address);
6157   %}
6158   ins_pipe(ialu_mem);
6159 %}
6160 
6161 //----------Store Instructions-------------------------------------------------
6162 
6163 // Store Byte
6164 instruct storeB(memory mem, rRegI src)
6165 %{
6166   match(Set mem (StoreB mem src));
6167 
6168   ins_cost(125); // XXX
6169   format %{ "movb    $mem, $src\t# byte" %}
6170   opcode(0x88);
6171   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
6172   ins_pipe(ialu_mem_reg);
6173 %}
6174 
6175 // Store Char/Short
6176 instruct storeC(memory mem, rRegI src)
6177 %{
6178   match(Set mem (StoreC mem src));
6179 
6180   ins_cost(125); // XXX
6181   format %{ "movw    $mem, $src\t# char/short" %}
6182   opcode(0x89);
6183   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6184   ins_pipe(ialu_mem_reg);
6185 %}
6186 
6187 // Store Integer
6188 instruct storeI(memory mem, rRegI src)
6189 %{
6190   match(Set mem (StoreI mem src));
6191 
6192   ins_cost(125); // XXX
6193   format %{ "movl    $mem, $src\t# int" %}
6194   opcode(0x89);
6195   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6196   ins_pipe(ialu_mem_reg);
6197 %}
6198 
6199 // Store Long
6200 instruct storeL(memory mem, rRegL src)
6201 %{
6202   match(Set mem (StoreL mem src));
6203 
6204   ins_cost(125); // XXX
6205   format %{ "movq    $mem, $src\t# long" %}
6206   opcode(0x89);
6207   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6208   ins_pipe(ialu_mem_reg); // XXX
6209 %}
6210 
6211 // Store Pointer
6212 instruct storeP(memory mem, any_RegP src)
6213 %{
6214   match(Set mem (StoreP mem src));
6215 
6216   ins_cost(125); // XXX
6217   format %{ "movq    $mem, $src\t# ptr" %}
6218   opcode(0x89);
6219   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6220   ins_pipe(ialu_mem_reg);
6221 %}
6222 
6223 instruct storeImmP0(memory mem, immP0 zero)
6224 %{
6225   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6226   match(Set mem (StoreP mem zero));
6227 
6228   ins_cost(125); // XXX
6229   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
6230   ins_encode %{
6231     __ movq($mem$$Address, r12);
6232   %}
6233   ins_pipe(ialu_mem_reg);
6234 %}
6235 
6236 // Store NULL Pointer, mark word, or other simple pointer constant.
6237 instruct storeImmP(memory mem, immP31 src)
6238 %{
6239   match(Set mem (StoreP mem src));
6240 
6241   ins_cost(150); // XXX
6242   format %{ "movq    $mem, $src\t# ptr" %}
6243   opcode(0xC7); /* C7 /0 */
6244   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6245   ins_pipe(ialu_mem_imm);
6246 %}
6247 
6248 // Store Compressed Pointer
6249 instruct storeN(memory mem, rRegN src)
6250 %{
6251   match(Set mem (StoreN mem src));
6252 
6253   ins_cost(125); // XXX
6254   format %{ "movl    $mem, $src\t# compressed ptr" %}
6255   ins_encode %{
6256     __ movl($mem$$Address, $src$$Register);
6257   %}
6258   ins_pipe(ialu_mem_reg);
6259 %}
6260 
6261 instruct storeNKlass(memory mem, rRegN src)
6262 %{
6263   match(Set mem (StoreNKlass mem src));
6264 
6265   ins_cost(125); // XXX
6266   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6267   ins_encode %{
6268     __ movl($mem$$Address, $src$$Register);
6269   %}
6270   ins_pipe(ialu_mem_reg);
6271 %}
6272 
6273 instruct storeImmN0(memory mem, immN0 zero)
6274 %{
6275   predicate(CompressedOops::base() == NULL && CompressedKlassPointers::base() == NULL);
6276   match(Set mem (StoreN mem zero));
6277 
6278   ins_cost(125); // XXX
6279   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
6280   ins_encode %{
6281     __ movl($mem$$Address, r12);
6282   %}
6283   ins_pipe(ialu_mem_reg);
6284 %}
6285 
6286 instruct storeImmN(memory mem, immN src)
6287 %{
6288   match(Set mem (StoreN mem src));
6289 
6290   ins_cost(150); // XXX
6291   format %{ "movl    $mem, $src\t# compressed ptr" %}
6292   ins_encode %{
6293     address con = (address)$src$$constant;
6294     if (con == NULL) {
6295       __ movl($mem$$Address, (int32_t)0);
6296     } else {
6297       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
6298     }
6299   %}
6300   ins_pipe(ialu_mem_imm);
6301 %}
6302 
6303 instruct storeImmNKlass(memory mem, immNKlass src)
6304 %{
6305   match(Set mem (StoreNKlass mem src));
6306 
6307   ins_cost(150); // XXX
6308   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6309   ins_encode %{
6310     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
6311   %}
6312   ins_pipe(ialu_mem_imm);
6313 %}
6314 
6315 // Store Integer Immediate
6316 instruct storeImmI0(memory mem, immI0 zero)
6317 %{
6318   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6319   match(Set mem (StoreI mem zero));
6320 
6321   ins_cost(125); // XXX
6322   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6323   ins_encode %{
6324     __ movl($mem$$Address, r12);
6325   %}
6326   ins_pipe(ialu_mem_reg);
6327 %}
6328 
6329 instruct storeImmI(memory mem, immI src)
6330 %{
6331   match(Set mem (StoreI mem src));
6332 
6333   ins_cost(150);
6334   format %{ "movl    $mem, $src\t# int" %}
6335   opcode(0xC7); /* C7 /0 */
6336   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6337   ins_pipe(ialu_mem_imm);
6338 %}
6339 
6340 // Store Long Immediate
6341 instruct storeImmL0(memory mem, immL0 zero)
6342 %{
6343   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6344   match(Set mem (StoreL mem zero));
6345 
6346   ins_cost(125); // XXX
6347   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6348   ins_encode %{
6349     __ movq($mem$$Address, r12);
6350   %}
6351   ins_pipe(ialu_mem_reg);
6352 %}
6353 
6354 instruct storeImmL(memory mem, immL32 src)
6355 %{
6356   match(Set mem (StoreL mem src));
6357 
6358   ins_cost(150);
6359   format %{ "movq    $mem, $src\t# long" %}
6360   opcode(0xC7); /* C7 /0 */
6361   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6362   ins_pipe(ialu_mem_imm);
6363 %}
6364 
6365 // Store Short/Char Immediate
6366 instruct storeImmC0(memory mem, immI0 zero)
6367 %{
6368   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6369   match(Set mem (StoreC mem zero));
6370 
6371   ins_cost(125); // XXX
6372   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6373   ins_encode %{
6374     __ movw($mem$$Address, r12);
6375   %}
6376   ins_pipe(ialu_mem_reg);
6377 %}
6378 
6379 instruct storeImmI16(memory mem, immI16 src)
6380 %{
6381   predicate(UseStoreImmI16);
6382   match(Set mem (StoreC mem src));
6383 
6384   ins_cost(150);
6385   format %{ "movw    $mem, $src\t# short/char" %}
6386   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6387   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6388   ins_pipe(ialu_mem_imm);
6389 %}
6390 
6391 // Store Byte Immediate
6392 instruct storeImmB0(memory mem, immI0 zero)
6393 %{
6394   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6395   match(Set mem (StoreB mem zero));
6396 
6397   ins_cost(125); // XXX
6398   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6399   ins_encode %{
6400     __ movb($mem$$Address, r12);
6401   %}
6402   ins_pipe(ialu_mem_reg);
6403 %}
6404 
6405 instruct storeImmB(memory mem, immI8 src)
6406 %{
6407   match(Set mem (StoreB mem src));
6408 
6409   ins_cost(150); // XXX
6410   format %{ "movb    $mem, $src\t# byte" %}
6411   opcode(0xC6); /* C6 /0 */
6412   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6413   ins_pipe(ialu_mem_imm);
6414 %}
6415 
6416 // Store CMS card-mark Immediate
6417 instruct storeImmCM0_reg(memory mem, immI0 zero)
6418 %{
6419   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6420   match(Set mem (StoreCM mem zero));
6421 
6422   ins_cost(125); // XXX
6423   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6424   ins_encode %{
6425     __ movb($mem$$Address, r12);
6426   %}
6427   ins_pipe(ialu_mem_reg);
6428 %}
6429 
6430 instruct storeImmCM0(memory mem, immI0 src)
6431 %{
6432   match(Set mem (StoreCM mem src));
6433 
6434   ins_cost(150); // XXX
6435   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6436   opcode(0xC6); /* C6 /0 */
6437   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6438   ins_pipe(ialu_mem_imm);
6439 %}
6440 
6441 // Store Float
6442 instruct storeF(memory mem, regF src)
6443 %{
6444   match(Set mem (StoreF mem src));
6445 
6446   ins_cost(95); // XXX
6447   format %{ "movss   $mem, $src\t# float" %}
6448   ins_encode %{
6449     __ movflt($mem$$Address, $src$$XMMRegister);
6450   %}
6451   ins_pipe(pipe_slow); // XXX
6452 %}
6453 
6454 // Store immediate Float value (it is faster than store from XMM register)
6455 instruct storeF0(memory mem, immF0 zero)
6456 %{
6457   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6458   match(Set mem (StoreF mem zero));
6459 
6460   ins_cost(25); // XXX
6461   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6462   ins_encode %{
6463     __ movl($mem$$Address, r12);
6464   %}
6465   ins_pipe(ialu_mem_reg);
6466 %}
6467 
6468 instruct storeF_imm(memory mem, immF src)
6469 %{
6470   match(Set mem (StoreF mem src));
6471 
6472   ins_cost(50);
6473   format %{ "movl    $mem, $src\t# float" %}
6474   opcode(0xC7); /* C7 /0 */
6475   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6476   ins_pipe(ialu_mem_imm);
6477 %}
6478 
6479 // Store Double
6480 instruct storeD(memory mem, regD src)
6481 %{
6482   match(Set mem (StoreD mem src));
6483 
6484   ins_cost(95); // XXX
6485   format %{ "movsd   $mem, $src\t# double" %}
6486   ins_encode %{
6487     __ movdbl($mem$$Address, $src$$XMMRegister);
6488   %}
6489   ins_pipe(pipe_slow); // XXX
6490 %}
6491 
6492 // Store immediate double 0.0 (it is faster than store from XMM register)
6493 instruct storeD0_imm(memory mem, immD0 src)
6494 %{
6495   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
6496   match(Set mem (StoreD mem src));
6497 
6498   ins_cost(50);
6499   format %{ "movq    $mem, $src\t# double 0." %}
6500   opcode(0xC7); /* C7 /0 */
6501   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6502   ins_pipe(ialu_mem_imm);
6503 %}
6504 
6505 instruct storeD0(memory mem, immD0 zero)
6506 %{
6507   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6508   match(Set mem (StoreD mem zero));
6509 
6510   ins_cost(25); // XXX
6511   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6512   ins_encode %{
6513     __ movq($mem$$Address, r12);
6514   %}
6515   ins_pipe(ialu_mem_reg);
6516 %}
6517 
6518 instruct storeSSI(stackSlotI dst, rRegI src)
6519 %{
6520   match(Set dst src);
6521 
6522   ins_cost(100);
6523   format %{ "movl    $dst, $src\t# int stk" %}
6524   opcode(0x89);
6525   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6526   ins_pipe( ialu_mem_reg );
6527 %}
6528 
6529 instruct storeSSL(stackSlotL dst, rRegL src)
6530 %{
6531   match(Set dst src);
6532 
6533   ins_cost(100);
6534   format %{ "movq    $dst, $src\t# long stk" %}
6535   opcode(0x89);
6536   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6537   ins_pipe(ialu_mem_reg);
6538 %}
6539 
6540 instruct storeSSP(stackSlotP dst, rRegP src)
6541 %{
6542   match(Set dst src);
6543 
6544   ins_cost(100);
6545   format %{ "movq    $dst, $src\t# ptr stk" %}
6546   opcode(0x89);
6547   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6548   ins_pipe(ialu_mem_reg);
6549 %}
6550 
6551 instruct storeSSF(stackSlotF dst, regF src)
6552 %{
6553   match(Set dst src);
6554 
6555   ins_cost(95); // XXX
6556   format %{ "movss   $dst, $src\t# float stk" %}
6557   ins_encode %{
6558     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6559   %}
6560   ins_pipe(pipe_slow); // XXX
6561 %}
6562 
6563 instruct storeSSD(stackSlotD dst, regD src)
6564 %{
6565   match(Set dst src);
6566 
6567   ins_cost(95); // XXX
6568   format %{ "movsd   $dst, $src\t# double stk" %}
6569   ins_encode %{
6570     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6571   %}
6572   ins_pipe(pipe_slow); // XXX
6573 %}
6574 
6575 //----------BSWAP Instructions-------------------------------------------------
6576 instruct bytes_reverse_int(rRegI dst) %{
6577   match(Set dst (ReverseBytesI dst));
6578 
6579   format %{ "bswapl  $dst" %}
6580   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6581   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6582   ins_pipe( ialu_reg );
6583 %}
6584 
6585 instruct bytes_reverse_long(rRegL dst) %{
6586   match(Set dst (ReverseBytesL dst));
6587 
6588   format %{ "bswapq  $dst" %}
6589   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6590   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6591   ins_pipe( ialu_reg);
6592 %}
6593 
6594 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6595   match(Set dst (ReverseBytesUS dst));
6596   effect(KILL cr);
6597 
6598   format %{ "bswapl  $dst\n\t"
6599             "shrl    $dst,16\n\t" %}
6600   ins_encode %{
6601     __ bswapl($dst$$Register);
6602     __ shrl($dst$$Register, 16);
6603   %}
6604   ins_pipe( ialu_reg );
6605 %}
6606 
6607 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6608   match(Set dst (ReverseBytesS dst));
6609   effect(KILL cr);
6610 
6611   format %{ "bswapl  $dst\n\t"
6612             "sar     $dst,16\n\t" %}
6613   ins_encode %{
6614     __ bswapl($dst$$Register);
6615     __ sarl($dst$$Register, 16);
6616   %}
6617   ins_pipe( ialu_reg );
6618 %}
6619 
6620 //---------- Zeros Count Instructions ------------------------------------------
6621 
6622 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6623   predicate(UseCountLeadingZerosInstruction);
6624   match(Set dst (CountLeadingZerosI src));
6625   effect(KILL cr);
6626 
6627   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6628   ins_encode %{
6629     __ lzcntl($dst$$Register, $src$$Register);
6630   %}
6631   ins_pipe(ialu_reg);
6632 %}
6633 
6634 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6635   predicate(!UseCountLeadingZerosInstruction);
6636   match(Set dst (CountLeadingZerosI src));
6637   effect(KILL cr);
6638 
6639   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6640             "jnz     skip\n\t"
6641             "movl    $dst, -1\n"
6642       "skip:\n\t"
6643             "negl    $dst\n\t"
6644             "addl    $dst, 31" %}
6645   ins_encode %{
6646     Register Rdst = $dst$$Register;
6647     Register Rsrc = $src$$Register;
6648     Label skip;
6649     __ bsrl(Rdst, Rsrc);
6650     __ jccb(Assembler::notZero, skip);
6651     __ movl(Rdst, -1);
6652     __ bind(skip);
6653     __ negl(Rdst);
6654     __ addl(Rdst, BitsPerInt - 1);
6655   %}
6656   ins_pipe(ialu_reg);
6657 %}
6658 
6659 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6660   predicate(UseCountLeadingZerosInstruction);
6661   match(Set dst (CountLeadingZerosL src));
6662   effect(KILL cr);
6663 
6664   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6665   ins_encode %{
6666     __ lzcntq($dst$$Register, $src$$Register);
6667   %}
6668   ins_pipe(ialu_reg);
6669 %}
6670 
6671 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6672   predicate(!UseCountLeadingZerosInstruction);
6673   match(Set dst (CountLeadingZerosL src));
6674   effect(KILL cr);
6675 
6676   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6677             "jnz     skip\n\t"
6678             "movl    $dst, -1\n"
6679       "skip:\n\t"
6680             "negl    $dst\n\t"
6681             "addl    $dst, 63" %}
6682   ins_encode %{
6683     Register Rdst = $dst$$Register;
6684     Register Rsrc = $src$$Register;
6685     Label skip;
6686     __ bsrq(Rdst, Rsrc);
6687     __ jccb(Assembler::notZero, skip);
6688     __ movl(Rdst, -1);
6689     __ bind(skip);
6690     __ negl(Rdst);
6691     __ addl(Rdst, BitsPerLong - 1);
6692   %}
6693   ins_pipe(ialu_reg);
6694 %}
6695 
6696 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6697   predicate(UseCountTrailingZerosInstruction);
6698   match(Set dst (CountTrailingZerosI src));
6699   effect(KILL cr);
6700 
6701   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6702   ins_encode %{
6703     __ tzcntl($dst$$Register, $src$$Register);
6704   %}
6705   ins_pipe(ialu_reg);
6706 %}
6707 
6708 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6709   predicate(!UseCountTrailingZerosInstruction);
6710   match(Set dst (CountTrailingZerosI src));
6711   effect(KILL cr);
6712 
6713   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6714             "jnz     done\n\t"
6715             "movl    $dst, 32\n"
6716       "done:" %}
6717   ins_encode %{
6718     Register Rdst = $dst$$Register;
6719     Label done;
6720     __ bsfl(Rdst, $src$$Register);
6721     __ jccb(Assembler::notZero, done);
6722     __ movl(Rdst, BitsPerInt);
6723     __ bind(done);
6724   %}
6725   ins_pipe(ialu_reg);
6726 %}
6727 
6728 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6729   predicate(UseCountTrailingZerosInstruction);
6730   match(Set dst (CountTrailingZerosL src));
6731   effect(KILL cr);
6732 
6733   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6734   ins_encode %{
6735     __ tzcntq($dst$$Register, $src$$Register);
6736   %}
6737   ins_pipe(ialu_reg);
6738 %}
6739 
6740 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6741   predicate(!UseCountTrailingZerosInstruction);
6742   match(Set dst (CountTrailingZerosL src));
6743   effect(KILL cr);
6744 
6745   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6746             "jnz     done\n\t"
6747             "movl    $dst, 64\n"
6748       "done:" %}
6749   ins_encode %{
6750     Register Rdst = $dst$$Register;
6751     Label done;
6752     __ bsfq(Rdst, $src$$Register);
6753     __ jccb(Assembler::notZero, done);
6754     __ movl(Rdst, BitsPerLong);
6755     __ bind(done);
6756   %}
6757   ins_pipe(ialu_reg);
6758 %}
6759 
6760 
6761 //---------- Population Count Instructions -------------------------------------
6762 
6763 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6764   predicate(UsePopCountInstruction);
6765   match(Set dst (PopCountI src));
6766   effect(KILL cr);
6767 
6768   format %{ "popcnt  $dst, $src" %}
6769   ins_encode %{
6770     __ popcntl($dst$$Register, $src$$Register);
6771   %}
6772   ins_pipe(ialu_reg);
6773 %}
6774 
6775 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6776   predicate(UsePopCountInstruction);
6777   match(Set dst (PopCountI (LoadI mem)));
6778   effect(KILL cr);
6779 
6780   format %{ "popcnt  $dst, $mem" %}
6781   ins_encode %{
6782     __ popcntl($dst$$Register, $mem$$Address);
6783   %}
6784   ins_pipe(ialu_reg);
6785 %}
6786 
6787 // Note: Long.bitCount(long) returns an int.
6788 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6789   predicate(UsePopCountInstruction);
6790   match(Set dst (PopCountL src));
6791   effect(KILL cr);
6792 
6793   format %{ "popcnt  $dst, $src" %}
6794   ins_encode %{
6795     __ popcntq($dst$$Register, $src$$Register);
6796   %}
6797   ins_pipe(ialu_reg);
6798 %}
6799 
6800 // Note: Long.bitCount(long) returns an int.
6801 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6802   predicate(UsePopCountInstruction);
6803   match(Set dst (PopCountL (LoadL mem)));
6804   effect(KILL cr);
6805 
6806   format %{ "popcnt  $dst, $mem" %}
6807   ins_encode %{
6808     __ popcntq($dst$$Register, $mem$$Address);
6809   %}
6810   ins_pipe(ialu_reg);
6811 %}
6812 
6813 
6814 //----------MemBar Instructions-----------------------------------------------
6815 // Memory barrier flavors
6816 
6817 instruct membar_acquire()
6818 %{
6819   match(MemBarAcquire);
6820   match(LoadFence);
6821   ins_cost(0);
6822 
6823   size(0);
6824   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6825   ins_encode();
6826   ins_pipe(empty);
6827 %}
6828 
6829 instruct membar_acquire_lock()
6830 %{
6831   match(MemBarAcquireLock);
6832   ins_cost(0);
6833 
6834   size(0);
6835   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6836   ins_encode();
6837   ins_pipe(empty);
6838 %}
6839 
6840 instruct membar_release()
6841 %{
6842   match(MemBarRelease);
6843   match(StoreFence);
6844   ins_cost(0);
6845 
6846   size(0);
6847   format %{ "MEMBAR-release ! (empty encoding)" %}
6848   ins_encode();
6849   ins_pipe(empty);
6850 %}
6851 
6852 instruct membar_release_lock()
6853 %{
6854   match(MemBarReleaseLock);
6855   ins_cost(0);
6856 
6857   size(0);
6858   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6859   ins_encode();
6860   ins_pipe(empty);
6861 %}
6862 
6863 instruct membar_volatile(rFlagsReg cr) %{
6864   match(MemBarVolatile);
6865   effect(KILL cr);
6866   ins_cost(400);
6867 
6868   format %{
6869     $$template
6870     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6871   %}
6872   ins_encode %{
6873     __ membar(Assembler::StoreLoad);
6874   %}
6875   ins_pipe(pipe_slow);
6876 %}
6877 
6878 instruct unnecessary_membar_volatile()
6879 %{
6880   match(MemBarVolatile);
6881   predicate(Matcher::post_store_load_barrier(n));
6882   ins_cost(0);
6883 
6884   size(0);
6885   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6886   ins_encode();
6887   ins_pipe(empty);
6888 %}
6889 
6890 instruct membar_storestore() %{
6891   match(MemBarStoreStore);
6892   ins_cost(0);
6893 
6894   size(0);
6895   format %{ "MEMBAR-storestore (empty encoding)" %}
6896   ins_encode( );
6897   ins_pipe(empty);
6898 %}
6899 
6900 //----------Move Instructions--------------------------------------------------
6901 
6902 instruct castX2P(rRegP dst, rRegL src)
6903 %{
6904   match(Set dst (CastX2P src));
6905 
6906   format %{ "movq    $dst, $src\t# long->ptr" %}
6907   ins_encode %{
6908     if ($dst$$reg != $src$$reg) {
6909       __ movptr($dst$$Register, $src$$Register);
6910     }
6911   %}
6912   ins_pipe(ialu_reg_reg); // XXX
6913 %}
6914 
6915 instruct castP2X(rRegL dst, rRegP src)
6916 %{
6917   match(Set dst (CastP2X src));
6918 
6919   format %{ "movq    $dst, $src\t# ptr -> long" %}
6920   ins_encode %{
6921     if ($dst$$reg != $src$$reg) {
6922       __ movptr($dst$$Register, $src$$Register);
6923     }
6924   %}
6925   ins_pipe(ialu_reg_reg); // XXX
6926 %}
6927 
6928 // Convert oop into int for vectors alignment masking
6929 instruct convP2I(rRegI dst, rRegP src)
6930 %{
6931   match(Set dst (ConvL2I (CastP2X src)));
6932 
6933   format %{ "movl    $dst, $src\t# ptr -> int" %}
6934   ins_encode %{
6935     __ movl($dst$$Register, $src$$Register);
6936   %}
6937   ins_pipe(ialu_reg_reg); // XXX
6938 %}
6939 
6940 // Convert compressed oop into int for vectors alignment masking
6941 // in case of 32bit oops (heap < 4Gb).
6942 instruct convN2I(rRegI dst, rRegN src)
6943 %{
6944   predicate(CompressedOops::shift() == 0);
6945   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6946 
6947   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6948   ins_encode %{
6949     __ movl($dst$$Register, $src$$Register);
6950   %}
6951   ins_pipe(ialu_reg_reg); // XXX
6952 %}
6953 
6954 // Convert oop pointer into compressed form
6955 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6956   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6957   match(Set dst (EncodeP src));
6958   effect(KILL cr);
6959   format %{ "encode_heap_oop $dst,$src" %}
6960   ins_encode %{
6961     Register s = $src$$Register;
6962     Register d = $dst$$Register;
6963     if (s != d) {
6964       __ movq(d, s);
6965     }
6966     __ encode_heap_oop(d);
6967   %}
6968   ins_pipe(ialu_reg_long);
6969 %}
6970 
6971 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6972   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6973   match(Set dst (EncodeP src));
6974   effect(KILL cr);
6975   format %{ "encode_heap_oop_not_null $dst,$src" %}
6976   ins_encode %{
6977     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6978   %}
6979   ins_pipe(ialu_reg_long);
6980 %}
6981 
6982 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6983   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6984             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6985   match(Set dst (DecodeN src));
6986   effect(KILL cr);
6987   format %{ "decode_heap_oop $dst,$src" %}
6988   ins_encode %{
6989     Register s = $src$$Register;
6990     Register d = $dst$$Register;
6991     if (s != d) {
6992       __ movq(d, s);
6993     }
6994     __ decode_heap_oop(d);
6995   %}
6996   ins_pipe(ialu_reg_long);
6997 %}
6998 
6999 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7000   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7001             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7002   match(Set dst (DecodeN src));
7003   effect(KILL cr);
7004   format %{ "decode_heap_oop_not_null $dst,$src" %}
7005   ins_encode %{
7006     Register s = $src$$Register;
7007     Register d = $dst$$Register;
7008     if (s != d) {
7009       __ decode_heap_oop_not_null(d, s);
7010     } else {
7011       __ decode_heap_oop_not_null(d);
7012     }
7013   %}
7014   ins_pipe(ialu_reg_long);
7015 %}
7016 
7017 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
7018   match(Set dst (EncodePKlass src));
7019   effect(KILL cr);
7020   format %{ "encode_klass_not_null $dst,$src" %}
7021   ins_encode %{
7022     __ encode_klass_not_null($dst$$Register, $src$$Register);
7023   %}
7024   ins_pipe(ialu_reg_long);
7025 %}
7026 
7027 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7028   match(Set dst (DecodeNKlass src));
7029   effect(KILL cr);
7030   format %{ "decode_klass_not_null $dst,$src" %}
7031   ins_encode %{
7032     Register s = $src$$Register;
7033     Register d = $dst$$Register;
7034     if (s != d) {
7035       __ decode_klass_not_null(d, s);
7036     } else {
7037       __ decode_klass_not_null(d);
7038     }
7039   %}
7040   ins_pipe(ialu_reg_long);
7041 %}
7042 
7043 
7044 //----------Conditional Move---------------------------------------------------
7045 // Jump
7046 // dummy instruction for generating temp registers
7047 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
7048   match(Jump (LShiftL switch_val shift));
7049   ins_cost(350);
7050   predicate(false);
7051   effect(TEMP dest);
7052 
7053   format %{ "leaq    $dest, [$constantaddress]\n\t"
7054             "jmp     [$dest + $switch_val << $shift]\n\t" %}
7055   ins_encode %{
7056     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7057     // to do that and the compiler is using that register as one it can allocate.
7058     // So we build it all by hand.
7059     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
7060     // ArrayAddress dispatch(table, index);
7061     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
7062     __ lea($dest$$Register, $constantaddress);
7063     __ jmp(dispatch);
7064   %}
7065   ins_pipe(pipe_jmp);
7066 %}
7067 
7068 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
7069   match(Jump (AddL (LShiftL switch_val shift) offset));
7070   ins_cost(350);
7071   effect(TEMP dest);
7072 
7073   format %{ "leaq    $dest, [$constantaddress]\n\t"
7074             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
7075   ins_encode %{
7076     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7077     // to do that and the compiler is using that register as one it can allocate.
7078     // So we build it all by hand.
7079     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7080     // ArrayAddress dispatch(table, index);
7081     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7082     __ lea($dest$$Register, $constantaddress);
7083     __ jmp(dispatch);
7084   %}
7085   ins_pipe(pipe_jmp);
7086 %}
7087 
7088 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
7089   match(Jump switch_val);
7090   ins_cost(350);
7091   effect(TEMP dest);
7092 
7093   format %{ "leaq    $dest, [$constantaddress]\n\t"
7094             "jmp     [$dest + $switch_val]\n\t" %}
7095   ins_encode %{
7096     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7097     // to do that and the compiler is using that register as one it can allocate.
7098     // So we build it all by hand.
7099     // Address index(noreg, switch_reg, Address::times_1);
7100     // ArrayAddress dispatch(table, index);
7101     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
7102     __ lea($dest$$Register, $constantaddress);
7103     __ jmp(dispatch);
7104   %}
7105   ins_pipe(pipe_jmp);
7106 %}
7107 
7108 // Conditional move
7109 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
7110 %{
7111   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7112 
7113   ins_cost(200); // XXX
7114   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7115   opcode(0x0F, 0x40);
7116   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7117   ins_pipe(pipe_cmov_reg);
7118 %}
7119 
7120 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
7121   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7122 
7123   ins_cost(200); // XXX
7124   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7125   opcode(0x0F, 0x40);
7126   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7127   ins_pipe(pipe_cmov_reg);
7128 %}
7129 
7130 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
7131   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7132   ins_cost(200);
7133   expand %{
7134     cmovI_regU(cop, cr, dst, src);
7135   %}
7136 %}
7137 
7138 // Conditional move
7139 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
7140   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7141 
7142   ins_cost(250); // XXX
7143   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7144   opcode(0x0F, 0x40);
7145   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7146   ins_pipe(pipe_cmov_mem);
7147 %}
7148 
7149 // Conditional move
7150 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
7151 %{
7152   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7153 
7154   ins_cost(250); // XXX
7155   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7156   opcode(0x0F, 0x40);
7157   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7158   ins_pipe(pipe_cmov_mem);
7159 %}
7160 
7161 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
7162   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7163   ins_cost(250);
7164   expand %{
7165     cmovI_memU(cop, cr, dst, src);
7166   %}
7167 %}
7168 
7169 // Conditional move
7170 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
7171 %{
7172   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7173 
7174   ins_cost(200); // XXX
7175   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
7176   opcode(0x0F, 0x40);
7177   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7178   ins_pipe(pipe_cmov_reg);
7179 %}
7180 
7181 // Conditional move
7182 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
7183 %{
7184   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7185 
7186   ins_cost(200); // XXX
7187   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
7188   opcode(0x0F, 0x40);
7189   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7190   ins_pipe(pipe_cmov_reg);
7191 %}
7192 
7193 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
7194   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7195   ins_cost(200);
7196   expand %{
7197     cmovN_regU(cop, cr, dst, src);
7198   %}
7199 %}
7200 
7201 // Conditional move
7202 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
7203 %{
7204   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7205 
7206   ins_cost(200); // XXX
7207   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
7208   opcode(0x0F, 0x40);
7209   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7210   ins_pipe(pipe_cmov_reg);  // XXX
7211 %}
7212 
7213 // Conditional move
7214 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
7215 %{
7216   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7217 
7218   ins_cost(200); // XXX
7219   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
7220   opcode(0x0F, 0x40);
7221   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7222   ins_pipe(pipe_cmov_reg); // XXX
7223 %}
7224 
7225 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
7226   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7227   ins_cost(200);
7228   expand %{
7229     cmovP_regU(cop, cr, dst, src);
7230   %}
7231 %}
7232 
7233 // DISABLED: Requires the ADLC to emit a bottom_type call that
7234 // correctly meets the two pointer arguments; one is an incoming
7235 // register but the other is a memory operand.  ALSO appears to
7236 // be buggy with implicit null checks.
7237 //
7238 //// Conditional move
7239 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
7240 //%{
7241 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7242 //  ins_cost(250);
7243 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7244 //  opcode(0x0F,0x40);
7245 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7246 //  ins_pipe( pipe_cmov_mem );
7247 //%}
7248 //
7249 //// Conditional move
7250 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
7251 //%{
7252 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7253 //  ins_cost(250);
7254 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7255 //  opcode(0x0F,0x40);
7256 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7257 //  ins_pipe( pipe_cmov_mem );
7258 //%}
7259 
7260 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
7261 %{
7262   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7263 
7264   ins_cost(200); // XXX
7265   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7266   opcode(0x0F, 0x40);
7267   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7268   ins_pipe(pipe_cmov_reg);  // XXX
7269 %}
7270 
7271 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
7272 %{
7273   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7274 
7275   ins_cost(200); // XXX
7276   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7277   opcode(0x0F, 0x40);
7278   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7279   ins_pipe(pipe_cmov_mem);  // XXX
7280 %}
7281 
7282 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
7283 %{
7284   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7285 
7286   ins_cost(200); // XXX
7287   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7288   opcode(0x0F, 0x40);
7289   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7290   ins_pipe(pipe_cmov_reg); // XXX
7291 %}
7292 
7293 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7294   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7295   ins_cost(200);
7296   expand %{
7297     cmovL_regU(cop, cr, dst, src);
7298   %}
7299 %}
7300 
7301 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7302 %{
7303   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7304 
7305   ins_cost(200); // XXX
7306   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7307   opcode(0x0F, 0x40);
7308   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7309   ins_pipe(pipe_cmov_mem); // XXX
7310 %}
7311 
7312 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7313   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7314   ins_cost(200);
7315   expand %{
7316     cmovL_memU(cop, cr, dst, src);
7317   %}
7318 %}
7319 
7320 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7321 %{
7322   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7323 
7324   ins_cost(200); // XXX
7325   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7326             "movss     $dst, $src\n"
7327     "skip:" %}
7328   ins_encode %{
7329     Label Lskip;
7330     // Invert sense of branch from sense of CMOV
7331     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7332     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7333     __ bind(Lskip);
7334   %}
7335   ins_pipe(pipe_slow);
7336 %}
7337 
7338 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7339 // %{
7340 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7341 
7342 //   ins_cost(200); // XXX
7343 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7344 //             "movss     $dst, $src\n"
7345 //     "skip:" %}
7346 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7347 //   ins_pipe(pipe_slow);
7348 // %}
7349 
7350 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7351 %{
7352   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7353 
7354   ins_cost(200); // XXX
7355   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7356             "movss     $dst, $src\n"
7357     "skip:" %}
7358   ins_encode %{
7359     Label Lskip;
7360     // Invert sense of branch from sense of CMOV
7361     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7362     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7363     __ bind(Lskip);
7364   %}
7365   ins_pipe(pipe_slow);
7366 %}
7367 
7368 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7369   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7370   ins_cost(200);
7371   expand %{
7372     cmovF_regU(cop, cr, dst, src);
7373   %}
7374 %}
7375 
7376 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7377 %{
7378   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7379 
7380   ins_cost(200); // XXX
7381   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7382             "movsd     $dst, $src\n"
7383     "skip:" %}
7384   ins_encode %{
7385     Label Lskip;
7386     // Invert sense of branch from sense of CMOV
7387     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7388     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7389     __ bind(Lskip);
7390   %}
7391   ins_pipe(pipe_slow);
7392 %}
7393 
7394 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7395 %{
7396   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7397 
7398   ins_cost(200); // XXX
7399   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7400             "movsd     $dst, $src\n"
7401     "skip:" %}
7402   ins_encode %{
7403     Label Lskip;
7404     // Invert sense of branch from sense of CMOV
7405     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7406     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7407     __ bind(Lskip);
7408   %}
7409   ins_pipe(pipe_slow);
7410 %}
7411 
7412 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7413   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7414   ins_cost(200);
7415   expand %{
7416     cmovD_regU(cop, cr, dst, src);
7417   %}
7418 %}
7419 
7420 //----------Arithmetic Instructions--------------------------------------------
7421 //----------Addition Instructions----------------------------------------------
7422 
7423 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7424 %{
7425   match(Set dst (AddI dst src));
7426   effect(KILL cr);
7427 
7428   format %{ "addl    $dst, $src\t# int" %}
7429   opcode(0x03);
7430   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7431   ins_pipe(ialu_reg_reg);
7432 %}
7433 
7434 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7435 %{
7436   match(Set dst (AddI dst src));
7437   effect(KILL cr);
7438 
7439   format %{ "addl    $dst, $src\t# int" %}
7440   opcode(0x81, 0x00); /* /0 id */
7441   ins_encode(OpcSErm(dst, src), Con8or32(src));
7442   ins_pipe( ialu_reg );
7443 %}
7444 
7445 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7446 %{
7447   match(Set dst (AddI dst (LoadI src)));
7448   effect(KILL cr);
7449 
7450   ins_cost(125); // XXX
7451   format %{ "addl    $dst, $src\t# int" %}
7452   opcode(0x03);
7453   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7454   ins_pipe(ialu_reg_mem);
7455 %}
7456 
7457 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7458 %{
7459   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7460   effect(KILL cr);
7461 
7462   ins_cost(150); // XXX
7463   format %{ "addl    $dst, $src\t# int" %}
7464   opcode(0x01); /* Opcode 01 /r */
7465   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7466   ins_pipe(ialu_mem_reg);
7467 %}
7468 
7469 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7470 %{
7471   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7472   effect(KILL cr);
7473 
7474   ins_cost(125); // XXX
7475   format %{ "addl    $dst, $src\t# int" %}
7476   opcode(0x81); /* Opcode 81 /0 id */
7477   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7478   ins_pipe(ialu_mem_imm);
7479 %}
7480 
7481 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7482 %{
7483   predicate(UseIncDec);
7484   match(Set dst (AddI dst src));
7485   effect(KILL cr);
7486 
7487   format %{ "incl    $dst\t# int" %}
7488   opcode(0xFF, 0x00); // FF /0
7489   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7490   ins_pipe(ialu_reg);
7491 %}
7492 
7493 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7494 %{
7495   predicate(UseIncDec);
7496   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7497   effect(KILL cr);
7498 
7499   ins_cost(125); // XXX
7500   format %{ "incl    $dst\t# int" %}
7501   opcode(0xFF); /* Opcode FF /0 */
7502   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7503   ins_pipe(ialu_mem_imm);
7504 %}
7505 
7506 // XXX why does that use AddI
7507 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7508 %{
7509   predicate(UseIncDec);
7510   match(Set dst (AddI dst src));
7511   effect(KILL cr);
7512 
7513   format %{ "decl    $dst\t# int" %}
7514   opcode(0xFF, 0x01); // FF /1
7515   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7516   ins_pipe(ialu_reg);
7517 %}
7518 
7519 // XXX why does that use AddI
7520 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7521 %{
7522   predicate(UseIncDec);
7523   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7524   effect(KILL cr);
7525 
7526   ins_cost(125); // XXX
7527   format %{ "decl    $dst\t# int" %}
7528   opcode(0xFF); /* Opcode FF /1 */
7529   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7530   ins_pipe(ialu_mem_imm);
7531 %}
7532 
7533 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7534 %{
7535   match(Set dst (AddI src0 src1));
7536 
7537   ins_cost(110);
7538   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7539   opcode(0x8D); /* 0x8D /r */
7540   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7541   ins_pipe(ialu_reg_reg);
7542 %}
7543 
7544 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7545 %{
7546   match(Set dst (AddL dst src));
7547   effect(KILL cr);
7548 
7549   format %{ "addq    $dst, $src\t# long" %}
7550   opcode(0x03);
7551   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7552   ins_pipe(ialu_reg_reg);
7553 %}
7554 
7555 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7556 %{
7557   match(Set dst (AddL dst src));
7558   effect(KILL cr);
7559 
7560   format %{ "addq    $dst, $src\t# long" %}
7561   opcode(0x81, 0x00); /* /0 id */
7562   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7563   ins_pipe( ialu_reg );
7564 %}
7565 
7566 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7567 %{
7568   match(Set dst (AddL dst (LoadL src)));
7569   effect(KILL cr);
7570 
7571   ins_cost(125); // XXX
7572   format %{ "addq    $dst, $src\t# long" %}
7573   opcode(0x03);
7574   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7575   ins_pipe(ialu_reg_mem);
7576 %}
7577 
7578 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7579 %{
7580   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7581   effect(KILL cr);
7582 
7583   ins_cost(150); // XXX
7584   format %{ "addq    $dst, $src\t# long" %}
7585   opcode(0x01); /* Opcode 01 /r */
7586   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7587   ins_pipe(ialu_mem_reg);
7588 %}
7589 
7590 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7591 %{
7592   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7593   effect(KILL cr);
7594 
7595   ins_cost(125); // XXX
7596   format %{ "addq    $dst, $src\t# long" %}
7597   opcode(0x81); /* Opcode 81 /0 id */
7598   ins_encode(REX_mem_wide(dst),
7599              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7600   ins_pipe(ialu_mem_imm);
7601 %}
7602 
7603 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7604 %{
7605   predicate(UseIncDec);
7606   match(Set dst (AddL dst src));
7607   effect(KILL cr);
7608 
7609   format %{ "incq    $dst\t# long" %}
7610   opcode(0xFF, 0x00); // FF /0
7611   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7612   ins_pipe(ialu_reg);
7613 %}
7614 
7615 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7616 %{
7617   predicate(UseIncDec);
7618   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7619   effect(KILL cr);
7620 
7621   ins_cost(125); // XXX
7622   format %{ "incq    $dst\t# long" %}
7623   opcode(0xFF); /* Opcode FF /0 */
7624   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7625   ins_pipe(ialu_mem_imm);
7626 %}
7627 
7628 // XXX why does that use AddL
7629 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7630 %{
7631   predicate(UseIncDec);
7632   match(Set dst (AddL dst src));
7633   effect(KILL cr);
7634 
7635   format %{ "decq    $dst\t# long" %}
7636   opcode(0xFF, 0x01); // FF /1
7637   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7638   ins_pipe(ialu_reg);
7639 %}
7640 
7641 // XXX why does that use AddL
7642 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7643 %{
7644   predicate(UseIncDec);
7645   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7646   effect(KILL cr);
7647 
7648   ins_cost(125); // XXX
7649   format %{ "decq    $dst\t# long" %}
7650   opcode(0xFF); /* Opcode FF /1 */
7651   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7652   ins_pipe(ialu_mem_imm);
7653 %}
7654 
7655 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7656 %{
7657   match(Set dst (AddL src0 src1));
7658 
7659   ins_cost(110);
7660   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7661   opcode(0x8D); /* 0x8D /r */
7662   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7663   ins_pipe(ialu_reg_reg);
7664 %}
7665 
7666 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7667 %{
7668   match(Set dst (AddP dst src));
7669   effect(KILL cr);
7670 
7671   format %{ "addq    $dst, $src\t# ptr" %}
7672   opcode(0x03);
7673   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7674   ins_pipe(ialu_reg_reg);
7675 %}
7676 
7677 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7678 %{
7679   match(Set dst (AddP dst src));
7680   effect(KILL cr);
7681 
7682   format %{ "addq    $dst, $src\t# ptr" %}
7683   opcode(0x81, 0x00); /* /0 id */
7684   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7685   ins_pipe( ialu_reg );
7686 %}
7687 
7688 // XXX addP mem ops ????
7689 
7690 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7691 %{
7692   match(Set dst (AddP src0 src1));
7693 
7694   ins_cost(110);
7695   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7696   opcode(0x8D); /* 0x8D /r */
7697   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7698   ins_pipe(ialu_reg_reg);
7699 %}
7700 
7701 instruct checkCastPP(rRegP dst)
7702 %{
7703   match(Set dst (CheckCastPP dst));
7704 
7705   size(0);
7706   format %{ "# checkcastPP of $dst" %}
7707   ins_encode(/* empty encoding */);
7708   ins_pipe(empty);
7709 %}
7710 
7711 instruct castPP(rRegP dst)
7712 %{
7713   match(Set dst (CastPP dst));
7714 
7715   size(0);
7716   format %{ "# castPP of $dst" %}
7717   ins_encode(/* empty encoding */);
7718   ins_pipe(empty);
7719 %}
7720 
7721 instruct castII(rRegI dst)
7722 %{
7723   match(Set dst (CastII dst));
7724 
7725   size(0);
7726   format %{ "# castII of $dst" %}
7727   ins_encode(/* empty encoding */);
7728   ins_cost(0);
7729   ins_pipe(empty);
7730 %}
7731 
7732 // LoadP-locked same as a regular LoadP when used with compare-swap
7733 instruct loadPLocked(rRegP dst, memory mem)
7734 %{
7735   match(Set dst (LoadPLocked mem));
7736 
7737   ins_cost(125); // XXX
7738   format %{ "movq    $dst, $mem\t# ptr locked" %}
7739   opcode(0x8B);
7740   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7741   ins_pipe(ialu_reg_mem); // XXX
7742 %}
7743 
7744 // Conditional-store of the updated heap-top.
7745 // Used during allocation of the shared heap.
7746 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7747 
7748 instruct storePConditional(memory heap_top_ptr,
7749                            rax_RegP oldval, rRegP newval,
7750                            rFlagsReg cr)
7751 %{
7752   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7753 
7754   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7755             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7756   opcode(0x0F, 0xB1);
7757   ins_encode(lock_prefix,
7758              REX_reg_mem_wide(newval, heap_top_ptr),
7759              OpcP, OpcS,
7760              reg_mem(newval, heap_top_ptr));
7761   ins_pipe(pipe_cmpxchg);
7762 %}
7763 
7764 // Conditional-store of an int value.
7765 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7766 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7767 %{
7768   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7769   effect(KILL oldval);
7770 
7771   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7772   opcode(0x0F, 0xB1);
7773   ins_encode(lock_prefix,
7774              REX_reg_mem(newval, mem),
7775              OpcP, OpcS,
7776              reg_mem(newval, mem));
7777   ins_pipe(pipe_cmpxchg);
7778 %}
7779 
7780 // Conditional-store of a long value.
7781 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7782 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7783 %{
7784   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7785   effect(KILL oldval);
7786 
7787   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7788   opcode(0x0F, 0xB1);
7789   ins_encode(lock_prefix,
7790              REX_reg_mem_wide(newval, mem),
7791              OpcP, OpcS,
7792              reg_mem(newval, mem));
7793   ins_pipe(pipe_cmpxchg);
7794 %}
7795 
7796 
7797 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7798 instruct compareAndSwapP(rRegI res,
7799                          memory mem_ptr,
7800                          rax_RegP oldval, rRegP newval,
7801                          rFlagsReg cr)
7802 %{
7803   predicate(VM_Version::supports_cx8());
7804   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7805   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7806   effect(KILL cr, KILL oldval);
7807 
7808   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7809             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7810             "sete    $res\n\t"
7811             "movzbl  $res, $res" %}
7812   opcode(0x0F, 0xB1);
7813   ins_encode(lock_prefix,
7814              REX_reg_mem_wide(newval, mem_ptr),
7815              OpcP, OpcS,
7816              reg_mem(newval, mem_ptr),
7817              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7818              REX_reg_breg(res, res), // movzbl
7819              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7820   ins_pipe( pipe_cmpxchg );
7821 %}
7822 
7823 instruct compareAndSwapL(rRegI res,
7824                          memory mem_ptr,
7825                          rax_RegL oldval, rRegL newval,
7826                          rFlagsReg cr)
7827 %{
7828   predicate(VM_Version::supports_cx8());
7829   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7830   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7831   effect(KILL cr, KILL oldval);
7832 
7833   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7834             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7835             "sete    $res\n\t"
7836             "movzbl  $res, $res" %}
7837   opcode(0x0F, 0xB1);
7838   ins_encode(lock_prefix,
7839              REX_reg_mem_wide(newval, mem_ptr),
7840              OpcP, OpcS,
7841              reg_mem(newval, mem_ptr),
7842              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7843              REX_reg_breg(res, res), // movzbl
7844              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7845   ins_pipe( pipe_cmpxchg );
7846 %}
7847 
7848 instruct compareAndSwapI(rRegI res,
7849                          memory mem_ptr,
7850                          rax_RegI oldval, rRegI newval,
7851                          rFlagsReg cr)
7852 %{
7853   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7854   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7855   effect(KILL cr, KILL oldval);
7856 
7857   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7858             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7859             "sete    $res\n\t"
7860             "movzbl  $res, $res" %}
7861   opcode(0x0F, 0xB1);
7862   ins_encode(lock_prefix,
7863              REX_reg_mem(newval, mem_ptr),
7864              OpcP, OpcS,
7865              reg_mem(newval, mem_ptr),
7866              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7867              REX_reg_breg(res, res), // movzbl
7868              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7869   ins_pipe( pipe_cmpxchg );
7870 %}
7871 
7872 instruct compareAndSwapB(rRegI res,
7873                          memory mem_ptr,
7874                          rax_RegI oldval, rRegI newval,
7875                          rFlagsReg cr)
7876 %{
7877   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7878   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7879   effect(KILL cr, KILL oldval);
7880 
7881   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7882             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7883             "sete    $res\n\t"
7884             "movzbl  $res, $res" %}
7885   opcode(0x0F, 0xB0);
7886   ins_encode(lock_prefix,
7887              REX_breg_mem(newval, mem_ptr),
7888              OpcP, OpcS,
7889              reg_mem(newval, mem_ptr),
7890              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7891              REX_reg_breg(res, res), // movzbl
7892              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7893   ins_pipe( pipe_cmpxchg );
7894 %}
7895 
7896 instruct compareAndSwapS(rRegI res,
7897                          memory mem_ptr,
7898                          rax_RegI oldval, rRegI newval,
7899                          rFlagsReg cr)
7900 %{
7901   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7902   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7903   effect(KILL cr, KILL oldval);
7904 
7905   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7906             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7907             "sete    $res\n\t"
7908             "movzbl  $res, $res" %}
7909   opcode(0x0F, 0xB1);
7910   ins_encode(lock_prefix,
7911              SizePrefix,
7912              REX_reg_mem(newval, mem_ptr),
7913              OpcP, OpcS,
7914              reg_mem(newval, mem_ptr),
7915              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7916              REX_reg_breg(res, res), // movzbl
7917              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7918   ins_pipe( pipe_cmpxchg );
7919 %}
7920 
7921 instruct compareAndSwapN(rRegI res,
7922                           memory mem_ptr,
7923                           rax_RegN oldval, rRegN newval,
7924                           rFlagsReg cr) %{
7925   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7926   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7927   effect(KILL cr, KILL oldval);
7928 
7929   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7930             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7931             "sete    $res\n\t"
7932             "movzbl  $res, $res" %}
7933   opcode(0x0F, 0xB1);
7934   ins_encode(lock_prefix,
7935              REX_reg_mem(newval, mem_ptr),
7936              OpcP, OpcS,
7937              reg_mem(newval, mem_ptr),
7938              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7939              REX_reg_breg(res, res), // movzbl
7940              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7941   ins_pipe( pipe_cmpxchg );
7942 %}
7943 
7944 instruct compareAndExchangeB(
7945                          memory mem_ptr,
7946                          rax_RegI oldval, rRegI newval,
7947                          rFlagsReg cr)
7948 %{
7949   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7950   effect(KILL cr);
7951 
7952   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7953             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7954   opcode(0x0F, 0xB0);
7955   ins_encode(lock_prefix,
7956              REX_breg_mem(newval, mem_ptr),
7957              OpcP, OpcS,
7958              reg_mem(newval, mem_ptr) // lock cmpxchg
7959              );
7960   ins_pipe( pipe_cmpxchg );
7961 %}
7962 
7963 instruct compareAndExchangeS(
7964                          memory mem_ptr,
7965                          rax_RegI oldval, rRegI newval,
7966                          rFlagsReg cr)
7967 %{
7968   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7969   effect(KILL cr);
7970 
7971   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7972             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7973   opcode(0x0F, 0xB1);
7974   ins_encode(lock_prefix,
7975              SizePrefix,
7976              REX_reg_mem(newval, mem_ptr),
7977              OpcP, OpcS,
7978              reg_mem(newval, mem_ptr) // lock cmpxchg
7979              );
7980   ins_pipe( pipe_cmpxchg );
7981 %}
7982 
7983 instruct compareAndExchangeI(
7984                          memory mem_ptr,
7985                          rax_RegI oldval, rRegI newval,
7986                          rFlagsReg cr)
7987 %{
7988   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7989   effect(KILL cr);
7990 
7991   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7992             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7993   opcode(0x0F, 0xB1);
7994   ins_encode(lock_prefix,
7995              REX_reg_mem(newval, mem_ptr),
7996              OpcP, OpcS,
7997              reg_mem(newval, mem_ptr) // lock cmpxchg
7998              );
7999   ins_pipe( pipe_cmpxchg );
8000 %}
8001 
8002 instruct compareAndExchangeL(
8003                          memory mem_ptr,
8004                          rax_RegL oldval, rRegL newval,
8005                          rFlagsReg cr)
8006 %{
8007   predicate(VM_Version::supports_cx8());
8008   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
8009   effect(KILL cr);
8010 
8011   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8012             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8013   opcode(0x0F, 0xB1);
8014   ins_encode(lock_prefix,
8015              REX_reg_mem_wide(newval, mem_ptr),
8016              OpcP, OpcS,
8017              reg_mem(newval, mem_ptr)  // lock cmpxchg
8018             );
8019   ins_pipe( pipe_cmpxchg );
8020 %}
8021 
8022 instruct compareAndExchangeN(
8023                           memory mem_ptr,
8024                           rax_RegN oldval, rRegN newval,
8025                           rFlagsReg cr) %{
8026   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
8027   effect(KILL cr);
8028 
8029   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8030             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8031   opcode(0x0F, 0xB1);
8032   ins_encode(lock_prefix,
8033              REX_reg_mem(newval, mem_ptr),
8034              OpcP, OpcS,
8035              reg_mem(newval, mem_ptr)  // lock cmpxchg
8036           );
8037   ins_pipe( pipe_cmpxchg );
8038 %}
8039 
8040 instruct compareAndExchangeP(
8041                          memory mem_ptr,
8042                          rax_RegP oldval, rRegP newval,
8043                          rFlagsReg cr)
8044 %{
8045   predicate(VM_Version::supports_cx8());
8046   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
8047   effect(KILL cr);
8048 
8049   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8050             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8051   opcode(0x0F, 0xB1);
8052   ins_encode(lock_prefix,
8053              REX_reg_mem_wide(newval, mem_ptr),
8054              OpcP, OpcS,
8055              reg_mem(newval, mem_ptr)  // lock cmpxchg
8056           );
8057   ins_pipe( pipe_cmpxchg );
8058 %}
8059 
8060 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8061   predicate(n->as_LoadStore()->result_not_used());
8062   match(Set dummy (GetAndAddB mem add));
8063   effect(KILL cr);
8064   format %{ "ADDB  [$mem],$add" %}
8065   ins_encode %{
8066     __ lock();
8067     __ addb($mem$$Address, $add$$constant);
8068   %}
8069   ins_pipe( pipe_cmpxchg );
8070 %}
8071 
8072 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
8073   match(Set newval (GetAndAddB mem newval));
8074   effect(KILL cr);
8075   format %{ "XADDB  [$mem],$newval" %}
8076   ins_encode %{
8077     __ lock();
8078     __ xaddb($mem$$Address, $newval$$Register);
8079   %}
8080   ins_pipe( pipe_cmpxchg );
8081 %}
8082 
8083 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8084   predicate(n->as_LoadStore()->result_not_used());
8085   match(Set dummy (GetAndAddS mem add));
8086   effect(KILL cr);
8087   format %{ "ADDW  [$mem],$add" %}
8088   ins_encode %{
8089     __ lock();
8090     __ addw($mem$$Address, $add$$constant);
8091   %}
8092   ins_pipe( pipe_cmpxchg );
8093 %}
8094 
8095 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
8096   match(Set newval (GetAndAddS mem newval));
8097   effect(KILL cr);
8098   format %{ "XADDW  [$mem],$newval" %}
8099   ins_encode %{
8100     __ lock();
8101     __ xaddw($mem$$Address, $newval$$Register);
8102   %}
8103   ins_pipe( pipe_cmpxchg );
8104 %}
8105 
8106 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8107   predicate(n->as_LoadStore()->result_not_used());
8108   match(Set dummy (GetAndAddI mem add));
8109   effect(KILL cr);
8110   format %{ "ADDL  [$mem],$add" %}
8111   ins_encode %{
8112     __ lock();
8113     __ addl($mem$$Address, $add$$constant);
8114   %}
8115   ins_pipe( pipe_cmpxchg );
8116 %}
8117 
8118 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
8119   match(Set newval (GetAndAddI mem newval));
8120   effect(KILL cr);
8121   format %{ "XADDL  [$mem],$newval" %}
8122   ins_encode %{
8123     __ lock();
8124     __ xaddl($mem$$Address, $newval$$Register);
8125   %}
8126   ins_pipe( pipe_cmpxchg );
8127 %}
8128 
8129 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
8130   predicate(n->as_LoadStore()->result_not_used());
8131   match(Set dummy (GetAndAddL mem add));
8132   effect(KILL cr);
8133   format %{ "ADDQ  [$mem],$add" %}
8134   ins_encode %{
8135     __ lock();
8136     __ addq($mem$$Address, $add$$constant);
8137   %}
8138   ins_pipe( pipe_cmpxchg );
8139 %}
8140 
8141 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
8142   match(Set newval (GetAndAddL mem newval));
8143   effect(KILL cr);
8144   format %{ "XADDQ  [$mem],$newval" %}
8145   ins_encode %{
8146     __ lock();
8147     __ xaddq($mem$$Address, $newval$$Register);
8148   %}
8149   ins_pipe( pipe_cmpxchg );
8150 %}
8151 
8152 instruct xchgB( memory mem, rRegI newval) %{
8153   match(Set newval (GetAndSetB mem newval));
8154   format %{ "XCHGB  $newval,[$mem]" %}
8155   ins_encode %{
8156     __ xchgb($newval$$Register, $mem$$Address);
8157   %}
8158   ins_pipe( pipe_cmpxchg );
8159 %}
8160 
8161 instruct xchgS( memory mem, rRegI newval) %{
8162   match(Set newval (GetAndSetS mem newval));
8163   format %{ "XCHGW  $newval,[$mem]" %}
8164   ins_encode %{
8165     __ xchgw($newval$$Register, $mem$$Address);
8166   %}
8167   ins_pipe( pipe_cmpxchg );
8168 %}
8169 
8170 instruct xchgI( memory mem, rRegI newval) %{
8171   match(Set newval (GetAndSetI mem newval));
8172   format %{ "XCHGL  $newval,[$mem]" %}
8173   ins_encode %{
8174     __ xchgl($newval$$Register, $mem$$Address);
8175   %}
8176   ins_pipe( pipe_cmpxchg );
8177 %}
8178 
8179 instruct xchgL( memory mem, rRegL newval) %{
8180   match(Set newval (GetAndSetL mem newval));
8181   format %{ "XCHGL  $newval,[$mem]" %}
8182   ins_encode %{
8183     __ xchgq($newval$$Register, $mem$$Address);
8184   %}
8185   ins_pipe( pipe_cmpxchg );
8186 %}
8187 
8188 instruct xchgP( memory mem, rRegP newval) %{
8189   match(Set newval (GetAndSetP mem newval));
8190   format %{ "XCHGQ  $newval,[$mem]" %}
8191   ins_encode %{
8192     __ xchgq($newval$$Register, $mem$$Address);
8193   %}
8194   ins_pipe( pipe_cmpxchg );
8195 %}
8196 
8197 instruct xchgN( memory mem, rRegN newval) %{
8198   match(Set newval (GetAndSetN mem newval));
8199   format %{ "XCHGL  $newval,$mem]" %}
8200   ins_encode %{
8201     __ xchgl($newval$$Register, $mem$$Address);
8202   %}
8203   ins_pipe( pipe_cmpxchg );
8204 %}
8205 
8206 //----------Abs Instructions-------------------------------------------
8207 
8208 // Integer Absolute Instructions
8209 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
8210 %{
8211   match(Set dst (AbsI src));
8212   effect(TEMP dst, TEMP tmp, KILL cr);
8213   format %{ "movl $tmp, $src\n\t"
8214             "sarl $tmp, 31\n\t"
8215             "movl $dst, $src\n\t"
8216             "xorl $dst, $tmp\n\t"
8217             "subl $dst, $tmp\n"
8218           %}
8219   ins_encode %{
8220     __ movl($tmp$$Register, $src$$Register);
8221     __ sarl($tmp$$Register, 31);
8222     __ movl($dst$$Register, $src$$Register);
8223     __ xorl($dst$$Register, $tmp$$Register);
8224     __ subl($dst$$Register, $tmp$$Register);
8225   %}
8226 
8227   ins_pipe(ialu_reg_reg);
8228 %}
8229 
8230 // Long Absolute Instructions
8231 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
8232 %{
8233   match(Set dst (AbsL src));
8234   effect(TEMP dst, TEMP tmp, KILL cr);
8235   format %{ "movq $tmp, $src\n\t"
8236             "sarq $tmp, 63\n\t"
8237             "movq $dst, $src\n\t"
8238             "xorq $dst, $tmp\n\t"
8239             "subq $dst, $tmp\n"
8240           %}
8241   ins_encode %{
8242     __ movq($tmp$$Register, $src$$Register);
8243     __ sarq($tmp$$Register, 63);
8244     __ movq($dst$$Register, $src$$Register);
8245     __ xorq($dst$$Register, $tmp$$Register);
8246     __ subq($dst$$Register, $tmp$$Register);
8247   %}
8248 
8249   ins_pipe(ialu_reg_reg);
8250 %}
8251 
8252 //----------Subtraction Instructions-------------------------------------------
8253 
8254 // Integer Subtraction Instructions
8255 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8256 %{
8257   match(Set dst (SubI dst src));
8258   effect(KILL cr);
8259 
8260   format %{ "subl    $dst, $src\t# int" %}
8261   opcode(0x2B);
8262   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8263   ins_pipe(ialu_reg_reg);
8264 %}
8265 
8266 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8267 %{
8268   match(Set dst (SubI dst src));
8269   effect(KILL cr);
8270 
8271   format %{ "subl    $dst, $src\t# int" %}
8272   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8273   ins_encode(OpcSErm(dst, src), Con8or32(src));
8274   ins_pipe(ialu_reg);
8275 %}
8276 
8277 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8278 %{
8279   match(Set dst (SubI dst (LoadI src)));
8280   effect(KILL cr);
8281 
8282   ins_cost(125);
8283   format %{ "subl    $dst, $src\t# int" %}
8284   opcode(0x2B);
8285   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8286   ins_pipe(ialu_reg_mem);
8287 %}
8288 
8289 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8290 %{
8291   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8292   effect(KILL cr);
8293 
8294   ins_cost(150);
8295   format %{ "subl    $dst, $src\t# int" %}
8296   opcode(0x29); /* Opcode 29 /r */
8297   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8298   ins_pipe(ialu_mem_reg);
8299 %}
8300 
8301 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
8302 %{
8303   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8304   effect(KILL cr);
8305 
8306   ins_cost(125); // XXX
8307   format %{ "subl    $dst, $src\t# int" %}
8308   opcode(0x81); /* Opcode 81 /5 id */
8309   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8310   ins_pipe(ialu_mem_imm);
8311 %}
8312 
8313 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8314 %{
8315   match(Set dst (SubL dst src));
8316   effect(KILL cr);
8317 
8318   format %{ "subq    $dst, $src\t# long" %}
8319   opcode(0x2B);
8320   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8321   ins_pipe(ialu_reg_reg);
8322 %}
8323 
8324 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
8325 %{
8326   match(Set dst (SubL dst src));
8327   effect(KILL cr);
8328 
8329   format %{ "subq    $dst, $src\t# long" %}
8330   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8331   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8332   ins_pipe(ialu_reg);
8333 %}
8334 
8335 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8336 %{
8337   match(Set dst (SubL dst (LoadL src)));
8338   effect(KILL cr);
8339 
8340   ins_cost(125);
8341   format %{ "subq    $dst, $src\t# long" %}
8342   opcode(0x2B);
8343   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8344   ins_pipe(ialu_reg_mem);
8345 %}
8346 
8347 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8348 %{
8349   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8350   effect(KILL cr);
8351 
8352   ins_cost(150);
8353   format %{ "subq    $dst, $src\t# long" %}
8354   opcode(0x29); /* Opcode 29 /r */
8355   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8356   ins_pipe(ialu_mem_reg);
8357 %}
8358 
8359 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8360 %{
8361   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8362   effect(KILL cr);
8363 
8364   ins_cost(125); // XXX
8365   format %{ "subq    $dst, $src\t# long" %}
8366   opcode(0x81); /* Opcode 81 /5 id */
8367   ins_encode(REX_mem_wide(dst),
8368              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8369   ins_pipe(ialu_mem_imm);
8370 %}
8371 
8372 // Subtract from a pointer
8373 // XXX hmpf???
8374 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8375 %{
8376   match(Set dst (AddP dst (SubI zero src)));
8377   effect(KILL cr);
8378 
8379   format %{ "subq    $dst, $src\t# ptr - int" %}
8380   opcode(0x2B);
8381   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8382   ins_pipe(ialu_reg_reg);
8383 %}
8384 
8385 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8386 %{
8387   match(Set dst (SubI zero dst));
8388   effect(KILL cr);
8389 
8390   format %{ "negl    $dst\t# int" %}
8391   opcode(0xF7, 0x03);  // Opcode F7 /3
8392   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8393   ins_pipe(ialu_reg);
8394 %}
8395 
8396 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8397 %{
8398   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8399   effect(KILL cr);
8400 
8401   format %{ "negl    $dst\t# int" %}
8402   opcode(0xF7, 0x03);  // Opcode F7 /3
8403   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8404   ins_pipe(ialu_reg);
8405 %}
8406 
8407 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8408 %{
8409   match(Set dst (SubL zero dst));
8410   effect(KILL cr);
8411 
8412   format %{ "negq    $dst\t# long" %}
8413   opcode(0xF7, 0x03);  // Opcode F7 /3
8414   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8415   ins_pipe(ialu_reg);
8416 %}
8417 
8418 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8419 %{
8420   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8421   effect(KILL cr);
8422 
8423   format %{ "negq    $dst\t# long" %}
8424   opcode(0xF7, 0x03);  // Opcode F7 /3
8425   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8426   ins_pipe(ialu_reg);
8427 %}
8428 
8429 //----------Multiplication/Division Instructions-------------------------------
8430 // Integer Multiplication Instructions
8431 // Multiply Register
8432 
8433 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8434 %{
8435   match(Set dst (MulI dst src));
8436   effect(KILL cr);
8437 
8438   ins_cost(300);
8439   format %{ "imull   $dst, $src\t# int" %}
8440   opcode(0x0F, 0xAF);
8441   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8442   ins_pipe(ialu_reg_reg_alu0);
8443 %}
8444 
8445 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8446 %{
8447   match(Set dst (MulI src imm));
8448   effect(KILL cr);
8449 
8450   ins_cost(300);
8451   format %{ "imull   $dst, $src, $imm\t# int" %}
8452   opcode(0x69); /* 69 /r id */
8453   ins_encode(REX_reg_reg(dst, src),
8454              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8455   ins_pipe(ialu_reg_reg_alu0);
8456 %}
8457 
8458 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8459 %{
8460   match(Set dst (MulI dst (LoadI src)));
8461   effect(KILL cr);
8462 
8463   ins_cost(350);
8464   format %{ "imull   $dst, $src\t# int" %}
8465   opcode(0x0F, 0xAF);
8466   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8467   ins_pipe(ialu_reg_mem_alu0);
8468 %}
8469 
8470 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8471 %{
8472   match(Set dst (MulI (LoadI src) imm));
8473   effect(KILL cr);
8474 
8475   ins_cost(300);
8476   format %{ "imull   $dst, $src, $imm\t# int" %}
8477   opcode(0x69); /* 69 /r id */
8478   ins_encode(REX_reg_mem(dst, src),
8479              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8480   ins_pipe(ialu_reg_mem_alu0);
8481 %}
8482 
8483 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
8484 %{
8485   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
8486   effect(KILL cr, KILL src2);
8487 
8488   expand %{ mulI_rReg(dst, src1, cr);
8489            mulI_rReg(src2, src3, cr);
8490            addI_rReg(dst, src2, cr); %}
8491 %}
8492 
8493 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8494 %{
8495   match(Set dst (MulL dst src));
8496   effect(KILL cr);
8497 
8498   ins_cost(300);
8499   format %{ "imulq   $dst, $src\t# long" %}
8500   opcode(0x0F, 0xAF);
8501   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8502   ins_pipe(ialu_reg_reg_alu0);
8503 %}
8504 
8505 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8506 %{
8507   match(Set dst (MulL src imm));
8508   effect(KILL cr);
8509 
8510   ins_cost(300);
8511   format %{ "imulq   $dst, $src, $imm\t# long" %}
8512   opcode(0x69); /* 69 /r id */
8513   ins_encode(REX_reg_reg_wide(dst, src),
8514              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8515   ins_pipe(ialu_reg_reg_alu0);
8516 %}
8517 
8518 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8519 %{
8520   match(Set dst (MulL dst (LoadL src)));
8521   effect(KILL cr);
8522 
8523   ins_cost(350);
8524   format %{ "imulq   $dst, $src\t# long" %}
8525   opcode(0x0F, 0xAF);
8526   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8527   ins_pipe(ialu_reg_mem_alu0);
8528 %}
8529 
8530 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8531 %{
8532   match(Set dst (MulL (LoadL src) imm));
8533   effect(KILL cr);
8534 
8535   ins_cost(300);
8536   format %{ "imulq   $dst, $src, $imm\t# long" %}
8537   opcode(0x69); /* 69 /r id */
8538   ins_encode(REX_reg_mem_wide(dst, src),
8539              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8540   ins_pipe(ialu_reg_mem_alu0);
8541 %}
8542 
8543 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8544 %{
8545   match(Set dst (MulHiL src rax));
8546   effect(USE_KILL rax, KILL cr);
8547 
8548   ins_cost(300);
8549   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8550   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8551   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8552   ins_pipe(ialu_reg_reg_alu0);
8553 %}
8554 
8555 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8556                    rFlagsReg cr)
8557 %{
8558   match(Set rax (DivI rax div));
8559   effect(KILL rdx, KILL cr);
8560 
8561   ins_cost(30*100+10*100); // XXX
8562   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8563             "jne,s   normal\n\t"
8564             "xorl    rdx, rdx\n\t"
8565             "cmpl    $div, -1\n\t"
8566             "je,s    done\n"
8567     "normal: cdql\n\t"
8568             "idivl   $div\n"
8569     "done:"        %}
8570   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8571   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8572   ins_pipe(ialu_reg_reg_alu0);
8573 %}
8574 
8575 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8576                    rFlagsReg cr)
8577 %{
8578   match(Set rax (DivL rax div));
8579   effect(KILL rdx, KILL cr);
8580 
8581   ins_cost(30*100+10*100); // XXX
8582   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8583             "cmpq    rax, rdx\n\t"
8584             "jne,s   normal\n\t"
8585             "xorl    rdx, rdx\n\t"
8586             "cmpq    $div, -1\n\t"
8587             "je,s    done\n"
8588     "normal: cdqq\n\t"
8589             "idivq   $div\n"
8590     "done:"        %}
8591   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8592   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8593   ins_pipe(ialu_reg_reg_alu0);
8594 %}
8595 
8596 // Integer DIVMOD with Register, both quotient and mod results
8597 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8598                              rFlagsReg cr)
8599 %{
8600   match(DivModI rax div);
8601   effect(KILL cr);
8602 
8603   ins_cost(30*100+10*100); // XXX
8604   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8605             "jne,s   normal\n\t"
8606             "xorl    rdx, rdx\n\t"
8607             "cmpl    $div, -1\n\t"
8608             "je,s    done\n"
8609     "normal: cdql\n\t"
8610             "idivl   $div\n"
8611     "done:"        %}
8612   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8613   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8614   ins_pipe(pipe_slow);
8615 %}
8616 
8617 // Long DIVMOD with Register, both quotient and mod results
8618 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8619                              rFlagsReg cr)
8620 %{
8621   match(DivModL rax div);
8622   effect(KILL cr);
8623 
8624   ins_cost(30*100+10*100); // XXX
8625   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8626             "cmpq    rax, rdx\n\t"
8627             "jne,s   normal\n\t"
8628             "xorl    rdx, rdx\n\t"
8629             "cmpq    $div, -1\n\t"
8630             "je,s    done\n"
8631     "normal: cdqq\n\t"
8632             "idivq   $div\n"
8633     "done:"        %}
8634   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8635   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8636   ins_pipe(pipe_slow);
8637 %}
8638 
8639 //----------- DivL-By-Constant-Expansions--------------------------------------
8640 // DivI cases are handled by the compiler
8641 
8642 // Magic constant, reciprocal of 10
8643 instruct loadConL_0x6666666666666667(rRegL dst)
8644 %{
8645   effect(DEF dst);
8646 
8647   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8648   ins_encode(load_immL(dst, 0x6666666666666667));
8649   ins_pipe(ialu_reg);
8650 %}
8651 
8652 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8653 %{
8654   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8655 
8656   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8657   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8658   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8659   ins_pipe(ialu_reg_reg_alu0);
8660 %}
8661 
8662 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8663 %{
8664   effect(USE_DEF dst, KILL cr);
8665 
8666   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8667   opcode(0xC1, 0x7); /* C1 /7 ib */
8668   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8669   ins_pipe(ialu_reg);
8670 %}
8671 
8672 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8673 %{
8674   effect(USE_DEF dst, KILL cr);
8675 
8676   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8677   opcode(0xC1, 0x7); /* C1 /7 ib */
8678   ins_encode(reg_opc_imm_wide(dst, 0x2));
8679   ins_pipe(ialu_reg);
8680 %}
8681 
8682 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8683 %{
8684   match(Set dst (DivL src div));
8685 
8686   ins_cost((5+8)*100);
8687   expand %{
8688     rax_RegL rax;                     // Killed temp
8689     rFlagsReg cr;                     // Killed
8690     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8691     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8692     sarL_rReg_63(src, cr);            // sarq  src, 63
8693     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8694     subL_rReg(dst, src, cr);          // subl  rdx, src
8695   %}
8696 %}
8697 
8698 //-----------------------------------------------------------------------------
8699 
8700 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8701                    rFlagsReg cr)
8702 %{
8703   match(Set rdx (ModI rax div));
8704   effect(KILL rax, KILL cr);
8705 
8706   ins_cost(300); // XXX
8707   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8708             "jne,s   normal\n\t"
8709             "xorl    rdx, rdx\n\t"
8710             "cmpl    $div, -1\n\t"
8711             "je,s    done\n"
8712     "normal: cdql\n\t"
8713             "idivl   $div\n"
8714     "done:"        %}
8715   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8716   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8717   ins_pipe(ialu_reg_reg_alu0);
8718 %}
8719 
8720 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8721                    rFlagsReg cr)
8722 %{
8723   match(Set rdx (ModL rax div));
8724   effect(KILL rax, KILL cr);
8725 
8726   ins_cost(300); // XXX
8727   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8728             "cmpq    rax, rdx\n\t"
8729             "jne,s   normal\n\t"
8730             "xorl    rdx, rdx\n\t"
8731             "cmpq    $div, -1\n\t"
8732             "je,s    done\n"
8733     "normal: cdqq\n\t"
8734             "idivq   $div\n"
8735     "done:"        %}
8736   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8737   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8738   ins_pipe(ialu_reg_reg_alu0);
8739 %}
8740 
8741 // Integer Shift Instructions
8742 // Shift Left by one
8743 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8744 %{
8745   match(Set dst (LShiftI dst shift));
8746   effect(KILL cr);
8747 
8748   format %{ "sall    $dst, $shift" %}
8749   opcode(0xD1, 0x4); /* D1 /4 */
8750   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8751   ins_pipe(ialu_reg);
8752 %}
8753 
8754 // Shift Left by one
8755 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8756 %{
8757   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8758   effect(KILL cr);
8759 
8760   format %{ "sall    $dst, $shift\t" %}
8761   opcode(0xD1, 0x4); /* D1 /4 */
8762   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8763   ins_pipe(ialu_mem_imm);
8764 %}
8765 
8766 // Shift Left by 8-bit immediate
8767 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8768 %{
8769   match(Set dst (LShiftI dst shift));
8770   effect(KILL cr);
8771 
8772   format %{ "sall    $dst, $shift" %}
8773   opcode(0xC1, 0x4); /* C1 /4 ib */
8774   ins_encode(reg_opc_imm(dst, shift));
8775   ins_pipe(ialu_reg);
8776 %}
8777 
8778 // Shift Left by 8-bit immediate
8779 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8780 %{
8781   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8782   effect(KILL cr);
8783 
8784   format %{ "sall    $dst, $shift" %}
8785   opcode(0xC1, 0x4); /* C1 /4 ib */
8786   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8787   ins_pipe(ialu_mem_imm);
8788 %}
8789 
8790 // Shift Left by variable
8791 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8792 %{
8793   match(Set dst (LShiftI dst shift));
8794   effect(KILL cr);
8795 
8796   format %{ "sall    $dst, $shift" %}
8797   opcode(0xD3, 0x4); /* D3 /4 */
8798   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8799   ins_pipe(ialu_reg_reg);
8800 %}
8801 
8802 // Shift Left by variable
8803 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8804 %{
8805   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8806   effect(KILL cr);
8807 
8808   format %{ "sall    $dst, $shift" %}
8809   opcode(0xD3, 0x4); /* D3 /4 */
8810   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8811   ins_pipe(ialu_mem_reg);
8812 %}
8813 
8814 // Arithmetic shift right by one
8815 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8816 %{
8817   match(Set dst (RShiftI dst shift));
8818   effect(KILL cr);
8819 
8820   format %{ "sarl    $dst, $shift" %}
8821   opcode(0xD1, 0x7); /* D1 /7 */
8822   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8823   ins_pipe(ialu_reg);
8824 %}
8825 
8826 // Arithmetic shift right by one
8827 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8828 %{
8829   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8830   effect(KILL cr);
8831 
8832   format %{ "sarl    $dst, $shift" %}
8833   opcode(0xD1, 0x7); /* D1 /7 */
8834   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8835   ins_pipe(ialu_mem_imm);
8836 %}
8837 
8838 // Arithmetic Shift Right by 8-bit immediate
8839 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8840 %{
8841   match(Set dst (RShiftI dst shift));
8842   effect(KILL cr);
8843 
8844   format %{ "sarl    $dst, $shift" %}
8845   opcode(0xC1, 0x7); /* C1 /7 ib */
8846   ins_encode(reg_opc_imm(dst, shift));
8847   ins_pipe(ialu_mem_imm);
8848 %}
8849 
8850 // Arithmetic Shift Right by 8-bit immediate
8851 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8852 %{
8853   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8854   effect(KILL cr);
8855 
8856   format %{ "sarl    $dst, $shift" %}
8857   opcode(0xC1, 0x7); /* C1 /7 ib */
8858   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8859   ins_pipe(ialu_mem_imm);
8860 %}
8861 
8862 // Arithmetic Shift Right by variable
8863 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8864 %{
8865   match(Set dst (RShiftI dst shift));
8866   effect(KILL cr);
8867 
8868   format %{ "sarl    $dst, $shift" %}
8869   opcode(0xD3, 0x7); /* D3 /7 */
8870   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8871   ins_pipe(ialu_reg_reg);
8872 %}
8873 
8874 // Arithmetic Shift Right by variable
8875 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8876 %{
8877   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8878   effect(KILL cr);
8879 
8880   format %{ "sarl    $dst, $shift" %}
8881   opcode(0xD3, 0x7); /* D3 /7 */
8882   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8883   ins_pipe(ialu_mem_reg);
8884 %}
8885 
8886 // Logical shift right by one
8887 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8888 %{
8889   match(Set dst (URShiftI dst shift));
8890   effect(KILL cr);
8891 
8892   format %{ "shrl    $dst, $shift" %}
8893   opcode(0xD1, 0x5); /* D1 /5 */
8894   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8895   ins_pipe(ialu_reg);
8896 %}
8897 
8898 // Logical shift right by one
8899 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8900 %{
8901   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8902   effect(KILL cr);
8903 
8904   format %{ "shrl    $dst, $shift" %}
8905   opcode(0xD1, 0x5); /* D1 /5 */
8906   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8907   ins_pipe(ialu_mem_imm);
8908 %}
8909 
8910 // Logical Shift Right by 8-bit immediate
8911 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8912 %{
8913   match(Set dst (URShiftI dst shift));
8914   effect(KILL cr);
8915 
8916   format %{ "shrl    $dst, $shift" %}
8917   opcode(0xC1, 0x5); /* C1 /5 ib */
8918   ins_encode(reg_opc_imm(dst, shift));
8919   ins_pipe(ialu_reg);
8920 %}
8921 
8922 // Logical Shift Right by 8-bit immediate
8923 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8924 %{
8925   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8926   effect(KILL cr);
8927 
8928   format %{ "shrl    $dst, $shift" %}
8929   opcode(0xC1, 0x5); /* C1 /5 ib */
8930   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8931   ins_pipe(ialu_mem_imm);
8932 %}
8933 
8934 // Logical Shift Right by variable
8935 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8936 %{
8937   match(Set dst (URShiftI dst shift));
8938   effect(KILL cr);
8939 
8940   format %{ "shrl    $dst, $shift" %}
8941   opcode(0xD3, 0x5); /* D3 /5 */
8942   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8943   ins_pipe(ialu_reg_reg);
8944 %}
8945 
8946 // Logical Shift Right by variable
8947 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8948 %{
8949   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8950   effect(KILL cr);
8951 
8952   format %{ "shrl    $dst, $shift" %}
8953   opcode(0xD3, 0x5); /* D3 /5 */
8954   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8955   ins_pipe(ialu_mem_reg);
8956 %}
8957 
8958 // Long Shift Instructions
8959 // Shift Left by one
8960 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8961 %{
8962   match(Set dst (LShiftL dst shift));
8963   effect(KILL cr);
8964 
8965   format %{ "salq    $dst, $shift" %}
8966   opcode(0xD1, 0x4); /* D1 /4 */
8967   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8968   ins_pipe(ialu_reg);
8969 %}
8970 
8971 // Shift Left by one
8972 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8973 %{
8974   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8975   effect(KILL cr);
8976 
8977   format %{ "salq    $dst, $shift" %}
8978   opcode(0xD1, 0x4); /* D1 /4 */
8979   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8980   ins_pipe(ialu_mem_imm);
8981 %}
8982 
8983 // Shift Left by 8-bit immediate
8984 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8985 %{
8986   match(Set dst (LShiftL dst shift));
8987   effect(KILL cr);
8988 
8989   format %{ "salq    $dst, $shift" %}
8990   opcode(0xC1, 0x4); /* C1 /4 ib */
8991   ins_encode(reg_opc_imm_wide(dst, shift));
8992   ins_pipe(ialu_reg);
8993 %}
8994 
8995 // Shift Left by 8-bit immediate
8996 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8997 %{
8998   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8999   effect(KILL cr);
9000 
9001   format %{ "salq    $dst, $shift" %}
9002   opcode(0xC1, 0x4); /* C1 /4 ib */
9003   ins_encode(REX_mem_wide(dst), OpcP,
9004              RM_opc_mem(secondary, dst), Con8or32(shift));
9005   ins_pipe(ialu_mem_imm);
9006 %}
9007 
9008 // Shift Left by variable
9009 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9010 %{
9011   match(Set dst (LShiftL dst shift));
9012   effect(KILL cr);
9013 
9014   format %{ "salq    $dst, $shift" %}
9015   opcode(0xD3, 0x4); /* D3 /4 */
9016   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9017   ins_pipe(ialu_reg_reg);
9018 %}
9019 
9020 // Shift Left by variable
9021 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9022 %{
9023   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
9024   effect(KILL cr);
9025 
9026   format %{ "salq    $dst, $shift" %}
9027   opcode(0xD3, 0x4); /* D3 /4 */
9028   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9029   ins_pipe(ialu_mem_reg);
9030 %}
9031 
9032 // Arithmetic shift right by one
9033 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9034 %{
9035   match(Set dst (RShiftL dst shift));
9036   effect(KILL cr);
9037 
9038   format %{ "sarq    $dst, $shift" %}
9039   opcode(0xD1, 0x7); /* D1 /7 */
9040   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9041   ins_pipe(ialu_reg);
9042 %}
9043 
9044 // Arithmetic shift right by one
9045 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9046 %{
9047   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9048   effect(KILL cr);
9049 
9050   format %{ "sarq    $dst, $shift" %}
9051   opcode(0xD1, 0x7); /* D1 /7 */
9052   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9053   ins_pipe(ialu_mem_imm);
9054 %}
9055 
9056 // Arithmetic Shift Right by 8-bit immediate
9057 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9058 %{
9059   match(Set dst (RShiftL dst shift));
9060   effect(KILL cr);
9061 
9062   format %{ "sarq    $dst, $shift" %}
9063   opcode(0xC1, 0x7); /* C1 /7 ib */
9064   ins_encode(reg_opc_imm_wide(dst, shift));
9065   ins_pipe(ialu_mem_imm);
9066 %}
9067 
9068 // Arithmetic Shift Right by 8-bit immediate
9069 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9070 %{
9071   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9072   effect(KILL cr);
9073 
9074   format %{ "sarq    $dst, $shift" %}
9075   opcode(0xC1, 0x7); /* C1 /7 ib */
9076   ins_encode(REX_mem_wide(dst), OpcP,
9077              RM_opc_mem(secondary, dst), Con8or32(shift));
9078   ins_pipe(ialu_mem_imm);
9079 %}
9080 
9081 // Arithmetic Shift Right by variable
9082 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9083 %{
9084   match(Set dst (RShiftL dst shift));
9085   effect(KILL cr);
9086 
9087   format %{ "sarq    $dst, $shift" %}
9088   opcode(0xD3, 0x7); /* D3 /7 */
9089   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9090   ins_pipe(ialu_reg_reg);
9091 %}
9092 
9093 // Arithmetic Shift Right by variable
9094 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9095 %{
9096   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9097   effect(KILL cr);
9098 
9099   format %{ "sarq    $dst, $shift" %}
9100   opcode(0xD3, 0x7); /* D3 /7 */
9101   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9102   ins_pipe(ialu_mem_reg);
9103 %}
9104 
9105 // Logical shift right by one
9106 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9107 %{
9108   match(Set dst (URShiftL dst shift));
9109   effect(KILL cr);
9110 
9111   format %{ "shrq    $dst, $shift" %}
9112   opcode(0xD1, 0x5); /* D1 /5 */
9113   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
9114   ins_pipe(ialu_reg);
9115 %}
9116 
9117 // Logical shift right by one
9118 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9119 %{
9120   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9121   effect(KILL cr);
9122 
9123   format %{ "shrq    $dst, $shift" %}
9124   opcode(0xD1, 0x5); /* D1 /5 */
9125   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9126   ins_pipe(ialu_mem_imm);
9127 %}
9128 
9129 // Logical Shift Right by 8-bit immediate
9130 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9131 %{
9132   match(Set dst (URShiftL dst shift));
9133   effect(KILL cr);
9134 
9135   format %{ "shrq    $dst, $shift" %}
9136   opcode(0xC1, 0x5); /* C1 /5 ib */
9137   ins_encode(reg_opc_imm_wide(dst, shift));
9138   ins_pipe(ialu_reg);
9139 %}
9140 
9141 
9142 // Logical Shift Right by 8-bit immediate
9143 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9144 %{
9145   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9146   effect(KILL cr);
9147 
9148   format %{ "shrq    $dst, $shift" %}
9149   opcode(0xC1, 0x5); /* C1 /5 ib */
9150   ins_encode(REX_mem_wide(dst), OpcP,
9151              RM_opc_mem(secondary, dst), Con8or32(shift));
9152   ins_pipe(ialu_mem_imm);
9153 %}
9154 
9155 // Logical Shift Right by variable
9156 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9157 %{
9158   match(Set dst (URShiftL dst shift));
9159   effect(KILL cr);
9160 
9161   format %{ "shrq    $dst, $shift" %}
9162   opcode(0xD3, 0x5); /* D3 /5 */
9163   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9164   ins_pipe(ialu_reg_reg);
9165 %}
9166 
9167 // Logical Shift Right by variable
9168 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9169 %{
9170   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9171   effect(KILL cr);
9172 
9173   format %{ "shrq    $dst, $shift" %}
9174   opcode(0xD3, 0x5); /* D3 /5 */
9175   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9176   ins_pipe(ialu_mem_reg);
9177 %}
9178 
9179 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
9180 // This idiom is used by the compiler for the i2b bytecode.
9181 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
9182 %{
9183   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
9184 
9185   format %{ "movsbl  $dst, $src\t# i2b" %}
9186   opcode(0x0F, 0xBE);
9187   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9188   ins_pipe(ialu_reg_reg);
9189 %}
9190 
9191 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
9192 // This idiom is used by the compiler the i2s bytecode.
9193 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
9194 %{
9195   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
9196 
9197   format %{ "movswl  $dst, $src\t# i2s" %}
9198   opcode(0x0F, 0xBF);
9199   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9200   ins_pipe(ialu_reg_reg);
9201 %}
9202 
9203 // ROL/ROR instructions
9204 
9205 // ROL expand
9206 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
9207   effect(KILL cr, USE_DEF dst);
9208 
9209   format %{ "roll    $dst" %}
9210   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9211   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9212   ins_pipe(ialu_reg);
9213 %}
9214 
9215 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
9216   effect(USE_DEF dst, USE shift, KILL cr);
9217 
9218   format %{ "roll    $dst, $shift" %}
9219   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9220   ins_encode( reg_opc_imm(dst, shift) );
9221   ins_pipe(ialu_reg);
9222 %}
9223 
9224 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9225 %{
9226   effect(USE_DEF dst, USE shift, KILL cr);
9227 
9228   format %{ "roll    $dst, $shift" %}
9229   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9230   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9231   ins_pipe(ialu_reg_reg);
9232 %}
9233 // end of ROL expand
9234 
9235 // Rotate Left by one
9236 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9237 %{
9238   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9239 
9240   expand %{
9241     rolI_rReg_imm1(dst, cr);
9242   %}
9243 %}
9244 
9245 // Rotate Left by 8-bit immediate
9246 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9247 %{
9248   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9249   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9250 
9251   expand %{
9252     rolI_rReg_imm8(dst, lshift, cr);
9253   %}
9254 %}
9255 
9256 // Rotate Left by variable
9257 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9258 %{
9259   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9260 
9261   expand %{
9262     rolI_rReg_CL(dst, shift, cr);
9263   %}
9264 %}
9265 
9266 // Rotate Left by variable
9267 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9268 %{
9269   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9270 
9271   expand %{
9272     rolI_rReg_CL(dst, shift, cr);
9273   %}
9274 %}
9275 
9276 // ROR expand
9277 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
9278 %{
9279   effect(USE_DEF dst, KILL cr);
9280 
9281   format %{ "rorl    $dst" %}
9282   opcode(0xD1, 0x1); /* D1 /1 */
9283   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9284   ins_pipe(ialu_reg);
9285 %}
9286 
9287 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
9288 %{
9289   effect(USE_DEF dst, USE shift, KILL cr);
9290 
9291   format %{ "rorl    $dst, $shift" %}
9292   opcode(0xC1, 0x1); /* C1 /1 ib */
9293   ins_encode(reg_opc_imm(dst, shift));
9294   ins_pipe(ialu_reg);
9295 %}
9296 
9297 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9298 %{
9299   effect(USE_DEF dst, USE shift, KILL cr);
9300 
9301   format %{ "rorl    $dst, $shift" %}
9302   opcode(0xD3, 0x1); /* D3 /1 */
9303   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9304   ins_pipe(ialu_reg_reg);
9305 %}
9306 // end of ROR expand
9307 
9308 // Rotate Right by one
9309 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9310 %{
9311   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9312 
9313   expand %{
9314     rorI_rReg_imm1(dst, cr);
9315   %}
9316 %}
9317 
9318 // Rotate Right by 8-bit immediate
9319 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9320 %{
9321   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9322   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9323 
9324   expand %{
9325     rorI_rReg_imm8(dst, rshift, cr);
9326   %}
9327 %}
9328 
9329 // Rotate Right by variable
9330 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9331 %{
9332   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9333 
9334   expand %{
9335     rorI_rReg_CL(dst, shift, cr);
9336   %}
9337 %}
9338 
9339 // Rotate Right by variable
9340 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9341 %{
9342   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9343 
9344   expand %{
9345     rorI_rReg_CL(dst, shift, cr);
9346   %}
9347 %}
9348 
9349 // for long rotate
9350 // ROL expand
9351 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9352   effect(USE_DEF dst, KILL cr);
9353 
9354   format %{ "rolq    $dst" %}
9355   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9356   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9357   ins_pipe(ialu_reg);
9358 %}
9359 
9360 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9361   effect(USE_DEF dst, USE shift, KILL cr);
9362 
9363   format %{ "rolq    $dst, $shift" %}
9364   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9365   ins_encode( reg_opc_imm_wide(dst, shift) );
9366   ins_pipe(ialu_reg);
9367 %}
9368 
9369 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9370 %{
9371   effect(USE_DEF dst, USE shift, KILL cr);
9372 
9373   format %{ "rolq    $dst, $shift" %}
9374   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9375   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9376   ins_pipe(ialu_reg_reg);
9377 %}
9378 // end of ROL expand
9379 
9380 // Rotate Left by one
9381 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9382 %{
9383   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9384 
9385   expand %{
9386     rolL_rReg_imm1(dst, cr);
9387   %}
9388 %}
9389 
9390 // Rotate Left by 8-bit immediate
9391 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9392 %{
9393   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9394   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9395 
9396   expand %{
9397     rolL_rReg_imm8(dst, lshift, cr);
9398   %}
9399 %}
9400 
9401 // Rotate Left by variable
9402 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9403 %{
9404   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9405 
9406   expand %{
9407     rolL_rReg_CL(dst, shift, cr);
9408   %}
9409 %}
9410 
9411 // Rotate Left by variable
9412 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9413 %{
9414   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9415 
9416   expand %{
9417     rolL_rReg_CL(dst, shift, cr);
9418   %}
9419 %}
9420 
9421 // ROR expand
9422 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9423 %{
9424   effect(USE_DEF dst, KILL cr);
9425 
9426   format %{ "rorq    $dst" %}
9427   opcode(0xD1, 0x1); /* D1 /1 */
9428   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9429   ins_pipe(ialu_reg);
9430 %}
9431 
9432 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9433 %{
9434   effect(USE_DEF dst, USE shift, KILL cr);
9435 
9436   format %{ "rorq    $dst, $shift" %}
9437   opcode(0xC1, 0x1); /* C1 /1 ib */
9438   ins_encode(reg_opc_imm_wide(dst, shift));
9439   ins_pipe(ialu_reg);
9440 %}
9441 
9442 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9443 %{
9444   effect(USE_DEF dst, USE shift, KILL cr);
9445 
9446   format %{ "rorq    $dst, $shift" %}
9447   opcode(0xD3, 0x1); /* D3 /1 */
9448   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9449   ins_pipe(ialu_reg_reg);
9450 %}
9451 // end of ROR expand
9452 
9453 // Rotate Right by one
9454 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9455 %{
9456   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9457 
9458   expand %{
9459     rorL_rReg_imm1(dst, cr);
9460   %}
9461 %}
9462 
9463 // Rotate Right by 8-bit immediate
9464 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9465 %{
9466   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9467   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9468 
9469   expand %{
9470     rorL_rReg_imm8(dst, rshift, cr);
9471   %}
9472 %}
9473 
9474 // Rotate Right by variable
9475 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9476 %{
9477   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9478 
9479   expand %{
9480     rorL_rReg_CL(dst, shift, cr);
9481   %}
9482 %}
9483 
9484 // Rotate Right by variable
9485 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9486 %{
9487   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9488 
9489   expand %{
9490     rorL_rReg_CL(dst, shift, cr);
9491   %}
9492 %}
9493 
9494 // Logical Instructions
9495 
9496 // Integer Logical Instructions
9497 
9498 // And Instructions
9499 // And Register with Register
9500 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9501 %{
9502   match(Set dst (AndI dst src));
9503   effect(KILL cr);
9504 
9505   format %{ "andl    $dst, $src\t# int" %}
9506   opcode(0x23);
9507   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9508   ins_pipe(ialu_reg_reg);
9509 %}
9510 
9511 // And Register with Immediate 255
9512 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9513 %{
9514   match(Set dst (AndI dst src));
9515 
9516   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9517   opcode(0x0F, 0xB6);
9518   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9519   ins_pipe(ialu_reg);
9520 %}
9521 
9522 // And Register with Immediate 255 and promote to long
9523 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9524 %{
9525   match(Set dst (ConvI2L (AndI src mask)));
9526 
9527   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9528   opcode(0x0F, 0xB6);
9529   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9530   ins_pipe(ialu_reg);
9531 %}
9532 
9533 // And Register with Immediate 65535
9534 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9535 %{
9536   match(Set dst (AndI dst src));
9537 
9538   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9539   opcode(0x0F, 0xB7);
9540   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9541   ins_pipe(ialu_reg);
9542 %}
9543 
9544 // And Register with Immediate 65535 and promote to long
9545 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9546 %{
9547   match(Set dst (ConvI2L (AndI src mask)));
9548 
9549   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9550   opcode(0x0F, 0xB7);
9551   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9552   ins_pipe(ialu_reg);
9553 %}
9554 
9555 // And Register with Immediate
9556 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9557 %{
9558   match(Set dst (AndI dst src));
9559   effect(KILL cr);
9560 
9561   format %{ "andl    $dst, $src\t# int" %}
9562   opcode(0x81, 0x04); /* Opcode 81 /4 */
9563   ins_encode(OpcSErm(dst, src), Con8or32(src));
9564   ins_pipe(ialu_reg);
9565 %}
9566 
9567 // And Register with Memory
9568 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9569 %{
9570   match(Set dst (AndI dst (LoadI src)));
9571   effect(KILL cr);
9572 
9573   ins_cost(125);
9574   format %{ "andl    $dst, $src\t# int" %}
9575   opcode(0x23);
9576   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9577   ins_pipe(ialu_reg_mem);
9578 %}
9579 
9580 // And Memory with Register
9581 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9582 %{
9583   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
9584   effect(KILL cr);
9585 
9586   ins_cost(150);
9587   format %{ "andb    $dst, $src\t# byte" %}
9588   opcode(0x20);
9589   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9590   ins_pipe(ialu_mem_reg);
9591 %}
9592 
9593 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9594 %{
9595   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9596   effect(KILL cr);
9597 
9598   ins_cost(150);
9599   format %{ "andl    $dst, $src\t# int" %}
9600   opcode(0x21); /* Opcode 21 /r */
9601   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9602   ins_pipe(ialu_mem_reg);
9603 %}
9604 
9605 // And Memory with Immediate
9606 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9607 %{
9608   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9609   effect(KILL cr);
9610 
9611   ins_cost(125);
9612   format %{ "andl    $dst, $src\t# int" %}
9613   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9614   ins_encode(REX_mem(dst), OpcSE(src),
9615              RM_opc_mem(secondary, dst), Con8or32(src));
9616   ins_pipe(ialu_mem_imm);
9617 %}
9618 
9619 // BMI1 instructions
9620 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9621   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9622   predicate(UseBMI1Instructions);
9623   effect(KILL cr);
9624 
9625   ins_cost(125);
9626   format %{ "andnl  $dst, $src1, $src2" %}
9627 
9628   ins_encode %{
9629     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9630   %}
9631   ins_pipe(ialu_reg_mem);
9632 %}
9633 
9634 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9635   match(Set dst (AndI (XorI src1 minus_1) src2));
9636   predicate(UseBMI1Instructions);
9637   effect(KILL cr);
9638 
9639   format %{ "andnl  $dst, $src1, $src2" %}
9640 
9641   ins_encode %{
9642     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9643   %}
9644   ins_pipe(ialu_reg);
9645 %}
9646 
9647 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9648   match(Set dst (AndI (SubI imm_zero src) src));
9649   predicate(UseBMI1Instructions);
9650   effect(KILL cr);
9651 
9652   format %{ "blsil  $dst, $src" %}
9653 
9654   ins_encode %{
9655     __ blsil($dst$$Register, $src$$Register);
9656   %}
9657   ins_pipe(ialu_reg);
9658 %}
9659 
9660 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9661   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9662   predicate(UseBMI1Instructions);
9663   effect(KILL cr);
9664 
9665   ins_cost(125);
9666   format %{ "blsil  $dst, $src" %}
9667 
9668   ins_encode %{
9669     __ blsil($dst$$Register, $src$$Address);
9670   %}
9671   ins_pipe(ialu_reg_mem);
9672 %}
9673 
9674 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9675 %{
9676   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9677   predicate(UseBMI1Instructions);
9678   effect(KILL cr);
9679 
9680   ins_cost(125);
9681   format %{ "blsmskl $dst, $src" %}
9682 
9683   ins_encode %{
9684     __ blsmskl($dst$$Register, $src$$Address);
9685   %}
9686   ins_pipe(ialu_reg_mem);
9687 %}
9688 
9689 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9690 %{
9691   match(Set dst (XorI (AddI src minus_1) src));
9692   predicate(UseBMI1Instructions);
9693   effect(KILL cr);
9694 
9695   format %{ "blsmskl $dst, $src" %}
9696 
9697   ins_encode %{
9698     __ blsmskl($dst$$Register, $src$$Register);
9699   %}
9700 
9701   ins_pipe(ialu_reg);
9702 %}
9703 
9704 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9705 %{
9706   match(Set dst (AndI (AddI src minus_1) src) );
9707   predicate(UseBMI1Instructions);
9708   effect(KILL cr);
9709 
9710   format %{ "blsrl  $dst, $src" %}
9711 
9712   ins_encode %{
9713     __ blsrl($dst$$Register, $src$$Register);
9714   %}
9715 
9716   ins_pipe(ialu_reg_mem);
9717 %}
9718 
9719 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9720 %{
9721   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9722   predicate(UseBMI1Instructions);
9723   effect(KILL cr);
9724 
9725   ins_cost(125);
9726   format %{ "blsrl  $dst, $src" %}
9727 
9728   ins_encode %{
9729     __ blsrl($dst$$Register, $src$$Address);
9730   %}
9731 
9732   ins_pipe(ialu_reg);
9733 %}
9734 
9735 // Or Instructions
9736 // Or Register with Register
9737 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9738 %{
9739   match(Set dst (OrI dst src));
9740   effect(KILL cr);
9741 
9742   format %{ "orl     $dst, $src\t# int" %}
9743   opcode(0x0B);
9744   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9745   ins_pipe(ialu_reg_reg);
9746 %}
9747 
9748 // Or Register with Immediate
9749 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9750 %{
9751   match(Set dst (OrI dst src));
9752   effect(KILL cr);
9753 
9754   format %{ "orl     $dst, $src\t# int" %}
9755   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9756   ins_encode(OpcSErm(dst, src), Con8or32(src));
9757   ins_pipe(ialu_reg);
9758 %}
9759 
9760 // Or Register with Memory
9761 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9762 %{
9763   match(Set dst (OrI dst (LoadI src)));
9764   effect(KILL cr);
9765 
9766   ins_cost(125);
9767   format %{ "orl     $dst, $src\t# int" %}
9768   opcode(0x0B);
9769   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9770   ins_pipe(ialu_reg_mem);
9771 %}
9772 
9773 // Or Memory with Register
9774 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9775 %{
9776   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
9777   effect(KILL cr);
9778 
9779   ins_cost(150);
9780   format %{ "orb    $dst, $src\t# byte" %}
9781   opcode(0x08);
9782   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9783   ins_pipe(ialu_mem_reg);
9784 %}
9785 
9786 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9787 %{
9788   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9789   effect(KILL cr);
9790 
9791   ins_cost(150);
9792   format %{ "orl     $dst, $src\t# int" %}
9793   opcode(0x09); /* Opcode 09 /r */
9794   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9795   ins_pipe(ialu_mem_reg);
9796 %}
9797 
9798 // Or Memory with Immediate
9799 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9800 %{
9801   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9802   effect(KILL cr);
9803 
9804   ins_cost(125);
9805   format %{ "orl     $dst, $src\t# int" %}
9806   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9807   ins_encode(REX_mem(dst), OpcSE(src),
9808              RM_opc_mem(secondary, dst), Con8or32(src));
9809   ins_pipe(ialu_mem_imm);
9810 %}
9811 
9812 // Xor Instructions
9813 // Xor Register with Register
9814 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9815 %{
9816   match(Set dst (XorI dst src));
9817   effect(KILL cr);
9818 
9819   format %{ "xorl    $dst, $src\t# int" %}
9820   opcode(0x33);
9821   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9822   ins_pipe(ialu_reg_reg);
9823 %}
9824 
9825 // Xor Register with Immediate -1
9826 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9827   match(Set dst (XorI dst imm));
9828 
9829   format %{ "not    $dst" %}
9830   ins_encode %{
9831      __ notl($dst$$Register);
9832   %}
9833   ins_pipe(ialu_reg);
9834 %}
9835 
9836 // Xor Register with Immediate
9837 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9838 %{
9839   match(Set dst (XorI dst src));
9840   effect(KILL cr);
9841 
9842   format %{ "xorl    $dst, $src\t# int" %}
9843   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9844   ins_encode(OpcSErm(dst, src), Con8or32(src));
9845   ins_pipe(ialu_reg);
9846 %}
9847 
9848 // Xor Register with Memory
9849 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9850 %{
9851   match(Set dst (XorI dst (LoadI src)));
9852   effect(KILL cr);
9853 
9854   ins_cost(125);
9855   format %{ "xorl    $dst, $src\t# int" %}
9856   opcode(0x33);
9857   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9858   ins_pipe(ialu_reg_mem);
9859 %}
9860 
9861 // Xor Memory with Register
9862 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9863 %{
9864   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
9865   effect(KILL cr);
9866 
9867   ins_cost(150);
9868   format %{ "xorb    $dst, $src\t# byte" %}
9869   opcode(0x30);
9870   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9871   ins_pipe(ialu_mem_reg);
9872 %}
9873 
9874 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9875 %{
9876   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9877   effect(KILL cr);
9878 
9879   ins_cost(150);
9880   format %{ "xorl    $dst, $src\t# int" %}
9881   opcode(0x31); /* Opcode 31 /r */
9882   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9883   ins_pipe(ialu_mem_reg);
9884 %}
9885 
9886 // Xor Memory with Immediate
9887 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9888 %{
9889   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9890   effect(KILL cr);
9891 
9892   ins_cost(125);
9893   format %{ "xorl    $dst, $src\t# int" %}
9894   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9895   ins_encode(REX_mem(dst), OpcSE(src),
9896              RM_opc_mem(secondary, dst), Con8or32(src));
9897   ins_pipe(ialu_mem_imm);
9898 %}
9899 
9900 
9901 // Long Logical Instructions
9902 
9903 // And Instructions
9904 // And Register with Register
9905 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9906 %{
9907   match(Set dst (AndL dst src));
9908   effect(KILL cr);
9909 
9910   format %{ "andq    $dst, $src\t# long" %}
9911   opcode(0x23);
9912   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9913   ins_pipe(ialu_reg_reg);
9914 %}
9915 
9916 // And Register with Immediate 255
9917 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9918 %{
9919   match(Set dst (AndL dst src));
9920 
9921   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9922   opcode(0x0F, 0xB6);
9923   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9924   ins_pipe(ialu_reg);
9925 %}
9926 
9927 // And Register with Immediate 65535
9928 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9929 %{
9930   match(Set dst (AndL dst src));
9931 
9932   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9933   opcode(0x0F, 0xB7);
9934   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9935   ins_pipe(ialu_reg);
9936 %}
9937 
9938 // And Register with Immediate
9939 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9940 %{
9941   match(Set dst (AndL dst src));
9942   effect(KILL cr);
9943 
9944   format %{ "andq    $dst, $src\t# long" %}
9945   opcode(0x81, 0x04); /* Opcode 81 /4 */
9946   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9947   ins_pipe(ialu_reg);
9948 %}
9949 
9950 // And Register with Memory
9951 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9952 %{
9953   match(Set dst (AndL dst (LoadL src)));
9954   effect(KILL cr);
9955 
9956   ins_cost(125);
9957   format %{ "andq    $dst, $src\t# long" %}
9958   opcode(0x23);
9959   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9960   ins_pipe(ialu_reg_mem);
9961 %}
9962 
9963 // And Memory with Register
9964 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9965 %{
9966   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9967   effect(KILL cr);
9968 
9969   ins_cost(150);
9970   format %{ "andq    $dst, $src\t# long" %}
9971   opcode(0x21); /* Opcode 21 /r */
9972   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9973   ins_pipe(ialu_mem_reg);
9974 %}
9975 
9976 // And Memory with Immediate
9977 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9978 %{
9979   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9980   effect(KILL cr);
9981 
9982   ins_cost(125);
9983   format %{ "andq    $dst, $src\t# long" %}
9984   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9985   ins_encode(REX_mem_wide(dst), OpcSE(src),
9986              RM_opc_mem(secondary, dst), Con8or32(src));
9987   ins_pipe(ialu_mem_imm);
9988 %}
9989 
9990 // BMI1 instructions
9991 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9992   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9993   predicate(UseBMI1Instructions);
9994   effect(KILL cr);
9995 
9996   ins_cost(125);
9997   format %{ "andnq  $dst, $src1, $src2" %}
9998 
9999   ins_encode %{
10000     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10001   %}
10002   ins_pipe(ialu_reg_mem);
10003 %}
10004 
10005 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10006   match(Set dst (AndL (XorL src1 minus_1) src2));
10007   predicate(UseBMI1Instructions);
10008   effect(KILL cr);
10009 
10010   format %{ "andnq  $dst, $src1, $src2" %}
10011 
10012   ins_encode %{
10013   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10014   %}
10015   ins_pipe(ialu_reg_mem);
10016 %}
10017 
10018 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10019   match(Set dst (AndL (SubL imm_zero src) src));
10020   predicate(UseBMI1Instructions);
10021   effect(KILL cr);
10022 
10023   format %{ "blsiq  $dst, $src" %}
10024 
10025   ins_encode %{
10026     __ blsiq($dst$$Register, $src$$Register);
10027   %}
10028   ins_pipe(ialu_reg);
10029 %}
10030 
10031 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10032   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10033   predicate(UseBMI1Instructions);
10034   effect(KILL cr);
10035 
10036   ins_cost(125);
10037   format %{ "blsiq  $dst, $src" %}
10038 
10039   ins_encode %{
10040     __ blsiq($dst$$Register, $src$$Address);
10041   %}
10042   ins_pipe(ialu_reg_mem);
10043 %}
10044 
10045 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10046 %{
10047   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10048   predicate(UseBMI1Instructions);
10049   effect(KILL cr);
10050 
10051   ins_cost(125);
10052   format %{ "blsmskq $dst, $src" %}
10053 
10054   ins_encode %{
10055     __ blsmskq($dst$$Register, $src$$Address);
10056   %}
10057   ins_pipe(ialu_reg_mem);
10058 %}
10059 
10060 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10061 %{
10062   match(Set dst (XorL (AddL src minus_1) src));
10063   predicate(UseBMI1Instructions);
10064   effect(KILL cr);
10065 
10066   format %{ "blsmskq $dst, $src" %}
10067 
10068   ins_encode %{
10069     __ blsmskq($dst$$Register, $src$$Register);
10070   %}
10071 
10072   ins_pipe(ialu_reg);
10073 %}
10074 
10075 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10076 %{
10077   match(Set dst (AndL (AddL src minus_1) src) );
10078   predicate(UseBMI1Instructions);
10079   effect(KILL cr);
10080 
10081   format %{ "blsrq  $dst, $src" %}
10082 
10083   ins_encode %{
10084     __ blsrq($dst$$Register, $src$$Register);
10085   %}
10086 
10087   ins_pipe(ialu_reg);
10088 %}
10089 
10090 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10091 %{
10092   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10093   predicate(UseBMI1Instructions);
10094   effect(KILL cr);
10095 
10096   ins_cost(125);
10097   format %{ "blsrq  $dst, $src" %}
10098 
10099   ins_encode %{
10100     __ blsrq($dst$$Register, $src$$Address);
10101   %}
10102 
10103   ins_pipe(ialu_reg);
10104 %}
10105 
10106 // Or Instructions
10107 // Or Register with Register
10108 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10109 %{
10110   match(Set dst (OrL dst src));
10111   effect(KILL cr);
10112 
10113   format %{ "orq     $dst, $src\t# long" %}
10114   opcode(0x0B);
10115   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10116   ins_pipe(ialu_reg_reg);
10117 %}
10118 
10119 // Use any_RegP to match R15 (TLS register) without spilling.
10120 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10121   match(Set dst (OrL dst (CastP2X src)));
10122   effect(KILL cr);
10123 
10124   format %{ "orq     $dst, $src\t# long" %}
10125   opcode(0x0B);
10126   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10127   ins_pipe(ialu_reg_reg);
10128 %}
10129 
10130 
10131 // Or Register with Immediate
10132 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10133 %{
10134   match(Set dst (OrL dst src));
10135   effect(KILL cr);
10136 
10137   format %{ "orq     $dst, $src\t# long" %}
10138   opcode(0x81, 0x01); /* Opcode 81 /1 id */
10139   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10140   ins_pipe(ialu_reg);
10141 %}
10142 
10143 // Or Register with Memory
10144 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10145 %{
10146   match(Set dst (OrL dst (LoadL src)));
10147   effect(KILL cr);
10148 
10149   ins_cost(125);
10150   format %{ "orq     $dst, $src\t# long" %}
10151   opcode(0x0B);
10152   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10153   ins_pipe(ialu_reg_mem);
10154 %}
10155 
10156 // Or Memory with Register
10157 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10158 %{
10159   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10160   effect(KILL cr);
10161 
10162   ins_cost(150);
10163   format %{ "orq     $dst, $src\t# long" %}
10164   opcode(0x09); /* Opcode 09 /r */
10165   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10166   ins_pipe(ialu_mem_reg);
10167 %}
10168 
10169 // Or Memory with Immediate
10170 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10171 %{
10172   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10173   effect(KILL cr);
10174 
10175   ins_cost(125);
10176   format %{ "orq     $dst, $src\t# long" %}
10177   opcode(0x81, 0x1); /* Opcode 81 /1 id */
10178   ins_encode(REX_mem_wide(dst), OpcSE(src),
10179              RM_opc_mem(secondary, dst), Con8or32(src));
10180   ins_pipe(ialu_mem_imm);
10181 %}
10182 
10183 // Xor Instructions
10184 // Xor Register with Register
10185 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10186 %{
10187   match(Set dst (XorL dst src));
10188   effect(KILL cr);
10189 
10190   format %{ "xorq    $dst, $src\t# long" %}
10191   opcode(0x33);
10192   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10193   ins_pipe(ialu_reg_reg);
10194 %}
10195 
10196 // Xor Register with Immediate -1
10197 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10198   match(Set dst (XorL dst imm));
10199 
10200   format %{ "notq   $dst" %}
10201   ins_encode %{
10202      __ notq($dst$$Register);
10203   %}
10204   ins_pipe(ialu_reg);
10205 %}
10206 
10207 // Xor Register with Immediate
10208 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10209 %{
10210   match(Set dst (XorL dst src));
10211   effect(KILL cr);
10212 
10213   format %{ "xorq    $dst, $src\t# long" %}
10214   opcode(0x81, 0x06); /* Opcode 81 /6 id */
10215   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10216   ins_pipe(ialu_reg);
10217 %}
10218 
10219 // Xor Register with Memory
10220 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10221 %{
10222   match(Set dst (XorL dst (LoadL src)));
10223   effect(KILL cr);
10224 
10225   ins_cost(125);
10226   format %{ "xorq    $dst, $src\t# long" %}
10227   opcode(0x33);
10228   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10229   ins_pipe(ialu_reg_mem);
10230 %}
10231 
10232 // Xor Memory with Register
10233 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10234 %{
10235   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10236   effect(KILL cr);
10237 
10238   ins_cost(150);
10239   format %{ "xorq    $dst, $src\t# long" %}
10240   opcode(0x31); /* Opcode 31 /r */
10241   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10242   ins_pipe(ialu_mem_reg);
10243 %}
10244 
10245 // Xor Memory with Immediate
10246 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10247 %{
10248   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10249   effect(KILL cr);
10250 
10251   ins_cost(125);
10252   format %{ "xorq    $dst, $src\t# long" %}
10253   opcode(0x81, 0x6); /* Opcode 81 /6 id */
10254   ins_encode(REX_mem_wide(dst), OpcSE(src),
10255              RM_opc_mem(secondary, dst), Con8or32(src));
10256   ins_pipe(ialu_mem_imm);
10257 %}
10258 
10259 // Convert Int to Boolean
10260 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10261 %{
10262   match(Set dst (Conv2B src));
10263   effect(KILL cr);
10264 
10265   format %{ "testl   $src, $src\t# ci2b\n\t"
10266             "setnz   $dst\n\t"
10267             "movzbl  $dst, $dst" %}
10268   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
10269              setNZ_reg(dst),
10270              REX_reg_breg(dst, dst), // movzbl
10271              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10272   ins_pipe(pipe_slow); // XXX
10273 %}
10274 
10275 // Convert Pointer to Boolean
10276 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10277 %{
10278   match(Set dst (Conv2B src));
10279   effect(KILL cr);
10280 
10281   format %{ "testq   $src, $src\t# cp2b\n\t"
10282             "setnz   $dst\n\t"
10283             "movzbl  $dst, $dst" %}
10284   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
10285              setNZ_reg(dst),
10286              REX_reg_breg(dst, dst), // movzbl
10287              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10288   ins_pipe(pipe_slow); // XXX
10289 %}
10290 
10291 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10292 %{
10293   match(Set dst (CmpLTMask p q));
10294   effect(KILL cr);
10295 
10296   ins_cost(400);
10297   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10298             "setlt   $dst\n\t"
10299             "movzbl  $dst, $dst\n\t"
10300             "negl    $dst" %}
10301   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
10302              setLT_reg(dst),
10303              REX_reg_breg(dst, dst), // movzbl
10304              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
10305              neg_reg(dst));
10306   ins_pipe(pipe_slow);
10307 %}
10308 
10309 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
10310 %{
10311   match(Set dst (CmpLTMask dst zero));
10312   effect(KILL cr);
10313 
10314   ins_cost(100);
10315   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10316   ins_encode %{
10317   __ sarl($dst$$Register, 31);
10318   %}
10319   ins_pipe(ialu_reg);
10320 %}
10321 
10322 /* Better to save a register than avoid a branch */
10323 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10324 %{
10325   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10326   effect(KILL cr);
10327   ins_cost(300);
10328   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10329             "jge     done\n\t"
10330             "addl    $p,$y\n"
10331             "done:   " %}
10332   ins_encode %{
10333     Register Rp = $p$$Register;
10334     Register Rq = $q$$Register;
10335     Register Ry = $y$$Register;
10336     Label done;
10337     __ subl(Rp, Rq);
10338     __ jccb(Assembler::greaterEqual, done);
10339     __ addl(Rp, Ry);
10340     __ bind(done);
10341   %}
10342   ins_pipe(pipe_cmplt);
10343 %}
10344 
10345 /* Better to save a register than avoid a branch */
10346 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10347 %{
10348   match(Set y (AndI (CmpLTMask p q) y));
10349   effect(KILL cr);
10350 
10351   ins_cost(300);
10352 
10353   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10354             "jlt     done\n\t"
10355             "xorl    $y, $y\n"
10356             "done:   " %}
10357   ins_encode %{
10358     Register Rp = $p$$Register;
10359     Register Rq = $q$$Register;
10360     Register Ry = $y$$Register;
10361     Label done;
10362     __ cmpl(Rp, Rq);
10363     __ jccb(Assembler::less, done);
10364     __ xorl(Ry, Ry);
10365     __ bind(done);
10366   %}
10367   ins_pipe(pipe_cmplt);
10368 %}
10369 
10370 
10371 //---------- FP Instructions------------------------------------------------
10372 
10373 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10374 %{
10375   match(Set cr (CmpF src1 src2));
10376 
10377   ins_cost(145);
10378   format %{ "ucomiss $src1, $src2\n\t"
10379             "jnp,s   exit\n\t"
10380             "pushfq\t# saw NaN, set CF\n\t"
10381             "andq    [rsp], #0xffffff2b\n\t"
10382             "popfq\n"
10383     "exit:" %}
10384   ins_encode %{
10385     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10386     emit_cmpfp_fixup(_masm);
10387   %}
10388   ins_pipe(pipe_slow);
10389 %}
10390 
10391 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10392   match(Set cr (CmpF src1 src2));
10393 
10394   ins_cost(100);
10395   format %{ "ucomiss $src1, $src2" %}
10396   ins_encode %{
10397     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10398   %}
10399   ins_pipe(pipe_slow);
10400 %}
10401 
10402 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10403 %{
10404   match(Set cr (CmpF src1 (LoadF src2)));
10405 
10406   ins_cost(145);
10407   format %{ "ucomiss $src1, $src2\n\t"
10408             "jnp,s   exit\n\t"
10409             "pushfq\t# saw NaN, set CF\n\t"
10410             "andq    [rsp], #0xffffff2b\n\t"
10411             "popfq\n"
10412     "exit:" %}
10413   ins_encode %{
10414     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10415     emit_cmpfp_fixup(_masm);
10416   %}
10417   ins_pipe(pipe_slow);
10418 %}
10419 
10420 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10421   match(Set cr (CmpF src1 (LoadF src2)));
10422 
10423   ins_cost(100);
10424   format %{ "ucomiss $src1, $src2" %}
10425   ins_encode %{
10426     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10427   %}
10428   ins_pipe(pipe_slow);
10429 %}
10430 
10431 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10432   match(Set cr (CmpF src con));
10433 
10434   ins_cost(145);
10435   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10436             "jnp,s   exit\n\t"
10437             "pushfq\t# saw NaN, set CF\n\t"
10438             "andq    [rsp], #0xffffff2b\n\t"
10439             "popfq\n"
10440     "exit:" %}
10441   ins_encode %{
10442     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10443     emit_cmpfp_fixup(_masm);
10444   %}
10445   ins_pipe(pipe_slow);
10446 %}
10447 
10448 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10449   match(Set cr (CmpF src con));
10450   ins_cost(100);
10451   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10452   ins_encode %{
10453     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10454   %}
10455   ins_pipe(pipe_slow);
10456 %}
10457 
10458 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10459 %{
10460   match(Set cr (CmpD src1 src2));
10461 
10462   ins_cost(145);
10463   format %{ "ucomisd $src1, $src2\n\t"
10464             "jnp,s   exit\n\t"
10465             "pushfq\t# saw NaN, set CF\n\t"
10466             "andq    [rsp], #0xffffff2b\n\t"
10467             "popfq\n"
10468     "exit:" %}
10469   ins_encode %{
10470     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10471     emit_cmpfp_fixup(_masm);
10472   %}
10473   ins_pipe(pipe_slow);
10474 %}
10475 
10476 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10477   match(Set cr (CmpD src1 src2));
10478 
10479   ins_cost(100);
10480   format %{ "ucomisd $src1, $src2 test" %}
10481   ins_encode %{
10482     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10483   %}
10484   ins_pipe(pipe_slow);
10485 %}
10486 
10487 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10488 %{
10489   match(Set cr (CmpD src1 (LoadD src2)));
10490 
10491   ins_cost(145);
10492   format %{ "ucomisd $src1, $src2\n\t"
10493             "jnp,s   exit\n\t"
10494             "pushfq\t# saw NaN, set CF\n\t"
10495             "andq    [rsp], #0xffffff2b\n\t"
10496             "popfq\n"
10497     "exit:" %}
10498   ins_encode %{
10499     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10500     emit_cmpfp_fixup(_masm);
10501   %}
10502   ins_pipe(pipe_slow);
10503 %}
10504 
10505 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10506   match(Set cr (CmpD src1 (LoadD src2)));
10507 
10508   ins_cost(100);
10509   format %{ "ucomisd $src1, $src2" %}
10510   ins_encode %{
10511     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10512   %}
10513   ins_pipe(pipe_slow);
10514 %}
10515 
10516 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10517   match(Set cr (CmpD src con));
10518 
10519   ins_cost(145);
10520   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10521             "jnp,s   exit\n\t"
10522             "pushfq\t# saw NaN, set CF\n\t"
10523             "andq    [rsp], #0xffffff2b\n\t"
10524             "popfq\n"
10525     "exit:" %}
10526   ins_encode %{
10527     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10528     emit_cmpfp_fixup(_masm);
10529   %}
10530   ins_pipe(pipe_slow);
10531 %}
10532 
10533 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10534   match(Set cr (CmpD src con));
10535   ins_cost(100);
10536   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10537   ins_encode %{
10538     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10539   %}
10540   ins_pipe(pipe_slow);
10541 %}
10542 
10543 // Compare into -1,0,1
10544 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10545 %{
10546   match(Set dst (CmpF3 src1 src2));
10547   effect(KILL cr);
10548 
10549   ins_cost(275);
10550   format %{ "ucomiss $src1, $src2\n\t"
10551             "movl    $dst, #-1\n\t"
10552             "jp,s    done\n\t"
10553             "jb,s    done\n\t"
10554             "setne   $dst\n\t"
10555             "movzbl  $dst, $dst\n"
10556     "done:" %}
10557   ins_encode %{
10558     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10559     emit_cmpfp3(_masm, $dst$$Register);
10560   %}
10561   ins_pipe(pipe_slow);
10562 %}
10563 
10564 // Compare into -1,0,1
10565 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10566 %{
10567   match(Set dst (CmpF3 src1 (LoadF src2)));
10568   effect(KILL cr);
10569 
10570   ins_cost(275);
10571   format %{ "ucomiss $src1, $src2\n\t"
10572             "movl    $dst, #-1\n\t"
10573             "jp,s    done\n\t"
10574             "jb,s    done\n\t"
10575             "setne   $dst\n\t"
10576             "movzbl  $dst, $dst\n"
10577     "done:" %}
10578   ins_encode %{
10579     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10580     emit_cmpfp3(_masm, $dst$$Register);
10581   %}
10582   ins_pipe(pipe_slow);
10583 %}
10584 
10585 // Compare into -1,0,1
10586 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10587   match(Set dst (CmpF3 src con));
10588   effect(KILL cr);
10589 
10590   ins_cost(275);
10591   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10592             "movl    $dst, #-1\n\t"
10593             "jp,s    done\n\t"
10594             "jb,s    done\n\t"
10595             "setne   $dst\n\t"
10596             "movzbl  $dst, $dst\n"
10597     "done:" %}
10598   ins_encode %{
10599     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10600     emit_cmpfp3(_masm, $dst$$Register);
10601   %}
10602   ins_pipe(pipe_slow);
10603 %}
10604 
10605 // Compare into -1,0,1
10606 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10607 %{
10608   match(Set dst (CmpD3 src1 src2));
10609   effect(KILL cr);
10610 
10611   ins_cost(275);
10612   format %{ "ucomisd $src1, $src2\n\t"
10613             "movl    $dst, #-1\n\t"
10614             "jp,s    done\n\t"
10615             "jb,s    done\n\t"
10616             "setne   $dst\n\t"
10617             "movzbl  $dst, $dst\n"
10618     "done:" %}
10619   ins_encode %{
10620     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10621     emit_cmpfp3(_masm, $dst$$Register);
10622   %}
10623   ins_pipe(pipe_slow);
10624 %}
10625 
10626 // Compare into -1,0,1
10627 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10628 %{
10629   match(Set dst (CmpD3 src1 (LoadD src2)));
10630   effect(KILL cr);
10631 
10632   ins_cost(275);
10633   format %{ "ucomisd $src1, $src2\n\t"
10634             "movl    $dst, #-1\n\t"
10635             "jp,s    done\n\t"
10636             "jb,s    done\n\t"
10637             "setne   $dst\n\t"
10638             "movzbl  $dst, $dst\n"
10639     "done:" %}
10640   ins_encode %{
10641     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10642     emit_cmpfp3(_masm, $dst$$Register);
10643   %}
10644   ins_pipe(pipe_slow);
10645 %}
10646 
10647 // Compare into -1,0,1
10648 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10649   match(Set dst (CmpD3 src con));
10650   effect(KILL cr);
10651 
10652   ins_cost(275);
10653   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10654             "movl    $dst, #-1\n\t"
10655             "jp,s    done\n\t"
10656             "jb,s    done\n\t"
10657             "setne   $dst\n\t"
10658             "movzbl  $dst, $dst\n"
10659     "done:" %}
10660   ins_encode %{
10661     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10662     emit_cmpfp3(_masm, $dst$$Register);
10663   %}
10664   ins_pipe(pipe_slow);
10665 %}
10666 
10667 //----------Arithmetic Conversion Instructions---------------------------------
10668 
10669 instruct roundFloat_nop(regF dst)
10670 %{
10671   match(Set dst (RoundFloat dst));
10672 
10673   ins_cost(0);
10674   ins_encode();
10675   ins_pipe(empty);
10676 %}
10677 
10678 instruct roundDouble_nop(regD dst)
10679 %{
10680   match(Set dst (RoundDouble dst));
10681 
10682   ins_cost(0);
10683   ins_encode();
10684   ins_pipe(empty);
10685 %}
10686 
10687 instruct convF2D_reg_reg(regD dst, regF src)
10688 %{
10689   match(Set dst (ConvF2D src));
10690 
10691   format %{ "cvtss2sd $dst, $src" %}
10692   ins_encode %{
10693     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10694   %}
10695   ins_pipe(pipe_slow); // XXX
10696 %}
10697 
10698 instruct convF2D_reg_mem(regD dst, memory src)
10699 %{
10700   match(Set dst (ConvF2D (LoadF src)));
10701 
10702   format %{ "cvtss2sd $dst, $src" %}
10703   ins_encode %{
10704     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10705   %}
10706   ins_pipe(pipe_slow); // XXX
10707 %}
10708 
10709 instruct convD2F_reg_reg(regF dst, regD src)
10710 %{
10711   match(Set dst (ConvD2F src));
10712 
10713   format %{ "cvtsd2ss $dst, $src" %}
10714   ins_encode %{
10715     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10716   %}
10717   ins_pipe(pipe_slow); // XXX
10718 %}
10719 
10720 instruct convD2F_reg_mem(regF dst, memory src)
10721 %{
10722   match(Set dst (ConvD2F (LoadD src)));
10723 
10724   format %{ "cvtsd2ss $dst, $src" %}
10725   ins_encode %{
10726     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10727   %}
10728   ins_pipe(pipe_slow); // XXX
10729 %}
10730 
10731 // XXX do mem variants
10732 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10733 %{
10734   match(Set dst (ConvF2I src));
10735   effect(KILL cr);
10736 
10737   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10738             "cmpl    $dst, #0x80000000\n\t"
10739             "jne,s   done\n\t"
10740             "subq    rsp, #8\n\t"
10741             "movss   [rsp], $src\n\t"
10742             "call    f2i_fixup\n\t"
10743             "popq    $dst\n"
10744     "done:   "%}
10745   ins_encode %{
10746     Label done;
10747     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10748     __ cmpl($dst$$Register, 0x80000000);
10749     __ jccb(Assembler::notEqual, done);
10750     __ subptr(rsp, 8);
10751     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10752     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10753     __ pop($dst$$Register);
10754     __ bind(done);
10755   %}
10756   ins_pipe(pipe_slow);
10757 %}
10758 
10759 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10760 %{
10761   match(Set dst (ConvF2L src));
10762   effect(KILL cr);
10763 
10764   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10765             "cmpq    $dst, [0x8000000000000000]\n\t"
10766             "jne,s   done\n\t"
10767             "subq    rsp, #8\n\t"
10768             "movss   [rsp], $src\n\t"
10769             "call    f2l_fixup\n\t"
10770             "popq    $dst\n"
10771     "done:   "%}
10772   ins_encode %{
10773     Label done;
10774     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10775     __ cmp64($dst$$Register,
10776              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10777     __ jccb(Assembler::notEqual, done);
10778     __ subptr(rsp, 8);
10779     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10780     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10781     __ pop($dst$$Register);
10782     __ bind(done);
10783   %}
10784   ins_pipe(pipe_slow);
10785 %}
10786 
10787 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10788 %{
10789   match(Set dst (ConvD2I src));
10790   effect(KILL cr);
10791 
10792   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10793             "cmpl    $dst, #0x80000000\n\t"
10794             "jne,s   done\n\t"
10795             "subq    rsp, #8\n\t"
10796             "movsd   [rsp], $src\n\t"
10797             "call    d2i_fixup\n\t"
10798             "popq    $dst\n"
10799     "done:   "%}
10800   ins_encode %{
10801     Label done;
10802     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10803     __ cmpl($dst$$Register, 0x80000000);
10804     __ jccb(Assembler::notEqual, done);
10805     __ subptr(rsp, 8);
10806     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10807     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10808     __ pop($dst$$Register);
10809     __ bind(done);
10810   %}
10811   ins_pipe(pipe_slow);
10812 %}
10813 
10814 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10815 %{
10816   match(Set dst (ConvD2L src));
10817   effect(KILL cr);
10818 
10819   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10820             "cmpq    $dst, [0x8000000000000000]\n\t"
10821             "jne,s   done\n\t"
10822             "subq    rsp, #8\n\t"
10823             "movsd   [rsp], $src\n\t"
10824             "call    d2l_fixup\n\t"
10825             "popq    $dst\n"
10826     "done:   "%}
10827   ins_encode %{
10828     Label done;
10829     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10830     __ cmp64($dst$$Register,
10831              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10832     __ jccb(Assembler::notEqual, done);
10833     __ subptr(rsp, 8);
10834     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10835     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10836     __ pop($dst$$Register);
10837     __ bind(done);
10838   %}
10839   ins_pipe(pipe_slow);
10840 %}
10841 
10842 instruct convI2F_reg_reg(regF dst, rRegI src)
10843 %{
10844   predicate(!UseXmmI2F);
10845   match(Set dst (ConvI2F src));
10846 
10847   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10848   ins_encode %{
10849     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10850   %}
10851   ins_pipe(pipe_slow); // XXX
10852 %}
10853 
10854 instruct convI2F_reg_mem(regF dst, memory src)
10855 %{
10856   match(Set dst (ConvI2F (LoadI src)));
10857 
10858   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10859   ins_encode %{
10860     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10861   %}
10862   ins_pipe(pipe_slow); // XXX
10863 %}
10864 
10865 instruct convI2D_reg_reg(regD dst, rRegI src)
10866 %{
10867   predicate(!UseXmmI2D);
10868   match(Set dst (ConvI2D src));
10869 
10870   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10871   ins_encode %{
10872     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10873   %}
10874   ins_pipe(pipe_slow); // XXX
10875 %}
10876 
10877 instruct convI2D_reg_mem(regD dst, memory src)
10878 %{
10879   match(Set dst (ConvI2D (LoadI src)));
10880 
10881   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10882   ins_encode %{
10883     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10884   %}
10885   ins_pipe(pipe_slow); // XXX
10886 %}
10887 
10888 instruct convXI2F_reg(regF dst, rRegI src)
10889 %{
10890   predicate(UseXmmI2F);
10891   match(Set dst (ConvI2F src));
10892 
10893   format %{ "movdl $dst, $src\n\t"
10894             "cvtdq2psl $dst, $dst\t# i2f" %}
10895   ins_encode %{
10896     __ movdl($dst$$XMMRegister, $src$$Register);
10897     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10898   %}
10899   ins_pipe(pipe_slow); // XXX
10900 %}
10901 
10902 instruct convXI2D_reg(regD dst, rRegI src)
10903 %{
10904   predicate(UseXmmI2D);
10905   match(Set dst (ConvI2D src));
10906 
10907   format %{ "movdl $dst, $src\n\t"
10908             "cvtdq2pdl $dst, $dst\t# i2d" %}
10909   ins_encode %{
10910     __ movdl($dst$$XMMRegister, $src$$Register);
10911     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10912   %}
10913   ins_pipe(pipe_slow); // XXX
10914 %}
10915 
10916 instruct convL2F_reg_reg(regF dst, rRegL src)
10917 %{
10918   match(Set dst (ConvL2F src));
10919 
10920   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10921   ins_encode %{
10922     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10923   %}
10924   ins_pipe(pipe_slow); // XXX
10925 %}
10926 
10927 instruct convL2F_reg_mem(regF dst, memory src)
10928 %{
10929   match(Set dst (ConvL2F (LoadL src)));
10930 
10931   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10932   ins_encode %{
10933     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10934   %}
10935   ins_pipe(pipe_slow); // XXX
10936 %}
10937 
10938 instruct convL2D_reg_reg(regD dst, rRegL src)
10939 %{
10940   match(Set dst (ConvL2D src));
10941 
10942   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10943   ins_encode %{
10944     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10945   %}
10946   ins_pipe(pipe_slow); // XXX
10947 %}
10948 
10949 instruct convL2D_reg_mem(regD dst, memory src)
10950 %{
10951   match(Set dst (ConvL2D (LoadL src)));
10952 
10953   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10954   ins_encode %{
10955     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10956   %}
10957   ins_pipe(pipe_slow); // XXX
10958 %}
10959 
10960 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10961 %{
10962   match(Set dst (ConvI2L src));
10963 
10964   ins_cost(125);
10965   format %{ "movslq  $dst, $src\t# i2l" %}
10966   ins_encode %{
10967     __ movslq($dst$$Register, $src$$Register);
10968   %}
10969   ins_pipe(ialu_reg_reg);
10970 %}
10971 
10972 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10973 // %{
10974 //   match(Set dst (ConvI2L src));
10975 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10976 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10977 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10978 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10979 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10980 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10981 
10982 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10983 //   ins_encode(enc_copy(dst, src));
10984 // //   opcode(0x63); // needs REX.W
10985 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10986 //   ins_pipe(ialu_reg_reg);
10987 // %}
10988 
10989 // Zero-extend convert int to long
10990 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10991 %{
10992   match(Set dst (AndL (ConvI2L src) mask));
10993 
10994   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10995   ins_encode %{
10996     if ($dst$$reg != $src$$reg) {
10997       __ movl($dst$$Register, $src$$Register);
10998     }
10999   %}
11000   ins_pipe(ialu_reg_reg);
11001 %}
11002 
11003 // Zero-extend convert int to long
11004 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11005 %{
11006   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11007 
11008   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11009   ins_encode %{
11010     __ movl($dst$$Register, $src$$Address);
11011   %}
11012   ins_pipe(ialu_reg_mem);
11013 %}
11014 
11015 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11016 %{
11017   match(Set dst (AndL src mask));
11018 
11019   format %{ "movl    $dst, $src\t# zero-extend long" %}
11020   ins_encode %{
11021     __ movl($dst$$Register, $src$$Register);
11022   %}
11023   ins_pipe(ialu_reg_reg);
11024 %}
11025 
11026 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11027 %{
11028   match(Set dst (ConvL2I src));
11029 
11030   format %{ "movl    $dst, $src\t# l2i" %}
11031   ins_encode %{
11032     __ movl($dst$$Register, $src$$Register);
11033   %}
11034   ins_pipe(ialu_reg_reg);
11035 %}
11036 
11037 
11038 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11039   match(Set dst (MoveF2I src));
11040   effect(DEF dst, USE src);
11041 
11042   ins_cost(125);
11043   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11044   ins_encode %{
11045     __ movl($dst$$Register, Address(rsp, $src$$disp));
11046   %}
11047   ins_pipe(ialu_reg_mem);
11048 %}
11049 
11050 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11051   match(Set dst (MoveI2F src));
11052   effect(DEF dst, USE src);
11053 
11054   ins_cost(125);
11055   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11056   ins_encode %{
11057     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11058   %}
11059   ins_pipe(pipe_slow);
11060 %}
11061 
11062 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11063   match(Set dst (MoveD2L src));
11064   effect(DEF dst, USE src);
11065 
11066   ins_cost(125);
11067   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11068   ins_encode %{
11069     __ movq($dst$$Register, Address(rsp, $src$$disp));
11070   %}
11071   ins_pipe(ialu_reg_mem);
11072 %}
11073 
11074 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11075   predicate(!UseXmmLoadAndClearUpper);
11076   match(Set dst (MoveL2D src));
11077   effect(DEF dst, USE src);
11078 
11079   ins_cost(125);
11080   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11081   ins_encode %{
11082     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11083   %}
11084   ins_pipe(pipe_slow);
11085 %}
11086 
11087 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11088   predicate(UseXmmLoadAndClearUpper);
11089   match(Set dst (MoveL2D src));
11090   effect(DEF dst, USE src);
11091 
11092   ins_cost(125);
11093   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11094   ins_encode %{
11095     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11096   %}
11097   ins_pipe(pipe_slow);
11098 %}
11099 
11100 
11101 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11102   match(Set dst (MoveF2I src));
11103   effect(DEF dst, USE src);
11104 
11105   ins_cost(95); // XXX
11106   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11107   ins_encode %{
11108     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11109   %}
11110   ins_pipe(pipe_slow);
11111 %}
11112 
11113 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11114   match(Set dst (MoveI2F src));
11115   effect(DEF dst, USE src);
11116 
11117   ins_cost(100);
11118   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11119   ins_encode %{
11120     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11121   %}
11122   ins_pipe( ialu_mem_reg );
11123 %}
11124 
11125 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11126   match(Set dst (MoveD2L src));
11127   effect(DEF dst, USE src);
11128 
11129   ins_cost(95); // XXX
11130   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11131   ins_encode %{
11132     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11133   %}
11134   ins_pipe(pipe_slow);
11135 %}
11136 
11137 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11138   match(Set dst (MoveL2D src));
11139   effect(DEF dst, USE src);
11140 
11141   ins_cost(100);
11142   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11143   ins_encode %{
11144     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11145   %}
11146   ins_pipe(ialu_mem_reg);
11147 %}
11148 
11149 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11150   match(Set dst (MoveF2I src));
11151   effect(DEF dst, USE src);
11152   ins_cost(85);
11153   format %{ "movd    $dst,$src\t# MoveF2I" %}
11154   ins_encode %{
11155     __ movdl($dst$$Register, $src$$XMMRegister);
11156   %}
11157   ins_pipe( pipe_slow );
11158 %}
11159 
11160 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11161   match(Set dst (MoveD2L src));
11162   effect(DEF dst, USE src);
11163   ins_cost(85);
11164   format %{ "movd    $dst,$src\t# MoveD2L" %}
11165   ins_encode %{
11166     __ movdq($dst$$Register, $src$$XMMRegister);
11167   %}
11168   ins_pipe( pipe_slow );
11169 %}
11170 
11171 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11172   match(Set dst (MoveI2F src));
11173   effect(DEF dst, USE src);
11174   ins_cost(100);
11175   format %{ "movd    $dst,$src\t# MoveI2F" %}
11176   ins_encode %{
11177     __ movdl($dst$$XMMRegister, $src$$Register);
11178   %}
11179   ins_pipe( pipe_slow );
11180 %}
11181 
11182 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11183   match(Set dst (MoveL2D src));
11184   effect(DEF dst, USE src);
11185   ins_cost(100);
11186   format %{ "movd    $dst,$src\t# MoveL2D" %}
11187   ins_encode %{
11188      __ movdq($dst$$XMMRegister, $src$$Register);
11189   %}
11190   ins_pipe( pipe_slow );
11191 %}
11192 
11193 
11194 // =======================================================================
11195 // fast clearing of an array
11196 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11197                   Universe dummy, rFlagsReg cr)
11198 %{
11199   predicate(!((ClearArrayNode*)n)->is_large());
11200   match(Set dummy (ClearArray cnt base));
11201   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11202 
11203   format %{ $$template
11204     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11205     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11206     $$emit$$"jg      LARGE\n\t"
11207     $$emit$$"dec     rcx\n\t"
11208     $$emit$$"js      DONE\t# Zero length\n\t"
11209     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11210     $$emit$$"dec     rcx\n\t"
11211     $$emit$$"jge     LOOP\n\t"
11212     $$emit$$"jmp     DONE\n\t"
11213     $$emit$$"# LARGE:\n\t"
11214     if (UseFastStosb) {
11215        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11216        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11217     } else if (UseXMMForObjInit) {
11218        $$emit$$"mov     rdi,rax\n\t"
11219        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11220        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11221        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11222        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11223        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11224        $$emit$$"add     0x40,rax\n\t"
11225        $$emit$$"# L_zero_64_bytes:\n\t"
11226        $$emit$$"sub     0x8,rcx\n\t"
11227        $$emit$$"jge     L_loop\n\t"
11228        $$emit$$"add     0x4,rcx\n\t"
11229        $$emit$$"jl      L_tail\n\t"
11230        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11231        $$emit$$"add     0x20,rax\n\t"
11232        $$emit$$"sub     0x4,rcx\n\t"
11233        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11234        $$emit$$"add     0x4,rcx\n\t"
11235        $$emit$$"jle     L_end\n\t"
11236        $$emit$$"dec     rcx\n\t"
11237        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11238        $$emit$$"vmovq   xmm0,(rax)\n\t"
11239        $$emit$$"add     0x8,rax\n\t"
11240        $$emit$$"dec     rcx\n\t"
11241        $$emit$$"jge     L_sloop\n\t"
11242        $$emit$$"# L_end:\n\t"
11243     } else {
11244        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11245     }
11246     $$emit$$"# DONE"
11247   %}
11248   ins_encode %{
11249     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11250                  $tmp$$XMMRegister, false);
11251   %}
11252   ins_pipe(pipe_slow);
11253 %}
11254 
11255 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11256                         Universe dummy, rFlagsReg cr)
11257 %{
11258   predicate(((ClearArrayNode*)n)->is_large());
11259   match(Set dummy (ClearArray cnt base));
11260   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11261 
11262   format %{ $$template
11263     if (UseFastStosb) {
11264        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11265        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11266        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11267     } else if (UseXMMForObjInit) {
11268        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11269        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11270        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11271        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11272        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11273        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11274        $$emit$$"add     0x40,rax\n\t"
11275        $$emit$$"# L_zero_64_bytes:\n\t"
11276        $$emit$$"sub     0x8,rcx\n\t"
11277        $$emit$$"jge     L_loop\n\t"
11278        $$emit$$"add     0x4,rcx\n\t"
11279        $$emit$$"jl      L_tail\n\t"
11280        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11281        $$emit$$"add     0x20,rax\n\t"
11282        $$emit$$"sub     0x4,rcx\n\t"
11283        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11284        $$emit$$"add     0x4,rcx\n\t"
11285        $$emit$$"jle     L_end\n\t"
11286        $$emit$$"dec     rcx\n\t"
11287        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11288        $$emit$$"vmovq   xmm0,(rax)\n\t"
11289        $$emit$$"add     0x8,rax\n\t"
11290        $$emit$$"dec     rcx\n\t"
11291        $$emit$$"jge     L_sloop\n\t"
11292        $$emit$$"# L_end:\n\t"
11293     } else {
11294        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11295        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11296     }
11297   %}
11298   ins_encode %{
11299     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11300                  $tmp$$XMMRegister, true);
11301   %}
11302   ins_pipe(pipe_slow);
11303 %}
11304 
11305 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11306                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11307 %{
11308   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11309   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11310   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11311 
11312   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11313   ins_encode %{
11314     __ string_compare($str1$$Register, $str2$$Register,
11315                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11316                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
11317   %}
11318   ins_pipe( pipe_slow );
11319 %}
11320 
11321 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11322                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11323 %{
11324   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11325   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11326   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11327 
11328   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11329   ins_encode %{
11330     __ string_compare($str1$$Register, $str2$$Register,
11331                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11332                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
11333   %}
11334   ins_pipe( pipe_slow );
11335 %}
11336 
11337 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11338                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11339 %{
11340   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11341   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11342   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11343 
11344   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11345   ins_encode %{
11346     __ string_compare($str1$$Register, $str2$$Register,
11347                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11348                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
11349   %}
11350   ins_pipe( pipe_slow );
11351 %}
11352 
11353 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11354                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11355 %{
11356   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11357   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11358   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11359 
11360   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11361   ins_encode %{
11362     __ string_compare($str2$$Register, $str1$$Register,
11363                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11364                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
11365   %}
11366   ins_pipe( pipe_slow );
11367 %}
11368 
11369 // fast search of substring with known size.
11370 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11371                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11372 %{
11373   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11374   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11375   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11376 
11377   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11378   ins_encode %{
11379     int icnt2 = (int)$int_cnt2$$constant;
11380     if (icnt2 >= 16) {
11381       // IndexOf for constant substrings with size >= 16 elements
11382       // which don't need to be loaded through stack.
11383       __ string_indexofC8($str1$$Register, $str2$$Register,
11384                           $cnt1$$Register, $cnt2$$Register,
11385                           icnt2, $result$$Register,
11386                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11387     } else {
11388       // Small strings are loaded through stack if they cross page boundary.
11389       __ string_indexof($str1$$Register, $str2$$Register,
11390                         $cnt1$$Register, $cnt2$$Register,
11391                         icnt2, $result$$Register,
11392                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11393     }
11394   %}
11395   ins_pipe( pipe_slow );
11396 %}
11397 
11398 // fast search of substring with known size.
11399 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11400                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11401 %{
11402   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11403   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11404   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11405 
11406   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11407   ins_encode %{
11408     int icnt2 = (int)$int_cnt2$$constant;
11409     if (icnt2 >= 8) {
11410       // IndexOf for constant substrings with size >= 8 elements
11411       // which don't need to be loaded through stack.
11412       __ string_indexofC8($str1$$Register, $str2$$Register,
11413                           $cnt1$$Register, $cnt2$$Register,
11414                           icnt2, $result$$Register,
11415                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11416     } else {
11417       // Small strings are loaded through stack if they cross page boundary.
11418       __ string_indexof($str1$$Register, $str2$$Register,
11419                         $cnt1$$Register, $cnt2$$Register,
11420                         icnt2, $result$$Register,
11421                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11422     }
11423   %}
11424   ins_pipe( pipe_slow );
11425 %}
11426 
11427 // fast search of substring with known size.
11428 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11429                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11430 %{
11431   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11432   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11433   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11434 
11435   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11436   ins_encode %{
11437     int icnt2 = (int)$int_cnt2$$constant;
11438     if (icnt2 >= 8) {
11439       // IndexOf for constant substrings with size >= 8 elements
11440       // which don't need to be loaded through stack.
11441       __ string_indexofC8($str1$$Register, $str2$$Register,
11442                           $cnt1$$Register, $cnt2$$Register,
11443                           icnt2, $result$$Register,
11444                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11445     } else {
11446       // Small strings are loaded through stack if they cross page boundary.
11447       __ string_indexof($str1$$Register, $str2$$Register,
11448                         $cnt1$$Register, $cnt2$$Register,
11449                         icnt2, $result$$Register,
11450                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11451     }
11452   %}
11453   ins_pipe( pipe_slow );
11454 %}
11455 
11456 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11457                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11458 %{
11459   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11460   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11461   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11462 
11463   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11464   ins_encode %{
11465     __ string_indexof($str1$$Register, $str2$$Register,
11466                       $cnt1$$Register, $cnt2$$Register,
11467                       (-1), $result$$Register,
11468                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11469   %}
11470   ins_pipe( pipe_slow );
11471 %}
11472 
11473 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11474                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11475 %{
11476   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11477   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11478   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11479 
11480   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11481   ins_encode %{
11482     __ string_indexof($str1$$Register, $str2$$Register,
11483                       $cnt1$$Register, $cnt2$$Register,
11484                       (-1), $result$$Register,
11485                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11486   %}
11487   ins_pipe( pipe_slow );
11488 %}
11489 
11490 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11491                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11492 %{
11493   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11494   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11495   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11496 
11497   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11498   ins_encode %{
11499     __ string_indexof($str1$$Register, $str2$$Register,
11500                       $cnt1$$Register, $cnt2$$Register,
11501                       (-1), $result$$Register,
11502                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11503   %}
11504   ins_pipe( pipe_slow );
11505 %}
11506 
11507 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11508                               rbx_RegI result, legVecS vec1, legVecS vec2, legVecS vec3, rcx_RegI tmp, rFlagsReg cr)
11509 %{
11510   predicate(UseSSE42Intrinsics);
11511   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11512   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11513   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11514   ins_encode %{
11515     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11516                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11517   %}
11518   ins_pipe( pipe_slow );
11519 %}
11520 
11521 // fast string equals
11522 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11523                        legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11524 %{
11525   match(Set result (StrEquals (Binary str1 str2) cnt));
11526   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11527 
11528   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11529   ins_encode %{
11530     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11531                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11532                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11533   %}
11534   ins_pipe( pipe_slow );
11535 %}
11536 
11537 // fast array equals
11538 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11539                        legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11540 %{
11541   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11542   match(Set result (AryEq ary1 ary2));
11543   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11544 
11545   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11546   ins_encode %{
11547     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11548                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11549                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11550   %}
11551   ins_pipe( pipe_slow );
11552 %}
11553 
11554 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11555                       legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11556 %{
11557   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11558   match(Set result (AryEq ary1 ary2));
11559   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11560 
11561   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11562   ins_encode %{
11563     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11564                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11565                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11566   %}
11567   ins_pipe( pipe_slow );
11568 %}
11569 
11570 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11571                       legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11572 %{
11573   match(Set result (HasNegatives ary1 len));
11574   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11575 
11576   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11577   ins_encode %{
11578     __ has_negatives($ary1$$Register, $len$$Register,
11579                      $result$$Register, $tmp3$$Register,
11580                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11581   %}
11582   ins_pipe( pipe_slow );
11583 %}
11584 
11585 // fast char[] to byte[] compression
11586 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11587                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11588   match(Set result (StrCompressedCopy src (Binary dst len)));
11589   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11590 
11591   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11592   ins_encode %{
11593     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11594                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11595                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11596   %}
11597   ins_pipe( pipe_slow );
11598 %}
11599 
11600 // fast byte[] to char[] inflation
11601 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11602                         legVecS tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11603   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11604   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11605 
11606   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11607   ins_encode %{
11608     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11609                           $tmp1$$XMMRegister, $tmp2$$Register);
11610   %}
11611   ins_pipe( pipe_slow );
11612 %}
11613 
11614 // encode char[] to byte[] in ISO_8859_1
11615 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11616                           legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11617                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11618   match(Set result (EncodeISOArray src (Binary dst len)));
11619   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11620 
11621   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11622   ins_encode %{
11623     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11624                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11625                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11626   %}
11627   ins_pipe( pipe_slow );
11628 %}
11629 
11630 instruct getFP(rRegL dst) %{
11631   match(Set dst (GetFP));
11632   effect(DEF dst);
11633   ins_cost(1);
11634  
11635   ins_encode %{
11636     // Remove wordSize for return addr which is already pushed.
11637     int framesize = Compile::current()->frame_size_in_bytes() - wordSize;
11638     Address base(rsp, framesize);
11639     __ lea($dst$$Register, base);
11640   %}
11641   ins_pipe(ialu_reg_reg_long);
11642 %}
11643 
11644 //----------Overflow Math Instructions-----------------------------------------
11645 
11646 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11647 %{
11648   match(Set cr (OverflowAddI op1 op2));
11649   effect(DEF cr, USE_KILL op1, USE op2);
11650 
11651   format %{ "addl    $op1, $op2\t# overflow check int" %}
11652 
11653   ins_encode %{
11654     __ addl($op1$$Register, $op2$$Register);
11655   %}
11656   ins_pipe(ialu_reg_reg);
11657 %}
11658 
11659 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11660 %{
11661   match(Set cr (OverflowAddI op1 op2));
11662   effect(DEF cr, USE_KILL op1, USE op2);
11663 
11664   format %{ "addl    $op1, $op2\t# overflow check int" %}
11665 
11666   ins_encode %{
11667     __ addl($op1$$Register, $op2$$constant);
11668   %}
11669   ins_pipe(ialu_reg_reg);
11670 %}
11671 
11672 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11673 %{
11674   match(Set cr (OverflowAddL op1 op2));
11675   effect(DEF cr, USE_KILL op1, USE op2);
11676 
11677   format %{ "addq    $op1, $op2\t# overflow check long" %}
11678   ins_encode %{
11679     __ addq($op1$$Register, $op2$$Register);
11680   %}
11681   ins_pipe(ialu_reg_reg);
11682 %}
11683 
11684 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11685 %{
11686   match(Set cr (OverflowAddL op1 op2));
11687   effect(DEF cr, USE_KILL op1, USE op2);
11688 
11689   format %{ "addq    $op1, $op2\t# overflow check long" %}
11690   ins_encode %{
11691     __ addq($op1$$Register, $op2$$constant);
11692   %}
11693   ins_pipe(ialu_reg_reg);
11694 %}
11695 
11696 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11697 %{
11698   match(Set cr (OverflowSubI op1 op2));
11699 
11700   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11701   ins_encode %{
11702     __ cmpl($op1$$Register, $op2$$Register);
11703   %}
11704   ins_pipe(ialu_reg_reg);
11705 %}
11706 
11707 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11708 %{
11709   match(Set cr (OverflowSubI op1 op2));
11710 
11711   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11712   ins_encode %{
11713     __ cmpl($op1$$Register, $op2$$constant);
11714   %}
11715   ins_pipe(ialu_reg_reg);
11716 %}
11717 
11718 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11719 %{
11720   match(Set cr (OverflowSubL op1 op2));
11721 
11722   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11723   ins_encode %{
11724     __ cmpq($op1$$Register, $op2$$Register);
11725   %}
11726   ins_pipe(ialu_reg_reg);
11727 %}
11728 
11729 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11730 %{
11731   match(Set cr (OverflowSubL op1 op2));
11732 
11733   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11734   ins_encode %{
11735     __ cmpq($op1$$Register, $op2$$constant);
11736   %}
11737   ins_pipe(ialu_reg_reg);
11738 %}
11739 
11740 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11741 %{
11742   match(Set cr (OverflowSubI zero op2));
11743   effect(DEF cr, USE_KILL op2);
11744 
11745   format %{ "negl    $op2\t# overflow check int" %}
11746   ins_encode %{
11747     __ negl($op2$$Register);
11748   %}
11749   ins_pipe(ialu_reg_reg);
11750 %}
11751 
11752 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11753 %{
11754   match(Set cr (OverflowSubL zero op2));
11755   effect(DEF cr, USE_KILL op2);
11756 
11757   format %{ "negq    $op2\t# overflow check long" %}
11758   ins_encode %{
11759     __ negq($op2$$Register);
11760   %}
11761   ins_pipe(ialu_reg_reg);
11762 %}
11763 
11764 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11765 %{
11766   match(Set cr (OverflowMulI op1 op2));
11767   effect(DEF cr, USE_KILL op1, USE op2);
11768 
11769   format %{ "imull    $op1, $op2\t# overflow check int" %}
11770   ins_encode %{
11771     __ imull($op1$$Register, $op2$$Register);
11772   %}
11773   ins_pipe(ialu_reg_reg_alu0);
11774 %}
11775 
11776 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11777 %{
11778   match(Set cr (OverflowMulI op1 op2));
11779   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11780 
11781   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11782   ins_encode %{
11783     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11784   %}
11785   ins_pipe(ialu_reg_reg_alu0);
11786 %}
11787 
11788 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11789 %{
11790   match(Set cr (OverflowMulL op1 op2));
11791   effect(DEF cr, USE_KILL op1, USE op2);
11792 
11793   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11794   ins_encode %{
11795     __ imulq($op1$$Register, $op2$$Register);
11796   %}
11797   ins_pipe(ialu_reg_reg_alu0);
11798 %}
11799 
11800 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11801 %{
11802   match(Set cr (OverflowMulL op1 op2));
11803   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11804 
11805   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11806   ins_encode %{
11807     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11808   %}
11809   ins_pipe(ialu_reg_reg_alu0);
11810 %}
11811 
11812 
11813 //----------Control Flow Instructions------------------------------------------
11814 // Signed compare Instructions
11815 
11816 // XXX more variants!!
11817 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11818 %{
11819   match(Set cr (CmpI op1 op2));
11820   effect(DEF cr, USE op1, USE op2);
11821 
11822   format %{ "cmpl    $op1, $op2" %}
11823   opcode(0x3B);  /* Opcode 3B /r */
11824   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11825   ins_pipe(ialu_cr_reg_reg);
11826 %}
11827 
11828 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11829 %{
11830   match(Set cr (CmpI op1 op2));
11831 
11832   format %{ "cmpl    $op1, $op2" %}
11833   opcode(0x81, 0x07); /* Opcode 81 /7 */
11834   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11835   ins_pipe(ialu_cr_reg_imm);
11836 %}
11837 
11838 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11839 %{
11840   match(Set cr (CmpI op1 (LoadI op2)));
11841 
11842   ins_cost(500); // XXX
11843   format %{ "cmpl    $op1, $op2" %}
11844   opcode(0x3B); /* Opcode 3B /r */
11845   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11846   ins_pipe(ialu_cr_reg_mem);
11847 %}
11848 
11849 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11850 %{
11851   match(Set cr (CmpI src zero));
11852 
11853   format %{ "testl   $src, $src" %}
11854   opcode(0x85);
11855   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11856   ins_pipe(ialu_cr_reg_imm);
11857 %}
11858 
11859 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11860 %{
11861   match(Set cr (CmpI (AndI src con) zero));
11862 
11863   format %{ "testl   $src, $con" %}
11864   opcode(0xF7, 0x00);
11865   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11866   ins_pipe(ialu_cr_reg_imm);
11867 %}
11868 
11869 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11870 %{
11871   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11872 
11873   format %{ "testl   $src, $mem" %}
11874   opcode(0x85);
11875   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11876   ins_pipe(ialu_cr_reg_mem);
11877 %}
11878 
11879 // Unsigned compare Instructions; really, same as signed except they
11880 // produce an rFlagsRegU instead of rFlagsReg.
11881 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11882 %{
11883   match(Set cr (CmpU op1 op2));
11884 
11885   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11886   opcode(0x3B); /* Opcode 3B /r */
11887   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11888   ins_pipe(ialu_cr_reg_reg);
11889 %}
11890 
11891 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11892 %{
11893   match(Set cr (CmpU op1 op2));
11894 
11895   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11896   opcode(0x81,0x07); /* Opcode 81 /7 */
11897   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11898   ins_pipe(ialu_cr_reg_imm);
11899 %}
11900 
11901 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11902 %{
11903   match(Set cr (CmpU op1 (LoadI op2)));
11904 
11905   ins_cost(500); // XXX
11906   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11907   opcode(0x3B); /* Opcode 3B /r */
11908   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11909   ins_pipe(ialu_cr_reg_mem);
11910 %}
11911 
11912 // // // Cisc-spilled version of cmpU_rReg
11913 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11914 // //%{
11915 // //  match(Set cr (CmpU (LoadI op1) op2));
11916 // //
11917 // //  format %{ "CMPu   $op1,$op2" %}
11918 // //  ins_cost(500);
11919 // //  opcode(0x39);  /* Opcode 39 /r */
11920 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11921 // //%}
11922 
11923 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11924 %{
11925   match(Set cr (CmpU src zero));
11926 
11927   format %{ "testl   $src, $src\t# unsigned" %}
11928   opcode(0x85);
11929   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11930   ins_pipe(ialu_cr_reg_imm);
11931 %}
11932 
11933 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11934 %{
11935   match(Set cr (CmpP op1 op2));
11936 
11937   format %{ "cmpq    $op1, $op2\t# ptr" %}
11938   opcode(0x3B); /* Opcode 3B /r */
11939   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11940   ins_pipe(ialu_cr_reg_reg);
11941 %}
11942 
11943 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11944 %{
11945   match(Set cr (CmpP op1 (LoadP op2)));
11946 
11947   ins_cost(500); // XXX
11948   format %{ "cmpq    $op1, $op2\t# ptr" %}
11949   opcode(0x3B); /* Opcode 3B /r */
11950   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11951   ins_pipe(ialu_cr_reg_mem);
11952 %}
11953 
11954 // // // Cisc-spilled version of cmpP_rReg
11955 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11956 // //%{
11957 // //  match(Set cr (CmpP (LoadP op1) op2));
11958 // //
11959 // //  format %{ "CMPu   $op1,$op2" %}
11960 // //  ins_cost(500);
11961 // //  opcode(0x39);  /* Opcode 39 /r */
11962 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11963 // //%}
11964 
11965 // XXX this is generalized by compP_rReg_mem???
11966 // Compare raw pointer (used in out-of-heap check).
11967 // Only works because non-oop pointers must be raw pointers
11968 // and raw pointers have no anti-dependencies.
11969 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11970 %{
11971   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11972   match(Set cr (CmpP op1 (LoadP op2)));
11973 
11974   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11975   opcode(0x3B); /* Opcode 3B /r */
11976   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11977   ins_pipe(ialu_cr_reg_mem);
11978 %}
11979 
11980 // This will generate a signed flags result. This should be OK since
11981 // any compare to a zero should be eq/neq.
11982 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11983 %{
11984   match(Set cr (CmpP src zero));
11985 
11986   format %{ "testq   $src, $src\t# ptr" %}
11987   opcode(0x85);
11988   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11989   ins_pipe(ialu_cr_reg_imm);
11990 %}
11991 
11992 // This will generate a signed flags result. This should be OK since
11993 // any compare to a zero should be eq/neq.
11994 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11995 %{
11996   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
11997   match(Set cr (CmpP (LoadP op) zero));
11998 
11999   ins_cost(500); // XXX
12000   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12001   opcode(0xF7); /* Opcode F7 /0 */
12002   ins_encode(REX_mem_wide(op),
12003              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
12004   ins_pipe(ialu_cr_reg_imm);
12005 %}
12006 
12007 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12008 %{
12009   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
12010   match(Set cr (CmpP (LoadP mem) zero));
12011 
12012   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12013   ins_encode %{
12014     __ cmpq(r12, $mem$$Address);
12015   %}
12016   ins_pipe(ialu_cr_reg_mem);
12017 %}
12018 
12019 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12020 %{
12021   match(Set cr (CmpN op1 op2));
12022 
12023   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12024   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12025   ins_pipe(ialu_cr_reg_reg);
12026 %}
12027 
12028 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12029 %{
12030   match(Set cr (CmpN src (LoadN mem)));
12031 
12032   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12033   ins_encode %{
12034     __ cmpl($src$$Register, $mem$$Address);
12035   %}
12036   ins_pipe(ialu_cr_reg_mem);
12037 %}
12038 
12039 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12040   match(Set cr (CmpN op1 op2));
12041 
12042   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12043   ins_encode %{
12044     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12045   %}
12046   ins_pipe(ialu_cr_reg_imm);
12047 %}
12048 
12049 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12050 %{
12051   match(Set cr (CmpN src (LoadN mem)));
12052 
12053   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12054   ins_encode %{
12055     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12056   %}
12057   ins_pipe(ialu_cr_reg_mem);
12058 %}
12059 
12060 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12061   match(Set cr (CmpN op1 op2));
12062 
12063   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12064   ins_encode %{
12065     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12066   %}
12067   ins_pipe(ialu_cr_reg_imm);
12068 %}
12069 
12070 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12071 %{
12072   match(Set cr (CmpN src (LoadNKlass mem)));
12073 
12074   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12075   ins_encode %{
12076     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12077   %}
12078   ins_pipe(ialu_cr_reg_mem);
12079 %}
12080 
12081 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12082   match(Set cr (CmpN src zero));
12083 
12084   format %{ "testl   $src, $src\t# compressed ptr" %}
12085   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12086   ins_pipe(ialu_cr_reg_imm);
12087 %}
12088 
12089 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12090 %{
12091   predicate(CompressedOops::base() != NULL);
12092   match(Set cr (CmpN (LoadN mem) zero));
12093 
12094   ins_cost(500); // XXX
12095   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12096   ins_encode %{
12097     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12098   %}
12099   ins_pipe(ialu_cr_reg_mem);
12100 %}
12101 
12102 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12103 %{
12104   predicate(CompressedOops::base() == NULL && (CompressedKlassPointers::base() == NULL));
12105   match(Set cr (CmpN (LoadN mem) zero));
12106 
12107   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12108   ins_encode %{
12109     __ cmpl(r12, $mem$$Address);
12110   %}
12111   ins_pipe(ialu_cr_reg_mem);
12112 %}
12113 
12114 // Yanked all unsigned pointer compare operations.
12115 // Pointer compares are done with CmpP which is already unsigned.
12116 
12117 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12118 %{
12119   match(Set cr (CmpL op1 op2));
12120 
12121   format %{ "cmpq    $op1, $op2" %}
12122   opcode(0x3B);  /* Opcode 3B /r */
12123   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12124   ins_pipe(ialu_cr_reg_reg);
12125 %}
12126 
12127 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12128 %{
12129   match(Set cr (CmpL op1 op2));
12130 
12131   format %{ "cmpq    $op1, $op2" %}
12132   opcode(0x81, 0x07); /* Opcode 81 /7 */
12133   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12134   ins_pipe(ialu_cr_reg_imm);
12135 %}
12136 
12137 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12138 %{
12139   match(Set cr (CmpL op1 (LoadL op2)));
12140 
12141   format %{ "cmpq    $op1, $op2" %}
12142   opcode(0x3B); /* Opcode 3B /r */
12143   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12144   ins_pipe(ialu_cr_reg_mem);
12145 %}
12146 
12147 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12148 %{
12149   match(Set cr (CmpL src zero));
12150 
12151   format %{ "testq   $src, $src" %}
12152   opcode(0x85);
12153   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12154   ins_pipe(ialu_cr_reg_imm);
12155 %}
12156 
12157 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12158 %{
12159   match(Set cr (CmpL (AndL src con) zero));
12160 
12161   format %{ "testq   $src, $con\t# long" %}
12162   opcode(0xF7, 0x00);
12163   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
12164   ins_pipe(ialu_cr_reg_imm);
12165 %}
12166 
12167 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12168 %{
12169   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12170 
12171   format %{ "testq   $src, $mem" %}
12172   opcode(0x85);
12173   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12174   ins_pipe(ialu_cr_reg_mem);
12175 %}
12176 
12177 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12178 %{
12179   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12180 
12181   format %{ "testq   $src, $mem" %}
12182   opcode(0x85);
12183   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12184   ins_pipe(ialu_cr_reg_mem);
12185 %}
12186 
12187 // Manifest a CmpL result in an integer register.  Very painful.
12188 // This is the test to avoid.
12189 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12190 %{
12191   match(Set dst (CmpL3 src1 src2));
12192   effect(KILL flags);
12193 
12194   ins_cost(275); // XXX
12195   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12196             "movl    $dst, -1\n\t"
12197             "jl,s    done\n\t"
12198             "setne   $dst\n\t"
12199             "movzbl  $dst, $dst\n\t"
12200     "done:" %}
12201   ins_encode(cmpl3_flag(src1, src2, dst));
12202   ins_pipe(pipe_slow);
12203 %}
12204 
12205 // Unsigned long compare Instructions; really, same as signed long except they
12206 // produce an rFlagsRegU instead of rFlagsReg.
12207 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12208 %{
12209   match(Set cr (CmpUL op1 op2));
12210 
12211   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12212   opcode(0x3B);  /* Opcode 3B /r */
12213   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12214   ins_pipe(ialu_cr_reg_reg);
12215 %}
12216 
12217 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12218 %{
12219   match(Set cr (CmpUL op1 op2));
12220 
12221   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12222   opcode(0x81, 0x07); /* Opcode 81 /7 */
12223   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12224   ins_pipe(ialu_cr_reg_imm);
12225 %}
12226 
12227 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12228 %{
12229   match(Set cr (CmpUL op1 (LoadL op2)));
12230 
12231   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12232   opcode(0x3B); /* Opcode 3B /r */
12233   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12234   ins_pipe(ialu_cr_reg_mem);
12235 %}
12236 
12237 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12238 %{
12239   match(Set cr (CmpUL src zero));
12240 
12241   format %{ "testq   $src, $src\t# unsigned" %}
12242   opcode(0x85);
12243   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12244   ins_pipe(ialu_cr_reg_imm);
12245 %}
12246 
12247 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12248 %{
12249   match(Set cr (CmpI (LoadB mem) imm));
12250 
12251   ins_cost(125);
12252   format %{ "cmpb    $mem, $imm" %}
12253   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12254   ins_pipe(ialu_cr_reg_mem);
12255 %}
12256 
12257 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU8 imm, immI0 zero)
12258 %{
12259   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12260 
12261   ins_cost(125);
12262   format %{ "testb   $mem, $imm\t# ubyte" %}
12263   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12264   ins_pipe(ialu_cr_reg_mem);
12265 %}
12266 
12267 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
12268 %{
12269   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12270 
12271   ins_cost(125);
12272   format %{ "testb   $mem, $imm\t# byte" %}
12273   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12274   ins_pipe(ialu_cr_reg_mem);
12275 %}
12276 
12277 //----------Max and Min--------------------------------------------------------
12278 // Min Instructions
12279 
12280 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12281 %{
12282   effect(USE_DEF dst, USE src, USE cr);
12283 
12284   format %{ "cmovlgt $dst, $src\t# min" %}
12285   opcode(0x0F, 0x4F);
12286   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12287   ins_pipe(pipe_cmov_reg);
12288 %}
12289 
12290 
12291 instruct minI_rReg(rRegI dst, rRegI src)
12292 %{
12293   match(Set dst (MinI dst src));
12294 
12295   ins_cost(200);
12296   expand %{
12297     rFlagsReg cr;
12298     compI_rReg(cr, dst, src);
12299     cmovI_reg_g(dst, src, cr);
12300   %}
12301 %}
12302 
12303 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12304 %{
12305   effect(USE_DEF dst, USE src, USE cr);
12306 
12307   format %{ "cmovllt $dst, $src\t# max" %}
12308   opcode(0x0F, 0x4C);
12309   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12310   ins_pipe(pipe_cmov_reg);
12311 %}
12312 
12313 
12314 instruct maxI_rReg(rRegI dst, rRegI src)
12315 %{
12316   match(Set dst (MaxI dst src));
12317 
12318   ins_cost(200);
12319   expand %{
12320     rFlagsReg cr;
12321     compI_rReg(cr, dst, src);
12322     cmovI_reg_l(dst, src, cr);
12323   %}
12324 %}
12325 
12326 // ============================================================================
12327 // Branch Instructions
12328 
12329 // Jump Direct - Label defines a relative address from JMP+1
12330 instruct jmpDir(label labl)
12331 %{
12332   match(Goto);
12333   effect(USE labl);
12334 
12335   ins_cost(300);
12336   format %{ "jmp     $labl" %}
12337   size(5);
12338   ins_encode %{
12339     Label* L = $labl$$label;
12340     __ jmp(*L, false); // Always long jump
12341   %}
12342   ins_pipe(pipe_jmp);
12343 %}
12344 
12345 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12346 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12347 %{
12348   match(If cop cr);
12349   effect(USE labl);
12350 
12351   ins_cost(300);
12352   format %{ "j$cop     $labl" %}
12353   size(6);
12354   ins_encode %{
12355     Label* L = $labl$$label;
12356     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12357   %}
12358   ins_pipe(pipe_jcc);
12359 %}
12360 
12361 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12362 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12363 %{
12364   predicate(!n->has_vector_mask_set());
12365   match(CountedLoopEnd cop cr);
12366   effect(USE labl);
12367 
12368   ins_cost(300);
12369   format %{ "j$cop     $labl\t# loop end" %}
12370   size(6);
12371   ins_encode %{
12372     Label* L = $labl$$label;
12373     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12374   %}
12375   ins_pipe(pipe_jcc);
12376 %}
12377 
12378 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12379 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12380   predicate(!n->has_vector_mask_set());
12381   match(CountedLoopEnd cop cmp);
12382   effect(USE labl);
12383 
12384   ins_cost(300);
12385   format %{ "j$cop,u   $labl\t# loop end" %}
12386   size(6);
12387   ins_encode %{
12388     Label* L = $labl$$label;
12389     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12390   %}
12391   ins_pipe(pipe_jcc);
12392 %}
12393 
12394 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12395   predicate(!n->has_vector_mask_set());
12396   match(CountedLoopEnd cop cmp);
12397   effect(USE labl);
12398 
12399   ins_cost(200);
12400   format %{ "j$cop,u   $labl\t# loop end" %}
12401   size(6);
12402   ins_encode %{
12403     Label* L = $labl$$label;
12404     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12405   %}
12406   ins_pipe(pipe_jcc);
12407 %}
12408 
12409 // mask version
12410 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12411 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12412 %{
12413   predicate(n->has_vector_mask_set());
12414   match(CountedLoopEnd cop cr);
12415   effect(USE labl);
12416 
12417   ins_cost(400);
12418   format %{ "j$cop     $labl\t# loop end\n\t"
12419             "restorevectmask \t# vector mask restore for loops" %}
12420   size(10);
12421   ins_encode %{
12422     Label* L = $labl$$label;
12423     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12424     __ restorevectmask();
12425   %}
12426   ins_pipe(pipe_jcc);
12427 %}
12428 
12429 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12430 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12431   predicate(n->has_vector_mask_set());
12432   match(CountedLoopEnd cop cmp);
12433   effect(USE labl);
12434 
12435   ins_cost(400);
12436   format %{ "j$cop,u   $labl\t# loop end\n\t"
12437             "restorevectmask \t# vector mask restore for loops" %}
12438   size(10);
12439   ins_encode %{
12440     Label* L = $labl$$label;
12441     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12442     __ restorevectmask();
12443   %}
12444   ins_pipe(pipe_jcc);
12445 %}
12446 
12447 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12448   predicate(n->has_vector_mask_set());
12449   match(CountedLoopEnd cop cmp);
12450   effect(USE labl);
12451 
12452   ins_cost(300);
12453   format %{ "j$cop,u   $labl\t# loop end\n\t"
12454             "restorevectmask \t# vector mask restore for loops" %}
12455   size(10);
12456   ins_encode %{
12457     Label* L = $labl$$label;
12458     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12459     __ restorevectmask();
12460   %}
12461   ins_pipe(pipe_jcc);
12462 %}
12463 
12464 // Jump Direct Conditional - using unsigned comparison
12465 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12466   match(If cop cmp);
12467   effect(USE labl);
12468 
12469   ins_cost(300);
12470   format %{ "j$cop,u   $labl" %}
12471   size(6);
12472   ins_encode %{
12473     Label* L = $labl$$label;
12474     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12475   %}
12476   ins_pipe(pipe_jcc);
12477 %}
12478 
12479 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12480   match(If cop cmp);
12481   effect(USE labl);
12482 
12483   ins_cost(200);
12484   format %{ "j$cop,u   $labl" %}
12485   size(6);
12486   ins_encode %{
12487     Label* L = $labl$$label;
12488     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12489   %}
12490   ins_pipe(pipe_jcc);
12491 %}
12492 
12493 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12494   match(If cop cmp);
12495   effect(USE labl);
12496 
12497   ins_cost(200);
12498   format %{ $$template
12499     if ($cop$$cmpcode == Assembler::notEqual) {
12500       $$emit$$"jp,u    $labl\n\t"
12501       $$emit$$"j$cop,u   $labl"
12502     } else {
12503       $$emit$$"jp,u    done\n\t"
12504       $$emit$$"j$cop,u   $labl\n\t"
12505       $$emit$$"done:"
12506     }
12507   %}
12508   ins_encode %{
12509     Label* l = $labl$$label;
12510     if ($cop$$cmpcode == Assembler::notEqual) {
12511       __ jcc(Assembler::parity, *l, false);
12512       __ jcc(Assembler::notEqual, *l, false);
12513     } else if ($cop$$cmpcode == Assembler::equal) {
12514       Label done;
12515       __ jccb(Assembler::parity, done);
12516       __ jcc(Assembler::equal, *l, false);
12517       __ bind(done);
12518     } else {
12519        ShouldNotReachHere();
12520     }
12521   %}
12522   ins_pipe(pipe_jcc);
12523 %}
12524 
12525 // ============================================================================
12526 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12527 // superklass array for an instance of the superklass.  Set a hidden
12528 // internal cache on a hit (cache is checked with exposed code in
12529 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12530 // encoding ALSO sets flags.
12531 
12532 instruct partialSubtypeCheck(rdi_RegP result,
12533                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12534                              rFlagsReg cr)
12535 %{
12536   match(Set result (PartialSubtypeCheck sub super));
12537   effect(KILL rcx, KILL cr);
12538 
12539   ins_cost(1100);  // slightly larger than the next version
12540   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12541             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12542             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12543             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12544             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12545             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12546             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12547     "miss:\t" %}
12548 
12549   opcode(0x1); // Force a XOR of RDI
12550   ins_encode(enc_PartialSubtypeCheck());
12551   ins_pipe(pipe_slow);
12552 %}
12553 
12554 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12555                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12556                                      immP0 zero,
12557                                      rdi_RegP result)
12558 %{
12559   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12560   effect(KILL rcx, KILL result);
12561 
12562   ins_cost(1000);
12563   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12564             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12565             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12566             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12567             "jne,s   miss\t\t# Missed: flags nz\n\t"
12568             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12569     "miss:\t" %}
12570 
12571   opcode(0x0); // No need to XOR RDI
12572   ins_encode(enc_PartialSubtypeCheck());
12573   ins_pipe(pipe_slow);
12574 %}
12575 
12576 // ============================================================================
12577 // Branch Instructions -- short offset versions
12578 //
12579 // These instructions are used to replace jumps of a long offset (the default
12580 // match) with jumps of a shorter offset.  These instructions are all tagged
12581 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12582 // match rules in general matching.  Instead, the ADLC generates a conversion
12583 // method in the MachNode which can be used to do in-place replacement of the
12584 // long variant with the shorter variant.  The compiler will determine if a
12585 // branch can be taken by the is_short_branch_offset() predicate in the machine
12586 // specific code section of the file.
12587 
12588 // Jump Direct - Label defines a relative address from JMP+1
12589 instruct jmpDir_short(label labl) %{
12590   match(Goto);
12591   effect(USE labl);
12592 
12593   ins_cost(300);
12594   format %{ "jmp,s   $labl" %}
12595   size(2);
12596   ins_encode %{
12597     Label* L = $labl$$label;
12598     __ jmpb(*L);
12599   %}
12600   ins_pipe(pipe_jmp);
12601   ins_short_branch(1);
12602 %}
12603 
12604 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12605 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12606   match(If cop cr);
12607   effect(USE labl);
12608 
12609   ins_cost(300);
12610   format %{ "j$cop,s   $labl" %}
12611   size(2);
12612   ins_encode %{
12613     Label* L = $labl$$label;
12614     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12615   %}
12616   ins_pipe(pipe_jcc);
12617   ins_short_branch(1);
12618 %}
12619 
12620 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12621 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12622   match(CountedLoopEnd cop cr);
12623   effect(USE labl);
12624 
12625   ins_cost(300);
12626   format %{ "j$cop,s   $labl\t# loop end" %}
12627   size(2);
12628   ins_encode %{
12629     Label* L = $labl$$label;
12630     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12631   %}
12632   ins_pipe(pipe_jcc);
12633   ins_short_branch(1);
12634 %}
12635 
12636 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12637 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12638   match(CountedLoopEnd cop cmp);
12639   effect(USE labl);
12640 
12641   ins_cost(300);
12642   format %{ "j$cop,us  $labl\t# loop end" %}
12643   size(2);
12644   ins_encode %{
12645     Label* L = $labl$$label;
12646     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12647   %}
12648   ins_pipe(pipe_jcc);
12649   ins_short_branch(1);
12650 %}
12651 
12652 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12653   match(CountedLoopEnd cop cmp);
12654   effect(USE labl);
12655 
12656   ins_cost(300);
12657   format %{ "j$cop,us  $labl\t# loop end" %}
12658   size(2);
12659   ins_encode %{
12660     Label* L = $labl$$label;
12661     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12662   %}
12663   ins_pipe(pipe_jcc);
12664   ins_short_branch(1);
12665 %}
12666 
12667 // Jump Direct Conditional - using unsigned comparison
12668 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12669   match(If cop cmp);
12670   effect(USE labl);
12671 
12672   ins_cost(300);
12673   format %{ "j$cop,us  $labl" %}
12674   size(2);
12675   ins_encode %{
12676     Label* L = $labl$$label;
12677     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12678   %}
12679   ins_pipe(pipe_jcc);
12680   ins_short_branch(1);
12681 %}
12682 
12683 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12684   match(If cop cmp);
12685   effect(USE labl);
12686 
12687   ins_cost(300);
12688   format %{ "j$cop,us  $labl" %}
12689   size(2);
12690   ins_encode %{
12691     Label* L = $labl$$label;
12692     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12693   %}
12694   ins_pipe(pipe_jcc);
12695   ins_short_branch(1);
12696 %}
12697 
12698 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12699   match(If cop cmp);
12700   effect(USE labl);
12701 
12702   ins_cost(300);
12703   format %{ $$template
12704     if ($cop$$cmpcode == Assembler::notEqual) {
12705       $$emit$$"jp,u,s  $labl\n\t"
12706       $$emit$$"j$cop,u,s  $labl"
12707     } else {
12708       $$emit$$"jp,u,s  done\n\t"
12709       $$emit$$"j$cop,u,s  $labl\n\t"
12710       $$emit$$"done:"
12711     }
12712   %}
12713   size(4);
12714   ins_encode %{
12715     Label* l = $labl$$label;
12716     if ($cop$$cmpcode == Assembler::notEqual) {
12717       __ jccb(Assembler::parity, *l);
12718       __ jccb(Assembler::notEqual, *l);
12719     } else if ($cop$$cmpcode == Assembler::equal) {
12720       Label done;
12721       __ jccb(Assembler::parity, done);
12722       __ jccb(Assembler::equal, *l);
12723       __ bind(done);
12724     } else {
12725        ShouldNotReachHere();
12726     }
12727   %}
12728   ins_pipe(pipe_jcc);
12729   ins_short_branch(1);
12730 %}
12731 
12732 // ============================================================================
12733 // inlined locking and unlocking
12734 
12735 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12736   predicate(Compile::current()->use_rtm());
12737   match(Set cr (FastLock object box));
12738   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12739   ins_cost(300);
12740   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12741   ins_encode %{
12742     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12743                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12744                  _counters, _rtm_counters, _stack_rtm_counters,
12745                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12746                  true, ra_->C->profile_rtm());
12747   %}
12748   ins_pipe(pipe_slow);
12749 %}
12750 
12751 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12752   predicate(!Compile::current()->use_rtm());
12753   match(Set cr (FastLock object box));
12754   effect(TEMP tmp, TEMP scr, USE_KILL box);
12755   ins_cost(300);
12756   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12757   ins_encode %{
12758     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12759                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12760   %}
12761   ins_pipe(pipe_slow);
12762 %}
12763 
12764 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12765   match(Set cr (FastUnlock object box));
12766   effect(TEMP tmp, USE_KILL box);
12767   ins_cost(300);
12768   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12769   ins_encode %{
12770     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12771   %}
12772   ins_pipe(pipe_slow);
12773 %}
12774 
12775 
12776 // ============================================================================
12777 // Safepoint Instructions
12778 instruct safePoint_poll(rFlagsReg cr)
12779 %{
12780   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12781   match(SafePoint);
12782   effect(KILL cr);
12783 
12784   format %{ "testl   rax, [rip + #offset_to_poll_page]\t"
12785             "# Safepoint: poll for GC" %}
12786   ins_cost(125);
12787   ins_encode %{
12788     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12789     __ testl(rax, addr);
12790   %}
12791   ins_pipe(ialu_reg_mem);
12792 %}
12793 
12794 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12795 %{
12796   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12797   match(SafePoint poll);
12798   effect(KILL cr, USE poll);
12799 
12800   format %{ "testl   rax, [$poll]\t"
12801             "# Safepoint: poll for GC" %}
12802   ins_cost(125);
12803   ins_encode %{
12804     __ relocate(relocInfo::poll_type);
12805     __ testl(rax, Address($poll$$Register, 0));
12806   %}
12807   ins_pipe(ialu_reg_mem);
12808 %}
12809 
12810 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12811 %{
12812   predicate(SafepointMechanism::uses_thread_local_poll());
12813   match(SafePoint poll);
12814   effect(KILL cr, USE poll);
12815 
12816   format %{ "testl   rax, [$poll]\t"
12817             "# Safepoint: poll for GC" %}
12818   ins_cost(125);
12819   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12820   ins_encode %{
12821     __ relocate(relocInfo::poll_type);
12822     address pre_pc = __ pc();
12823     __ testl(rax, Address($poll$$Register, 0));
12824     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12825   %}
12826   ins_pipe(ialu_reg_mem);
12827 %}
12828 
12829 // ============================================================================
12830 // Procedure Call/Return Instructions
12831 // Call Java Static Instruction
12832 // Note: If this code changes, the corresponding ret_addr_offset() and
12833 //       compute_padding() functions will have to be adjusted.
12834 instruct CallStaticJavaDirect(method meth) %{
12835   match(CallStaticJava);
12836   effect(USE meth);
12837 
12838   ins_cost(300);
12839   format %{ "call,static " %}
12840   opcode(0xE8); /* E8 cd */
12841   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12842   ins_pipe(pipe_slow);
12843   ins_alignment(4);
12844 %}
12845 
12846 // Call Java Dynamic Instruction
12847 // Note: If this code changes, the corresponding ret_addr_offset() and
12848 //       compute_padding() functions will have to be adjusted.
12849 instruct CallDynamicJavaDirect(method meth)
12850 %{
12851   match(CallDynamicJava);
12852   effect(USE meth);
12853 
12854   ins_cost(300);
12855   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12856             "call,dynamic " %}
12857   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12858   ins_pipe(pipe_slow);
12859   ins_alignment(4);
12860 %}
12861 
12862 // Call Runtime Instruction
12863 instruct CallRuntimeDirect(method meth)
12864 %{
12865   match(CallRuntime);
12866   effect(USE meth);
12867 
12868   ins_cost(300);
12869   format %{ "call,runtime " %}
12870   ins_encode(clear_avx, Java_To_Runtime(meth));
12871   ins_pipe(pipe_slow);
12872 %}
12873 
12874 // Call runtime without safepoint
12875 instruct CallLeafDirect(method meth)
12876 %{
12877   match(CallLeaf);
12878   effect(USE meth);
12879 
12880   ins_cost(300);
12881   format %{ "call_leaf,runtime " %}
12882   ins_encode(clear_avx, Java_To_Runtime(meth));
12883   ins_pipe(pipe_slow);
12884 %}
12885 
12886 // Call runtime without safepoint
12887 instruct CallLeafNoFPDirect(method meth)
12888 %{
12889   match(CallLeafNoFP);
12890   effect(USE meth);
12891 
12892   ins_cost(300);
12893   format %{ "call_leaf_nofp,runtime " %}
12894   ins_encode(clear_avx, Java_To_Runtime(meth));
12895   ins_pipe(pipe_slow);
12896 %}
12897 
12898 // Return Instruction
12899 // Remove the return address & jump to it.
12900 // Notice: We always emit a nop after a ret to make sure there is room
12901 // for safepoint patching
12902 instruct Ret()
12903 %{
12904   match(Return);
12905 
12906   format %{ "ret" %}
12907   opcode(0xC3);
12908   ins_encode(OpcP);
12909   ins_pipe(pipe_jmp);
12910 %}
12911 
12912 // Tail Call; Jump from runtime stub to Java code.
12913 // Also known as an 'interprocedural jump'.
12914 // Target of jump will eventually return to caller.
12915 // TailJump below removes the return address.
12916 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12917 %{
12918   match(TailCall jump_target method_oop);
12919 
12920   ins_cost(300);
12921   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12922   opcode(0xFF, 0x4); /* Opcode FF /4 */
12923   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12924   ins_pipe(pipe_jmp);
12925 %}
12926 
12927 // Tail Jump; remove the return address; jump to target.
12928 // TailCall above leaves the return address around.
12929 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12930 %{
12931   match(TailJump jump_target ex_oop);
12932 
12933   ins_cost(300);
12934   format %{ "popq    rdx\t# pop return address\n\t"
12935             "jmp     $jump_target" %}
12936   opcode(0xFF, 0x4); /* Opcode FF /4 */
12937   ins_encode(Opcode(0x5a), // popq rdx
12938              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12939   ins_pipe(pipe_jmp);
12940 %}
12941 
12942 // Create exception oop: created by stack-crawling runtime code.
12943 // Created exception is now available to this handler, and is setup
12944 // just prior to jumping to this handler.  No code emitted.
12945 instruct CreateException(rax_RegP ex_oop)
12946 %{
12947   match(Set ex_oop (CreateEx));
12948 
12949   size(0);
12950   // use the following format syntax
12951   format %{ "# exception oop is in rax; no code emitted" %}
12952   ins_encode();
12953   ins_pipe(empty);
12954 %}
12955 
12956 // Rethrow exception:
12957 // The exception oop will come in the first argument position.
12958 // Then JUMP (not call) to the rethrow stub code.
12959 instruct RethrowException()
12960 %{
12961   match(Rethrow);
12962 
12963   // use the following format syntax
12964   format %{ "jmp     rethrow_stub" %}
12965   ins_encode(enc_rethrow);
12966   ins_pipe(pipe_jmp);
12967 %}
12968 
12969 // ============================================================================
12970 // This name is KNOWN by the ADLC and cannot be changed.
12971 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12972 // for this guy.
12973 instruct tlsLoadP(r15_RegP dst) %{
12974   match(Set dst (ThreadLocal));
12975   effect(DEF dst);
12976 
12977   size(0);
12978   format %{ "# TLS is in R15" %}
12979   ins_encode( /*empty encoding*/ );
12980   ins_pipe(ialu_reg_reg);
12981 %}
12982 
12983 
12984 //----------PEEPHOLE RULES-----------------------------------------------------
12985 // These must follow all instruction definitions as they use the names
12986 // defined in the instructions definitions.
12987 //
12988 // peepmatch ( root_instr_name [preceding_instruction]* );
12989 //
12990 // peepconstraint %{
12991 // (instruction_number.operand_name relational_op instruction_number.operand_name
12992 //  [, ...] );
12993 // // instruction numbers are zero-based using left to right order in peepmatch
12994 //
12995 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12996 // // provide an instruction_number.operand_name for each operand that appears
12997 // // in the replacement instruction's match rule
12998 //
12999 // ---------VM FLAGS---------------------------------------------------------
13000 //
13001 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13002 //
13003 // Each peephole rule is given an identifying number starting with zero and
13004 // increasing by one in the order seen by the parser.  An individual peephole
13005 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13006 // on the command-line.
13007 //
13008 // ---------CURRENT LIMITATIONS----------------------------------------------
13009 //
13010 // Only match adjacent instructions in same basic block
13011 // Only equality constraints
13012 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13013 // Only one replacement instruction
13014 //
13015 // ---------EXAMPLE----------------------------------------------------------
13016 //
13017 // // pertinent parts of existing instructions in architecture description
13018 // instruct movI(rRegI dst, rRegI src)
13019 // %{
13020 //   match(Set dst (CopyI src));
13021 // %}
13022 //
13023 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
13024 // %{
13025 //   match(Set dst (AddI dst src));
13026 //   effect(KILL cr);
13027 // %}
13028 //
13029 // // Change (inc mov) to lea
13030 // peephole %{
13031 //   // increment preceeded by register-register move
13032 //   peepmatch ( incI_rReg movI );
13033 //   // require that the destination register of the increment
13034 //   // match the destination register of the move
13035 //   peepconstraint ( 0.dst == 1.dst );
13036 //   // construct a replacement instruction that sets
13037 //   // the destination to ( move's source register + one )
13038 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13039 // %}
13040 //
13041 
13042 // Implementation no longer uses movX instructions since
13043 // machine-independent system no longer uses CopyX nodes.
13044 //
13045 // peephole
13046 // %{
13047 //   peepmatch (incI_rReg movI);
13048 //   peepconstraint (0.dst == 1.dst);
13049 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13050 // %}
13051 
13052 // peephole
13053 // %{
13054 //   peepmatch (decI_rReg movI);
13055 //   peepconstraint (0.dst == 1.dst);
13056 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13057 // %}
13058 
13059 // peephole
13060 // %{
13061 //   peepmatch (addI_rReg_imm movI);
13062 //   peepconstraint (0.dst == 1.dst);
13063 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13064 // %}
13065 
13066 // peephole
13067 // %{
13068 //   peepmatch (incL_rReg movL);
13069 //   peepconstraint (0.dst == 1.dst);
13070 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13071 // %}
13072 
13073 // peephole
13074 // %{
13075 //   peepmatch (decL_rReg movL);
13076 //   peepconstraint (0.dst == 1.dst);
13077 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13078 // %}
13079 
13080 // peephole
13081 // %{
13082 //   peepmatch (addL_rReg_imm movL);
13083 //   peepconstraint (0.dst == 1.dst);
13084 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13085 // %}
13086 
13087 // peephole
13088 // %{
13089 //   peepmatch (addP_rReg_imm movP);
13090 //   peepconstraint (0.dst == 1.dst);
13091 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13092 // %}
13093 
13094 // // Change load of spilled value to only a spill
13095 // instruct storeI(memory mem, rRegI src)
13096 // %{
13097 //   match(Set mem (StoreI mem src));
13098 // %}
13099 //
13100 // instruct loadI(rRegI dst, memory mem)
13101 // %{
13102 //   match(Set dst (LoadI mem));
13103 // %}
13104 //
13105 
13106 peephole
13107 %{
13108   peepmatch (loadI storeI);
13109   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13110   peepreplace (storeI(1.mem 1.mem 1.src));
13111 %}
13112 
13113 peephole
13114 %{
13115   peepmatch (loadL storeL);
13116   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13117   peepreplace (storeL(1.mem 1.mem 1.src));
13118 %}
13119 
13120 //----------SMARTSPILL RULES---------------------------------------------------
13121 // These must follow all instruction definitions as they use the names
13122 // defined in the instructions definitions.