1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer/long registers
 173 reg_class all_reg(RAX, RAX_H,
 174                   RDX, RDX_H,
 175                   RBP, RBP_H,
 176                   RDI, RDI_H,
 177                   RSI, RSI_H,
 178                   RCX, RCX_H,
 179                   RBX, RBX_H,
 180                   RSP, RSP_H,
 181                   R8,  R8_H,
 182                   R9,  R9_H,
 183                   R10, R10_H,
 184                   R11, R11_H,
 185                   R12, R12_H,
 186                   R13, R13_H,
 187                   R14, R14_H,
 188                   R15, R15_H);
 189 
 190 // Class for all int registers
 191 reg_class all_int_reg(RAX
 192                       RDX,
 193                       RBP,
 194                       RDI,
 195                       RSI,
 196                       RCX,
 197                       RBX,
 198                       R8,
 199                       R9,
 200                       R10,
 201                       R11,
 202                       R12,
 203                       R13,
 204                       R14);
 205 
 206 // Class for all pointer registers
 207 reg_class any_reg %{
 208   return _ANY_REG_mask;
 209 %}
 210 
 211 // Class for all pointer registers (excluding RSP)
 212 reg_class ptr_reg %{
 213   return _PTR_REG_mask;
 214 %}
 215 
 216 // Class for all pointer registers (excluding RSP and RBP)
 217 reg_class ptr_reg_no_rbp %{
 218   return _PTR_REG_NO_RBP_mask;
 219 %}
 220 
 221 // Class for all pointer registers (excluding RAX and RSP)
 222 reg_class ptr_no_rax_reg %{
 223   return _PTR_NO_RAX_REG_mask;
 224 %}
 225 
 226 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 227 reg_class ptr_no_rax_rbx_reg %{
 228   return _PTR_NO_RAX_RBX_REG_mask;
 229 %}
 230 
 231 // Class for all long registers (excluding RSP)
 232 reg_class long_reg %{
 233   return _LONG_REG_mask;
 234 %}
 235 
 236 // Class for all long registers (excluding RAX, RDX and RSP)
 237 reg_class long_no_rax_rdx_reg %{
 238   return _LONG_NO_RAX_RDX_REG_mask;
 239 %}
 240 
 241 // Class for all long registers (excluding RCX and RSP)
 242 reg_class long_no_rcx_reg %{
 243   return _LONG_NO_RCX_REG_mask;
 244 %}
 245 
 246 // Class for all int registers (excluding RSP)
 247 reg_class int_reg %{
 248   return _INT_REG_mask;
 249 %}
 250 
 251 // Class for all int registers (excluding RAX, RDX, and RSP)
 252 reg_class int_no_rax_rdx_reg %{
 253   return _INT_NO_RAX_RDX_REG_mask;
 254 %}
 255 
 256 // Class for all int registers (excluding RCX and RSP)
 257 reg_class int_no_rcx_reg %{
 258   return _INT_NO_RCX_REG_mask;
 259 %}
 260 
 261 // Singleton class for RAX pointer register
 262 reg_class ptr_rax_reg(RAX, RAX_H);
 263 
 264 // Singleton class for RBX pointer register
 265 reg_class ptr_rbx_reg(RBX, RBX_H);
 266 
 267 // Singleton class for RSI pointer register
 268 reg_class ptr_rsi_reg(RSI, RSI_H);
 269 
 270 // Singleton class for RDI pointer register
 271 reg_class ptr_rdi_reg(RDI, RDI_H);
 272 
 273 // Singleton class for stack pointer
 274 reg_class ptr_rsp_reg(RSP, RSP_H);
 275 
 276 // Singleton class for TLS pointer
 277 reg_class ptr_r15_reg(R15, R15_H);
 278 
 279 // Singleton class for RAX long register
 280 reg_class long_rax_reg(RAX, RAX_H);
 281 
 282 // Singleton class for RCX long register
 283 reg_class long_rcx_reg(RCX, RCX_H);
 284 
 285 // Singleton class for RDX long register
 286 reg_class long_rdx_reg(RDX, RDX_H);
 287 
 288 // Singleton class for RAX int register
 289 reg_class int_rax_reg(RAX);
 290 
 291 // Singleton class for RBX int register
 292 reg_class int_rbx_reg(RBX);
 293 
 294 // Singleton class for RCX int register
 295 reg_class int_rcx_reg(RCX);
 296 
 297 // Singleton class for RCX int register
 298 reg_class int_rdx_reg(RDX);
 299 
 300 // Singleton class for RCX int register
 301 reg_class int_rdi_reg(RDI);
 302 
 303 // Singleton class for instruction pointer
 304 // reg_class ip_reg(RIP);
 305 
 306 %}
 307 
 308 //----------SOURCE BLOCK-------------------------------------------------------
 309 // This is a block of C++ code which provides values, functions, and
 310 // definitions necessary in the rest of the architecture description
 311 source_hpp %{
 312 
 313 extern RegMask _ANY_REG_mask;
 314 extern RegMask _PTR_REG_mask;
 315 extern RegMask _PTR_REG_NO_RBP_mask;
 316 extern RegMask _PTR_NO_RAX_REG_mask;
 317 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
 318 extern RegMask _LONG_REG_mask;
 319 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
 320 extern RegMask _LONG_NO_RCX_REG_mask;
 321 extern RegMask _INT_REG_mask;
 322 extern RegMask _INT_NO_RAX_RDX_REG_mask;
 323 extern RegMask _INT_NO_RCX_REG_mask;
 324 
 325 extern RegMask _STACK_OR_PTR_REG_mask;
 326 extern RegMask _STACK_OR_LONG_REG_mask;
 327 extern RegMask _STACK_OR_INT_REG_mask;
 328 
 329 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
 330 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
 331 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
 332 
 333 %}
 334 
 335 source %{
 336 #define   RELOC_IMM64    Assembler::imm_operand
 337 #define   RELOC_DISP32   Assembler::disp32_operand
 338 
 339 #define __ _masm.
 340 
 341 RegMask _ANY_REG_mask;
 342 RegMask _PTR_REG_mask;
 343 RegMask _PTR_REG_NO_RBP_mask;
 344 RegMask _PTR_NO_RAX_REG_mask;
 345 RegMask _PTR_NO_RAX_RBX_REG_mask;
 346 RegMask _LONG_REG_mask;
 347 RegMask _LONG_NO_RAX_RDX_REG_mask;
 348 RegMask _LONG_NO_RCX_REG_mask;
 349 RegMask _INT_REG_mask;
 350 RegMask _INT_NO_RAX_RDX_REG_mask;
 351 RegMask _INT_NO_RCX_REG_mask;
 352 RegMask _STACK_OR_PTR_REG_mask;
 353 RegMask _STACK_OR_LONG_REG_mask;
 354 RegMask _STACK_OR_INT_REG_mask;
 355 
 356 static bool need_r12_heapbase() {
 357   return UseCompressedOops || UseCompressedClassPointers;
 358 }
 359 
 360 void reg_mask_init() {
 361   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
 362   // We derive a number of subsets from it.
 363   _ANY_REG_mask = _ALL_REG_mask;
 364 
 365   if (PreserveFramePointer) {
 366     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 367     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 368   }
 369   if (need_r12_heapbase()) {
 370     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 371     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
 372   }
 373 
 374   _PTR_REG_mask = _ANY_REG_mask;
 375   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
 376   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
 377   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
 378   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
 379 
 380   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
 381   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 382 
 383   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
 384   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 385   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 386 
 387   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
 388   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 389   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 390 
 391   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
 392   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
 393   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
 394 
 395   _LONG_REG_mask = _PTR_REG_mask;
 396   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
 397   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 398 
 399   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
 400   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 401   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 402   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 403   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
 404 
 405   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
 406   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 407   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
 408 
 409   _INT_REG_mask = _ALL_INT_REG_mask;
 410   if (PreserveFramePointer) {
 411     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 412   }
 413   if (need_r12_heapbase()) {
 414     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 415   }
 416 
 417   _STACK_OR_INT_REG_mask = _INT_REG_mask;
 418   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 419 
 420   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
 421   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 422   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 423 
 424   _INT_NO_RCX_REG_mask = _INT_REG_mask;
 425   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 426 }
 427 
 428 static bool generate_vzeroupper(Compile* C) {
 429   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 430 }
 431 
 432 static int clear_avx_size() {
 433   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 434 }
 435 
 436 // !!!!! Special hack to get all types of calls to specify the byte offset
 437 //       from the start of the call to the point where the return address
 438 //       will point.
 439 int MachCallStaticJavaNode::ret_addr_offset()
 440 {
 441   int offset = 5; // 5 bytes from start of call to where return address points
 442   offset += clear_avx_size();
 443   return offset;
 444 }
 445 
 446 int MachCallDynamicJavaNode::ret_addr_offset()
 447 {
 448   int offset = 15; // 15 bytes from start of call to where return address points
 449   offset += clear_avx_size();
 450   return offset;
 451 }
 452 
 453 int MachCallRuntimeNode::ret_addr_offset() {
 454   int offset = 13; // movq r10,#addr; callq (r10)
 455   offset += clear_avx_size();
 456   return offset;
 457 }
 458 
 459 // Indicate if the safepoint node needs the polling page as an input,
 460 // it does if the polling page is more than disp32 away.
 461 bool SafePointNode::needs_polling_address_input()
 462 {
 463   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 464 }
 465 
 466 //
 467 // Compute padding required for nodes which need alignment
 468 //
 469 
 470 // The address of the call instruction needs to be 4-byte aligned to
 471 // ensure that it does not span a cache line so that it can be patched.
 472 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 473 {
 474   current_offset += clear_avx_size(); // skip vzeroupper
 475   current_offset += 1; // skip call opcode byte
 476   return align_up(current_offset, alignment_required()) - current_offset;
 477 }
 478 
 479 // The address of the call instruction needs to be 4-byte aligned to
 480 // ensure that it does not span a cache line so that it can be patched.
 481 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 482 {
 483   current_offset += clear_avx_size(); // skip vzeroupper
 484   current_offset += 11; // skip movq instruction + call opcode byte
 485   return align_up(current_offset, alignment_required()) - current_offset;
 486 }
 487 
 488 // EMIT_RM()
 489 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 490   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 491   cbuf.insts()->emit_int8(c);
 492 }
 493 
 494 // EMIT_CC()
 495 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 496   unsigned char c = (unsigned char) (f1 | f2);
 497   cbuf.insts()->emit_int8(c);
 498 }
 499 
 500 // EMIT_OPCODE()
 501 void emit_opcode(CodeBuffer &cbuf, int code) {
 502   cbuf.insts()->emit_int8((unsigned char) code);
 503 }
 504 
 505 // EMIT_OPCODE() w/ relocation information
 506 void emit_opcode(CodeBuffer &cbuf,
 507                  int code, relocInfo::relocType reloc, int offset, int format)
 508 {
 509   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 510   emit_opcode(cbuf, code);
 511 }
 512 
 513 // EMIT_D8()
 514 void emit_d8(CodeBuffer &cbuf, int d8) {
 515   cbuf.insts()->emit_int8((unsigned char) d8);
 516 }
 517 
 518 // EMIT_D16()
 519 void emit_d16(CodeBuffer &cbuf, int d16) {
 520   cbuf.insts()->emit_int16(d16);
 521 }
 522 
 523 // EMIT_D32()
 524 void emit_d32(CodeBuffer &cbuf, int d32) {
 525   cbuf.insts()->emit_int32(d32);
 526 }
 527 
 528 // EMIT_D64()
 529 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 530   cbuf.insts()->emit_int64(d64);
 531 }
 532 
 533 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 534 void emit_d32_reloc(CodeBuffer& cbuf,
 535                     int d32,
 536                     relocInfo::relocType reloc,
 537                     int format)
 538 {
 539   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 540   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 541   cbuf.insts()->emit_int32(d32);
 542 }
 543 
 544 // emit 32 bit value and construct relocation entry from RelocationHolder
 545 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 546 #ifdef ASSERT
 547   if (rspec.reloc()->type() == relocInfo::oop_type &&
 548       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 549     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 550     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
 551   }
 552 #endif
 553   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 554   cbuf.insts()->emit_int32(d32);
 555 }
 556 
 557 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 558   address next_ip = cbuf.insts_end() + 4;
 559   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 560                  external_word_Relocation::spec(addr),
 561                  RELOC_DISP32);
 562 }
 563 
 564 
 565 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 566 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 567   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 568   cbuf.insts()->emit_int64(d64);
 569 }
 570 
 571 // emit 64 bit value and construct relocation entry from RelocationHolder
 572 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 573 #ifdef ASSERT
 574   if (rspec.reloc()->type() == relocInfo::oop_type &&
 575       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 576     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 577     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
 578   }
 579 #endif
 580   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 581   cbuf.insts()->emit_int64(d64);
 582 }
 583 
 584 // Access stack slot for load or store
 585 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 586 {
 587   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 588   if (-0x80 <= disp && disp < 0x80) {
 589     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 590     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 591     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 592   } else {
 593     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 594     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 595     emit_d32(cbuf, disp);     // Displacement // R/M byte
 596   }
 597 }
 598 
 599    // rRegI ereg, memory mem) %{    // emit_reg_mem
 600 void encode_RegMem(CodeBuffer &cbuf,
 601                    int reg,
 602                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 603 {
 604   assert(disp_reloc == relocInfo::none, "cannot have disp");
 605   int regenc = reg & 7;
 606   int baseenc = base & 7;
 607   int indexenc = index & 7;
 608 
 609   // There is no index & no scale, use form without SIB byte
 610   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 611     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 612     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 613       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 614     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 615       // If 8-bit displacement, mode 0x1
 616       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 617       emit_d8(cbuf, disp);
 618     } else {
 619       // If 32-bit displacement
 620       if (base == -1) { // Special flag for absolute address
 621         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 622         if (disp_reloc != relocInfo::none) {
 623           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 624         } else {
 625           emit_d32(cbuf, disp);
 626         }
 627       } else {
 628         // Normal base + offset
 629         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 630         if (disp_reloc != relocInfo::none) {
 631           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 632         } else {
 633           emit_d32(cbuf, disp);
 634         }
 635       }
 636     }
 637   } else {
 638     // Else, encode with the SIB byte
 639     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 640     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 641       // If no displacement
 642       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 643       emit_rm(cbuf, scale, indexenc, baseenc);
 644     } else {
 645       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 646         // If 8-bit displacement, mode 0x1
 647         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 648         emit_rm(cbuf, scale, indexenc, baseenc);
 649         emit_d8(cbuf, disp);
 650       } else {
 651         // If 32-bit displacement
 652         if (base == 0x04 ) {
 653           emit_rm(cbuf, 0x2, regenc, 0x4);
 654           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 655         } else {
 656           emit_rm(cbuf, 0x2, regenc, 0x4);
 657           emit_rm(cbuf, scale, indexenc, baseenc); // *
 658         }
 659         if (disp_reloc != relocInfo::none) {
 660           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 661         } else {
 662           emit_d32(cbuf, disp);
 663         }
 664       }
 665     }
 666   }
 667 }
 668 
 669 // This could be in MacroAssembler but it's fairly C2 specific
 670 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 671   Label exit;
 672   __ jccb(Assembler::noParity, exit);
 673   __ pushf();
 674   //
 675   // comiss/ucomiss instructions set ZF,PF,CF flags and
 676   // zero OF,AF,SF for NaN values.
 677   // Fixup flags by zeroing ZF,PF so that compare of NaN
 678   // values returns 'less than' result (CF is set).
 679   // Leave the rest of flags unchanged.
 680   //
 681   //    7 6 5 4 3 2 1 0
 682   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 683   //    0 0 1 0 1 0 1 1   (0x2B)
 684   //
 685   __ andq(Address(rsp, 0), 0xffffff2b);
 686   __ popf();
 687   __ bind(exit);
 688 }
 689 
 690 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 691   Label done;
 692   __ movl(dst, -1);
 693   __ jcc(Assembler::parity, done);
 694   __ jcc(Assembler::below, done);
 695   __ setb(Assembler::notEqual, dst);
 696   __ movzbl(dst, dst);
 697   __ bind(done);
 698 }
 699 
 700 // Math.min()    # Math.max()
 701 // --------------------------
 702 // ucomis[s/d]   #
 703 // ja   -> b     # a
 704 // jp   -> NaN   # NaN
 705 // jb   -> a     # b
 706 // je            #
 707 // |-jz -> a | b # a & b
 708 // |    -> a     #
 709 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
 710                      XMMRegister a, XMMRegister b,
 711                      XMMRegister xmmt, Register rt,
 712                      bool min, bool single) {
 713 
 714   Label nan, zero, below, above, done;
 715 
 716   if (single)
 717     __ ucomiss(a, b);
 718   else
 719     __ ucomisd(a, b);
 720 
 721   if (dst->encoding() != (min ? b : a)->encoding())
 722     __ jccb(Assembler::above, above); // CF=0 & ZF=0
 723   else
 724     __ jccb(Assembler::above, done);
 725 
 726   __ jccb(Assembler::parity, nan);  // PF=1
 727   __ jccb(Assembler::below, below); // CF=1
 728 
 729   // equal
 730   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
 731   if (single) {
 732     __ ucomiss(a, xmmt);
 733     __ jccb(Assembler::equal, zero);
 734 
 735     __ movflt(dst, a);
 736     __ jmp(done);
 737   }
 738   else {
 739     __ ucomisd(a, xmmt);
 740     __ jccb(Assembler::equal, zero);
 741 
 742     __ movdbl(dst, a);
 743     __ jmp(done);
 744   }
 745 
 746   __ bind(zero);
 747   if (min)
 748     __ vpor(dst, a, b, Assembler::AVX_128bit);
 749   else
 750     __ vpand(dst, a, b, Assembler::AVX_128bit);
 751 
 752   __ jmp(done);
 753 
 754   __ bind(above);
 755   if (single)
 756     __ movflt(dst, min ? b : a);
 757   else
 758     __ movdbl(dst, min ? b : a);
 759 
 760   __ jmp(done);
 761 
 762   __ bind(nan);
 763   if (single) {
 764     __ movl(rt, 0x7fc00000); // Float.NaN
 765     __ movdl(dst, rt);
 766   }
 767   else {
 768     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
 769     __ movdq(dst, rt);
 770   }
 771   __ jmp(done);
 772 
 773   __ bind(below);
 774   if (single)
 775     __ movflt(dst, min ? a : b);
 776   else
 777     __ movdbl(dst, min ? a : b);
 778 
 779   __ bind(done);
 780 }
 781 
 782 //=============================================================================
 783 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 784 
 785 int Compile::ConstantTable::calculate_table_base_offset() const {
 786   return 0;  // absolute addressing, no offset
 787 }
 788 
 789 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 790 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 791   ShouldNotReachHere();
 792 }
 793 
 794 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 795   // Empty encoding
 796 }
 797 
 798 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 799   return 0;
 800 }
 801 
 802 #ifndef PRODUCT
 803 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 804   st->print("# MachConstantBaseNode (empty encoding)");
 805 }
 806 #endif
 807 
 808 
 809 //=============================================================================
 810 #ifndef PRODUCT
 811 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 812   Compile* C = ra_->C;
 813 
 814   int framesize = C->frame_size_in_bytes();
 815   int bangsize = C->bang_size_in_bytes();
 816   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 817   // Remove wordSize for return addr which is already pushed.
 818   framesize -= wordSize;
 819 
 820   if (C->need_stack_bang(bangsize)) {
 821     framesize -= wordSize;
 822     st->print("# stack bang (%d bytes)", bangsize);
 823     st->print("\n\t");
 824     st->print("pushq   rbp\t# Save rbp");
 825     if (PreserveFramePointer) {
 826         st->print("\n\t");
 827         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 828     }
 829     if (framesize) {
 830       st->print("\n\t");
 831       st->print("subq    rsp, #%d\t# Create frame",framesize);
 832     }
 833   } else {
 834     st->print("subq    rsp, #%d\t# Create frame",framesize);
 835     st->print("\n\t");
 836     framesize -= wordSize;
 837     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 838     if (PreserveFramePointer) {
 839       st->print("\n\t");
 840       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 841       if (framesize > 0) {
 842         st->print("\n\t");
 843         st->print("addq    rbp, #%d", framesize);
 844       }
 845     }
 846   }
 847 
 848   if (VerifyStackAtCalls) {
 849     st->print("\n\t");
 850     framesize -= wordSize;
 851     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 852 #ifdef ASSERT
 853     st->print("\n\t");
 854     st->print("# stack alignment check");
 855 #endif
 856   }
 857   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 858     st->print("\n\t");
 859     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
 860     st->print("\n\t");
 861     st->print("je      fast_entry\t");
 862     st->print("\n\t");
 863     st->print("call    #nmethod_entry_barrier_stub\t");
 864     st->print("\n\tfast_entry:");
 865   }
 866   st->cr();
 867 }
 868 #endif
 869 
 870 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 871   Compile* C = ra_->C;
 872   MacroAssembler _masm(&cbuf);
 873 
 874   int framesize = C->frame_size_in_bytes();
 875   int bangsize = C->bang_size_in_bytes();
 876 
 877   if (C->clinit_barrier_on_entry()) {
 878     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 879     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 880 
 881     Label L_skip_barrier;
 882     Register klass = rscratch1;
 883 
 884     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
 885     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
 886 
 887     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
 888 
 889     __ bind(L_skip_barrier);
 890   }
 891 
 892   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
 893 
 894   C->set_frame_complete(cbuf.insts_size());
 895 
 896   if (C->has_mach_constant_base_node()) {
 897     // NOTE: We set the table base offset here because users might be
 898     // emitted before MachConstantBaseNode.
 899     Compile::ConstantTable& constant_table = C->constant_table();
 900     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 901   }
 902 }
 903 
 904 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 905 {
 906   return MachNode::size(ra_); // too many variables; just compute it
 907                               // the hard way
 908 }
 909 
 910 int MachPrologNode::reloc() const
 911 {
 912   return 0; // a large enough number
 913 }
 914 
 915 //=============================================================================
 916 #ifndef PRODUCT
 917 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 918 {
 919   Compile* C = ra_->C;
 920   if (generate_vzeroupper(C)) {
 921     st->print("vzeroupper");
 922     st->cr(); st->print("\t");
 923   }
 924 
 925   int framesize = C->frame_size_in_bytes();
 926   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 927   // Remove word for return adr already pushed
 928   // and RBP
 929   framesize -= 2*wordSize;
 930 
 931   if (framesize) {
 932     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 933     st->print("\t");
 934   }
 935 
 936   st->print_cr("popq    rbp");
 937   if (do_polling() && C->is_method_compilation()) {
 938     st->print("\t");
 939     if (SafepointMechanism::uses_thread_local_poll()) {
 940       st->print_cr("movq    rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 941                    "testl   rax, [rscratch1]\t"
 942                    "# Safepoint: poll for GC");
 943     } else if (Assembler::is_polling_page_far()) {
 944       st->print_cr("movq    rscratch1, #polling_page_address\n\t"
 945                    "testl   rax, [rscratch1]\t"
 946                    "# Safepoint: poll for GC");
 947     } else {
 948       st->print_cr("testl   rax, [rip + #offset_to_poll_page]\t"
 949                    "# Safepoint: poll for GC");
 950     }
 951   }
 952 }
 953 #endif
 954 
 955 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 956 {
 957   Compile* C = ra_->C;
 958   MacroAssembler _masm(&cbuf);
 959 
 960   if (generate_vzeroupper(C)) {
 961     // Clear upper bits of YMM registers when current compiled code uses
 962     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 963     __ vzeroupper();
 964   }
 965 
 966   int framesize = C->frame_size_in_bytes();
 967   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 968   // Remove word for return adr already pushed
 969   // and RBP
 970   framesize -= 2*wordSize;
 971 
 972   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 973 
 974   if (framesize) {
 975     emit_opcode(cbuf, Assembler::REX_W);
 976     if (framesize < 0x80) {
 977       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 978       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 979       emit_d8(cbuf, framesize);
 980     } else {
 981       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 982       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 983       emit_d32(cbuf, framesize);
 984     }
 985   }
 986 
 987   // popq rbp
 988   emit_opcode(cbuf, 0x58 | RBP_enc);
 989 
 990   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 991     __ reserved_stack_check();
 992   }
 993 
 994   if (do_polling() && C->is_method_compilation()) {
 995     MacroAssembler _masm(&cbuf);
 996     if (SafepointMechanism::uses_thread_local_poll()) {
 997       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
 998       __ relocate(relocInfo::poll_return_type);
 999       __ testl(rax, Address(rscratch1, 0));
1000     } else {
1001       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1002       if (Assembler::is_polling_page_far()) {
1003         __ lea(rscratch1, polling_page);
1004         __ relocate(relocInfo::poll_return_type);
1005         __ testl(rax, Address(rscratch1, 0));
1006       } else {
1007         __ testl(rax, polling_page);
1008       }
1009     }
1010   }
1011 }
1012 
1013 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1014 {
1015   return MachNode::size(ra_); // too many variables; just compute it
1016                               // the hard way
1017 }
1018 
1019 int MachEpilogNode::reloc() const
1020 {
1021   return 2; // a large enough number
1022 }
1023 
1024 const Pipeline* MachEpilogNode::pipeline() const
1025 {
1026   return MachNode::pipeline_class();
1027 }
1028 
1029 int MachEpilogNode::safepoint_offset() const
1030 {
1031   return 0;
1032 }
1033 
1034 //=============================================================================
1035 
1036 enum RC {
1037   rc_bad,
1038   rc_int,
1039   rc_float,
1040   rc_stack
1041 };
1042 
1043 static enum RC rc_class(OptoReg::Name reg)
1044 {
1045   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1046 
1047   if (OptoReg::is_stack(reg)) return rc_stack;
1048 
1049   VMReg r = OptoReg::as_VMReg(reg);
1050 
1051   if (r->is_Register()) return rc_int;
1052 
1053   assert(r->is_XMMRegister(), "must be");
1054   return rc_float;
1055 }
1056 
1057 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1058 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1059                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1060 
1061 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1062                             int stack_offset, int reg, uint ireg, outputStream* st);
1063 
1064 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1065                                       int dst_offset, uint ireg, outputStream* st) {
1066   if (cbuf) {
1067     MacroAssembler _masm(cbuf);
1068     switch (ireg) {
1069     case Op_VecS:
1070       __ movq(Address(rsp, -8), rax);
1071       __ movl(rax, Address(rsp, src_offset));
1072       __ movl(Address(rsp, dst_offset), rax);
1073       __ movq(rax, Address(rsp, -8));
1074       break;
1075     case Op_VecD:
1076       __ pushq(Address(rsp, src_offset));
1077       __ popq (Address(rsp, dst_offset));
1078       break;
1079     case Op_VecX:
1080       __ pushq(Address(rsp, src_offset));
1081       __ popq (Address(rsp, dst_offset));
1082       __ pushq(Address(rsp, src_offset+8));
1083       __ popq (Address(rsp, dst_offset+8));
1084       break;
1085     case Op_VecY:
1086       __ vmovdqu(Address(rsp, -32), xmm0);
1087       __ vmovdqu(xmm0, Address(rsp, src_offset));
1088       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1089       __ vmovdqu(xmm0, Address(rsp, -32));
1090       break;
1091     case Op_VecZ:
1092       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1093       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1094       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1095       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1096       break;
1097     default:
1098       ShouldNotReachHere();
1099     }
1100 #ifndef PRODUCT
1101   } else {
1102     switch (ireg) {
1103     case Op_VecS:
1104       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1105                 "movl    rax, [rsp + #%d]\n\t"
1106                 "movl    [rsp + #%d], rax\n\t"
1107                 "movq    rax, [rsp - #8]",
1108                 src_offset, dst_offset);
1109       break;
1110     case Op_VecD:
1111       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1112                 "popq    [rsp + #%d]",
1113                 src_offset, dst_offset);
1114       break;
1115      case Op_VecX:
1116       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1117                 "popq    [rsp + #%d]\n\t"
1118                 "pushq   [rsp + #%d]\n\t"
1119                 "popq    [rsp + #%d]",
1120                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1121       break;
1122     case Op_VecY:
1123       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1124                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1125                 "vmovdqu [rsp + #%d], xmm0\n\t"
1126                 "vmovdqu xmm0, [rsp - #32]",
1127                 src_offset, dst_offset);
1128       break;
1129     case Op_VecZ:
1130       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1131                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1132                 "vmovdqu [rsp + #%d], xmm0\n\t"
1133                 "vmovdqu xmm0, [rsp - #64]",
1134                 src_offset, dst_offset);
1135       break;
1136     default:
1137       ShouldNotReachHere();
1138     }
1139 #endif
1140   }
1141 }
1142 
1143 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1144                                        PhaseRegAlloc* ra_,
1145                                        bool do_size,
1146                                        outputStream* st) const {
1147   assert(cbuf != NULL || st  != NULL, "sanity");
1148   // Get registers to move
1149   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1150   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1151   OptoReg::Name dst_second = ra_->get_reg_second(this);
1152   OptoReg::Name dst_first = ra_->get_reg_first(this);
1153 
1154   enum RC src_second_rc = rc_class(src_second);
1155   enum RC src_first_rc = rc_class(src_first);
1156   enum RC dst_second_rc = rc_class(dst_second);
1157   enum RC dst_first_rc = rc_class(dst_first);
1158 
1159   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1160          "must move at least 1 register" );
1161 
1162   if (src_first == dst_first && src_second == dst_second) {
1163     // Self copy, no move
1164     return 0;
1165   }
1166   if (bottom_type()->isa_vect() != NULL) {
1167     uint ireg = ideal_reg();
1168     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1169     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1170     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1171       // mem -> mem
1172       int src_offset = ra_->reg2offset(src_first);
1173       int dst_offset = ra_->reg2offset(dst_first);
1174       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1175     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1176       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1177     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1178       int stack_offset = ra_->reg2offset(dst_first);
1179       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1180     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1181       int stack_offset = ra_->reg2offset(src_first);
1182       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1183     } else {
1184       ShouldNotReachHere();
1185     }
1186     return 0;
1187   }
1188   if (src_first_rc == rc_stack) {
1189     // mem ->
1190     if (dst_first_rc == rc_stack) {
1191       // mem -> mem
1192       assert(src_second != dst_first, "overlap");
1193       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1194           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1195         // 64-bit
1196         int src_offset = ra_->reg2offset(src_first);
1197         int dst_offset = ra_->reg2offset(dst_first);
1198         if (cbuf) {
1199           MacroAssembler _masm(cbuf);
1200           __ pushq(Address(rsp, src_offset));
1201           __ popq (Address(rsp, dst_offset));
1202 #ifndef PRODUCT
1203         } else {
1204           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1205                     "popq    [rsp + #%d]",
1206                      src_offset, dst_offset);
1207 #endif
1208         }
1209       } else {
1210         // 32-bit
1211         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1212         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1213         // No pushl/popl, so:
1214         int src_offset = ra_->reg2offset(src_first);
1215         int dst_offset = ra_->reg2offset(dst_first);
1216         if (cbuf) {
1217           MacroAssembler _masm(cbuf);
1218           __ movq(Address(rsp, -8), rax);
1219           __ movl(rax, Address(rsp, src_offset));
1220           __ movl(Address(rsp, dst_offset), rax);
1221           __ movq(rax, Address(rsp, -8));
1222 #ifndef PRODUCT
1223         } else {
1224           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1225                     "movl    rax, [rsp + #%d]\n\t"
1226                     "movl    [rsp + #%d], rax\n\t"
1227                     "movq    rax, [rsp - #8]",
1228                      src_offset, dst_offset);
1229 #endif
1230         }
1231       }
1232       return 0;
1233     } else if (dst_first_rc == rc_int) {
1234       // mem -> gpr
1235       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1236           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1237         // 64-bit
1238         int offset = ra_->reg2offset(src_first);
1239         if (cbuf) {
1240           MacroAssembler _masm(cbuf);
1241           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1242 #ifndef PRODUCT
1243         } else {
1244           st->print("movq    %s, [rsp + #%d]\t# spill",
1245                      Matcher::regName[dst_first],
1246                      offset);
1247 #endif
1248         }
1249       } else {
1250         // 32-bit
1251         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1252         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1253         int offset = ra_->reg2offset(src_first);
1254         if (cbuf) {
1255           MacroAssembler _masm(cbuf);
1256           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1257 #ifndef PRODUCT
1258         } else {
1259           st->print("movl    %s, [rsp + #%d]\t# spill",
1260                      Matcher::regName[dst_first],
1261                      offset);
1262 #endif
1263         }
1264       }
1265       return 0;
1266     } else if (dst_first_rc == rc_float) {
1267       // mem-> xmm
1268       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1269           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1270         // 64-bit
1271         int offset = ra_->reg2offset(src_first);
1272         if (cbuf) {
1273           MacroAssembler _masm(cbuf);
1274           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1275 #ifndef PRODUCT
1276         } else {
1277           st->print("%s  %s, [rsp + #%d]\t# spill",
1278                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1279                      Matcher::regName[dst_first],
1280                      offset);
1281 #endif
1282         }
1283       } else {
1284         // 32-bit
1285         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1286         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1287         int offset = ra_->reg2offset(src_first);
1288         if (cbuf) {
1289           MacroAssembler _masm(cbuf);
1290           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1291 #ifndef PRODUCT
1292         } else {
1293           st->print("movss   %s, [rsp + #%d]\t# spill",
1294                      Matcher::regName[dst_first],
1295                      offset);
1296 #endif
1297         }
1298       }
1299       return 0;
1300     }
1301   } else if (src_first_rc == rc_int) {
1302     // gpr ->
1303     if (dst_first_rc == rc_stack) {
1304       // gpr -> mem
1305       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1306           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1307         // 64-bit
1308         int offset = ra_->reg2offset(dst_first);
1309         if (cbuf) {
1310           MacroAssembler _masm(cbuf);
1311           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1312 #ifndef PRODUCT
1313         } else {
1314           st->print("movq    [rsp + #%d], %s\t# spill",
1315                      offset,
1316                      Matcher::regName[src_first]);
1317 #endif
1318         }
1319       } else {
1320         // 32-bit
1321         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1322         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1323         int offset = ra_->reg2offset(dst_first);
1324         if (cbuf) {
1325           MacroAssembler _masm(cbuf);
1326           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1327 #ifndef PRODUCT
1328         } else {
1329           st->print("movl    [rsp + #%d], %s\t# spill",
1330                      offset,
1331                      Matcher::regName[src_first]);
1332 #endif
1333         }
1334       }
1335       return 0;
1336     } else if (dst_first_rc == rc_int) {
1337       // gpr -> gpr
1338       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1339           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1340         // 64-bit
1341         if (cbuf) {
1342           MacroAssembler _masm(cbuf);
1343           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1344                   as_Register(Matcher::_regEncode[src_first]));
1345 #ifndef PRODUCT
1346         } else {
1347           st->print("movq    %s, %s\t# spill",
1348                      Matcher::regName[dst_first],
1349                      Matcher::regName[src_first]);
1350 #endif
1351         }
1352         return 0;
1353       } else {
1354         // 32-bit
1355         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1356         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1357         if (cbuf) {
1358           MacroAssembler _masm(cbuf);
1359           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1360                   as_Register(Matcher::_regEncode[src_first]));
1361 #ifndef PRODUCT
1362         } else {
1363           st->print("movl    %s, %s\t# spill",
1364                      Matcher::regName[dst_first],
1365                      Matcher::regName[src_first]);
1366 #endif
1367         }
1368         return 0;
1369       }
1370     } else if (dst_first_rc == rc_float) {
1371       // gpr -> xmm
1372       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1373           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1374         // 64-bit
1375         if (cbuf) {
1376           MacroAssembler _masm(cbuf);
1377           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1378 #ifndef PRODUCT
1379         } else {
1380           st->print("movdq   %s, %s\t# spill",
1381                      Matcher::regName[dst_first],
1382                      Matcher::regName[src_first]);
1383 #endif
1384         }
1385       } else {
1386         // 32-bit
1387         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1388         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1389         if (cbuf) {
1390           MacroAssembler _masm(cbuf);
1391           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1392 #ifndef PRODUCT
1393         } else {
1394           st->print("movdl   %s, %s\t# spill",
1395                      Matcher::regName[dst_first],
1396                      Matcher::regName[src_first]);
1397 #endif
1398         }
1399       }
1400       return 0;
1401     }
1402   } else if (src_first_rc == rc_float) {
1403     // xmm ->
1404     if (dst_first_rc == rc_stack) {
1405       // xmm -> mem
1406       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1407           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1408         // 64-bit
1409         int offset = ra_->reg2offset(dst_first);
1410         if (cbuf) {
1411           MacroAssembler _masm(cbuf);
1412           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1413 #ifndef PRODUCT
1414         } else {
1415           st->print("movsd   [rsp + #%d], %s\t# spill",
1416                      offset,
1417                      Matcher::regName[src_first]);
1418 #endif
1419         }
1420       } else {
1421         // 32-bit
1422         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1423         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1424         int offset = ra_->reg2offset(dst_first);
1425         if (cbuf) {
1426           MacroAssembler _masm(cbuf);
1427           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1428 #ifndef PRODUCT
1429         } else {
1430           st->print("movss   [rsp + #%d], %s\t# spill",
1431                      offset,
1432                      Matcher::regName[src_first]);
1433 #endif
1434         }
1435       }
1436       return 0;
1437     } else if (dst_first_rc == rc_int) {
1438       // xmm -> gpr
1439       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1440           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1441         // 64-bit
1442         if (cbuf) {
1443           MacroAssembler _masm(cbuf);
1444           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1445 #ifndef PRODUCT
1446         } else {
1447           st->print("movdq   %s, %s\t# spill",
1448                      Matcher::regName[dst_first],
1449                      Matcher::regName[src_first]);
1450 #endif
1451         }
1452       } else {
1453         // 32-bit
1454         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1455         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1456         if (cbuf) {
1457           MacroAssembler _masm(cbuf);
1458           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1459 #ifndef PRODUCT
1460         } else {
1461           st->print("movdl   %s, %s\t# spill",
1462                      Matcher::regName[dst_first],
1463                      Matcher::regName[src_first]);
1464 #endif
1465         }
1466       }
1467       return 0;
1468     } else if (dst_first_rc == rc_float) {
1469       // xmm -> xmm
1470       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1471           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1472         // 64-bit
1473         if (cbuf) {
1474           MacroAssembler _masm(cbuf);
1475           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1476 #ifndef PRODUCT
1477         } else {
1478           st->print("%s  %s, %s\t# spill",
1479                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1480                      Matcher::regName[dst_first],
1481                      Matcher::regName[src_first]);
1482 #endif
1483         }
1484       } else {
1485         // 32-bit
1486         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1487         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1488         if (cbuf) {
1489           MacroAssembler _masm(cbuf);
1490           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1491 #ifndef PRODUCT
1492         } else {
1493           st->print("%s  %s, %s\t# spill",
1494                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1495                      Matcher::regName[dst_first],
1496                      Matcher::regName[src_first]);
1497 #endif
1498         }
1499       }
1500       return 0;
1501     }
1502   }
1503 
1504   assert(0," foo ");
1505   Unimplemented();
1506   return 0;
1507 }
1508 
1509 #ifndef PRODUCT
1510 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1511   implementation(NULL, ra_, false, st);
1512 }
1513 #endif
1514 
1515 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1516   implementation(&cbuf, ra_, false, NULL);
1517 }
1518 
1519 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1520   return MachNode::size(ra_);
1521 }
1522 
1523 //=============================================================================
1524 #ifndef PRODUCT
1525 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1526 {
1527   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1528   int reg = ra_->get_reg_first(this);
1529   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1530             Matcher::regName[reg], offset);
1531 }
1532 #endif
1533 
1534 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1535 {
1536   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1537   int reg = ra_->get_encode(this);
1538   if (offset >= 0x80) {
1539     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1540     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1541     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1542     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1543     emit_d32(cbuf, offset);
1544   } else {
1545     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1546     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1547     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1548     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1549     emit_d8(cbuf, offset);
1550   }
1551 }
1552 
1553 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1554 {
1555   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1556   return (offset < 0x80) ? 5 : 8; // REX
1557 }
1558 
1559 //=============================================================================
1560 #ifndef PRODUCT
1561 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1562 {
1563   if (UseCompressedClassPointers) {
1564     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1565     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1566     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1567   } else {
1568     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1569                  "# Inline cache check");
1570   }
1571   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1572   st->print_cr("\tnop\t# nops to align entry point");
1573 }
1574 #endif
1575 
1576 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1577 {
1578   MacroAssembler masm(&cbuf);
1579   uint insts_size = cbuf.insts_size();
1580   if (UseCompressedClassPointers) {
1581     masm.load_klass(rscratch1, j_rarg0);
1582     masm.cmpptr(rax, rscratch1);
1583   } else {
1584     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1585   }
1586 
1587   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1588 
1589   /* WARNING these NOPs are critical so that verified entry point is properly
1590      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1591   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1592   if (OptoBreakpoint) {
1593     // Leave space for int3
1594     nops_cnt -= 1;
1595   }
1596   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1597   if (nops_cnt > 0)
1598     masm.nop(nops_cnt);
1599 }
1600 
1601 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1602 {
1603   return MachNode::size(ra_); // too many variables; just compute it
1604                               // the hard way
1605 }
1606 
1607 
1608 //=============================================================================
1609 
1610 int Matcher::regnum_to_fpu_offset(int regnum)
1611 {
1612   return regnum - 32; // The FP registers are in the second chunk
1613 }
1614 
1615 // This is UltraSparc specific, true just means we have fast l2f conversion
1616 const bool Matcher::convL2FSupported(void) {
1617   return true;
1618 }
1619 
1620 // Is this branch offset short enough that a short branch can be used?
1621 //
1622 // NOTE: If the platform does not provide any short branch variants, then
1623 //       this method should return false for offset 0.
1624 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1625   // The passed offset is relative to address of the branch.
1626   // On 86 a branch displacement is calculated relative to address
1627   // of a next instruction.
1628   offset -= br_size;
1629 
1630   // the short version of jmpConUCF2 contains multiple branches,
1631   // making the reach slightly less
1632   if (rule == jmpConUCF2_rule)
1633     return (-126 <= offset && offset <= 125);
1634   return (-128 <= offset && offset <= 127);
1635 }
1636 
1637 const bool Matcher::isSimpleConstant64(jlong value) {
1638   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1639   //return value == (int) value;  // Cf. storeImmL and immL32.
1640 
1641   // Probably always true, even if a temp register is required.
1642   return true;
1643 }
1644 
1645 // The ecx parameter to rep stosq for the ClearArray node is in words.
1646 const bool Matcher::init_array_count_is_in_bytes = false;
1647 
1648 // No additional cost for CMOVL.
1649 const int Matcher::long_cmove_cost() { return 0; }
1650 
1651 // No CMOVF/CMOVD with SSE2
1652 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1653 
1654 // Does the CPU require late expand (see block.cpp for description of late expand)?
1655 const bool Matcher::require_postalloc_expand = false;
1656 
1657 // Do we need to mask the count passed to shift instructions or does
1658 // the cpu only look at the lower 5/6 bits anyway?
1659 const bool Matcher::need_masked_shift_count = false;
1660 
1661 bool Matcher::narrow_oop_use_complex_address() {
1662   assert(UseCompressedOops, "only for compressed oops code");
1663   return (LogMinObjAlignmentInBytes <= 3);
1664 }
1665 
1666 bool Matcher::narrow_klass_use_complex_address() {
1667   assert(UseCompressedClassPointers, "only for compressed klass code");
1668   return (LogKlassAlignmentInBytes <= 3);
1669 }
1670 
1671 bool Matcher::const_oop_prefer_decode() {
1672   // Prefer ConN+DecodeN over ConP.
1673   return true;
1674 }
1675 
1676 bool Matcher::const_klass_prefer_decode() {
1677   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1678   //       or condisider the following:
1679   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1680   //return CompressedKlassPointers::base() == NULL;
1681   return true;
1682 }
1683 
1684 // Is it better to copy float constants, or load them directly from
1685 // memory?  Intel can load a float constant from a direct address,
1686 // requiring no extra registers.  Most RISCs will have to materialize
1687 // an address into a register first, so they would do better to copy
1688 // the constant from stack.
1689 const bool Matcher::rematerialize_float_constants = true; // XXX
1690 
1691 // If CPU can load and store mis-aligned doubles directly then no
1692 // fixup is needed.  Else we split the double into 2 integer pieces
1693 // and move it piece-by-piece.  Only happens when passing doubles into
1694 // C code as the Java calling convention forces doubles to be aligned.
1695 const bool Matcher::misaligned_doubles_ok = true;
1696 
1697 // No-op on amd64
1698 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1699 
1700 // Advertise here if the CPU requires explicit rounding operations to
1701 // implement the UseStrictFP mode.
1702 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1703 
1704 // Are floats conerted to double when stored to stack during deoptimization?
1705 // On x64 it is stored without convertion so we can use normal access.
1706 bool Matcher::float_in_double() { return false; }
1707 
1708 // Do ints take an entire long register or just half?
1709 const bool Matcher::int_in_long = true;
1710 
1711 // Return whether or not this register is ever used as an argument.
1712 // This function is used on startup to build the trampoline stubs in
1713 // generateOptoStub.  Registers not mentioned will be killed by the VM
1714 // call in the trampoline, and arguments in those registers not be
1715 // available to the callee.
1716 bool Matcher::can_be_java_arg(int reg)
1717 {
1718   return
1719     reg ==  RDI_num || reg == RDI_H_num ||
1720     reg ==  RSI_num || reg == RSI_H_num ||
1721     reg ==  RDX_num || reg == RDX_H_num ||
1722     reg ==  RCX_num || reg == RCX_H_num ||
1723     reg ==   R8_num || reg ==  R8_H_num ||
1724     reg ==   R9_num || reg ==  R9_H_num ||
1725     reg ==  R12_num || reg == R12_H_num ||
1726     reg == XMM0_num || reg == XMM0b_num ||
1727     reg == XMM1_num || reg == XMM1b_num ||
1728     reg == XMM2_num || reg == XMM2b_num ||
1729     reg == XMM3_num || reg == XMM3b_num ||
1730     reg == XMM4_num || reg == XMM4b_num ||
1731     reg == XMM5_num || reg == XMM5b_num ||
1732     reg == XMM6_num || reg == XMM6b_num ||
1733     reg == XMM7_num || reg == XMM7b_num;
1734 }
1735 
1736 bool Matcher::is_spillable_arg(int reg)
1737 {
1738   return can_be_java_arg(reg);
1739 }
1740 
1741 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1742   // In 64 bit mode a code which use multiply when
1743   // devisor is constant is faster than hardware
1744   // DIV instruction (it uses MulHiL).
1745   return false;
1746 }
1747 
1748 // Register for DIVI projection of divmodI
1749 RegMask Matcher::divI_proj_mask() {
1750   return INT_RAX_REG_mask();
1751 }
1752 
1753 // Register for MODI projection of divmodI
1754 RegMask Matcher::modI_proj_mask() {
1755   return INT_RDX_REG_mask();
1756 }
1757 
1758 // Register for DIVL projection of divmodL
1759 RegMask Matcher::divL_proj_mask() {
1760   return LONG_RAX_REG_mask();
1761 }
1762 
1763 // Register for MODL projection of divmodL
1764 RegMask Matcher::modL_proj_mask() {
1765   return LONG_RDX_REG_mask();
1766 }
1767 
1768 // Register for saving SP into on method handle invokes. Not used on x86_64.
1769 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1770     return NO_REG_mask();
1771 }
1772 
1773 %}
1774 
1775 //----------ENCODING BLOCK-----------------------------------------------------
1776 // This block specifies the encoding classes used by the compiler to
1777 // output byte streams.  Encoding classes are parameterized macros
1778 // used by Machine Instruction Nodes in order to generate the bit
1779 // encoding of the instruction.  Operands specify their base encoding
1780 // interface with the interface keyword.  There are currently
1781 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1782 // COND_INTER.  REG_INTER causes an operand to generate a function
1783 // which returns its register number when queried.  CONST_INTER causes
1784 // an operand to generate a function which returns the value of the
1785 // constant when queried.  MEMORY_INTER causes an operand to generate
1786 // four functions which return the Base Register, the Index Register,
1787 // the Scale Value, and the Offset Value of the operand when queried.
1788 // COND_INTER causes an operand to generate six functions which return
1789 // the encoding code (ie - encoding bits for the instruction)
1790 // associated with each basic boolean condition for a conditional
1791 // instruction.
1792 //
1793 // Instructions specify two basic values for encoding.  Again, a
1794 // function is available to check if the constant displacement is an
1795 // oop. They use the ins_encode keyword to specify their encoding
1796 // classes (which must be a sequence of enc_class names, and their
1797 // parameters, specified in the encoding block), and they use the
1798 // opcode keyword to specify, in order, their primary, secondary, and
1799 // tertiary opcode.  Only the opcode sections which a particular
1800 // instruction needs for encoding need to be specified.
1801 encode %{
1802   // Build emit functions for each basic byte or larger field in the
1803   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1804   // from C++ code in the enc_class source block.  Emit functions will
1805   // live in the main source block for now.  In future, we can
1806   // generalize this by adding a syntax that specifies the sizes of
1807   // fields in an order, so that the adlc can build the emit functions
1808   // automagically
1809 
1810   // Emit primary opcode
1811   enc_class OpcP
1812   %{
1813     emit_opcode(cbuf, $primary);
1814   %}
1815 
1816   // Emit secondary opcode
1817   enc_class OpcS
1818   %{
1819     emit_opcode(cbuf, $secondary);
1820   %}
1821 
1822   // Emit tertiary opcode
1823   enc_class OpcT
1824   %{
1825     emit_opcode(cbuf, $tertiary);
1826   %}
1827 
1828   // Emit opcode directly
1829   enc_class Opcode(immI d8)
1830   %{
1831     emit_opcode(cbuf, $d8$$constant);
1832   %}
1833 
1834   // Emit size prefix
1835   enc_class SizePrefix
1836   %{
1837     emit_opcode(cbuf, 0x66);
1838   %}
1839 
1840   enc_class reg(rRegI reg)
1841   %{
1842     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1843   %}
1844 
1845   enc_class reg_reg(rRegI dst, rRegI src)
1846   %{
1847     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1848   %}
1849 
1850   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1851   %{
1852     emit_opcode(cbuf, $opcode$$constant);
1853     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1854   %}
1855 
1856   enc_class cdql_enc(no_rax_rdx_RegI div)
1857   %{
1858     // Full implementation of Java idiv and irem; checks for
1859     // special case as described in JVM spec., p.243 & p.271.
1860     //
1861     //         normal case                           special case
1862     //
1863     // input : rax: dividend                         min_int
1864     //         reg: divisor                          -1
1865     //
1866     // output: rax: quotient  (= rax idiv reg)       min_int
1867     //         rdx: remainder (= rax irem reg)       0
1868     //
1869     //  Code sequnce:
1870     //
1871     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1872     //    5:   75 07/08                jne    e <normal>
1873     //    7:   33 d2                   xor    %edx,%edx
1874     //  [div >= 8 -> offset + 1]
1875     //  [REX_B]
1876     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1877     //    c:   74 03/04                je     11 <done>
1878     // 000000000000000e <normal>:
1879     //    e:   99                      cltd
1880     //  [div >= 8 -> offset + 1]
1881     //  [REX_B]
1882     //    f:   f7 f9                   idiv   $div
1883     // 0000000000000011 <done>:
1884 
1885     // cmp    $0x80000000,%eax
1886     emit_opcode(cbuf, 0x3d);
1887     emit_d8(cbuf, 0x00);
1888     emit_d8(cbuf, 0x00);
1889     emit_d8(cbuf, 0x00);
1890     emit_d8(cbuf, 0x80);
1891 
1892     // jne    e <normal>
1893     emit_opcode(cbuf, 0x75);
1894     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1895 
1896     // xor    %edx,%edx
1897     emit_opcode(cbuf, 0x33);
1898     emit_d8(cbuf, 0xD2);
1899 
1900     // cmp    $0xffffffffffffffff,%ecx
1901     if ($div$$reg >= 8) {
1902       emit_opcode(cbuf, Assembler::REX_B);
1903     }
1904     emit_opcode(cbuf, 0x83);
1905     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1906     emit_d8(cbuf, 0xFF);
1907 
1908     // je     11 <done>
1909     emit_opcode(cbuf, 0x74);
1910     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1911 
1912     // <normal>
1913     // cltd
1914     emit_opcode(cbuf, 0x99);
1915 
1916     // idivl (note: must be emitted by the user of this rule)
1917     // <done>
1918   %}
1919 
1920   enc_class cdqq_enc(no_rax_rdx_RegL div)
1921   %{
1922     // Full implementation of Java ldiv and lrem; checks for
1923     // special case as described in JVM spec., p.243 & p.271.
1924     //
1925     //         normal case                           special case
1926     //
1927     // input : rax: dividend                         min_long
1928     //         reg: divisor                          -1
1929     //
1930     // output: rax: quotient  (= rax idiv reg)       min_long
1931     //         rdx: remainder (= rax irem reg)       0
1932     //
1933     //  Code sequnce:
1934     //
1935     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1936     //    7:   00 00 80
1937     //    a:   48 39 d0                cmp    %rdx,%rax
1938     //    d:   75 08                   jne    17 <normal>
1939     //    f:   33 d2                   xor    %edx,%edx
1940     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1941     //   15:   74 05                   je     1c <done>
1942     // 0000000000000017 <normal>:
1943     //   17:   48 99                   cqto
1944     //   19:   48 f7 f9                idiv   $div
1945     // 000000000000001c <done>:
1946 
1947     // mov    $0x8000000000000000,%rdx
1948     emit_opcode(cbuf, Assembler::REX_W);
1949     emit_opcode(cbuf, 0xBA);
1950     emit_d8(cbuf, 0x00);
1951     emit_d8(cbuf, 0x00);
1952     emit_d8(cbuf, 0x00);
1953     emit_d8(cbuf, 0x00);
1954     emit_d8(cbuf, 0x00);
1955     emit_d8(cbuf, 0x00);
1956     emit_d8(cbuf, 0x00);
1957     emit_d8(cbuf, 0x80);
1958 
1959     // cmp    %rdx,%rax
1960     emit_opcode(cbuf, Assembler::REX_W);
1961     emit_opcode(cbuf, 0x39);
1962     emit_d8(cbuf, 0xD0);
1963 
1964     // jne    17 <normal>
1965     emit_opcode(cbuf, 0x75);
1966     emit_d8(cbuf, 0x08);
1967 
1968     // xor    %edx,%edx
1969     emit_opcode(cbuf, 0x33);
1970     emit_d8(cbuf, 0xD2);
1971 
1972     // cmp    $0xffffffffffffffff,$div
1973     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1974     emit_opcode(cbuf, 0x83);
1975     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1976     emit_d8(cbuf, 0xFF);
1977 
1978     // je     1e <done>
1979     emit_opcode(cbuf, 0x74);
1980     emit_d8(cbuf, 0x05);
1981 
1982     // <normal>
1983     // cqto
1984     emit_opcode(cbuf, Assembler::REX_W);
1985     emit_opcode(cbuf, 0x99);
1986 
1987     // idivq (note: must be emitted by the user of this rule)
1988     // <done>
1989   %}
1990 
1991   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1992   enc_class OpcSE(immI imm)
1993   %{
1994     // Emit primary opcode and set sign-extend bit
1995     // Check for 8-bit immediate, and set sign extend bit in opcode
1996     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1997       emit_opcode(cbuf, $primary | 0x02);
1998     } else {
1999       // 32-bit immediate
2000       emit_opcode(cbuf, $primary);
2001     }
2002   %}
2003 
2004   enc_class OpcSErm(rRegI dst, immI imm)
2005   %{
2006     // OpcSEr/m
2007     int dstenc = $dst$$reg;
2008     if (dstenc >= 8) {
2009       emit_opcode(cbuf, Assembler::REX_B);
2010       dstenc -= 8;
2011     }
2012     // Emit primary opcode and set sign-extend bit
2013     // Check for 8-bit immediate, and set sign extend bit in opcode
2014     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2015       emit_opcode(cbuf, $primary | 0x02);
2016     } else {
2017       // 32-bit immediate
2018       emit_opcode(cbuf, $primary);
2019     }
2020     // Emit r/m byte with secondary opcode, after primary opcode.
2021     emit_rm(cbuf, 0x3, $secondary, dstenc);
2022   %}
2023 
2024   enc_class OpcSErm_wide(rRegL dst, immI imm)
2025   %{
2026     // OpcSEr/m
2027     int dstenc = $dst$$reg;
2028     if (dstenc < 8) {
2029       emit_opcode(cbuf, Assembler::REX_W);
2030     } else {
2031       emit_opcode(cbuf, Assembler::REX_WB);
2032       dstenc -= 8;
2033     }
2034     // Emit primary opcode and set sign-extend bit
2035     // Check for 8-bit immediate, and set sign extend bit in opcode
2036     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2037       emit_opcode(cbuf, $primary | 0x02);
2038     } else {
2039       // 32-bit immediate
2040       emit_opcode(cbuf, $primary);
2041     }
2042     // Emit r/m byte with secondary opcode, after primary opcode.
2043     emit_rm(cbuf, 0x3, $secondary, dstenc);
2044   %}
2045 
2046   enc_class Con8or32(immI imm)
2047   %{
2048     // Check for 8-bit immediate, and set sign extend bit in opcode
2049     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2050       $$$emit8$imm$$constant;
2051     } else {
2052       // 32-bit immediate
2053       $$$emit32$imm$$constant;
2054     }
2055   %}
2056 
2057   enc_class opc2_reg(rRegI dst)
2058   %{
2059     // BSWAP
2060     emit_cc(cbuf, $secondary, $dst$$reg);
2061   %}
2062 
2063   enc_class opc3_reg(rRegI dst)
2064   %{
2065     // BSWAP
2066     emit_cc(cbuf, $tertiary, $dst$$reg);
2067   %}
2068 
2069   enc_class reg_opc(rRegI div)
2070   %{
2071     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2072     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2073   %}
2074 
2075   enc_class enc_cmov(cmpOp cop)
2076   %{
2077     // CMOV
2078     $$$emit8$primary;
2079     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2080   %}
2081 
2082   enc_class enc_PartialSubtypeCheck()
2083   %{
2084     Register Rrdi = as_Register(RDI_enc); // result register
2085     Register Rrax = as_Register(RAX_enc); // super class
2086     Register Rrcx = as_Register(RCX_enc); // killed
2087     Register Rrsi = as_Register(RSI_enc); // sub class
2088     Label miss;
2089     const bool set_cond_codes = true;
2090 
2091     MacroAssembler _masm(&cbuf);
2092     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2093                                      NULL, &miss,
2094                                      /*set_cond_codes:*/ true);
2095     if ($primary) {
2096       __ xorptr(Rrdi, Rrdi);
2097     }
2098     __ bind(miss);
2099   %}
2100 
2101   enc_class clear_avx %{
2102     debug_only(int off0 = cbuf.insts_size());
2103     if (generate_vzeroupper(Compile::current())) {
2104       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2105       // Clear upper bits of YMM registers when current compiled code uses
2106       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2107       MacroAssembler _masm(&cbuf);
2108       __ vzeroupper();
2109     }
2110     debug_only(int off1 = cbuf.insts_size());
2111     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2112   %}
2113 
2114   enc_class Java_To_Runtime(method meth) %{
2115     // No relocation needed
2116     MacroAssembler _masm(&cbuf);
2117     __ mov64(r10, (int64_t) $meth$$method);
2118     __ call(r10);
2119   %}
2120 
2121   enc_class Java_To_Interpreter(method meth)
2122   %{
2123     // CALL Java_To_Interpreter
2124     // This is the instruction starting address for relocation info.
2125     cbuf.set_insts_mark();
2126     $$$emit8$primary;
2127     // CALL directly to the runtime
2128     emit_d32_reloc(cbuf,
2129                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2130                    runtime_call_Relocation::spec(),
2131                    RELOC_DISP32);
2132   %}
2133 
2134   enc_class Java_Static_Call(method meth)
2135   %{
2136     // JAVA STATIC CALL
2137     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2138     // determine who we intended to call.
2139     cbuf.set_insts_mark();
2140     $$$emit8$primary;
2141 
2142     if (!_method) {
2143       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2144                      runtime_call_Relocation::spec(),
2145                      RELOC_DISP32);
2146     } else {
2147       int method_index = resolved_method_index(cbuf);
2148       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2149                                                   : static_call_Relocation::spec(method_index);
2150       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2151                      rspec, RELOC_DISP32);
2152       // Emit stubs for static call.
2153       address mark = cbuf.insts_mark();
2154       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2155       if (stub == NULL) {
2156         ciEnv::current()->record_failure("CodeCache is full");
2157         return;
2158       }
2159 #if INCLUDE_AOT
2160       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2161 #endif
2162     }
2163   %}
2164 
2165   enc_class Java_Dynamic_Call(method meth) %{
2166     MacroAssembler _masm(&cbuf);
2167     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2168   %}
2169 
2170   enc_class Java_Compiled_Call(method meth)
2171   %{
2172     // JAVA COMPILED CALL
2173     int disp = in_bytes(Method:: from_compiled_offset());
2174 
2175     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2176     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2177 
2178     // callq *disp(%rax)
2179     cbuf.set_insts_mark();
2180     $$$emit8$primary;
2181     if (disp < 0x80) {
2182       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2183       emit_d8(cbuf, disp); // Displacement
2184     } else {
2185       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2186       emit_d32(cbuf, disp); // Displacement
2187     }
2188   %}
2189 
2190   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2191   %{
2192     // SAL, SAR, SHR
2193     int dstenc = $dst$$reg;
2194     if (dstenc >= 8) {
2195       emit_opcode(cbuf, Assembler::REX_B);
2196       dstenc -= 8;
2197     }
2198     $$$emit8$primary;
2199     emit_rm(cbuf, 0x3, $secondary, dstenc);
2200     $$$emit8$shift$$constant;
2201   %}
2202 
2203   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2204   %{
2205     // SAL, SAR, SHR
2206     int dstenc = $dst$$reg;
2207     if (dstenc < 8) {
2208       emit_opcode(cbuf, Assembler::REX_W);
2209     } else {
2210       emit_opcode(cbuf, Assembler::REX_WB);
2211       dstenc -= 8;
2212     }
2213     $$$emit8$primary;
2214     emit_rm(cbuf, 0x3, $secondary, dstenc);
2215     $$$emit8$shift$$constant;
2216   %}
2217 
2218   enc_class load_immI(rRegI dst, immI src)
2219   %{
2220     int dstenc = $dst$$reg;
2221     if (dstenc >= 8) {
2222       emit_opcode(cbuf, Assembler::REX_B);
2223       dstenc -= 8;
2224     }
2225     emit_opcode(cbuf, 0xB8 | dstenc);
2226     $$$emit32$src$$constant;
2227   %}
2228 
2229   enc_class load_immL(rRegL dst, immL src)
2230   %{
2231     int dstenc = $dst$$reg;
2232     if (dstenc < 8) {
2233       emit_opcode(cbuf, Assembler::REX_W);
2234     } else {
2235       emit_opcode(cbuf, Assembler::REX_WB);
2236       dstenc -= 8;
2237     }
2238     emit_opcode(cbuf, 0xB8 | dstenc);
2239     emit_d64(cbuf, $src$$constant);
2240   %}
2241 
2242   enc_class load_immUL32(rRegL dst, immUL32 src)
2243   %{
2244     // same as load_immI, but this time we care about zeroes in the high word
2245     int dstenc = $dst$$reg;
2246     if (dstenc >= 8) {
2247       emit_opcode(cbuf, Assembler::REX_B);
2248       dstenc -= 8;
2249     }
2250     emit_opcode(cbuf, 0xB8 | dstenc);
2251     $$$emit32$src$$constant;
2252   %}
2253 
2254   enc_class load_immL32(rRegL dst, immL32 src)
2255   %{
2256     int dstenc = $dst$$reg;
2257     if (dstenc < 8) {
2258       emit_opcode(cbuf, Assembler::REX_W);
2259     } else {
2260       emit_opcode(cbuf, Assembler::REX_WB);
2261       dstenc -= 8;
2262     }
2263     emit_opcode(cbuf, 0xC7);
2264     emit_rm(cbuf, 0x03, 0x00, dstenc);
2265     $$$emit32$src$$constant;
2266   %}
2267 
2268   enc_class load_immP31(rRegP dst, immP32 src)
2269   %{
2270     // same as load_immI, but this time we care about zeroes in the high word
2271     int dstenc = $dst$$reg;
2272     if (dstenc >= 8) {
2273       emit_opcode(cbuf, Assembler::REX_B);
2274       dstenc -= 8;
2275     }
2276     emit_opcode(cbuf, 0xB8 | dstenc);
2277     $$$emit32$src$$constant;
2278   %}
2279 
2280   enc_class load_immP(rRegP dst, immP src)
2281   %{
2282     int dstenc = $dst$$reg;
2283     if (dstenc < 8) {
2284       emit_opcode(cbuf, Assembler::REX_W);
2285     } else {
2286       emit_opcode(cbuf, Assembler::REX_WB);
2287       dstenc -= 8;
2288     }
2289     emit_opcode(cbuf, 0xB8 | dstenc);
2290     // This next line should be generated from ADLC
2291     if ($src->constant_reloc() != relocInfo::none) {
2292       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2293     } else {
2294       emit_d64(cbuf, $src$$constant);
2295     }
2296   %}
2297 
2298   enc_class Con32(immI src)
2299   %{
2300     // Output immediate
2301     $$$emit32$src$$constant;
2302   %}
2303 
2304   enc_class Con32F_as_bits(immF src)
2305   %{
2306     // Output Float immediate bits
2307     jfloat jf = $src$$constant;
2308     jint jf_as_bits = jint_cast(jf);
2309     emit_d32(cbuf, jf_as_bits);
2310   %}
2311 
2312   enc_class Con16(immI src)
2313   %{
2314     // Output immediate
2315     $$$emit16$src$$constant;
2316   %}
2317 
2318   // How is this different from Con32??? XXX
2319   enc_class Con_d32(immI src)
2320   %{
2321     emit_d32(cbuf,$src$$constant);
2322   %}
2323 
2324   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2325     // Output immediate memory reference
2326     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2327     emit_d32(cbuf, 0x00);
2328   %}
2329 
2330   enc_class lock_prefix()
2331   %{
2332     emit_opcode(cbuf, 0xF0); // lock
2333   %}
2334 
2335   enc_class REX_mem(memory mem)
2336   %{
2337     if ($mem$$base >= 8) {
2338       if ($mem$$index < 8) {
2339         emit_opcode(cbuf, Assembler::REX_B);
2340       } else {
2341         emit_opcode(cbuf, Assembler::REX_XB);
2342       }
2343     } else {
2344       if ($mem$$index >= 8) {
2345         emit_opcode(cbuf, Assembler::REX_X);
2346       }
2347     }
2348   %}
2349 
2350   enc_class REX_mem_wide(memory mem)
2351   %{
2352     if ($mem$$base >= 8) {
2353       if ($mem$$index < 8) {
2354         emit_opcode(cbuf, Assembler::REX_WB);
2355       } else {
2356         emit_opcode(cbuf, Assembler::REX_WXB);
2357       }
2358     } else {
2359       if ($mem$$index < 8) {
2360         emit_opcode(cbuf, Assembler::REX_W);
2361       } else {
2362         emit_opcode(cbuf, Assembler::REX_WX);
2363       }
2364     }
2365   %}
2366 
2367   // for byte regs
2368   enc_class REX_breg(rRegI reg)
2369   %{
2370     if ($reg$$reg >= 4) {
2371       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2372     }
2373   %}
2374 
2375   // for byte regs
2376   enc_class REX_reg_breg(rRegI dst, rRegI src)
2377   %{
2378     if ($dst$$reg < 8) {
2379       if ($src$$reg >= 4) {
2380         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2381       }
2382     } else {
2383       if ($src$$reg < 8) {
2384         emit_opcode(cbuf, Assembler::REX_R);
2385       } else {
2386         emit_opcode(cbuf, Assembler::REX_RB);
2387       }
2388     }
2389   %}
2390 
2391   // for byte regs
2392   enc_class REX_breg_mem(rRegI reg, memory mem)
2393   %{
2394     if ($reg$$reg < 8) {
2395       if ($mem$$base < 8) {
2396         if ($mem$$index >= 8) {
2397           emit_opcode(cbuf, Assembler::REX_X);
2398         } else if ($reg$$reg >= 4) {
2399           emit_opcode(cbuf, Assembler::REX);
2400         }
2401       } else {
2402         if ($mem$$index < 8) {
2403           emit_opcode(cbuf, Assembler::REX_B);
2404         } else {
2405           emit_opcode(cbuf, Assembler::REX_XB);
2406         }
2407       }
2408     } else {
2409       if ($mem$$base < 8) {
2410         if ($mem$$index < 8) {
2411           emit_opcode(cbuf, Assembler::REX_R);
2412         } else {
2413           emit_opcode(cbuf, Assembler::REX_RX);
2414         }
2415       } else {
2416         if ($mem$$index < 8) {
2417           emit_opcode(cbuf, Assembler::REX_RB);
2418         } else {
2419           emit_opcode(cbuf, Assembler::REX_RXB);
2420         }
2421       }
2422     }
2423   %}
2424 
2425   enc_class REX_reg(rRegI reg)
2426   %{
2427     if ($reg$$reg >= 8) {
2428       emit_opcode(cbuf, Assembler::REX_B);
2429     }
2430   %}
2431 
2432   enc_class REX_reg_wide(rRegI reg)
2433   %{
2434     if ($reg$$reg < 8) {
2435       emit_opcode(cbuf, Assembler::REX_W);
2436     } else {
2437       emit_opcode(cbuf, Assembler::REX_WB);
2438     }
2439   %}
2440 
2441   enc_class REX_reg_reg(rRegI dst, rRegI src)
2442   %{
2443     if ($dst$$reg < 8) {
2444       if ($src$$reg >= 8) {
2445         emit_opcode(cbuf, Assembler::REX_B);
2446       }
2447     } else {
2448       if ($src$$reg < 8) {
2449         emit_opcode(cbuf, Assembler::REX_R);
2450       } else {
2451         emit_opcode(cbuf, Assembler::REX_RB);
2452       }
2453     }
2454   %}
2455 
2456   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2457   %{
2458     if ($dst$$reg < 8) {
2459       if ($src$$reg < 8) {
2460         emit_opcode(cbuf, Assembler::REX_W);
2461       } else {
2462         emit_opcode(cbuf, Assembler::REX_WB);
2463       }
2464     } else {
2465       if ($src$$reg < 8) {
2466         emit_opcode(cbuf, Assembler::REX_WR);
2467       } else {
2468         emit_opcode(cbuf, Assembler::REX_WRB);
2469       }
2470     }
2471   %}
2472 
2473   enc_class REX_reg_mem(rRegI reg, memory mem)
2474   %{
2475     if ($reg$$reg < 8) {
2476       if ($mem$$base < 8) {
2477         if ($mem$$index >= 8) {
2478           emit_opcode(cbuf, Assembler::REX_X);
2479         }
2480       } else {
2481         if ($mem$$index < 8) {
2482           emit_opcode(cbuf, Assembler::REX_B);
2483         } else {
2484           emit_opcode(cbuf, Assembler::REX_XB);
2485         }
2486       }
2487     } else {
2488       if ($mem$$base < 8) {
2489         if ($mem$$index < 8) {
2490           emit_opcode(cbuf, Assembler::REX_R);
2491         } else {
2492           emit_opcode(cbuf, Assembler::REX_RX);
2493         }
2494       } else {
2495         if ($mem$$index < 8) {
2496           emit_opcode(cbuf, Assembler::REX_RB);
2497         } else {
2498           emit_opcode(cbuf, Assembler::REX_RXB);
2499         }
2500       }
2501     }
2502   %}
2503 
2504   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2505   %{
2506     if ($reg$$reg < 8) {
2507       if ($mem$$base < 8) {
2508         if ($mem$$index < 8) {
2509           emit_opcode(cbuf, Assembler::REX_W);
2510         } else {
2511           emit_opcode(cbuf, Assembler::REX_WX);
2512         }
2513       } else {
2514         if ($mem$$index < 8) {
2515           emit_opcode(cbuf, Assembler::REX_WB);
2516         } else {
2517           emit_opcode(cbuf, Assembler::REX_WXB);
2518         }
2519       }
2520     } else {
2521       if ($mem$$base < 8) {
2522         if ($mem$$index < 8) {
2523           emit_opcode(cbuf, Assembler::REX_WR);
2524         } else {
2525           emit_opcode(cbuf, Assembler::REX_WRX);
2526         }
2527       } else {
2528         if ($mem$$index < 8) {
2529           emit_opcode(cbuf, Assembler::REX_WRB);
2530         } else {
2531           emit_opcode(cbuf, Assembler::REX_WRXB);
2532         }
2533       }
2534     }
2535   %}
2536 
2537   enc_class reg_mem(rRegI ereg, memory mem)
2538   %{
2539     // High registers handle in encode_RegMem
2540     int reg = $ereg$$reg;
2541     int base = $mem$$base;
2542     int index = $mem$$index;
2543     int scale = $mem$$scale;
2544     int disp = $mem$$disp;
2545     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2546 
2547     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2548   %}
2549 
2550   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2551   %{
2552     int rm_byte_opcode = $rm_opcode$$constant;
2553 
2554     // High registers handle in encode_RegMem
2555     int base = $mem$$base;
2556     int index = $mem$$index;
2557     int scale = $mem$$scale;
2558     int displace = $mem$$disp;
2559 
2560     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2561                                             // working with static
2562                                             // globals
2563     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2564                   disp_reloc);
2565   %}
2566 
2567   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2568   %{
2569     int reg_encoding = $dst$$reg;
2570     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2571     int index        = 0x04;            // 0x04 indicates no index
2572     int scale        = 0x00;            // 0x00 indicates no scale
2573     int displace     = $src1$$constant; // 0x00 indicates no displacement
2574     relocInfo::relocType disp_reloc = relocInfo::none;
2575     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2576                   disp_reloc);
2577   %}
2578 
2579   enc_class neg_reg(rRegI dst)
2580   %{
2581     int dstenc = $dst$$reg;
2582     if (dstenc >= 8) {
2583       emit_opcode(cbuf, Assembler::REX_B);
2584       dstenc -= 8;
2585     }
2586     // NEG $dst
2587     emit_opcode(cbuf, 0xF7);
2588     emit_rm(cbuf, 0x3, 0x03, dstenc);
2589   %}
2590 
2591   enc_class neg_reg_wide(rRegI dst)
2592   %{
2593     int dstenc = $dst$$reg;
2594     if (dstenc < 8) {
2595       emit_opcode(cbuf, Assembler::REX_W);
2596     } else {
2597       emit_opcode(cbuf, Assembler::REX_WB);
2598       dstenc -= 8;
2599     }
2600     // NEG $dst
2601     emit_opcode(cbuf, 0xF7);
2602     emit_rm(cbuf, 0x3, 0x03, dstenc);
2603   %}
2604 
2605   enc_class setLT_reg(rRegI dst)
2606   %{
2607     int dstenc = $dst$$reg;
2608     if (dstenc >= 8) {
2609       emit_opcode(cbuf, Assembler::REX_B);
2610       dstenc -= 8;
2611     } else if (dstenc >= 4) {
2612       emit_opcode(cbuf, Assembler::REX);
2613     }
2614     // SETLT $dst
2615     emit_opcode(cbuf, 0x0F);
2616     emit_opcode(cbuf, 0x9C);
2617     emit_rm(cbuf, 0x3, 0x0, dstenc);
2618   %}
2619 
2620   enc_class setNZ_reg(rRegI dst)
2621   %{
2622     int dstenc = $dst$$reg;
2623     if (dstenc >= 8) {
2624       emit_opcode(cbuf, Assembler::REX_B);
2625       dstenc -= 8;
2626     } else if (dstenc >= 4) {
2627       emit_opcode(cbuf, Assembler::REX);
2628     }
2629     // SETNZ $dst
2630     emit_opcode(cbuf, 0x0F);
2631     emit_opcode(cbuf, 0x95);
2632     emit_rm(cbuf, 0x3, 0x0, dstenc);
2633   %}
2634 
2635 
2636   // Compare the lonogs and set -1, 0, or 1 into dst
2637   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2638   %{
2639     int src1enc = $src1$$reg;
2640     int src2enc = $src2$$reg;
2641     int dstenc = $dst$$reg;
2642 
2643     // cmpq $src1, $src2
2644     if (src1enc < 8) {
2645       if (src2enc < 8) {
2646         emit_opcode(cbuf, Assembler::REX_W);
2647       } else {
2648         emit_opcode(cbuf, Assembler::REX_WB);
2649       }
2650     } else {
2651       if (src2enc < 8) {
2652         emit_opcode(cbuf, Assembler::REX_WR);
2653       } else {
2654         emit_opcode(cbuf, Assembler::REX_WRB);
2655       }
2656     }
2657     emit_opcode(cbuf, 0x3B);
2658     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2659 
2660     // movl $dst, -1
2661     if (dstenc >= 8) {
2662       emit_opcode(cbuf, Assembler::REX_B);
2663     }
2664     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2665     emit_d32(cbuf, -1);
2666 
2667     // jl,s done
2668     emit_opcode(cbuf, 0x7C);
2669     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2670 
2671     // setne $dst
2672     if (dstenc >= 4) {
2673       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2674     }
2675     emit_opcode(cbuf, 0x0F);
2676     emit_opcode(cbuf, 0x95);
2677     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2678 
2679     // movzbl $dst, $dst
2680     if (dstenc >= 4) {
2681       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2682     }
2683     emit_opcode(cbuf, 0x0F);
2684     emit_opcode(cbuf, 0xB6);
2685     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2686   %}
2687 
2688   enc_class Push_ResultXD(regD dst) %{
2689     MacroAssembler _masm(&cbuf);
2690     __ fstp_d(Address(rsp, 0));
2691     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2692     __ addptr(rsp, 8);
2693   %}
2694 
2695   enc_class Push_SrcXD(regD src) %{
2696     MacroAssembler _masm(&cbuf);
2697     __ subptr(rsp, 8);
2698     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2699     __ fld_d(Address(rsp, 0));
2700   %}
2701 
2702 
2703   enc_class enc_rethrow()
2704   %{
2705     cbuf.set_insts_mark();
2706     emit_opcode(cbuf, 0xE9); // jmp entry
2707     emit_d32_reloc(cbuf,
2708                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2709                    runtime_call_Relocation::spec(),
2710                    RELOC_DISP32);
2711   %}
2712 
2713 %}
2714 
2715 
2716 
2717 //----------FRAME--------------------------------------------------------------
2718 // Definition of frame structure and management information.
2719 //
2720 //  S T A C K   L A Y O U T    Allocators stack-slot number
2721 //                             |   (to get allocators register number
2722 //  G  Owned by    |        |  v    add OptoReg::stack0())
2723 //  r   CALLER     |        |
2724 //  o     |        +--------+      pad to even-align allocators stack-slot
2725 //  w     V        |  pad0  |        numbers; owned by CALLER
2726 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2727 //  h     ^        |   in   |  5
2728 //        |        |  args  |  4   Holes in incoming args owned by SELF
2729 //  |     |        |        |  3
2730 //  |     |        +--------+
2731 //  V     |        | old out|      Empty on Intel, window on Sparc
2732 //        |    old |preserve|      Must be even aligned.
2733 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2734 //        |        |   in   |  3   area for Intel ret address
2735 //     Owned by    |preserve|      Empty on Sparc.
2736 //       SELF      +--------+
2737 //        |        |  pad2  |  2   pad to align old SP
2738 //        |        +--------+  1
2739 //        |        | locks  |  0
2740 //        |        +--------+----> OptoReg::stack0(), even aligned
2741 //        |        |  pad1  | 11   pad to align new SP
2742 //        |        +--------+
2743 //        |        |        | 10
2744 //        |        | spills |  9   spills
2745 //        V        |        |  8   (pad0 slot for callee)
2746 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2747 //        ^        |  out   |  7
2748 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2749 //     Owned by    +--------+
2750 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2751 //        |    new |preserve|      Must be even-aligned.
2752 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2753 //        |        |        |
2754 //
2755 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2756 //         known from SELF's arguments and the Java calling convention.
2757 //         Region 6-7 is determined per call site.
2758 // Note 2: If the calling convention leaves holes in the incoming argument
2759 //         area, those holes are owned by SELF.  Holes in the outgoing area
2760 //         are owned by the CALLEE.  Holes should not be nessecary in the
2761 //         incoming area, as the Java calling convention is completely under
2762 //         the control of the AD file.  Doubles can be sorted and packed to
2763 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2764 //         varargs C calling conventions.
2765 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2766 //         even aligned with pad0 as needed.
2767 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2768 //         region 6-11 is even aligned; it may be padded out more so that
2769 //         the region from SP to FP meets the minimum stack alignment.
2770 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2771 //         alignment.  Region 11, pad1, may be dynamically extended so that
2772 //         SP meets the minimum alignment.
2773 
2774 frame
2775 %{
2776   // What direction does stack grow in (assumed to be same for C & Java)
2777   stack_direction(TOWARDS_LOW);
2778 
2779   // These three registers define part of the calling convention
2780   // between compiled code and the interpreter.
2781   inline_cache_reg(RAX);                // Inline Cache Register
2782   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2783                                         // calling interpreter
2784 
2785   // Optional: name the operand used by cisc-spilling to access
2786   // [stack_pointer + offset]
2787   cisc_spilling_operand_name(indOffset32);
2788 
2789   // Number of stack slots consumed by locking an object
2790   sync_stack_slots(2);
2791 
2792   // Compiled code's Frame Pointer
2793   frame_pointer(RSP);
2794 
2795   // Interpreter stores its frame pointer in a register which is
2796   // stored to the stack by I2CAdaptors.
2797   // I2CAdaptors convert from interpreted java to compiled java.
2798   interpreter_frame_pointer(RBP);
2799 
2800   // Stack alignment requirement
2801   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2802 
2803   // Number of stack slots between incoming argument block and the start of
2804   // a new frame.  The PROLOG must add this many slots to the stack.  The
2805   // EPILOG must remove this many slots.  amd64 needs two slots for
2806   // return address.
2807   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2808 
2809   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2810   // for calls to C.  Supports the var-args backing area for register parms.
2811   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2812 
2813   // The after-PROLOG location of the return address.  Location of
2814   // return address specifies a type (REG or STACK) and a number
2815   // representing the register number (i.e. - use a register name) or
2816   // stack slot.
2817   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2818   // Otherwise, it is above the locks and verification slot and alignment word
2819   return_addr(STACK - 2 +
2820               align_up((Compile::current()->in_preserve_stack_slots() +
2821                         Compile::current()->fixed_slots()),
2822                        stack_alignment_in_slots()));
2823 
2824   // Body of function which returns an integer array locating
2825   // arguments either in registers or in stack slots.  Passed an array
2826   // of ideal registers called "sig" and a "length" count.  Stack-slot
2827   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2828   // arguments for a CALLEE.  Incoming stack arguments are
2829   // automatically biased by the preserve_stack_slots field above.
2830 
2831   calling_convention
2832   %{
2833     // No difference between ingoing/outgoing just pass false
2834     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2835   %}
2836 
2837   c_calling_convention
2838   %{
2839     // This is obviously always outgoing
2840     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2841   %}
2842 
2843   // Location of compiled Java return values.  Same as C for now.
2844   return_value
2845   %{
2846     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2847            "only return normal values");
2848 
2849     static const int lo[Op_RegL + 1] = {
2850       0,
2851       0,
2852       RAX_num,  // Op_RegN
2853       RAX_num,  // Op_RegI
2854       RAX_num,  // Op_RegP
2855       XMM0_num, // Op_RegF
2856       XMM0_num, // Op_RegD
2857       RAX_num   // Op_RegL
2858     };
2859     static const int hi[Op_RegL + 1] = {
2860       0,
2861       0,
2862       OptoReg::Bad, // Op_RegN
2863       OptoReg::Bad, // Op_RegI
2864       RAX_H_num,    // Op_RegP
2865       OptoReg::Bad, // Op_RegF
2866       XMM0b_num,    // Op_RegD
2867       RAX_H_num     // Op_RegL
2868     };
2869     // Excluded flags and vector registers.
2870     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2871     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2872   %}
2873 %}
2874 
2875 //----------ATTRIBUTES---------------------------------------------------------
2876 //----------Operand Attributes-------------------------------------------------
2877 op_attrib op_cost(0);        // Required cost attribute
2878 
2879 //----------Instruction Attributes---------------------------------------------
2880 ins_attrib ins_cost(100);       // Required cost attribute
2881 ins_attrib ins_size(8);         // Required size attribute (in bits)
2882 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2883                                 // a non-matching short branch variant
2884                                 // of some long branch?
2885 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2886                                 // be a power of 2) specifies the
2887                                 // alignment that some part of the
2888                                 // instruction (not necessarily the
2889                                 // start) requires.  If > 1, a
2890                                 // compute_padding() function must be
2891                                 // provided for the instruction
2892 
2893 //----------OPERANDS-----------------------------------------------------------
2894 // Operand definitions must precede instruction definitions for correct parsing
2895 // in the ADLC because operands constitute user defined types which are used in
2896 // instruction definitions.
2897 
2898 //----------Simple Operands----------------------------------------------------
2899 // Immediate Operands
2900 // Integer Immediate
2901 operand immI()
2902 %{
2903   match(ConI);
2904 
2905   op_cost(10);
2906   format %{ %}
2907   interface(CONST_INTER);
2908 %}
2909 
2910 // Constant for test vs zero
2911 operand immI0()
2912 %{
2913   predicate(n->get_int() == 0);
2914   match(ConI);
2915 
2916   op_cost(0);
2917   format %{ %}
2918   interface(CONST_INTER);
2919 %}
2920 
2921 // Constant for increment
2922 operand immI1()
2923 %{
2924   predicate(n->get_int() == 1);
2925   match(ConI);
2926 
2927   op_cost(0);
2928   format %{ %}
2929   interface(CONST_INTER);
2930 %}
2931 
2932 // Constant for decrement
2933 operand immI_M1()
2934 %{
2935   predicate(n->get_int() == -1);
2936   match(ConI);
2937 
2938   op_cost(0);
2939   format %{ %}
2940   interface(CONST_INTER);
2941 %}
2942 
2943 // Valid scale values for addressing modes
2944 operand immI2()
2945 %{
2946   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2947   match(ConI);
2948 
2949   format %{ %}
2950   interface(CONST_INTER);
2951 %}
2952 
2953 operand immI8()
2954 %{
2955   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2956   match(ConI);
2957 
2958   op_cost(5);
2959   format %{ %}
2960   interface(CONST_INTER);
2961 %}
2962 
2963 operand immU8()
2964 %{
2965   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2966   match(ConI);
2967 
2968   op_cost(5);
2969   format %{ %}
2970   interface(CONST_INTER);
2971 %}
2972 
2973 operand immI16()
2974 %{
2975   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2976   match(ConI);
2977 
2978   op_cost(10);
2979   format %{ %}
2980   interface(CONST_INTER);
2981 %}
2982 
2983 // Int Immediate non-negative
2984 operand immU31()
2985 %{
2986   predicate(n->get_int() >= 0);
2987   match(ConI);
2988 
2989   op_cost(0);
2990   format %{ %}
2991   interface(CONST_INTER);
2992 %}
2993 
2994 // Constant for long shifts
2995 operand immI_32()
2996 %{
2997   predicate( n->get_int() == 32 );
2998   match(ConI);
2999 
3000   op_cost(0);
3001   format %{ %}
3002   interface(CONST_INTER);
3003 %}
3004 
3005 // Constant for long shifts
3006 operand immI_64()
3007 %{
3008   predicate( n->get_int() == 64 );
3009   match(ConI);
3010 
3011   op_cost(0);
3012   format %{ %}
3013   interface(CONST_INTER);
3014 %}
3015 
3016 // Pointer Immediate
3017 operand immP()
3018 %{
3019   match(ConP);
3020 
3021   op_cost(10);
3022   format %{ %}
3023   interface(CONST_INTER);
3024 %}
3025 
3026 // NULL Pointer Immediate
3027 operand immP0()
3028 %{
3029   predicate(n->get_ptr() == 0);
3030   match(ConP);
3031 
3032   op_cost(5);
3033   format %{ %}
3034   interface(CONST_INTER);
3035 %}
3036 
3037 // Pointer Immediate
3038 operand immN() %{
3039   match(ConN);
3040 
3041   op_cost(10);
3042   format %{ %}
3043   interface(CONST_INTER);
3044 %}
3045 
3046 operand immNKlass() %{
3047   match(ConNKlass);
3048 
3049   op_cost(10);
3050   format %{ %}
3051   interface(CONST_INTER);
3052 %}
3053 
3054 // NULL Pointer Immediate
3055 operand immN0() %{
3056   predicate(n->get_narrowcon() == 0);
3057   match(ConN);
3058 
3059   op_cost(5);
3060   format %{ %}
3061   interface(CONST_INTER);
3062 %}
3063 
3064 operand immP31()
3065 %{
3066   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3067             && (n->get_ptr() >> 31) == 0);
3068   match(ConP);
3069 
3070   op_cost(5);
3071   format %{ %}
3072   interface(CONST_INTER);
3073 %}
3074 
3075 
3076 // Long Immediate
3077 operand immL()
3078 %{
3079   match(ConL);
3080 
3081   op_cost(20);
3082   format %{ %}
3083   interface(CONST_INTER);
3084 %}
3085 
3086 // Long Immediate 8-bit
3087 operand immL8()
3088 %{
3089   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3090   match(ConL);
3091 
3092   op_cost(5);
3093   format %{ %}
3094   interface(CONST_INTER);
3095 %}
3096 
3097 // Long Immediate 32-bit unsigned
3098 operand immUL32()
3099 %{
3100   predicate(n->get_long() == (unsigned int) (n->get_long()));
3101   match(ConL);
3102 
3103   op_cost(10);
3104   format %{ %}
3105   interface(CONST_INTER);
3106 %}
3107 
3108 // Long Immediate 32-bit signed
3109 operand immL32()
3110 %{
3111   predicate(n->get_long() == (int) (n->get_long()));
3112   match(ConL);
3113 
3114   op_cost(15);
3115   format %{ %}
3116   interface(CONST_INTER);
3117 %}
3118 
3119 // Long Immediate zero
3120 operand immL0()
3121 %{
3122   predicate(n->get_long() == 0L);
3123   match(ConL);
3124 
3125   op_cost(10);
3126   format %{ %}
3127   interface(CONST_INTER);
3128 %}
3129 
3130 // Constant for increment
3131 operand immL1()
3132 %{
3133   predicate(n->get_long() == 1);
3134   match(ConL);
3135 
3136   format %{ %}
3137   interface(CONST_INTER);
3138 %}
3139 
3140 // Constant for decrement
3141 operand immL_M1()
3142 %{
3143   predicate(n->get_long() == -1);
3144   match(ConL);
3145 
3146   format %{ %}
3147   interface(CONST_INTER);
3148 %}
3149 
3150 // Long Immediate: the value 10
3151 operand immL10()
3152 %{
3153   predicate(n->get_long() == 10);
3154   match(ConL);
3155 
3156   format %{ %}
3157   interface(CONST_INTER);
3158 %}
3159 
3160 // Long immediate from 0 to 127.
3161 // Used for a shorter form of long mul by 10.
3162 operand immL_127()
3163 %{
3164   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3165   match(ConL);
3166 
3167   op_cost(10);
3168   format %{ %}
3169   interface(CONST_INTER);
3170 %}
3171 
3172 // Long Immediate: low 32-bit mask
3173 operand immL_32bits()
3174 %{
3175   predicate(n->get_long() == 0xFFFFFFFFL);
3176   match(ConL);
3177   op_cost(20);
3178 
3179   format %{ %}
3180   interface(CONST_INTER);
3181 %}
3182 
3183 // Float Immediate zero
3184 operand immF0()
3185 %{
3186   predicate(jint_cast(n->getf()) == 0);
3187   match(ConF);
3188 
3189   op_cost(5);
3190   format %{ %}
3191   interface(CONST_INTER);
3192 %}
3193 
3194 // Float Immediate
3195 operand immF()
3196 %{
3197   match(ConF);
3198 
3199   op_cost(15);
3200   format %{ %}
3201   interface(CONST_INTER);
3202 %}
3203 
3204 // Double Immediate zero
3205 operand immD0()
3206 %{
3207   predicate(jlong_cast(n->getd()) == 0);
3208   match(ConD);
3209 
3210   op_cost(5);
3211   format %{ %}
3212   interface(CONST_INTER);
3213 %}
3214 
3215 // Double Immediate
3216 operand immD()
3217 %{
3218   match(ConD);
3219 
3220   op_cost(15);
3221   format %{ %}
3222   interface(CONST_INTER);
3223 %}
3224 
3225 // Immediates for special shifts (sign extend)
3226 
3227 // Constants for increment
3228 operand immI_16()
3229 %{
3230   predicate(n->get_int() == 16);
3231   match(ConI);
3232 
3233   format %{ %}
3234   interface(CONST_INTER);
3235 %}
3236 
3237 operand immI_24()
3238 %{
3239   predicate(n->get_int() == 24);
3240   match(ConI);
3241 
3242   format %{ %}
3243   interface(CONST_INTER);
3244 %}
3245 
3246 // Constant for byte-wide masking
3247 operand immI_255()
3248 %{
3249   predicate(n->get_int() == 255);
3250   match(ConI);
3251 
3252   format %{ %}
3253   interface(CONST_INTER);
3254 %}
3255 
3256 // Constant for short-wide masking
3257 operand immI_65535()
3258 %{
3259   predicate(n->get_int() == 65535);
3260   match(ConI);
3261 
3262   format %{ %}
3263   interface(CONST_INTER);
3264 %}
3265 
3266 // Constant for byte-wide masking
3267 operand immL_255()
3268 %{
3269   predicate(n->get_long() == 255);
3270   match(ConL);
3271 
3272   format %{ %}
3273   interface(CONST_INTER);
3274 %}
3275 
3276 // Constant for short-wide masking
3277 operand immL_65535()
3278 %{
3279   predicate(n->get_long() == 65535);
3280   match(ConL);
3281 
3282   format %{ %}
3283   interface(CONST_INTER);
3284 %}
3285 
3286 // Register Operands
3287 // Integer Register
3288 operand rRegI()
3289 %{
3290   constraint(ALLOC_IN_RC(int_reg));
3291   match(RegI);
3292 
3293   match(rax_RegI);
3294   match(rbx_RegI);
3295   match(rcx_RegI);
3296   match(rdx_RegI);
3297   match(rdi_RegI);
3298 
3299   format %{ %}
3300   interface(REG_INTER);
3301 %}
3302 
3303 // Special Registers
3304 operand rax_RegI()
3305 %{
3306   constraint(ALLOC_IN_RC(int_rax_reg));
3307   match(RegI);
3308   match(rRegI);
3309 
3310   format %{ "RAX" %}
3311   interface(REG_INTER);
3312 %}
3313 
3314 // Special Registers
3315 operand rbx_RegI()
3316 %{
3317   constraint(ALLOC_IN_RC(int_rbx_reg));
3318   match(RegI);
3319   match(rRegI);
3320 
3321   format %{ "RBX" %}
3322   interface(REG_INTER);
3323 %}
3324 
3325 operand rcx_RegI()
3326 %{
3327   constraint(ALLOC_IN_RC(int_rcx_reg));
3328   match(RegI);
3329   match(rRegI);
3330 
3331   format %{ "RCX" %}
3332   interface(REG_INTER);
3333 %}
3334 
3335 operand rdx_RegI()
3336 %{
3337   constraint(ALLOC_IN_RC(int_rdx_reg));
3338   match(RegI);
3339   match(rRegI);
3340 
3341   format %{ "RDX" %}
3342   interface(REG_INTER);
3343 %}
3344 
3345 operand rdi_RegI()
3346 %{
3347   constraint(ALLOC_IN_RC(int_rdi_reg));
3348   match(RegI);
3349   match(rRegI);
3350 
3351   format %{ "RDI" %}
3352   interface(REG_INTER);
3353 %}
3354 
3355 operand no_rcx_RegI()
3356 %{
3357   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3358   match(RegI);
3359   match(rax_RegI);
3360   match(rbx_RegI);
3361   match(rdx_RegI);
3362   match(rdi_RegI);
3363 
3364   format %{ %}
3365   interface(REG_INTER);
3366 %}
3367 
3368 operand no_rax_rdx_RegI()
3369 %{
3370   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3371   match(RegI);
3372   match(rbx_RegI);
3373   match(rcx_RegI);
3374   match(rdi_RegI);
3375 
3376   format %{ %}
3377   interface(REG_INTER);
3378 %}
3379 
3380 // Pointer Register
3381 operand any_RegP()
3382 %{
3383   constraint(ALLOC_IN_RC(any_reg));
3384   match(RegP);
3385   match(rax_RegP);
3386   match(rbx_RegP);
3387   match(rdi_RegP);
3388   match(rsi_RegP);
3389   match(rbp_RegP);
3390   match(r15_RegP);
3391   match(rRegP);
3392 
3393   format %{ %}
3394   interface(REG_INTER);
3395 %}
3396 
3397 operand rRegP()
3398 %{
3399   constraint(ALLOC_IN_RC(ptr_reg));
3400   match(RegP);
3401   match(rax_RegP);
3402   match(rbx_RegP);
3403   match(rdi_RegP);
3404   match(rsi_RegP);
3405   match(rbp_RegP);  // See Q&A below about
3406   match(r15_RegP);  // r15_RegP and rbp_RegP.
3407 
3408   format %{ %}
3409   interface(REG_INTER);
3410 %}
3411 
3412 operand rRegN() %{
3413   constraint(ALLOC_IN_RC(int_reg));
3414   match(RegN);
3415 
3416   format %{ %}
3417   interface(REG_INTER);
3418 %}
3419 
3420 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3421 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3422 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3423 // The output of an instruction is controlled by the allocator, which respects
3424 // register class masks, not match rules.  Unless an instruction mentions
3425 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3426 // by the allocator as an input.
3427 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3428 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3429 // result, RBP is not included in the output of the instruction either.
3430 
3431 operand no_rax_RegP()
3432 %{
3433   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3434   match(RegP);
3435   match(rbx_RegP);
3436   match(rsi_RegP);
3437   match(rdi_RegP);
3438 
3439   format %{ %}
3440   interface(REG_INTER);
3441 %}
3442 
3443 // This operand is not allowed to use RBP even if
3444 // RBP is not used to hold the frame pointer.
3445 operand no_rbp_RegP()
3446 %{
3447   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3448   match(RegP);
3449   match(rbx_RegP);
3450   match(rsi_RegP);
3451   match(rdi_RegP);
3452 
3453   format %{ %}
3454   interface(REG_INTER);
3455 %}
3456 
3457 operand no_rax_rbx_RegP()
3458 %{
3459   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3460   match(RegP);
3461   match(rsi_RegP);
3462   match(rdi_RegP);
3463 
3464   format %{ %}
3465   interface(REG_INTER);
3466 %}
3467 
3468 // Special Registers
3469 // Return a pointer value
3470 operand rax_RegP()
3471 %{
3472   constraint(ALLOC_IN_RC(ptr_rax_reg));
3473   match(RegP);
3474   match(rRegP);
3475 
3476   format %{ %}
3477   interface(REG_INTER);
3478 %}
3479 
3480 // Special Registers
3481 // Return a compressed pointer value
3482 operand rax_RegN()
3483 %{
3484   constraint(ALLOC_IN_RC(int_rax_reg));
3485   match(RegN);
3486   match(rRegN);
3487 
3488   format %{ %}
3489   interface(REG_INTER);
3490 %}
3491 
3492 // Used in AtomicAdd
3493 operand rbx_RegP()
3494 %{
3495   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3496   match(RegP);
3497   match(rRegP);
3498 
3499   format %{ %}
3500   interface(REG_INTER);
3501 %}
3502 
3503 operand rsi_RegP()
3504 %{
3505   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3506   match(RegP);
3507   match(rRegP);
3508 
3509   format %{ %}
3510   interface(REG_INTER);
3511 %}
3512 
3513 // Used in rep stosq
3514 operand rdi_RegP()
3515 %{
3516   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3517   match(RegP);
3518   match(rRegP);
3519 
3520   format %{ %}
3521   interface(REG_INTER);
3522 %}
3523 
3524 operand r15_RegP()
3525 %{
3526   constraint(ALLOC_IN_RC(ptr_r15_reg));
3527   match(RegP);
3528   match(rRegP);
3529 
3530   format %{ %}
3531   interface(REG_INTER);
3532 %}
3533 
3534 operand rRegL()
3535 %{
3536   constraint(ALLOC_IN_RC(long_reg));
3537   match(RegL);
3538   match(rax_RegL);
3539   match(rdx_RegL);
3540 
3541   format %{ %}
3542   interface(REG_INTER);
3543 %}
3544 
3545 // Special Registers
3546 operand no_rax_rdx_RegL()
3547 %{
3548   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3549   match(RegL);
3550   match(rRegL);
3551 
3552   format %{ %}
3553   interface(REG_INTER);
3554 %}
3555 
3556 operand no_rax_RegL()
3557 %{
3558   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3559   match(RegL);
3560   match(rRegL);
3561   match(rdx_RegL);
3562 
3563   format %{ %}
3564   interface(REG_INTER);
3565 %}
3566 
3567 operand no_rcx_RegL()
3568 %{
3569   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3570   match(RegL);
3571   match(rRegL);
3572 
3573   format %{ %}
3574   interface(REG_INTER);
3575 %}
3576 
3577 operand rax_RegL()
3578 %{
3579   constraint(ALLOC_IN_RC(long_rax_reg));
3580   match(RegL);
3581   match(rRegL);
3582 
3583   format %{ "RAX" %}
3584   interface(REG_INTER);
3585 %}
3586 
3587 operand rcx_RegL()
3588 %{
3589   constraint(ALLOC_IN_RC(long_rcx_reg));
3590   match(RegL);
3591   match(rRegL);
3592 
3593   format %{ %}
3594   interface(REG_INTER);
3595 %}
3596 
3597 operand rdx_RegL()
3598 %{
3599   constraint(ALLOC_IN_RC(long_rdx_reg));
3600   match(RegL);
3601   match(rRegL);
3602 
3603   format %{ %}
3604   interface(REG_INTER);
3605 %}
3606 
3607 // Flags register, used as output of compare instructions
3608 operand rFlagsReg()
3609 %{
3610   constraint(ALLOC_IN_RC(int_flags));
3611   match(RegFlags);
3612 
3613   format %{ "RFLAGS" %}
3614   interface(REG_INTER);
3615 %}
3616 
3617 // Flags register, used as output of FLOATING POINT compare instructions
3618 operand rFlagsRegU()
3619 %{
3620   constraint(ALLOC_IN_RC(int_flags));
3621   match(RegFlags);
3622 
3623   format %{ "RFLAGS_U" %}
3624   interface(REG_INTER);
3625 %}
3626 
3627 operand rFlagsRegUCF() %{
3628   constraint(ALLOC_IN_RC(int_flags));
3629   match(RegFlags);
3630   predicate(false);
3631 
3632   format %{ "RFLAGS_U_CF" %}
3633   interface(REG_INTER);
3634 %}
3635 
3636 // Float register operands
3637 operand regF() %{
3638    constraint(ALLOC_IN_RC(float_reg));
3639    match(RegF);
3640 
3641    format %{ %}
3642    interface(REG_INTER);
3643 %}
3644 
3645 // Float register operands
3646 operand legRegF() %{
3647    constraint(ALLOC_IN_RC(float_reg_legacy));
3648    match(RegF);
3649 
3650    format %{ %}
3651    interface(REG_INTER);
3652 %}
3653 
3654 // Float register operands
3655 operand vlRegF() %{
3656    constraint(ALLOC_IN_RC(float_reg_vl));
3657    match(RegF);
3658 
3659    format %{ %}
3660    interface(REG_INTER);
3661 %}
3662 
3663 // Double register operands
3664 operand regD() %{
3665    constraint(ALLOC_IN_RC(double_reg));
3666    match(RegD);
3667 
3668    format %{ %}
3669    interface(REG_INTER);
3670 %}
3671 
3672 // Double register operands
3673 operand legRegD() %{
3674    constraint(ALLOC_IN_RC(double_reg_legacy));
3675    match(RegD);
3676 
3677    format %{ %}
3678    interface(REG_INTER);
3679 %}
3680 
3681 // Double register operands
3682 operand vlRegD() %{
3683    constraint(ALLOC_IN_RC(double_reg_vl));
3684    match(RegD);
3685 
3686    format %{ %}
3687    interface(REG_INTER);
3688 %}
3689 
3690 // Vectors
3691 operand vecS() %{
3692   constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
3693   match(VecS);
3694 
3695   format %{ %}
3696   interface(REG_INTER);
3697 %}
3698 
3699 // Vectors
3700 operand legVecS() %{
3701   constraint(ALLOC_IN_RC(vectors_reg_legacy));
3702   match(VecS);
3703 
3704   format %{ %}
3705   interface(REG_INTER);
3706 %}
3707 
3708 operand vecD() %{
3709   constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
3710   match(VecD);
3711 
3712   format %{ %}
3713   interface(REG_INTER);
3714 %}
3715 
3716 operand legVecD() %{
3717   constraint(ALLOC_IN_RC(vectord_reg_legacy));
3718   match(VecD);
3719 
3720   format %{ %}
3721   interface(REG_INTER);
3722 %}
3723 
3724 operand vecX() %{
3725   constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
3726   match(VecX);
3727 
3728   format %{ %}
3729   interface(REG_INTER);
3730 %}
3731 
3732 operand legVecX() %{
3733   constraint(ALLOC_IN_RC(vectorx_reg_legacy));
3734   match(VecX);
3735 
3736   format %{ %}
3737   interface(REG_INTER);
3738 %}
3739 
3740 operand vecY() %{
3741   constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
3742   match(VecY);
3743 
3744   format %{ %}
3745   interface(REG_INTER);
3746 %}
3747 
3748 operand legVecY() %{
3749   constraint(ALLOC_IN_RC(vectory_reg_legacy));
3750   match(VecY);
3751 
3752   format %{ %}
3753   interface(REG_INTER);
3754 %}
3755 
3756 //----------Memory Operands----------------------------------------------------
3757 // Direct Memory Operand
3758 // operand direct(immP addr)
3759 // %{
3760 //   match(addr);
3761 
3762 //   format %{ "[$addr]" %}
3763 //   interface(MEMORY_INTER) %{
3764 //     base(0xFFFFFFFF);
3765 //     index(0x4);
3766 //     scale(0x0);
3767 //     disp($addr);
3768 //   %}
3769 // %}
3770 
3771 // Indirect Memory Operand
3772 operand indirect(any_RegP reg)
3773 %{
3774   constraint(ALLOC_IN_RC(ptr_reg));
3775   match(reg);
3776 
3777   format %{ "[$reg]" %}
3778   interface(MEMORY_INTER) %{
3779     base($reg);
3780     index(0x4);
3781     scale(0x0);
3782     disp(0x0);
3783   %}
3784 %}
3785 
3786 // Indirect Memory Plus Short Offset Operand
3787 operand indOffset8(any_RegP reg, immL8 off)
3788 %{
3789   constraint(ALLOC_IN_RC(ptr_reg));
3790   match(AddP reg off);
3791 
3792   format %{ "[$reg + $off (8-bit)]" %}
3793   interface(MEMORY_INTER) %{
3794     base($reg);
3795     index(0x4);
3796     scale(0x0);
3797     disp($off);
3798   %}
3799 %}
3800 
3801 // Indirect Memory Plus Long Offset Operand
3802 operand indOffset32(any_RegP reg, immL32 off)
3803 %{
3804   constraint(ALLOC_IN_RC(ptr_reg));
3805   match(AddP reg off);
3806 
3807   format %{ "[$reg + $off (32-bit)]" %}
3808   interface(MEMORY_INTER) %{
3809     base($reg);
3810     index(0x4);
3811     scale(0x0);
3812     disp($off);
3813   %}
3814 %}
3815 
3816 // Indirect Memory Plus Index Register Plus Offset Operand
3817 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3818 %{
3819   constraint(ALLOC_IN_RC(ptr_reg));
3820   match(AddP (AddP reg lreg) off);
3821 
3822   op_cost(10);
3823   format %{"[$reg + $off + $lreg]" %}
3824   interface(MEMORY_INTER) %{
3825     base($reg);
3826     index($lreg);
3827     scale(0x0);
3828     disp($off);
3829   %}
3830 %}
3831 
3832 // Indirect Memory Plus Index Register Plus Offset Operand
3833 operand indIndex(any_RegP reg, rRegL lreg)
3834 %{
3835   constraint(ALLOC_IN_RC(ptr_reg));
3836   match(AddP reg lreg);
3837 
3838   op_cost(10);
3839   format %{"[$reg + $lreg]" %}
3840   interface(MEMORY_INTER) %{
3841     base($reg);
3842     index($lreg);
3843     scale(0x0);
3844     disp(0x0);
3845   %}
3846 %}
3847 
3848 // Indirect Memory Times Scale Plus Index Register
3849 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3850 %{
3851   constraint(ALLOC_IN_RC(ptr_reg));
3852   match(AddP reg (LShiftL lreg scale));
3853 
3854   op_cost(10);
3855   format %{"[$reg + $lreg << $scale]" %}
3856   interface(MEMORY_INTER) %{
3857     base($reg);
3858     index($lreg);
3859     scale($scale);
3860     disp(0x0);
3861   %}
3862 %}
3863 
3864 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3865 %{
3866   constraint(ALLOC_IN_RC(ptr_reg));
3867   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3868   match(AddP reg (LShiftL (ConvI2L idx) scale));
3869 
3870   op_cost(10);
3871   format %{"[$reg + pos $idx << $scale]" %}
3872   interface(MEMORY_INTER) %{
3873     base($reg);
3874     index($idx);
3875     scale($scale);
3876     disp(0x0);
3877   %}
3878 %}
3879 
3880 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3881 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3882 %{
3883   constraint(ALLOC_IN_RC(ptr_reg));
3884   match(AddP (AddP reg (LShiftL lreg scale)) off);
3885 
3886   op_cost(10);
3887   format %{"[$reg + $off + $lreg << $scale]" %}
3888   interface(MEMORY_INTER) %{
3889     base($reg);
3890     index($lreg);
3891     scale($scale);
3892     disp($off);
3893   %}
3894 %}
3895 
3896 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3897 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3898 %{
3899   constraint(ALLOC_IN_RC(ptr_reg));
3900   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3901   match(AddP (AddP reg (ConvI2L idx)) off);
3902 
3903   op_cost(10);
3904   format %{"[$reg + $off + $idx]" %}
3905   interface(MEMORY_INTER) %{
3906     base($reg);
3907     index($idx);
3908     scale(0x0);
3909     disp($off);
3910   %}
3911 %}
3912 
3913 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3914 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3915 %{
3916   constraint(ALLOC_IN_RC(ptr_reg));
3917   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3918   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3919 
3920   op_cost(10);
3921   format %{"[$reg + $off + $idx << $scale]" %}
3922   interface(MEMORY_INTER) %{
3923     base($reg);
3924     index($idx);
3925     scale($scale);
3926     disp($off);
3927   %}
3928 %}
3929 
3930 // Indirect Narrow Oop Plus Offset Operand
3931 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3932 // we can't free r12 even with CompressedOops::base() == NULL.
3933 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3934   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
3935   constraint(ALLOC_IN_RC(ptr_reg));
3936   match(AddP (DecodeN reg) off);
3937 
3938   op_cost(10);
3939   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3940   interface(MEMORY_INTER) %{
3941     base(0xc); // R12
3942     index($reg);
3943     scale(0x3);
3944     disp($off);
3945   %}
3946 %}
3947 
3948 // Indirect Memory Operand
3949 operand indirectNarrow(rRegN reg)
3950 %{
3951   predicate(CompressedOops::shift() == 0);
3952   constraint(ALLOC_IN_RC(ptr_reg));
3953   match(DecodeN reg);
3954 
3955   format %{ "[$reg]" %}
3956   interface(MEMORY_INTER) %{
3957     base($reg);
3958     index(0x4);
3959     scale(0x0);
3960     disp(0x0);
3961   %}
3962 %}
3963 
3964 // Indirect Memory Plus Short Offset Operand
3965 operand indOffset8Narrow(rRegN reg, immL8 off)
3966 %{
3967   predicate(CompressedOops::shift() == 0);
3968   constraint(ALLOC_IN_RC(ptr_reg));
3969   match(AddP (DecodeN reg) off);
3970 
3971   format %{ "[$reg + $off (8-bit)]" %}
3972   interface(MEMORY_INTER) %{
3973     base($reg);
3974     index(0x4);
3975     scale(0x0);
3976     disp($off);
3977   %}
3978 %}
3979 
3980 // Indirect Memory Plus Long Offset Operand
3981 operand indOffset32Narrow(rRegN reg, immL32 off)
3982 %{
3983   predicate(CompressedOops::shift() == 0);
3984   constraint(ALLOC_IN_RC(ptr_reg));
3985   match(AddP (DecodeN reg) off);
3986 
3987   format %{ "[$reg + $off (32-bit)]" %}
3988   interface(MEMORY_INTER) %{
3989     base($reg);
3990     index(0x4);
3991     scale(0x0);
3992     disp($off);
3993   %}
3994 %}
3995 
3996 // Indirect Memory Plus Index Register Plus Offset Operand
3997 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3998 %{
3999   predicate(CompressedOops::shift() == 0);
4000   constraint(ALLOC_IN_RC(ptr_reg));
4001   match(AddP (AddP (DecodeN reg) lreg) off);
4002 
4003   op_cost(10);
4004   format %{"[$reg + $off + $lreg]" %}
4005   interface(MEMORY_INTER) %{
4006     base($reg);
4007     index($lreg);
4008     scale(0x0);
4009     disp($off);
4010   %}
4011 %}
4012 
4013 // Indirect Memory Plus Index Register Plus Offset Operand
4014 operand indIndexNarrow(rRegN reg, rRegL lreg)
4015 %{
4016   predicate(CompressedOops::shift() == 0);
4017   constraint(ALLOC_IN_RC(ptr_reg));
4018   match(AddP (DecodeN reg) lreg);
4019 
4020   op_cost(10);
4021   format %{"[$reg + $lreg]" %}
4022   interface(MEMORY_INTER) %{
4023     base($reg);
4024     index($lreg);
4025     scale(0x0);
4026     disp(0x0);
4027   %}
4028 %}
4029 
4030 // Indirect Memory Times Scale Plus Index Register
4031 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
4032 %{
4033   predicate(CompressedOops::shift() == 0);
4034   constraint(ALLOC_IN_RC(ptr_reg));
4035   match(AddP (DecodeN reg) (LShiftL lreg scale));
4036 
4037   op_cost(10);
4038   format %{"[$reg + $lreg << $scale]" %}
4039   interface(MEMORY_INTER) %{
4040     base($reg);
4041     index($lreg);
4042     scale($scale);
4043     disp(0x0);
4044   %}
4045 %}
4046 
4047 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4048 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4049 %{
4050   predicate(CompressedOops::shift() == 0);
4051   constraint(ALLOC_IN_RC(ptr_reg));
4052   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4053 
4054   op_cost(10);
4055   format %{"[$reg + $off + $lreg << $scale]" %}
4056   interface(MEMORY_INTER) %{
4057     base($reg);
4058     index($lreg);
4059     scale($scale);
4060     disp($off);
4061   %}
4062 %}
4063 
4064 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4065 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4066 %{
4067   constraint(ALLOC_IN_RC(ptr_reg));
4068   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4069   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4070 
4071   op_cost(10);
4072   format %{"[$reg + $off + $idx]" %}
4073   interface(MEMORY_INTER) %{
4074     base($reg);
4075     index($idx);
4076     scale(0x0);
4077     disp($off);
4078   %}
4079 %}
4080 
4081 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4082 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4083 %{
4084   constraint(ALLOC_IN_RC(ptr_reg));
4085   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4086   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4087 
4088   op_cost(10);
4089   format %{"[$reg + $off + $idx << $scale]" %}
4090   interface(MEMORY_INTER) %{
4091     base($reg);
4092     index($idx);
4093     scale($scale);
4094     disp($off);
4095   %}
4096 %}
4097 
4098 //----------Special Memory Operands--------------------------------------------
4099 // Stack Slot Operand - This operand is used for loading and storing temporary
4100 //                      values on the stack where a match requires a value to
4101 //                      flow through memory.
4102 operand stackSlotP(sRegP reg)
4103 %{
4104   constraint(ALLOC_IN_RC(stack_slots));
4105   // No match rule because this operand is only generated in matching
4106 
4107   format %{ "[$reg]" %}
4108   interface(MEMORY_INTER) %{
4109     base(0x4);   // RSP
4110     index(0x4);  // No Index
4111     scale(0x0);  // No Scale
4112     disp($reg);  // Stack Offset
4113   %}
4114 %}
4115 
4116 operand stackSlotI(sRegI reg)
4117 %{
4118   constraint(ALLOC_IN_RC(stack_slots));
4119   // No match rule because this operand is only generated in matching
4120 
4121   format %{ "[$reg]" %}
4122   interface(MEMORY_INTER) %{
4123     base(0x4);   // RSP
4124     index(0x4);  // No Index
4125     scale(0x0);  // No Scale
4126     disp($reg);  // Stack Offset
4127   %}
4128 %}
4129 
4130 operand stackSlotF(sRegF reg)
4131 %{
4132   constraint(ALLOC_IN_RC(stack_slots));
4133   // No match rule because this operand is only generated in matching
4134 
4135   format %{ "[$reg]" %}
4136   interface(MEMORY_INTER) %{
4137     base(0x4);   // RSP
4138     index(0x4);  // No Index
4139     scale(0x0);  // No Scale
4140     disp($reg);  // Stack Offset
4141   %}
4142 %}
4143 
4144 operand stackSlotD(sRegD reg)
4145 %{
4146   constraint(ALLOC_IN_RC(stack_slots));
4147   // No match rule because this operand is only generated in matching
4148 
4149   format %{ "[$reg]" %}
4150   interface(MEMORY_INTER) %{
4151     base(0x4);   // RSP
4152     index(0x4);  // No Index
4153     scale(0x0);  // No Scale
4154     disp($reg);  // Stack Offset
4155   %}
4156 %}
4157 operand stackSlotL(sRegL reg)
4158 %{
4159   constraint(ALLOC_IN_RC(stack_slots));
4160   // No match rule because this operand is only generated in matching
4161 
4162   format %{ "[$reg]" %}
4163   interface(MEMORY_INTER) %{
4164     base(0x4);   // RSP
4165     index(0x4);  // No Index
4166     scale(0x0);  // No Scale
4167     disp($reg);  // Stack Offset
4168   %}
4169 %}
4170 
4171 //----------Conditional Branch Operands----------------------------------------
4172 // Comparison Op  - This is the operation of the comparison, and is limited to
4173 //                  the following set of codes:
4174 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4175 //
4176 // Other attributes of the comparison, such as unsignedness, are specified
4177 // by the comparison instruction that sets a condition code flags register.
4178 // That result is represented by a flags operand whose subtype is appropriate
4179 // to the unsignedness (etc.) of the comparison.
4180 //
4181 // Later, the instruction which matches both the Comparison Op (a Bool) and
4182 // the flags (produced by the Cmp) specifies the coding of the comparison op
4183 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4184 
4185 // Comparision Code
4186 operand cmpOp()
4187 %{
4188   match(Bool);
4189 
4190   format %{ "" %}
4191   interface(COND_INTER) %{
4192     equal(0x4, "e");
4193     not_equal(0x5, "ne");
4194     less(0xC, "l");
4195     greater_equal(0xD, "ge");
4196     less_equal(0xE, "le");
4197     greater(0xF, "g");
4198     overflow(0x0, "o");
4199     no_overflow(0x1, "no");
4200   %}
4201 %}
4202 
4203 // Comparison Code, unsigned compare.  Used by FP also, with
4204 // C2 (unordered) turned into GT or LT already.  The other bits
4205 // C0 and C3 are turned into Carry & Zero flags.
4206 operand cmpOpU()
4207 %{
4208   match(Bool);
4209 
4210   format %{ "" %}
4211   interface(COND_INTER) %{
4212     equal(0x4, "e");
4213     not_equal(0x5, "ne");
4214     less(0x2, "b");
4215     greater_equal(0x3, "nb");
4216     less_equal(0x6, "be");
4217     greater(0x7, "nbe");
4218     overflow(0x0, "o");
4219     no_overflow(0x1, "no");
4220   %}
4221 %}
4222 
4223 
4224 // Floating comparisons that don't require any fixup for the unordered case
4225 operand cmpOpUCF() %{
4226   match(Bool);
4227   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4228             n->as_Bool()->_test._test == BoolTest::ge ||
4229             n->as_Bool()->_test._test == BoolTest::le ||
4230             n->as_Bool()->_test._test == BoolTest::gt);
4231   format %{ "" %}
4232   interface(COND_INTER) %{
4233     equal(0x4, "e");
4234     not_equal(0x5, "ne");
4235     less(0x2, "b");
4236     greater_equal(0x3, "nb");
4237     less_equal(0x6, "be");
4238     greater(0x7, "nbe");
4239     overflow(0x0, "o");
4240     no_overflow(0x1, "no");
4241   %}
4242 %}
4243 
4244 
4245 // Floating comparisons that can be fixed up with extra conditional jumps
4246 operand cmpOpUCF2() %{
4247   match(Bool);
4248   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4249             n->as_Bool()->_test._test == BoolTest::eq);
4250   format %{ "" %}
4251   interface(COND_INTER) %{
4252     equal(0x4, "e");
4253     not_equal(0x5, "ne");
4254     less(0x2, "b");
4255     greater_equal(0x3, "nb");
4256     less_equal(0x6, "be");
4257     greater(0x7, "nbe");
4258     overflow(0x0, "o");
4259     no_overflow(0x1, "no");
4260   %}
4261 %}
4262 
4263 // Operands for bound floating pointer register arguments
4264 operand rxmm0() %{
4265   constraint(ALLOC_IN_RC(xmm0_reg));
4266   match(VecX);
4267   format%{%}
4268   interface(REG_INTER);
4269 %}
4270 operand rxmm1() %{
4271   constraint(ALLOC_IN_RC(xmm1_reg));
4272   match(VecX);
4273   format%{%}
4274   interface(REG_INTER);
4275 %}
4276 operand rxmm2() %{
4277   constraint(ALLOC_IN_RC(xmm2_reg));
4278   match(VecX);
4279   format%{%}
4280   interface(REG_INTER);
4281 %}
4282 operand rxmm3() %{
4283   constraint(ALLOC_IN_RC(xmm3_reg));
4284   match(VecX);
4285   format%{%}
4286   interface(REG_INTER);
4287 %}
4288 operand rxmm4() %{
4289   constraint(ALLOC_IN_RC(xmm4_reg));
4290   match(VecX);
4291   format%{%}
4292   interface(REG_INTER);
4293 %}
4294 operand rxmm5() %{
4295   constraint(ALLOC_IN_RC(xmm5_reg));
4296   match(VecX);
4297   format%{%}
4298   interface(REG_INTER);
4299 %}
4300 operand rxmm6() %{
4301   constraint(ALLOC_IN_RC(xmm6_reg));
4302   match(VecX);
4303   format%{%}
4304   interface(REG_INTER);
4305 %}
4306 operand rxmm7() %{
4307   constraint(ALLOC_IN_RC(xmm7_reg));
4308   match(VecX);
4309   format%{%}
4310   interface(REG_INTER);
4311 %}
4312 operand rxmm8() %{
4313   constraint(ALLOC_IN_RC(xmm8_reg));
4314   match(VecX);
4315   format%{%}
4316   interface(REG_INTER);
4317 %}
4318 operand rxmm9() %{
4319   constraint(ALLOC_IN_RC(xmm9_reg));
4320   match(VecX);
4321   format%{%}
4322   interface(REG_INTER);
4323 %}
4324 operand rxmm10() %{
4325   constraint(ALLOC_IN_RC(xmm10_reg));
4326   match(VecX);
4327   format%{%}
4328   interface(REG_INTER);
4329 %}
4330 operand rxmm11() %{
4331   constraint(ALLOC_IN_RC(xmm11_reg));
4332   match(VecX);
4333   format%{%}
4334   interface(REG_INTER);
4335 %}
4336 operand rxmm12() %{
4337   constraint(ALLOC_IN_RC(xmm12_reg));
4338   match(VecX);
4339   format%{%}
4340   interface(REG_INTER);
4341 %}
4342 operand rxmm13() %{
4343   constraint(ALLOC_IN_RC(xmm13_reg));
4344   match(VecX);
4345   format%{%}
4346   interface(REG_INTER);
4347 %}
4348 operand rxmm14() %{
4349   constraint(ALLOC_IN_RC(xmm14_reg));
4350   match(VecX);
4351   format%{%}
4352   interface(REG_INTER);
4353 %}
4354 operand rxmm15() %{
4355   constraint(ALLOC_IN_RC(xmm15_reg));
4356   match(VecX);
4357   format%{%}
4358   interface(REG_INTER);
4359 %}
4360 operand rxmm16() %{
4361   constraint(ALLOC_IN_RC(xmm16_reg));
4362   match(VecX);
4363   format%{%}
4364   interface(REG_INTER);
4365 %}
4366 operand rxmm17() %{
4367   constraint(ALLOC_IN_RC(xmm17_reg));
4368   match(VecX);
4369   format%{%}
4370   interface(REG_INTER);
4371 %}
4372 operand rxmm18() %{
4373   constraint(ALLOC_IN_RC(xmm18_reg));
4374   match(VecX);
4375   format%{%}
4376   interface(REG_INTER);
4377 %}
4378 operand rxmm19() %{
4379   constraint(ALLOC_IN_RC(xmm19_reg));
4380   match(VecX);
4381   format%{%}
4382   interface(REG_INTER);
4383 %}
4384 operand rxmm20() %{
4385   constraint(ALLOC_IN_RC(xmm20_reg));
4386   match(VecX);
4387   format%{%}
4388   interface(REG_INTER);
4389 %}
4390 operand rxmm21() %{
4391   constraint(ALLOC_IN_RC(xmm21_reg));
4392   match(VecX);
4393   format%{%}
4394   interface(REG_INTER);
4395 %}
4396 operand rxmm22() %{
4397   constraint(ALLOC_IN_RC(xmm22_reg));
4398   match(VecX);
4399   format%{%}
4400   interface(REG_INTER);
4401 %}
4402 operand rxmm23() %{
4403   constraint(ALLOC_IN_RC(xmm23_reg));
4404   match(VecX);
4405   format%{%}
4406   interface(REG_INTER);
4407 %}
4408 operand rxmm24() %{
4409   constraint(ALLOC_IN_RC(xmm24_reg));
4410   match(VecX);
4411   format%{%}
4412   interface(REG_INTER);
4413 %}
4414 operand rxmm25() %{
4415   constraint(ALLOC_IN_RC(xmm25_reg));
4416   match(VecX);
4417   format%{%}
4418   interface(REG_INTER);
4419 %}
4420 operand rxmm26() %{
4421   constraint(ALLOC_IN_RC(xmm26_reg));
4422   match(VecX);
4423   format%{%}
4424   interface(REG_INTER);
4425 %}
4426 operand rxmm27() %{
4427   constraint(ALLOC_IN_RC(xmm27_reg));
4428   match(VecX);
4429   format%{%}
4430   interface(REG_INTER);
4431 %}
4432 operand rxmm28() %{
4433   constraint(ALLOC_IN_RC(xmm28_reg));
4434   match(VecX);
4435   format%{%}
4436   interface(REG_INTER);
4437 %}
4438 operand rxmm29() %{
4439   constraint(ALLOC_IN_RC(xmm29_reg));
4440   match(VecX);
4441   format%{%}
4442   interface(REG_INTER);
4443 %}
4444 operand rxmm30() %{
4445   constraint(ALLOC_IN_RC(xmm30_reg));
4446   match(VecX);
4447   format%{%}
4448   interface(REG_INTER);
4449 %}
4450 operand rxmm31() %{
4451   constraint(ALLOC_IN_RC(xmm31_reg));
4452   match(VecX);
4453   format%{%}
4454   interface(REG_INTER);
4455 %}
4456 
4457 //----------OPERAND CLASSES----------------------------------------------------
4458 // Operand Classes are groups of operands that are used as to simplify
4459 // instruction definitions by not requiring the AD writer to specify separate
4460 // instructions for every form of operand when the instruction accepts
4461 // multiple operand types with the same basic encoding and format.  The classic
4462 // case of this is memory operands.
4463 
4464 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4465                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4466                indCompressedOopOffset,
4467                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4468                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4469                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4470 
4471 //----------PIPELINE-----------------------------------------------------------
4472 // Rules which define the behavior of the target architectures pipeline.
4473 pipeline %{
4474 
4475 //----------ATTRIBUTES---------------------------------------------------------
4476 attributes %{
4477   variable_size_instructions;        // Fixed size instructions
4478   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4479   instruction_unit_size = 1;         // An instruction is 1 bytes long
4480   instruction_fetch_unit_size = 16;  // The processor fetches one line
4481   instruction_fetch_units = 1;       // of 16 bytes
4482 
4483   // List of nop instructions
4484   nops( MachNop );
4485 %}
4486 
4487 //----------RESOURCES----------------------------------------------------------
4488 // Resources are the functional units available to the machine
4489 
4490 // Generic P2/P3 pipeline
4491 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4492 // 3 instructions decoded per cycle.
4493 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4494 // 3 ALU op, only ALU0 handles mul instructions.
4495 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4496            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4497            BR, FPU,
4498            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4499 
4500 //----------PIPELINE DESCRIPTION-----------------------------------------------
4501 // Pipeline Description specifies the stages in the machine's pipeline
4502 
4503 // Generic P2/P3 pipeline
4504 pipe_desc(S0, S1, S2, S3, S4, S5);
4505 
4506 //----------PIPELINE CLASSES---------------------------------------------------
4507 // Pipeline Classes describe the stages in which input and output are
4508 // referenced by the hardware pipeline.
4509 
4510 // Naming convention: ialu or fpu
4511 // Then: _reg
4512 // Then: _reg if there is a 2nd register
4513 // Then: _long if it's a pair of instructions implementing a long
4514 // Then: _fat if it requires the big decoder
4515 //   Or: _mem if it requires the big decoder and a memory unit.
4516 
4517 // Integer ALU reg operation
4518 pipe_class ialu_reg(rRegI dst)
4519 %{
4520     single_instruction;
4521     dst    : S4(write);
4522     dst    : S3(read);
4523     DECODE : S0;        // any decoder
4524     ALU    : S3;        // any alu
4525 %}
4526 
4527 // Long ALU reg operation
4528 pipe_class ialu_reg_long(rRegL dst)
4529 %{
4530     instruction_count(2);
4531     dst    : S4(write);
4532     dst    : S3(read);
4533     DECODE : S0(2);     // any 2 decoders
4534     ALU    : S3(2);     // both alus
4535 %}
4536 
4537 // Integer ALU reg operation using big decoder
4538 pipe_class ialu_reg_fat(rRegI dst)
4539 %{
4540     single_instruction;
4541     dst    : S4(write);
4542     dst    : S3(read);
4543     D0     : S0;        // big decoder only
4544     ALU    : S3;        // any alu
4545 %}
4546 
4547 // Long ALU reg operation using big decoder
4548 pipe_class ialu_reg_long_fat(rRegL dst)
4549 %{
4550     instruction_count(2);
4551     dst    : S4(write);
4552     dst    : S3(read);
4553     D0     : S0(2);     // big decoder only; twice
4554     ALU    : S3(2);     // any 2 alus
4555 %}
4556 
4557 // Integer ALU reg-reg operation
4558 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4559 %{
4560     single_instruction;
4561     dst    : S4(write);
4562     src    : S3(read);
4563     DECODE : S0;        // any decoder
4564     ALU    : S3;        // any alu
4565 %}
4566 
4567 // Long ALU reg-reg operation
4568 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4569 %{
4570     instruction_count(2);
4571     dst    : S4(write);
4572     src    : S3(read);
4573     DECODE : S0(2);     // any 2 decoders
4574     ALU    : S3(2);     // both alus
4575 %}
4576 
4577 // Integer ALU reg-reg operation
4578 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4579 %{
4580     single_instruction;
4581     dst    : S4(write);
4582     src    : S3(read);
4583     D0     : S0;        // big decoder only
4584     ALU    : S3;        // any alu
4585 %}
4586 
4587 // Long ALU reg-reg operation
4588 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4589 %{
4590     instruction_count(2);
4591     dst    : S4(write);
4592     src    : S3(read);
4593     D0     : S0(2);     // big decoder only; twice
4594     ALU    : S3(2);     // both alus
4595 %}
4596 
4597 // Integer ALU reg-mem operation
4598 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4599 %{
4600     single_instruction;
4601     dst    : S5(write);
4602     mem    : S3(read);
4603     D0     : S0;        // big decoder only
4604     ALU    : S4;        // any alu
4605     MEM    : S3;        // any mem
4606 %}
4607 
4608 // Integer mem operation (prefetch)
4609 pipe_class ialu_mem(memory mem)
4610 %{
4611     single_instruction;
4612     mem    : S3(read);
4613     D0     : S0;        // big decoder only
4614     MEM    : S3;        // any mem
4615 %}
4616 
4617 // Integer Store to Memory
4618 pipe_class ialu_mem_reg(memory mem, rRegI src)
4619 %{
4620     single_instruction;
4621     mem    : S3(read);
4622     src    : S5(read);
4623     D0     : S0;        // big decoder only
4624     ALU    : S4;        // any alu
4625     MEM    : S3;
4626 %}
4627 
4628 // // Long Store to Memory
4629 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4630 // %{
4631 //     instruction_count(2);
4632 //     mem    : S3(read);
4633 //     src    : S5(read);
4634 //     D0     : S0(2);          // big decoder only; twice
4635 //     ALU    : S4(2);     // any 2 alus
4636 //     MEM    : S3(2);  // Both mems
4637 // %}
4638 
4639 // Integer Store to Memory
4640 pipe_class ialu_mem_imm(memory mem)
4641 %{
4642     single_instruction;
4643     mem    : S3(read);
4644     D0     : S0;        // big decoder only
4645     ALU    : S4;        // any alu
4646     MEM    : S3;
4647 %}
4648 
4649 // Integer ALU0 reg-reg operation
4650 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4651 %{
4652     single_instruction;
4653     dst    : S4(write);
4654     src    : S3(read);
4655     D0     : S0;        // Big decoder only
4656     ALU0   : S3;        // only alu0
4657 %}
4658 
4659 // Integer ALU0 reg-mem operation
4660 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4661 %{
4662     single_instruction;
4663     dst    : S5(write);
4664     mem    : S3(read);
4665     D0     : S0;        // big decoder only
4666     ALU0   : S4;        // ALU0 only
4667     MEM    : S3;        // any mem
4668 %}
4669 
4670 // Integer ALU reg-reg operation
4671 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4672 %{
4673     single_instruction;
4674     cr     : S4(write);
4675     src1   : S3(read);
4676     src2   : S3(read);
4677     DECODE : S0;        // any decoder
4678     ALU    : S3;        // any alu
4679 %}
4680 
4681 // Integer ALU reg-imm operation
4682 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4683 %{
4684     single_instruction;
4685     cr     : S4(write);
4686     src1   : S3(read);
4687     DECODE : S0;        // any decoder
4688     ALU    : S3;        // any alu
4689 %}
4690 
4691 // Integer ALU reg-mem operation
4692 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4693 %{
4694     single_instruction;
4695     cr     : S4(write);
4696     src1   : S3(read);
4697     src2   : S3(read);
4698     D0     : S0;        // big decoder only
4699     ALU    : S4;        // any alu
4700     MEM    : S3;
4701 %}
4702 
4703 // Conditional move reg-reg
4704 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4705 %{
4706     instruction_count(4);
4707     y      : S4(read);
4708     q      : S3(read);
4709     p      : S3(read);
4710     DECODE : S0(4);     // any decoder
4711 %}
4712 
4713 // Conditional move reg-reg
4714 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4715 %{
4716     single_instruction;
4717     dst    : S4(write);
4718     src    : S3(read);
4719     cr     : S3(read);
4720     DECODE : S0;        // any decoder
4721 %}
4722 
4723 // Conditional move reg-mem
4724 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4725 %{
4726     single_instruction;
4727     dst    : S4(write);
4728     src    : S3(read);
4729     cr     : S3(read);
4730     DECODE : S0;        // any decoder
4731     MEM    : S3;
4732 %}
4733 
4734 // Conditional move reg-reg long
4735 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4736 %{
4737     single_instruction;
4738     dst    : S4(write);
4739     src    : S3(read);
4740     cr     : S3(read);
4741     DECODE : S0(2);     // any 2 decoders
4742 %}
4743 
4744 // XXX
4745 // // Conditional move double reg-reg
4746 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4747 // %{
4748 //     single_instruction;
4749 //     dst    : S4(write);
4750 //     src    : S3(read);
4751 //     cr     : S3(read);
4752 //     DECODE : S0;     // any decoder
4753 // %}
4754 
4755 // Float reg-reg operation
4756 pipe_class fpu_reg(regD dst)
4757 %{
4758     instruction_count(2);
4759     dst    : S3(read);
4760     DECODE : S0(2);     // any 2 decoders
4761     FPU    : S3;
4762 %}
4763 
4764 // Float reg-reg operation
4765 pipe_class fpu_reg_reg(regD dst, regD src)
4766 %{
4767     instruction_count(2);
4768     dst    : S4(write);
4769     src    : S3(read);
4770     DECODE : S0(2);     // any 2 decoders
4771     FPU    : S3;
4772 %}
4773 
4774 // Float reg-reg operation
4775 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4776 %{
4777     instruction_count(3);
4778     dst    : S4(write);
4779     src1   : S3(read);
4780     src2   : S3(read);
4781     DECODE : S0(3);     // any 3 decoders
4782     FPU    : S3(2);
4783 %}
4784 
4785 // Float reg-reg operation
4786 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4787 %{
4788     instruction_count(4);
4789     dst    : S4(write);
4790     src1   : S3(read);
4791     src2   : S3(read);
4792     src3   : S3(read);
4793     DECODE : S0(4);     // any 3 decoders
4794     FPU    : S3(2);
4795 %}
4796 
4797 // Float reg-reg operation
4798 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4799 %{
4800     instruction_count(4);
4801     dst    : S4(write);
4802     src1   : S3(read);
4803     src2   : S3(read);
4804     src3   : S3(read);
4805     DECODE : S1(3);     // any 3 decoders
4806     D0     : S0;        // Big decoder only
4807     FPU    : S3(2);
4808     MEM    : S3;
4809 %}
4810 
4811 // Float reg-mem operation
4812 pipe_class fpu_reg_mem(regD dst, memory mem)
4813 %{
4814     instruction_count(2);
4815     dst    : S5(write);
4816     mem    : S3(read);
4817     D0     : S0;        // big decoder only
4818     DECODE : S1;        // any decoder for FPU POP
4819     FPU    : S4;
4820     MEM    : S3;        // any mem
4821 %}
4822 
4823 // Float reg-mem operation
4824 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4825 %{
4826     instruction_count(3);
4827     dst    : S5(write);
4828     src1   : S3(read);
4829     mem    : S3(read);
4830     D0     : S0;        // big decoder only
4831     DECODE : S1(2);     // any decoder for FPU POP
4832     FPU    : S4;
4833     MEM    : S3;        // any mem
4834 %}
4835 
4836 // Float mem-reg operation
4837 pipe_class fpu_mem_reg(memory mem, regD src)
4838 %{
4839     instruction_count(2);
4840     src    : S5(read);
4841     mem    : S3(read);
4842     DECODE : S0;        // any decoder for FPU PUSH
4843     D0     : S1;        // big decoder only
4844     FPU    : S4;
4845     MEM    : S3;        // any mem
4846 %}
4847 
4848 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4849 %{
4850     instruction_count(3);
4851     src1   : S3(read);
4852     src2   : S3(read);
4853     mem    : S3(read);
4854     DECODE : S0(2);     // any decoder for FPU PUSH
4855     D0     : S1;        // big decoder only
4856     FPU    : S4;
4857     MEM    : S3;        // any mem
4858 %}
4859 
4860 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4861 %{
4862     instruction_count(3);
4863     src1   : S3(read);
4864     src2   : S3(read);
4865     mem    : S4(read);
4866     DECODE : S0;        // any decoder for FPU PUSH
4867     D0     : S0(2);     // big decoder only
4868     FPU    : S4;
4869     MEM    : S3(2);     // any mem
4870 %}
4871 
4872 pipe_class fpu_mem_mem(memory dst, memory src1)
4873 %{
4874     instruction_count(2);
4875     src1   : S3(read);
4876     dst    : S4(read);
4877     D0     : S0(2);     // big decoder only
4878     MEM    : S3(2);     // any mem
4879 %}
4880 
4881 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4882 %{
4883     instruction_count(3);
4884     src1   : S3(read);
4885     src2   : S3(read);
4886     dst    : S4(read);
4887     D0     : S0(3);     // big decoder only
4888     FPU    : S4;
4889     MEM    : S3(3);     // any mem
4890 %}
4891 
4892 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4893 %{
4894     instruction_count(3);
4895     src1   : S4(read);
4896     mem    : S4(read);
4897     DECODE : S0;        // any decoder for FPU PUSH
4898     D0     : S0(2);     // big decoder only
4899     FPU    : S4;
4900     MEM    : S3(2);     // any mem
4901 %}
4902 
4903 // Float load constant
4904 pipe_class fpu_reg_con(regD dst)
4905 %{
4906     instruction_count(2);
4907     dst    : S5(write);
4908     D0     : S0;        // big decoder only for the load
4909     DECODE : S1;        // any decoder for FPU POP
4910     FPU    : S4;
4911     MEM    : S3;        // any mem
4912 %}
4913 
4914 // Float load constant
4915 pipe_class fpu_reg_reg_con(regD dst, regD src)
4916 %{
4917     instruction_count(3);
4918     dst    : S5(write);
4919     src    : S3(read);
4920     D0     : S0;        // big decoder only for the load
4921     DECODE : S1(2);     // any decoder for FPU POP
4922     FPU    : S4;
4923     MEM    : S3;        // any mem
4924 %}
4925 
4926 // UnConditional branch
4927 pipe_class pipe_jmp(label labl)
4928 %{
4929     single_instruction;
4930     BR   : S3;
4931 %}
4932 
4933 // Conditional branch
4934 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4935 %{
4936     single_instruction;
4937     cr    : S1(read);
4938     BR    : S3;
4939 %}
4940 
4941 // Allocation idiom
4942 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4943 %{
4944     instruction_count(1); force_serialization;
4945     fixed_latency(6);
4946     heap_ptr : S3(read);
4947     DECODE   : S0(3);
4948     D0       : S2;
4949     MEM      : S3;
4950     ALU      : S3(2);
4951     dst      : S5(write);
4952     BR       : S5;
4953 %}
4954 
4955 // Generic big/slow expanded idiom
4956 pipe_class pipe_slow()
4957 %{
4958     instruction_count(10); multiple_bundles; force_serialization;
4959     fixed_latency(100);
4960     D0  : S0(2);
4961     MEM : S3(2);
4962 %}
4963 
4964 // The real do-nothing guy
4965 pipe_class empty()
4966 %{
4967     instruction_count(0);
4968 %}
4969 
4970 // Define the class for the Nop node
4971 define
4972 %{
4973    MachNop = empty;
4974 %}
4975 
4976 %}
4977 
4978 //----------INSTRUCTIONS-------------------------------------------------------
4979 //
4980 // match      -- States which machine-independent subtree may be replaced
4981 //               by this instruction.
4982 // ins_cost   -- The estimated cost of this instruction is used by instruction
4983 //               selection to identify a minimum cost tree of machine
4984 //               instructions that matches a tree of machine-independent
4985 //               instructions.
4986 // format     -- A string providing the disassembly for this instruction.
4987 //               The value of an instruction's operand may be inserted
4988 //               by referring to it with a '$' prefix.
4989 // opcode     -- Three instruction opcodes may be provided.  These are referred
4990 //               to within an encode class as $primary, $secondary, and $tertiary
4991 //               rrspectively.  The primary opcode is commonly used to
4992 //               indicate the type of machine instruction, while secondary
4993 //               and tertiary are often used for prefix options or addressing
4994 //               modes.
4995 // ins_encode -- A list of encode classes with parameters. The encode class
4996 //               name must have been defined in an 'enc_class' specification
4997 //               in the encode section of the architecture description.
4998 
4999 
5000 //----------Load/Store/Move Instructions---------------------------------------
5001 //----------Load Instructions--------------------------------------------------
5002 
5003 // Load Byte (8 bit signed)
5004 instruct loadB(rRegI dst, memory mem)
5005 %{
5006   match(Set dst (LoadB mem));
5007 
5008   ins_cost(125);
5009   format %{ "movsbl  $dst, $mem\t# byte" %}
5010 
5011   ins_encode %{
5012     __ movsbl($dst$$Register, $mem$$Address);
5013   %}
5014 
5015   ins_pipe(ialu_reg_mem);
5016 %}
5017 
5018 // Load Byte (8 bit signed) into Long Register
5019 instruct loadB2L(rRegL dst, memory mem)
5020 %{
5021   match(Set dst (ConvI2L (LoadB mem)));
5022 
5023   ins_cost(125);
5024   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
5025 
5026   ins_encode %{
5027     __ movsbq($dst$$Register, $mem$$Address);
5028   %}
5029 
5030   ins_pipe(ialu_reg_mem);
5031 %}
5032 
5033 // Load Unsigned Byte (8 bit UNsigned)
5034 instruct loadUB(rRegI dst, memory mem)
5035 %{
5036   match(Set dst (LoadUB mem));
5037 
5038   ins_cost(125);
5039   format %{ "movzbl  $dst, $mem\t# ubyte" %}
5040 
5041   ins_encode %{
5042     __ movzbl($dst$$Register, $mem$$Address);
5043   %}
5044 
5045   ins_pipe(ialu_reg_mem);
5046 %}
5047 
5048 // Load Unsigned Byte (8 bit UNsigned) into Long Register
5049 instruct loadUB2L(rRegL dst, memory mem)
5050 %{
5051   match(Set dst (ConvI2L (LoadUB mem)));
5052 
5053   ins_cost(125);
5054   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
5055 
5056   ins_encode %{
5057     __ movzbq($dst$$Register, $mem$$Address);
5058   %}
5059 
5060   ins_pipe(ialu_reg_mem);
5061 %}
5062 
5063 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
5064 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5065   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5066   effect(KILL cr);
5067 
5068   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
5069             "andl    $dst, right_n_bits($mask, 8)" %}
5070   ins_encode %{
5071     Register Rdst = $dst$$Register;
5072     __ movzbq(Rdst, $mem$$Address);
5073     __ andl(Rdst, $mask$$constant & right_n_bits(8));
5074   %}
5075   ins_pipe(ialu_reg_mem);
5076 %}
5077 
5078 // Load Short (16 bit signed)
5079 instruct loadS(rRegI dst, memory mem)
5080 %{
5081   match(Set dst (LoadS mem));
5082 
5083   ins_cost(125);
5084   format %{ "movswl $dst, $mem\t# short" %}
5085 
5086   ins_encode %{
5087     __ movswl($dst$$Register, $mem$$Address);
5088   %}
5089 
5090   ins_pipe(ialu_reg_mem);
5091 %}
5092 
5093 // Load Short (16 bit signed) to Byte (8 bit signed)
5094 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5095   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5096 
5097   ins_cost(125);
5098   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5099   ins_encode %{
5100     __ movsbl($dst$$Register, $mem$$Address);
5101   %}
5102   ins_pipe(ialu_reg_mem);
5103 %}
5104 
5105 // Load Short (16 bit signed) into Long Register
5106 instruct loadS2L(rRegL dst, memory mem)
5107 %{
5108   match(Set dst (ConvI2L (LoadS mem)));
5109 
5110   ins_cost(125);
5111   format %{ "movswq $dst, $mem\t# short -> long" %}
5112 
5113   ins_encode %{
5114     __ movswq($dst$$Register, $mem$$Address);
5115   %}
5116 
5117   ins_pipe(ialu_reg_mem);
5118 %}
5119 
5120 // Load Unsigned Short/Char (16 bit UNsigned)
5121 instruct loadUS(rRegI dst, memory mem)
5122 %{
5123   match(Set dst (LoadUS mem));
5124 
5125   ins_cost(125);
5126   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5127 
5128   ins_encode %{
5129     __ movzwl($dst$$Register, $mem$$Address);
5130   %}
5131 
5132   ins_pipe(ialu_reg_mem);
5133 %}
5134 
5135 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5136 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5137   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5138 
5139   ins_cost(125);
5140   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5141   ins_encode %{
5142     __ movsbl($dst$$Register, $mem$$Address);
5143   %}
5144   ins_pipe(ialu_reg_mem);
5145 %}
5146 
5147 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5148 instruct loadUS2L(rRegL dst, memory mem)
5149 %{
5150   match(Set dst (ConvI2L (LoadUS mem)));
5151 
5152   ins_cost(125);
5153   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5154 
5155   ins_encode %{
5156     __ movzwq($dst$$Register, $mem$$Address);
5157   %}
5158 
5159   ins_pipe(ialu_reg_mem);
5160 %}
5161 
5162 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5163 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5164   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5165 
5166   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5167   ins_encode %{
5168     __ movzbq($dst$$Register, $mem$$Address);
5169   %}
5170   ins_pipe(ialu_reg_mem);
5171 %}
5172 
5173 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5174 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5175   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5176   effect(KILL cr);
5177 
5178   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5179             "andl    $dst, right_n_bits($mask, 16)" %}
5180   ins_encode %{
5181     Register Rdst = $dst$$Register;
5182     __ movzwq(Rdst, $mem$$Address);
5183     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5184   %}
5185   ins_pipe(ialu_reg_mem);
5186 %}
5187 
5188 // Load Integer
5189 instruct loadI(rRegI dst, memory mem)
5190 %{
5191   match(Set dst (LoadI mem));
5192 
5193   ins_cost(125);
5194   format %{ "movl    $dst, $mem\t# int" %}
5195 
5196   ins_encode %{
5197     __ movl($dst$$Register, $mem$$Address);
5198   %}
5199 
5200   ins_pipe(ialu_reg_mem);
5201 %}
5202 
5203 // Load Integer (32 bit signed) to Byte (8 bit signed)
5204 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5205   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5206 
5207   ins_cost(125);
5208   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5209   ins_encode %{
5210     __ movsbl($dst$$Register, $mem$$Address);
5211   %}
5212   ins_pipe(ialu_reg_mem);
5213 %}
5214 
5215 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5216 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5217   match(Set dst (AndI (LoadI mem) mask));
5218 
5219   ins_cost(125);
5220   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5221   ins_encode %{
5222     __ movzbl($dst$$Register, $mem$$Address);
5223   %}
5224   ins_pipe(ialu_reg_mem);
5225 %}
5226 
5227 // Load Integer (32 bit signed) to Short (16 bit signed)
5228 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5229   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5230 
5231   ins_cost(125);
5232   format %{ "movswl  $dst, $mem\t# int -> short" %}
5233   ins_encode %{
5234     __ movswl($dst$$Register, $mem$$Address);
5235   %}
5236   ins_pipe(ialu_reg_mem);
5237 %}
5238 
5239 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5240 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5241   match(Set dst (AndI (LoadI mem) mask));
5242 
5243   ins_cost(125);
5244   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5245   ins_encode %{
5246     __ movzwl($dst$$Register, $mem$$Address);
5247   %}
5248   ins_pipe(ialu_reg_mem);
5249 %}
5250 
5251 // Load Integer into Long Register
5252 instruct loadI2L(rRegL dst, memory mem)
5253 %{
5254   match(Set dst (ConvI2L (LoadI mem)));
5255 
5256   ins_cost(125);
5257   format %{ "movslq  $dst, $mem\t# int -> long" %}
5258 
5259   ins_encode %{
5260     __ movslq($dst$$Register, $mem$$Address);
5261   %}
5262 
5263   ins_pipe(ialu_reg_mem);
5264 %}
5265 
5266 // Load Integer with mask 0xFF into Long Register
5267 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5268   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5269 
5270   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5271   ins_encode %{
5272     __ movzbq($dst$$Register, $mem$$Address);
5273   %}
5274   ins_pipe(ialu_reg_mem);
5275 %}
5276 
5277 // Load Integer with mask 0xFFFF into Long Register
5278 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5279   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5280 
5281   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5282   ins_encode %{
5283     __ movzwq($dst$$Register, $mem$$Address);
5284   %}
5285   ins_pipe(ialu_reg_mem);
5286 %}
5287 
5288 // Load Integer with a 31-bit mask into Long Register
5289 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5290   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5291   effect(KILL cr);
5292 
5293   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5294             "andl    $dst, $mask" %}
5295   ins_encode %{
5296     Register Rdst = $dst$$Register;
5297     __ movl(Rdst, $mem$$Address);
5298     __ andl(Rdst, $mask$$constant);
5299   %}
5300   ins_pipe(ialu_reg_mem);
5301 %}
5302 
5303 // Load Unsigned Integer into Long Register
5304 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5305 %{
5306   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5307 
5308   ins_cost(125);
5309   format %{ "movl    $dst, $mem\t# uint -> long" %}
5310 
5311   ins_encode %{
5312     __ movl($dst$$Register, $mem$$Address);
5313   %}
5314 
5315   ins_pipe(ialu_reg_mem);
5316 %}
5317 
5318 // Load Long
5319 instruct loadL(rRegL dst, memory mem)
5320 %{
5321   match(Set dst (LoadL mem));
5322 
5323   ins_cost(125);
5324   format %{ "movq    $dst, $mem\t# long" %}
5325 
5326   ins_encode %{
5327     __ movq($dst$$Register, $mem$$Address);
5328   %}
5329 
5330   ins_pipe(ialu_reg_mem); // XXX
5331 %}
5332 
5333 // Load Range
5334 instruct loadRange(rRegI dst, memory mem)
5335 %{
5336   match(Set dst (LoadRange mem));
5337 
5338   ins_cost(125); // XXX
5339   format %{ "movl    $dst, $mem\t# range" %}
5340   opcode(0x8B);
5341   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5342   ins_pipe(ialu_reg_mem);
5343 %}
5344 
5345 // Load Pointer
5346 instruct loadP(rRegP dst, memory mem)
5347 %{
5348   match(Set dst (LoadP mem));
5349 
5350   ins_cost(125); // XXX
5351   format %{ "movq    $dst, $mem\t# ptr" %}
5352   opcode(0x8B);
5353   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5354   ins_pipe(ialu_reg_mem); // XXX
5355 %}
5356 
5357 // Load Compressed Pointer
5358 instruct loadN(rRegN dst, memory mem)
5359 %{
5360    match(Set dst (LoadN mem));
5361 
5362    ins_cost(125); // XXX
5363    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5364    ins_encode %{
5365      __ movl($dst$$Register, $mem$$Address);
5366    %}
5367    ins_pipe(ialu_reg_mem); // XXX
5368 %}
5369 
5370 
5371 // Load Klass Pointer
5372 instruct loadKlass(rRegP dst, memory mem)
5373 %{
5374   match(Set dst (LoadKlass mem));
5375 
5376   ins_cost(125); // XXX
5377   format %{ "movq    $dst, $mem\t# class" %}
5378   opcode(0x8B);
5379   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5380   ins_pipe(ialu_reg_mem); // XXX
5381 %}
5382 
5383 // Load narrow Klass Pointer
5384 instruct loadNKlass(rRegN dst, memory mem)
5385 %{
5386   match(Set dst (LoadNKlass mem));
5387 
5388   ins_cost(125); // XXX
5389   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5390   ins_encode %{
5391     __ movl($dst$$Register, $mem$$Address);
5392   %}
5393   ins_pipe(ialu_reg_mem); // XXX
5394 %}
5395 
5396 // Load Float
5397 instruct loadF(regF dst, memory mem)
5398 %{
5399   match(Set dst (LoadF mem));
5400 
5401   ins_cost(145); // XXX
5402   format %{ "movss   $dst, $mem\t# float" %}
5403   ins_encode %{
5404     __ movflt($dst$$XMMRegister, $mem$$Address);
5405   %}
5406   ins_pipe(pipe_slow); // XXX
5407 %}
5408 
5409 // Load Float
5410 instruct MoveF2VL(vlRegF dst, regF src) %{
5411   match(Set dst src);
5412   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5413   ins_encode %{
5414     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5415   %}
5416   ins_pipe( fpu_reg_reg );
5417 %}
5418 
5419 // Load Float
5420 instruct MoveF2LEG(legRegF dst, regF src) %{
5421   match(Set dst src);
5422   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
5423   ins_encode %{
5424     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5425   %}
5426   ins_pipe( fpu_reg_reg );
5427 %}
5428 
5429 // Load Float
5430 instruct MoveVL2F(regF dst, vlRegF src) %{
5431   match(Set dst src);
5432   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5433   ins_encode %{
5434     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5435   %}
5436   ins_pipe( fpu_reg_reg );
5437 %}
5438 
5439 // Load Float
5440 instruct MoveLEG2F(regF dst, legRegF src) %{
5441   match(Set dst src);
5442   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
5443   ins_encode %{
5444     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5445   %}
5446   ins_pipe( fpu_reg_reg );
5447 %}
5448 
5449 // Load Double
5450 instruct loadD_partial(regD dst, memory mem)
5451 %{
5452   predicate(!UseXmmLoadAndClearUpper);
5453   match(Set dst (LoadD mem));
5454 
5455   ins_cost(145); // XXX
5456   format %{ "movlpd  $dst, $mem\t# double" %}
5457   ins_encode %{
5458     __ movdbl($dst$$XMMRegister, $mem$$Address);
5459   %}
5460   ins_pipe(pipe_slow); // XXX
5461 %}
5462 
5463 instruct loadD(regD dst, memory mem)
5464 %{
5465   predicate(UseXmmLoadAndClearUpper);
5466   match(Set dst (LoadD mem));
5467 
5468   ins_cost(145); // XXX
5469   format %{ "movsd   $dst, $mem\t# double" %}
5470   ins_encode %{
5471     __ movdbl($dst$$XMMRegister, $mem$$Address);
5472   %}
5473   ins_pipe(pipe_slow); // XXX
5474 %}
5475 
5476 // Load Double
5477 instruct MoveD2VL(vlRegD dst, regD src) %{
5478   match(Set dst src);
5479   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5480   ins_encode %{
5481     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5482   %}
5483   ins_pipe( fpu_reg_reg );
5484 %}
5485 
5486 // Load Double
5487 instruct MoveD2LEG(legRegD dst, regD src) %{
5488   match(Set dst src);
5489   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
5490   ins_encode %{
5491     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5492   %}
5493   ins_pipe( fpu_reg_reg );
5494 %}
5495 
5496 // Load Double
5497 instruct MoveVL2D(regD dst, vlRegD src) %{
5498   match(Set dst src);
5499   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5500   ins_encode %{
5501     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5502   %}
5503   ins_pipe( fpu_reg_reg );
5504 %}
5505 
5506 // Load Double
5507 instruct MoveLEG2D(regD dst, legRegD src) %{
5508   match(Set dst src);
5509   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
5510   ins_encode %{
5511     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5512   %}
5513   ins_pipe( fpu_reg_reg );
5514 %}
5515 
5516 // Following pseudo code describes the algorithm for max[FD]:
5517 // Min algorithm is on similar lines
5518 //  btmp = (b < +0.0) ? a : b
5519 //  atmp = (b < +0.0) ? b : a
5520 //  Tmp  = Max_Float(atmp , btmp)
5521 //  Res  = (atmp == NaN) ? atmp : Tmp
5522 
5523 // max = java.lang.Math.max(float a, float b)
5524 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
5525   predicate(UseAVX > 0 && !n->is_reduction());
5526   match(Set dst (MaxF a b));
5527   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5528   format %{
5529      "blendvps         $btmp,$b,$a,$b           \n\t"
5530      "blendvps         $atmp,$a,$b,$b           \n\t"
5531      "vmaxss           $tmp,$atmp,$btmp         \n\t"
5532      "cmpps.unordered  $btmp,$atmp,$atmp        \n\t"
5533      "blendvps         $dst,$tmp,$atmp,$btmp    \n\t"
5534   %}
5535   ins_encode %{
5536     int vector_len = Assembler::AVX_128bit;
5537     __ blendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
5538     __ blendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
5539     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5540     __ cmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5541     __ blendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5542  %}
5543   ins_pipe( pipe_slow );
5544 %}
5545 
5546 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
5547   predicate(UseAVX > 0 && n->is_reduction());
5548   match(Set dst (MaxF a b));
5549   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5550 
5551   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
5552   ins_encode %{
5553     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5554                     false /*min*/, true /*single*/);
5555   %}
5556   ins_pipe( pipe_slow );
5557 %}
5558 
5559 // max = java.lang.Math.max(double a, double b)
5560 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
5561   predicate(UseAVX > 0 && !n->is_reduction());
5562   match(Set dst (MaxD a b));
5563   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
5564   format %{
5565      "blendvpd         $btmp,$b,$a,$b            \n\t"
5566      "blendvpd         $atmp,$a,$b,$b            \n\t"
5567      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
5568      "cmppd.unordered  $btmp,$atmp,$atmp         \n\t"
5569      "blendvpd         $dst,$tmp,$atmp,$btmp     \n\t"
5570   %}
5571   ins_encode %{
5572     int vector_len = Assembler::AVX_128bit;
5573     __ blendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
5574     __ blendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
5575     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5576     __ cmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5577     __ blendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5578   %}
5579   ins_pipe( pipe_slow );
5580 %}
5581 
5582 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
5583   predicate(UseAVX > 0 && n->is_reduction());
5584   match(Set dst (MaxD a b));
5585   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5586 
5587   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
5588   ins_encode %{
5589     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5590                     false /*min*/, false /*single*/);
5591   %}
5592   ins_pipe( pipe_slow );
5593 %}
5594 
5595 // min = java.lang.Math.min(float a, float b)
5596 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
5597   predicate(UseAVX > 0 && !n->is_reduction());
5598   match(Set dst (MinF a b));
5599   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5600   format %{
5601      "blendvps         $atmp,$a,$b,$a             \n\t"
5602      "blendvps         $btmp,$b,$a,$a             \n\t"
5603      "vminss           $tmp,$atmp,$btmp           \n\t"
5604      "cmpps.unordered  $btmp,$atmp,$atmp          \n\t"
5605      "blendvps         $dst,$tmp,$atmp,$btmp      \n\t"
5606   %}
5607   ins_encode %{
5608     int vector_len = Assembler::AVX_128bit;
5609     __ blendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
5610     __ blendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
5611     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5612     __ cmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5613     __ blendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5614   %}
5615   ins_pipe( pipe_slow );
5616 %}
5617 
5618 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
5619   predicate(UseAVX > 0 && n->is_reduction());
5620   match(Set dst (MinF a b));
5621   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5622 
5623   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
5624   ins_encode %{
5625     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5626                     true /*min*/, true /*single*/);
5627   %}
5628   ins_pipe( pipe_slow );
5629 %}
5630 
5631 // min = java.lang.Math.min(double a, double b)
5632 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
5633   predicate(UseAVX > 0 && !n->is_reduction());
5634   match(Set dst (MinD a b));
5635   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5636   format %{
5637      "blendvpd         $atmp,$a,$b,$a           \n\t"
5638      "blendvpd         $btmp,$b,$a,$a           \n\t"
5639      "vminsd           $tmp,$atmp,$btmp         \n\t"
5640      "cmppd.unordered  $btmp,$atmp,$atmp        \n\t"
5641      "blendvpd         $dst,$tmp,$atmp,$btmp    \n\t"
5642   %}
5643   ins_encode %{
5644     int vector_len = Assembler::AVX_128bit;
5645     __ blendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
5646     __ blendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
5647     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5648     __ cmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5649     __ blendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5650   %}
5651   ins_pipe( pipe_slow );
5652 %}
5653 
5654 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
5655   predicate(UseAVX > 0 && n->is_reduction());
5656   match(Set dst (MinD a b));
5657   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5658 
5659   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
5660   ins_encode %{
5661     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5662                     true /*min*/, false /*single*/);
5663   %}
5664   ins_pipe( pipe_slow );
5665 %}
5666 
5667 // Load Effective Address
5668 instruct leaP8(rRegP dst, indOffset8 mem)
5669 %{
5670   match(Set dst mem);
5671 
5672   ins_cost(110); // XXX
5673   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5674   opcode(0x8D);
5675   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5676   ins_pipe(ialu_reg_reg_fat);
5677 %}
5678 
5679 instruct leaP32(rRegP dst, indOffset32 mem)
5680 %{
5681   match(Set dst mem);
5682 
5683   ins_cost(110);
5684   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5685   opcode(0x8D);
5686   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5687   ins_pipe(ialu_reg_reg_fat);
5688 %}
5689 
5690 // instruct leaPIdx(rRegP dst, indIndex mem)
5691 // %{
5692 //   match(Set dst mem);
5693 
5694 //   ins_cost(110);
5695 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5696 //   opcode(0x8D);
5697 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5698 //   ins_pipe(ialu_reg_reg_fat);
5699 // %}
5700 
5701 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5702 %{
5703   match(Set dst mem);
5704 
5705   ins_cost(110);
5706   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5707   opcode(0x8D);
5708   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5709   ins_pipe(ialu_reg_reg_fat);
5710 %}
5711 
5712 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5713 %{
5714   match(Set dst mem);
5715 
5716   ins_cost(110);
5717   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5718   opcode(0x8D);
5719   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5720   ins_pipe(ialu_reg_reg_fat);
5721 %}
5722 
5723 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5724 %{
5725   match(Set dst mem);
5726 
5727   ins_cost(110);
5728   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5729   opcode(0x8D);
5730   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5731   ins_pipe(ialu_reg_reg_fat);
5732 %}
5733 
5734 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5735 %{
5736   match(Set dst mem);
5737 
5738   ins_cost(110);
5739   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5740   opcode(0x8D);
5741   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5742   ins_pipe(ialu_reg_reg_fat);
5743 %}
5744 
5745 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5746 %{
5747   match(Set dst mem);
5748 
5749   ins_cost(110);
5750   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5751   opcode(0x8D);
5752   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5753   ins_pipe(ialu_reg_reg_fat);
5754 %}
5755 
5756 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5757 %{
5758   match(Set dst mem);
5759 
5760   ins_cost(110);
5761   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5762   opcode(0x8D);
5763   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5764   ins_pipe(ialu_reg_reg_fat);
5765 %}
5766 
5767 // Load Effective Address which uses Narrow (32-bits) oop
5768 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5769 %{
5770   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
5771   match(Set dst mem);
5772 
5773   ins_cost(110);
5774   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5775   opcode(0x8D);
5776   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5777   ins_pipe(ialu_reg_reg_fat);
5778 %}
5779 
5780 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5781 %{
5782   predicate(CompressedOops::shift() == 0);
5783   match(Set dst mem);
5784 
5785   ins_cost(110); // XXX
5786   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5787   opcode(0x8D);
5788   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5789   ins_pipe(ialu_reg_reg_fat);
5790 %}
5791 
5792 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5793 %{
5794   predicate(CompressedOops::shift() == 0);
5795   match(Set dst mem);
5796 
5797   ins_cost(110);
5798   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5799   opcode(0x8D);
5800   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5801   ins_pipe(ialu_reg_reg_fat);
5802 %}
5803 
5804 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5805 %{
5806   predicate(CompressedOops::shift() == 0);
5807   match(Set dst mem);
5808 
5809   ins_cost(110);
5810   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5811   opcode(0x8D);
5812   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5813   ins_pipe(ialu_reg_reg_fat);
5814 %}
5815 
5816 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5817 %{
5818   predicate(CompressedOops::shift() == 0);
5819   match(Set dst mem);
5820 
5821   ins_cost(110);
5822   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5823   opcode(0x8D);
5824   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5825   ins_pipe(ialu_reg_reg_fat);
5826 %}
5827 
5828 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5829 %{
5830   predicate(CompressedOops::shift() == 0);
5831   match(Set dst mem);
5832 
5833   ins_cost(110);
5834   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5835   opcode(0x8D);
5836   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5837   ins_pipe(ialu_reg_reg_fat);
5838 %}
5839 
5840 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5841 %{
5842   predicate(CompressedOops::shift() == 0);
5843   match(Set dst mem);
5844 
5845   ins_cost(110);
5846   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5847   opcode(0x8D);
5848   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5849   ins_pipe(ialu_reg_reg_fat);
5850 %}
5851 
5852 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5853 %{
5854   predicate(CompressedOops::shift() == 0);
5855   match(Set dst mem);
5856 
5857   ins_cost(110);
5858   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5859   opcode(0x8D);
5860   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5861   ins_pipe(ialu_reg_reg_fat);
5862 %}
5863 
5864 instruct loadConI(rRegI dst, immI src)
5865 %{
5866   match(Set dst src);
5867 
5868   format %{ "movl    $dst, $src\t# int" %}
5869   ins_encode(load_immI(dst, src));
5870   ins_pipe(ialu_reg_fat); // XXX
5871 %}
5872 
5873 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5874 %{
5875   match(Set dst src);
5876   effect(KILL cr);
5877 
5878   ins_cost(50);
5879   format %{ "xorl    $dst, $dst\t# int" %}
5880   opcode(0x33); /* + rd */
5881   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5882   ins_pipe(ialu_reg);
5883 %}
5884 
5885 instruct loadConL(rRegL dst, immL src)
5886 %{
5887   match(Set dst src);
5888 
5889   ins_cost(150);
5890   format %{ "movq    $dst, $src\t# long" %}
5891   ins_encode(load_immL(dst, src));
5892   ins_pipe(ialu_reg);
5893 %}
5894 
5895 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5896 %{
5897   match(Set dst src);
5898   effect(KILL cr);
5899 
5900   ins_cost(50);
5901   format %{ "xorl    $dst, $dst\t# long" %}
5902   opcode(0x33); /* + rd */
5903   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5904   ins_pipe(ialu_reg); // XXX
5905 %}
5906 
5907 instruct loadConUL32(rRegL dst, immUL32 src)
5908 %{
5909   match(Set dst src);
5910 
5911   ins_cost(60);
5912   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5913   ins_encode(load_immUL32(dst, src));
5914   ins_pipe(ialu_reg);
5915 %}
5916 
5917 instruct loadConL32(rRegL dst, immL32 src)
5918 %{
5919   match(Set dst src);
5920 
5921   ins_cost(70);
5922   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5923   ins_encode(load_immL32(dst, src));
5924   ins_pipe(ialu_reg);
5925 %}
5926 
5927 instruct loadConP(rRegP dst, immP con) %{
5928   match(Set dst con);
5929 
5930   format %{ "movq    $dst, $con\t# ptr" %}
5931   ins_encode(load_immP(dst, con));
5932   ins_pipe(ialu_reg_fat); // XXX
5933 %}
5934 
5935 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5936 %{
5937   match(Set dst src);
5938   effect(KILL cr);
5939 
5940   ins_cost(50);
5941   format %{ "xorl    $dst, $dst\t# ptr" %}
5942   opcode(0x33); /* + rd */
5943   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5944   ins_pipe(ialu_reg);
5945 %}
5946 
5947 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5948 %{
5949   match(Set dst src);
5950   effect(KILL cr);
5951 
5952   ins_cost(60);
5953   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5954   ins_encode(load_immP31(dst, src));
5955   ins_pipe(ialu_reg);
5956 %}
5957 
5958 instruct loadConF(regF dst, immF con) %{
5959   match(Set dst con);
5960   ins_cost(125);
5961   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5962   ins_encode %{
5963     __ movflt($dst$$XMMRegister, $constantaddress($con));
5964   %}
5965   ins_pipe(pipe_slow);
5966 %}
5967 
5968 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5969   match(Set dst src);
5970   effect(KILL cr);
5971   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5972   ins_encode %{
5973     __ xorq($dst$$Register, $dst$$Register);
5974   %}
5975   ins_pipe(ialu_reg);
5976 %}
5977 
5978 instruct loadConN(rRegN dst, immN src) %{
5979   match(Set dst src);
5980 
5981   ins_cost(125);
5982   format %{ "movl    $dst, $src\t# compressed ptr" %}
5983   ins_encode %{
5984     address con = (address)$src$$constant;
5985     if (con == NULL) {
5986       ShouldNotReachHere();
5987     } else {
5988       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5989     }
5990   %}
5991   ins_pipe(ialu_reg_fat); // XXX
5992 %}
5993 
5994 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5995   match(Set dst src);
5996 
5997   ins_cost(125);
5998   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5999   ins_encode %{
6000     address con = (address)$src$$constant;
6001     if (con == NULL) {
6002       ShouldNotReachHere();
6003     } else {
6004       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
6005     }
6006   %}
6007   ins_pipe(ialu_reg_fat); // XXX
6008 %}
6009 
6010 instruct loadConF0(regF dst, immF0 src)
6011 %{
6012   match(Set dst src);
6013   ins_cost(100);
6014 
6015   format %{ "xorps   $dst, $dst\t# float 0.0" %}
6016   ins_encode %{
6017     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
6018   %}
6019   ins_pipe(pipe_slow);
6020 %}
6021 
6022 // Use the same format since predicate() can not be used here.
6023 instruct loadConD(regD dst, immD con) %{
6024   match(Set dst con);
6025   ins_cost(125);
6026   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
6027   ins_encode %{
6028     __ movdbl($dst$$XMMRegister, $constantaddress($con));
6029   %}
6030   ins_pipe(pipe_slow);
6031 %}
6032 
6033 instruct loadConD0(regD dst, immD0 src)
6034 %{
6035   match(Set dst src);
6036   ins_cost(100);
6037 
6038   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
6039   ins_encode %{
6040     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
6041   %}
6042   ins_pipe(pipe_slow);
6043 %}
6044 
6045 instruct loadSSI(rRegI dst, stackSlotI src)
6046 %{
6047   match(Set dst src);
6048 
6049   ins_cost(125);
6050   format %{ "movl    $dst, $src\t# int stk" %}
6051   opcode(0x8B);
6052   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6053   ins_pipe(ialu_reg_mem);
6054 %}
6055 
6056 instruct loadSSL(rRegL dst, stackSlotL src)
6057 %{
6058   match(Set dst src);
6059 
6060   ins_cost(125);
6061   format %{ "movq    $dst, $src\t# long stk" %}
6062   opcode(0x8B);
6063   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6064   ins_pipe(ialu_reg_mem);
6065 %}
6066 
6067 instruct loadSSP(rRegP dst, stackSlotP src)
6068 %{
6069   match(Set dst src);
6070 
6071   ins_cost(125);
6072   format %{ "movq    $dst, $src\t# ptr stk" %}
6073   opcode(0x8B);
6074   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6075   ins_pipe(ialu_reg_mem);
6076 %}
6077 
6078 instruct loadSSF(regF dst, stackSlotF src)
6079 %{
6080   match(Set dst src);
6081 
6082   ins_cost(125);
6083   format %{ "movss   $dst, $src\t# float stk" %}
6084   ins_encode %{
6085     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
6086   %}
6087   ins_pipe(pipe_slow); // XXX
6088 %}
6089 
6090 // Use the same format since predicate() can not be used here.
6091 instruct loadSSD(regD dst, stackSlotD src)
6092 %{
6093   match(Set dst src);
6094 
6095   ins_cost(125);
6096   format %{ "movsd   $dst, $src\t# double stk" %}
6097   ins_encode  %{
6098     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
6099   %}
6100   ins_pipe(pipe_slow); // XXX
6101 %}
6102 
6103 // Prefetch instructions for allocation.
6104 // Must be safe to execute with invalid address (cannot fault).
6105 
6106 instruct prefetchAlloc( memory mem ) %{
6107   predicate(AllocatePrefetchInstr==3);
6108   match(PrefetchAllocation mem);
6109   ins_cost(125);
6110 
6111   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
6112   ins_encode %{
6113     __ prefetchw($mem$$Address);
6114   %}
6115   ins_pipe(ialu_mem);
6116 %}
6117 
6118 instruct prefetchAllocNTA( memory mem ) %{
6119   predicate(AllocatePrefetchInstr==0);
6120   match(PrefetchAllocation mem);
6121   ins_cost(125);
6122 
6123   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
6124   ins_encode %{
6125     __ prefetchnta($mem$$Address);
6126   %}
6127   ins_pipe(ialu_mem);
6128 %}
6129 
6130 instruct prefetchAllocT0( memory mem ) %{
6131   predicate(AllocatePrefetchInstr==1);
6132   match(PrefetchAllocation mem);
6133   ins_cost(125);
6134 
6135   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
6136   ins_encode %{
6137     __ prefetcht0($mem$$Address);
6138   %}
6139   ins_pipe(ialu_mem);
6140 %}
6141 
6142 instruct prefetchAllocT2( memory mem ) %{
6143   predicate(AllocatePrefetchInstr==2);
6144   match(PrefetchAllocation mem);
6145   ins_cost(125);
6146 
6147   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
6148   ins_encode %{
6149     __ prefetcht2($mem$$Address);
6150   %}
6151   ins_pipe(ialu_mem);
6152 %}
6153 
6154 //----------Store Instructions-------------------------------------------------
6155 
6156 // Store Byte
6157 instruct storeB(memory mem, rRegI src)
6158 %{
6159   match(Set mem (StoreB mem src));
6160 
6161   ins_cost(125); // XXX
6162   format %{ "movb    $mem, $src\t# byte" %}
6163   opcode(0x88);
6164   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
6165   ins_pipe(ialu_mem_reg);
6166 %}
6167 
6168 // Store Char/Short
6169 instruct storeC(memory mem, rRegI src)
6170 %{
6171   match(Set mem (StoreC mem src));
6172 
6173   ins_cost(125); // XXX
6174   format %{ "movw    $mem, $src\t# char/short" %}
6175   opcode(0x89);
6176   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6177   ins_pipe(ialu_mem_reg);
6178 %}
6179 
6180 // Store Integer
6181 instruct storeI(memory mem, rRegI src)
6182 %{
6183   match(Set mem (StoreI mem src));
6184 
6185   ins_cost(125); // XXX
6186   format %{ "movl    $mem, $src\t# int" %}
6187   opcode(0x89);
6188   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6189   ins_pipe(ialu_mem_reg);
6190 %}
6191 
6192 // Store Long
6193 instruct storeL(memory mem, rRegL src)
6194 %{
6195   match(Set mem (StoreL mem src));
6196 
6197   ins_cost(125); // XXX
6198   format %{ "movq    $mem, $src\t# long" %}
6199   opcode(0x89);
6200   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6201   ins_pipe(ialu_mem_reg); // XXX
6202 %}
6203 
6204 // Store Pointer
6205 instruct storeP(memory mem, any_RegP src)
6206 %{
6207   match(Set mem (StoreP mem src));
6208 
6209   ins_cost(125); // XXX
6210   format %{ "movq    $mem, $src\t# ptr" %}
6211   opcode(0x89);
6212   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6213   ins_pipe(ialu_mem_reg);
6214 %}
6215 
6216 instruct storeImmP0(memory mem, immP0 zero)
6217 %{
6218   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6219   match(Set mem (StoreP mem zero));
6220 
6221   ins_cost(125); // XXX
6222   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
6223   ins_encode %{
6224     __ movq($mem$$Address, r12);
6225   %}
6226   ins_pipe(ialu_mem_reg);
6227 %}
6228 
6229 // Store NULL Pointer, mark word, or other simple pointer constant.
6230 instruct storeImmP(memory mem, immP31 src)
6231 %{
6232   match(Set mem (StoreP mem src));
6233 
6234   ins_cost(150); // XXX
6235   format %{ "movq    $mem, $src\t# ptr" %}
6236   opcode(0xC7); /* C7 /0 */
6237   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6238   ins_pipe(ialu_mem_imm);
6239 %}
6240 
6241 // Store Compressed Pointer
6242 instruct storeN(memory mem, rRegN src)
6243 %{
6244   match(Set mem (StoreN mem src));
6245 
6246   ins_cost(125); // XXX
6247   format %{ "movl    $mem, $src\t# compressed ptr" %}
6248   ins_encode %{
6249     __ movl($mem$$Address, $src$$Register);
6250   %}
6251   ins_pipe(ialu_mem_reg);
6252 %}
6253 
6254 instruct storeNKlass(memory mem, rRegN src)
6255 %{
6256   match(Set mem (StoreNKlass mem src));
6257 
6258   ins_cost(125); // XXX
6259   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6260   ins_encode %{
6261     __ movl($mem$$Address, $src$$Register);
6262   %}
6263   ins_pipe(ialu_mem_reg);
6264 %}
6265 
6266 instruct storeImmN0(memory mem, immN0 zero)
6267 %{
6268   predicate(CompressedOops::base() == NULL && CompressedKlassPointers::base() == NULL);
6269   match(Set mem (StoreN mem zero));
6270 
6271   ins_cost(125); // XXX
6272   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
6273   ins_encode %{
6274     __ movl($mem$$Address, r12);
6275   %}
6276   ins_pipe(ialu_mem_reg);
6277 %}
6278 
6279 instruct storeImmN(memory mem, immN src)
6280 %{
6281   match(Set mem (StoreN mem src));
6282 
6283   ins_cost(150); // XXX
6284   format %{ "movl    $mem, $src\t# compressed ptr" %}
6285   ins_encode %{
6286     address con = (address)$src$$constant;
6287     if (con == NULL) {
6288       __ movl($mem$$Address, (int32_t)0);
6289     } else {
6290       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
6291     }
6292   %}
6293   ins_pipe(ialu_mem_imm);
6294 %}
6295 
6296 instruct storeImmNKlass(memory mem, immNKlass src)
6297 %{
6298   match(Set mem (StoreNKlass mem src));
6299 
6300   ins_cost(150); // XXX
6301   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6302   ins_encode %{
6303     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
6304   %}
6305   ins_pipe(ialu_mem_imm);
6306 %}
6307 
6308 // Store Integer Immediate
6309 instruct storeImmI0(memory mem, immI0 zero)
6310 %{
6311   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6312   match(Set mem (StoreI mem zero));
6313 
6314   ins_cost(125); // XXX
6315   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6316   ins_encode %{
6317     __ movl($mem$$Address, r12);
6318   %}
6319   ins_pipe(ialu_mem_reg);
6320 %}
6321 
6322 instruct storeImmI(memory mem, immI src)
6323 %{
6324   match(Set mem (StoreI mem src));
6325 
6326   ins_cost(150);
6327   format %{ "movl    $mem, $src\t# int" %}
6328   opcode(0xC7); /* C7 /0 */
6329   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6330   ins_pipe(ialu_mem_imm);
6331 %}
6332 
6333 // Store Long Immediate
6334 instruct storeImmL0(memory mem, immL0 zero)
6335 %{
6336   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6337   match(Set mem (StoreL mem zero));
6338 
6339   ins_cost(125); // XXX
6340   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6341   ins_encode %{
6342     __ movq($mem$$Address, r12);
6343   %}
6344   ins_pipe(ialu_mem_reg);
6345 %}
6346 
6347 instruct storeImmL(memory mem, immL32 src)
6348 %{
6349   match(Set mem (StoreL mem src));
6350 
6351   ins_cost(150);
6352   format %{ "movq    $mem, $src\t# long" %}
6353   opcode(0xC7); /* C7 /0 */
6354   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6355   ins_pipe(ialu_mem_imm);
6356 %}
6357 
6358 // Store Short/Char Immediate
6359 instruct storeImmC0(memory mem, immI0 zero)
6360 %{
6361   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6362   match(Set mem (StoreC mem zero));
6363 
6364   ins_cost(125); // XXX
6365   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6366   ins_encode %{
6367     __ movw($mem$$Address, r12);
6368   %}
6369   ins_pipe(ialu_mem_reg);
6370 %}
6371 
6372 instruct storeImmI16(memory mem, immI16 src)
6373 %{
6374   predicate(UseStoreImmI16);
6375   match(Set mem (StoreC mem src));
6376 
6377   ins_cost(150);
6378   format %{ "movw    $mem, $src\t# short/char" %}
6379   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6380   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6381   ins_pipe(ialu_mem_imm);
6382 %}
6383 
6384 // Store Byte Immediate
6385 instruct storeImmB0(memory mem, immI0 zero)
6386 %{
6387   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6388   match(Set mem (StoreB mem zero));
6389 
6390   ins_cost(125); // XXX
6391   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6392   ins_encode %{
6393     __ movb($mem$$Address, r12);
6394   %}
6395   ins_pipe(ialu_mem_reg);
6396 %}
6397 
6398 instruct storeImmB(memory mem, immI8 src)
6399 %{
6400   match(Set mem (StoreB mem src));
6401 
6402   ins_cost(150); // XXX
6403   format %{ "movb    $mem, $src\t# byte" %}
6404   opcode(0xC6); /* C6 /0 */
6405   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6406   ins_pipe(ialu_mem_imm);
6407 %}
6408 
6409 // Store CMS card-mark Immediate
6410 instruct storeImmCM0_reg(memory mem, immI0 zero)
6411 %{
6412   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6413   match(Set mem (StoreCM mem zero));
6414 
6415   ins_cost(125); // XXX
6416   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6417   ins_encode %{
6418     __ movb($mem$$Address, r12);
6419   %}
6420   ins_pipe(ialu_mem_reg);
6421 %}
6422 
6423 instruct storeImmCM0(memory mem, immI0 src)
6424 %{
6425   match(Set mem (StoreCM mem src));
6426 
6427   ins_cost(150); // XXX
6428   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6429   opcode(0xC6); /* C6 /0 */
6430   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6431   ins_pipe(ialu_mem_imm);
6432 %}
6433 
6434 // Store Float
6435 instruct storeF(memory mem, regF src)
6436 %{
6437   match(Set mem (StoreF mem src));
6438 
6439   ins_cost(95); // XXX
6440   format %{ "movss   $mem, $src\t# float" %}
6441   ins_encode %{
6442     __ movflt($mem$$Address, $src$$XMMRegister);
6443   %}
6444   ins_pipe(pipe_slow); // XXX
6445 %}
6446 
6447 // Store immediate Float value (it is faster than store from XMM register)
6448 instruct storeF0(memory mem, immF0 zero)
6449 %{
6450   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6451   match(Set mem (StoreF mem zero));
6452 
6453   ins_cost(25); // XXX
6454   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6455   ins_encode %{
6456     __ movl($mem$$Address, r12);
6457   %}
6458   ins_pipe(ialu_mem_reg);
6459 %}
6460 
6461 instruct storeF_imm(memory mem, immF src)
6462 %{
6463   match(Set mem (StoreF mem src));
6464 
6465   ins_cost(50);
6466   format %{ "movl    $mem, $src\t# float" %}
6467   opcode(0xC7); /* C7 /0 */
6468   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6469   ins_pipe(ialu_mem_imm);
6470 %}
6471 
6472 // Store Double
6473 instruct storeD(memory mem, regD src)
6474 %{
6475   match(Set mem (StoreD mem src));
6476 
6477   ins_cost(95); // XXX
6478   format %{ "movsd   $mem, $src\t# double" %}
6479   ins_encode %{
6480     __ movdbl($mem$$Address, $src$$XMMRegister);
6481   %}
6482   ins_pipe(pipe_slow); // XXX
6483 %}
6484 
6485 // Store immediate double 0.0 (it is faster than store from XMM register)
6486 instruct storeD0_imm(memory mem, immD0 src)
6487 %{
6488   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
6489   match(Set mem (StoreD mem src));
6490 
6491   ins_cost(50);
6492   format %{ "movq    $mem, $src\t# double 0." %}
6493   opcode(0xC7); /* C7 /0 */
6494   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6495   ins_pipe(ialu_mem_imm);
6496 %}
6497 
6498 instruct storeD0(memory mem, immD0 zero)
6499 %{
6500   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6501   match(Set mem (StoreD mem zero));
6502 
6503   ins_cost(25); // XXX
6504   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6505   ins_encode %{
6506     __ movq($mem$$Address, r12);
6507   %}
6508   ins_pipe(ialu_mem_reg);
6509 %}
6510 
6511 instruct storeSSI(stackSlotI dst, rRegI src)
6512 %{
6513   match(Set dst src);
6514 
6515   ins_cost(100);
6516   format %{ "movl    $dst, $src\t# int stk" %}
6517   opcode(0x89);
6518   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6519   ins_pipe( ialu_mem_reg );
6520 %}
6521 
6522 instruct storeSSL(stackSlotL dst, rRegL src)
6523 %{
6524   match(Set dst src);
6525 
6526   ins_cost(100);
6527   format %{ "movq    $dst, $src\t# long stk" %}
6528   opcode(0x89);
6529   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6530   ins_pipe(ialu_mem_reg);
6531 %}
6532 
6533 instruct storeSSP(stackSlotP dst, rRegP src)
6534 %{
6535   match(Set dst src);
6536 
6537   ins_cost(100);
6538   format %{ "movq    $dst, $src\t# ptr stk" %}
6539   opcode(0x89);
6540   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6541   ins_pipe(ialu_mem_reg);
6542 %}
6543 
6544 instruct storeSSF(stackSlotF dst, regF src)
6545 %{
6546   match(Set dst src);
6547 
6548   ins_cost(95); // XXX
6549   format %{ "movss   $dst, $src\t# float stk" %}
6550   ins_encode %{
6551     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6552   %}
6553   ins_pipe(pipe_slow); // XXX
6554 %}
6555 
6556 instruct storeSSD(stackSlotD dst, regD src)
6557 %{
6558   match(Set dst src);
6559 
6560   ins_cost(95); // XXX
6561   format %{ "movsd   $dst, $src\t# double stk" %}
6562   ins_encode %{
6563     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6564   %}
6565   ins_pipe(pipe_slow); // XXX
6566 %}
6567 
6568 //----------BSWAP Instructions-------------------------------------------------
6569 instruct bytes_reverse_int(rRegI dst) %{
6570   match(Set dst (ReverseBytesI dst));
6571 
6572   format %{ "bswapl  $dst" %}
6573   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6574   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6575   ins_pipe( ialu_reg );
6576 %}
6577 
6578 instruct bytes_reverse_long(rRegL dst) %{
6579   match(Set dst (ReverseBytesL dst));
6580 
6581   format %{ "bswapq  $dst" %}
6582   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6583   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6584   ins_pipe( ialu_reg);
6585 %}
6586 
6587 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6588   match(Set dst (ReverseBytesUS dst));
6589   effect(KILL cr);
6590 
6591   format %{ "bswapl  $dst\n\t"
6592             "shrl    $dst,16\n\t" %}
6593   ins_encode %{
6594     __ bswapl($dst$$Register);
6595     __ shrl($dst$$Register, 16);
6596   %}
6597   ins_pipe( ialu_reg );
6598 %}
6599 
6600 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6601   match(Set dst (ReverseBytesS dst));
6602   effect(KILL cr);
6603 
6604   format %{ "bswapl  $dst\n\t"
6605             "sar     $dst,16\n\t" %}
6606   ins_encode %{
6607     __ bswapl($dst$$Register);
6608     __ sarl($dst$$Register, 16);
6609   %}
6610   ins_pipe( ialu_reg );
6611 %}
6612 
6613 //---------- Zeros Count Instructions ------------------------------------------
6614 
6615 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6616   predicate(UseCountLeadingZerosInstruction);
6617   match(Set dst (CountLeadingZerosI src));
6618   effect(KILL cr);
6619 
6620   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6621   ins_encode %{
6622     __ lzcntl($dst$$Register, $src$$Register);
6623   %}
6624   ins_pipe(ialu_reg);
6625 %}
6626 
6627 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6628   predicate(!UseCountLeadingZerosInstruction);
6629   match(Set dst (CountLeadingZerosI src));
6630   effect(KILL cr);
6631 
6632   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6633             "jnz     skip\n\t"
6634             "movl    $dst, -1\n"
6635       "skip:\n\t"
6636             "negl    $dst\n\t"
6637             "addl    $dst, 31" %}
6638   ins_encode %{
6639     Register Rdst = $dst$$Register;
6640     Register Rsrc = $src$$Register;
6641     Label skip;
6642     __ bsrl(Rdst, Rsrc);
6643     __ jccb(Assembler::notZero, skip);
6644     __ movl(Rdst, -1);
6645     __ bind(skip);
6646     __ negl(Rdst);
6647     __ addl(Rdst, BitsPerInt - 1);
6648   %}
6649   ins_pipe(ialu_reg);
6650 %}
6651 
6652 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6653   predicate(UseCountLeadingZerosInstruction);
6654   match(Set dst (CountLeadingZerosL src));
6655   effect(KILL cr);
6656 
6657   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6658   ins_encode %{
6659     __ lzcntq($dst$$Register, $src$$Register);
6660   %}
6661   ins_pipe(ialu_reg);
6662 %}
6663 
6664 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6665   predicate(!UseCountLeadingZerosInstruction);
6666   match(Set dst (CountLeadingZerosL src));
6667   effect(KILL cr);
6668 
6669   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6670             "jnz     skip\n\t"
6671             "movl    $dst, -1\n"
6672       "skip:\n\t"
6673             "negl    $dst\n\t"
6674             "addl    $dst, 63" %}
6675   ins_encode %{
6676     Register Rdst = $dst$$Register;
6677     Register Rsrc = $src$$Register;
6678     Label skip;
6679     __ bsrq(Rdst, Rsrc);
6680     __ jccb(Assembler::notZero, skip);
6681     __ movl(Rdst, -1);
6682     __ bind(skip);
6683     __ negl(Rdst);
6684     __ addl(Rdst, BitsPerLong - 1);
6685   %}
6686   ins_pipe(ialu_reg);
6687 %}
6688 
6689 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6690   predicate(UseCountTrailingZerosInstruction);
6691   match(Set dst (CountTrailingZerosI src));
6692   effect(KILL cr);
6693 
6694   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6695   ins_encode %{
6696     __ tzcntl($dst$$Register, $src$$Register);
6697   %}
6698   ins_pipe(ialu_reg);
6699 %}
6700 
6701 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6702   predicate(!UseCountTrailingZerosInstruction);
6703   match(Set dst (CountTrailingZerosI src));
6704   effect(KILL cr);
6705 
6706   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6707             "jnz     done\n\t"
6708             "movl    $dst, 32\n"
6709       "done:" %}
6710   ins_encode %{
6711     Register Rdst = $dst$$Register;
6712     Label done;
6713     __ bsfl(Rdst, $src$$Register);
6714     __ jccb(Assembler::notZero, done);
6715     __ movl(Rdst, BitsPerInt);
6716     __ bind(done);
6717   %}
6718   ins_pipe(ialu_reg);
6719 %}
6720 
6721 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6722   predicate(UseCountTrailingZerosInstruction);
6723   match(Set dst (CountTrailingZerosL src));
6724   effect(KILL cr);
6725 
6726   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6727   ins_encode %{
6728     __ tzcntq($dst$$Register, $src$$Register);
6729   %}
6730   ins_pipe(ialu_reg);
6731 %}
6732 
6733 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6734   predicate(!UseCountTrailingZerosInstruction);
6735   match(Set dst (CountTrailingZerosL src));
6736   effect(KILL cr);
6737 
6738   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6739             "jnz     done\n\t"
6740             "movl    $dst, 64\n"
6741       "done:" %}
6742   ins_encode %{
6743     Register Rdst = $dst$$Register;
6744     Label done;
6745     __ bsfq(Rdst, $src$$Register);
6746     __ jccb(Assembler::notZero, done);
6747     __ movl(Rdst, BitsPerLong);
6748     __ bind(done);
6749   %}
6750   ins_pipe(ialu_reg);
6751 %}
6752 
6753 
6754 //---------- Population Count Instructions -------------------------------------
6755 
6756 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6757   predicate(UsePopCountInstruction);
6758   match(Set dst (PopCountI src));
6759   effect(KILL cr);
6760 
6761   format %{ "popcnt  $dst, $src" %}
6762   ins_encode %{
6763     __ popcntl($dst$$Register, $src$$Register);
6764   %}
6765   ins_pipe(ialu_reg);
6766 %}
6767 
6768 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6769   predicate(UsePopCountInstruction);
6770   match(Set dst (PopCountI (LoadI mem)));
6771   effect(KILL cr);
6772 
6773   format %{ "popcnt  $dst, $mem" %}
6774   ins_encode %{
6775     __ popcntl($dst$$Register, $mem$$Address);
6776   %}
6777   ins_pipe(ialu_reg);
6778 %}
6779 
6780 // Note: Long.bitCount(long) returns an int.
6781 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6782   predicate(UsePopCountInstruction);
6783   match(Set dst (PopCountL src));
6784   effect(KILL cr);
6785 
6786   format %{ "popcnt  $dst, $src" %}
6787   ins_encode %{
6788     __ popcntq($dst$$Register, $src$$Register);
6789   %}
6790   ins_pipe(ialu_reg);
6791 %}
6792 
6793 // Note: Long.bitCount(long) returns an int.
6794 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6795   predicate(UsePopCountInstruction);
6796   match(Set dst (PopCountL (LoadL mem)));
6797   effect(KILL cr);
6798 
6799   format %{ "popcnt  $dst, $mem" %}
6800   ins_encode %{
6801     __ popcntq($dst$$Register, $mem$$Address);
6802   %}
6803   ins_pipe(ialu_reg);
6804 %}
6805 
6806 
6807 //----------MemBar Instructions-----------------------------------------------
6808 // Memory barrier flavors
6809 
6810 instruct membar_acquire()
6811 %{
6812   match(MemBarAcquire);
6813   match(LoadFence);
6814   ins_cost(0);
6815 
6816   size(0);
6817   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6818   ins_encode();
6819   ins_pipe(empty);
6820 %}
6821 
6822 instruct membar_acquire_lock()
6823 %{
6824   match(MemBarAcquireLock);
6825   ins_cost(0);
6826 
6827   size(0);
6828   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6829   ins_encode();
6830   ins_pipe(empty);
6831 %}
6832 
6833 instruct membar_release()
6834 %{
6835   match(MemBarRelease);
6836   match(StoreFence);
6837   ins_cost(0);
6838 
6839   size(0);
6840   format %{ "MEMBAR-release ! (empty encoding)" %}
6841   ins_encode();
6842   ins_pipe(empty);
6843 %}
6844 
6845 instruct membar_release_lock()
6846 %{
6847   match(MemBarReleaseLock);
6848   ins_cost(0);
6849 
6850   size(0);
6851   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6852   ins_encode();
6853   ins_pipe(empty);
6854 %}
6855 
6856 instruct membar_volatile(rFlagsReg cr) %{
6857   match(MemBarVolatile);
6858   effect(KILL cr);
6859   ins_cost(400);
6860 
6861   format %{
6862     $$template
6863     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6864   %}
6865   ins_encode %{
6866     __ membar(Assembler::StoreLoad);
6867   %}
6868   ins_pipe(pipe_slow);
6869 %}
6870 
6871 instruct unnecessary_membar_volatile()
6872 %{
6873   match(MemBarVolatile);
6874   predicate(Matcher::post_store_load_barrier(n));
6875   ins_cost(0);
6876 
6877   size(0);
6878   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6879   ins_encode();
6880   ins_pipe(empty);
6881 %}
6882 
6883 instruct membar_storestore() %{
6884   match(MemBarStoreStore);
6885   ins_cost(0);
6886 
6887   size(0);
6888   format %{ "MEMBAR-storestore (empty encoding)" %}
6889   ins_encode( );
6890   ins_pipe(empty);
6891 %}
6892 
6893 //----------Move Instructions--------------------------------------------------
6894 
6895 instruct castX2P(rRegP dst, rRegL src)
6896 %{
6897   match(Set dst (CastX2P src));
6898 
6899   format %{ "movq    $dst, $src\t# long->ptr" %}
6900   ins_encode %{
6901     if ($dst$$reg != $src$$reg) {
6902       __ movptr($dst$$Register, $src$$Register);
6903     }
6904   %}
6905   ins_pipe(ialu_reg_reg); // XXX
6906 %}
6907 
6908 instruct castP2X(rRegL dst, rRegP src)
6909 %{
6910   match(Set dst (CastP2X src));
6911 
6912   format %{ "movq    $dst, $src\t# ptr -> long" %}
6913   ins_encode %{
6914     if ($dst$$reg != $src$$reg) {
6915       __ movptr($dst$$Register, $src$$Register);
6916     }
6917   %}
6918   ins_pipe(ialu_reg_reg); // XXX
6919 %}
6920 
6921 // Convert oop into int for vectors alignment masking
6922 instruct convP2I(rRegI dst, rRegP src)
6923 %{
6924   match(Set dst (ConvL2I (CastP2X src)));
6925 
6926   format %{ "movl    $dst, $src\t# ptr -> int" %}
6927   ins_encode %{
6928     __ movl($dst$$Register, $src$$Register);
6929   %}
6930   ins_pipe(ialu_reg_reg); // XXX
6931 %}
6932 
6933 // Convert compressed oop into int for vectors alignment masking
6934 // in case of 32bit oops (heap < 4Gb).
6935 instruct convN2I(rRegI dst, rRegN src)
6936 %{
6937   predicate(CompressedOops::shift() == 0);
6938   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6939 
6940   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6941   ins_encode %{
6942     __ movl($dst$$Register, $src$$Register);
6943   %}
6944   ins_pipe(ialu_reg_reg); // XXX
6945 %}
6946 
6947 // Convert oop pointer into compressed form
6948 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6949   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6950   match(Set dst (EncodeP src));
6951   effect(KILL cr);
6952   format %{ "encode_heap_oop $dst,$src" %}
6953   ins_encode %{
6954     Register s = $src$$Register;
6955     Register d = $dst$$Register;
6956     if (s != d) {
6957       __ movq(d, s);
6958     }
6959     __ encode_heap_oop(d);
6960   %}
6961   ins_pipe(ialu_reg_long);
6962 %}
6963 
6964 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6965   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6966   match(Set dst (EncodeP src));
6967   effect(KILL cr);
6968   format %{ "encode_heap_oop_not_null $dst,$src" %}
6969   ins_encode %{
6970     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6971   %}
6972   ins_pipe(ialu_reg_long);
6973 %}
6974 
6975 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6976   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6977             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6978   match(Set dst (DecodeN src));
6979   effect(KILL cr);
6980   format %{ "decode_heap_oop $dst,$src" %}
6981   ins_encode %{
6982     Register s = $src$$Register;
6983     Register d = $dst$$Register;
6984     if (s != d) {
6985       __ movq(d, s);
6986     }
6987     __ decode_heap_oop(d);
6988   %}
6989   ins_pipe(ialu_reg_long);
6990 %}
6991 
6992 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6993   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6994             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6995   match(Set dst (DecodeN src));
6996   effect(KILL cr);
6997   format %{ "decode_heap_oop_not_null $dst,$src" %}
6998   ins_encode %{
6999     Register s = $src$$Register;
7000     Register d = $dst$$Register;
7001     if (s != d) {
7002       __ decode_heap_oop_not_null(d, s);
7003     } else {
7004       __ decode_heap_oop_not_null(d);
7005     }
7006   %}
7007   ins_pipe(ialu_reg_long);
7008 %}
7009 
7010 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
7011   match(Set dst (EncodePKlass src));
7012   effect(KILL cr);
7013   format %{ "encode_klass_not_null $dst,$src" %}
7014   ins_encode %{
7015     __ encode_klass_not_null($dst$$Register, $src$$Register);
7016   %}
7017   ins_pipe(ialu_reg_long);
7018 %}
7019 
7020 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7021   match(Set dst (DecodeNKlass src));
7022   effect(KILL cr);
7023   format %{ "decode_klass_not_null $dst,$src" %}
7024   ins_encode %{
7025     Register s = $src$$Register;
7026     Register d = $dst$$Register;
7027     if (s != d) {
7028       __ decode_klass_not_null(d, s);
7029     } else {
7030       __ decode_klass_not_null(d);
7031     }
7032   %}
7033   ins_pipe(ialu_reg_long);
7034 %}
7035 
7036 
7037 //----------Conditional Move---------------------------------------------------
7038 // Jump
7039 // dummy instruction for generating temp registers
7040 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
7041   match(Jump (LShiftL switch_val shift));
7042   ins_cost(350);
7043   predicate(false);
7044   effect(TEMP dest);
7045 
7046   format %{ "leaq    $dest, [$constantaddress]\n\t"
7047             "jmp     [$dest + $switch_val << $shift]\n\t" %}
7048   ins_encode %{
7049     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7050     // to do that and the compiler is using that register as one it can allocate.
7051     // So we build it all by hand.
7052     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
7053     // ArrayAddress dispatch(table, index);
7054     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
7055     __ lea($dest$$Register, $constantaddress);
7056     __ jmp(dispatch);
7057   %}
7058   ins_pipe(pipe_jmp);
7059 %}
7060 
7061 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
7062   match(Jump (AddL (LShiftL switch_val shift) offset));
7063   ins_cost(350);
7064   effect(TEMP dest);
7065 
7066   format %{ "leaq    $dest, [$constantaddress]\n\t"
7067             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
7068   ins_encode %{
7069     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7070     // to do that and the compiler is using that register as one it can allocate.
7071     // So we build it all by hand.
7072     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7073     // ArrayAddress dispatch(table, index);
7074     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7075     __ lea($dest$$Register, $constantaddress);
7076     __ jmp(dispatch);
7077   %}
7078   ins_pipe(pipe_jmp);
7079 %}
7080 
7081 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
7082   match(Jump switch_val);
7083   ins_cost(350);
7084   effect(TEMP dest);
7085 
7086   format %{ "leaq    $dest, [$constantaddress]\n\t"
7087             "jmp     [$dest + $switch_val]\n\t" %}
7088   ins_encode %{
7089     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7090     // to do that and the compiler is using that register as one it can allocate.
7091     // So we build it all by hand.
7092     // Address index(noreg, switch_reg, Address::times_1);
7093     // ArrayAddress dispatch(table, index);
7094     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
7095     __ lea($dest$$Register, $constantaddress);
7096     __ jmp(dispatch);
7097   %}
7098   ins_pipe(pipe_jmp);
7099 %}
7100 
7101 // Conditional move
7102 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
7103 %{
7104   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7105 
7106   ins_cost(200); // XXX
7107   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7108   opcode(0x0F, 0x40);
7109   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7110   ins_pipe(pipe_cmov_reg);
7111 %}
7112 
7113 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
7114   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7115 
7116   ins_cost(200); // XXX
7117   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7118   opcode(0x0F, 0x40);
7119   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7120   ins_pipe(pipe_cmov_reg);
7121 %}
7122 
7123 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
7124   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7125   ins_cost(200);
7126   expand %{
7127     cmovI_regU(cop, cr, dst, src);
7128   %}
7129 %}
7130 
7131 // Conditional move
7132 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
7133   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7134 
7135   ins_cost(250); // XXX
7136   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7137   opcode(0x0F, 0x40);
7138   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7139   ins_pipe(pipe_cmov_mem);
7140 %}
7141 
7142 // Conditional move
7143 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
7144 %{
7145   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7146 
7147   ins_cost(250); // XXX
7148   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7149   opcode(0x0F, 0x40);
7150   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7151   ins_pipe(pipe_cmov_mem);
7152 %}
7153 
7154 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
7155   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7156   ins_cost(250);
7157   expand %{
7158     cmovI_memU(cop, cr, dst, src);
7159   %}
7160 %}
7161 
7162 // Conditional move
7163 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
7164 %{
7165   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7166 
7167   ins_cost(200); // XXX
7168   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
7169   opcode(0x0F, 0x40);
7170   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7171   ins_pipe(pipe_cmov_reg);
7172 %}
7173 
7174 // Conditional move
7175 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
7176 %{
7177   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7178 
7179   ins_cost(200); // XXX
7180   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
7181   opcode(0x0F, 0x40);
7182   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7183   ins_pipe(pipe_cmov_reg);
7184 %}
7185 
7186 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
7187   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7188   ins_cost(200);
7189   expand %{
7190     cmovN_regU(cop, cr, dst, src);
7191   %}
7192 %}
7193 
7194 // Conditional move
7195 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
7196 %{
7197   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7198 
7199   ins_cost(200); // XXX
7200   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
7201   opcode(0x0F, 0x40);
7202   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7203   ins_pipe(pipe_cmov_reg);  // XXX
7204 %}
7205 
7206 // Conditional move
7207 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
7208 %{
7209   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7210 
7211   ins_cost(200); // XXX
7212   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
7213   opcode(0x0F, 0x40);
7214   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7215   ins_pipe(pipe_cmov_reg); // XXX
7216 %}
7217 
7218 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
7219   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7220   ins_cost(200);
7221   expand %{
7222     cmovP_regU(cop, cr, dst, src);
7223   %}
7224 %}
7225 
7226 // DISABLED: Requires the ADLC to emit a bottom_type call that
7227 // correctly meets the two pointer arguments; one is an incoming
7228 // register but the other is a memory operand.  ALSO appears to
7229 // be buggy with implicit null checks.
7230 //
7231 //// Conditional move
7232 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
7233 //%{
7234 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7235 //  ins_cost(250);
7236 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7237 //  opcode(0x0F,0x40);
7238 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7239 //  ins_pipe( pipe_cmov_mem );
7240 //%}
7241 //
7242 //// Conditional move
7243 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
7244 //%{
7245 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7246 //  ins_cost(250);
7247 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7248 //  opcode(0x0F,0x40);
7249 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7250 //  ins_pipe( pipe_cmov_mem );
7251 //%}
7252 
7253 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
7254 %{
7255   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7256 
7257   ins_cost(200); // XXX
7258   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7259   opcode(0x0F, 0x40);
7260   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7261   ins_pipe(pipe_cmov_reg);  // XXX
7262 %}
7263 
7264 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
7265 %{
7266   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7267 
7268   ins_cost(200); // XXX
7269   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7270   opcode(0x0F, 0x40);
7271   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7272   ins_pipe(pipe_cmov_mem);  // XXX
7273 %}
7274 
7275 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
7276 %{
7277   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7278 
7279   ins_cost(200); // XXX
7280   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7281   opcode(0x0F, 0x40);
7282   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7283   ins_pipe(pipe_cmov_reg); // XXX
7284 %}
7285 
7286 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7287   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7288   ins_cost(200);
7289   expand %{
7290     cmovL_regU(cop, cr, dst, src);
7291   %}
7292 %}
7293 
7294 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7295 %{
7296   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7297 
7298   ins_cost(200); // XXX
7299   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7300   opcode(0x0F, 0x40);
7301   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7302   ins_pipe(pipe_cmov_mem); // XXX
7303 %}
7304 
7305 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7306   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7307   ins_cost(200);
7308   expand %{
7309     cmovL_memU(cop, cr, dst, src);
7310   %}
7311 %}
7312 
7313 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7314 %{
7315   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7316 
7317   ins_cost(200); // XXX
7318   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7319             "movss     $dst, $src\n"
7320     "skip:" %}
7321   ins_encode %{
7322     Label Lskip;
7323     // Invert sense of branch from sense of CMOV
7324     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7325     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7326     __ bind(Lskip);
7327   %}
7328   ins_pipe(pipe_slow);
7329 %}
7330 
7331 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7332 // %{
7333 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7334 
7335 //   ins_cost(200); // XXX
7336 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7337 //             "movss     $dst, $src\n"
7338 //     "skip:" %}
7339 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7340 //   ins_pipe(pipe_slow);
7341 // %}
7342 
7343 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7344 %{
7345   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7346 
7347   ins_cost(200); // XXX
7348   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7349             "movss     $dst, $src\n"
7350     "skip:" %}
7351   ins_encode %{
7352     Label Lskip;
7353     // Invert sense of branch from sense of CMOV
7354     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7355     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7356     __ bind(Lskip);
7357   %}
7358   ins_pipe(pipe_slow);
7359 %}
7360 
7361 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7362   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7363   ins_cost(200);
7364   expand %{
7365     cmovF_regU(cop, cr, dst, src);
7366   %}
7367 %}
7368 
7369 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7370 %{
7371   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7372 
7373   ins_cost(200); // XXX
7374   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7375             "movsd     $dst, $src\n"
7376     "skip:" %}
7377   ins_encode %{
7378     Label Lskip;
7379     // Invert sense of branch from sense of CMOV
7380     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7381     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7382     __ bind(Lskip);
7383   %}
7384   ins_pipe(pipe_slow);
7385 %}
7386 
7387 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7388 %{
7389   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7390 
7391   ins_cost(200); // XXX
7392   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7393             "movsd     $dst, $src\n"
7394     "skip:" %}
7395   ins_encode %{
7396     Label Lskip;
7397     // Invert sense of branch from sense of CMOV
7398     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7399     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7400     __ bind(Lskip);
7401   %}
7402   ins_pipe(pipe_slow);
7403 %}
7404 
7405 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7406   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7407   ins_cost(200);
7408   expand %{
7409     cmovD_regU(cop, cr, dst, src);
7410   %}
7411 %}
7412 
7413 //----------Arithmetic Instructions--------------------------------------------
7414 //----------Addition Instructions----------------------------------------------
7415 
7416 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7417 %{
7418   match(Set dst (AddI dst src));
7419   effect(KILL cr);
7420 
7421   format %{ "addl    $dst, $src\t# int" %}
7422   opcode(0x03);
7423   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7424   ins_pipe(ialu_reg_reg);
7425 %}
7426 
7427 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7428 %{
7429   match(Set dst (AddI dst src));
7430   effect(KILL cr);
7431 
7432   format %{ "addl    $dst, $src\t# int" %}
7433   opcode(0x81, 0x00); /* /0 id */
7434   ins_encode(OpcSErm(dst, src), Con8or32(src));
7435   ins_pipe( ialu_reg );
7436 %}
7437 
7438 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7439 %{
7440   match(Set dst (AddI dst (LoadI src)));
7441   effect(KILL cr);
7442 
7443   ins_cost(125); // XXX
7444   format %{ "addl    $dst, $src\t# int" %}
7445   opcode(0x03);
7446   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7447   ins_pipe(ialu_reg_mem);
7448 %}
7449 
7450 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7451 %{
7452   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7453   effect(KILL cr);
7454 
7455   ins_cost(150); // XXX
7456   format %{ "addl    $dst, $src\t# int" %}
7457   opcode(0x01); /* Opcode 01 /r */
7458   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7459   ins_pipe(ialu_mem_reg);
7460 %}
7461 
7462 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7463 %{
7464   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7465   effect(KILL cr);
7466 
7467   ins_cost(125); // XXX
7468   format %{ "addl    $dst, $src\t# int" %}
7469   opcode(0x81); /* Opcode 81 /0 id */
7470   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7471   ins_pipe(ialu_mem_imm);
7472 %}
7473 
7474 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7475 %{
7476   predicate(UseIncDec);
7477   match(Set dst (AddI dst src));
7478   effect(KILL cr);
7479 
7480   format %{ "incl    $dst\t# int" %}
7481   opcode(0xFF, 0x00); // FF /0
7482   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7483   ins_pipe(ialu_reg);
7484 %}
7485 
7486 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7487 %{
7488   predicate(UseIncDec);
7489   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7490   effect(KILL cr);
7491 
7492   ins_cost(125); // XXX
7493   format %{ "incl    $dst\t# int" %}
7494   opcode(0xFF); /* Opcode FF /0 */
7495   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7496   ins_pipe(ialu_mem_imm);
7497 %}
7498 
7499 // XXX why does that use AddI
7500 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7501 %{
7502   predicate(UseIncDec);
7503   match(Set dst (AddI dst src));
7504   effect(KILL cr);
7505 
7506   format %{ "decl    $dst\t# int" %}
7507   opcode(0xFF, 0x01); // FF /1
7508   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7509   ins_pipe(ialu_reg);
7510 %}
7511 
7512 // XXX why does that use AddI
7513 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7514 %{
7515   predicate(UseIncDec);
7516   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7517   effect(KILL cr);
7518 
7519   ins_cost(125); // XXX
7520   format %{ "decl    $dst\t# int" %}
7521   opcode(0xFF); /* Opcode FF /1 */
7522   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7523   ins_pipe(ialu_mem_imm);
7524 %}
7525 
7526 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7527 %{
7528   match(Set dst (AddI src0 src1));
7529 
7530   ins_cost(110);
7531   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7532   opcode(0x8D); /* 0x8D /r */
7533   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7534   ins_pipe(ialu_reg_reg);
7535 %}
7536 
7537 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7538 %{
7539   match(Set dst (AddL dst src));
7540   effect(KILL cr);
7541 
7542   format %{ "addq    $dst, $src\t# long" %}
7543   opcode(0x03);
7544   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7545   ins_pipe(ialu_reg_reg);
7546 %}
7547 
7548 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7549 %{
7550   match(Set dst (AddL dst src));
7551   effect(KILL cr);
7552 
7553   format %{ "addq    $dst, $src\t# long" %}
7554   opcode(0x81, 0x00); /* /0 id */
7555   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7556   ins_pipe( ialu_reg );
7557 %}
7558 
7559 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7560 %{
7561   match(Set dst (AddL dst (LoadL src)));
7562   effect(KILL cr);
7563 
7564   ins_cost(125); // XXX
7565   format %{ "addq    $dst, $src\t# long" %}
7566   opcode(0x03);
7567   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7568   ins_pipe(ialu_reg_mem);
7569 %}
7570 
7571 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7572 %{
7573   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7574   effect(KILL cr);
7575 
7576   ins_cost(150); // XXX
7577   format %{ "addq    $dst, $src\t# long" %}
7578   opcode(0x01); /* Opcode 01 /r */
7579   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7580   ins_pipe(ialu_mem_reg);
7581 %}
7582 
7583 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7584 %{
7585   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7586   effect(KILL cr);
7587 
7588   ins_cost(125); // XXX
7589   format %{ "addq    $dst, $src\t# long" %}
7590   opcode(0x81); /* Opcode 81 /0 id */
7591   ins_encode(REX_mem_wide(dst),
7592              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7593   ins_pipe(ialu_mem_imm);
7594 %}
7595 
7596 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7597 %{
7598   predicate(UseIncDec);
7599   match(Set dst (AddL dst src));
7600   effect(KILL cr);
7601 
7602   format %{ "incq    $dst\t# long" %}
7603   opcode(0xFF, 0x00); // FF /0
7604   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7605   ins_pipe(ialu_reg);
7606 %}
7607 
7608 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7609 %{
7610   predicate(UseIncDec);
7611   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7612   effect(KILL cr);
7613 
7614   ins_cost(125); // XXX
7615   format %{ "incq    $dst\t# long" %}
7616   opcode(0xFF); /* Opcode FF /0 */
7617   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7618   ins_pipe(ialu_mem_imm);
7619 %}
7620 
7621 // XXX why does that use AddL
7622 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7623 %{
7624   predicate(UseIncDec);
7625   match(Set dst (AddL dst src));
7626   effect(KILL cr);
7627 
7628   format %{ "decq    $dst\t# long" %}
7629   opcode(0xFF, 0x01); // FF /1
7630   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7631   ins_pipe(ialu_reg);
7632 %}
7633 
7634 // XXX why does that use AddL
7635 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7636 %{
7637   predicate(UseIncDec);
7638   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7639   effect(KILL cr);
7640 
7641   ins_cost(125); // XXX
7642   format %{ "decq    $dst\t# long" %}
7643   opcode(0xFF); /* Opcode FF /1 */
7644   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7645   ins_pipe(ialu_mem_imm);
7646 %}
7647 
7648 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7649 %{
7650   match(Set dst (AddL src0 src1));
7651 
7652   ins_cost(110);
7653   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7654   opcode(0x8D); /* 0x8D /r */
7655   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7656   ins_pipe(ialu_reg_reg);
7657 %}
7658 
7659 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7660 %{
7661   match(Set dst (AddP dst src));
7662   effect(KILL cr);
7663 
7664   format %{ "addq    $dst, $src\t# ptr" %}
7665   opcode(0x03);
7666   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7667   ins_pipe(ialu_reg_reg);
7668 %}
7669 
7670 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7671 %{
7672   match(Set dst (AddP dst src));
7673   effect(KILL cr);
7674 
7675   format %{ "addq    $dst, $src\t# ptr" %}
7676   opcode(0x81, 0x00); /* /0 id */
7677   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7678   ins_pipe( ialu_reg );
7679 %}
7680 
7681 // XXX addP mem ops ????
7682 
7683 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7684 %{
7685   match(Set dst (AddP src0 src1));
7686 
7687   ins_cost(110);
7688   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7689   opcode(0x8D); /* 0x8D /r */
7690   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7691   ins_pipe(ialu_reg_reg);
7692 %}
7693 
7694 instruct checkCastPP(rRegP dst)
7695 %{
7696   match(Set dst (CheckCastPP dst));
7697 
7698   size(0);
7699   format %{ "# checkcastPP of $dst" %}
7700   ins_encode(/* empty encoding */);
7701   ins_pipe(empty);
7702 %}
7703 
7704 instruct castPP(rRegP dst)
7705 %{
7706   match(Set dst (CastPP dst));
7707 
7708   size(0);
7709   format %{ "# castPP of $dst" %}
7710   ins_encode(/* empty encoding */);
7711   ins_pipe(empty);
7712 %}
7713 
7714 instruct castII(rRegI dst)
7715 %{
7716   match(Set dst (CastII dst));
7717 
7718   size(0);
7719   format %{ "# castII of $dst" %}
7720   ins_encode(/* empty encoding */);
7721   ins_cost(0);
7722   ins_pipe(empty);
7723 %}
7724 
7725 // LoadP-locked same as a regular LoadP when used with compare-swap
7726 instruct loadPLocked(rRegP dst, memory mem)
7727 %{
7728   match(Set dst (LoadPLocked mem));
7729 
7730   ins_cost(125); // XXX
7731   format %{ "movq    $dst, $mem\t# ptr locked" %}
7732   opcode(0x8B);
7733   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7734   ins_pipe(ialu_reg_mem); // XXX
7735 %}
7736 
7737 // Conditional-store of the updated heap-top.
7738 // Used during allocation of the shared heap.
7739 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7740 
7741 instruct storePConditional(memory heap_top_ptr,
7742                            rax_RegP oldval, rRegP newval,
7743                            rFlagsReg cr)
7744 %{
7745   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7746 
7747   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7748             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7749   opcode(0x0F, 0xB1);
7750   ins_encode(lock_prefix,
7751              REX_reg_mem_wide(newval, heap_top_ptr),
7752              OpcP, OpcS,
7753              reg_mem(newval, heap_top_ptr));
7754   ins_pipe(pipe_cmpxchg);
7755 %}
7756 
7757 // Conditional-store of an int value.
7758 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7759 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7760 %{
7761   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7762   effect(KILL oldval);
7763 
7764   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7765   opcode(0x0F, 0xB1);
7766   ins_encode(lock_prefix,
7767              REX_reg_mem(newval, mem),
7768              OpcP, OpcS,
7769              reg_mem(newval, mem));
7770   ins_pipe(pipe_cmpxchg);
7771 %}
7772 
7773 // Conditional-store of a long value.
7774 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7775 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7776 %{
7777   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7778   effect(KILL oldval);
7779 
7780   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7781   opcode(0x0F, 0xB1);
7782   ins_encode(lock_prefix,
7783              REX_reg_mem_wide(newval, mem),
7784              OpcP, OpcS,
7785              reg_mem(newval, mem));
7786   ins_pipe(pipe_cmpxchg);
7787 %}
7788 
7789 
7790 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7791 instruct compareAndSwapP(rRegI res,
7792                          memory mem_ptr,
7793                          rax_RegP oldval, rRegP newval,
7794                          rFlagsReg cr)
7795 %{
7796   predicate(VM_Version::supports_cx8());
7797   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7798   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7799   effect(KILL cr, KILL oldval);
7800 
7801   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7802             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7803             "sete    $res\n\t"
7804             "movzbl  $res, $res" %}
7805   opcode(0x0F, 0xB1);
7806   ins_encode(lock_prefix,
7807              REX_reg_mem_wide(newval, mem_ptr),
7808              OpcP, OpcS,
7809              reg_mem(newval, mem_ptr),
7810              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7811              REX_reg_breg(res, res), // movzbl
7812              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7813   ins_pipe( pipe_cmpxchg );
7814 %}
7815 
7816 instruct compareAndSwapL(rRegI res,
7817                          memory mem_ptr,
7818                          rax_RegL oldval, rRegL newval,
7819                          rFlagsReg cr)
7820 %{
7821   predicate(VM_Version::supports_cx8());
7822   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7823   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7824   effect(KILL cr, KILL oldval);
7825 
7826   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7827             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7828             "sete    $res\n\t"
7829             "movzbl  $res, $res" %}
7830   opcode(0x0F, 0xB1);
7831   ins_encode(lock_prefix,
7832              REX_reg_mem_wide(newval, mem_ptr),
7833              OpcP, OpcS,
7834              reg_mem(newval, mem_ptr),
7835              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7836              REX_reg_breg(res, res), // movzbl
7837              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7838   ins_pipe( pipe_cmpxchg );
7839 %}
7840 
7841 instruct compareAndSwapI(rRegI res,
7842                          memory mem_ptr,
7843                          rax_RegI oldval, rRegI newval,
7844                          rFlagsReg cr)
7845 %{
7846   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7847   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7848   effect(KILL cr, KILL oldval);
7849 
7850   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7851             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7852             "sete    $res\n\t"
7853             "movzbl  $res, $res" %}
7854   opcode(0x0F, 0xB1);
7855   ins_encode(lock_prefix,
7856              REX_reg_mem(newval, mem_ptr),
7857              OpcP, OpcS,
7858              reg_mem(newval, mem_ptr),
7859              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7860              REX_reg_breg(res, res), // movzbl
7861              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7862   ins_pipe( pipe_cmpxchg );
7863 %}
7864 
7865 instruct compareAndSwapB(rRegI res,
7866                          memory mem_ptr,
7867                          rax_RegI oldval, rRegI newval,
7868                          rFlagsReg cr)
7869 %{
7870   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7871   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7872   effect(KILL cr, KILL oldval);
7873 
7874   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7875             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7876             "sete    $res\n\t"
7877             "movzbl  $res, $res" %}
7878   opcode(0x0F, 0xB0);
7879   ins_encode(lock_prefix,
7880              REX_breg_mem(newval, mem_ptr),
7881              OpcP, OpcS,
7882              reg_mem(newval, mem_ptr),
7883              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7884              REX_reg_breg(res, res), // movzbl
7885              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7886   ins_pipe( pipe_cmpxchg );
7887 %}
7888 
7889 instruct compareAndSwapS(rRegI res,
7890                          memory mem_ptr,
7891                          rax_RegI oldval, rRegI newval,
7892                          rFlagsReg cr)
7893 %{
7894   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7895   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7896   effect(KILL cr, KILL oldval);
7897 
7898   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7899             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7900             "sete    $res\n\t"
7901             "movzbl  $res, $res" %}
7902   opcode(0x0F, 0xB1);
7903   ins_encode(lock_prefix,
7904              SizePrefix,
7905              REX_reg_mem(newval, mem_ptr),
7906              OpcP, OpcS,
7907              reg_mem(newval, mem_ptr),
7908              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7909              REX_reg_breg(res, res), // movzbl
7910              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7911   ins_pipe( pipe_cmpxchg );
7912 %}
7913 
7914 instruct compareAndSwapN(rRegI res,
7915                           memory mem_ptr,
7916                           rax_RegN oldval, rRegN newval,
7917                           rFlagsReg cr) %{
7918   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7919   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7920   effect(KILL cr, KILL oldval);
7921 
7922   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7923             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7924             "sete    $res\n\t"
7925             "movzbl  $res, $res" %}
7926   opcode(0x0F, 0xB1);
7927   ins_encode(lock_prefix,
7928              REX_reg_mem(newval, mem_ptr),
7929              OpcP, OpcS,
7930              reg_mem(newval, mem_ptr),
7931              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7932              REX_reg_breg(res, res), // movzbl
7933              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7934   ins_pipe( pipe_cmpxchg );
7935 %}
7936 
7937 instruct compareAndExchangeB(
7938                          memory mem_ptr,
7939                          rax_RegI oldval, rRegI newval,
7940                          rFlagsReg cr)
7941 %{
7942   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7943   effect(KILL cr);
7944 
7945   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7946             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7947   opcode(0x0F, 0xB0);
7948   ins_encode(lock_prefix,
7949              REX_breg_mem(newval, mem_ptr),
7950              OpcP, OpcS,
7951              reg_mem(newval, mem_ptr) // lock cmpxchg
7952              );
7953   ins_pipe( pipe_cmpxchg );
7954 %}
7955 
7956 instruct compareAndExchangeS(
7957                          memory mem_ptr,
7958                          rax_RegI oldval, rRegI newval,
7959                          rFlagsReg cr)
7960 %{
7961   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7962   effect(KILL cr);
7963 
7964   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7965             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7966   opcode(0x0F, 0xB1);
7967   ins_encode(lock_prefix,
7968              SizePrefix,
7969              REX_reg_mem(newval, mem_ptr),
7970              OpcP, OpcS,
7971              reg_mem(newval, mem_ptr) // lock cmpxchg
7972              );
7973   ins_pipe( pipe_cmpxchg );
7974 %}
7975 
7976 instruct compareAndExchangeI(
7977                          memory mem_ptr,
7978                          rax_RegI oldval, rRegI newval,
7979                          rFlagsReg cr)
7980 %{
7981   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7982   effect(KILL cr);
7983 
7984   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7985             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7986   opcode(0x0F, 0xB1);
7987   ins_encode(lock_prefix,
7988              REX_reg_mem(newval, mem_ptr),
7989              OpcP, OpcS,
7990              reg_mem(newval, mem_ptr) // lock cmpxchg
7991              );
7992   ins_pipe( pipe_cmpxchg );
7993 %}
7994 
7995 instruct compareAndExchangeL(
7996                          memory mem_ptr,
7997                          rax_RegL oldval, rRegL newval,
7998                          rFlagsReg cr)
7999 %{
8000   predicate(VM_Version::supports_cx8());
8001   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
8002   effect(KILL cr);
8003 
8004   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8005             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8006   opcode(0x0F, 0xB1);
8007   ins_encode(lock_prefix,
8008              REX_reg_mem_wide(newval, mem_ptr),
8009              OpcP, OpcS,
8010              reg_mem(newval, mem_ptr)  // lock cmpxchg
8011             );
8012   ins_pipe( pipe_cmpxchg );
8013 %}
8014 
8015 instruct compareAndExchangeN(
8016                           memory mem_ptr,
8017                           rax_RegN oldval, rRegN newval,
8018                           rFlagsReg cr) %{
8019   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
8020   effect(KILL cr);
8021 
8022   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8023             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8024   opcode(0x0F, 0xB1);
8025   ins_encode(lock_prefix,
8026              REX_reg_mem(newval, mem_ptr),
8027              OpcP, OpcS,
8028              reg_mem(newval, mem_ptr)  // lock cmpxchg
8029           );
8030   ins_pipe( pipe_cmpxchg );
8031 %}
8032 
8033 instruct compareAndExchangeP(
8034                          memory mem_ptr,
8035                          rax_RegP oldval, rRegP newval,
8036                          rFlagsReg cr)
8037 %{
8038   predicate(VM_Version::supports_cx8());
8039   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
8040   effect(KILL cr);
8041 
8042   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8043             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8044   opcode(0x0F, 0xB1);
8045   ins_encode(lock_prefix,
8046              REX_reg_mem_wide(newval, mem_ptr),
8047              OpcP, OpcS,
8048              reg_mem(newval, mem_ptr)  // lock cmpxchg
8049           );
8050   ins_pipe( pipe_cmpxchg );
8051 %}
8052 
8053 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8054   predicate(n->as_LoadStore()->result_not_used());
8055   match(Set dummy (GetAndAddB mem add));
8056   effect(KILL cr);
8057   format %{ "ADDB  [$mem],$add" %}
8058   ins_encode %{
8059     __ lock();
8060     __ addb($mem$$Address, $add$$constant);
8061   %}
8062   ins_pipe( pipe_cmpxchg );
8063 %}
8064 
8065 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
8066   match(Set newval (GetAndAddB mem newval));
8067   effect(KILL cr);
8068   format %{ "XADDB  [$mem],$newval" %}
8069   ins_encode %{
8070     __ lock();
8071     __ xaddb($mem$$Address, $newval$$Register);
8072   %}
8073   ins_pipe( pipe_cmpxchg );
8074 %}
8075 
8076 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8077   predicate(n->as_LoadStore()->result_not_used());
8078   match(Set dummy (GetAndAddS mem add));
8079   effect(KILL cr);
8080   format %{ "ADDW  [$mem],$add" %}
8081   ins_encode %{
8082     __ lock();
8083     __ addw($mem$$Address, $add$$constant);
8084   %}
8085   ins_pipe( pipe_cmpxchg );
8086 %}
8087 
8088 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
8089   match(Set newval (GetAndAddS mem newval));
8090   effect(KILL cr);
8091   format %{ "XADDW  [$mem],$newval" %}
8092   ins_encode %{
8093     __ lock();
8094     __ xaddw($mem$$Address, $newval$$Register);
8095   %}
8096   ins_pipe( pipe_cmpxchg );
8097 %}
8098 
8099 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8100   predicate(n->as_LoadStore()->result_not_used());
8101   match(Set dummy (GetAndAddI mem add));
8102   effect(KILL cr);
8103   format %{ "ADDL  [$mem],$add" %}
8104   ins_encode %{
8105     __ lock();
8106     __ addl($mem$$Address, $add$$constant);
8107   %}
8108   ins_pipe( pipe_cmpxchg );
8109 %}
8110 
8111 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
8112   match(Set newval (GetAndAddI mem newval));
8113   effect(KILL cr);
8114   format %{ "XADDL  [$mem],$newval" %}
8115   ins_encode %{
8116     __ lock();
8117     __ xaddl($mem$$Address, $newval$$Register);
8118   %}
8119   ins_pipe( pipe_cmpxchg );
8120 %}
8121 
8122 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
8123   predicate(n->as_LoadStore()->result_not_used());
8124   match(Set dummy (GetAndAddL mem add));
8125   effect(KILL cr);
8126   format %{ "ADDQ  [$mem],$add" %}
8127   ins_encode %{
8128     __ lock();
8129     __ addq($mem$$Address, $add$$constant);
8130   %}
8131   ins_pipe( pipe_cmpxchg );
8132 %}
8133 
8134 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
8135   match(Set newval (GetAndAddL mem newval));
8136   effect(KILL cr);
8137   format %{ "XADDQ  [$mem],$newval" %}
8138   ins_encode %{
8139     __ lock();
8140     __ xaddq($mem$$Address, $newval$$Register);
8141   %}
8142   ins_pipe( pipe_cmpxchg );
8143 %}
8144 
8145 instruct xchgB( memory mem, rRegI newval) %{
8146   match(Set newval (GetAndSetB mem newval));
8147   format %{ "XCHGB  $newval,[$mem]" %}
8148   ins_encode %{
8149     __ xchgb($newval$$Register, $mem$$Address);
8150   %}
8151   ins_pipe( pipe_cmpxchg );
8152 %}
8153 
8154 instruct xchgS( memory mem, rRegI newval) %{
8155   match(Set newval (GetAndSetS mem newval));
8156   format %{ "XCHGW  $newval,[$mem]" %}
8157   ins_encode %{
8158     __ xchgw($newval$$Register, $mem$$Address);
8159   %}
8160   ins_pipe( pipe_cmpxchg );
8161 %}
8162 
8163 instruct xchgI( memory mem, rRegI newval) %{
8164   match(Set newval (GetAndSetI mem newval));
8165   format %{ "XCHGL  $newval,[$mem]" %}
8166   ins_encode %{
8167     __ xchgl($newval$$Register, $mem$$Address);
8168   %}
8169   ins_pipe( pipe_cmpxchg );
8170 %}
8171 
8172 instruct xchgL( memory mem, rRegL newval) %{
8173   match(Set newval (GetAndSetL mem newval));
8174   format %{ "XCHGL  $newval,[$mem]" %}
8175   ins_encode %{
8176     __ xchgq($newval$$Register, $mem$$Address);
8177   %}
8178   ins_pipe( pipe_cmpxchg );
8179 %}
8180 
8181 instruct xchgP( memory mem, rRegP newval) %{
8182   match(Set newval (GetAndSetP mem newval));
8183   format %{ "XCHGQ  $newval,[$mem]" %}
8184   ins_encode %{
8185     __ xchgq($newval$$Register, $mem$$Address);
8186   %}
8187   ins_pipe( pipe_cmpxchg );
8188 %}
8189 
8190 instruct xchgN( memory mem, rRegN newval) %{
8191   match(Set newval (GetAndSetN mem newval));
8192   format %{ "XCHGL  $newval,$mem]" %}
8193   ins_encode %{
8194     __ xchgl($newval$$Register, $mem$$Address);
8195   %}
8196   ins_pipe( pipe_cmpxchg );
8197 %}
8198 
8199 //----------Abs Instructions-------------------------------------------
8200 
8201 // Integer Absolute Instructions
8202 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
8203 %{
8204   match(Set dst (AbsI src));
8205   effect(TEMP dst, TEMP tmp, KILL cr);
8206   format %{ "movl $tmp, $src\n\t"
8207             "sarl $tmp, 31\n\t"
8208             "movl $dst, $src\n\t"
8209             "xorl $dst, $tmp\n\t"
8210             "subl $dst, $tmp\n"
8211           %}
8212   ins_encode %{
8213     __ movl($tmp$$Register, $src$$Register);
8214     __ sarl($tmp$$Register, 31);
8215     __ movl($dst$$Register, $src$$Register);
8216     __ xorl($dst$$Register, $tmp$$Register);
8217     __ subl($dst$$Register, $tmp$$Register);
8218   %}
8219 
8220   ins_pipe(ialu_reg_reg);
8221 %}
8222 
8223 // Long Absolute Instructions
8224 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
8225 %{
8226   match(Set dst (AbsL src));
8227   effect(TEMP dst, TEMP tmp, KILL cr);
8228   format %{ "movq $tmp, $src\n\t"
8229             "sarq $tmp, 63\n\t"
8230             "movq $dst, $src\n\t"
8231             "xorq $dst, $tmp\n\t"
8232             "subq $dst, $tmp\n"
8233           %}
8234   ins_encode %{
8235     __ movq($tmp$$Register, $src$$Register);
8236     __ sarq($tmp$$Register, 63);
8237     __ movq($dst$$Register, $src$$Register);
8238     __ xorq($dst$$Register, $tmp$$Register);
8239     __ subq($dst$$Register, $tmp$$Register);
8240   %}
8241 
8242   ins_pipe(ialu_reg_reg);
8243 %}
8244 
8245 //----------Subtraction Instructions-------------------------------------------
8246 
8247 // Integer Subtraction Instructions
8248 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8249 %{
8250   match(Set dst (SubI dst src));
8251   effect(KILL cr);
8252 
8253   format %{ "subl    $dst, $src\t# int" %}
8254   opcode(0x2B);
8255   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8256   ins_pipe(ialu_reg_reg);
8257 %}
8258 
8259 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8260 %{
8261   match(Set dst (SubI dst src));
8262   effect(KILL cr);
8263 
8264   format %{ "subl    $dst, $src\t# int" %}
8265   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8266   ins_encode(OpcSErm(dst, src), Con8or32(src));
8267   ins_pipe(ialu_reg);
8268 %}
8269 
8270 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8271 %{
8272   match(Set dst (SubI dst (LoadI src)));
8273   effect(KILL cr);
8274 
8275   ins_cost(125);
8276   format %{ "subl    $dst, $src\t# int" %}
8277   opcode(0x2B);
8278   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8279   ins_pipe(ialu_reg_mem);
8280 %}
8281 
8282 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8283 %{
8284   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8285   effect(KILL cr);
8286 
8287   ins_cost(150);
8288   format %{ "subl    $dst, $src\t# int" %}
8289   opcode(0x29); /* Opcode 29 /r */
8290   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8291   ins_pipe(ialu_mem_reg);
8292 %}
8293 
8294 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
8295 %{
8296   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8297   effect(KILL cr);
8298 
8299   ins_cost(125); // XXX
8300   format %{ "subl    $dst, $src\t# int" %}
8301   opcode(0x81); /* Opcode 81 /5 id */
8302   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8303   ins_pipe(ialu_mem_imm);
8304 %}
8305 
8306 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8307 %{
8308   match(Set dst (SubL dst src));
8309   effect(KILL cr);
8310 
8311   format %{ "subq    $dst, $src\t# long" %}
8312   opcode(0x2B);
8313   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8314   ins_pipe(ialu_reg_reg);
8315 %}
8316 
8317 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
8318 %{
8319   match(Set dst (SubL dst src));
8320   effect(KILL cr);
8321 
8322   format %{ "subq    $dst, $src\t# long" %}
8323   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8324   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8325   ins_pipe(ialu_reg);
8326 %}
8327 
8328 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8329 %{
8330   match(Set dst (SubL dst (LoadL src)));
8331   effect(KILL cr);
8332 
8333   ins_cost(125);
8334   format %{ "subq    $dst, $src\t# long" %}
8335   opcode(0x2B);
8336   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8337   ins_pipe(ialu_reg_mem);
8338 %}
8339 
8340 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8341 %{
8342   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8343   effect(KILL cr);
8344 
8345   ins_cost(150);
8346   format %{ "subq    $dst, $src\t# long" %}
8347   opcode(0x29); /* Opcode 29 /r */
8348   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8349   ins_pipe(ialu_mem_reg);
8350 %}
8351 
8352 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8353 %{
8354   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8355   effect(KILL cr);
8356 
8357   ins_cost(125); // XXX
8358   format %{ "subq    $dst, $src\t# long" %}
8359   opcode(0x81); /* Opcode 81 /5 id */
8360   ins_encode(REX_mem_wide(dst),
8361              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8362   ins_pipe(ialu_mem_imm);
8363 %}
8364 
8365 // Subtract from a pointer
8366 // XXX hmpf???
8367 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8368 %{
8369   match(Set dst (AddP dst (SubI zero src)));
8370   effect(KILL cr);
8371 
8372   format %{ "subq    $dst, $src\t# ptr - int" %}
8373   opcode(0x2B);
8374   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8375   ins_pipe(ialu_reg_reg);
8376 %}
8377 
8378 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8379 %{
8380   match(Set dst (SubI zero dst));
8381   effect(KILL cr);
8382 
8383   format %{ "negl    $dst\t# int" %}
8384   opcode(0xF7, 0x03);  // Opcode F7 /3
8385   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8386   ins_pipe(ialu_reg);
8387 %}
8388 
8389 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8390 %{
8391   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8392   effect(KILL cr);
8393 
8394   format %{ "negl    $dst\t# int" %}
8395   opcode(0xF7, 0x03);  // Opcode F7 /3
8396   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8397   ins_pipe(ialu_reg);
8398 %}
8399 
8400 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8401 %{
8402   match(Set dst (SubL zero dst));
8403   effect(KILL cr);
8404 
8405   format %{ "negq    $dst\t# long" %}
8406   opcode(0xF7, 0x03);  // Opcode F7 /3
8407   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8408   ins_pipe(ialu_reg);
8409 %}
8410 
8411 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8412 %{
8413   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8414   effect(KILL cr);
8415 
8416   format %{ "negq    $dst\t# long" %}
8417   opcode(0xF7, 0x03);  // Opcode F7 /3
8418   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8419   ins_pipe(ialu_reg);
8420 %}
8421 
8422 //----------Multiplication/Division Instructions-------------------------------
8423 // Integer Multiplication Instructions
8424 // Multiply Register
8425 
8426 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8427 %{
8428   match(Set dst (MulI dst src));
8429   effect(KILL cr);
8430 
8431   ins_cost(300);
8432   format %{ "imull   $dst, $src\t# int" %}
8433   opcode(0x0F, 0xAF);
8434   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8435   ins_pipe(ialu_reg_reg_alu0);
8436 %}
8437 
8438 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8439 %{
8440   match(Set dst (MulI src imm));
8441   effect(KILL cr);
8442 
8443   ins_cost(300);
8444   format %{ "imull   $dst, $src, $imm\t# int" %}
8445   opcode(0x69); /* 69 /r id */
8446   ins_encode(REX_reg_reg(dst, src),
8447              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8448   ins_pipe(ialu_reg_reg_alu0);
8449 %}
8450 
8451 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8452 %{
8453   match(Set dst (MulI dst (LoadI src)));
8454   effect(KILL cr);
8455 
8456   ins_cost(350);
8457   format %{ "imull   $dst, $src\t# int" %}
8458   opcode(0x0F, 0xAF);
8459   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8460   ins_pipe(ialu_reg_mem_alu0);
8461 %}
8462 
8463 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8464 %{
8465   match(Set dst (MulI (LoadI src) imm));
8466   effect(KILL cr);
8467 
8468   ins_cost(300);
8469   format %{ "imull   $dst, $src, $imm\t# int" %}
8470   opcode(0x69); /* 69 /r id */
8471   ins_encode(REX_reg_mem(dst, src),
8472              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8473   ins_pipe(ialu_reg_mem_alu0);
8474 %}
8475 
8476 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
8477 %{
8478   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
8479   effect(KILL cr, KILL src2);
8480 
8481   expand %{ mulI_rReg(dst, src1, cr);
8482            mulI_rReg(src2, src3, cr);
8483            addI_rReg(dst, src2, cr); %}
8484 %}
8485 
8486 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8487 %{
8488   match(Set dst (MulL dst src));
8489   effect(KILL cr);
8490 
8491   ins_cost(300);
8492   format %{ "imulq   $dst, $src\t# long" %}
8493   opcode(0x0F, 0xAF);
8494   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8495   ins_pipe(ialu_reg_reg_alu0);
8496 %}
8497 
8498 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8499 %{
8500   match(Set dst (MulL src imm));
8501   effect(KILL cr);
8502 
8503   ins_cost(300);
8504   format %{ "imulq   $dst, $src, $imm\t# long" %}
8505   opcode(0x69); /* 69 /r id */
8506   ins_encode(REX_reg_reg_wide(dst, src),
8507              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8508   ins_pipe(ialu_reg_reg_alu0);
8509 %}
8510 
8511 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8512 %{
8513   match(Set dst (MulL dst (LoadL src)));
8514   effect(KILL cr);
8515 
8516   ins_cost(350);
8517   format %{ "imulq   $dst, $src\t# long" %}
8518   opcode(0x0F, 0xAF);
8519   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8520   ins_pipe(ialu_reg_mem_alu0);
8521 %}
8522 
8523 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8524 %{
8525   match(Set dst (MulL (LoadL src) imm));
8526   effect(KILL cr);
8527 
8528   ins_cost(300);
8529   format %{ "imulq   $dst, $src, $imm\t# long" %}
8530   opcode(0x69); /* 69 /r id */
8531   ins_encode(REX_reg_mem_wide(dst, src),
8532              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8533   ins_pipe(ialu_reg_mem_alu0);
8534 %}
8535 
8536 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8537 %{
8538   match(Set dst (MulHiL src rax));
8539   effect(USE_KILL rax, KILL cr);
8540 
8541   ins_cost(300);
8542   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8543   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8544   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8545   ins_pipe(ialu_reg_reg_alu0);
8546 %}
8547 
8548 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8549                    rFlagsReg cr)
8550 %{
8551   match(Set rax (DivI rax div));
8552   effect(KILL rdx, KILL cr);
8553 
8554   ins_cost(30*100+10*100); // XXX
8555   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8556             "jne,s   normal\n\t"
8557             "xorl    rdx, rdx\n\t"
8558             "cmpl    $div, -1\n\t"
8559             "je,s    done\n"
8560     "normal: cdql\n\t"
8561             "idivl   $div\n"
8562     "done:"        %}
8563   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8564   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8565   ins_pipe(ialu_reg_reg_alu0);
8566 %}
8567 
8568 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8569                    rFlagsReg cr)
8570 %{
8571   match(Set rax (DivL rax div));
8572   effect(KILL rdx, KILL cr);
8573 
8574   ins_cost(30*100+10*100); // XXX
8575   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8576             "cmpq    rax, rdx\n\t"
8577             "jne,s   normal\n\t"
8578             "xorl    rdx, rdx\n\t"
8579             "cmpq    $div, -1\n\t"
8580             "je,s    done\n"
8581     "normal: cdqq\n\t"
8582             "idivq   $div\n"
8583     "done:"        %}
8584   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8585   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8586   ins_pipe(ialu_reg_reg_alu0);
8587 %}
8588 
8589 // Integer DIVMOD with Register, both quotient and mod results
8590 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8591                              rFlagsReg cr)
8592 %{
8593   match(DivModI rax div);
8594   effect(KILL cr);
8595 
8596   ins_cost(30*100+10*100); // XXX
8597   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8598             "jne,s   normal\n\t"
8599             "xorl    rdx, rdx\n\t"
8600             "cmpl    $div, -1\n\t"
8601             "je,s    done\n"
8602     "normal: cdql\n\t"
8603             "idivl   $div\n"
8604     "done:"        %}
8605   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8606   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8607   ins_pipe(pipe_slow);
8608 %}
8609 
8610 // Long DIVMOD with Register, both quotient and mod results
8611 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8612                              rFlagsReg cr)
8613 %{
8614   match(DivModL rax div);
8615   effect(KILL cr);
8616 
8617   ins_cost(30*100+10*100); // XXX
8618   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8619             "cmpq    rax, rdx\n\t"
8620             "jne,s   normal\n\t"
8621             "xorl    rdx, rdx\n\t"
8622             "cmpq    $div, -1\n\t"
8623             "je,s    done\n"
8624     "normal: cdqq\n\t"
8625             "idivq   $div\n"
8626     "done:"        %}
8627   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8628   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8629   ins_pipe(pipe_slow);
8630 %}
8631 
8632 //----------- DivL-By-Constant-Expansions--------------------------------------
8633 // DivI cases are handled by the compiler
8634 
8635 // Magic constant, reciprocal of 10
8636 instruct loadConL_0x6666666666666667(rRegL dst)
8637 %{
8638   effect(DEF dst);
8639 
8640   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8641   ins_encode(load_immL(dst, 0x6666666666666667));
8642   ins_pipe(ialu_reg);
8643 %}
8644 
8645 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8646 %{
8647   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8648 
8649   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8650   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8651   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8652   ins_pipe(ialu_reg_reg_alu0);
8653 %}
8654 
8655 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8656 %{
8657   effect(USE_DEF dst, KILL cr);
8658 
8659   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8660   opcode(0xC1, 0x7); /* C1 /7 ib */
8661   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8662   ins_pipe(ialu_reg);
8663 %}
8664 
8665 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8666 %{
8667   effect(USE_DEF dst, KILL cr);
8668 
8669   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8670   opcode(0xC1, 0x7); /* C1 /7 ib */
8671   ins_encode(reg_opc_imm_wide(dst, 0x2));
8672   ins_pipe(ialu_reg);
8673 %}
8674 
8675 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8676 %{
8677   match(Set dst (DivL src div));
8678 
8679   ins_cost((5+8)*100);
8680   expand %{
8681     rax_RegL rax;                     // Killed temp
8682     rFlagsReg cr;                     // Killed
8683     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8684     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8685     sarL_rReg_63(src, cr);            // sarq  src, 63
8686     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8687     subL_rReg(dst, src, cr);          // subl  rdx, src
8688   %}
8689 %}
8690 
8691 //-----------------------------------------------------------------------------
8692 
8693 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8694                    rFlagsReg cr)
8695 %{
8696   match(Set rdx (ModI rax div));
8697   effect(KILL rax, KILL cr);
8698 
8699   ins_cost(300); // XXX
8700   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8701             "jne,s   normal\n\t"
8702             "xorl    rdx, rdx\n\t"
8703             "cmpl    $div, -1\n\t"
8704             "je,s    done\n"
8705     "normal: cdql\n\t"
8706             "idivl   $div\n"
8707     "done:"        %}
8708   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8709   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8710   ins_pipe(ialu_reg_reg_alu0);
8711 %}
8712 
8713 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8714                    rFlagsReg cr)
8715 %{
8716   match(Set rdx (ModL rax div));
8717   effect(KILL rax, KILL cr);
8718 
8719   ins_cost(300); // XXX
8720   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8721             "cmpq    rax, rdx\n\t"
8722             "jne,s   normal\n\t"
8723             "xorl    rdx, rdx\n\t"
8724             "cmpq    $div, -1\n\t"
8725             "je,s    done\n"
8726     "normal: cdqq\n\t"
8727             "idivq   $div\n"
8728     "done:"        %}
8729   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8730   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8731   ins_pipe(ialu_reg_reg_alu0);
8732 %}
8733 
8734 // Integer Shift Instructions
8735 // Shift Left by one
8736 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8737 %{
8738   match(Set dst (LShiftI dst shift));
8739   effect(KILL cr);
8740 
8741   format %{ "sall    $dst, $shift" %}
8742   opcode(0xD1, 0x4); /* D1 /4 */
8743   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8744   ins_pipe(ialu_reg);
8745 %}
8746 
8747 // Shift Left by one
8748 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8749 %{
8750   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8751   effect(KILL cr);
8752 
8753   format %{ "sall    $dst, $shift\t" %}
8754   opcode(0xD1, 0x4); /* D1 /4 */
8755   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8756   ins_pipe(ialu_mem_imm);
8757 %}
8758 
8759 // Shift Left by 8-bit immediate
8760 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8761 %{
8762   match(Set dst (LShiftI dst shift));
8763   effect(KILL cr);
8764 
8765   format %{ "sall    $dst, $shift" %}
8766   opcode(0xC1, 0x4); /* C1 /4 ib */
8767   ins_encode(reg_opc_imm(dst, shift));
8768   ins_pipe(ialu_reg);
8769 %}
8770 
8771 // Shift Left by 8-bit immediate
8772 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8773 %{
8774   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8775   effect(KILL cr);
8776 
8777   format %{ "sall    $dst, $shift" %}
8778   opcode(0xC1, 0x4); /* C1 /4 ib */
8779   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8780   ins_pipe(ialu_mem_imm);
8781 %}
8782 
8783 // Shift Left by variable
8784 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8785 %{
8786   match(Set dst (LShiftI dst shift));
8787   effect(KILL cr);
8788 
8789   format %{ "sall    $dst, $shift" %}
8790   opcode(0xD3, 0x4); /* D3 /4 */
8791   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8792   ins_pipe(ialu_reg_reg);
8793 %}
8794 
8795 // Shift Left by variable
8796 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8797 %{
8798   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8799   effect(KILL cr);
8800 
8801   format %{ "sall    $dst, $shift" %}
8802   opcode(0xD3, 0x4); /* D3 /4 */
8803   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8804   ins_pipe(ialu_mem_reg);
8805 %}
8806 
8807 // Arithmetic shift right by one
8808 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8809 %{
8810   match(Set dst (RShiftI dst shift));
8811   effect(KILL cr);
8812 
8813   format %{ "sarl    $dst, $shift" %}
8814   opcode(0xD1, 0x7); /* D1 /7 */
8815   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8816   ins_pipe(ialu_reg);
8817 %}
8818 
8819 // Arithmetic shift right by one
8820 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8821 %{
8822   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8823   effect(KILL cr);
8824 
8825   format %{ "sarl    $dst, $shift" %}
8826   opcode(0xD1, 0x7); /* D1 /7 */
8827   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8828   ins_pipe(ialu_mem_imm);
8829 %}
8830 
8831 // Arithmetic Shift Right by 8-bit immediate
8832 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8833 %{
8834   match(Set dst (RShiftI dst shift));
8835   effect(KILL cr);
8836 
8837   format %{ "sarl    $dst, $shift" %}
8838   opcode(0xC1, 0x7); /* C1 /7 ib */
8839   ins_encode(reg_opc_imm(dst, shift));
8840   ins_pipe(ialu_mem_imm);
8841 %}
8842 
8843 // Arithmetic Shift Right by 8-bit immediate
8844 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8845 %{
8846   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8847   effect(KILL cr);
8848 
8849   format %{ "sarl    $dst, $shift" %}
8850   opcode(0xC1, 0x7); /* C1 /7 ib */
8851   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8852   ins_pipe(ialu_mem_imm);
8853 %}
8854 
8855 // Arithmetic Shift Right by variable
8856 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8857 %{
8858   match(Set dst (RShiftI dst shift));
8859   effect(KILL cr);
8860 
8861   format %{ "sarl    $dst, $shift" %}
8862   opcode(0xD3, 0x7); /* D3 /7 */
8863   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8864   ins_pipe(ialu_reg_reg);
8865 %}
8866 
8867 // Arithmetic Shift Right by variable
8868 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8869 %{
8870   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8871   effect(KILL cr);
8872 
8873   format %{ "sarl    $dst, $shift" %}
8874   opcode(0xD3, 0x7); /* D3 /7 */
8875   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8876   ins_pipe(ialu_mem_reg);
8877 %}
8878 
8879 // Logical shift right by one
8880 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8881 %{
8882   match(Set dst (URShiftI dst shift));
8883   effect(KILL cr);
8884 
8885   format %{ "shrl    $dst, $shift" %}
8886   opcode(0xD1, 0x5); /* D1 /5 */
8887   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8888   ins_pipe(ialu_reg);
8889 %}
8890 
8891 // Logical shift right by one
8892 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8893 %{
8894   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8895   effect(KILL cr);
8896 
8897   format %{ "shrl    $dst, $shift" %}
8898   opcode(0xD1, 0x5); /* D1 /5 */
8899   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8900   ins_pipe(ialu_mem_imm);
8901 %}
8902 
8903 // Logical Shift Right by 8-bit immediate
8904 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8905 %{
8906   match(Set dst (URShiftI dst shift));
8907   effect(KILL cr);
8908 
8909   format %{ "shrl    $dst, $shift" %}
8910   opcode(0xC1, 0x5); /* C1 /5 ib */
8911   ins_encode(reg_opc_imm(dst, shift));
8912   ins_pipe(ialu_reg);
8913 %}
8914 
8915 // Logical Shift Right by 8-bit immediate
8916 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8917 %{
8918   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8919   effect(KILL cr);
8920 
8921   format %{ "shrl    $dst, $shift" %}
8922   opcode(0xC1, 0x5); /* C1 /5 ib */
8923   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8924   ins_pipe(ialu_mem_imm);
8925 %}
8926 
8927 // Logical Shift Right by variable
8928 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8929 %{
8930   match(Set dst (URShiftI dst shift));
8931   effect(KILL cr);
8932 
8933   format %{ "shrl    $dst, $shift" %}
8934   opcode(0xD3, 0x5); /* D3 /5 */
8935   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8936   ins_pipe(ialu_reg_reg);
8937 %}
8938 
8939 // Logical Shift Right by variable
8940 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8941 %{
8942   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8943   effect(KILL cr);
8944 
8945   format %{ "shrl    $dst, $shift" %}
8946   opcode(0xD3, 0x5); /* D3 /5 */
8947   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8948   ins_pipe(ialu_mem_reg);
8949 %}
8950 
8951 // Long Shift Instructions
8952 // Shift Left by one
8953 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8954 %{
8955   match(Set dst (LShiftL dst shift));
8956   effect(KILL cr);
8957 
8958   format %{ "salq    $dst, $shift" %}
8959   opcode(0xD1, 0x4); /* D1 /4 */
8960   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8961   ins_pipe(ialu_reg);
8962 %}
8963 
8964 // Shift Left by one
8965 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8966 %{
8967   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8968   effect(KILL cr);
8969 
8970   format %{ "salq    $dst, $shift" %}
8971   opcode(0xD1, 0x4); /* D1 /4 */
8972   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8973   ins_pipe(ialu_mem_imm);
8974 %}
8975 
8976 // Shift Left by 8-bit immediate
8977 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8978 %{
8979   match(Set dst (LShiftL dst shift));
8980   effect(KILL cr);
8981 
8982   format %{ "salq    $dst, $shift" %}
8983   opcode(0xC1, 0x4); /* C1 /4 ib */
8984   ins_encode(reg_opc_imm_wide(dst, shift));
8985   ins_pipe(ialu_reg);
8986 %}
8987 
8988 // Shift Left by 8-bit immediate
8989 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8990 %{
8991   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8992   effect(KILL cr);
8993 
8994   format %{ "salq    $dst, $shift" %}
8995   opcode(0xC1, 0x4); /* C1 /4 ib */
8996   ins_encode(REX_mem_wide(dst), OpcP,
8997              RM_opc_mem(secondary, dst), Con8or32(shift));
8998   ins_pipe(ialu_mem_imm);
8999 %}
9000 
9001 // Shift Left by variable
9002 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9003 %{
9004   match(Set dst (LShiftL dst shift));
9005   effect(KILL cr);
9006 
9007   format %{ "salq    $dst, $shift" %}
9008   opcode(0xD3, 0x4); /* D3 /4 */
9009   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9010   ins_pipe(ialu_reg_reg);
9011 %}
9012 
9013 // Shift Left by variable
9014 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9015 %{
9016   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
9017   effect(KILL cr);
9018 
9019   format %{ "salq    $dst, $shift" %}
9020   opcode(0xD3, 0x4); /* D3 /4 */
9021   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9022   ins_pipe(ialu_mem_reg);
9023 %}
9024 
9025 // Arithmetic shift right by one
9026 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9027 %{
9028   match(Set dst (RShiftL dst shift));
9029   effect(KILL cr);
9030 
9031   format %{ "sarq    $dst, $shift" %}
9032   opcode(0xD1, 0x7); /* D1 /7 */
9033   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9034   ins_pipe(ialu_reg);
9035 %}
9036 
9037 // Arithmetic shift right by one
9038 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9039 %{
9040   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9041   effect(KILL cr);
9042 
9043   format %{ "sarq    $dst, $shift" %}
9044   opcode(0xD1, 0x7); /* D1 /7 */
9045   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9046   ins_pipe(ialu_mem_imm);
9047 %}
9048 
9049 // Arithmetic Shift Right by 8-bit immediate
9050 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9051 %{
9052   match(Set dst (RShiftL dst shift));
9053   effect(KILL cr);
9054 
9055   format %{ "sarq    $dst, $shift" %}
9056   opcode(0xC1, 0x7); /* C1 /7 ib */
9057   ins_encode(reg_opc_imm_wide(dst, shift));
9058   ins_pipe(ialu_mem_imm);
9059 %}
9060 
9061 // Arithmetic Shift Right by 8-bit immediate
9062 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9063 %{
9064   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9065   effect(KILL cr);
9066 
9067   format %{ "sarq    $dst, $shift" %}
9068   opcode(0xC1, 0x7); /* C1 /7 ib */
9069   ins_encode(REX_mem_wide(dst), OpcP,
9070              RM_opc_mem(secondary, dst), Con8or32(shift));
9071   ins_pipe(ialu_mem_imm);
9072 %}
9073 
9074 // Arithmetic Shift Right by variable
9075 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9076 %{
9077   match(Set dst (RShiftL dst shift));
9078   effect(KILL cr);
9079 
9080   format %{ "sarq    $dst, $shift" %}
9081   opcode(0xD3, 0x7); /* D3 /7 */
9082   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9083   ins_pipe(ialu_reg_reg);
9084 %}
9085 
9086 // Arithmetic Shift Right by variable
9087 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9088 %{
9089   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9090   effect(KILL cr);
9091 
9092   format %{ "sarq    $dst, $shift" %}
9093   opcode(0xD3, 0x7); /* D3 /7 */
9094   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9095   ins_pipe(ialu_mem_reg);
9096 %}
9097 
9098 // Logical shift right by one
9099 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9100 %{
9101   match(Set dst (URShiftL dst shift));
9102   effect(KILL cr);
9103 
9104   format %{ "shrq    $dst, $shift" %}
9105   opcode(0xD1, 0x5); /* D1 /5 */
9106   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
9107   ins_pipe(ialu_reg);
9108 %}
9109 
9110 // Logical shift right by one
9111 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9112 %{
9113   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9114   effect(KILL cr);
9115 
9116   format %{ "shrq    $dst, $shift" %}
9117   opcode(0xD1, 0x5); /* D1 /5 */
9118   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9119   ins_pipe(ialu_mem_imm);
9120 %}
9121 
9122 // Logical Shift Right by 8-bit immediate
9123 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9124 %{
9125   match(Set dst (URShiftL dst shift));
9126   effect(KILL cr);
9127 
9128   format %{ "shrq    $dst, $shift" %}
9129   opcode(0xC1, 0x5); /* C1 /5 ib */
9130   ins_encode(reg_opc_imm_wide(dst, shift));
9131   ins_pipe(ialu_reg);
9132 %}
9133 
9134 
9135 // Logical Shift Right by 8-bit immediate
9136 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9137 %{
9138   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9139   effect(KILL cr);
9140 
9141   format %{ "shrq    $dst, $shift" %}
9142   opcode(0xC1, 0x5); /* C1 /5 ib */
9143   ins_encode(REX_mem_wide(dst), OpcP,
9144              RM_opc_mem(secondary, dst), Con8or32(shift));
9145   ins_pipe(ialu_mem_imm);
9146 %}
9147 
9148 // Logical Shift Right by variable
9149 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9150 %{
9151   match(Set dst (URShiftL dst shift));
9152   effect(KILL cr);
9153 
9154   format %{ "shrq    $dst, $shift" %}
9155   opcode(0xD3, 0x5); /* D3 /5 */
9156   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9157   ins_pipe(ialu_reg_reg);
9158 %}
9159 
9160 // Logical Shift Right by variable
9161 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9162 %{
9163   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9164   effect(KILL cr);
9165 
9166   format %{ "shrq    $dst, $shift" %}
9167   opcode(0xD3, 0x5); /* D3 /5 */
9168   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9169   ins_pipe(ialu_mem_reg);
9170 %}
9171 
9172 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
9173 // This idiom is used by the compiler for the i2b bytecode.
9174 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
9175 %{
9176   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
9177 
9178   format %{ "movsbl  $dst, $src\t# i2b" %}
9179   opcode(0x0F, 0xBE);
9180   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9181   ins_pipe(ialu_reg_reg);
9182 %}
9183 
9184 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
9185 // This idiom is used by the compiler the i2s bytecode.
9186 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
9187 %{
9188   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
9189 
9190   format %{ "movswl  $dst, $src\t# i2s" %}
9191   opcode(0x0F, 0xBF);
9192   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9193   ins_pipe(ialu_reg_reg);
9194 %}
9195 
9196 // ROL/ROR instructions
9197 
9198 // ROL expand
9199 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
9200   effect(KILL cr, USE_DEF dst);
9201 
9202   format %{ "roll    $dst" %}
9203   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9204   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9205   ins_pipe(ialu_reg);
9206 %}
9207 
9208 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
9209   effect(USE_DEF dst, USE shift, KILL cr);
9210 
9211   format %{ "roll    $dst, $shift" %}
9212   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9213   ins_encode( reg_opc_imm(dst, shift) );
9214   ins_pipe(ialu_reg);
9215 %}
9216 
9217 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9218 %{
9219   effect(USE_DEF dst, USE shift, KILL cr);
9220 
9221   format %{ "roll    $dst, $shift" %}
9222   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9223   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9224   ins_pipe(ialu_reg_reg);
9225 %}
9226 // end of ROL expand
9227 
9228 // Rotate Left by one
9229 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9230 %{
9231   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9232 
9233   expand %{
9234     rolI_rReg_imm1(dst, cr);
9235   %}
9236 %}
9237 
9238 // Rotate Left by 8-bit immediate
9239 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9240 %{
9241   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9242   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9243 
9244   expand %{
9245     rolI_rReg_imm8(dst, lshift, cr);
9246   %}
9247 %}
9248 
9249 // Rotate Left by variable
9250 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9251 %{
9252   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9253 
9254   expand %{
9255     rolI_rReg_CL(dst, shift, cr);
9256   %}
9257 %}
9258 
9259 // Rotate Left by variable
9260 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9261 %{
9262   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9263 
9264   expand %{
9265     rolI_rReg_CL(dst, shift, cr);
9266   %}
9267 %}
9268 
9269 // ROR expand
9270 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
9271 %{
9272   effect(USE_DEF dst, KILL cr);
9273 
9274   format %{ "rorl    $dst" %}
9275   opcode(0xD1, 0x1); /* D1 /1 */
9276   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9277   ins_pipe(ialu_reg);
9278 %}
9279 
9280 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
9281 %{
9282   effect(USE_DEF dst, USE shift, KILL cr);
9283 
9284   format %{ "rorl    $dst, $shift" %}
9285   opcode(0xC1, 0x1); /* C1 /1 ib */
9286   ins_encode(reg_opc_imm(dst, shift));
9287   ins_pipe(ialu_reg);
9288 %}
9289 
9290 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9291 %{
9292   effect(USE_DEF dst, USE shift, KILL cr);
9293 
9294   format %{ "rorl    $dst, $shift" %}
9295   opcode(0xD3, 0x1); /* D3 /1 */
9296   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9297   ins_pipe(ialu_reg_reg);
9298 %}
9299 // end of ROR expand
9300 
9301 // Rotate Right by one
9302 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9303 %{
9304   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9305 
9306   expand %{
9307     rorI_rReg_imm1(dst, cr);
9308   %}
9309 %}
9310 
9311 // Rotate Right by 8-bit immediate
9312 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9313 %{
9314   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9315   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9316 
9317   expand %{
9318     rorI_rReg_imm8(dst, rshift, cr);
9319   %}
9320 %}
9321 
9322 // Rotate Right by variable
9323 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9324 %{
9325   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9326 
9327   expand %{
9328     rorI_rReg_CL(dst, shift, cr);
9329   %}
9330 %}
9331 
9332 // Rotate Right by variable
9333 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9334 %{
9335   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9336 
9337   expand %{
9338     rorI_rReg_CL(dst, shift, cr);
9339   %}
9340 %}
9341 
9342 // for long rotate
9343 // ROL expand
9344 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9345   effect(USE_DEF dst, KILL cr);
9346 
9347   format %{ "rolq    $dst" %}
9348   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9349   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9350   ins_pipe(ialu_reg);
9351 %}
9352 
9353 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9354   effect(USE_DEF dst, USE shift, KILL cr);
9355 
9356   format %{ "rolq    $dst, $shift" %}
9357   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9358   ins_encode( reg_opc_imm_wide(dst, shift) );
9359   ins_pipe(ialu_reg);
9360 %}
9361 
9362 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9363 %{
9364   effect(USE_DEF dst, USE shift, KILL cr);
9365 
9366   format %{ "rolq    $dst, $shift" %}
9367   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9368   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9369   ins_pipe(ialu_reg_reg);
9370 %}
9371 // end of ROL expand
9372 
9373 // Rotate Left by one
9374 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9375 %{
9376   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9377 
9378   expand %{
9379     rolL_rReg_imm1(dst, cr);
9380   %}
9381 %}
9382 
9383 // Rotate Left by 8-bit immediate
9384 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9385 %{
9386   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9387   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9388 
9389   expand %{
9390     rolL_rReg_imm8(dst, lshift, cr);
9391   %}
9392 %}
9393 
9394 // Rotate Left by variable
9395 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9396 %{
9397   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9398 
9399   expand %{
9400     rolL_rReg_CL(dst, shift, cr);
9401   %}
9402 %}
9403 
9404 // Rotate Left by variable
9405 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9406 %{
9407   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9408 
9409   expand %{
9410     rolL_rReg_CL(dst, shift, cr);
9411   %}
9412 %}
9413 
9414 // ROR expand
9415 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9416 %{
9417   effect(USE_DEF dst, KILL cr);
9418 
9419   format %{ "rorq    $dst" %}
9420   opcode(0xD1, 0x1); /* D1 /1 */
9421   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9422   ins_pipe(ialu_reg);
9423 %}
9424 
9425 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9426 %{
9427   effect(USE_DEF dst, USE shift, KILL cr);
9428 
9429   format %{ "rorq    $dst, $shift" %}
9430   opcode(0xC1, 0x1); /* C1 /1 ib */
9431   ins_encode(reg_opc_imm_wide(dst, shift));
9432   ins_pipe(ialu_reg);
9433 %}
9434 
9435 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9436 %{
9437   effect(USE_DEF dst, USE shift, KILL cr);
9438 
9439   format %{ "rorq    $dst, $shift" %}
9440   opcode(0xD3, 0x1); /* D3 /1 */
9441   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9442   ins_pipe(ialu_reg_reg);
9443 %}
9444 // end of ROR expand
9445 
9446 // Rotate Right by one
9447 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9448 %{
9449   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9450 
9451   expand %{
9452     rorL_rReg_imm1(dst, cr);
9453   %}
9454 %}
9455 
9456 // Rotate Right by 8-bit immediate
9457 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9458 %{
9459   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9460   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9461 
9462   expand %{
9463     rorL_rReg_imm8(dst, rshift, cr);
9464   %}
9465 %}
9466 
9467 // Rotate Right by variable
9468 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9469 %{
9470   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9471 
9472   expand %{
9473     rorL_rReg_CL(dst, shift, cr);
9474   %}
9475 %}
9476 
9477 // Rotate Right by variable
9478 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9479 %{
9480   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9481 
9482   expand %{
9483     rorL_rReg_CL(dst, shift, cr);
9484   %}
9485 %}
9486 
9487 // Logical Instructions
9488 
9489 // Integer Logical Instructions
9490 
9491 // And Instructions
9492 // And Register with Register
9493 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9494 %{
9495   match(Set dst (AndI dst src));
9496   effect(KILL cr);
9497 
9498   format %{ "andl    $dst, $src\t# int" %}
9499   opcode(0x23);
9500   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9501   ins_pipe(ialu_reg_reg);
9502 %}
9503 
9504 // And Register with Immediate 255
9505 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9506 %{
9507   match(Set dst (AndI dst src));
9508 
9509   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9510   opcode(0x0F, 0xB6);
9511   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9512   ins_pipe(ialu_reg);
9513 %}
9514 
9515 // And Register with Immediate 255 and promote to long
9516 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9517 %{
9518   match(Set dst (ConvI2L (AndI src mask)));
9519 
9520   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9521   opcode(0x0F, 0xB6);
9522   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9523   ins_pipe(ialu_reg);
9524 %}
9525 
9526 // And Register with Immediate 65535
9527 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9528 %{
9529   match(Set dst (AndI dst src));
9530 
9531   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9532   opcode(0x0F, 0xB7);
9533   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9534   ins_pipe(ialu_reg);
9535 %}
9536 
9537 // And Register with Immediate 65535 and promote to long
9538 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9539 %{
9540   match(Set dst (ConvI2L (AndI src mask)));
9541 
9542   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9543   opcode(0x0F, 0xB7);
9544   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9545   ins_pipe(ialu_reg);
9546 %}
9547 
9548 // And Register with Immediate
9549 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9550 %{
9551   match(Set dst (AndI dst src));
9552   effect(KILL cr);
9553 
9554   format %{ "andl    $dst, $src\t# int" %}
9555   opcode(0x81, 0x04); /* Opcode 81 /4 */
9556   ins_encode(OpcSErm(dst, src), Con8or32(src));
9557   ins_pipe(ialu_reg);
9558 %}
9559 
9560 // And Register with Memory
9561 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9562 %{
9563   match(Set dst (AndI dst (LoadI src)));
9564   effect(KILL cr);
9565 
9566   ins_cost(125);
9567   format %{ "andl    $dst, $src\t# int" %}
9568   opcode(0x23);
9569   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9570   ins_pipe(ialu_reg_mem);
9571 %}
9572 
9573 // And Memory with Register
9574 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9575 %{
9576   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
9577   effect(KILL cr);
9578 
9579   ins_cost(150);
9580   format %{ "andb    $dst, $src\t# byte" %}
9581   opcode(0x20);
9582   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9583   ins_pipe(ialu_mem_reg);
9584 %}
9585 
9586 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9587 %{
9588   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9589   effect(KILL cr);
9590 
9591   ins_cost(150);
9592   format %{ "andl    $dst, $src\t# int" %}
9593   opcode(0x21); /* Opcode 21 /r */
9594   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9595   ins_pipe(ialu_mem_reg);
9596 %}
9597 
9598 // And Memory with Immediate
9599 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9600 %{
9601   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9602   effect(KILL cr);
9603 
9604   ins_cost(125);
9605   format %{ "andl    $dst, $src\t# int" %}
9606   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9607   ins_encode(REX_mem(dst), OpcSE(src),
9608              RM_opc_mem(secondary, dst), Con8or32(src));
9609   ins_pipe(ialu_mem_imm);
9610 %}
9611 
9612 // BMI1 instructions
9613 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9614   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9615   predicate(UseBMI1Instructions);
9616   effect(KILL cr);
9617 
9618   ins_cost(125);
9619   format %{ "andnl  $dst, $src1, $src2" %}
9620 
9621   ins_encode %{
9622     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9623   %}
9624   ins_pipe(ialu_reg_mem);
9625 %}
9626 
9627 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9628   match(Set dst (AndI (XorI src1 minus_1) src2));
9629   predicate(UseBMI1Instructions);
9630   effect(KILL cr);
9631 
9632   format %{ "andnl  $dst, $src1, $src2" %}
9633 
9634   ins_encode %{
9635     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9636   %}
9637   ins_pipe(ialu_reg);
9638 %}
9639 
9640 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9641   match(Set dst (AndI (SubI imm_zero src) src));
9642   predicate(UseBMI1Instructions);
9643   effect(KILL cr);
9644 
9645   format %{ "blsil  $dst, $src" %}
9646 
9647   ins_encode %{
9648     __ blsil($dst$$Register, $src$$Register);
9649   %}
9650   ins_pipe(ialu_reg);
9651 %}
9652 
9653 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9654   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9655   predicate(UseBMI1Instructions);
9656   effect(KILL cr);
9657 
9658   ins_cost(125);
9659   format %{ "blsil  $dst, $src" %}
9660 
9661   ins_encode %{
9662     __ blsil($dst$$Register, $src$$Address);
9663   %}
9664   ins_pipe(ialu_reg_mem);
9665 %}
9666 
9667 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9668 %{
9669   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9670   predicate(UseBMI1Instructions);
9671   effect(KILL cr);
9672 
9673   ins_cost(125);
9674   format %{ "blsmskl $dst, $src" %}
9675 
9676   ins_encode %{
9677     __ blsmskl($dst$$Register, $src$$Address);
9678   %}
9679   ins_pipe(ialu_reg_mem);
9680 %}
9681 
9682 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9683 %{
9684   match(Set dst (XorI (AddI src minus_1) src));
9685   predicate(UseBMI1Instructions);
9686   effect(KILL cr);
9687 
9688   format %{ "blsmskl $dst, $src" %}
9689 
9690   ins_encode %{
9691     __ blsmskl($dst$$Register, $src$$Register);
9692   %}
9693 
9694   ins_pipe(ialu_reg);
9695 %}
9696 
9697 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9698 %{
9699   match(Set dst (AndI (AddI src minus_1) src) );
9700   predicate(UseBMI1Instructions);
9701   effect(KILL cr);
9702 
9703   format %{ "blsrl  $dst, $src" %}
9704 
9705   ins_encode %{
9706     __ blsrl($dst$$Register, $src$$Register);
9707   %}
9708 
9709   ins_pipe(ialu_reg_mem);
9710 %}
9711 
9712 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9713 %{
9714   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9715   predicate(UseBMI1Instructions);
9716   effect(KILL cr);
9717 
9718   ins_cost(125);
9719   format %{ "blsrl  $dst, $src" %}
9720 
9721   ins_encode %{
9722     __ blsrl($dst$$Register, $src$$Address);
9723   %}
9724 
9725   ins_pipe(ialu_reg);
9726 %}
9727 
9728 // Or Instructions
9729 // Or Register with Register
9730 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9731 %{
9732   match(Set dst (OrI dst src));
9733   effect(KILL cr);
9734 
9735   format %{ "orl     $dst, $src\t# int" %}
9736   opcode(0x0B);
9737   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9738   ins_pipe(ialu_reg_reg);
9739 %}
9740 
9741 // Or Register with Immediate
9742 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9743 %{
9744   match(Set dst (OrI dst src));
9745   effect(KILL cr);
9746 
9747   format %{ "orl     $dst, $src\t# int" %}
9748   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9749   ins_encode(OpcSErm(dst, src), Con8or32(src));
9750   ins_pipe(ialu_reg);
9751 %}
9752 
9753 // Or Register with Memory
9754 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9755 %{
9756   match(Set dst (OrI dst (LoadI src)));
9757   effect(KILL cr);
9758 
9759   ins_cost(125);
9760   format %{ "orl     $dst, $src\t# int" %}
9761   opcode(0x0B);
9762   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9763   ins_pipe(ialu_reg_mem);
9764 %}
9765 
9766 // Or Memory with Register
9767 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9768 %{
9769   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
9770   effect(KILL cr);
9771 
9772   ins_cost(150);
9773   format %{ "orb    $dst, $src\t# byte" %}
9774   opcode(0x08);
9775   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9776   ins_pipe(ialu_mem_reg);
9777 %}
9778 
9779 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9780 %{
9781   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9782   effect(KILL cr);
9783 
9784   ins_cost(150);
9785   format %{ "orl     $dst, $src\t# int" %}
9786   opcode(0x09); /* Opcode 09 /r */
9787   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9788   ins_pipe(ialu_mem_reg);
9789 %}
9790 
9791 // Or Memory with Immediate
9792 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9793 %{
9794   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9795   effect(KILL cr);
9796 
9797   ins_cost(125);
9798   format %{ "orl     $dst, $src\t# int" %}
9799   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9800   ins_encode(REX_mem(dst), OpcSE(src),
9801              RM_opc_mem(secondary, dst), Con8or32(src));
9802   ins_pipe(ialu_mem_imm);
9803 %}
9804 
9805 // Xor Instructions
9806 // Xor Register with Register
9807 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9808 %{
9809   match(Set dst (XorI dst src));
9810   effect(KILL cr);
9811 
9812   format %{ "xorl    $dst, $src\t# int" %}
9813   opcode(0x33);
9814   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9815   ins_pipe(ialu_reg_reg);
9816 %}
9817 
9818 // Xor Register with Immediate -1
9819 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9820   match(Set dst (XorI dst imm));
9821 
9822   format %{ "not    $dst" %}
9823   ins_encode %{
9824      __ notl($dst$$Register);
9825   %}
9826   ins_pipe(ialu_reg);
9827 %}
9828 
9829 // Xor Register with Immediate
9830 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9831 %{
9832   match(Set dst (XorI dst src));
9833   effect(KILL cr);
9834 
9835   format %{ "xorl    $dst, $src\t# int" %}
9836   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9837   ins_encode(OpcSErm(dst, src), Con8or32(src));
9838   ins_pipe(ialu_reg);
9839 %}
9840 
9841 // Xor Register with Memory
9842 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9843 %{
9844   match(Set dst (XorI dst (LoadI src)));
9845   effect(KILL cr);
9846 
9847   ins_cost(125);
9848   format %{ "xorl    $dst, $src\t# int" %}
9849   opcode(0x33);
9850   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9851   ins_pipe(ialu_reg_mem);
9852 %}
9853 
9854 // Xor Memory with Register
9855 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9856 %{
9857   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
9858   effect(KILL cr);
9859 
9860   ins_cost(150);
9861   format %{ "xorb    $dst, $src\t# byte" %}
9862   opcode(0x30);
9863   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9864   ins_pipe(ialu_mem_reg);
9865 %}
9866 
9867 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9868 %{
9869   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9870   effect(KILL cr);
9871 
9872   ins_cost(150);
9873   format %{ "xorl    $dst, $src\t# int" %}
9874   opcode(0x31); /* Opcode 31 /r */
9875   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9876   ins_pipe(ialu_mem_reg);
9877 %}
9878 
9879 // Xor Memory with Immediate
9880 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9881 %{
9882   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9883   effect(KILL cr);
9884 
9885   ins_cost(125);
9886   format %{ "xorl    $dst, $src\t# int" %}
9887   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9888   ins_encode(REX_mem(dst), OpcSE(src),
9889              RM_opc_mem(secondary, dst), Con8or32(src));
9890   ins_pipe(ialu_mem_imm);
9891 %}
9892 
9893 
9894 // Long Logical Instructions
9895 
9896 // And Instructions
9897 // And Register with Register
9898 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9899 %{
9900   match(Set dst (AndL dst src));
9901   effect(KILL cr);
9902 
9903   format %{ "andq    $dst, $src\t# long" %}
9904   opcode(0x23);
9905   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9906   ins_pipe(ialu_reg_reg);
9907 %}
9908 
9909 // And Register with Immediate 255
9910 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9911 %{
9912   match(Set dst (AndL dst src));
9913 
9914   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9915   opcode(0x0F, 0xB6);
9916   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9917   ins_pipe(ialu_reg);
9918 %}
9919 
9920 // And Register with Immediate 65535
9921 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9922 %{
9923   match(Set dst (AndL dst src));
9924 
9925   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9926   opcode(0x0F, 0xB7);
9927   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9928   ins_pipe(ialu_reg);
9929 %}
9930 
9931 // And Register with Immediate
9932 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9933 %{
9934   match(Set dst (AndL dst src));
9935   effect(KILL cr);
9936 
9937   format %{ "andq    $dst, $src\t# long" %}
9938   opcode(0x81, 0x04); /* Opcode 81 /4 */
9939   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9940   ins_pipe(ialu_reg);
9941 %}
9942 
9943 // And Register with Memory
9944 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9945 %{
9946   match(Set dst (AndL dst (LoadL src)));
9947   effect(KILL cr);
9948 
9949   ins_cost(125);
9950   format %{ "andq    $dst, $src\t# long" %}
9951   opcode(0x23);
9952   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9953   ins_pipe(ialu_reg_mem);
9954 %}
9955 
9956 // And Memory with Register
9957 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9958 %{
9959   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9960   effect(KILL cr);
9961 
9962   ins_cost(150);
9963   format %{ "andq    $dst, $src\t# long" %}
9964   opcode(0x21); /* Opcode 21 /r */
9965   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9966   ins_pipe(ialu_mem_reg);
9967 %}
9968 
9969 // And Memory with Immediate
9970 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9971 %{
9972   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9973   effect(KILL cr);
9974 
9975   ins_cost(125);
9976   format %{ "andq    $dst, $src\t# long" %}
9977   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9978   ins_encode(REX_mem_wide(dst), OpcSE(src),
9979              RM_opc_mem(secondary, dst), Con8or32(src));
9980   ins_pipe(ialu_mem_imm);
9981 %}
9982 
9983 // BMI1 instructions
9984 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9985   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9986   predicate(UseBMI1Instructions);
9987   effect(KILL cr);
9988 
9989   ins_cost(125);
9990   format %{ "andnq  $dst, $src1, $src2" %}
9991 
9992   ins_encode %{
9993     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9994   %}
9995   ins_pipe(ialu_reg_mem);
9996 %}
9997 
9998 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9999   match(Set dst (AndL (XorL src1 minus_1) src2));
10000   predicate(UseBMI1Instructions);
10001   effect(KILL cr);
10002 
10003   format %{ "andnq  $dst, $src1, $src2" %}
10004 
10005   ins_encode %{
10006   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10007   %}
10008   ins_pipe(ialu_reg_mem);
10009 %}
10010 
10011 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10012   match(Set dst (AndL (SubL imm_zero src) src));
10013   predicate(UseBMI1Instructions);
10014   effect(KILL cr);
10015 
10016   format %{ "blsiq  $dst, $src" %}
10017 
10018   ins_encode %{
10019     __ blsiq($dst$$Register, $src$$Register);
10020   %}
10021   ins_pipe(ialu_reg);
10022 %}
10023 
10024 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10025   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10026   predicate(UseBMI1Instructions);
10027   effect(KILL cr);
10028 
10029   ins_cost(125);
10030   format %{ "blsiq  $dst, $src" %}
10031 
10032   ins_encode %{
10033     __ blsiq($dst$$Register, $src$$Address);
10034   %}
10035   ins_pipe(ialu_reg_mem);
10036 %}
10037 
10038 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10039 %{
10040   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10041   predicate(UseBMI1Instructions);
10042   effect(KILL cr);
10043 
10044   ins_cost(125);
10045   format %{ "blsmskq $dst, $src" %}
10046 
10047   ins_encode %{
10048     __ blsmskq($dst$$Register, $src$$Address);
10049   %}
10050   ins_pipe(ialu_reg_mem);
10051 %}
10052 
10053 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10054 %{
10055   match(Set dst (XorL (AddL src minus_1) src));
10056   predicate(UseBMI1Instructions);
10057   effect(KILL cr);
10058 
10059   format %{ "blsmskq $dst, $src" %}
10060 
10061   ins_encode %{
10062     __ blsmskq($dst$$Register, $src$$Register);
10063   %}
10064 
10065   ins_pipe(ialu_reg);
10066 %}
10067 
10068 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10069 %{
10070   match(Set dst (AndL (AddL src minus_1) src) );
10071   predicate(UseBMI1Instructions);
10072   effect(KILL cr);
10073 
10074   format %{ "blsrq  $dst, $src" %}
10075 
10076   ins_encode %{
10077     __ blsrq($dst$$Register, $src$$Register);
10078   %}
10079 
10080   ins_pipe(ialu_reg);
10081 %}
10082 
10083 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10084 %{
10085   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10086   predicate(UseBMI1Instructions);
10087   effect(KILL cr);
10088 
10089   ins_cost(125);
10090   format %{ "blsrq  $dst, $src" %}
10091 
10092   ins_encode %{
10093     __ blsrq($dst$$Register, $src$$Address);
10094   %}
10095 
10096   ins_pipe(ialu_reg);
10097 %}
10098 
10099 // Or Instructions
10100 // Or Register with Register
10101 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10102 %{
10103   match(Set dst (OrL dst src));
10104   effect(KILL cr);
10105 
10106   format %{ "orq     $dst, $src\t# long" %}
10107   opcode(0x0B);
10108   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10109   ins_pipe(ialu_reg_reg);
10110 %}
10111 
10112 // Use any_RegP to match R15 (TLS register) without spilling.
10113 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10114   match(Set dst (OrL dst (CastP2X src)));
10115   effect(KILL cr);
10116 
10117   format %{ "orq     $dst, $src\t# long" %}
10118   opcode(0x0B);
10119   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10120   ins_pipe(ialu_reg_reg);
10121 %}
10122 
10123 
10124 // Or Register with Immediate
10125 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10126 %{
10127   match(Set dst (OrL dst src));
10128   effect(KILL cr);
10129 
10130   format %{ "orq     $dst, $src\t# long" %}
10131   opcode(0x81, 0x01); /* Opcode 81 /1 id */
10132   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10133   ins_pipe(ialu_reg);
10134 %}
10135 
10136 // Or Register with Memory
10137 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10138 %{
10139   match(Set dst (OrL dst (LoadL src)));
10140   effect(KILL cr);
10141 
10142   ins_cost(125);
10143   format %{ "orq     $dst, $src\t# long" %}
10144   opcode(0x0B);
10145   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10146   ins_pipe(ialu_reg_mem);
10147 %}
10148 
10149 // Or Memory with Register
10150 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10151 %{
10152   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10153   effect(KILL cr);
10154 
10155   ins_cost(150);
10156   format %{ "orq     $dst, $src\t# long" %}
10157   opcode(0x09); /* Opcode 09 /r */
10158   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10159   ins_pipe(ialu_mem_reg);
10160 %}
10161 
10162 // Or Memory with Immediate
10163 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10164 %{
10165   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10166   effect(KILL cr);
10167 
10168   ins_cost(125);
10169   format %{ "orq     $dst, $src\t# long" %}
10170   opcode(0x81, 0x1); /* Opcode 81 /1 id */
10171   ins_encode(REX_mem_wide(dst), OpcSE(src),
10172              RM_opc_mem(secondary, dst), Con8or32(src));
10173   ins_pipe(ialu_mem_imm);
10174 %}
10175 
10176 // Xor Instructions
10177 // Xor Register with Register
10178 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10179 %{
10180   match(Set dst (XorL dst src));
10181   effect(KILL cr);
10182 
10183   format %{ "xorq    $dst, $src\t# long" %}
10184   opcode(0x33);
10185   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10186   ins_pipe(ialu_reg_reg);
10187 %}
10188 
10189 // Xor Register with Immediate -1
10190 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10191   match(Set dst (XorL dst imm));
10192 
10193   format %{ "notq   $dst" %}
10194   ins_encode %{
10195      __ notq($dst$$Register);
10196   %}
10197   ins_pipe(ialu_reg);
10198 %}
10199 
10200 // Xor Register with Immediate
10201 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10202 %{
10203   match(Set dst (XorL dst src));
10204   effect(KILL cr);
10205 
10206   format %{ "xorq    $dst, $src\t# long" %}
10207   opcode(0x81, 0x06); /* Opcode 81 /6 id */
10208   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10209   ins_pipe(ialu_reg);
10210 %}
10211 
10212 // Xor Register with Memory
10213 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10214 %{
10215   match(Set dst (XorL dst (LoadL src)));
10216   effect(KILL cr);
10217 
10218   ins_cost(125);
10219   format %{ "xorq    $dst, $src\t# long" %}
10220   opcode(0x33);
10221   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10222   ins_pipe(ialu_reg_mem);
10223 %}
10224 
10225 // Xor Memory with Register
10226 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10227 %{
10228   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10229   effect(KILL cr);
10230 
10231   ins_cost(150);
10232   format %{ "xorq    $dst, $src\t# long" %}
10233   opcode(0x31); /* Opcode 31 /r */
10234   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10235   ins_pipe(ialu_mem_reg);
10236 %}
10237 
10238 // Xor Memory with Immediate
10239 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10240 %{
10241   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10242   effect(KILL cr);
10243 
10244   ins_cost(125);
10245   format %{ "xorq    $dst, $src\t# long" %}
10246   opcode(0x81, 0x6); /* Opcode 81 /6 id */
10247   ins_encode(REX_mem_wide(dst), OpcSE(src),
10248              RM_opc_mem(secondary, dst), Con8or32(src));
10249   ins_pipe(ialu_mem_imm);
10250 %}
10251 
10252 // Convert Int to Boolean
10253 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10254 %{
10255   match(Set dst (Conv2B src));
10256   effect(KILL cr);
10257 
10258   format %{ "testl   $src, $src\t# ci2b\n\t"
10259             "setnz   $dst\n\t"
10260             "movzbl  $dst, $dst" %}
10261   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
10262              setNZ_reg(dst),
10263              REX_reg_breg(dst, dst), // movzbl
10264              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10265   ins_pipe(pipe_slow); // XXX
10266 %}
10267 
10268 // Convert Pointer to Boolean
10269 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10270 %{
10271   match(Set dst (Conv2B src));
10272   effect(KILL cr);
10273 
10274   format %{ "testq   $src, $src\t# cp2b\n\t"
10275             "setnz   $dst\n\t"
10276             "movzbl  $dst, $dst" %}
10277   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
10278              setNZ_reg(dst),
10279              REX_reg_breg(dst, dst), // movzbl
10280              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10281   ins_pipe(pipe_slow); // XXX
10282 %}
10283 
10284 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10285 %{
10286   match(Set dst (CmpLTMask p q));
10287   effect(KILL cr);
10288 
10289   ins_cost(400);
10290   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10291             "setlt   $dst\n\t"
10292             "movzbl  $dst, $dst\n\t"
10293             "negl    $dst" %}
10294   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
10295              setLT_reg(dst),
10296              REX_reg_breg(dst, dst), // movzbl
10297              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
10298              neg_reg(dst));
10299   ins_pipe(pipe_slow);
10300 %}
10301 
10302 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
10303 %{
10304   match(Set dst (CmpLTMask dst zero));
10305   effect(KILL cr);
10306 
10307   ins_cost(100);
10308   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10309   ins_encode %{
10310   __ sarl($dst$$Register, 31);
10311   %}
10312   ins_pipe(ialu_reg);
10313 %}
10314 
10315 /* Better to save a register than avoid a branch */
10316 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10317 %{
10318   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10319   effect(KILL cr);
10320   ins_cost(300);
10321   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10322             "jge     done\n\t"
10323             "addl    $p,$y\n"
10324             "done:   " %}
10325   ins_encode %{
10326     Register Rp = $p$$Register;
10327     Register Rq = $q$$Register;
10328     Register Ry = $y$$Register;
10329     Label done;
10330     __ subl(Rp, Rq);
10331     __ jccb(Assembler::greaterEqual, done);
10332     __ addl(Rp, Ry);
10333     __ bind(done);
10334   %}
10335   ins_pipe(pipe_cmplt);
10336 %}
10337 
10338 /* Better to save a register than avoid a branch */
10339 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10340 %{
10341   match(Set y (AndI (CmpLTMask p q) y));
10342   effect(KILL cr);
10343 
10344   ins_cost(300);
10345 
10346   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10347             "jlt     done\n\t"
10348             "xorl    $y, $y\n"
10349             "done:   " %}
10350   ins_encode %{
10351     Register Rp = $p$$Register;
10352     Register Rq = $q$$Register;
10353     Register Ry = $y$$Register;
10354     Label done;
10355     __ cmpl(Rp, Rq);
10356     __ jccb(Assembler::less, done);
10357     __ xorl(Ry, Ry);
10358     __ bind(done);
10359   %}
10360   ins_pipe(pipe_cmplt);
10361 %}
10362 
10363 
10364 //---------- FP Instructions------------------------------------------------
10365 
10366 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10367 %{
10368   match(Set cr (CmpF src1 src2));
10369 
10370   ins_cost(145);
10371   format %{ "ucomiss $src1, $src2\n\t"
10372             "jnp,s   exit\n\t"
10373             "pushfq\t# saw NaN, set CF\n\t"
10374             "andq    [rsp], #0xffffff2b\n\t"
10375             "popfq\n"
10376     "exit:" %}
10377   ins_encode %{
10378     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10379     emit_cmpfp_fixup(_masm);
10380   %}
10381   ins_pipe(pipe_slow);
10382 %}
10383 
10384 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10385   match(Set cr (CmpF src1 src2));
10386 
10387   ins_cost(100);
10388   format %{ "ucomiss $src1, $src2" %}
10389   ins_encode %{
10390     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10391   %}
10392   ins_pipe(pipe_slow);
10393 %}
10394 
10395 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10396 %{
10397   match(Set cr (CmpF src1 (LoadF src2)));
10398 
10399   ins_cost(145);
10400   format %{ "ucomiss $src1, $src2\n\t"
10401             "jnp,s   exit\n\t"
10402             "pushfq\t# saw NaN, set CF\n\t"
10403             "andq    [rsp], #0xffffff2b\n\t"
10404             "popfq\n"
10405     "exit:" %}
10406   ins_encode %{
10407     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10408     emit_cmpfp_fixup(_masm);
10409   %}
10410   ins_pipe(pipe_slow);
10411 %}
10412 
10413 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10414   match(Set cr (CmpF src1 (LoadF src2)));
10415 
10416   ins_cost(100);
10417   format %{ "ucomiss $src1, $src2" %}
10418   ins_encode %{
10419     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10420   %}
10421   ins_pipe(pipe_slow);
10422 %}
10423 
10424 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10425   match(Set cr (CmpF src con));
10426 
10427   ins_cost(145);
10428   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10429             "jnp,s   exit\n\t"
10430             "pushfq\t# saw NaN, set CF\n\t"
10431             "andq    [rsp], #0xffffff2b\n\t"
10432             "popfq\n"
10433     "exit:" %}
10434   ins_encode %{
10435     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10436     emit_cmpfp_fixup(_masm);
10437   %}
10438   ins_pipe(pipe_slow);
10439 %}
10440 
10441 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10442   match(Set cr (CmpF src con));
10443   ins_cost(100);
10444   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10445   ins_encode %{
10446     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10447   %}
10448   ins_pipe(pipe_slow);
10449 %}
10450 
10451 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10452 %{
10453   match(Set cr (CmpD src1 src2));
10454 
10455   ins_cost(145);
10456   format %{ "ucomisd $src1, $src2\n\t"
10457             "jnp,s   exit\n\t"
10458             "pushfq\t# saw NaN, set CF\n\t"
10459             "andq    [rsp], #0xffffff2b\n\t"
10460             "popfq\n"
10461     "exit:" %}
10462   ins_encode %{
10463     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10464     emit_cmpfp_fixup(_masm);
10465   %}
10466   ins_pipe(pipe_slow);
10467 %}
10468 
10469 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10470   match(Set cr (CmpD src1 src2));
10471 
10472   ins_cost(100);
10473   format %{ "ucomisd $src1, $src2 test" %}
10474   ins_encode %{
10475     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10476   %}
10477   ins_pipe(pipe_slow);
10478 %}
10479 
10480 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10481 %{
10482   match(Set cr (CmpD src1 (LoadD src2)));
10483 
10484   ins_cost(145);
10485   format %{ "ucomisd $src1, $src2\n\t"
10486             "jnp,s   exit\n\t"
10487             "pushfq\t# saw NaN, set CF\n\t"
10488             "andq    [rsp], #0xffffff2b\n\t"
10489             "popfq\n"
10490     "exit:" %}
10491   ins_encode %{
10492     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10493     emit_cmpfp_fixup(_masm);
10494   %}
10495   ins_pipe(pipe_slow);
10496 %}
10497 
10498 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10499   match(Set cr (CmpD src1 (LoadD src2)));
10500 
10501   ins_cost(100);
10502   format %{ "ucomisd $src1, $src2" %}
10503   ins_encode %{
10504     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10505   %}
10506   ins_pipe(pipe_slow);
10507 %}
10508 
10509 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10510   match(Set cr (CmpD src con));
10511 
10512   ins_cost(145);
10513   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10514             "jnp,s   exit\n\t"
10515             "pushfq\t# saw NaN, set CF\n\t"
10516             "andq    [rsp], #0xffffff2b\n\t"
10517             "popfq\n"
10518     "exit:" %}
10519   ins_encode %{
10520     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10521     emit_cmpfp_fixup(_masm);
10522   %}
10523   ins_pipe(pipe_slow);
10524 %}
10525 
10526 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10527   match(Set cr (CmpD src con));
10528   ins_cost(100);
10529   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10530   ins_encode %{
10531     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10532   %}
10533   ins_pipe(pipe_slow);
10534 %}
10535 
10536 // Compare into -1,0,1
10537 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10538 %{
10539   match(Set dst (CmpF3 src1 src2));
10540   effect(KILL cr);
10541 
10542   ins_cost(275);
10543   format %{ "ucomiss $src1, $src2\n\t"
10544             "movl    $dst, #-1\n\t"
10545             "jp,s    done\n\t"
10546             "jb,s    done\n\t"
10547             "setne   $dst\n\t"
10548             "movzbl  $dst, $dst\n"
10549     "done:" %}
10550   ins_encode %{
10551     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10552     emit_cmpfp3(_masm, $dst$$Register);
10553   %}
10554   ins_pipe(pipe_slow);
10555 %}
10556 
10557 // Compare into -1,0,1
10558 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10559 %{
10560   match(Set dst (CmpF3 src1 (LoadF src2)));
10561   effect(KILL cr);
10562 
10563   ins_cost(275);
10564   format %{ "ucomiss $src1, $src2\n\t"
10565             "movl    $dst, #-1\n\t"
10566             "jp,s    done\n\t"
10567             "jb,s    done\n\t"
10568             "setne   $dst\n\t"
10569             "movzbl  $dst, $dst\n"
10570     "done:" %}
10571   ins_encode %{
10572     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10573     emit_cmpfp3(_masm, $dst$$Register);
10574   %}
10575   ins_pipe(pipe_slow);
10576 %}
10577 
10578 // Compare into -1,0,1
10579 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10580   match(Set dst (CmpF3 src con));
10581   effect(KILL cr);
10582 
10583   ins_cost(275);
10584   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10585             "movl    $dst, #-1\n\t"
10586             "jp,s    done\n\t"
10587             "jb,s    done\n\t"
10588             "setne   $dst\n\t"
10589             "movzbl  $dst, $dst\n"
10590     "done:" %}
10591   ins_encode %{
10592     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10593     emit_cmpfp3(_masm, $dst$$Register);
10594   %}
10595   ins_pipe(pipe_slow);
10596 %}
10597 
10598 // Compare into -1,0,1
10599 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10600 %{
10601   match(Set dst (CmpD3 src1 src2));
10602   effect(KILL cr);
10603 
10604   ins_cost(275);
10605   format %{ "ucomisd $src1, $src2\n\t"
10606             "movl    $dst, #-1\n\t"
10607             "jp,s    done\n\t"
10608             "jb,s    done\n\t"
10609             "setne   $dst\n\t"
10610             "movzbl  $dst, $dst\n"
10611     "done:" %}
10612   ins_encode %{
10613     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10614     emit_cmpfp3(_masm, $dst$$Register);
10615   %}
10616   ins_pipe(pipe_slow);
10617 %}
10618 
10619 // Compare into -1,0,1
10620 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10621 %{
10622   match(Set dst (CmpD3 src1 (LoadD src2)));
10623   effect(KILL cr);
10624 
10625   ins_cost(275);
10626   format %{ "ucomisd $src1, $src2\n\t"
10627             "movl    $dst, #-1\n\t"
10628             "jp,s    done\n\t"
10629             "jb,s    done\n\t"
10630             "setne   $dst\n\t"
10631             "movzbl  $dst, $dst\n"
10632     "done:" %}
10633   ins_encode %{
10634     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10635     emit_cmpfp3(_masm, $dst$$Register);
10636   %}
10637   ins_pipe(pipe_slow);
10638 %}
10639 
10640 // Compare into -1,0,1
10641 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10642   match(Set dst (CmpD3 src con));
10643   effect(KILL cr);
10644 
10645   ins_cost(275);
10646   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10647             "movl    $dst, #-1\n\t"
10648             "jp,s    done\n\t"
10649             "jb,s    done\n\t"
10650             "setne   $dst\n\t"
10651             "movzbl  $dst, $dst\n"
10652     "done:" %}
10653   ins_encode %{
10654     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10655     emit_cmpfp3(_masm, $dst$$Register);
10656   %}
10657   ins_pipe(pipe_slow);
10658 %}
10659 
10660 //----------Arithmetic Conversion Instructions---------------------------------
10661 
10662 instruct roundFloat_nop(regF dst)
10663 %{
10664   match(Set dst (RoundFloat dst));
10665 
10666   ins_cost(0);
10667   ins_encode();
10668   ins_pipe(empty);
10669 %}
10670 
10671 instruct roundDouble_nop(regD dst)
10672 %{
10673   match(Set dst (RoundDouble dst));
10674 
10675   ins_cost(0);
10676   ins_encode();
10677   ins_pipe(empty);
10678 %}
10679 
10680 instruct convF2D_reg_reg(regD dst, regF src)
10681 %{
10682   match(Set dst (ConvF2D src));
10683 
10684   format %{ "cvtss2sd $dst, $src" %}
10685   ins_encode %{
10686     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10687   %}
10688   ins_pipe(pipe_slow); // XXX
10689 %}
10690 
10691 instruct convF2D_reg_mem(regD dst, memory src)
10692 %{
10693   match(Set dst (ConvF2D (LoadF src)));
10694 
10695   format %{ "cvtss2sd $dst, $src" %}
10696   ins_encode %{
10697     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10698   %}
10699   ins_pipe(pipe_slow); // XXX
10700 %}
10701 
10702 instruct convD2F_reg_reg(regF dst, regD src)
10703 %{
10704   match(Set dst (ConvD2F src));
10705 
10706   format %{ "cvtsd2ss $dst, $src" %}
10707   ins_encode %{
10708     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10709   %}
10710   ins_pipe(pipe_slow); // XXX
10711 %}
10712 
10713 instruct convD2F_reg_mem(regF dst, memory src)
10714 %{
10715   match(Set dst (ConvD2F (LoadD src)));
10716 
10717   format %{ "cvtsd2ss $dst, $src" %}
10718   ins_encode %{
10719     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10720   %}
10721   ins_pipe(pipe_slow); // XXX
10722 %}
10723 
10724 // XXX do mem variants
10725 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10726 %{
10727   match(Set dst (ConvF2I src));
10728   effect(KILL cr);
10729 
10730   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10731             "cmpl    $dst, #0x80000000\n\t"
10732             "jne,s   done\n\t"
10733             "subq    rsp, #8\n\t"
10734             "movss   [rsp], $src\n\t"
10735             "call    f2i_fixup\n\t"
10736             "popq    $dst\n"
10737     "done:   "%}
10738   ins_encode %{
10739     Label done;
10740     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10741     __ cmpl($dst$$Register, 0x80000000);
10742     __ jccb(Assembler::notEqual, done);
10743     __ subptr(rsp, 8);
10744     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10745     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10746     __ pop($dst$$Register);
10747     __ bind(done);
10748   %}
10749   ins_pipe(pipe_slow);
10750 %}
10751 
10752 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10753 %{
10754   match(Set dst (ConvF2L src));
10755   effect(KILL cr);
10756 
10757   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10758             "cmpq    $dst, [0x8000000000000000]\n\t"
10759             "jne,s   done\n\t"
10760             "subq    rsp, #8\n\t"
10761             "movss   [rsp], $src\n\t"
10762             "call    f2l_fixup\n\t"
10763             "popq    $dst\n"
10764     "done:   "%}
10765   ins_encode %{
10766     Label done;
10767     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10768     __ cmp64($dst$$Register,
10769              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10770     __ jccb(Assembler::notEqual, done);
10771     __ subptr(rsp, 8);
10772     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10773     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10774     __ pop($dst$$Register);
10775     __ bind(done);
10776   %}
10777   ins_pipe(pipe_slow);
10778 %}
10779 
10780 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10781 %{
10782   match(Set dst (ConvD2I src));
10783   effect(KILL cr);
10784 
10785   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10786             "cmpl    $dst, #0x80000000\n\t"
10787             "jne,s   done\n\t"
10788             "subq    rsp, #8\n\t"
10789             "movsd   [rsp], $src\n\t"
10790             "call    d2i_fixup\n\t"
10791             "popq    $dst\n"
10792     "done:   "%}
10793   ins_encode %{
10794     Label done;
10795     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10796     __ cmpl($dst$$Register, 0x80000000);
10797     __ jccb(Assembler::notEqual, done);
10798     __ subptr(rsp, 8);
10799     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10800     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10801     __ pop($dst$$Register);
10802     __ bind(done);
10803   %}
10804   ins_pipe(pipe_slow);
10805 %}
10806 
10807 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10808 %{
10809   match(Set dst (ConvD2L src));
10810   effect(KILL cr);
10811 
10812   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10813             "cmpq    $dst, [0x8000000000000000]\n\t"
10814             "jne,s   done\n\t"
10815             "subq    rsp, #8\n\t"
10816             "movsd   [rsp], $src\n\t"
10817             "call    d2l_fixup\n\t"
10818             "popq    $dst\n"
10819     "done:   "%}
10820   ins_encode %{
10821     Label done;
10822     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10823     __ cmp64($dst$$Register,
10824              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10825     __ jccb(Assembler::notEqual, done);
10826     __ subptr(rsp, 8);
10827     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10828     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10829     __ pop($dst$$Register);
10830     __ bind(done);
10831   %}
10832   ins_pipe(pipe_slow);
10833 %}
10834 
10835 instruct convI2F_reg_reg(regF dst, rRegI src)
10836 %{
10837   predicate(!UseXmmI2F);
10838   match(Set dst (ConvI2F src));
10839 
10840   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10841   ins_encode %{
10842     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10843   %}
10844   ins_pipe(pipe_slow); // XXX
10845 %}
10846 
10847 instruct convI2F_reg_mem(regF dst, memory src)
10848 %{
10849   match(Set dst (ConvI2F (LoadI src)));
10850 
10851   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10852   ins_encode %{
10853     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10854   %}
10855   ins_pipe(pipe_slow); // XXX
10856 %}
10857 
10858 instruct convI2D_reg_reg(regD dst, rRegI src)
10859 %{
10860   predicate(!UseXmmI2D);
10861   match(Set dst (ConvI2D src));
10862 
10863   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10864   ins_encode %{
10865     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10866   %}
10867   ins_pipe(pipe_slow); // XXX
10868 %}
10869 
10870 instruct convI2D_reg_mem(regD dst, memory src)
10871 %{
10872   match(Set dst (ConvI2D (LoadI src)));
10873 
10874   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10875   ins_encode %{
10876     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10877   %}
10878   ins_pipe(pipe_slow); // XXX
10879 %}
10880 
10881 instruct convXI2F_reg(regF dst, rRegI src)
10882 %{
10883   predicate(UseXmmI2F);
10884   match(Set dst (ConvI2F src));
10885 
10886   format %{ "movdl $dst, $src\n\t"
10887             "cvtdq2psl $dst, $dst\t# i2f" %}
10888   ins_encode %{
10889     __ movdl($dst$$XMMRegister, $src$$Register);
10890     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10891   %}
10892   ins_pipe(pipe_slow); // XXX
10893 %}
10894 
10895 instruct convXI2D_reg(regD dst, rRegI src)
10896 %{
10897   predicate(UseXmmI2D);
10898   match(Set dst (ConvI2D src));
10899 
10900   format %{ "movdl $dst, $src\n\t"
10901             "cvtdq2pdl $dst, $dst\t# i2d" %}
10902   ins_encode %{
10903     __ movdl($dst$$XMMRegister, $src$$Register);
10904     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10905   %}
10906   ins_pipe(pipe_slow); // XXX
10907 %}
10908 
10909 instruct convL2F_reg_reg(regF dst, rRegL src)
10910 %{
10911   match(Set dst (ConvL2F src));
10912 
10913   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10914   ins_encode %{
10915     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10916   %}
10917   ins_pipe(pipe_slow); // XXX
10918 %}
10919 
10920 instruct convL2F_reg_mem(regF dst, memory src)
10921 %{
10922   match(Set dst (ConvL2F (LoadL src)));
10923 
10924   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10925   ins_encode %{
10926     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10927   %}
10928   ins_pipe(pipe_slow); // XXX
10929 %}
10930 
10931 instruct convL2D_reg_reg(regD dst, rRegL src)
10932 %{
10933   match(Set dst (ConvL2D src));
10934 
10935   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10936   ins_encode %{
10937     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10938   %}
10939   ins_pipe(pipe_slow); // XXX
10940 %}
10941 
10942 instruct convL2D_reg_mem(regD dst, memory src)
10943 %{
10944   match(Set dst (ConvL2D (LoadL src)));
10945 
10946   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10947   ins_encode %{
10948     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10949   %}
10950   ins_pipe(pipe_slow); // XXX
10951 %}
10952 
10953 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10954 %{
10955   match(Set dst (ConvI2L src));
10956 
10957   ins_cost(125);
10958   format %{ "movslq  $dst, $src\t# i2l" %}
10959   ins_encode %{
10960     __ movslq($dst$$Register, $src$$Register);
10961   %}
10962   ins_pipe(ialu_reg_reg);
10963 %}
10964 
10965 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10966 // %{
10967 //   match(Set dst (ConvI2L src));
10968 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10969 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10970 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10971 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10972 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10973 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10974 
10975 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10976 //   ins_encode(enc_copy(dst, src));
10977 // //   opcode(0x63); // needs REX.W
10978 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10979 //   ins_pipe(ialu_reg_reg);
10980 // %}
10981 
10982 // Zero-extend convert int to long
10983 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10984 %{
10985   match(Set dst (AndL (ConvI2L src) mask));
10986 
10987   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10988   ins_encode %{
10989     if ($dst$$reg != $src$$reg) {
10990       __ movl($dst$$Register, $src$$Register);
10991     }
10992   %}
10993   ins_pipe(ialu_reg_reg);
10994 %}
10995 
10996 // Zero-extend convert int to long
10997 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10998 %{
10999   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11000 
11001   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11002   ins_encode %{
11003     __ movl($dst$$Register, $src$$Address);
11004   %}
11005   ins_pipe(ialu_reg_mem);
11006 %}
11007 
11008 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11009 %{
11010   match(Set dst (AndL src mask));
11011 
11012   format %{ "movl    $dst, $src\t# zero-extend long" %}
11013   ins_encode %{
11014     __ movl($dst$$Register, $src$$Register);
11015   %}
11016   ins_pipe(ialu_reg_reg);
11017 %}
11018 
11019 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11020 %{
11021   match(Set dst (ConvL2I src));
11022 
11023   format %{ "movl    $dst, $src\t# l2i" %}
11024   ins_encode %{
11025     __ movl($dst$$Register, $src$$Register);
11026   %}
11027   ins_pipe(ialu_reg_reg);
11028 %}
11029 
11030 
11031 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11032   match(Set dst (MoveF2I src));
11033   effect(DEF dst, USE src);
11034 
11035   ins_cost(125);
11036   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11037   ins_encode %{
11038     __ movl($dst$$Register, Address(rsp, $src$$disp));
11039   %}
11040   ins_pipe(ialu_reg_mem);
11041 %}
11042 
11043 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11044   match(Set dst (MoveI2F src));
11045   effect(DEF dst, USE src);
11046 
11047   ins_cost(125);
11048   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11049   ins_encode %{
11050     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11051   %}
11052   ins_pipe(pipe_slow);
11053 %}
11054 
11055 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11056   match(Set dst (MoveD2L src));
11057   effect(DEF dst, USE src);
11058 
11059   ins_cost(125);
11060   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11061   ins_encode %{
11062     __ movq($dst$$Register, Address(rsp, $src$$disp));
11063   %}
11064   ins_pipe(ialu_reg_mem);
11065 %}
11066 
11067 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11068   predicate(!UseXmmLoadAndClearUpper);
11069   match(Set dst (MoveL2D src));
11070   effect(DEF dst, USE src);
11071 
11072   ins_cost(125);
11073   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11074   ins_encode %{
11075     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11076   %}
11077   ins_pipe(pipe_slow);
11078 %}
11079 
11080 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11081   predicate(UseXmmLoadAndClearUpper);
11082   match(Set dst (MoveL2D src));
11083   effect(DEF dst, USE src);
11084 
11085   ins_cost(125);
11086   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11087   ins_encode %{
11088     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11089   %}
11090   ins_pipe(pipe_slow);
11091 %}
11092 
11093 
11094 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11095   match(Set dst (MoveF2I src));
11096   effect(DEF dst, USE src);
11097 
11098   ins_cost(95); // XXX
11099   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11100   ins_encode %{
11101     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11102   %}
11103   ins_pipe(pipe_slow);
11104 %}
11105 
11106 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11107   match(Set dst (MoveI2F src));
11108   effect(DEF dst, USE src);
11109 
11110   ins_cost(100);
11111   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11112   ins_encode %{
11113     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11114   %}
11115   ins_pipe( ialu_mem_reg );
11116 %}
11117 
11118 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11119   match(Set dst (MoveD2L src));
11120   effect(DEF dst, USE src);
11121 
11122   ins_cost(95); // XXX
11123   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11124   ins_encode %{
11125     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11126   %}
11127   ins_pipe(pipe_slow);
11128 %}
11129 
11130 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11131   match(Set dst (MoveL2D src));
11132   effect(DEF dst, USE src);
11133 
11134   ins_cost(100);
11135   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11136   ins_encode %{
11137     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11138   %}
11139   ins_pipe(ialu_mem_reg);
11140 %}
11141 
11142 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11143   match(Set dst (MoveF2I src));
11144   effect(DEF dst, USE src);
11145   ins_cost(85);
11146   format %{ "movd    $dst,$src\t# MoveF2I" %}
11147   ins_encode %{
11148     __ movdl($dst$$Register, $src$$XMMRegister);
11149   %}
11150   ins_pipe( pipe_slow );
11151 %}
11152 
11153 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11154   match(Set dst (MoveD2L src));
11155   effect(DEF dst, USE src);
11156   ins_cost(85);
11157   format %{ "movd    $dst,$src\t# MoveD2L" %}
11158   ins_encode %{
11159     __ movdq($dst$$Register, $src$$XMMRegister);
11160   %}
11161   ins_pipe( pipe_slow );
11162 %}
11163 
11164 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11165   match(Set dst (MoveI2F src));
11166   effect(DEF dst, USE src);
11167   ins_cost(100);
11168   format %{ "movd    $dst,$src\t# MoveI2F" %}
11169   ins_encode %{
11170     __ movdl($dst$$XMMRegister, $src$$Register);
11171   %}
11172   ins_pipe( pipe_slow );
11173 %}
11174 
11175 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11176   match(Set dst (MoveL2D src));
11177   effect(DEF dst, USE src);
11178   ins_cost(100);
11179   format %{ "movd    $dst,$src\t# MoveL2D" %}
11180   ins_encode %{
11181      __ movdq($dst$$XMMRegister, $src$$Register);
11182   %}
11183   ins_pipe( pipe_slow );
11184 %}
11185 
11186 
11187 // =======================================================================
11188 // fast clearing of an array
11189 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11190                   Universe dummy, rFlagsReg cr)
11191 %{
11192   predicate(!((ClearArrayNode*)n)->is_large());
11193   match(Set dummy (ClearArray cnt base));
11194   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11195 
11196   format %{ $$template
11197     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11198     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11199     $$emit$$"jg      LARGE\n\t"
11200     $$emit$$"dec     rcx\n\t"
11201     $$emit$$"js      DONE\t# Zero length\n\t"
11202     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11203     $$emit$$"dec     rcx\n\t"
11204     $$emit$$"jge     LOOP\n\t"
11205     $$emit$$"jmp     DONE\n\t"
11206     $$emit$$"# LARGE:\n\t"
11207     if (UseFastStosb) {
11208        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11209        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11210     } else if (UseXMMForObjInit) {
11211        $$emit$$"mov     rdi,rax\n\t"
11212        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11213        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11214        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11215        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11216        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11217        $$emit$$"add     0x40,rax\n\t"
11218        $$emit$$"# L_zero_64_bytes:\n\t"
11219        $$emit$$"sub     0x8,rcx\n\t"
11220        $$emit$$"jge     L_loop\n\t"
11221        $$emit$$"add     0x4,rcx\n\t"
11222        $$emit$$"jl      L_tail\n\t"
11223        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11224        $$emit$$"add     0x20,rax\n\t"
11225        $$emit$$"sub     0x4,rcx\n\t"
11226        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11227        $$emit$$"add     0x4,rcx\n\t"
11228        $$emit$$"jle     L_end\n\t"
11229        $$emit$$"dec     rcx\n\t"
11230        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11231        $$emit$$"vmovq   xmm0,(rax)\n\t"
11232        $$emit$$"add     0x8,rax\n\t"
11233        $$emit$$"dec     rcx\n\t"
11234        $$emit$$"jge     L_sloop\n\t"
11235        $$emit$$"# L_end:\n\t"
11236     } else {
11237        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11238     }
11239     $$emit$$"# DONE"
11240   %}
11241   ins_encode %{
11242     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11243                  $tmp$$XMMRegister, false);
11244   %}
11245   ins_pipe(pipe_slow);
11246 %}
11247 
11248 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11249                         Universe dummy, rFlagsReg cr)
11250 %{
11251   predicate(((ClearArrayNode*)n)->is_large());
11252   match(Set dummy (ClearArray cnt base));
11253   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11254 
11255   format %{ $$template
11256     if (UseFastStosb) {
11257        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11258        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11259        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11260     } else if (UseXMMForObjInit) {
11261        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11262        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11263        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11264        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11265        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11266        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11267        $$emit$$"add     0x40,rax\n\t"
11268        $$emit$$"# L_zero_64_bytes:\n\t"
11269        $$emit$$"sub     0x8,rcx\n\t"
11270        $$emit$$"jge     L_loop\n\t"
11271        $$emit$$"add     0x4,rcx\n\t"
11272        $$emit$$"jl      L_tail\n\t"
11273        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11274        $$emit$$"add     0x20,rax\n\t"
11275        $$emit$$"sub     0x4,rcx\n\t"
11276        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11277        $$emit$$"add     0x4,rcx\n\t"
11278        $$emit$$"jle     L_end\n\t"
11279        $$emit$$"dec     rcx\n\t"
11280        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11281        $$emit$$"vmovq   xmm0,(rax)\n\t"
11282        $$emit$$"add     0x8,rax\n\t"
11283        $$emit$$"dec     rcx\n\t"
11284        $$emit$$"jge     L_sloop\n\t"
11285        $$emit$$"# L_end:\n\t"
11286     } else {
11287        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11288        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11289     }
11290   %}
11291   ins_encode %{
11292     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11293                  $tmp$$XMMRegister, true);
11294   %}
11295   ins_pipe(pipe_slow);
11296 %}
11297 
11298 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11299                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11300 %{
11301   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11302   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11303   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11304 
11305   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11306   ins_encode %{
11307     __ string_compare($str1$$Register, $str2$$Register,
11308                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11309                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
11310   %}
11311   ins_pipe( pipe_slow );
11312 %}
11313 
11314 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11315                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11316 %{
11317   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11318   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11319   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11320 
11321   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11322   ins_encode %{
11323     __ string_compare($str1$$Register, $str2$$Register,
11324                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11325                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
11326   %}
11327   ins_pipe( pipe_slow );
11328 %}
11329 
11330 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11331                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11332 %{
11333   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11334   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11335   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11336 
11337   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11338   ins_encode %{
11339     __ string_compare($str1$$Register, $str2$$Register,
11340                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11341                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
11342   %}
11343   ins_pipe( pipe_slow );
11344 %}
11345 
11346 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11347                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11348 %{
11349   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11350   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11351   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11352 
11353   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11354   ins_encode %{
11355     __ string_compare($str2$$Register, $str1$$Register,
11356                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11357                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
11358   %}
11359   ins_pipe( pipe_slow );
11360 %}
11361 
11362 // fast search of substring with known size.
11363 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11364                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11365 %{
11366   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11367   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11368   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11369 
11370   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11371   ins_encode %{
11372     int icnt2 = (int)$int_cnt2$$constant;
11373     if (icnt2 >= 16) {
11374       // IndexOf for constant substrings with size >= 16 elements
11375       // which don't need to be loaded through stack.
11376       __ string_indexofC8($str1$$Register, $str2$$Register,
11377                           $cnt1$$Register, $cnt2$$Register,
11378                           icnt2, $result$$Register,
11379                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11380     } else {
11381       // Small strings are loaded through stack if they cross page boundary.
11382       __ string_indexof($str1$$Register, $str2$$Register,
11383                         $cnt1$$Register, $cnt2$$Register,
11384                         icnt2, $result$$Register,
11385                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11386     }
11387   %}
11388   ins_pipe( pipe_slow );
11389 %}
11390 
11391 // fast search of substring with known size.
11392 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11393                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11394 %{
11395   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11396   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11397   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11398 
11399   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11400   ins_encode %{
11401     int icnt2 = (int)$int_cnt2$$constant;
11402     if (icnt2 >= 8) {
11403       // IndexOf for constant substrings with size >= 8 elements
11404       // which don't need to be loaded through stack.
11405       __ string_indexofC8($str1$$Register, $str2$$Register,
11406                           $cnt1$$Register, $cnt2$$Register,
11407                           icnt2, $result$$Register,
11408                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11409     } else {
11410       // Small strings are loaded through stack if they cross page boundary.
11411       __ string_indexof($str1$$Register, $str2$$Register,
11412                         $cnt1$$Register, $cnt2$$Register,
11413                         icnt2, $result$$Register,
11414                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11415     }
11416   %}
11417   ins_pipe( pipe_slow );
11418 %}
11419 
11420 // fast search of substring with known size.
11421 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11422                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11423 %{
11424   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11425   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11426   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11427 
11428   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11429   ins_encode %{
11430     int icnt2 = (int)$int_cnt2$$constant;
11431     if (icnt2 >= 8) {
11432       // IndexOf for constant substrings with size >= 8 elements
11433       // which don't need to be loaded through stack.
11434       __ string_indexofC8($str1$$Register, $str2$$Register,
11435                           $cnt1$$Register, $cnt2$$Register,
11436                           icnt2, $result$$Register,
11437                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11438     } else {
11439       // Small strings are loaded through stack if they cross page boundary.
11440       __ string_indexof($str1$$Register, $str2$$Register,
11441                         $cnt1$$Register, $cnt2$$Register,
11442                         icnt2, $result$$Register,
11443                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11444     }
11445   %}
11446   ins_pipe( pipe_slow );
11447 %}
11448 
11449 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11450                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11451 %{
11452   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11453   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11454   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11455 
11456   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11457   ins_encode %{
11458     __ string_indexof($str1$$Register, $str2$$Register,
11459                       $cnt1$$Register, $cnt2$$Register,
11460                       (-1), $result$$Register,
11461                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11462   %}
11463   ins_pipe( pipe_slow );
11464 %}
11465 
11466 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11467                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11468 %{
11469   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11470   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11471   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11472 
11473   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11474   ins_encode %{
11475     __ string_indexof($str1$$Register, $str2$$Register,
11476                       $cnt1$$Register, $cnt2$$Register,
11477                       (-1), $result$$Register,
11478                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11479   %}
11480   ins_pipe( pipe_slow );
11481 %}
11482 
11483 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11484                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11485 %{
11486   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11487   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11488   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11489 
11490   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11491   ins_encode %{
11492     __ string_indexof($str1$$Register, $str2$$Register,
11493                       $cnt1$$Register, $cnt2$$Register,
11494                       (-1), $result$$Register,
11495                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11496   %}
11497   ins_pipe( pipe_slow );
11498 %}
11499 
11500 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11501                               rbx_RegI result, legVecS vec1, legVecS vec2, legVecS vec3, rcx_RegI tmp, rFlagsReg cr)
11502 %{
11503   predicate(UseSSE42Intrinsics);
11504   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11505   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11506   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11507   ins_encode %{
11508     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11509                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11510   %}
11511   ins_pipe( pipe_slow );
11512 %}
11513 
11514 // fast string equals
11515 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11516                        legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11517 %{
11518   match(Set result (StrEquals (Binary str1 str2) cnt));
11519   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11520 
11521   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11522   ins_encode %{
11523     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11524                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11525                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11526   %}
11527   ins_pipe( pipe_slow );
11528 %}
11529 
11530 // fast array equals
11531 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11532                        legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11533 %{
11534   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11535   match(Set result (AryEq ary1 ary2));
11536   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11537 
11538   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11539   ins_encode %{
11540     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11541                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11542                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11543   %}
11544   ins_pipe( pipe_slow );
11545 %}
11546 
11547 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11548                       legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11549 %{
11550   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11551   match(Set result (AryEq ary1 ary2));
11552   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11553 
11554   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11555   ins_encode %{
11556     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11557                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11558                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11559   %}
11560   ins_pipe( pipe_slow );
11561 %}
11562 
11563 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11564                       legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11565 %{
11566   match(Set result (HasNegatives ary1 len));
11567   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11568 
11569   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11570   ins_encode %{
11571     __ has_negatives($ary1$$Register, $len$$Register,
11572                      $result$$Register, $tmp3$$Register,
11573                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11574   %}
11575   ins_pipe( pipe_slow );
11576 %}
11577 
11578 // fast char[] to byte[] compression
11579 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11580                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11581   match(Set result (StrCompressedCopy src (Binary dst len)));
11582   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11583 
11584   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11585   ins_encode %{
11586     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11587                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11588                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11589   %}
11590   ins_pipe( pipe_slow );
11591 %}
11592 
11593 // fast byte[] to char[] inflation
11594 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11595                         legVecS tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11596   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11597   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11598 
11599   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11600   ins_encode %{
11601     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11602                           $tmp1$$XMMRegister, $tmp2$$Register);
11603   %}
11604   ins_pipe( pipe_slow );
11605 %}
11606 
11607 // encode char[] to byte[] in ISO_8859_1
11608 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11609                           legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11610                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11611   match(Set result (EncodeISOArray src (Binary dst len)));
11612   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11613 
11614   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11615   ins_encode %{
11616     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11617                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11618                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11619   %}
11620   ins_pipe( pipe_slow );
11621 %}
11622 
11623 //----------Overflow Math Instructions-----------------------------------------
11624 
11625 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11626 %{
11627   match(Set cr (OverflowAddI op1 op2));
11628   effect(DEF cr, USE_KILL op1, USE op2);
11629 
11630   format %{ "addl    $op1, $op2\t# overflow check int" %}
11631 
11632   ins_encode %{
11633     __ addl($op1$$Register, $op2$$Register);
11634   %}
11635   ins_pipe(ialu_reg_reg);
11636 %}
11637 
11638 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11639 %{
11640   match(Set cr (OverflowAddI op1 op2));
11641   effect(DEF cr, USE_KILL op1, USE op2);
11642 
11643   format %{ "addl    $op1, $op2\t# overflow check int" %}
11644 
11645   ins_encode %{
11646     __ addl($op1$$Register, $op2$$constant);
11647   %}
11648   ins_pipe(ialu_reg_reg);
11649 %}
11650 
11651 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11652 %{
11653   match(Set cr (OverflowAddL op1 op2));
11654   effect(DEF cr, USE_KILL op1, USE op2);
11655 
11656   format %{ "addq    $op1, $op2\t# overflow check long" %}
11657   ins_encode %{
11658     __ addq($op1$$Register, $op2$$Register);
11659   %}
11660   ins_pipe(ialu_reg_reg);
11661 %}
11662 
11663 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11664 %{
11665   match(Set cr (OverflowAddL op1 op2));
11666   effect(DEF cr, USE_KILL op1, USE op2);
11667 
11668   format %{ "addq    $op1, $op2\t# overflow check long" %}
11669   ins_encode %{
11670     __ addq($op1$$Register, $op2$$constant);
11671   %}
11672   ins_pipe(ialu_reg_reg);
11673 %}
11674 
11675 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11676 %{
11677   match(Set cr (OverflowSubI op1 op2));
11678 
11679   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11680   ins_encode %{
11681     __ cmpl($op1$$Register, $op2$$Register);
11682   %}
11683   ins_pipe(ialu_reg_reg);
11684 %}
11685 
11686 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11687 %{
11688   match(Set cr (OverflowSubI op1 op2));
11689 
11690   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11691   ins_encode %{
11692     __ cmpl($op1$$Register, $op2$$constant);
11693   %}
11694   ins_pipe(ialu_reg_reg);
11695 %}
11696 
11697 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11698 %{
11699   match(Set cr (OverflowSubL op1 op2));
11700 
11701   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11702   ins_encode %{
11703     __ cmpq($op1$$Register, $op2$$Register);
11704   %}
11705   ins_pipe(ialu_reg_reg);
11706 %}
11707 
11708 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11709 %{
11710   match(Set cr (OverflowSubL op1 op2));
11711 
11712   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11713   ins_encode %{
11714     __ cmpq($op1$$Register, $op2$$constant);
11715   %}
11716   ins_pipe(ialu_reg_reg);
11717 %}
11718 
11719 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11720 %{
11721   match(Set cr (OverflowSubI zero op2));
11722   effect(DEF cr, USE_KILL op2);
11723 
11724   format %{ "negl    $op2\t# overflow check int" %}
11725   ins_encode %{
11726     __ negl($op2$$Register);
11727   %}
11728   ins_pipe(ialu_reg_reg);
11729 %}
11730 
11731 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11732 %{
11733   match(Set cr (OverflowSubL zero op2));
11734   effect(DEF cr, USE_KILL op2);
11735 
11736   format %{ "negq    $op2\t# overflow check long" %}
11737   ins_encode %{
11738     __ negq($op2$$Register);
11739   %}
11740   ins_pipe(ialu_reg_reg);
11741 %}
11742 
11743 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11744 %{
11745   match(Set cr (OverflowMulI op1 op2));
11746   effect(DEF cr, USE_KILL op1, USE op2);
11747 
11748   format %{ "imull    $op1, $op2\t# overflow check int" %}
11749   ins_encode %{
11750     __ imull($op1$$Register, $op2$$Register);
11751   %}
11752   ins_pipe(ialu_reg_reg_alu0);
11753 %}
11754 
11755 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11756 %{
11757   match(Set cr (OverflowMulI op1 op2));
11758   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11759 
11760   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11761   ins_encode %{
11762     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11763   %}
11764   ins_pipe(ialu_reg_reg_alu0);
11765 %}
11766 
11767 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11768 %{
11769   match(Set cr (OverflowMulL op1 op2));
11770   effect(DEF cr, USE_KILL op1, USE op2);
11771 
11772   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11773   ins_encode %{
11774     __ imulq($op1$$Register, $op2$$Register);
11775   %}
11776   ins_pipe(ialu_reg_reg_alu0);
11777 %}
11778 
11779 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11780 %{
11781   match(Set cr (OverflowMulL op1 op2));
11782   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11783 
11784   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11785   ins_encode %{
11786     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11787   %}
11788   ins_pipe(ialu_reg_reg_alu0);
11789 %}
11790 
11791 
11792 //----------Control Flow Instructions------------------------------------------
11793 // Signed compare Instructions
11794 
11795 // XXX more variants!!
11796 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11797 %{
11798   match(Set cr (CmpI op1 op2));
11799   effect(DEF cr, USE op1, USE op2);
11800 
11801   format %{ "cmpl    $op1, $op2" %}
11802   opcode(0x3B);  /* Opcode 3B /r */
11803   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11804   ins_pipe(ialu_cr_reg_reg);
11805 %}
11806 
11807 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11808 %{
11809   match(Set cr (CmpI op1 op2));
11810 
11811   format %{ "cmpl    $op1, $op2" %}
11812   opcode(0x81, 0x07); /* Opcode 81 /7 */
11813   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11814   ins_pipe(ialu_cr_reg_imm);
11815 %}
11816 
11817 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11818 %{
11819   match(Set cr (CmpI op1 (LoadI op2)));
11820 
11821   ins_cost(500); // XXX
11822   format %{ "cmpl    $op1, $op2" %}
11823   opcode(0x3B); /* Opcode 3B /r */
11824   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11825   ins_pipe(ialu_cr_reg_mem);
11826 %}
11827 
11828 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11829 %{
11830   match(Set cr (CmpI src zero));
11831 
11832   format %{ "testl   $src, $src" %}
11833   opcode(0x85);
11834   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11835   ins_pipe(ialu_cr_reg_imm);
11836 %}
11837 
11838 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11839 %{
11840   match(Set cr (CmpI (AndI src con) zero));
11841 
11842   format %{ "testl   $src, $con" %}
11843   opcode(0xF7, 0x00);
11844   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11845   ins_pipe(ialu_cr_reg_imm);
11846 %}
11847 
11848 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11849 %{
11850   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11851 
11852   format %{ "testl   $src, $mem" %}
11853   opcode(0x85);
11854   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11855   ins_pipe(ialu_cr_reg_mem);
11856 %}
11857 
11858 // Unsigned compare Instructions; really, same as signed except they
11859 // produce an rFlagsRegU instead of rFlagsReg.
11860 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11861 %{
11862   match(Set cr (CmpU op1 op2));
11863 
11864   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11865   opcode(0x3B); /* Opcode 3B /r */
11866   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11867   ins_pipe(ialu_cr_reg_reg);
11868 %}
11869 
11870 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11871 %{
11872   match(Set cr (CmpU op1 op2));
11873 
11874   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11875   opcode(0x81,0x07); /* Opcode 81 /7 */
11876   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11877   ins_pipe(ialu_cr_reg_imm);
11878 %}
11879 
11880 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11881 %{
11882   match(Set cr (CmpU op1 (LoadI op2)));
11883 
11884   ins_cost(500); // XXX
11885   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11886   opcode(0x3B); /* Opcode 3B /r */
11887   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11888   ins_pipe(ialu_cr_reg_mem);
11889 %}
11890 
11891 // // // Cisc-spilled version of cmpU_rReg
11892 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11893 // //%{
11894 // //  match(Set cr (CmpU (LoadI op1) op2));
11895 // //
11896 // //  format %{ "CMPu   $op1,$op2" %}
11897 // //  ins_cost(500);
11898 // //  opcode(0x39);  /* Opcode 39 /r */
11899 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11900 // //%}
11901 
11902 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11903 %{
11904   match(Set cr (CmpU src zero));
11905 
11906   format %{ "testl   $src, $src\t# unsigned" %}
11907   opcode(0x85);
11908   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11909   ins_pipe(ialu_cr_reg_imm);
11910 %}
11911 
11912 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11913 %{
11914   match(Set cr (CmpP op1 op2));
11915 
11916   format %{ "cmpq    $op1, $op2\t# ptr" %}
11917   opcode(0x3B); /* Opcode 3B /r */
11918   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11919   ins_pipe(ialu_cr_reg_reg);
11920 %}
11921 
11922 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11923 %{
11924   match(Set cr (CmpP op1 (LoadP op2)));
11925 
11926   ins_cost(500); // XXX
11927   format %{ "cmpq    $op1, $op2\t# ptr" %}
11928   opcode(0x3B); /* Opcode 3B /r */
11929   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11930   ins_pipe(ialu_cr_reg_mem);
11931 %}
11932 
11933 // // // Cisc-spilled version of cmpP_rReg
11934 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11935 // //%{
11936 // //  match(Set cr (CmpP (LoadP op1) op2));
11937 // //
11938 // //  format %{ "CMPu   $op1,$op2" %}
11939 // //  ins_cost(500);
11940 // //  opcode(0x39);  /* Opcode 39 /r */
11941 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11942 // //%}
11943 
11944 // XXX this is generalized by compP_rReg_mem???
11945 // Compare raw pointer (used in out-of-heap check).
11946 // Only works because non-oop pointers must be raw pointers
11947 // and raw pointers have no anti-dependencies.
11948 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11949 %{
11950   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11951   match(Set cr (CmpP op1 (LoadP op2)));
11952 
11953   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11954   opcode(0x3B); /* Opcode 3B /r */
11955   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11956   ins_pipe(ialu_cr_reg_mem);
11957 %}
11958 
11959 // This will generate a signed flags result. This should be OK since
11960 // any compare to a zero should be eq/neq.
11961 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11962 %{
11963   match(Set cr (CmpP src zero));
11964 
11965   format %{ "testq   $src, $src\t# ptr" %}
11966   opcode(0x85);
11967   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11968   ins_pipe(ialu_cr_reg_imm);
11969 %}
11970 
11971 // This will generate a signed flags result. This should be OK since
11972 // any compare to a zero should be eq/neq.
11973 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11974 %{
11975   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
11976   match(Set cr (CmpP (LoadP op) zero));
11977 
11978   ins_cost(500); // XXX
11979   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11980   opcode(0xF7); /* Opcode F7 /0 */
11981   ins_encode(REX_mem_wide(op),
11982              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11983   ins_pipe(ialu_cr_reg_imm);
11984 %}
11985 
11986 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11987 %{
11988   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
11989   match(Set cr (CmpP (LoadP mem) zero));
11990 
11991   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11992   ins_encode %{
11993     __ cmpq(r12, $mem$$Address);
11994   %}
11995   ins_pipe(ialu_cr_reg_mem);
11996 %}
11997 
11998 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11999 %{
12000   match(Set cr (CmpN op1 op2));
12001 
12002   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12003   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12004   ins_pipe(ialu_cr_reg_reg);
12005 %}
12006 
12007 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12008 %{
12009   match(Set cr (CmpN src (LoadN mem)));
12010 
12011   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12012   ins_encode %{
12013     __ cmpl($src$$Register, $mem$$Address);
12014   %}
12015   ins_pipe(ialu_cr_reg_mem);
12016 %}
12017 
12018 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12019   match(Set cr (CmpN op1 op2));
12020 
12021   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12022   ins_encode %{
12023     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12024   %}
12025   ins_pipe(ialu_cr_reg_imm);
12026 %}
12027 
12028 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12029 %{
12030   match(Set cr (CmpN src (LoadN mem)));
12031 
12032   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12033   ins_encode %{
12034     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12035   %}
12036   ins_pipe(ialu_cr_reg_mem);
12037 %}
12038 
12039 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12040   match(Set cr (CmpN op1 op2));
12041 
12042   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12043   ins_encode %{
12044     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12045   %}
12046   ins_pipe(ialu_cr_reg_imm);
12047 %}
12048 
12049 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12050 %{
12051   match(Set cr (CmpN src (LoadNKlass mem)));
12052 
12053   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12054   ins_encode %{
12055     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12056   %}
12057   ins_pipe(ialu_cr_reg_mem);
12058 %}
12059 
12060 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12061   match(Set cr (CmpN src zero));
12062 
12063   format %{ "testl   $src, $src\t# compressed ptr" %}
12064   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12065   ins_pipe(ialu_cr_reg_imm);
12066 %}
12067 
12068 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12069 %{
12070   predicate(CompressedOops::base() != NULL);
12071   match(Set cr (CmpN (LoadN mem) zero));
12072 
12073   ins_cost(500); // XXX
12074   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12075   ins_encode %{
12076     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12077   %}
12078   ins_pipe(ialu_cr_reg_mem);
12079 %}
12080 
12081 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12082 %{
12083   predicate(CompressedOops::base() == NULL && (CompressedKlassPointers::base() == NULL));
12084   match(Set cr (CmpN (LoadN mem) zero));
12085 
12086   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12087   ins_encode %{
12088     __ cmpl(r12, $mem$$Address);
12089   %}
12090   ins_pipe(ialu_cr_reg_mem);
12091 %}
12092 
12093 // Yanked all unsigned pointer compare operations.
12094 // Pointer compares are done with CmpP which is already unsigned.
12095 
12096 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12097 %{
12098   match(Set cr (CmpL op1 op2));
12099 
12100   format %{ "cmpq    $op1, $op2" %}
12101   opcode(0x3B);  /* Opcode 3B /r */
12102   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12103   ins_pipe(ialu_cr_reg_reg);
12104 %}
12105 
12106 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12107 %{
12108   match(Set cr (CmpL op1 op2));
12109 
12110   format %{ "cmpq    $op1, $op2" %}
12111   opcode(0x81, 0x07); /* Opcode 81 /7 */
12112   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12113   ins_pipe(ialu_cr_reg_imm);
12114 %}
12115 
12116 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12117 %{
12118   match(Set cr (CmpL op1 (LoadL op2)));
12119 
12120   format %{ "cmpq    $op1, $op2" %}
12121   opcode(0x3B); /* Opcode 3B /r */
12122   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12123   ins_pipe(ialu_cr_reg_mem);
12124 %}
12125 
12126 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12127 %{
12128   match(Set cr (CmpL src zero));
12129 
12130   format %{ "testq   $src, $src" %}
12131   opcode(0x85);
12132   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12133   ins_pipe(ialu_cr_reg_imm);
12134 %}
12135 
12136 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12137 %{
12138   match(Set cr (CmpL (AndL src con) zero));
12139 
12140   format %{ "testq   $src, $con\t# long" %}
12141   opcode(0xF7, 0x00);
12142   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
12143   ins_pipe(ialu_cr_reg_imm);
12144 %}
12145 
12146 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12147 %{
12148   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12149 
12150   format %{ "testq   $src, $mem" %}
12151   opcode(0x85);
12152   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12153   ins_pipe(ialu_cr_reg_mem);
12154 %}
12155 
12156 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12157 %{
12158   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12159 
12160   format %{ "testq   $src, $mem" %}
12161   opcode(0x85);
12162   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12163   ins_pipe(ialu_cr_reg_mem);
12164 %}
12165 
12166 // Manifest a CmpL result in an integer register.  Very painful.
12167 // This is the test to avoid.
12168 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12169 %{
12170   match(Set dst (CmpL3 src1 src2));
12171   effect(KILL flags);
12172 
12173   ins_cost(275); // XXX
12174   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12175             "movl    $dst, -1\n\t"
12176             "jl,s    done\n\t"
12177             "setne   $dst\n\t"
12178             "movzbl  $dst, $dst\n\t"
12179     "done:" %}
12180   ins_encode(cmpl3_flag(src1, src2, dst));
12181   ins_pipe(pipe_slow);
12182 %}
12183 
12184 // Unsigned long compare Instructions; really, same as signed long except they
12185 // produce an rFlagsRegU instead of rFlagsReg.
12186 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12187 %{
12188   match(Set cr (CmpUL op1 op2));
12189 
12190   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12191   opcode(0x3B);  /* Opcode 3B /r */
12192   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12193   ins_pipe(ialu_cr_reg_reg);
12194 %}
12195 
12196 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12197 %{
12198   match(Set cr (CmpUL op1 op2));
12199 
12200   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12201   opcode(0x81, 0x07); /* Opcode 81 /7 */
12202   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12203   ins_pipe(ialu_cr_reg_imm);
12204 %}
12205 
12206 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12207 %{
12208   match(Set cr (CmpUL op1 (LoadL op2)));
12209 
12210   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12211   opcode(0x3B); /* Opcode 3B /r */
12212   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12213   ins_pipe(ialu_cr_reg_mem);
12214 %}
12215 
12216 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12217 %{
12218   match(Set cr (CmpUL src zero));
12219 
12220   format %{ "testq   $src, $src\t# unsigned" %}
12221   opcode(0x85);
12222   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12223   ins_pipe(ialu_cr_reg_imm);
12224 %}
12225 
12226 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12227 %{
12228   match(Set cr (CmpI (LoadB mem) imm));
12229 
12230   ins_cost(125);
12231   format %{ "cmpb    $mem, $imm" %}
12232   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12233   ins_pipe(ialu_cr_reg_mem);
12234 %}
12235 
12236 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU8 imm, immI0 zero)
12237 %{
12238   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12239 
12240   ins_cost(125);
12241   format %{ "testb   $mem, $imm\t# ubyte" %}
12242   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12243   ins_pipe(ialu_cr_reg_mem);
12244 %}
12245 
12246 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
12247 %{
12248   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12249 
12250   ins_cost(125);
12251   format %{ "testb   $mem, $imm\t# byte" %}
12252   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12253   ins_pipe(ialu_cr_reg_mem);
12254 %}
12255 
12256 //----------Max and Min--------------------------------------------------------
12257 // Min Instructions
12258 
12259 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12260 %{
12261   effect(USE_DEF dst, USE src, USE cr);
12262 
12263   format %{ "cmovlgt $dst, $src\t# min" %}
12264   opcode(0x0F, 0x4F);
12265   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12266   ins_pipe(pipe_cmov_reg);
12267 %}
12268 
12269 
12270 instruct minI_rReg(rRegI dst, rRegI src)
12271 %{
12272   match(Set dst (MinI dst src));
12273 
12274   ins_cost(200);
12275   expand %{
12276     rFlagsReg cr;
12277     compI_rReg(cr, dst, src);
12278     cmovI_reg_g(dst, src, cr);
12279   %}
12280 %}
12281 
12282 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12283 %{
12284   effect(USE_DEF dst, USE src, USE cr);
12285 
12286   format %{ "cmovllt $dst, $src\t# max" %}
12287   opcode(0x0F, 0x4C);
12288   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12289   ins_pipe(pipe_cmov_reg);
12290 %}
12291 
12292 
12293 instruct maxI_rReg(rRegI dst, rRegI src)
12294 %{
12295   match(Set dst (MaxI dst src));
12296 
12297   ins_cost(200);
12298   expand %{
12299     rFlagsReg cr;
12300     compI_rReg(cr, dst, src);
12301     cmovI_reg_l(dst, src, cr);
12302   %}
12303 %}
12304 
12305 // ============================================================================
12306 // Branch Instructions
12307 
12308 // Jump Direct - Label defines a relative address from JMP+1
12309 instruct jmpDir(label labl)
12310 %{
12311   match(Goto);
12312   effect(USE labl);
12313 
12314   ins_cost(300);
12315   format %{ "jmp     $labl" %}
12316   size(5);
12317   ins_encode %{
12318     Label* L = $labl$$label;
12319     __ jmp(*L, false); // Always long jump
12320   %}
12321   ins_pipe(pipe_jmp);
12322 %}
12323 
12324 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12325 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12326 %{
12327   match(If cop cr);
12328   effect(USE labl);
12329 
12330   ins_cost(300);
12331   format %{ "j$cop     $labl" %}
12332   size(6);
12333   ins_encode %{
12334     Label* L = $labl$$label;
12335     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12336   %}
12337   ins_pipe(pipe_jcc);
12338 %}
12339 
12340 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12341 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12342 %{
12343   predicate(!n->has_vector_mask_set());
12344   match(CountedLoopEnd cop cr);
12345   effect(USE labl);
12346 
12347   ins_cost(300);
12348   format %{ "j$cop     $labl\t# loop end" %}
12349   size(6);
12350   ins_encode %{
12351     Label* L = $labl$$label;
12352     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12353   %}
12354   ins_pipe(pipe_jcc);
12355 %}
12356 
12357 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12358 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12359   predicate(!n->has_vector_mask_set());
12360   match(CountedLoopEnd cop cmp);
12361   effect(USE labl);
12362 
12363   ins_cost(300);
12364   format %{ "j$cop,u   $labl\t# loop end" %}
12365   size(6);
12366   ins_encode %{
12367     Label* L = $labl$$label;
12368     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12369   %}
12370   ins_pipe(pipe_jcc);
12371 %}
12372 
12373 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12374   predicate(!n->has_vector_mask_set());
12375   match(CountedLoopEnd cop cmp);
12376   effect(USE labl);
12377 
12378   ins_cost(200);
12379   format %{ "j$cop,u   $labl\t# loop end" %}
12380   size(6);
12381   ins_encode %{
12382     Label* L = $labl$$label;
12383     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12384   %}
12385   ins_pipe(pipe_jcc);
12386 %}
12387 
12388 // mask version
12389 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12390 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12391 %{
12392   predicate(n->has_vector_mask_set());
12393   match(CountedLoopEnd cop cr);
12394   effect(USE labl);
12395 
12396   ins_cost(400);
12397   format %{ "j$cop     $labl\t# loop end\n\t"
12398             "restorevectmask \t# vector mask restore for loops" %}
12399   size(10);
12400   ins_encode %{
12401     Label* L = $labl$$label;
12402     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12403     __ restorevectmask();
12404   %}
12405   ins_pipe(pipe_jcc);
12406 %}
12407 
12408 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12409 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12410   predicate(n->has_vector_mask_set());
12411   match(CountedLoopEnd cop cmp);
12412   effect(USE labl);
12413 
12414   ins_cost(400);
12415   format %{ "j$cop,u   $labl\t# loop end\n\t"
12416             "restorevectmask \t# vector mask restore for loops" %}
12417   size(10);
12418   ins_encode %{
12419     Label* L = $labl$$label;
12420     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12421     __ restorevectmask();
12422   %}
12423   ins_pipe(pipe_jcc);
12424 %}
12425 
12426 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12427   predicate(n->has_vector_mask_set());
12428   match(CountedLoopEnd cop cmp);
12429   effect(USE labl);
12430 
12431   ins_cost(300);
12432   format %{ "j$cop,u   $labl\t# loop end\n\t"
12433             "restorevectmask \t# vector mask restore for loops" %}
12434   size(10);
12435   ins_encode %{
12436     Label* L = $labl$$label;
12437     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12438     __ restorevectmask();
12439   %}
12440   ins_pipe(pipe_jcc);
12441 %}
12442 
12443 // Jump Direct Conditional - using unsigned comparison
12444 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12445   match(If cop cmp);
12446   effect(USE labl);
12447 
12448   ins_cost(300);
12449   format %{ "j$cop,u   $labl" %}
12450   size(6);
12451   ins_encode %{
12452     Label* L = $labl$$label;
12453     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12454   %}
12455   ins_pipe(pipe_jcc);
12456 %}
12457 
12458 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12459   match(If cop cmp);
12460   effect(USE labl);
12461 
12462   ins_cost(200);
12463   format %{ "j$cop,u   $labl" %}
12464   size(6);
12465   ins_encode %{
12466     Label* L = $labl$$label;
12467     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12468   %}
12469   ins_pipe(pipe_jcc);
12470 %}
12471 
12472 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12473   match(If cop cmp);
12474   effect(USE labl);
12475 
12476   ins_cost(200);
12477   format %{ $$template
12478     if ($cop$$cmpcode == Assembler::notEqual) {
12479       $$emit$$"jp,u    $labl\n\t"
12480       $$emit$$"j$cop,u   $labl"
12481     } else {
12482       $$emit$$"jp,u    done\n\t"
12483       $$emit$$"j$cop,u   $labl\n\t"
12484       $$emit$$"done:"
12485     }
12486   %}
12487   ins_encode %{
12488     Label* l = $labl$$label;
12489     if ($cop$$cmpcode == Assembler::notEqual) {
12490       __ jcc(Assembler::parity, *l, false);
12491       __ jcc(Assembler::notEqual, *l, false);
12492     } else if ($cop$$cmpcode == Assembler::equal) {
12493       Label done;
12494       __ jccb(Assembler::parity, done);
12495       __ jcc(Assembler::equal, *l, false);
12496       __ bind(done);
12497     } else {
12498        ShouldNotReachHere();
12499     }
12500   %}
12501   ins_pipe(pipe_jcc);
12502 %}
12503 
12504 // ============================================================================
12505 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12506 // superklass array for an instance of the superklass.  Set a hidden
12507 // internal cache on a hit (cache is checked with exposed code in
12508 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12509 // encoding ALSO sets flags.
12510 
12511 instruct partialSubtypeCheck(rdi_RegP result,
12512                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12513                              rFlagsReg cr)
12514 %{
12515   match(Set result (PartialSubtypeCheck sub super));
12516   effect(KILL rcx, KILL cr);
12517 
12518   ins_cost(1100);  // slightly larger than the next version
12519   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12520             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12521             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12522             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12523             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12524             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12525             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12526     "miss:\t" %}
12527 
12528   opcode(0x1); // Force a XOR of RDI
12529   ins_encode(enc_PartialSubtypeCheck());
12530   ins_pipe(pipe_slow);
12531 %}
12532 
12533 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12534                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12535                                      immP0 zero,
12536                                      rdi_RegP result)
12537 %{
12538   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12539   effect(KILL rcx, KILL result);
12540 
12541   ins_cost(1000);
12542   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12543             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12544             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12545             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12546             "jne,s   miss\t\t# Missed: flags nz\n\t"
12547             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12548     "miss:\t" %}
12549 
12550   opcode(0x0); // No need to XOR RDI
12551   ins_encode(enc_PartialSubtypeCheck());
12552   ins_pipe(pipe_slow);
12553 %}
12554 
12555 // ============================================================================
12556 // Branch Instructions -- short offset versions
12557 //
12558 // These instructions are used to replace jumps of a long offset (the default
12559 // match) with jumps of a shorter offset.  These instructions are all tagged
12560 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12561 // match rules in general matching.  Instead, the ADLC generates a conversion
12562 // method in the MachNode which can be used to do in-place replacement of the
12563 // long variant with the shorter variant.  The compiler will determine if a
12564 // branch can be taken by the is_short_branch_offset() predicate in the machine
12565 // specific code section of the file.
12566 
12567 // Jump Direct - Label defines a relative address from JMP+1
12568 instruct jmpDir_short(label labl) %{
12569   match(Goto);
12570   effect(USE labl);
12571 
12572   ins_cost(300);
12573   format %{ "jmp,s   $labl" %}
12574   size(2);
12575   ins_encode %{
12576     Label* L = $labl$$label;
12577     __ jmpb(*L);
12578   %}
12579   ins_pipe(pipe_jmp);
12580   ins_short_branch(1);
12581 %}
12582 
12583 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12584 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12585   match(If cop cr);
12586   effect(USE labl);
12587 
12588   ins_cost(300);
12589   format %{ "j$cop,s   $labl" %}
12590   size(2);
12591   ins_encode %{
12592     Label* L = $labl$$label;
12593     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12594   %}
12595   ins_pipe(pipe_jcc);
12596   ins_short_branch(1);
12597 %}
12598 
12599 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12600 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12601   match(CountedLoopEnd cop cr);
12602   effect(USE labl);
12603 
12604   ins_cost(300);
12605   format %{ "j$cop,s   $labl\t# loop end" %}
12606   size(2);
12607   ins_encode %{
12608     Label* L = $labl$$label;
12609     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12610   %}
12611   ins_pipe(pipe_jcc);
12612   ins_short_branch(1);
12613 %}
12614 
12615 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12616 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12617   match(CountedLoopEnd cop cmp);
12618   effect(USE labl);
12619 
12620   ins_cost(300);
12621   format %{ "j$cop,us  $labl\t# loop end" %}
12622   size(2);
12623   ins_encode %{
12624     Label* L = $labl$$label;
12625     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12626   %}
12627   ins_pipe(pipe_jcc);
12628   ins_short_branch(1);
12629 %}
12630 
12631 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12632   match(CountedLoopEnd cop cmp);
12633   effect(USE labl);
12634 
12635   ins_cost(300);
12636   format %{ "j$cop,us  $labl\t# loop end" %}
12637   size(2);
12638   ins_encode %{
12639     Label* L = $labl$$label;
12640     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12641   %}
12642   ins_pipe(pipe_jcc);
12643   ins_short_branch(1);
12644 %}
12645 
12646 // Jump Direct Conditional - using unsigned comparison
12647 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12648   match(If cop cmp);
12649   effect(USE labl);
12650 
12651   ins_cost(300);
12652   format %{ "j$cop,us  $labl" %}
12653   size(2);
12654   ins_encode %{
12655     Label* L = $labl$$label;
12656     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12657   %}
12658   ins_pipe(pipe_jcc);
12659   ins_short_branch(1);
12660 %}
12661 
12662 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12663   match(If cop cmp);
12664   effect(USE labl);
12665 
12666   ins_cost(300);
12667   format %{ "j$cop,us  $labl" %}
12668   size(2);
12669   ins_encode %{
12670     Label* L = $labl$$label;
12671     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12672   %}
12673   ins_pipe(pipe_jcc);
12674   ins_short_branch(1);
12675 %}
12676 
12677 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12678   match(If cop cmp);
12679   effect(USE labl);
12680 
12681   ins_cost(300);
12682   format %{ $$template
12683     if ($cop$$cmpcode == Assembler::notEqual) {
12684       $$emit$$"jp,u,s  $labl\n\t"
12685       $$emit$$"j$cop,u,s  $labl"
12686     } else {
12687       $$emit$$"jp,u,s  done\n\t"
12688       $$emit$$"j$cop,u,s  $labl\n\t"
12689       $$emit$$"done:"
12690     }
12691   %}
12692   size(4);
12693   ins_encode %{
12694     Label* l = $labl$$label;
12695     if ($cop$$cmpcode == Assembler::notEqual) {
12696       __ jccb(Assembler::parity, *l);
12697       __ jccb(Assembler::notEqual, *l);
12698     } else if ($cop$$cmpcode == Assembler::equal) {
12699       Label done;
12700       __ jccb(Assembler::parity, done);
12701       __ jccb(Assembler::equal, *l);
12702       __ bind(done);
12703     } else {
12704        ShouldNotReachHere();
12705     }
12706   %}
12707   ins_pipe(pipe_jcc);
12708   ins_short_branch(1);
12709 %}
12710 
12711 // ============================================================================
12712 // inlined locking and unlocking
12713 
12714 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12715   predicate(Compile::current()->use_rtm());
12716   match(Set cr (FastLock object box));
12717   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12718   ins_cost(300);
12719   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12720   ins_encode %{
12721     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12722                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12723                  _counters, _rtm_counters, _stack_rtm_counters,
12724                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12725                  true, ra_->C->profile_rtm());
12726   %}
12727   ins_pipe(pipe_slow);
12728 %}
12729 
12730 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12731   predicate(!Compile::current()->use_rtm());
12732   match(Set cr (FastLock object box));
12733   effect(TEMP tmp, TEMP scr, USE_KILL box);
12734   ins_cost(300);
12735   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12736   ins_encode %{
12737     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12738                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12739   %}
12740   ins_pipe(pipe_slow);
12741 %}
12742 
12743 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12744   match(Set cr (FastUnlock object box));
12745   effect(TEMP tmp, USE_KILL box);
12746   ins_cost(300);
12747   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12748   ins_encode %{
12749     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12750   %}
12751   ins_pipe(pipe_slow);
12752 %}
12753 
12754 
12755 // ============================================================================
12756 // Safepoint Instructions
12757 instruct safePoint_poll(rFlagsReg cr)
12758 %{
12759   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12760   match(SafePoint);
12761   effect(KILL cr);
12762 
12763   format %{ "testl   rax, [rip + #offset_to_poll_page]\t"
12764             "# Safepoint: poll for GC" %}
12765   ins_cost(125);
12766   ins_encode %{
12767     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12768     __ testl(rax, addr);
12769   %}
12770   ins_pipe(ialu_reg_mem);
12771 %}
12772 
12773 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12774 %{
12775   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12776   match(SafePoint poll);
12777   effect(KILL cr, USE poll);
12778 
12779   format %{ "testl   rax, [$poll]\t"
12780             "# Safepoint: poll for GC" %}
12781   ins_cost(125);
12782   ins_encode %{
12783     __ relocate(relocInfo::poll_type);
12784     __ testl(rax, Address($poll$$Register, 0));
12785   %}
12786   ins_pipe(ialu_reg_mem);
12787 %}
12788 
12789 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12790 %{
12791   predicate(SafepointMechanism::uses_thread_local_poll());
12792   match(SafePoint poll);
12793   effect(KILL cr, USE poll);
12794 
12795   format %{ "testl   rax, [$poll]\t"
12796             "# Safepoint: poll for GC" %}
12797   ins_cost(125);
12798   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12799   ins_encode %{
12800     __ relocate(relocInfo::poll_type);
12801     address pre_pc = __ pc();
12802     __ testl(rax, Address($poll$$Register, 0));
12803     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12804   %}
12805   ins_pipe(ialu_reg_mem);
12806 %}
12807 
12808 // ============================================================================
12809 // Procedure Call/Return Instructions
12810 // Call Java Static Instruction
12811 // Note: If this code changes, the corresponding ret_addr_offset() and
12812 //       compute_padding() functions will have to be adjusted.
12813 instruct CallStaticJavaDirect(method meth) %{
12814   match(CallStaticJava);
12815   effect(USE meth);
12816 
12817   ins_cost(300);
12818   format %{ "call,static " %}
12819   opcode(0xE8); /* E8 cd */
12820   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12821   ins_pipe(pipe_slow);
12822   ins_alignment(4);
12823 %}
12824 
12825 // Call Java Dynamic Instruction
12826 // Note: If this code changes, the corresponding ret_addr_offset() and
12827 //       compute_padding() functions will have to be adjusted.
12828 instruct CallDynamicJavaDirect(method meth)
12829 %{
12830   match(CallDynamicJava);
12831   effect(USE meth);
12832 
12833   ins_cost(300);
12834   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12835             "call,dynamic " %}
12836   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12837   ins_pipe(pipe_slow);
12838   ins_alignment(4);
12839 %}
12840 
12841 // Call Runtime Instruction
12842 instruct CallRuntimeDirect(method meth)
12843 %{
12844   match(CallRuntime);
12845   effect(USE meth);
12846 
12847   ins_cost(300);
12848   format %{ "call,runtime " %}
12849   ins_encode(clear_avx, Java_To_Runtime(meth));
12850   ins_pipe(pipe_slow);
12851 %}
12852 
12853 // Call runtime without safepoint
12854 instruct CallLeafDirect(method meth)
12855 %{
12856   match(CallLeaf);
12857   effect(USE meth);
12858 
12859   ins_cost(300);
12860   format %{ "call_leaf,runtime " %}
12861   ins_encode(clear_avx, Java_To_Runtime(meth));
12862   ins_pipe(pipe_slow);
12863 %}
12864 
12865 // Call runtime without safepoint
12866 instruct CallLeafNoFPDirect(method meth)
12867 %{
12868   match(CallLeafNoFP);
12869   effect(USE meth);
12870 
12871   ins_cost(300);
12872   format %{ "call_leaf_nofp,runtime " %}
12873   ins_encode(clear_avx, Java_To_Runtime(meth));
12874   ins_pipe(pipe_slow);
12875 %}
12876 
12877 // Return Instruction
12878 // Remove the return address & jump to it.
12879 // Notice: We always emit a nop after a ret to make sure there is room
12880 // for safepoint patching
12881 instruct Ret()
12882 %{
12883   match(Return);
12884 
12885   format %{ "ret" %}
12886   opcode(0xC3);
12887   ins_encode(OpcP);
12888   ins_pipe(pipe_jmp);
12889 %}
12890 
12891 // Tail Call; Jump from runtime stub to Java code.
12892 // Also known as an 'interprocedural jump'.
12893 // Target of jump will eventually return to caller.
12894 // TailJump below removes the return address.
12895 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12896 %{
12897   match(TailCall jump_target method_oop);
12898 
12899   ins_cost(300);
12900   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12901   opcode(0xFF, 0x4); /* Opcode FF /4 */
12902   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12903   ins_pipe(pipe_jmp);
12904 %}
12905 
12906 // Tail Jump; remove the return address; jump to target.
12907 // TailCall above leaves the return address around.
12908 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12909 %{
12910   match(TailJump jump_target ex_oop);
12911 
12912   ins_cost(300);
12913   format %{ "popq    rdx\t# pop return address\n\t"
12914             "jmp     $jump_target" %}
12915   opcode(0xFF, 0x4); /* Opcode FF /4 */
12916   ins_encode(Opcode(0x5a), // popq rdx
12917              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12918   ins_pipe(pipe_jmp);
12919 %}
12920 
12921 // Create exception oop: created by stack-crawling runtime code.
12922 // Created exception is now available to this handler, and is setup
12923 // just prior to jumping to this handler.  No code emitted.
12924 instruct CreateException(rax_RegP ex_oop)
12925 %{
12926   match(Set ex_oop (CreateEx));
12927 
12928   size(0);
12929   // use the following format syntax
12930   format %{ "# exception oop is in rax; no code emitted" %}
12931   ins_encode();
12932   ins_pipe(empty);
12933 %}
12934 
12935 // Rethrow exception:
12936 // The exception oop will come in the first argument position.
12937 // Then JUMP (not call) to the rethrow stub code.
12938 instruct RethrowException()
12939 %{
12940   match(Rethrow);
12941 
12942   // use the following format syntax
12943   format %{ "jmp     rethrow_stub" %}
12944   ins_encode(enc_rethrow);
12945   ins_pipe(pipe_jmp);
12946 %}
12947 
12948 // ============================================================================
12949 // This name is KNOWN by the ADLC and cannot be changed.
12950 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12951 // for this guy.
12952 instruct tlsLoadP(r15_RegP dst) %{
12953   match(Set dst (ThreadLocal));
12954   effect(DEF dst);
12955 
12956   size(0);
12957   format %{ "# TLS is in R15" %}
12958   ins_encode( /*empty encoding*/ );
12959   ins_pipe(ialu_reg_reg);
12960 %}
12961 
12962 
12963 //----------PEEPHOLE RULES-----------------------------------------------------
12964 // These must follow all instruction definitions as they use the names
12965 // defined in the instructions definitions.
12966 //
12967 // peepmatch ( root_instr_name [preceding_instruction]* );
12968 //
12969 // peepconstraint %{
12970 // (instruction_number.operand_name relational_op instruction_number.operand_name
12971 //  [, ...] );
12972 // // instruction numbers are zero-based using left to right order in peepmatch
12973 //
12974 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12975 // // provide an instruction_number.operand_name for each operand that appears
12976 // // in the replacement instruction's match rule
12977 //
12978 // ---------VM FLAGS---------------------------------------------------------
12979 //
12980 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12981 //
12982 // Each peephole rule is given an identifying number starting with zero and
12983 // increasing by one in the order seen by the parser.  An individual peephole
12984 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12985 // on the command-line.
12986 //
12987 // ---------CURRENT LIMITATIONS----------------------------------------------
12988 //
12989 // Only match adjacent instructions in same basic block
12990 // Only equality constraints
12991 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12992 // Only one replacement instruction
12993 //
12994 // ---------EXAMPLE----------------------------------------------------------
12995 //
12996 // // pertinent parts of existing instructions in architecture description
12997 // instruct movI(rRegI dst, rRegI src)
12998 // %{
12999 //   match(Set dst (CopyI src));
13000 // %}
13001 //
13002 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
13003 // %{
13004 //   match(Set dst (AddI dst src));
13005 //   effect(KILL cr);
13006 // %}
13007 //
13008 // // Change (inc mov) to lea
13009 // peephole %{
13010 //   // increment preceeded by register-register move
13011 //   peepmatch ( incI_rReg movI );
13012 //   // require that the destination register of the increment
13013 //   // match the destination register of the move
13014 //   peepconstraint ( 0.dst == 1.dst );
13015 //   // construct a replacement instruction that sets
13016 //   // the destination to ( move's source register + one )
13017 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13018 // %}
13019 //
13020 
13021 // Implementation no longer uses movX instructions since
13022 // machine-independent system no longer uses CopyX nodes.
13023 //
13024 // peephole
13025 // %{
13026 //   peepmatch (incI_rReg movI);
13027 //   peepconstraint (0.dst == 1.dst);
13028 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13029 // %}
13030 
13031 // peephole
13032 // %{
13033 //   peepmatch (decI_rReg movI);
13034 //   peepconstraint (0.dst == 1.dst);
13035 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13036 // %}
13037 
13038 // peephole
13039 // %{
13040 //   peepmatch (addI_rReg_imm movI);
13041 //   peepconstraint (0.dst == 1.dst);
13042 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13043 // %}
13044 
13045 // peephole
13046 // %{
13047 //   peepmatch (incL_rReg movL);
13048 //   peepconstraint (0.dst == 1.dst);
13049 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13050 // %}
13051 
13052 // peephole
13053 // %{
13054 //   peepmatch (decL_rReg movL);
13055 //   peepconstraint (0.dst == 1.dst);
13056 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13057 // %}
13058 
13059 // peephole
13060 // %{
13061 //   peepmatch (addL_rReg_imm movL);
13062 //   peepconstraint (0.dst == 1.dst);
13063 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13064 // %}
13065 
13066 // peephole
13067 // %{
13068 //   peepmatch (addP_rReg_imm movP);
13069 //   peepconstraint (0.dst == 1.dst);
13070 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13071 // %}
13072 
13073 // // Change load of spilled value to only a spill
13074 // instruct storeI(memory mem, rRegI src)
13075 // %{
13076 //   match(Set mem (StoreI mem src));
13077 // %}
13078 //
13079 // instruct loadI(rRegI dst, memory mem)
13080 // %{
13081 //   match(Set dst (LoadI mem));
13082 // %}
13083 //
13084 
13085 peephole
13086 %{
13087   peepmatch (loadI storeI);
13088   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13089   peepreplace (storeI(1.mem 1.mem 1.src));
13090 %}
13091 
13092 peephole
13093 %{
13094   peepmatch (loadL storeL);
13095   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13096   peepreplace (storeL(1.mem 1.mem 1.src));
13097 %}
13098 
13099 //----------SMARTSPILL RULES---------------------------------------------------
13100 // These must follow all instruction definitions as they use the names
13101 // defined in the instructions definitions.