1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RDX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RDI int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 
  323 source_hpp %{
  324 
  325 #include "peephole_x86_64.hpp"
  326 
  327 %}
  328 
  329 // Register masks
  330 source_hpp %{
  331 
  332 extern RegMask _ANY_REG_mask;
  333 extern RegMask _PTR_REG_mask;
  334 extern RegMask _PTR_REG_NO_RBP_mask;
  335 extern RegMask _PTR_NO_RAX_REG_mask;
  336 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  337 extern RegMask _LONG_REG_mask;
  338 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  339 extern RegMask _LONG_NO_RCX_REG_mask;
  340 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  341 extern RegMask _INT_REG_mask;
  342 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  343 extern RegMask _INT_NO_RCX_REG_mask;
  344 extern RegMask _INT_NO_RBP_R13_REG_mask;
  345 extern RegMask _FLOAT_REG_mask;
  346 
  347 extern RegMask _STACK_OR_PTR_REG_mask;
  348 extern RegMask _STACK_OR_LONG_REG_mask;
  349 extern RegMask _STACK_OR_INT_REG_mask;
  350 
  351 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  352 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  353 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_INT_REG_mask;
  380 
  381 static bool need_r12_heapbase() {
  382   return UseCompressedOops;
  383 }
  384 
  385 void reg_mask_init() {
  386   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  387   // We derive a number of subsets from it.
  388   _ANY_REG_mask = _ALL_REG_mask;
  389 
  390   if (PreserveFramePointer) {
  391     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  392     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  393   }
  394   if (need_r12_heapbase()) {
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  396     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  397   }
  398 
  399   _PTR_REG_mask = _ANY_REG_mask;
  400   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  401   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  404 
  405   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  406   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  407 
  408   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  409   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  410   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  411 
  412   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  413   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  414   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  415 
  416   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  417   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  418   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  419 
  420   _LONG_REG_mask = _PTR_REG_mask;
  421   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  422   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  423 
  424   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  425   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  426   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  429 
  430   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  431   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  432   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  433 
  434   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  435   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  436   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  439 
  440   _INT_REG_mask = _ALL_INT_REG_mask;
  441   if (PreserveFramePointer) {
  442     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  443   }
  444   if (need_r12_heapbase()) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  446   }
  447 
  448   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  449   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  450 
  451   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  452   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  453   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  454 
  455   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  456   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  457 
  458   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  459   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  460   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  461 
  462   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  463   // from the float_reg_legacy/float_reg_evex register class.
  464   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  465 }
  466 
  467 static bool generate_vzeroupper(Compile* C) {
  468   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  469 }
  470 
  471 static int clear_avx_size() {
  472   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  473 }
  474 
  475 // !!!!! Special hack to get all types of calls to specify the byte offset
  476 //       from the start of the call to the point where the return address
  477 //       will point.
  478 int MachCallStaticJavaNode::ret_addr_offset()
  479 {
  480   int offset = 5; // 5 bytes from start of call to where return address points
  481   offset += clear_avx_size();
  482   return offset;
  483 }
  484 
  485 int MachCallDynamicJavaNode::ret_addr_offset()
  486 {
  487   int offset = 15; // 15 bytes from start of call to where return address points
  488   offset += clear_avx_size();
  489   return offset;
  490 }
  491 
  492 int MachCallRuntimeNode::ret_addr_offset() {
  493   int offset = 13; // movq r10,#addr; callq (r10)
  494   if (this->ideal_Opcode() != Op_CallLeafVector) {
  495     offset += clear_avx_size();
  496   }
  497   return offset;
  498 }
  499 //
  500 // Compute padding required for nodes which need alignment
  501 //
  502 
  503 // The address of the call instruction needs to be 4-byte aligned to
  504 // ensure that it does not span a cache line so that it can be patched.
  505 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  506 {
  507   current_offset += clear_avx_size(); // skip vzeroupper
  508   current_offset += 1; // skip call opcode byte
  509   return align_up(current_offset, alignment_required()) - current_offset;
  510 }
  511 
  512 // The address of the call instruction needs to be 4-byte aligned to
  513 // ensure that it does not span a cache line so that it can be patched.
  514 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  515 {
  516   current_offset += clear_avx_size(); // skip vzeroupper
  517   current_offset += 11; // skip movq instruction + call opcode byte
  518   return align_up(current_offset, alignment_required()) - current_offset;
  519 }
  520 
  521 // EMIT_RM()
  522 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  523   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  524   cbuf.insts()->emit_int8(c);
  525 }
  526 
  527 // EMIT_CC()
  528 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  529   unsigned char c = (unsigned char) (f1 | f2);
  530   cbuf.insts()->emit_int8(c);
  531 }
  532 
  533 // EMIT_OPCODE()
  534 void emit_opcode(CodeBuffer &cbuf, int code) {
  535   cbuf.insts()->emit_int8((unsigned char) code);
  536 }
  537 
  538 // EMIT_OPCODE() w/ relocation information
  539 void emit_opcode(CodeBuffer &cbuf,
  540                  int code, relocInfo::relocType reloc, int offset, int format)
  541 {
  542   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  543   emit_opcode(cbuf, code);
  544 }
  545 
  546 // EMIT_D8()
  547 void emit_d8(CodeBuffer &cbuf, int d8) {
  548   cbuf.insts()->emit_int8((unsigned char) d8);
  549 }
  550 
  551 // EMIT_D16()
  552 void emit_d16(CodeBuffer &cbuf, int d16) {
  553   cbuf.insts()->emit_int16(d16);
  554 }
  555 
  556 // EMIT_D32()
  557 void emit_d32(CodeBuffer &cbuf, int d32) {
  558   cbuf.insts()->emit_int32(d32);
  559 }
  560 
  561 // EMIT_D64()
  562 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  563   cbuf.insts()->emit_int64(d64);
  564 }
  565 
  566 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  567 void emit_d32_reloc(CodeBuffer& cbuf,
  568                     int d32,
  569                     relocInfo::relocType reloc,
  570                     int format)
  571 {
  572   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  573   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  574   cbuf.insts()->emit_int32(d32);
  575 }
  576 
  577 // emit 32 bit value and construct relocation entry from RelocationHolder
  578 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  579 #ifdef ASSERT
  580   if (rspec.reloc()->type() == relocInfo::oop_type &&
  581       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  582     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  583     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  584   }
  585 #endif
  586   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  587   cbuf.insts()->emit_int32(d32);
  588 }
  589 
  590 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  591   address next_ip = cbuf.insts_end() + 4;
  592   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  593                  external_word_Relocation::spec(addr),
  594                  RELOC_DISP32);
  595 }
  596 
  597 
  598 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  599 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  600   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  601   cbuf.insts()->emit_int64(d64);
  602 }
  603 
  604 // emit 64 bit value and construct relocation entry from RelocationHolder
  605 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  606 #ifdef ASSERT
  607   if (rspec.reloc()->type() == relocInfo::oop_type &&
  608       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  609     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  610     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  611   }
  612 #endif
  613   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  614   cbuf.insts()->emit_int64(d64);
  615 }
  616 
  617 // Access stack slot for load or store
  618 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  619 {
  620   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  621   if (-0x80 <= disp && disp < 0x80) {
  622     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  623     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  624     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  625   } else {
  626     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  627     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  628     emit_d32(cbuf, disp);     // Displacement // R/M byte
  629   }
  630 }
  631 
  632    // rRegI ereg, memory mem) %{    // emit_reg_mem
  633 void encode_RegMem(CodeBuffer &cbuf,
  634                    int reg,
  635                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  636 {
  637   assert(disp_reloc == relocInfo::none, "cannot have disp");
  638   int regenc = reg & 7;
  639   int baseenc = base & 7;
  640   int indexenc = index & 7;
  641 
  642   // There is no index & no scale, use form without SIB byte
  643   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  644     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  645     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  646       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  647     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  648       // If 8-bit displacement, mode 0x1
  649       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  650       emit_d8(cbuf, disp);
  651     } else {
  652       // If 32-bit displacement
  653       if (base == -1) { // Special flag for absolute address
  654         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  655         if (disp_reloc != relocInfo::none) {
  656           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  657         } else {
  658           emit_d32(cbuf, disp);
  659         }
  660       } else {
  661         // Normal base + offset
  662         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  663         if (disp_reloc != relocInfo::none) {
  664           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  665         } else {
  666           emit_d32(cbuf, disp);
  667         }
  668       }
  669     }
  670   } else {
  671     // Else, encode with the SIB byte
  672     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  673     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  674       // If no displacement
  675       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  676       emit_rm(cbuf, scale, indexenc, baseenc);
  677     } else {
  678       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  679         // If 8-bit displacement, mode 0x1
  680         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  681         emit_rm(cbuf, scale, indexenc, baseenc);
  682         emit_d8(cbuf, disp);
  683       } else {
  684         // If 32-bit displacement
  685         if (base == 0x04 ) {
  686           emit_rm(cbuf, 0x2, regenc, 0x4);
  687           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  688         } else {
  689           emit_rm(cbuf, 0x2, regenc, 0x4);
  690           emit_rm(cbuf, scale, indexenc, baseenc); // *
  691         }
  692         if (disp_reloc != relocInfo::none) {
  693           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  694         } else {
  695           emit_d32(cbuf, disp);
  696         }
  697       }
  698     }
  699   }
  700 }
  701 
  702 // This could be in MacroAssembler but it's fairly C2 specific
  703 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  704   Label exit;
  705   __ jccb(Assembler::noParity, exit);
  706   __ pushf();
  707   //
  708   // comiss/ucomiss instructions set ZF,PF,CF flags and
  709   // zero OF,AF,SF for NaN values.
  710   // Fixup flags by zeroing ZF,PF so that compare of NaN
  711   // values returns 'less than' result (CF is set).
  712   // Leave the rest of flags unchanged.
  713   //
  714   //    7 6 5 4 3 2 1 0
  715   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  716   //    0 0 1 0 1 0 1 1   (0x2B)
  717   //
  718   __ andq(Address(rsp, 0), 0xffffff2b);
  719   __ popf();
  720   __ bind(exit);
  721 }
  722 
  723 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  724   Label done;
  725   __ movl(dst, -1);
  726   __ jcc(Assembler::parity, done);
  727   __ jcc(Assembler::below, done);
  728   __ setb(Assembler::notEqual, dst);
  729   __ movzbl(dst, dst);
  730   __ bind(done);
  731 }
  732 
  733 // Math.min()    # Math.max()
  734 // --------------------------
  735 // ucomis[s/d]   #
  736 // ja   -> b     # a
  737 // jp   -> NaN   # NaN
  738 // jb   -> a     # b
  739 // je            #
  740 // |-jz -> a | b # a & b
  741 // |    -> a     #
  742 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  743                      XMMRegister a, XMMRegister b,
  744                      XMMRegister xmmt, Register rt,
  745                      bool min, bool single) {
  746 
  747   Label nan, zero, below, above, done;
  748 
  749   if (single)
  750     __ ucomiss(a, b);
  751   else
  752     __ ucomisd(a, b);
  753 
  754   if (dst->encoding() != (min ? b : a)->encoding())
  755     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  756   else
  757     __ jccb(Assembler::above, done);
  758 
  759   __ jccb(Assembler::parity, nan);  // PF=1
  760   __ jccb(Assembler::below, below); // CF=1
  761 
  762   // equal
  763   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  764   if (single) {
  765     __ ucomiss(a, xmmt);
  766     __ jccb(Assembler::equal, zero);
  767 
  768     __ movflt(dst, a);
  769     __ jmp(done);
  770   }
  771   else {
  772     __ ucomisd(a, xmmt);
  773     __ jccb(Assembler::equal, zero);
  774 
  775     __ movdbl(dst, a);
  776     __ jmp(done);
  777   }
  778 
  779   __ bind(zero);
  780   if (min)
  781     __ vpor(dst, a, b, Assembler::AVX_128bit);
  782   else
  783     __ vpand(dst, a, b, Assembler::AVX_128bit);
  784 
  785   __ jmp(done);
  786 
  787   __ bind(above);
  788   if (single)
  789     __ movflt(dst, min ? b : a);
  790   else
  791     __ movdbl(dst, min ? b : a);
  792 
  793   __ jmp(done);
  794 
  795   __ bind(nan);
  796   if (single) {
  797     __ movl(rt, 0x7fc00000); // Float.NaN
  798     __ movdl(dst, rt);
  799   }
  800   else {
  801     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  802     __ movdq(dst, rt);
  803   }
  804   __ jmp(done);
  805 
  806   __ bind(below);
  807   if (single)
  808     __ movflt(dst, min ? a : b);
  809   else
  810     __ movdbl(dst, min ? a : b);
  811 
  812   __ bind(done);
  813 }
  814 
  815 //=============================================================================
  816 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  817 
  818 int ConstantTable::calculate_table_base_offset() const {
  819   return 0;  // absolute addressing, no offset
  820 }
  821 
  822 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  823 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  824   ShouldNotReachHere();
  825 }
  826 
  827 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  828   // Empty encoding
  829 }
  830 
  831 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  832   return 0;
  833 }
  834 
  835 #ifndef PRODUCT
  836 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  837   st->print("# MachConstantBaseNode (empty encoding)");
  838 }
  839 #endif
  840 
  841 
  842 //=============================================================================
  843 #ifndef PRODUCT
  844 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  845   Compile* C = ra_->C;
  846 
  847   int framesize = C->output()->frame_size_in_bytes();
  848   int bangsize = C->output()->bang_size_in_bytes();
  849   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  850   // Remove wordSize for return addr which is already pushed.
  851   framesize -= wordSize;
  852 
  853   if (C->output()->need_stack_bang(bangsize)) {
  854     framesize -= wordSize;
  855     st->print("# stack bang (%d bytes)", bangsize);
  856     st->print("\n\t");
  857     st->print("pushq   rbp\t# Save rbp");
  858     if (PreserveFramePointer) {
  859         st->print("\n\t");
  860         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  861     }
  862     if (framesize) {
  863       st->print("\n\t");
  864       st->print("subq    rsp, #%d\t# Create frame",framesize);
  865     }
  866   } else {
  867     st->print("subq    rsp, #%d\t# Create frame",framesize);
  868     st->print("\n\t");
  869     framesize -= wordSize;
  870     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  871     if (PreserveFramePointer) {
  872       st->print("\n\t");
  873       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  874       if (framesize > 0) {
  875         st->print("\n\t");
  876         st->print("addq    rbp, #%d", framesize);
  877       }
  878     }
  879   }
  880 
  881   if (VerifyStackAtCalls) {
  882     st->print("\n\t");
  883     framesize -= wordSize;
  884     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  885 #ifdef ASSERT
  886     st->print("\n\t");
  887     st->print("# stack alignment check");
  888 #endif
  889   }
  890   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  891     st->print("\n\t");
  892     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  893     st->print("\n\t");
  894     st->print("je      fast_entry\t");
  895     st->print("\n\t");
  896     st->print("call    #nmethod_entry_barrier_stub\t");
  897     st->print("\n\tfast_entry:");
  898   }
  899   st->cr();
  900 }
  901 #endif
  902 
  903 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  904   Compile* C = ra_->C;
  905   C2_MacroAssembler _masm(&cbuf);
  906 
  907   int framesize = C->output()->frame_size_in_bytes();
  908   int bangsize = C->output()->bang_size_in_bytes();
  909 
  910   if (C->clinit_barrier_on_entry()) {
  911     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  912     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  913 
  914     Label L_skip_barrier;
  915     Register klass = rscratch1;
  916 
  917     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  918     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  919 
  920     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  921 
  922     __ bind(L_skip_barrier);
  923   }
  924 
  925   int max_monitors = C->method() != NULL ? C->max_monitors() : 0;
  926   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL, max_monitors);
  927 
  928   C->output()->set_frame_complete(cbuf.insts_size());
  929 
  930   if (C->has_mach_constant_base_node()) {
  931     // NOTE: We set the table base offset here because users might be
  932     // emitted before MachConstantBaseNode.
  933     ConstantTable& constant_table = C->output()->constant_table();
  934     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  935   }
  936 }
  937 
  938 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  939 {
  940   return MachNode::size(ra_); // too many variables; just compute it
  941                               // the hard way
  942 }
  943 
  944 int MachPrologNode::reloc() const
  945 {
  946   return 0; // a large enough number
  947 }
  948 
  949 //=============================================================================
  950 #ifndef PRODUCT
  951 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  952 {
  953   Compile* C = ra_->C;
  954   if (generate_vzeroupper(C)) {
  955     st->print("vzeroupper");
  956     st->cr(); st->print("\t");
  957   }
  958 
  959   int framesize = C->output()->frame_size_in_bytes();
  960   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  961   // Remove word for return adr already pushed
  962   // and RBP
  963   framesize -= 2*wordSize;
  964 
  965   if (framesize) {
  966     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  967     st->print("\t");
  968   }
  969 
  970   st->print_cr("popq    rbp");
  971   if (do_polling() && C->is_method_compilation()) {
  972     st->print("\t");
  973     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  974                  "ja      #safepoint_stub\t"
  975                  "# Safepoint: poll for GC");
  976   }
  977 }
  978 #endif
  979 
  980 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  981 {
  982   Compile* C = ra_->C;
  983   MacroAssembler _masm(&cbuf);
  984 
  985   if (generate_vzeroupper(C)) {
  986     // Clear upper bits of YMM registers when current compiled code uses
  987     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  988     __ vzeroupper();
  989   }
  990 
  991   int framesize = C->output()->frame_size_in_bytes();
  992   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  993   // Remove word for return adr already pushed
  994   // and RBP
  995   framesize -= 2*wordSize;
  996 
  997   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  998 
  999   if (framesize) {
 1000     emit_opcode(cbuf, Assembler::REX_W);
 1001     if (framesize < 0x80) {
 1002       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 1003       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 1004       emit_d8(cbuf, framesize);
 1005     } else {
 1006       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 1007       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 1008       emit_d32(cbuf, framesize);
 1009     }
 1010   }
 1011 
 1012   // popq rbp
 1013   emit_opcode(cbuf, 0x58 | RBP_enc);
 1014 
 1015   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1016     __ reserved_stack_check();
 1017   }
 1018 
 1019   if (do_polling() && C->is_method_compilation()) {
 1020     MacroAssembler _masm(&cbuf);
 1021     Label dummy_label;
 1022     Label* code_stub = &dummy_label;
 1023     if (!C->output()->in_scratch_emit_size()) {
 1024       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1025       C->output()->add_stub(stub);
 1026       code_stub = &stub->entry();
 1027     }
 1028     __ relocate(relocInfo::poll_return_type);
 1029     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1030   }
 1031 }
 1032 
 1033 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1034 {
 1035   return MachNode::size(ra_); // too many variables; just compute it
 1036                               // the hard way
 1037 }
 1038 
 1039 int MachEpilogNode::reloc() const
 1040 {
 1041   return 2; // a large enough number
 1042 }
 1043 
 1044 const Pipeline* MachEpilogNode::pipeline() const
 1045 {
 1046   return MachNode::pipeline_class();
 1047 }
 1048 
 1049 //=============================================================================
 1050 
 1051 enum RC {
 1052   rc_bad,
 1053   rc_int,
 1054   rc_kreg,
 1055   rc_float,
 1056   rc_stack
 1057 };
 1058 
 1059 static enum RC rc_class(OptoReg::Name reg)
 1060 {
 1061   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1062 
 1063   if (OptoReg::is_stack(reg)) return rc_stack;
 1064 
 1065   VMReg r = OptoReg::as_VMReg(reg);
 1066 
 1067   if (r->is_Register()) return rc_int;
 1068 
 1069   if (r->is_KRegister()) return rc_kreg;
 1070 
 1071   assert(r->is_XMMRegister(), "must be");
 1072   return rc_float;
 1073 }
 1074 
 1075 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1076 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1077                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1078 
 1079 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1080                      int stack_offset, int reg, uint ireg, outputStream* st);
 1081 
 1082 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1083                                       int dst_offset, uint ireg, outputStream* st) {
 1084   if (cbuf) {
 1085     MacroAssembler _masm(cbuf);
 1086     switch (ireg) {
 1087     case Op_VecS:
 1088       __ movq(Address(rsp, -8), rax);
 1089       __ movl(rax, Address(rsp, src_offset));
 1090       __ movl(Address(rsp, dst_offset), rax);
 1091       __ movq(rax, Address(rsp, -8));
 1092       break;
 1093     case Op_VecD:
 1094       __ pushq(Address(rsp, src_offset));
 1095       __ popq (Address(rsp, dst_offset));
 1096       break;
 1097     case Op_VecX:
 1098       __ pushq(Address(rsp, src_offset));
 1099       __ popq (Address(rsp, dst_offset));
 1100       __ pushq(Address(rsp, src_offset+8));
 1101       __ popq (Address(rsp, dst_offset+8));
 1102       break;
 1103     case Op_VecY:
 1104       __ vmovdqu(Address(rsp, -32), xmm0);
 1105       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1106       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1107       __ vmovdqu(xmm0, Address(rsp, -32));
 1108       break;
 1109     case Op_VecZ:
 1110       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1111       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1112       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1113       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1114       break;
 1115     default:
 1116       ShouldNotReachHere();
 1117     }
 1118 #ifndef PRODUCT
 1119   } else {
 1120     switch (ireg) {
 1121     case Op_VecS:
 1122       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1123                 "movl    rax, [rsp + #%d]\n\t"
 1124                 "movl    [rsp + #%d], rax\n\t"
 1125                 "movq    rax, [rsp - #8]",
 1126                 src_offset, dst_offset);
 1127       break;
 1128     case Op_VecD:
 1129       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1130                 "popq    [rsp + #%d]",
 1131                 src_offset, dst_offset);
 1132       break;
 1133      case Op_VecX:
 1134       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1135                 "popq    [rsp + #%d]\n\t"
 1136                 "pushq   [rsp + #%d]\n\t"
 1137                 "popq    [rsp + #%d]",
 1138                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1139       break;
 1140     case Op_VecY:
 1141       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1142                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1143                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1144                 "vmovdqu xmm0, [rsp - #32]",
 1145                 src_offset, dst_offset);
 1146       break;
 1147     case Op_VecZ:
 1148       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1149                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1150                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1151                 "vmovdqu xmm0, [rsp - #64]",
 1152                 src_offset, dst_offset);
 1153       break;
 1154     default:
 1155       ShouldNotReachHere();
 1156     }
 1157 #endif
 1158   }
 1159 }
 1160 
 1161 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1162                                        PhaseRegAlloc* ra_,
 1163                                        bool do_size,
 1164                                        outputStream* st) const {
 1165   assert(cbuf != NULL || st  != NULL, "sanity");
 1166   // Get registers to move
 1167   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1168   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1169   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1170   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1171 
 1172   enum RC src_second_rc = rc_class(src_second);
 1173   enum RC src_first_rc = rc_class(src_first);
 1174   enum RC dst_second_rc = rc_class(dst_second);
 1175   enum RC dst_first_rc = rc_class(dst_first);
 1176 
 1177   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1178          "must move at least 1 register" );
 1179 
 1180   if (src_first == dst_first && src_second == dst_second) {
 1181     // Self copy, no move
 1182     return 0;
 1183   }
 1184   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1185     uint ireg = ideal_reg();
 1186     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1187     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1188     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1189       // mem -> mem
 1190       int src_offset = ra_->reg2offset(src_first);
 1191       int dst_offset = ra_->reg2offset(dst_first);
 1192       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1193     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1194       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1195     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1196       int stack_offset = ra_->reg2offset(dst_first);
 1197       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1198     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1199       int stack_offset = ra_->reg2offset(src_first);
 1200       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1201     } else {
 1202       ShouldNotReachHere();
 1203     }
 1204     return 0;
 1205   }
 1206   if (src_first_rc == rc_stack) {
 1207     // mem ->
 1208     if (dst_first_rc == rc_stack) {
 1209       // mem -> mem
 1210       assert(src_second != dst_first, "overlap");
 1211       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1212           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1213         // 64-bit
 1214         int src_offset = ra_->reg2offset(src_first);
 1215         int dst_offset = ra_->reg2offset(dst_first);
 1216         if (cbuf) {
 1217           MacroAssembler _masm(cbuf);
 1218           __ pushq(Address(rsp, src_offset));
 1219           __ popq (Address(rsp, dst_offset));
 1220 #ifndef PRODUCT
 1221         } else {
 1222           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1223                     "popq    [rsp + #%d]",
 1224                      src_offset, dst_offset);
 1225 #endif
 1226         }
 1227       } else {
 1228         // 32-bit
 1229         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1230         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1231         // No pushl/popl, so:
 1232         int src_offset = ra_->reg2offset(src_first);
 1233         int dst_offset = ra_->reg2offset(dst_first);
 1234         if (cbuf) {
 1235           MacroAssembler _masm(cbuf);
 1236           __ movq(Address(rsp, -8), rax);
 1237           __ movl(rax, Address(rsp, src_offset));
 1238           __ movl(Address(rsp, dst_offset), rax);
 1239           __ movq(rax, Address(rsp, -8));
 1240 #ifndef PRODUCT
 1241         } else {
 1242           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1243                     "movl    rax, [rsp + #%d]\n\t"
 1244                     "movl    [rsp + #%d], rax\n\t"
 1245                     "movq    rax, [rsp - #8]",
 1246                      src_offset, dst_offset);
 1247 #endif
 1248         }
 1249       }
 1250       return 0;
 1251     } else if (dst_first_rc == rc_int) {
 1252       // mem -> gpr
 1253       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1254           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1255         // 64-bit
 1256         int offset = ra_->reg2offset(src_first);
 1257         if (cbuf) {
 1258           MacroAssembler _masm(cbuf);
 1259           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1260 #ifndef PRODUCT
 1261         } else {
 1262           st->print("movq    %s, [rsp + #%d]\t# spill",
 1263                      Matcher::regName[dst_first],
 1264                      offset);
 1265 #endif
 1266         }
 1267       } else {
 1268         // 32-bit
 1269         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1270         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1271         int offset = ra_->reg2offset(src_first);
 1272         if (cbuf) {
 1273           MacroAssembler _masm(cbuf);
 1274           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1275 #ifndef PRODUCT
 1276         } else {
 1277           st->print("movl    %s, [rsp + #%d]\t# spill",
 1278                      Matcher::regName[dst_first],
 1279                      offset);
 1280 #endif
 1281         }
 1282       }
 1283       return 0;
 1284     } else if (dst_first_rc == rc_float) {
 1285       // mem-> xmm
 1286       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1287           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1288         // 64-bit
 1289         int offset = ra_->reg2offset(src_first);
 1290         if (cbuf) {
 1291           MacroAssembler _masm(cbuf);
 1292           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1293 #ifndef PRODUCT
 1294         } else {
 1295           st->print("%s  %s, [rsp + #%d]\t# spill",
 1296                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1297                      Matcher::regName[dst_first],
 1298                      offset);
 1299 #endif
 1300         }
 1301       } else {
 1302         // 32-bit
 1303         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1304         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1305         int offset = ra_->reg2offset(src_first);
 1306         if (cbuf) {
 1307           MacroAssembler _masm(cbuf);
 1308           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1309 #ifndef PRODUCT
 1310         } else {
 1311           st->print("movss   %s, [rsp + #%d]\t# spill",
 1312                      Matcher::regName[dst_first],
 1313                      offset);
 1314 #endif
 1315         }
 1316       }
 1317       return 0;
 1318     } else if (dst_first_rc == rc_kreg) {
 1319       // mem -> kreg
 1320       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1321           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1322         // 64-bit
 1323         int offset = ra_->reg2offset(src_first);
 1324         if (cbuf) {
 1325           MacroAssembler _masm(cbuf);
 1326           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1327 #ifndef PRODUCT
 1328         } else {
 1329           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1330                      Matcher::regName[dst_first],
 1331                      offset);
 1332 #endif
 1333         }
 1334       }
 1335       return 0;
 1336     }
 1337   } else if (src_first_rc == rc_int) {
 1338     // gpr ->
 1339     if (dst_first_rc == rc_stack) {
 1340       // gpr -> mem
 1341       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1342           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1343         // 64-bit
 1344         int offset = ra_->reg2offset(dst_first);
 1345         if (cbuf) {
 1346           MacroAssembler _masm(cbuf);
 1347           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1348 #ifndef PRODUCT
 1349         } else {
 1350           st->print("movq    [rsp + #%d], %s\t# spill",
 1351                      offset,
 1352                      Matcher::regName[src_first]);
 1353 #endif
 1354         }
 1355       } else {
 1356         // 32-bit
 1357         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1358         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1359         int offset = ra_->reg2offset(dst_first);
 1360         if (cbuf) {
 1361           MacroAssembler _masm(cbuf);
 1362           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1363 #ifndef PRODUCT
 1364         } else {
 1365           st->print("movl    [rsp + #%d], %s\t# spill",
 1366                      offset,
 1367                      Matcher::regName[src_first]);
 1368 #endif
 1369         }
 1370       }
 1371       return 0;
 1372     } else if (dst_first_rc == rc_int) {
 1373       // gpr -> gpr
 1374       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1375           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1376         // 64-bit
 1377         if (cbuf) {
 1378           MacroAssembler _masm(cbuf);
 1379           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1380                   as_Register(Matcher::_regEncode[src_first]));
 1381 #ifndef PRODUCT
 1382         } else {
 1383           st->print("movq    %s, %s\t# spill",
 1384                      Matcher::regName[dst_first],
 1385                      Matcher::regName[src_first]);
 1386 #endif
 1387         }
 1388         return 0;
 1389       } else {
 1390         // 32-bit
 1391         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1392         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1393         if (cbuf) {
 1394           MacroAssembler _masm(cbuf);
 1395           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1396                   as_Register(Matcher::_regEncode[src_first]));
 1397 #ifndef PRODUCT
 1398         } else {
 1399           st->print("movl    %s, %s\t# spill",
 1400                      Matcher::regName[dst_first],
 1401                      Matcher::regName[src_first]);
 1402 #endif
 1403         }
 1404         return 0;
 1405       }
 1406     } else if (dst_first_rc == rc_float) {
 1407       // gpr -> xmm
 1408       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1409           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1410         // 64-bit
 1411         if (cbuf) {
 1412           MacroAssembler _masm(cbuf);
 1413           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1414 #ifndef PRODUCT
 1415         } else {
 1416           st->print("movdq   %s, %s\t# spill",
 1417                      Matcher::regName[dst_first],
 1418                      Matcher::regName[src_first]);
 1419 #endif
 1420         }
 1421       } else {
 1422         // 32-bit
 1423         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1424         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1425         if (cbuf) {
 1426           MacroAssembler _masm(cbuf);
 1427           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1428 #ifndef PRODUCT
 1429         } else {
 1430           st->print("movdl   %s, %s\t# spill",
 1431                      Matcher::regName[dst_first],
 1432                      Matcher::regName[src_first]);
 1433 #endif
 1434         }
 1435       }
 1436       return 0;
 1437     } else if (dst_first_rc == rc_kreg) {
 1438       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1439           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1440         // 64-bit
 1441         if (cbuf) {
 1442           MacroAssembler _masm(cbuf);
 1443           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1444   #ifndef PRODUCT
 1445         } else {
 1446            st->print("kmovq   %s, %s\t# spill",
 1447                        Matcher::regName[dst_first],
 1448                        Matcher::regName[src_first]);
 1449   #endif
 1450         }
 1451       }
 1452       Unimplemented();
 1453       return 0;
 1454     }
 1455   } else if (src_first_rc == rc_float) {
 1456     // xmm ->
 1457     if (dst_first_rc == rc_stack) {
 1458       // xmm -> mem
 1459       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1460           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1461         // 64-bit
 1462         int offset = ra_->reg2offset(dst_first);
 1463         if (cbuf) {
 1464           MacroAssembler _masm(cbuf);
 1465           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1466 #ifndef PRODUCT
 1467         } else {
 1468           st->print("movsd   [rsp + #%d], %s\t# spill",
 1469                      offset,
 1470                      Matcher::regName[src_first]);
 1471 #endif
 1472         }
 1473       } else {
 1474         // 32-bit
 1475         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1476         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1477         int offset = ra_->reg2offset(dst_first);
 1478         if (cbuf) {
 1479           MacroAssembler _masm(cbuf);
 1480           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1481 #ifndef PRODUCT
 1482         } else {
 1483           st->print("movss   [rsp + #%d], %s\t# spill",
 1484                      offset,
 1485                      Matcher::regName[src_first]);
 1486 #endif
 1487         }
 1488       }
 1489       return 0;
 1490     } else if (dst_first_rc == rc_int) {
 1491       // xmm -> gpr
 1492       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1493           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1494         // 64-bit
 1495         if (cbuf) {
 1496           MacroAssembler _masm(cbuf);
 1497           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1498 #ifndef PRODUCT
 1499         } else {
 1500           st->print("movdq   %s, %s\t# spill",
 1501                      Matcher::regName[dst_first],
 1502                      Matcher::regName[src_first]);
 1503 #endif
 1504         }
 1505       } else {
 1506         // 32-bit
 1507         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1508         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1509         if (cbuf) {
 1510           MacroAssembler _masm(cbuf);
 1511           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1512 #ifndef PRODUCT
 1513         } else {
 1514           st->print("movdl   %s, %s\t# spill",
 1515                      Matcher::regName[dst_first],
 1516                      Matcher::regName[src_first]);
 1517 #endif
 1518         }
 1519       }
 1520       return 0;
 1521     } else if (dst_first_rc == rc_float) {
 1522       // xmm -> xmm
 1523       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1524           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1525         // 64-bit
 1526         if (cbuf) {
 1527           MacroAssembler _masm(cbuf);
 1528           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1529 #ifndef PRODUCT
 1530         } else {
 1531           st->print("%s  %s, %s\t# spill",
 1532                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1533                      Matcher::regName[dst_first],
 1534                      Matcher::regName[src_first]);
 1535 #endif
 1536         }
 1537       } else {
 1538         // 32-bit
 1539         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1540         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1541         if (cbuf) {
 1542           MacroAssembler _masm(cbuf);
 1543           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1544 #ifndef PRODUCT
 1545         } else {
 1546           st->print("%s  %s, %s\t# spill",
 1547                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1548                      Matcher::regName[dst_first],
 1549                      Matcher::regName[src_first]);
 1550 #endif
 1551         }
 1552       }
 1553       return 0;
 1554     } else if (dst_first_rc == rc_kreg) {
 1555       assert(false, "Illegal spilling");
 1556       return 0;
 1557     }
 1558   } else if (src_first_rc == rc_kreg) {
 1559     if (dst_first_rc == rc_stack) {
 1560       // mem -> kreg
 1561       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1562           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1563         // 64-bit
 1564         int offset = ra_->reg2offset(dst_first);
 1565         if (cbuf) {
 1566           MacroAssembler _masm(cbuf);
 1567           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1568 #ifndef PRODUCT
 1569         } else {
 1570           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1571                      offset,
 1572                      Matcher::regName[src_first]);
 1573 #endif
 1574         }
 1575       }
 1576       return 0;
 1577     } else if (dst_first_rc == rc_int) {
 1578       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1579           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1580         // 64-bit
 1581         if (cbuf) {
 1582           MacroAssembler _masm(cbuf);
 1583           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1584 #ifndef PRODUCT
 1585         } else {
 1586          st->print("kmovq   %s, %s\t# spill",
 1587                      Matcher::regName[dst_first],
 1588                      Matcher::regName[src_first]);
 1589 #endif
 1590         }
 1591       }
 1592       Unimplemented();
 1593       return 0;
 1594     } else if (dst_first_rc == rc_kreg) {
 1595       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1596           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1597         // 64-bit
 1598         if (cbuf) {
 1599           MacroAssembler _masm(cbuf);
 1600           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1601 #ifndef PRODUCT
 1602         } else {
 1603          st->print("kmovq   %s, %s\t# spill",
 1604                      Matcher::regName[dst_first],
 1605                      Matcher::regName[src_first]);
 1606 #endif
 1607         }
 1608       }
 1609       return 0;
 1610     } else if (dst_first_rc == rc_float) {
 1611       assert(false, "Illegal spill");
 1612       return 0;
 1613     }
 1614   }
 1615 
 1616   assert(0," foo ");
 1617   Unimplemented();
 1618   return 0;
 1619 }
 1620 
 1621 #ifndef PRODUCT
 1622 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1623   implementation(NULL, ra_, false, st);
 1624 }
 1625 #endif
 1626 
 1627 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1628   implementation(&cbuf, ra_, false, NULL);
 1629 }
 1630 
 1631 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1632   return MachNode::size(ra_);
 1633 }
 1634 
 1635 //=============================================================================
 1636 #ifndef PRODUCT
 1637 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1638 {
 1639   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1640   int reg = ra_->get_reg_first(this);
 1641   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1642             Matcher::regName[reg], offset);
 1643 }
 1644 #endif
 1645 
 1646 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1647 {
 1648   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1649   int reg = ra_->get_encode(this);
 1650   if (offset >= 0x80) {
 1651     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1652     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1653     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1654     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1655     emit_d32(cbuf, offset);
 1656   } else {
 1657     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1658     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1659     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1660     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1661     emit_d8(cbuf, offset);
 1662   }
 1663 }
 1664 
 1665 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1666 {
 1667   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1668   return (offset < 0x80) ? 5 : 8; // REX
 1669 }
 1670 
 1671 //=============================================================================
 1672 #ifndef PRODUCT
 1673 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1674 {
 1675   if (UseCompressedClassPointers) {
 1676     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1677     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1678     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1679   } else {
 1680     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1681                  "# Inline cache check");
 1682   }
 1683   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1684   st->print_cr("\tnop\t# nops to align entry point");
 1685 }
 1686 #endif
 1687 
 1688 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1689 {
 1690   MacroAssembler masm(&cbuf);
 1691   uint insts_size = cbuf.insts_size();
 1692   if (UseCompressedClassPointers) {
 1693     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1694     masm.cmpptr(rax, rscratch1);
 1695   } else {
 1696     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1697   }
 1698 
 1699   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1700 
 1701   /* WARNING these NOPs are critical so that verified entry point is properly
 1702      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1703   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1704   if (OptoBreakpoint) {
 1705     // Leave space for int3
 1706     nops_cnt -= 1;
 1707   }
 1708   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1709   if (nops_cnt > 0)
 1710     masm.nop(nops_cnt);
 1711 }
 1712 
 1713 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1714 {
 1715   return MachNode::size(ra_); // too many variables; just compute it
 1716                               // the hard way
 1717 }
 1718 
 1719 
 1720 //=============================================================================
 1721 
 1722 const bool Matcher::supports_vector_calling_convention(void) {
 1723   if (EnableVectorSupport && UseVectorStubs) {
 1724     return true;
 1725   }
 1726   return false;
 1727 }
 1728 
 1729 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1730   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1731   int lo = XMM0_num;
 1732   int hi = XMM0b_num;
 1733   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1734   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1735   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1736   return OptoRegPair(hi, lo);
 1737 }
 1738 
 1739 // Is this branch offset short enough that a short branch can be used?
 1740 //
 1741 // NOTE: If the platform does not provide any short branch variants, then
 1742 //       this method should return false for offset 0.
 1743 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1744   // The passed offset is relative to address of the branch.
 1745   // On 86 a branch displacement is calculated relative to address
 1746   // of a next instruction.
 1747   offset -= br_size;
 1748 
 1749   // the short version of jmpConUCF2 contains multiple branches,
 1750   // making the reach slightly less
 1751   if (rule == jmpConUCF2_rule)
 1752     return (-126 <= offset && offset <= 125);
 1753   return (-128 <= offset && offset <= 127);
 1754 }
 1755 
 1756 // Return whether or not this register is ever used as an argument.
 1757 // This function is used on startup to build the trampoline stubs in
 1758 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1759 // call in the trampoline, and arguments in those registers not be
 1760 // available to the callee.
 1761 bool Matcher::can_be_java_arg(int reg)
 1762 {
 1763   return
 1764     reg ==  RDI_num || reg == RDI_H_num ||
 1765     reg ==  RSI_num || reg == RSI_H_num ||
 1766     reg ==  RDX_num || reg == RDX_H_num ||
 1767     reg ==  RCX_num || reg == RCX_H_num ||
 1768     reg ==   R8_num || reg ==  R8_H_num ||
 1769     reg ==   R9_num || reg ==  R9_H_num ||
 1770     reg ==  R12_num || reg == R12_H_num ||
 1771     reg == XMM0_num || reg == XMM0b_num ||
 1772     reg == XMM1_num || reg == XMM1b_num ||
 1773     reg == XMM2_num || reg == XMM2b_num ||
 1774     reg == XMM3_num || reg == XMM3b_num ||
 1775     reg == XMM4_num || reg == XMM4b_num ||
 1776     reg == XMM5_num || reg == XMM5b_num ||
 1777     reg == XMM6_num || reg == XMM6b_num ||
 1778     reg == XMM7_num || reg == XMM7b_num;
 1779 }
 1780 
 1781 bool Matcher::is_spillable_arg(int reg)
 1782 {
 1783   return can_be_java_arg(reg);
 1784 }
 1785 
 1786 uint Matcher::int_pressure_limit()
 1787 {
 1788   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1789 }
 1790 
 1791 uint Matcher::float_pressure_limit()
 1792 {
 1793   // After experiment around with different values, the following default threshold
 1794   // works best for LCM's register pressure scheduling on x64.
 1795   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1796   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1797   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1798 }
 1799 
 1800 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1801   // In 64 bit mode a code which use multiply when
 1802   // devisor is constant is faster than hardware
 1803   // DIV instruction (it uses MulHiL).
 1804   return false;
 1805 }
 1806 
 1807 // Register for DIVI projection of divmodI
 1808 RegMask Matcher::divI_proj_mask() {
 1809   return INT_RAX_REG_mask();
 1810 }
 1811 
 1812 // Register for MODI projection of divmodI
 1813 RegMask Matcher::modI_proj_mask() {
 1814   return INT_RDX_REG_mask();
 1815 }
 1816 
 1817 // Register for DIVL projection of divmodL
 1818 RegMask Matcher::divL_proj_mask() {
 1819   return LONG_RAX_REG_mask();
 1820 }
 1821 
 1822 // Register for MODL projection of divmodL
 1823 RegMask Matcher::modL_proj_mask() {
 1824   return LONG_RDX_REG_mask();
 1825 }
 1826 
 1827 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1828 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1829     return NO_REG_mask();
 1830 }
 1831 
 1832 %}
 1833 
 1834 //----------ENCODING BLOCK-----------------------------------------------------
 1835 // This block specifies the encoding classes used by the compiler to
 1836 // output byte streams.  Encoding classes are parameterized macros
 1837 // used by Machine Instruction Nodes in order to generate the bit
 1838 // encoding of the instruction.  Operands specify their base encoding
 1839 // interface with the interface keyword.  There are currently
 1840 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1841 // COND_INTER.  REG_INTER causes an operand to generate a function
 1842 // which returns its register number when queried.  CONST_INTER causes
 1843 // an operand to generate a function which returns the value of the
 1844 // constant when queried.  MEMORY_INTER causes an operand to generate
 1845 // four functions which return the Base Register, the Index Register,
 1846 // the Scale Value, and the Offset Value of the operand when queried.
 1847 // COND_INTER causes an operand to generate six functions which return
 1848 // the encoding code (ie - encoding bits for the instruction)
 1849 // associated with each basic boolean condition for a conditional
 1850 // instruction.
 1851 //
 1852 // Instructions specify two basic values for encoding.  Again, a
 1853 // function is available to check if the constant displacement is an
 1854 // oop. They use the ins_encode keyword to specify their encoding
 1855 // classes (which must be a sequence of enc_class names, and their
 1856 // parameters, specified in the encoding block), and they use the
 1857 // opcode keyword to specify, in order, their primary, secondary, and
 1858 // tertiary opcode.  Only the opcode sections which a particular
 1859 // instruction needs for encoding need to be specified.
 1860 encode %{
 1861   // Build emit functions for each basic byte or larger field in the
 1862   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1863   // from C++ code in the enc_class source block.  Emit functions will
 1864   // live in the main source block for now.  In future, we can
 1865   // generalize this by adding a syntax that specifies the sizes of
 1866   // fields in an order, so that the adlc can build the emit functions
 1867   // automagically
 1868 
 1869   // Emit primary opcode
 1870   enc_class OpcP
 1871   %{
 1872     emit_opcode(cbuf, $primary);
 1873   %}
 1874 
 1875   // Emit secondary opcode
 1876   enc_class OpcS
 1877   %{
 1878     emit_opcode(cbuf, $secondary);
 1879   %}
 1880 
 1881   // Emit tertiary opcode
 1882   enc_class OpcT
 1883   %{
 1884     emit_opcode(cbuf, $tertiary);
 1885   %}
 1886 
 1887   // Emit opcode directly
 1888   enc_class Opcode(immI d8)
 1889   %{
 1890     emit_opcode(cbuf, $d8$$constant);
 1891   %}
 1892 
 1893   // Emit size prefix
 1894   enc_class SizePrefix
 1895   %{
 1896     emit_opcode(cbuf, 0x66);
 1897   %}
 1898 
 1899   enc_class reg(rRegI reg)
 1900   %{
 1901     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1902   %}
 1903 
 1904   enc_class reg_reg(rRegI dst, rRegI src)
 1905   %{
 1906     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1907   %}
 1908 
 1909   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1910   %{
 1911     emit_opcode(cbuf, $opcode$$constant);
 1912     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1913   %}
 1914 
 1915   enc_class cdql_enc(no_rax_rdx_RegI div)
 1916   %{
 1917     // Full implementation of Java idiv and irem; checks for
 1918     // special case as described in JVM spec., p.243 & p.271.
 1919     //
 1920     //         normal case                           special case
 1921     //
 1922     // input : rax: dividend                         min_int
 1923     //         reg: divisor                          -1
 1924     //
 1925     // output: rax: quotient  (= rax idiv reg)       min_int
 1926     //         rdx: remainder (= rax irem reg)       0
 1927     //
 1928     //  Code sequnce:
 1929     //
 1930     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1931     //    5:   75 07/08                jne    e <normal>
 1932     //    7:   33 d2                   xor    %edx,%edx
 1933     //  [div >= 8 -> offset + 1]
 1934     //  [REX_B]
 1935     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1936     //    c:   74 03/04                je     11 <done>
 1937     // 000000000000000e <normal>:
 1938     //    e:   99                      cltd
 1939     //  [div >= 8 -> offset + 1]
 1940     //  [REX_B]
 1941     //    f:   f7 f9                   idiv   $div
 1942     // 0000000000000011 <done>:
 1943     MacroAssembler _masm(&cbuf);
 1944     Label normal;
 1945     Label done;
 1946 
 1947     // cmp    $0x80000000,%eax
 1948     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1949 
 1950     // jne    e <normal>
 1951     __ jccb(Assembler::notEqual, normal);
 1952 
 1953     // xor    %edx,%edx
 1954     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1955 
 1956     // cmp    $0xffffffffffffffff,%ecx
 1957     __ cmpl($div$$Register, -1);
 1958 
 1959     // je     11 <done>
 1960     __ jccb(Assembler::equal, done);
 1961 
 1962     // <normal>
 1963     // cltd
 1964     __ bind(normal);
 1965     __ cdql();
 1966 
 1967     // idivl
 1968     // <done>
 1969     __ idivl($div$$Register);
 1970     __ bind(done);
 1971   %}
 1972 
 1973   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1974   %{
 1975     // Full implementation of Java ldiv and lrem; checks for
 1976     // special case as described in JVM spec., p.243 & p.271.
 1977     //
 1978     //         normal case                           special case
 1979     //
 1980     // input : rax: dividend                         min_long
 1981     //         reg: divisor                          -1
 1982     //
 1983     // output: rax: quotient  (= rax idiv reg)       min_long
 1984     //         rdx: remainder (= rax irem reg)       0
 1985     //
 1986     //  Code sequnce:
 1987     //
 1988     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1989     //    7:   00 00 80
 1990     //    a:   48 39 d0                cmp    %rdx,%rax
 1991     //    d:   75 08                   jne    17 <normal>
 1992     //    f:   33 d2                   xor    %edx,%edx
 1993     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1994     //   15:   74 05                   je     1c <done>
 1995     // 0000000000000017 <normal>:
 1996     //   17:   48 99                   cqto
 1997     //   19:   48 f7 f9                idiv   $div
 1998     // 000000000000001c <done>:
 1999     MacroAssembler _masm(&cbuf);
 2000     Label normal;
 2001     Label done;
 2002 
 2003     // mov    $0x8000000000000000,%rdx
 2004     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 2005 
 2006     // cmp    %rdx,%rax
 2007     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 2008 
 2009     // jne    17 <normal>
 2010     __ jccb(Assembler::notEqual, normal);
 2011 
 2012     // xor    %edx,%edx
 2013     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 2014 
 2015     // cmp    $0xffffffffffffffff,$div
 2016     __ cmpq($div$$Register, -1);
 2017 
 2018     // je     1e <done>
 2019     __ jccb(Assembler::equal, done);
 2020 
 2021     // <normal>
 2022     // cqto
 2023     __ bind(normal);
 2024     __ cdqq();
 2025 
 2026     // idivq (note: must be emitted by the user of this rule)
 2027     // <done>
 2028     __ idivq($div$$Register);
 2029     __ bind(done);
 2030   %}
 2031 
 2032   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2033   enc_class OpcSE(immI imm)
 2034   %{
 2035     // Emit primary opcode and set sign-extend bit
 2036     // Check for 8-bit immediate, and set sign extend bit in opcode
 2037     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2038       emit_opcode(cbuf, $primary | 0x02);
 2039     } else {
 2040       // 32-bit immediate
 2041       emit_opcode(cbuf, $primary);
 2042     }
 2043   %}
 2044 
 2045   enc_class OpcSErm(rRegI dst, immI imm)
 2046   %{
 2047     // OpcSEr/m
 2048     int dstenc = $dst$$reg;
 2049     if (dstenc >= 8) {
 2050       emit_opcode(cbuf, Assembler::REX_B);
 2051       dstenc -= 8;
 2052     }
 2053     // Emit primary opcode and set sign-extend bit
 2054     // Check for 8-bit immediate, and set sign extend bit in opcode
 2055     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2056       emit_opcode(cbuf, $primary | 0x02);
 2057     } else {
 2058       // 32-bit immediate
 2059       emit_opcode(cbuf, $primary);
 2060     }
 2061     // Emit r/m byte with secondary opcode, after primary opcode.
 2062     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2063   %}
 2064 
 2065   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2066   %{
 2067     // OpcSEr/m
 2068     int dstenc = $dst$$reg;
 2069     if (dstenc < 8) {
 2070       emit_opcode(cbuf, Assembler::REX_W);
 2071     } else {
 2072       emit_opcode(cbuf, Assembler::REX_WB);
 2073       dstenc -= 8;
 2074     }
 2075     // Emit primary opcode and set sign-extend bit
 2076     // Check for 8-bit immediate, and set sign extend bit in opcode
 2077     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2078       emit_opcode(cbuf, $primary | 0x02);
 2079     } else {
 2080       // 32-bit immediate
 2081       emit_opcode(cbuf, $primary);
 2082     }
 2083     // Emit r/m byte with secondary opcode, after primary opcode.
 2084     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2085   %}
 2086 
 2087   enc_class Con8or32(immI imm)
 2088   %{
 2089     // Check for 8-bit immediate, and set sign extend bit in opcode
 2090     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2091       $$$emit8$imm$$constant;
 2092     } else {
 2093       // 32-bit immediate
 2094       $$$emit32$imm$$constant;
 2095     }
 2096   %}
 2097 
 2098   enc_class opc2_reg(rRegI dst)
 2099   %{
 2100     // BSWAP
 2101     emit_cc(cbuf, $secondary, $dst$$reg);
 2102   %}
 2103 
 2104   enc_class opc3_reg(rRegI dst)
 2105   %{
 2106     // BSWAP
 2107     emit_cc(cbuf, $tertiary, $dst$$reg);
 2108   %}
 2109 
 2110   enc_class reg_opc(rRegI div)
 2111   %{
 2112     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2113     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2114   %}
 2115 
 2116   enc_class enc_cmov(cmpOp cop)
 2117   %{
 2118     // CMOV
 2119     $$$emit8$primary;
 2120     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2121   %}
 2122 
 2123   enc_class enc_PartialSubtypeCheck()
 2124   %{
 2125     Register Rrdi = as_Register(RDI_enc); // result register
 2126     Register Rrax = as_Register(RAX_enc); // super class
 2127     Register Rrcx = as_Register(RCX_enc); // killed
 2128     Register Rrsi = as_Register(RSI_enc); // sub class
 2129     Label miss;
 2130     const bool set_cond_codes = true;
 2131 
 2132     MacroAssembler _masm(&cbuf);
 2133     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2134                                      NULL, &miss,
 2135                                      /*set_cond_codes:*/ true);
 2136     if ($primary) {
 2137       __ xorptr(Rrdi, Rrdi);
 2138     }
 2139     __ bind(miss);
 2140   %}
 2141 
 2142   enc_class clear_avx %{
 2143     debug_only(int off0 = cbuf.insts_size());
 2144     if (generate_vzeroupper(Compile::current())) {
 2145       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2146       // Clear upper bits of YMM registers when current compiled code uses
 2147       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2148       MacroAssembler _masm(&cbuf);
 2149       __ vzeroupper();
 2150     }
 2151     debug_only(int off1 = cbuf.insts_size());
 2152     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2153   %}
 2154 
 2155   enc_class Java_To_Runtime(method meth) %{
 2156     // No relocation needed
 2157     MacroAssembler _masm(&cbuf);
 2158     __ mov64(r10, (int64_t) $meth$$method);
 2159     __ call(r10);
 2160     __ post_call_nop();
 2161   %}
 2162 
 2163   enc_class Java_Static_Call(method meth)
 2164   %{
 2165     // JAVA STATIC CALL
 2166     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2167     // determine who we intended to call.
 2168     MacroAssembler _masm(&cbuf);
 2169     cbuf.set_insts_mark();
 2170 
 2171     if (!_method) {
 2172       $$$emit8$primary;
 2173       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2174                      runtime_call_Relocation::spec(),
 2175                      RELOC_DISP32);
 2176     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 2177       // The NOP here is purely to ensure that eliding a call to
 2178       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 2179       __ addr_nop_5();
 2180       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 2181     } else {
 2182       $$$emit8$primary;
 2183       int method_index = resolved_method_index(cbuf);
 2184       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2185                                                   : static_call_Relocation::spec(method_index);
 2186       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2187                      rspec, RELOC_DISP32);
 2188       address mark = cbuf.insts_mark();
 2189       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 2190         // Calls of the same statically bound method can share
 2191         // a stub to the interpreter.
 2192         cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
 2193       } else {
 2194         // Emit stubs for static call.
 2195         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2196         if (stub == NULL) {
 2197           ciEnv::current()->record_failure("CodeCache is full");
 2198           return;
 2199         }
 2200       }
 2201     }
 2202     _masm.clear_inst_mark();
 2203     __ post_call_nop();
 2204   %}
 2205 
 2206   enc_class Java_Dynamic_Call(method meth) %{
 2207     MacroAssembler _masm(&cbuf);
 2208     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2209     __ post_call_nop();
 2210   %}
 2211 
 2212   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2213   %{
 2214     // SAL, SAR, SHR
 2215     int dstenc = $dst$$reg;
 2216     if (dstenc >= 8) {
 2217       emit_opcode(cbuf, Assembler::REX_B);
 2218       dstenc -= 8;
 2219     }
 2220     $$$emit8$primary;
 2221     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2222     $$$emit8$shift$$constant;
 2223   %}
 2224 
 2225   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2226   %{
 2227     // SAL, SAR, SHR
 2228     int dstenc = $dst$$reg;
 2229     if (dstenc < 8) {
 2230       emit_opcode(cbuf, Assembler::REX_W);
 2231     } else {
 2232       emit_opcode(cbuf, Assembler::REX_WB);
 2233       dstenc -= 8;
 2234     }
 2235     $$$emit8$primary;
 2236     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2237     $$$emit8$shift$$constant;
 2238   %}
 2239 
 2240   enc_class load_immI(rRegI dst, immI src)
 2241   %{
 2242     int dstenc = $dst$$reg;
 2243     if (dstenc >= 8) {
 2244       emit_opcode(cbuf, Assembler::REX_B);
 2245       dstenc -= 8;
 2246     }
 2247     emit_opcode(cbuf, 0xB8 | dstenc);
 2248     $$$emit32$src$$constant;
 2249   %}
 2250 
 2251   enc_class load_immL(rRegL dst, immL src)
 2252   %{
 2253     int dstenc = $dst$$reg;
 2254     if (dstenc < 8) {
 2255       emit_opcode(cbuf, Assembler::REX_W);
 2256     } else {
 2257       emit_opcode(cbuf, Assembler::REX_WB);
 2258       dstenc -= 8;
 2259     }
 2260     emit_opcode(cbuf, 0xB8 | dstenc);
 2261     emit_d64(cbuf, $src$$constant);
 2262   %}
 2263 
 2264   enc_class load_immUL32(rRegL dst, immUL32 src)
 2265   %{
 2266     // same as load_immI, but this time we care about zeroes in the high word
 2267     int dstenc = $dst$$reg;
 2268     if (dstenc >= 8) {
 2269       emit_opcode(cbuf, Assembler::REX_B);
 2270       dstenc -= 8;
 2271     }
 2272     emit_opcode(cbuf, 0xB8 | dstenc);
 2273     $$$emit32$src$$constant;
 2274   %}
 2275 
 2276   enc_class load_immL32(rRegL dst, immL32 src)
 2277   %{
 2278     int dstenc = $dst$$reg;
 2279     if (dstenc < 8) {
 2280       emit_opcode(cbuf, Assembler::REX_W);
 2281     } else {
 2282       emit_opcode(cbuf, Assembler::REX_WB);
 2283       dstenc -= 8;
 2284     }
 2285     emit_opcode(cbuf, 0xC7);
 2286     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2287     $$$emit32$src$$constant;
 2288   %}
 2289 
 2290   enc_class load_immP31(rRegP dst, immP32 src)
 2291   %{
 2292     // same as load_immI, but this time we care about zeroes in the high word
 2293     int dstenc = $dst$$reg;
 2294     if (dstenc >= 8) {
 2295       emit_opcode(cbuf, Assembler::REX_B);
 2296       dstenc -= 8;
 2297     }
 2298     emit_opcode(cbuf, 0xB8 | dstenc);
 2299     $$$emit32$src$$constant;
 2300   %}
 2301 
 2302   enc_class load_immP(rRegP dst, immP src)
 2303   %{
 2304     int dstenc = $dst$$reg;
 2305     if (dstenc < 8) {
 2306       emit_opcode(cbuf, Assembler::REX_W);
 2307     } else {
 2308       emit_opcode(cbuf, Assembler::REX_WB);
 2309       dstenc -= 8;
 2310     }
 2311     emit_opcode(cbuf, 0xB8 | dstenc);
 2312     // This next line should be generated from ADLC
 2313     if ($src->constant_reloc() != relocInfo::none) {
 2314       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2315     } else {
 2316       emit_d64(cbuf, $src$$constant);
 2317     }
 2318   %}
 2319 
 2320   enc_class Con32(immI src)
 2321   %{
 2322     // Output immediate
 2323     $$$emit32$src$$constant;
 2324   %}
 2325 
 2326   enc_class Con32F_as_bits(immF src)
 2327   %{
 2328     // Output Float immediate bits
 2329     jfloat jf = $src$$constant;
 2330     jint jf_as_bits = jint_cast(jf);
 2331     emit_d32(cbuf, jf_as_bits);
 2332   %}
 2333 
 2334   enc_class Con16(immI src)
 2335   %{
 2336     // Output immediate
 2337     $$$emit16$src$$constant;
 2338   %}
 2339 
 2340   // How is this different from Con32??? XXX
 2341   enc_class Con_d32(immI src)
 2342   %{
 2343     emit_d32(cbuf,$src$$constant);
 2344   %}
 2345 
 2346   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2347     // Output immediate memory reference
 2348     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2349     emit_d32(cbuf, 0x00);
 2350   %}
 2351 
 2352   enc_class lock_prefix()
 2353   %{
 2354     emit_opcode(cbuf, 0xF0); // lock
 2355   %}
 2356 
 2357   enc_class REX_mem(memory mem)
 2358   %{
 2359     if ($mem$$base >= 8) {
 2360       if ($mem$$index < 8) {
 2361         emit_opcode(cbuf, Assembler::REX_B);
 2362       } else {
 2363         emit_opcode(cbuf, Assembler::REX_XB);
 2364       }
 2365     } else {
 2366       if ($mem$$index >= 8) {
 2367         emit_opcode(cbuf, Assembler::REX_X);
 2368       }
 2369     }
 2370   %}
 2371 
 2372   enc_class REX_mem_wide(memory mem)
 2373   %{
 2374     if ($mem$$base >= 8) {
 2375       if ($mem$$index < 8) {
 2376         emit_opcode(cbuf, Assembler::REX_WB);
 2377       } else {
 2378         emit_opcode(cbuf, Assembler::REX_WXB);
 2379       }
 2380     } else {
 2381       if ($mem$$index < 8) {
 2382         emit_opcode(cbuf, Assembler::REX_W);
 2383       } else {
 2384         emit_opcode(cbuf, Assembler::REX_WX);
 2385       }
 2386     }
 2387   %}
 2388 
 2389   // for byte regs
 2390   enc_class REX_breg(rRegI reg)
 2391   %{
 2392     if ($reg$$reg >= 4) {
 2393       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2394     }
 2395   %}
 2396 
 2397   // for byte regs
 2398   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2399   %{
 2400     if ($dst$$reg < 8) {
 2401       if ($src$$reg >= 4) {
 2402         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2403       }
 2404     } else {
 2405       if ($src$$reg < 8) {
 2406         emit_opcode(cbuf, Assembler::REX_R);
 2407       } else {
 2408         emit_opcode(cbuf, Assembler::REX_RB);
 2409       }
 2410     }
 2411   %}
 2412 
 2413   // for byte regs
 2414   enc_class REX_breg_mem(rRegI reg, memory mem)
 2415   %{
 2416     if ($reg$$reg < 8) {
 2417       if ($mem$$base < 8) {
 2418         if ($mem$$index >= 8) {
 2419           emit_opcode(cbuf, Assembler::REX_X);
 2420         } else if ($reg$$reg >= 4) {
 2421           emit_opcode(cbuf, Assembler::REX);
 2422         }
 2423       } else {
 2424         if ($mem$$index < 8) {
 2425           emit_opcode(cbuf, Assembler::REX_B);
 2426         } else {
 2427           emit_opcode(cbuf, Assembler::REX_XB);
 2428         }
 2429       }
 2430     } else {
 2431       if ($mem$$base < 8) {
 2432         if ($mem$$index < 8) {
 2433           emit_opcode(cbuf, Assembler::REX_R);
 2434         } else {
 2435           emit_opcode(cbuf, Assembler::REX_RX);
 2436         }
 2437       } else {
 2438         if ($mem$$index < 8) {
 2439           emit_opcode(cbuf, Assembler::REX_RB);
 2440         } else {
 2441           emit_opcode(cbuf, Assembler::REX_RXB);
 2442         }
 2443       }
 2444     }
 2445   %}
 2446 
 2447   enc_class REX_reg(rRegI reg)
 2448   %{
 2449     if ($reg$$reg >= 8) {
 2450       emit_opcode(cbuf, Assembler::REX_B);
 2451     }
 2452   %}
 2453 
 2454   enc_class REX_reg_wide(rRegI reg)
 2455   %{
 2456     if ($reg$$reg < 8) {
 2457       emit_opcode(cbuf, Assembler::REX_W);
 2458     } else {
 2459       emit_opcode(cbuf, Assembler::REX_WB);
 2460     }
 2461   %}
 2462 
 2463   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2464   %{
 2465     if ($dst$$reg < 8) {
 2466       if ($src$$reg >= 8) {
 2467         emit_opcode(cbuf, Assembler::REX_B);
 2468       }
 2469     } else {
 2470       if ($src$$reg < 8) {
 2471         emit_opcode(cbuf, Assembler::REX_R);
 2472       } else {
 2473         emit_opcode(cbuf, Assembler::REX_RB);
 2474       }
 2475     }
 2476   %}
 2477 
 2478   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2479   %{
 2480     if ($dst$$reg < 8) {
 2481       if ($src$$reg < 8) {
 2482         emit_opcode(cbuf, Assembler::REX_W);
 2483       } else {
 2484         emit_opcode(cbuf, Assembler::REX_WB);
 2485       }
 2486     } else {
 2487       if ($src$$reg < 8) {
 2488         emit_opcode(cbuf, Assembler::REX_WR);
 2489       } else {
 2490         emit_opcode(cbuf, Assembler::REX_WRB);
 2491       }
 2492     }
 2493   %}
 2494 
 2495   enc_class REX_reg_mem(rRegI reg, memory mem)
 2496   %{
 2497     if ($reg$$reg < 8) {
 2498       if ($mem$$base < 8) {
 2499         if ($mem$$index >= 8) {
 2500           emit_opcode(cbuf, Assembler::REX_X);
 2501         }
 2502       } else {
 2503         if ($mem$$index < 8) {
 2504           emit_opcode(cbuf, Assembler::REX_B);
 2505         } else {
 2506           emit_opcode(cbuf, Assembler::REX_XB);
 2507         }
 2508       }
 2509     } else {
 2510       if ($mem$$base < 8) {
 2511         if ($mem$$index < 8) {
 2512           emit_opcode(cbuf, Assembler::REX_R);
 2513         } else {
 2514           emit_opcode(cbuf, Assembler::REX_RX);
 2515         }
 2516       } else {
 2517         if ($mem$$index < 8) {
 2518           emit_opcode(cbuf, Assembler::REX_RB);
 2519         } else {
 2520           emit_opcode(cbuf, Assembler::REX_RXB);
 2521         }
 2522       }
 2523     }
 2524   %}
 2525 
 2526   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2527   %{
 2528     if ($reg$$reg < 8) {
 2529       if ($mem$$base < 8) {
 2530         if ($mem$$index < 8) {
 2531           emit_opcode(cbuf, Assembler::REX_W);
 2532         } else {
 2533           emit_opcode(cbuf, Assembler::REX_WX);
 2534         }
 2535       } else {
 2536         if ($mem$$index < 8) {
 2537           emit_opcode(cbuf, Assembler::REX_WB);
 2538         } else {
 2539           emit_opcode(cbuf, Assembler::REX_WXB);
 2540         }
 2541       }
 2542     } else {
 2543       if ($mem$$base < 8) {
 2544         if ($mem$$index < 8) {
 2545           emit_opcode(cbuf, Assembler::REX_WR);
 2546         } else {
 2547           emit_opcode(cbuf, Assembler::REX_WRX);
 2548         }
 2549       } else {
 2550         if ($mem$$index < 8) {
 2551           emit_opcode(cbuf, Assembler::REX_WRB);
 2552         } else {
 2553           emit_opcode(cbuf, Assembler::REX_WRXB);
 2554         }
 2555       }
 2556     }
 2557   %}
 2558 
 2559   enc_class reg_mem(rRegI ereg, memory mem)
 2560   %{
 2561     // High registers handle in encode_RegMem
 2562     int reg = $ereg$$reg;
 2563     int base = $mem$$base;
 2564     int index = $mem$$index;
 2565     int scale = $mem$$scale;
 2566     int disp = $mem$$disp;
 2567     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2568 
 2569     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2570   %}
 2571 
 2572   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2573   %{
 2574     int rm_byte_opcode = $rm_opcode$$constant;
 2575 
 2576     // High registers handle in encode_RegMem
 2577     int base = $mem$$base;
 2578     int index = $mem$$index;
 2579     int scale = $mem$$scale;
 2580     int displace = $mem$$disp;
 2581 
 2582     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2583                                             // working with static
 2584                                             // globals
 2585     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2586                   disp_reloc);
 2587   %}
 2588 
 2589   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2590   %{
 2591     int reg_encoding = $dst$$reg;
 2592     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2593     int index        = 0x04;            // 0x04 indicates no index
 2594     int scale        = 0x00;            // 0x00 indicates no scale
 2595     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2596     relocInfo::relocType disp_reloc = relocInfo::none;
 2597     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2598                   disp_reloc);
 2599   %}
 2600 
 2601   enc_class neg_reg(rRegI dst)
 2602   %{
 2603     int dstenc = $dst$$reg;
 2604     if (dstenc >= 8) {
 2605       emit_opcode(cbuf, Assembler::REX_B);
 2606       dstenc -= 8;
 2607     }
 2608     // NEG $dst
 2609     emit_opcode(cbuf, 0xF7);
 2610     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2611   %}
 2612 
 2613   enc_class neg_reg_wide(rRegI dst)
 2614   %{
 2615     int dstenc = $dst$$reg;
 2616     if (dstenc < 8) {
 2617       emit_opcode(cbuf, Assembler::REX_W);
 2618     } else {
 2619       emit_opcode(cbuf, Assembler::REX_WB);
 2620       dstenc -= 8;
 2621     }
 2622     // NEG $dst
 2623     emit_opcode(cbuf, 0xF7);
 2624     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2625   %}
 2626 
 2627   enc_class setLT_reg(rRegI dst)
 2628   %{
 2629     int dstenc = $dst$$reg;
 2630     if (dstenc >= 8) {
 2631       emit_opcode(cbuf, Assembler::REX_B);
 2632       dstenc -= 8;
 2633     } else if (dstenc >= 4) {
 2634       emit_opcode(cbuf, Assembler::REX);
 2635     }
 2636     // SETLT $dst
 2637     emit_opcode(cbuf, 0x0F);
 2638     emit_opcode(cbuf, 0x9C);
 2639     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2640   %}
 2641 
 2642   enc_class setNZ_reg(rRegI dst)
 2643   %{
 2644     int dstenc = $dst$$reg;
 2645     if (dstenc >= 8) {
 2646       emit_opcode(cbuf, Assembler::REX_B);
 2647       dstenc -= 8;
 2648     } else if (dstenc >= 4) {
 2649       emit_opcode(cbuf, Assembler::REX);
 2650     }
 2651     // SETNZ $dst
 2652     emit_opcode(cbuf, 0x0F);
 2653     emit_opcode(cbuf, 0x95);
 2654     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2655   %}
 2656 
 2657 
 2658   // Compare the lonogs and set -1, 0, or 1 into dst
 2659   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2660   %{
 2661     int src1enc = $src1$$reg;
 2662     int src2enc = $src2$$reg;
 2663     int dstenc = $dst$$reg;
 2664 
 2665     // cmpq $src1, $src2
 2666     if (src1enc < 8) {
 2667       if (src2enc < 8) {
 2668         emit_opcode(cbuf, Assembler::REX_W);
 2669       } else {
 2670         emit_opcode(cbuf, Assembler::REX_WB);
 2671       }
 2672     } else {
 2673       if (src2enc < 8) {
 2674         emit_opcode(cbuf, Assembler::REX_WR);
 2675       } else {
 2676         emit_opcode(cbuf, Assembler::REX_WRB);
 2677       }
 2678     }
 2679     emit_opcode(cbuf, 0x3B);
 2680     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2681 
 2682     // movl $dst, -1
 2683     if (dstenc >= 8) {
 2684       emit_opcode(cbuf, Assembler::REX_B);
 2685     }
 2686     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2687     emit_d32(cbuf, -1);
 2688 
 2689     // jl,s done
 2690     emit_opcode(cbuf, 0x7C);
 2691     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2692 
 2693     // setne $dst
 2694     if (dstenc >= 4) {
 2695       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2696     }
 2697     emit_opcode(cbuf, 0x0F);
 2698     emit_opcode(cbuf, 0x95);
 2699     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2700 
 2701     // movzbl $dst, $dst
 2702     if (dstenc >= 4) {
 2703       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2704     }
 2705     emit_opcode(cbuf, 0x0F);
 2706     emit_opcode(cbuf, 0xB6);
 2707     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2708   %}
 2709 
 2710   enc_class Push_ResultXD(regD dst) %{
 2711     MacroAssembler _masm(&cbuf);
 2712     __ fstp_d(Address(rsp, 0));
 2713     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2714     __ addptr(rsp, 8);
 2715   %}
 2716 
 2717   enc_class Push_SrcXD(regD src) %{
 2718     MacroAssembler _masm(&cbuf);
 2719     __ subptr(rsp, 8);
 2720     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2721     __ fld_d(Address(rsp, 0));
 2722   %}
 2723 
 2724 
 2725   enc_class enc_rethrow()
 2726   %{
 2727     cbuf.set_insts_mark();
 2728     emit_opcode(cbuf, 0xE9); // jmp entry
 2729     emit_d32_reloc(cbuf,
 2730                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2731                    runtime_call_Relocation::spec(),
 2732                    RELOC_DISP32);
 2733   %}
 2734 
 2735 %}
 2736 
 2737 
 2738 
 2739 //----------FRAME--------------------------------------------------------------
 2740 // Definition of frame structure and management information.
 2741 //
 2742 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2743 //                             |   (to get allocators register number
 2744 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2745 //  r   CALLER     |        |
 2746 //  o     |        +--------+      pad to even-align allocators stack-slot
 2747 //  w     V        |  pad0  |        numbers; owned by CALLER
 2748 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2749 //  h     ^        |   in   |  5
 2750 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2751 //  |     |        |        |  3
 2752 //  |     |        +--------+
 2753 //  V     |        | old out|      Empty on Intel, window on Sparc
 2754 //        |    old |preserve|      Must be even aligned.
 2755 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2756 //        |        |   in   |  3   area for Intel ret address
 2757 //     Owned by    |preserve|      Empty on Sparc.
 2758 //       SELF      +--------+
 2759 //        |        |  pad2  |  2   pad to align old SP
 2760 //        |        +--------+  1
 2761 //        |        | locks  |  0
 2762 //        |        +--------+----> OptoReg::stack0(), even aligned
 2763 //        |        |  pad1  | 11   pad to align new SP
 2764 //        |        +--------+
 2765 //        |        |        | 10
 2766 //        |        | spills |  9   spills
 2767 //        V        |        |  8   (pad0 slot for callee)
 2768 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2769 //        ^        |  out   |  7
 2770 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2771 //     Owned by    +--------+
 2772 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2773 //        |    new |preserve|      Must be even-aligned.
 2774 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2775 //        |        |        |
 2776 //
 2777 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2778 //         known from SELF's arguments and the Java calling convention.
 2779 //         Region 6-7 is determined per call site.
 2780 // Note 2: If the calling convention leaves holes in the incoming argument
 2781 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2782 //         are owned by the CALLEE.  Holes should not be necessary in the
 2783 //         incoming area, as the Java calling convention is completely under
 2784 //         the control of the AD file.  Doubles can be sorted and packed to
 2785 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2786 //         varargs C calling conventions.
 2787 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2788 //         even aligned with pad0 as needed.
 2789 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2790 //         region 6-11 is even aligned; it may be padded out more so that
 2791 //         the region from SP to FP meets the minimum stack alignment.
 2792 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2793 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2794 //         SP meets the minimum alignment.
 2795 
 2796 frame
 2797 %{
 2798   // These three registers define part of the calling convention
 2799   // between compiled code and the interpreter.
 2800   inline_cache_reg(RAX);                // Inline Cache Register
 2801 
 2802   // Optional: name the operand used by cisc-spilling to access
 2803   // [stack_pointer + offset]
 2804   cisc_spilling_operand_name(indOffset32);
 2805 
 2806   // Number of stack slots consumed by locking an object
 2807   sync_stack_slots(2);
 2808 
 2809   // Compiled code's Frame Pointer
 2810   frame_pointer(RSP);
 2811 
 2812   // Interpreter stores its frame pointer in a register which is
 2813   // stored to the stack by I2CAdaptors.
 2814   // I2CAdaptors convert from interpreted java to compiled java.
 2815   interpreter_frame_pointer(RBP);
 2816 
 2817   // Stack alignment requirement
 2818   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2819 
 2820   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2821   // for calls to C.  Supports the var-args backing area for register parms.
 2822   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2823 
 2824   // The after-PROLOG location of the return address.  Location of
 2825   // return address specifies a type (REG or STACK) and a number
 2826   // representing the register number (i.e. - use a register name) or
 2827   // stack slot.
 2828   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2829   // Otherwise, it is above the locks and verification slot and alignment word
 2830   return_addr(STACK - 2 +
 2831               align_up((Compile::current()->in_preserve_stack_slots() +
 2832                         Compile::current()->fixed_slots()),
 2833                        stack_alignment_in_slots()));
 2834 
 2835   // Location of compiled Java return values.  Same as C for now.
 2836   return_value
 2837   %{
 2838     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2839            "only return normal values");
 2840 
 2841     static const int lo[Op_RegL + 1] = {
 2842       0,
 2843       0,
 2844       RAX_num,  // Op_RegN
 2845       RAX_num,  // Op_RegI
 2846       RAX_num,  // Op_RegP
 2847       XMM0_num, // Op_RegF
 2848       XMM0_num, // Op_RegD
 2849       RAX_num   // Op_RegL
 2850     };
 2851     static const int hi[Op_RegL + 1] = {
 2852       0,
 2853       0,
 2854       OptoReg::Bad, // Op_RegN
 2855       OptoReg::Bad, // Op_RegI
 2856       RAX_H_num,    // Op_RegP
 2857       OptoReg::Bad, // Op_RegF
 2858       XMM0b_num,    // Op_RegD
 2859       RAX_H_num     // Op_RegL
 2860     };
 2861     // Excluded flags and vector registers.
 2862     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2863     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2864   %}
 2865 %}
 2866 
 2867 //----------ATTRIBUTES---------------------------------------------------------
 2868 //----------Operand Attributes-------------------------------------------------
 2869 op_attrib op_cost(0);        // Required cost attribute
 2870 
 2871 //----------Instruction Attributes---------------------------------------------
 2872 ins_attrib ins_cost(100);       // Required cost attribute
 2873 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2874 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2875                                 // a non-matching short branch variant
 2876                                 // of some long branch?
 2877 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2878                                 // be a power of 2) specifies the
 2879                                 // alignment that some part of the
 2880                                 // instruction (not necessarily the
 2881                                 // start) requires.  If > 1, a
 2882                                 // compute_padding() function must be
 2883                                 // provided for the instruction
 2884 
 2885 //----------OPERANDS-----------------------------------------------------------
 2886 // Operand definitions must precede instruction definitions for correct parsing
 2887 // in the ADLC because operands constitute user defined types which are used in
 2888 // instruction definitions.
 2889 
 2890 //----------Simple Operands----------------------------------------------------
 2891 // Immediate Operands
 2892 // Integer Immediate
 2893 operand immI()
 2894 %{
 2895   match(ConI);
 2896 
 2897   op_cost(10);
 2898   format %{ %}
 2899   interface(CONST_INTER);
 2900 %}
 2901 
 2902 // Constant for test vs zero
 2903 operand immI_0()
 2904 %{
 2905   predicate(n->get_int() == 0);
 2906   match(ConI);
 2907 
 2908   op_cost(0);
 2909   format %{ %}
 2910   interface(CONST_INTER);
 2911 %}
 2912 
 2913 // Constant for increment
 2914 operand immI_1()
 2915 %{
 2916   predicate(n->get_int() == 1);
 2917   match(ConI);
 2918 
 2919   op_cost(0);
 2920   format %{ %}
 2921   interface(CONST_INTER);
 2922 %}
 2923 
 2924 // Constant for decrement
 2925 operand immI_M1()
 2926 %{
 2927   predicate(n->get_int() == -1);
 2928   match(ConI);
 2929 
 2930   op_cost(0);
 2931   format %{ %}
 2932   interface(CONST_INTER);
 2933 %}
 2934 
 2935 operand immI_2()
 2936 %{
 2937   predicate(n->get_int() == 2);
 2938   match(ConI);
 2939 
 2940   op_cost(0);
 2941   format %{ %}
 2942   interface(CONST_INTER);
 2943 %}
 2944 
 2945 operand immI_4()
 2946 %{
 2947   predicate(n->get_int() == 4);
 2948   match(ConI);
 2949 
 2950   op_cost(0);
 2951   format %{ %}
 2952   interface(CONST_INTER);
 2953 %}
 2954 
 2955 operand immI_8()
 2956 %{
 2957   predicate(n->get_int() == 8);
 2958   match(ConI);
 2959 
 2960   op_cost(0);
 2961   format %{ %}
 2962   interface(CONST_INTER);
 2963 %}
 2964 
 2965 // Valid scale values for addressing modes
 2966 operand immI2()
 2967 %{
 2968   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2969   match(ConI);
 2970 
 2971   format %{ %}
 2972   interface(CONST_INTER);
 2973 %}
 2974 
 2975 operand immU7()
 2976 %{
 2977   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2978   match(ConI);
 2979 
 2980   op_cost(5);
 2981   format %{ %}
 2982   interface(CONST_INTER);
 2983 %}
 2984 
 2985 operand immI8()
 2986 %{
 2987   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2988   match(ConI);
 2989 
 2990   op_cost(5);
 2991   format %{ %}
 2992   interface(CONST_INTER);
 2993 %}
 2994 
 2995 operand immU8()
 2996 %{
 2997   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2998   match(ConI);
 2999 
 3000   op_cost(5);
 3001   format %{ %}
 3002   interface(CONST_INTER);
 3003 %}
 3004 
 3005 operand immI16()
 3006 %{
 3007   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 3008   match(ConI);
 3009 
 3010   op_cost(10);
 3011   format %{ %}
 3012   interface(CONST_INTER);
 3013 %}
 3014 
 3015 // Int Immediate non-negative
 3016 operand immU31()
 3017 %{
 3018   predicate(n->get_int() >= 0);
 3019   match(ConI);
 3020 
 3021   op_cost(0);
 3022   format %{ %}
 3023   interface(CONST_INTER);
 3024 %}
 3025 
 3026 // Constant for long shifts
 3027 operand immI_32()
 3028 %{
 3029   predicate( n->get_int() == 32 );
 3030   match(ConI);
 3031 
 3032   op_cost(0);
 3033   format %{ %}
 3034   interface(CONST_INTER);
 3035 %}
 3036 
 3037 // Constant for long shifts
 3038 operand immI_64()
 3039 %{
 3040   predicate( n->get_int() == 64 );
 3041   match(ConI);
 3042 
 3043   op_cost(0);
 3044   format %{ %}
 3045   interface(CONST_INTER);
 3046 %}
 3047 
 3048 // Pointer Immediate
 3049 operand immP()
 3050 %{
 3051   match(ConP);
 3052 
 3053   op_cost(10);
 3054   format %{ %}
 3055   interface(CONST_INTER);
 3056 %}
 3057 
 3058 // NULL Pointer Immediate
 3059 operand immP0()
 3060 %{
 3061   predicate(n->get_ptr() == 0);
 3062   match(ConP);
 3063 
 3064   op_cost(5);
 3065   format %{ %}
 3066   interface(CONST_INTER);
 3067 %}
 3068 
 3069 // Pointer Immediate
 3070 operand immN() %{
 3071   match(ConN);
 3072 
 3073   op_cost(10);
 3074   format %{ %}
 3075   interface(CONST_INTER);
 3076 %}
 3077 
 3078 operand immNKlass() %{
 3079   match(ConNKlass);
 3080 
 3081   op_cost(10);
 3082   format %{ %}
 3083   interface(CONST_INTER);
 3084 %}
 3085 
 3086 // NULL Pointer Immediate
 3087 operand immN0() %{
 3088   predicate(n->get_narrowcon() == 0);
 3089   match(ConN);
 3090 
 3091   op_cost(5);
 3092   format %{ %}
 3093   interface(CONST_INTER);
 3094 %}
 3095 
 3096 operand immP31()
 3097 %{
 3098   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3099             && (n->get_ptr() >> 31) == 0);
 3100   match(ConP);
 3101 
 3102   op_cost(5);
 3103   format %{ %}
 3104   interface(CONST_INTER);
 3105 %}
 3106 
 3107 
 3108 // Long Immediate
 3109 operand immL()
 3110 %{
 3111   match(ConL);
 3112 
 3113   op_cost(20);
 3114   format %{ %}
 3115   interface(CONST_INTER);
 3116 %}
 3117 
 3118 // Long Immediate 8-bit
 3119 operand immL8()
 3120 %{
 3121   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3122   match(ConL);
 3123 
 3124   op_cost(5);
 3125   format %{ %}
 3126   interface(CONST_INTER);
 3127 %}
 3128 
 3129 // Long Immediate 32-bit unsigned
 3130 operand immUL32()
 3131 %{
 3132   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3133   match(ConL);
 3134 
 3135   op_cost(10);
 3136   format %{ %}
 3137   interface(CONST_INTER);
 3138 %}
 3139 
 3140 // Long Immediate 32-bit signed
 3141 operand immL32()
 3142 %{
 3143   predicate(n->get_long() == (int) (n->get_long()));
 3144   match(ConL);
 3145 
 3146   op_cost(15);
 3147   format %{ %}
 3148   interface(CONST_INTER);
 3149 %}
 3150 
 3151 operand immL_Pow2()
 3152 %{
 3153   predicate(is_power_of_2((julong)n->get_long()));
 3154   match(ConL);
 3155 
 3156   op_cost(15);
 3157   format %{ %}
 3158   interface(CONST_INTER);
 3159 %}
 3160 
 3161 operand immL_NotPow2()
 3162 %{
 3163   predicate(is_power_of_2((julong)~n->get_long()));
 3164   match(ConL);
 3165 
 3166   op_cost(15);
 3167   format %{ %}
 3168   interface(CONST_INTER);
 3169 %}
 3170 
 3171 // Long Immediate zero
 3172 operand immL0()
 3173 %{
 3174   predicate(n->get_long() == 0L);
 3175   match(ConL);
 3176 
 3177   op_cost(10);
 3178   format %{ %}
 3179   interface(CONST_INTER);
 3180 %}
 3181 
 3182 // Constant for increment
 3183 operand immL1()
 3184 %{
 3185   predicate(n->get_long() == 1);
 3186   match(ConL);
 3187 
 3188   format %{ %}
 3189   interface(CONST_INTER);
 3190 %}
 3191 
 3192 // Constant for decrement
 3193 operand immL_M1()
 3194 %{
 3195   predicate(n->get_long() == -1);
 3196   match(ConL);
 3197 
 3198   format %{ %}
 3199   interface(CONST_INTER);
 3200 %}
 3201 
 3202 // Long Immediate: the value 10
 3203 operand immL10()
 3204 %{
 3205   predicate(n->get_long() == 10);
 3206   match(ConL);
 3207 
 3208   format %{ %}
 3209   interface(CONST_INTER);
 3210 %}
 3211 
 3212 // Long immediate from 0 to 127.
 3213 // Used for a shorter form of long mul by 10.
 3214 operand immL_127()
 3215 %{
 3216   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3217   match(ConL);
 3218 
 3219   op_cost(10);
 3220   format %{ %}
 3221   interface(CONST_INTER);
 3222 %}
 3223 
 3224 // Long Immediate: low 32-bit mask
 3225 operand immL_32bits()
 3226 %{
 3227   predicate(n->get_long() == 0xFFFFFFFFL);
 3228   match(ConL);
 3229   op_cost(20);
 3230 
 3231   format %{ %}
 3232   interface(CONST_INTER);
 3233 %}
 3234 
 3235 // Int Immediate: 2^n-1, positive
 3236 operand immI_Pow2M1()
 3237 %{
 3238   predicate((n->get_int() > 0)
 3239             && is_power_of_2(n->get_int() + 1));
 3240   match(ConI);
 3241 
 3242   op_cost(20);
 3243   format %{ %}
 3244   interface(CONST_INTER);
 3245 %}
 3246 
 3247 // Float Immediate zero
 3248 operand immF0()
 3249 %{
 3250   predicate(jint_cast(n->getf()) == 0);
 3251   match(ConF);
 3252 
 3253   op_cost(5);
 3254   format %{ %}
 3255   interface(CONST_INTER);
 3256 %}
 3257 
 3258 // Float Immediate
 3259 operand immF()
 3260 %{
 3261   match(ConF);
 3262 
 3263   op_cost(15);
 3264   format %{ %}
 3265   interface(CONST_INTER);
 3266 %}
 3267 
 3268 // Double Immediate zero
 3269 operand immD0()
 3270 %{
 3271   predicate(jlong_cast(n->getd()) == 0);
 3272   match(ConD);
 3273 
 3274   op_cost(5);
 3275   format %{ %}
 3276   interface(CONST_INTER);
 3277 %}
 3278 
 3279 // Double Immediate
 3280 operand immD()
 3281 %{
 3282   match(ConD);
 3283 
 3284   op_cost(15);
 3285   format %{ %}
 3286   interface(CONST_INTER);
 3287 %}
 3288 
 3289 // Immediates for special shifts (sign extend)
 3290 
 3291 // Constants for increment
 3292 operand immI_16()
 3293 %{
 3294   predicate(n->get_int() == 16);
 3295   match(ConI);
 3296 
 3297   format %{ %}
 3298   interface(CONST_INTER);
 3299 %}
 3300 
 3301 operand immI_24()
 3302 %{
 3303   predicate(n->get_int() == 24);
 3304   match(ConI);
 3305 
 3306   format %{ %}
 3307   interface(CONST_INTER);
 3308 %}
 3309 
 3310 // Constant for byte-wide masking
 3311 operand immI_255()
 3312 %{
 3313   predicate(n->get_int() == 255);
 3314   match(ConI);
 3315 
 3316   format %{ %}
 3317   interface(CONST_INTER);
 3318 %}
 3319 
 3320 // Constant for short-wide masking
 3321 operand immI_65535()
 3322 %{
 3323   predicate(n->get_int() == 65535);
 3324   match(ConI);
 3325 
 3326   format %{ %}
 3327   interface(CONST_INTER);
 3328 %}
 3329 
 3330 // Constant for byte-wide masking
 3331 operand immL_255()
 3332 %{
 3333   predicate(n->get_long() == 255);
 3334   match(ConL);
 3335 
 3336   format %{ %}
 3337   interface(CONST_INTER);
 3338 %}
 3339 
 3340 // Constant for short-wide masking
 3341 operand immL_65535()
 3342 %{
 3343   predicate(n->get_long() == 65535);
 3344   match(ConL);
 3345 
 3346   format %{ %}
 3347   interface(CONST_INTER);
 3348 %}
 3349 
 3350 operand kReg()
 3351 %{
 3352   constraint(ALLOC_IN_RC(vectmask_reg));
 3353   match(RegVectMask);
 3354   format %{%}
 3355   interface(REG_INTER);
 3356 %}
 3357 
 3358 operand kReg_K1()
 3359 %{
 3360   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3361   match(RegVectMask);
 3362   format %{%}
 3363   interface(REG_INTER);
 3364 %}
 3365 
 3366 operand kReg_K2()
 3367 %{
 3368   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3369   match(RegVectMask);
 3370   format %{%}
 3371   interface(REG_INTER);
 3372 %}
 3373 
 3374 // Special Registers
 3375 operand kReg_K3()
 3376 %{
 3377   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3378   match(RegVectMask);
 3379   format %{%}
 3380   interface(REG_INTER);
 3381 %}
 3382 
 3383 operand kReg_K4()
 3384 %{
 3385   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3386   match(RegVectMask);
 3387   format %{%}
 3388   interface(REG_INTER);
 3389 %}
 3390 
 3391 operand kReg_K5()
 3392 %{
 3393   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3394   match(RegVectMask);
 3395   format %{%}
 3396   interface(REG_INTER);
 3397 %}
 3398 
 3399 operand kReg_K6()
 3400 %{
 3401   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3402   match(RegVectMask);
 3403   format %{%}
 3404   interface(REG_INTER);
 3405 %}
 3406 
 3407 // Special Registers
 3408 operand kReg_K7()
 3409 %{
 3410   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3411   match(RegVectMask);
 3412   format %{%}
 3413   interface(REG_INTER);
 3414 %}
 3415 
 3416 // Register Operands
 3417 // Integer Register
 3418 operand rRegI()
 3419 %{
 3420   constraint(ALLOC_IN_RC(int_reg));
 3421   match(RegI);
 3422 
 3423   match(rax_RegI);
 3424   match(rbx_RegI);
 3425   match(rcx_RegI);
 3426   match(rdx_RegI);
 3427   match(rdi_RegI);
 3428 
 3429   format %{ %}
 3430   interface(REG_INTER);
 3431 %}
 3432 
 3433 // Special Registers
 3434 operand rax_RegI()
 3435 %{
 3436   constraint(ALLOC_IN_RC(int_rax_reg));
 3437   match(RegI);
 3438   match(rRegI);
 3439 
 3440   format %{ "RAX" %}
 3441   interface(REG_INTER);
 3442 %}
 3443 
 3444 // Special Registers
 3445 operand rbx_RegI()
 3446 %{
 3447   constraint(ALLOC_IN_RC(int_rbx_reg));
 3448   match(RegI);
 3449   match(rRegI);
 3450 
 3451   format %{ "RBX" %}
 3452   interface(REG_INTER);
 3453 %}
 3454 
 3455 operand rcx_RegI()
 3456 %{
 3457   constraint(ALLOC_IN_RC(int_rcx_reg));
 3458   match(RegI);
 3459   match(rRegI);
 3460 
 3461   format %{ "RCX" %}
 3462   interface(REG_INTER);
 3463 %}
 3464 
 3465 operand rdx_RegI()
 3466 %{
 3467   constraint(ALLOC_IN_RC(int_rdx_reg));
 3468   match(RegI);
 3469   match(rRegI);
 3470 
 3471   format %{ "RDX" %}
 3472   interface(REG_INTER);
 3473 %}
 3474 
 3475 operand rdi_RegI()
 3476 %{
 3477   constraint(ALLOC_IN_RC(int_rdi_reg));
 3478   match(RegI);
 3479   match(rRegI);
 3480 
 3481   format %{ "RDI" %}
 3482   interface(REG_INTER);
 3483 %}
 3484 
 3485 operand no_rax_rdx_RegI()
 3486 %{
 3487   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3488   match(RegI);
 3489   match(rbx_RegI);
 3490   match(rcx_RegI);
 3491   match(rdi_RegI);
 3492 
 3493   format %{ %}
 3494   interface(REG_INTER);
 3495 %}
 3496 
 3497 operand no_rbp_r13_RegI()
 3498 %{
 3499   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3500   match(RegI);
 3501   match(rRegI);
 3502   match(rax_RegI);
 3503   match(rbx_RegI);
 3504   match(rcx_RegI);
 3505   match(rdx_RegI);
 3506   match(rdi_RegI);
 3507 
 3508   format %{ %}
 3509   interface(REG_INTER);
 3510 %}
 3511 
 3512 // Pointer Register
 3513 operand any_RegP()
 3514 %{
 3515   constraint(ALLOC_IN_RC(any_reg));
 3516   match(RegP);
 3517   match(rax_RegP);
 3518   match(rbx_RegP);
 3519   match(rdi_RegP);
 3520   match(rsi_RegP);
 3521   match(rbp_RegP);
 3522   match(r15_RegP);
 3523   match(rRegP);
 3524 
 3525   format %{ %}
 3526   interface(REG_INTER);
 3527 %}
 3528 
 3529 operand rRegP()
 3530 %{
 3531   constraint(ALLOC_IN_RC(ptr_reg));
 3532   match(RegP);
 3533   match(rax_RegP);
 3534   match(rbx_RegP);
 3535   match(rdi_RegP);
 3536   match(rsi_RegP);
 3537   match(rbp_RegP);  // See Q&A below about
 3538   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3539 
 3540   format %{ %}
 3541   interface(REG_INTER);
 3542 %}
 3543 
 3544 operand rRegN() %{
 3545   constraint(ALLOC_IN_RC(int_reg));
 3546   match(RegN);
 3547 
 3548   format %{ %}
 3549   interface(REG_INTER);
 3550 %}
 3551 
 3552 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3553 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3554 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3555 // The output of an instruction is controlled by the allocator, which respects
 3556 // register class masks, not match rules.  Unless an instruction mentions
 3557 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3558 // by the allocator as an input.
 3559 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3560 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3561 // result, RBP is not included in the output of the instruction either.
 3562 
 3563 operand no_rax_RegP()
 3564 %{
 3565   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3566   match(RegP);
 3567   match(rbx_RegP);
 3568   match(rsi_RegP);
 3569   match(rdi_RegP);
 3570 
 3571   format %{ %}
 3572   interface(REG_INTER);
 3573 %}
 3574 
 3575 // This operand is not allowed to use RBP even if
 3576 // RBP is not used to hold the frame pointer.
 3577 operand no_rbp_RegP()
 3578 %{
 3579   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3580   match(RegP);
 3581   match(rbx_RegP);
 3582   match(rsi_RegP);
 3583   match(rdi_RegP);
 3584 
 3585   format %{ %}
 3586   interface(REG_INTER);
 3587 %}
 3588 
 3589 operand no_rax_rbx_RegP()
 3590 %{
 3591   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3592   match(RegP);
 3593   match(rsi_RegP);
 3594   match(rdi_RegP);
 3595 
 3596   format %{ %}
 3597   interface(REG_INTER);
 3598 %}
 3599 
 3600 // Special Registers
 3601 // Return a pointer value
 3602 operand rax_RegP()
 3603 %{
 3604   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3605   match(RegP);
 3606   match(rRegP);
 3607 
 3608   format %{ %}
 3609   interface(REG_INTER);
 3610 %}
 3611 
 3612 // Special Registers
 3613 // Return a compressed pointer value
 3614 operand rax_RegN()
 3615 %{
 3616   constraint(ALLOC_IN_RC(int_rax_reg));
 3617   match(RegN);
 3618   match(rRegN);
 3619 
 3620   format %{ %}
 3621   interface(REG_INTER);
 3622 %}
 3623 
 3624 // Used in AtomicAdd
 3625 operand rbx_RegP()
 3626 %{
 3627   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3628   match(RegP);
 3629   match(rRegP);
 3630 
 3631   format %{ %}
 3632   interface(REG_INTER);
 3633 %}
 3634 
 3635 operand rsi_RegP()
 3636 %{
 3637   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3638   match(RegP);
 3639   match(rRegP);
 3640 
 3641   format %{ %}
 3642   interface(REG_INTER);
 3643 %}
 3644 
 3645 operand rbp_RegP()
 3646 %{
 3647   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3648   match(RegP);
 3649   match(rRegP);
 3650 
 3651   format %{ %}
 3652   interface(REG_INTER);
 3653 %}
 3654 
 3655 // Used in rep stosq
 3656 operand rdi_RegP()
 3657 %{
 3658   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3659   match(RegP);
 3660   match(rRegP);
 3661 
 3662   format %{ %}
 3663   interface(REG_INTER);
 3664 %}
 3665 
 3666 operand r15_RegP()
 3667 %{
 3668   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3669   match(RegP);
 3670   match(rRegP);
 3671 
 3672   format %{ %}
 3673   interface(REG_INTER);
 3674 %}
 3675 
 3676 operand rRegL()
 3677 %{
 3678   constraint(ALLOC_IN_RC(long_reg));
 3679   match(RegL);
 3680   match(rax_RegL);
 3681   match(rdx_RegL);
 3682 
 3683   format %{ %}
 3684   interface(REG_INTER);
 3685 %}
 3686 
 3687 // Special Registers
 3688 operand no_rax_rdx_RegL()
 3689 %{
 3690   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3691   match(RegL);
 3692   match(rRegL);
 3693 
 3694   format %{ %}
 3695   interface(REG_INTER);
 3696 %}
 3697 
 3698 operand rax_RegL()
 3699 %{
 3700   constraint(ALLOC_IN_RC(long_rax_reg));
 3701   match(RegL);
 3702   match(rRegL);
 3703 
 3704   format %{ "RAX" %}
 3705   interface(REG_INTER);
 3706 %}
 3707 
 3708 operand rcx_RegL()
 3709 %{
 3710   constraint(ALLOC_IN_RC(long_rcx_reg));
 3711   match(RegL);
 3712   match(rRegL);
 3713 
 3714   format %{ %}
 3715   interface(REG_INTER);
 3716 %}
 3717 
 3718 operand rdx_RegL()
 3719 %{
 3720   constraint(ALLOC_IN_RC(long_rdx_reg));
 3721   match(RegL);
 3722   match(rRegL);
 3723 
 3724   format %{ %}
 3725   interface(REG_INTER);
 3726 %}
 3727 
 3728 operand no_rbp_r13_RegL()
 3729 %{
 3730   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3731   match(RegL);
 3732   match(rRegL);
 3733   match(rax_RegL);
 3734   match(rcx_RegL);
 3735   match(rdx_RegL);
 3736 
 3737   format %{ %}
 3738   interface(REG_INTER);
 3739 %}
 3740 
 3741 // Flags register, used as output of compare instructions
 3742 operand rFlagsReg()
 3743 %{
 3744   constraint(ALLOC_IN_RC(int_flags));
 3745   match(RegFlags);
 3746 
 3747   format %{ "RFLAGS" %}
 3748   interface(REG_INTER);
 3749 %}
 3750 
 3751 // Flags register, used as output of FLOATING POINT compare instructions
 3752 operand rFlagsRegU()
 3753 %{
 3754   constraint(ALLOC_IN_RC(int_flags));
 3755   match(RegFlags);
 3756 
 3757   format %{ "RFLAGS_U" %}
 3758   interface(REG_INTER);
 3759 %}
 3760 
 3761 operand rFlagsRegUCF() %{
 3762   constraint(ALLOC_IN_RC(int_flags));
 3763   match(RegFlags);
 3764   predicate(false);
 3765 
 3766   format %{ "RFLAGS_U_CF" %}
 3767   interface(REG_INTER);
 3768 %}
 3769 
 3770 // Float register operands
 3771 operand regF() %{
 3772    constraint(ALLOC_IN_RC(float_reg));
 3773    match(RegF);
 3774 
 3775    format %{ %}
 3776    interface(REG_INTER);
 3777 %}
 3778 
 3779 // Float register operands
 3780 operand legRegF() %{
 3781    constraint(ALLOC_IN_RC(float_reg_legacy));
 3782    match(RegF);
 3783 
 3784    format %{ %}
 3785    interface(REG_INTER);
 3786 %}
 3787 
 3788 // Float register operands
 3789 operand vlRegF() %{
 3790    constraint(ALLOC_IN_RC(float_reg_vl));
 3791    match(RegF);
 3792 
 3793    format %{ %}
 3794    interface(REG_INTER);
 3795 %}
 3796 
 3797 // Double register operands
 3798 operand regD() %{
 3799    constraint(ALLOC_IN_RC(double_reg));
 3800    match(RegD);
 3801 
 3802    format %{ %}
 3803    interface(REG_INTER);
 3804 %}
 3805 
 3806 // Double register operands
 3807 operand legRegD() %{
 3808    constraint(ALLOC_IN_RC(double_reg_legacy));
 3809    match(RegD);
 3810 
 3811    format %{ %}
 3812    interface(REG_INTER);
 3813 %}
 3814 
 3815 // Double register operands
 3816 operand vlRegD() %{
 3817    constraint(ALLOC_IN_RC(double_reg_vl));
 3818    match(RegD);
 3819 
 3820    format %{ %}
 3821    interface(REG_INTER);
 3822 %}
 3823 
 3824 //----------Memory Operands----------------------------------------------------
 3825 // Direct Memory Operand
 3826 // operand direct(immP addr)
 3827 // %{
 3828 //   match(addr);
 3829 
 3830 //   format %{ "[$addr]" %}
 3831 //   interface(MEMORY_INTER) %{
 3832 //     base(0xFFFFFFFF);
 3833 //     index(0x4);
 3834 //     scale(0x0);
 3835 //     disp($addr);
 3836 //   %}
 3837 // %}
 3838 
 3839 // Indirect Memory Operand
 3840 operand indirect(any_RegP reg)
 3841 %{
 3842   constraint(ALLOC_IN_RC(ptr_reg));
 3843   match(reg);
 3844 
 3845   format %{ "[$reg]" %}
 3846   interface(MEMORY_INTER) %{
 3847     base($reg);
 3848     index(0x4);
 3849     scale(0x0);
 3850     disp(0x0);
 3851   %}
 3852 %}
 3853 
 3854 // Indirect Memory Plus Short Offset Operand
 3855 operand indOffset8(any_RegP reg, immL8 off)
 3856 %{
 3857   constraint(ALLOC_IN_RC(ptr_reg));
 3858   match(AddP reg off);
 3859 
 3860   format %{ "[$reg + $off (8-bit)]" %}
 3861   interface(MEMORY_INTER) %{
 3862     base($reg);
 3863     index(0x4);
 3864     scale(0x0);
 3865     disp($off);
 3866   %}
 3867 %}
 3868 
 3869 // Indirect Memory Plus Long Offset Operand
 3870 operand indOffset32(any_RegP reg, immL32 off)
 3871 %{
 3872   constraint(ALLOC_IN_RC(ptr_reg));
 3873   match(AddP reg off);
 3874 
 3875   format %{ "[$reg + $off (32-bit)]" %}
 3876   interface(MEMORY_INTER) %{
 3877     base($reg);
 3878     index(0x4);
 3879     scale(0x0);
 3880     disp($off);
 3881   %}
 3882 %}
 3883 
 3884 // Indirect Memory Plus Index Register Plus Offset Operand
 3885 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3886 %{
 3887   constraint(ALLOC_IN_RC(ptr_reg));
 3888   match(AddP (AddP reg lreg) off);
 3889 
 3890   op_cost(10);
 3891   format %{"[$reg + $off + $lreg]" %}
 3892   interface(MEMORY_INTER) %{
 3893     base($reg);
 3894     index($lreg);
 3895     scale(0x0);
 3896     disp($off);
 3897   %}
 3898 %}
 3899 
 3900 // Indirect Memory Plus Index Register Plus Offset Operand
 3901 operand indIndex(any_RegP reg, rRegL lreg)
 3902 %{
 3903   constraint(ALLOC_IN_RC(ptr_reg));
 3904   match(AddP reg lreg);
 3905 
 3906   op_cost(10);
 3907   format %{"[$reg + $lreg]" %}
 3908   interface(MEMORY_INTER) %{
 3909     base($reg);
 3910     index($lreg);
 3911     scale(0x0);
 3912     disp(0x0);
 3913   %}
 3914 %}
 3915 
 3916 // Indirect Memory Times Scale Plus Index Register
 3917 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3918 %{
 3919   constraint(ALLOC_IN_RC(ptr_reg));
 3920   match(AddP reg (LShiftL lreg scale));
 3921 
 3922   op_cost(10);
 3923   format %{"[$reg + $lreg << $scale]" %}
 3924   interface(MEMORY_INTER) %{
 3925     base($reg);
 3926     index($lreg);
 3927     scale($scale);
 3928     disp(0x0);
 3929   %}
 3930 %}
 3931 
 3932 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3933 %{
 3934   constraint(ALLOC_IN_RC(ptr_reg));
 3935   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3936   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3937 
 3938   op_cost(10);
 3939   format %{"[$reg + pos $idx << $scale]" %}
 3940   interface(MEMORY_INTER) %{
 3941     base($reg);
 3942     index($idx);
 3943     scale($scale);
 3944     disp(0x0);
 3945   %}
 3946 %}
 3947 
 3948 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3949 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3950 %{
 3951   constraint(ALLOC_IN_RC(ptr_reg));
 3952   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3953 
 3954   op_cost(10);
 3955   format %{"[$reg + $off + $lreg << $scale]" %}
 3956   interface(MEMORY_INTER) %{
 3957     base($reg);
 3958     index($lreg);
 3959     scale($scale);
 3960     disp($off);
 3961   %}
 3962 %}
 3963 
 3964 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3965 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3966 %{
 3967   constraint(ALLOC_IN_RC(ptr_reg));
 3968   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3969   match(AddP (AddP reg (ConvI2L idx)) off);
 3970 
 3971   op_cost(10);
 3972   format %{"[$reg + $off + $idx]" %}
 3973   interface(MEMORY_INTER) %{
 3974     base($reg);
 3975     index($idx);
 3976     scale(0x0);
 3977     disp($off);
 3978   %}
 3979 %}
 3980 
 3981 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3982 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3983 %{
 3984   constraint(ALLOC_IN_RC(ptr_reg));
 3985   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3986   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3987 
 3988   op_cost(10);
 3989   format %{"[$reg + $off + $idx << $scale]" %}
 3990   interface(MEMORY_INTER) %{
 3991     base($reg);
 3992     index($idx);
 3993     scale($scale);
 3994     disp($off);
 3995   %}
 3996 %}
 3997 
 3998 // Indirect Narrow Oop Plus Offset Operand
 3999 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 4000 // we can't free r12 even with CompressedOops::base() == NULL.
 4001 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 4002   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 4003   constraint(ALLOC_IN_RC(ptr_reg));
 4004   match(AddP (DecodeN reg) off);
 4005 
 4006   op_cost(10);
 4007   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 4008   interface(MEMORY_INTER) %{
 4009     base(0xc); // R12
 4010     index($reg);
 4011     scale(0x3);
 4012     disp($off);
 4013   %}
 4014 %}
 4015 
 4016 // Indirect Memory Operand
 4017 operand indirectNarrow(rRegN reg)
 4018 %{
 4019   predicate(CompressedOops::shift() == 0);
 4020   constraint(ALLOC_IN_RC(ptr_reg));
 4021   match(DecodeN reg);
 4022 
 4023   format %{ "[$reg]" %}
 4024   interface(MEMORY_INTER) %{
 4025     base($reg);
 4026     index(0x4);
 4027     scale(0x0);
 4028     disp(0x0);
 4029   %}
 4030 %}
 4031 
 4032 // Indirect Memory Plus Short Offset Operand
 4033 operand indOffset8Narrow(rRegN reg, immL8 off)
 4034 %{
 4035   predicate(CompressedOops::shift() == 0);
 4036   constraint(ALLOC_IN_RC(ptr_reg));
 4037   match(AddP (DecodeN reg) off);
 4038 
 4039   format %{ "[$reg + $off (8-bit)]" %}
 4040   interface(MEMORY_INTER) %{
 4041     base($reg);
 4042     index(0x4);
 4043     scale(0x0);
 4044     disp($off);
 4045   %}
 4046 %}
 4047 
 4048 // Indirect Memory Plus Long Offset Operand
 4049 operand indOffset32Narrow(rRegN reg, immL32 off)
 4050 %{
 4051   predicate(CompressedOops::shift() == 0);
 4052   constraint(ALLOC_IN_RC(ptr_reg));
 4053   match(AddP (DecodeN reg) off);
 4054 
 4055   format %{ "[$reg + $off (32-bit)]" %}
 4056   interface(MEMORY_INTER) %{
 4057     base($reg);
 4058     index(0x4);
 4059     scale(0x0);
 4060     disp($off);
 4061   %}
 4062 %}
 4063 
 4064 // Indirect Memory Plus Index Register Plus Offset Operand
 4065 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4066 %{
 4067   predicate(CompressedOops::shift() == 0);
 4068   constraint(ALLOC_IN_RC(ptr_reg));
 4069   match(AddP (AddP (DecodeN reg) lreg) off);
 4070 
 4071   op_cost(10);
 4072   format %{"[$reg + $off + $lreg]" %}
 4073   interface(MEMORY_INTER) %{
 4074     base($reg);
 4075     index($lreg);
 4076     scale(0x0);
 4077     disp($off);
 4078   %}
 4079 %}
 4080 
 4081 // Indirect Memory Plus Index Register Plus Offset Operand
 4082 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4083 %{
 4084   predicate(CompressedOops::shift() == 0);
 4085   constraint(ALLOC_IN_RC(ptr_reg));
 4086   match(AddP (DecodeN reg) lreg);
 4087 
 4088   op_cost(10);
 4089   format %{"[$reg + $lreg]" %}
 4090   interface(MEMORY_INTER) %{
 4091     base($reg);
 4092     index($lreg);
 4093     scale(0x0);
 4094     disp(0x0);
 4095   %}
 4096 %}
 4097 
 4098 // Indirect Memory Times Scale Plus Index Register
 4099 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4100 %{
 4101   predicate(CompressedOops::shift() == 0);
 4102   constraint(ALLOC_IN_RC(ptr_reg));
 4103   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4104 
 4105   op_cost(10);
 4106   format %{"[$reg + $lreg << $scale]" %}
 4107   interface(MEMORY_INTER) %{
 4108     base($reg);
 4109     index($lreg);
 4110     scale($scale);
 4111     disp(0x0);
 4112   %}
 4113 %}
 4114 
 4115 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4116 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4117 %{
 4118   predicate(CompressedOops::shift() == 0);
 4119   constraint(ALLOC_IN_RC(ptr_reg));
 4120   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4121 
 4122   op_cost(10);
 4123   format %{"[$reg + $off + $lreg << $scale]" %}
 4124   interface(MEMORY_INTER) %{
 4125     base($reg);
 4126     index($lreg);
 4127     scale($scale);
 4128     disp($off);
 4129   %}
 4130 %}
 4131 
 4132 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4133 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4134 %{
 4135   constraint(ALLOC_IN_RC(ptr_reg));
 4136   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4137   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4138 
 4139   op_cost(10);
 4140   format %{"[$reg + $off + $idx]" %}
 4141   interface(MEMORY_INTER) %{
 4142     base($reg);
 4143     index($idx);
 4144     scale(0x0);
 4145     disp($off);
 4146   %}
 4147 %}
 4148 
 4149 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4150 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4151 %{
 4152   constraint(ALLOC_IN_RC(ptr_reg));
 4153   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4154   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4155 
 4156   op_cost(10);
 4157   format %{"[$reg + $off + $idx << $scale]" %}
 4158   interface(MEMORY_INTER) %{
 4159     base($reg);
 4160     index($idx);
 4161     scale($scale);
 4162     disp($off);
 4163   %}
 4164 %}
 4165 
 4166 //----------Special Memory Operands--------------------------------------------
 4167 // Stack Slot Operand - This operand is used for loading and storing temporary
 4168 //                      values on the stack where a match requires a value to
 4169 //                      flow through memory.
 4170 operand stackSlotP(sRegP reg)
 4171 %{
 4172   constraint(ALLOC_IN_RC(stack_slots));
 4173   // No match rule because this operand is only generated in matching
 4174 
 4175   format %{ "[$reg]" %}
 4176   interface(MEMORY_INTER) %{
 4177     base(0x4);   // RSP
 4178     index(0x4);  // No Index
 4179     scale(0x0);  // No Scale
 4180     disp($reg);  // Stack Offset
 4181   %}
 4182 %}
 4183 
 4184 operand stackSlotI(sRegI reg)
 4185 %{
 4186   constraint(ALLOC_IN_RC(stack_slots));
 4187   // No match rule because this operand is only generated in matching
 4188 
 4189   format %{ "[$reg]" %}
 4190   interface(MEMORY_INTER) %{
 4191     base(0x4);   // RSP
 4192     index(0x4);  // No Index
 4193     scale(0x0);  // No Scale
 4194     disp($reg);  // Stack Offset
 4195   %}
 4196 %}
 4197 
 4198 operand stackSlotF(sRegF reg)
 4199 %{
 4200   constraint(ALLOC_IN_RC(stack_slots));
 4201   // No match rule because this operand is only generated in matching
 4202 
 4203   format %{ "[$reg]" %}
 4204   interface(MEMORY_INTER) %{
 4205     base(0x4);   // RSP
 4206     index(0x4);  // No Index
 4207     scale(0x0);  // No Scale
 4208     disp($reg);  // Stack Offset
 4209   %}
 4210 %}
 4211 
 4212 operand stackSlotD(sRegD reg)
 4213 %{
 4214   constraint(ALLOC_IN_RC(stack_slots));
 4215   // No match rule because this operand is only generated in matching
 4216 
 4217   format %{ "[$reg]" %}
 4218   interface(MEMORY_INTER) %{
 4219     base(0x4);   // RSP
 4220     index(0x4);  // No Index
 4221     scale(0x0);  // No Scale
 4222     disp($reg);  // Stack Offset
 4223   %}
 4224 %}
 4225 operand stackSlotL(sRegL reg)
 4226 %{
 4227   constraint(ALLOC_IN_RC(stack_slots));
 4228   // No match rule because this operand is only generated in matching
 4229 
 4230   format %{ "[$reg]" %}
 4231   interface(MEMORY_INTER) %{
 4232     base(0x4);   // RSP
 4233     index(0x4);  // No Index
 4234     scale(0x0);  // No Scale
 4235     disp($reg);  // Stack Offset
 4236   %}
 4237 %}
 4238 
 4239 //----------Conditional Branch Operands----------------------------------------
 4240 // Comparison Op  - This is the operation of the comparison, and is limited to
 4241 //                  the following set of codes:
 4242 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4243 //
 4244 // Other attributes of the comparison, such as unsignedness, are specified
 4245 // by the comparison instruction that sets a condition code flags register.
 4246 // That result is represented by a flags operand whose subtype is appropriate
 4247 // to the unsignedness (etc.) of the comparison.
 4248 //
 4249 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4250 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4251 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4252 
 4253 // Comparison Code
 4254 operand cmpOp()
 4255 %{
 4256   match(Bool);
 4257 
 4258   format %{ "" %}
 4259   interface(COND_INTER) %{
 4260     equal(0x4, "e");
 4261     not_equal(0x5, "ne");
 4262     less(0xC, "l");
 4263     greater_equal(0xD, "ge");
 4264     less_equal(0xE, "le");
 4265     greater(0xF, "g");
 4266     overflow(0x0, "o");
 4267     no_overflow(0x1, "no");
 4268   %}
 4269 %}
 4270 
 4271 // Comparison Code, unsigned compare.  Used by FP also, with
 4272 // C2 (unordered) turned into GT or LT already.  The other bits
 4273 // C0 and C3 are turned into Carry & Zero flags.
 4274 operand cmpOpU()
 4275 %{
 4276   match(Bool);
 4277 
 4278   format %{ "" %}
 4279   interface(COND_INTER) %{
 4280     equal(0x4, "e");
 4281     not_equal(0x5, "ne");
 4282     less(0x2, "b");
 4283     greater_equal(0x3, "ae");
 4284     less_equal(0x6, "be");
 4285     greater(0x7, "a");
 4286     overflow(0x0, "o");
 4287     no_overflow(0x1, "no");
 4288   %}
 4289 %}
 4290 
 4291 
 4292 // Floating comparisons that don't require any fixup for the unordered case,
 4293 // If both inputs of the comparison are the same, ZF is always set so we
 4294 // don't need to use cmpOpUCF2 for eq/ne
 4295 operand cmpOpUCF() %{
 4296   match(Bool);
 4297   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4298             n->as_Bool()->_test._test == BoolTest::ge ||
 4299             n->as_Bool()->_test._test == BoolTest::le ||
 4300             n->as_Bool()->_test._test == BoolTest::gt ||
 4301             n->in(1)->in(1) == n->in(1)->in(2));
 4302   format %{ "" %}
 4303   interface(COND_INTER) %{
 4304     equal(0xb, "np");
 4305     not_equal(0xa, "p");
 4306     less(0x2, "b");
 4307     greater_equal(0x3, "ae");
 4308     less_equal(0x6, "be");
 4309     greater(0x7, "a");
 4310     overflow(0x0, "o");
 4311     no_overflow(0x1, "no");
 4312   %}
 4313 %}
 4314 
 4315 
 4316 // Floating comparisons that can be fixed up with extra conditional jumps
 4317 operand cmpOpUCF2() %{
 4318   match(Bool);
 4319   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 4320              n->as_Bool()->_test._test == BoolTest::eq) &&
 4321             n->in(1)->in(1) != n->in(1)->in(2));
 4322   format %{ "" %}
 4323   interface(COND_INTER) %{
 4324     equal(0x4, "e");
 4325     not_equal(0x5, "ne");
 4326     less(0x2, "b");
 4327     greater_equal(0x3, "ae");
 4328     less_equal(0x6, "be");
 4329     greater(0x7, "a");
 4330     overflow(0x0, "o");
 4331     no_overflow(0x1, "no");
 4332   %}
 4333 %}
 4334 
 4335 //----------OPERAND CLASSES----------------------------------------------------
 4336 // Operand Classes are groups of operands that are used as to simplify
 4337 // instruction definitions by not requiring the AD writer to specify separate
 4338 // instructions for every form of operand when the instruction accepts
 4339 // multiple operand types with the same basic encoding and format.  The classic
 4340 // case of this is memory operands.
 4341 
 4342 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4343                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4344                indCompressedOopOffset,
 4345                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4346                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4347                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4348 
 4349 //----------PIPELINE-----------------------------------------------------------
 4350 // Rules which define the behavior of the target architectures pipeline.
 4351 pipeline %{
 4352 
 4353 //----------ATTRIBUTES---------------------------------------------------------
 4354 attributes %{
 4355   variable_size_instructions;        // Fixed size instructions
 4356   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4357   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4358   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4359   instruction_fetch_units = 1;       // of 16 bytes
 4360 
 4361   // List of nop instructions
 4362   nops( MachNop );
 4363 %}
 4364 
 4365 //----------RESOURCES----------------------------------------------------------
 4366 // Resources are the functional units available to the machine
 4367 
 4368 // Generic P2/P3 pipeline
 4369 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4370 // 3 instructions decoded per cycle.
 4371 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4372 // 3 ALU op, only ALU0 handles mul instructions.
 4373 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4374            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4375            BR, FPU,
 4376            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4377 
 4378 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4379 // Pipeline Description specifies the stages in the machine's pipeline
 4380 
 4381 // Generic P2/P3 pipeline
 4382 pipe_desc(S0, S1, S2, S3, S4, S5);
 4383 
 4384 //----------PIPELINE CLASSES---------------------------------------------------
 4385 // Pipeline Classes describe the stages in which input and output are
 4386 // referenced by the hardware pipeline.
 4387 
 4388 // Naming convention: ialu or fpu
 4389 // Then: _reg
 4390 // Then: _reg if there is a 2nd register
 4391 // Then: _long if it's a pair of instructions implementing a long
 4392 // Then: _fat if it requires the big decoder
 4393 //   Or: _mem if it requires the big decoder and a memory unit.
 4394 
 4395 // Integer ALU reg operation
 4396 pipe_class ialu_reg(rRegI dst)
 4397 %{
 4398     single_instruction;
 4399     dst    : S4(write);
 4400     dst    : S3(read);
 4401     DECODE : S0;        // any decoder
 4402     ALU    : S3;        // any alu
 4403 %}
 4404 
 4405 // Long ALU reg operation
 4406 pipe_class ialu_reg_long(rRegL dst)
 4407 %{
 4408     instruction_count(2);
 4409     dst    : S4(write);
 4410     dst    : S3(read);
 4411     DECODE : S0(2);     // any 2 decoders
 4412     ALU    : S3(2);     // both alus
 4413 %}
 4414 
 4415 // Integer ALU reg operation using big decoder
 4416 pipe_class ialu_reg_fat(rRegI dst)
 4417 %{
 4418     single_instruction;
 4419     dst    : S4(write);
 4420     dst    : S3(read);
 4421     D0     : S0;        // big decoder only
 4422     ALU    : S3;        // any alu
 4423 %}
 4424 
 4425 // Integer ALU reg-reg operation
 4426 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4427 %{
 4428     single_instruction;
 4429     dst    : S4(write);
 4430     src    : S3(read);
 4431     DECODE : S0;        // any decoder
 4432     ALU    : S3;        // any alu
 4433 %}
 4434 
 4435 // Integer ALU reg-reg operation
 4436 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4437 %{
 4438     single_instruction;
 4439     dst    : S4(write);
 4440     src    : S3(read);
 4441     D0     : S0;        // big decoder only
 4442     ALU    : S3;        // any alu
 4443 %}
 4444 
 4445 // Integer ALU reg-mem operation
 4446 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4447 %{
 4448     single_instruction;
 4449     dst    : S5(write);
 4450     mem    : S3(read);
 4451     D0     : S0;        // big decoder only
 4452     ALU    : S4;        // any alu
 4453     MEM    : S3;        // any mem
 4454 %}
 4455 
 4456 // Integer mem operation (prefetch)
 4457 pipe_class ialu_mem(memory mem)
 4458 %{
 4459     single_instruction;
 4460     mem    : S3(read);
 4461     D0     : S0;        // big decoder only
 4462     MEM    : S3;        // any mem
 4463 %}
 4464 
 4465 // Integer Store to Memory
 4466 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4467 %{
 4468     single_instruction;
 4469     mem    : S3(read);
 4470     src    : S5(read);
 4471     D0     : S0;        // big decoder only
 4472     ALU    : S4;        // any alu
 4473     MEM    : S3;
 4474 %}
 4475 
 4476 // // Long Store to Memory
 4477 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4478 // %{
 4479 //     instruction_count(2);
 4480 //     mem    : S3(read);
 4481 //     src    : S5(read);
 4482 //     D0     : S0(2);          // big decoder only; twice
 4483 //     ALU    : S4(2);     // any 2 alus
 4484 //     MEM    : S3(2);  // Both mems
 4485 // %}
 4486 
 4487 // Integer Store to Memory
 4488 pipe_class ialu_mem_imm(memory mem)
 4489 %{
 4490     single_instruction;
 4491     mem    : S3(read);
 4492     D0     : S0;        // big decoder only
 4493     ALU    : S4;        // any alu
 4494     MEM    : S3;
 4495 %}
 4496 
 4497 // Integer ALU0 reg-reg operation
 4498 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4499 %{
 4500     single_instruction;
 4501     dst    : S4(write);
 4502     src    : S3(read);
 4503     D0     : S0;        // Big decoder only
 4504     ALU0   : S3;        // only alu0
 4505 %}
 4506 
 4507 // Integer ALU0 reg-mem operation
 4508 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4509 %{
 4510     single_instruction;
 4511     dst    : S5(write);
 4512     mem    : S3(read);
 4513     D0     : S0;        // big decoder only
 4514     ALU0   : S4;        // ALU0 only
 4515     MEM    : S3;        // any mem
 4516 %}
 4517 
 4518 // Integer ALU reg-reg operation
 4519 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4520 %{
 4521     single_instruction;
 4522     cr     : S4(write);
 4523     src1   : S3(read);
 4524     src2   : S3(read);
 4525     DECODE : S0;        // any decoder
 4526     ALU    : S3;        // any alu
 4527 %}
 4528 
 4529 // Integer ALU reg-imm operation
 4530 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4531 %{
 4532     single_instruction;
 4533     cr     : S4(write);
 4534     src1   : S3(read);
 4535     DECODE : S0;        // any decoder
 4536     ALU    : S3;        // any alu
 4537 %}
 4538 
 4539 // Integer ALU reg-mem operation
 4540 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4541 %{
 4542     single_instruction;
 4543     cr     : S4(write);
 4544     src1   : S3(read);
 4545     src2   : S3(read);
 4546     D0     : S0;        // big decoder only
 4547     ALU    : S4;        // any alu
 4548     MEM    : S3;
 4549 %}
 4550 
 4551 // Conditional move reg-reg
 4552 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4553 %{
 4554     instruction_count(4);
 4555     y      : S4(read);
 4556     q      : S3(read);
 4557     p      : S3(read);
 4558     DECODE : S0(4);     // any decoder
 4559 %}
 4560 
 4561 // Conditional move reg-reg
 4562 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4563 %{
 4564     single_instruction;
 4565     dst    : S4(write);
 4566     src    : S3(read);
 4567     cr     : S3(read);
 4568     DECODE : S0;        // any decoder
 4569 %}
 4570 
 4571 // Conditional move reg-mem
 4572 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4573 %{
 4574     single_instruction;
 4575     dst    : S4(write);
 4576     src    : S3(read);
 4577     cr     : S3(read);
 4578     DECODE : S0;        // any decoder
 4579     MEM    : S3;
 4580 %}
 4581 
 4582 // Conditional move reg-reg long
 4583 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4584 %{
 4585     single_instruction;
 4586     dst    : S4(write);
 4587     src    : S3(read);
 4588     cr     : S3(read);
 4589     DECODE : S0(2);     // any 2 decoders
 4590 %}
 4591 
 4592 // XXX
 4593 // // Conditional move double reg-reg
 4594 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4595 // %{
 4596 //     single_instruction;
 4597 //     dst    : S4(write);
 4598 //     src    : S3(read);
 4599 //     cr     : S3(read);
 4600 //     DECODE : S0;     // any decoder
 4601 // %}
 4602 
 4603 // Float reg-reg operation
 4604 pipe_class fpu_reg(regD dst)
 4605 %{
 4606     instruction_count(2);
 4607     dst    : S3(read);
 4608     DECODE : S0(2);     // any 2 decoders
 4609     FPU    : S3;
 4610 %}
 4611 
 4612 // Float reg-reg operation
 4613 pipe_class fpu_reg_reg(regD dst, regD src)
 4614 %{
 4615     instruction_count(2);
 4616     dst    : S4(write);
 4617     src    : S3(read);
 4618     DECODE : S0(2);     // any 2 decoders
 4619     FPU    : S3;
 4620 %}
 4621 
 4622 // Float reg-reg operation
 4623 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4624 %{
 4625     instruction_count(3);
 4626     dst    : S4(write);
 4627     src1   : S3(read);
 4628     src2   : S3(read);
 4629     DECODE : S0(3);     // any 3 decoders
 4630     FPU    : S3(2);
 4631 %}
 4632 
 4633 // Float reg-reg operation
 4634 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4635 %{
 4636     instruction_count(4);
 4637     dst    : S4(write);
 4638     src1   : S3(read);
 4639     src2   : S3(read);
 4640     src3   : S3(read);
 4641     DECODE : S0(4);     // any 3 decoders
 4642     FPU    : S3(2);
 4643 %}
 4644 
 4645 // Float reg-reg operation
 4646 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4647 %{
 4648     instruction_count(4);
 4649     dst    : S4(write);
 4650     src1   : S3(read);
 4651     src2   : S3(read);
 4652     src3   : S3(read);
 4653     DECODE : S1(3);     // any 3 decoders
 4654     D0     : S0;        // Big decoder only
 4655     FPU    : S3(2);
 4656     MEM    : S3;
 4657 %}
 4658 
 4659 // Float reg-mem operation
 4660 pipe_class fpu_reg_mem(regD dst, memory mem)
 4661 %{
 4662     instruction_count(2);
 4663     dst    : S5(write);
 4664     mem    : S3(read);
 4665     D0     : S0;        // big decoder only
 4666     DECODE : S1;        // any decoder for FPU POP
 4667     FPU    : S4;
 4668     MEM    : S3;        // any mem
 4669 %}
 4670 
 4671 // Float reg-mem operation
 4672 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4673 %{
 4674     instruction_count(3);
 4675     dst    : S5(write);
 4676     src1   : S3(read);
 4677     mem    : S3(read);
 4678     D0     : S0;        // big decoder only
 4679     DECODE : S1(2);     // any decoder for FPU POP
 4680     FPU    : S4;
 4681     MEM    : S3;        // any mem
 4682 %}
 4683 
 4684 // Float mem-reg operation
 4685 pipe_class fpu_mem_reg(memory mem, regD src)
 4686 %{
 4687     instruction_count(2);
 4688     src    : S5(read);
 4689     mem    : S3(read);
 4690     DECODE : S0;        // any decoder for FPU PUSH
 4691     D0     : S1;        // big decoder only
 4692     FPU    : S4;
 4693     MEM    : S3;        // any mem
 4694 %}
 4695 
 4696 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4697 %{
 4698     instruction_count(3);
 4699     src1   : S3(read);
 4700     src2   : S3(read);
 4701     mem    : S3(read);
 4702     DECODE : S0(2);     // any decoder for FPU PUSH
 4703     D0     : S1;        // big decoder only
 4704     FPU    : S4;
 4705     MEM    : S3;        // any mem
 4706 %}
 4707 
 4708 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4709 %{
 4710     instruction_count(3);
 4711     src1   : S3(read);
 4712     src2   : S3(read);
 4713     mem    : S4(read);
 4714     DECODE : S0;        // any decoder for FPU PUSH
 4715     D0     : S0(2);     // big decoder only
 4716     FPU    : S4;
 4717     MEM    : S3(2);     // any mem
 4718 %}
 4719 
 4720 pipe_class fpu_mem_mem(memory dst, memory src1)
 4721 %{
 4722     instruction_count(2);
 4723     src1   : S3(read);
 4724     dst    : S4(read);
 4725     D0     : S0(2);     // big decoder only
 4726     MEM    : S3(2);     // any mem
 4727 %}
 4728 
 4729 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4730 %{
 4731     instruction_count(3);
 4732     src1   : S3(read);
 4733     src2   : S3(read);
 4734     dst    : S4(read);
 4735     D0     : S0(3);     // big decoder only
 4736     FPU    : S4;
 4737     MEM    : S3(3);     // any mem
 4738 %}
 4739 
 4740 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4741 %{
 4742     instruction_count(3);
 4743     src1   : S4(read);
 4744     mem    : S4(read);
 4745     DECODE : S0;        // any decoder for FPU PUSH
 4746     D0     : S0(2);     // big decoder only
 4747     FPU    : S4;
 4748     MEM    : S3(2);     // any mem
 4749 %}
 4750 
 4751 // Float load constant
 4752 pipe_class fpu_reg_con(regD dst)
 4753 %{
 4754     instruction_count(2);
 4755     dst    : S5(write);
 4756     D0     : S0;        // big decoder only for the load
 4757     DECODE : S1;        // any decoder for FPU POP
 4758     FPU    : S4;
 4759     MEM    : S3;        // any mem
 4760 %}
 4761 
 4762 // Float load constant
 4763 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4764 %{
 4765     instruction_count(3);
 4766     dst    : S5(write);
 4767     src    : S3(read);
 4768     D0     : S0;        // big decoder only for the load
 4769     DECODE : S1(2);     // any decoder for FPU POP
 4770     FPU    : S4;
 4771     MEM    : S3;        // any mem
 4772 %}
 4773 
 4774 // UnConditional branch
 4775 pipe_class pipe_jmp(label labl)
 4776 %{
 4777     single_instruction;
 4778     BR   : S3;
 4779 %}
 4780 
 4781 // Conditional branch
 4782 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4783 %{
 4784     single_instruction;
 4785     cr    : S1(read);
 4786     BR    : S3;
 4787 %}
 4788 
 4789 // Allocation idiom
 4790 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4791 %{
 4792     instruction_count(1); force_serialization;
 4793     fixed_latency(6);
 4794     heap_ptr : S3(read);
 4795     DECODE   : S0(3);
 4796     D0       : S2;
 4797     MEM      : S3;
 4798     ALU      : S3(2);
 4799     dst      : S5(write);
 4800     BR       : S5;
 4801 %}
 4802 
 4803 // Generic big/slow expanded idiom
 4804 pipe_class pipe_slow()
 4805 %{
 4806     instruction_count(10); multiple_bundles; force_serialization;
 4807     fixed_latency(100);
 4808     D0  : S0(2);
 4809     MEM : S3(2);
 4810 %}
 4811 
 4812 // The real do-nothing guy
 4813 pipe_class empty()
 4814 %{
 4815     instruction_count(0);
 4816 %}
 4817 
 4818 // Define the class for the Nop node
 4819 define
 4820 %{
 4821    MachNop = empty;
 4822 %}
 4823 
 4824 %}
 4825 
 4826 //----------INSTRUCTIONS-------------------------------------------------------
 4827 //
 4828 // match      -- States which machine-independent subtree may be replaced
 4829 //               by this instruction.
 4830 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4831 //               selection to identify a minimum cost tree of machine
 4832 //               instructions that matches a tree of machine-independent
 4833 //               instructions.
 4834 // format     -- A string providing the disassembly for this instruction.
 4835 //               The value of an instruction's operand may be inserted
 4836 //               by referring to it with a '$' prefix.
 4837 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4838 //               to within an encode class as $primary, $secondary, and $tertiary
 4839 //               rrspectively.  The primary opcode is commonly used to
 4840 //               indicate the type of machine instruction, while secondary
 4841 //               and tertiary are often used for prefix options or addressing
 4842 //               modes.
 4843 // ins_encode -- A list of encode classes with parameters. The encode class
 4844 //               name must have been defined in an 'enc_class' specification
 4845 //               in the encode section of the architecture description.
 4846 
 4847 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4848 // Load Float
 4849 instruct MoveF2VL(vlRegF dst, regF src) %{
 4850   match(Set dst src);
 4851   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4852   ins_encode %{
 4853     ShouldNotReachHere();
 4854   %}
 4855   ins_pipe( fpu_reg_reg );
 4856 %}
 4857 
 4858 // Load Float
 4859 instruct MoveF2LEG(legRegF dst, regF src) %{
 4860   match(Set dst src);
 4861   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4862   ins_encode %{
 4863     ShouldNotReachHere();
 4864   %}
 4865   ins_pipe( fpu_reg_reg );
 4866 %}
 4867 
 4868 // Load Float
 4869 instruct MoveVL2F(regF dst, vlRegF src) %{
 4870   match(Set dst src);
 4871   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4872   ins_encode %{
 4873     ShouldNotReachHere();
 4874   %}
 4875   ins_pipe( fpu_reg_reg );
 4876 %}
 4877 
 4878 // Load Float
 4879 instruct MoveLEG2F(regF dst, legRegF src) %{
 4880   match(Set dst src);
 4881   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4882   ins_encode %{
 4883     ShouldNotReachHere();
 4884   %}
 4885   ins_pipe( fpu_reg_reg );
 4886 %}
 4887 
 4888 // Load Double
 4889 instruct MoveD2VL(vlRegD dst, regD src) %{
 4890   match(Set dst src);
 4891   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4892   ins_encode %{
 4893     ShouldNotReachHere();
 4894   %}
 4895   ins_pipe( fpu_reg_reg );
 4896 %}
 4897 
 4898 // Load Double
 4899 instruct MoveD2LEG(legRegD dst, regD src) %{
 4900   match(Set dst src);
 4901   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4902   ins_encode %{
 4903     ShouldNotReachHere();
 4904   %}
 4905   ins_pipe( fpu_reg_reg );
 4906 %}
 4907 
 4908 // Load Double
 4909 instruct MoveVL2D(regD dst, vlRegD src) %{
 4910   match(Set dst src);
 4911   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4912   ins_encode %{
 4913     ShouldNotReachHere();
 4914   %}
 4915   ins_pipe( fpu_reg_reg );
 4916 %}
 4917 
 4918 // Load Double
 4919 instruct MoveLEG2D(regD dst, legRegD src) %{
 4920   match(Set dst src);
 4921   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4922   ins_encode %{
 4923     ShouldNotReachHere();
 4924   %}
 4925   ins_pipe( fpu_reg_reg );
 4926 %}
 4927 
 4928 //----------Load/Store/Move Instructions---------------------------------------
 4929 //----------Load Instructions--------------------------------------------------
 4930 
 4931 // Load Byte (8 bit signed)
 4932 instruct loadB(rRegI dst, memory mem)
 4933 %{
 4934   match(Set dst (LoadB mem));
 4935 
 4936   ins_cost(125);
 4937   format %{ "movsbl  $dst, $mem\t# byte" %}
 4938 
 4939   ins_encode %{
 4940     __ movsbl($dst$$Register, $mem$$Address);
 4941   %}
 4942 
 4943   ins_pipe(ialu_reg_mem);
 4944 %}
 4945 
 4946 // Load Byte (8 bit signed) into Long Register
 4947 instruct loadB2L(rRegL dst, memory mem)
 4948 %{
 4949   match(Set dst (ConvI2L (LoadB mem)));
 4950 
 4951   ins_cost(125);
 4952   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4953 
 4954   ins_encode %{
 4955     __ movsbq($dst$$Register, $mem$$Address);
 4956   %}
 4957 
 4958   ins_pipe(ialu_reg_mem);
 4959 %}
 4960 
 4961 // Load Unsigned Byte (8 bit UNsigned)
 4962 instruct loadUB(rRegI dst, memory mem)
 4963 %{
 4964   match(Set dst (LoadUB mem));
 4965 
 4966   ins_cost(125);
 4967   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4968 
 4969   ins_encode %{
 4970     __ movzbl($dst$$Register, $mem$$Address);
 4971   %}
 4972 
 4973   ins_pipe(ialu_reg_mem);
 4974 %}
 4975 
 4976 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4977 instruct loadUB2L(rRegL dst, memory mem)
 4978 %{
 4979   match(Set dst (ConvI2L (LoadUB mem)));
 4980 
 4981   ins_cost(125);
 4982   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4983 
 4984   ins_encode %{
 4985     __ movzbq($dst$$Register, $mem$$Address);
 4986   %}
 4987 
 4988   ins_pipe(ialu_reg_mem);
 4989 %}
 4990 
 4991 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4992 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4993   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4994   effect(KILL cr);
 4995 
 4996   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4997             "andl    $dst, right_n_bits($mask, 8)" %}
 4998   ins_encode %{
 4999     Register Rdst = $dst$$Register;
 5000     __ movzbq(Rdst, $mem$$Address);
 5001     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 5002   %}
 5003   ins_pipe(ialu_reg_mem);
 5004 %}
 5005 
 5006 // Load Short (16 bit signed)
 5007 instruct loadS(rRegI dst, memory mem)
 5008 %{
 5009   match(Set dst (LoadS mem));
 5010 
 5011   ins_cost(125);
 5012   format %{ "movswl $dst, $mem\t# short" %}
 5013 
 5014   ins_encode %{
 5015     __ movswl($dst$$Register, $mem$$Address);
 5016   %}
 5017 
 5018   ins_pipe(ialu_reg_mem);
 5019 %}
 5020 
 5021 // Load Short (16 bit signed) to Byte (8 bit signed)
 5022 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5023   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5024 
 5025   ins_cost(125);
 5026   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5027   ins_encode %{
 5028     __ movsbl($dst$$Register, $mem$$Address);
 5029   %}
 5030   ins_pipe(ialu_reg_mem);
 5031 %}
 5032 
 5033 // Load Short (16 bit signed) into Long Register
 5034 instruct loadS2L(rRegL dst, memory mem)
 5035 %{
 5036   match(Set dst (ConvI2L (LoadS mem)));
 5037 
 5038   ins_cost(125);
 5039   format %{ "movswq $dst, $mem\t# short -> long" %}
 5040 
 5041   ins_encode %{
 5042     __ movswq($dst$$Register, $mem$$Address);
 5043   %}
 5044 
 5045   ins_pipe(ialu_reg_mem);
 5046 %}
 5047 
 5048 // Load Unsigned Short/Char (16 bit UNsigned)
 5049 instruct loadUS(rRegI dst, memory mem)
 5050 %{
 5051   match(Set dst (LoadUS mem));
 5052 
 5053   ins_cost(125);
 5054   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5055 
 5056   ins_encode %{
 5057     __ movzwl($dst$$Register, $mem$$Address);
 5058   %}
 5059 
 5060   ins_pipe(ialu_reg_mem);
 5061 %}
 5062 
 5063 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5064 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5065   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5066 
 5067   ins_cost(125);
 5068   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5069   ins_encode %{
 5070     __ movsbl($dst$$Register, $mem$$Address);
 5071   %}
 5072   ins_pipe(ialu_reg_mem);
 5073 %}
 5074 
 5075 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5076 instruct loadUS2L(rRegL dst, memory mem)
 5077 %{
 5078   match(Set dst (ConvI2L (LoadUS mem)));
 5079 
 5080   ins_cost(125);
 5081   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5082 
 5083   ins_encode %{
 5084     __ movzwq($dst$$Register, $mem$$Address);
 5085   %}
 5086 
 5087   ins_pipe(ialu_reg_mem);
 5088 %}
 5089 
 5090 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5091 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5092   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5093 
 5094   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5095   ins_encode %{
 5096     __ movzbq($dst$$Register, $mem$$Address);
 5097   %}
 5098   ins_pipe(ialu_reg_mem);
 5099 %}
 5100 
 5101 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5102 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5103   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5104   effect(KILL cr);
 5105 
 5106   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5107             "andl    $dst, right_n_bits($mask, 16)" %}
 5108   ins_encode %{
 5109     Register Rdst = $dst$$Register;
 5110     __ movzwq(Rdst, $mem$$Address);
 5111     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5112   %}
 5113   ins_pipe(ialu_reg_mem);
 5114 %}
 5115 
 5116 // Load Integer
 5117 instruct loadI(rRegI dst, memory mem)
 5118 %{
 5119   match(Set dst (LoadI mem));
 5120 
 5121   ins_cost(125);
 5122   format %{ "movl    $dst, $mem\t# int" %}
 5123 
 5124   ins_encode %{
 5125     __ movl($dst$$Register, $mem$$Address);
 5126   %}
 5127 
 5128   ins_pipe(ialu_reg_mem);
 5129 %}
 5130 
 5131 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5132 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5133   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5134 
 5135   ins_cost(125);
 5136   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5137   ins_encode %{
 5138     __ movsbl($dst$$Register, $mem$$Address);
 5139   %}
 5140   ins_pipe(ialu_reg_mem);
 5141 %}
 5142 
 5143 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5144 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5145   match(Set dst (AndI (LoadI mem) mask));
 5146 
 5147   ins_cost(125);
 5148   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5149   ins_encode %{
 5150     __ movzbl($dst$$Register, $mem$$Address);
 5151   %}
 5152   ins_pipe(ialu_reg_mem);
 5153 %}
 5154 
 5155 // Load Integer (32 bit signed) to Short (16 bit signed)
 5156 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5157   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5158 
 5159   ins_cost(125);
 5160   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5161   ins_encode %{
 5162     __ movswl($dst$$Register, $mem$$Address);
 5163   %}
 5164   ins_pipe(ialu_reg_mem);
 5165 %}
 5166 
 5167 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5168 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5169   match(Set dst (AndI (LoadI mem) mask));
 5170 
 5171   ins_cost(125);
 5172   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5173   ins_encode %{
 5174     __ movzwl($dst$$Register, $mem$$Address);
 5175   %}
 5176   ins_pipe(ialu_reg_mem);
 5177 %}
 5178 
 5179 // Load Integer into Long Register
 5180 instruct loadI2L(rRegL dst, memory mem)
 5181 %{
 5182   match(Set dst (ConvI2L (LoadI mem)));
 5183 
 5184   ins_cost(125);
 5185   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5186 
 5187   ins_encode %{
 5188     __ movslq($dst$$Register, $mem$$Address);
 5189   %}
 5190 
 5191   ins_pipe(ialu_reg_mem);
 5192 %}
 5193 
 5194 // Load Integer with mask 0xFF into Long Register
 5195 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5196   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5197 
 5198   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5199   ins_encode %{
 5200     __ movzbq($dst$$Register, $mem$$Address);
 5201   %}
 5202   ins_pipe(ialu_reg_mem);
 5203 %}
 5204 
 5205 // Load Integer with mask 0xFFFF into Long Register
 5206 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5207   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5208 
 5209   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5210   ins_encode %{
 5211     __ movzwq($dst$$Register, $mem$$Address);
 5212   %}
 5213   ins_pipe(ialu_reg_mem);
 5214 %}
 5215 
 5216 // Load Integer with a 31-bit mask into Long Register
 5217 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5218   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5219   effect(KILL cr);
 5220 
 5221   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5222             "andl    $dst, $mask" %}
 5223   ins_encode %{
 5224     Register Rdst = $dst$$Register;
 5225     __ movl(Rdst, $mem$$Address);
 5226     __ andl(Rdst, $mask$$constant);
 5227   %}
 5228   ins_pipe(ialu_reg_mem);
 5229 %}
 5230 
 5231 // Load Unsigned Integer into Long Register
 5232 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5233 %{
 5234   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5235 
 5236   ins_cost(125);
 5237   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5238 
 5239   ins_encode %{
 5240     __ movl($dst$$Register, $mem$$Address);
 5241   %}
 5242 
 5243   ins_pipe(ialu_reg_mem);
 5244 %}
 5245 
 5246 // Load Long
 5247 instruct loadL(rRegL dst, memory mem)
 5248 %{
 5249   match(Set dst (LoadL mem));
 5250 
 5251   ins_cost(125);
 5252   format %{ "movq    $dst, $mem\t# long" %}
 5253 
 5254   ins_encode %{
 5255     __ movq($dst$$Register, $mem$$Address);
 5256   %}
 5257 
 5258   ins_pipe(ialu_reg_mem); // XXX
 5259 %}
 5260 
 5261 // Load Range
 5262 instruct loadRange(rRegI dst, memory mem)
 5263 %{
 5264   match(Set dst (LoadRange mem));
 5265 
 5266   ins_cost(125); // XXX
 5267   format %{ "movl    $dst, $mem\t# range" %}
 5268   ins_encode %{
 5269     __ movl($dst$$Register, $mem$$Address);
 5270   %}
 5271   ins_pipe(ialu_reg_mem);
 5272 %}
 5273 
 5274 // Load Pointer
 5275 instruct loadP(rRegP dst, memory mem)
 5276 %{
 5277   match(Set dst (LoadP mem));
 5278   predicate(n->as_Load()->barrier_data() == 0);
 5279 
 5280   ins_cost(125); // XXX
 5281   format %{ "movq    $dst, $mem\t# ptr" %}
 5282   ins_encode %{
 5283     __ movq($dst$$Register, $mem$$Address);
 5284   %}
 5285   ins_pipe(ialu_reg_mem); // XXX
 5286 %}
 5287 
 5288 // Load Compressed Pointer
 5289 instruct loadN(rRegN dst, memory mem)
 5290 %{
 5291    match(Set dst (LoadN mem));
 5292 
 5293    ins_cost(125); // XXX
 5294    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5295    ins_encode %{
 5296      __ movl($dst$$Register, $mem$$Address);
 5297    %}
 5298    ins_pipe(ialu_reg_mem); // XXX
 5299 %}
 5300 
 5301 
 5302 // Load Klass Pointer
 5303 instruct loadKlass(rRegP dst, memory mem)
 5304 %{
 5305   match(Set dst (LoadKlass mem));
 5306 
 5307   ins_cost(125); // XXX
 5308   format %{ "movq    $dst, $mem\t# class" %}
 5309   ins_encode %{
 5310     __ movq($dst$$Register, $mem$$Address);
 5311   %}
 5312   ins_pipe(ialu_reg_mem); // XXX
 5313 %}
 5314 
 5315 // Load narrow Klass Pointer
 5316 instruct loadNKlass(rRegN dst, memory mem)
 5317 %{
 5318   predicate(!UseCompactObjectHeaders);
 5319   match(Set dst (LoadNKlass mem));
 5320 
 5321   ins_cost(125); // XXX
 5322   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5323   ins_encode %{
 5324     __ movl($dst$$Register, $mem$$Address);
 5325   %}
 5326   ins_pipe(ialu_reg_mem); // XXX
 5327 %}
 5328 
 5329 instruct loadNKlassLilliput(rRegN dst, indOffset8 mem, rFlagsReg cr)
 5330 %{
 5331   predicate(UseCompactObjectHeaders);
 5332   match(Set dst (LoadNKlass mem));
 5333   effect(TEMP_DEF dst, KILL cr);
 5334   ins_cost(125); // XXX
 5335   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5336   ins_encode %{
 5337     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset 4, but got: %d", $mem$$disp);
 5338     assert($mem$$index == 4, "expect no index register: %d", $mem$$index);
 5339     Register dst = $dst$$Register;
 5340     Register obj = $mem$$base$$Register;
 5341     C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
 5342     Compile::current()->output()->add_stub(stub);
 5343     __ movq(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
 5344     __ testb(dst, markWord::monitor_value);
 5345     __ jcc(Assembler::notZero, stub->entry());
 5346     __ bind(stub->continuation());
 5347     __ shrq(dst, markWord::klass_shift);
 5348   %}
 5349   ins_pipe(pipe_slow); // XXX
 5350 %}
 5351 
 5352 // Load Float
 5353 instruct loadF(regF dst, memory mem)
 5354 %{
 5355   match(Set dst (LoadF mem));
 5356 
 5357   ins_cost(145); // XXX
 5358   format %{ "movss   $dst, $mem\t# float" %}
 5359   ins_encode %{
 5360     __ movflt($dst$$XMMRegister, $mem$$Address);
 5361   %}
 5362   ins_pipe(pipe_slow); // XXX
 5363 %}
 5364 
 5365 // Load Double
 5366 instruct loadD_partial(regD dst, memory mem)
 5367 %{
 5368   predicate(!UseXmmLoadAndClearUpper);
 5369   match(Set dst (LoadD mem));
 5370 
 5371   ins_cost(145); // XXX
 5372   format %{ "movlpd  $dst, $mem\t# double" %}
 5373   ins_encode %{
 5374     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5375   %}
 5376   ins_pipe(pipe_slow); // XXX
 5377 %}
 5378 
 5379 instruct loadD(regD dst, memory mem)
 5380 %{
 5381   predicate(UseXmmLoadAndClearUpper);
 5382   match(Set dst (LoadD mem));
 5383 
 5384   ins_cost(145); // XXX
 5385   format %{ "movsd   $dst, $mem\t# double" %}
 5386   ins_encode %{
 5387     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5388   %}
 5389   ins_pipe(pipe_slow); // XXX
 5390 %}
 5391 
 5392 
 5393 // Following pseudo code describes the algorithm for max[FD]:
 5394 // Min algorithm is on similar lines
 5395 //  btmp = (b < +0.0) ? a : b
 5396 //  atmp = (b < +0.0) ? b : a
 5397 //  Tmp  = Max_Float(atmp , btmp)
 5398 //  Res  = (atmp == NaN) ? atmp : Tmp
 5399 
 5400 // max = java.lang.Math.max(float a, float b)
 5401 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5402   predicate(UseAVX > 0 && !n->is_reduction());
 5403   match(Set dst (MaxF a b));
 5404   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5405   format %{
 5406      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5407      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5408      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5409      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5410      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5411   %}
 5412   ins_encode %{
 5413     int vector_len = Assembler::AVX_128bit;
 5414     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5415     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5416     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5417     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5418     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5419  %}
 5420   ins_pipe( pipe_slow );
 5421 %}
 5422 
 5423 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5424   predicate(UseAVX > 0 && n->is_reduction());
 5425   match(Set dst (MaxF a b));
 5426   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5427 
 5428   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5429   ins_encode %{
 5430     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5431                     false /*min*/, true /*single*/);
 5432   %}
 5433   ins_pipe( pipe_slow );
 5434 %}
 5435 
 5436 // max = java.lang.Math.max(double a, double b)
 5437 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5438   predicate(UseAVX > 0 && !n->is_reduction());
 5439   match(Set dst (MaxD a b));
 5440   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5441   format %{
 5442      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5443      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5444      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5445      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5446      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5447   %}
 5448   ins_encode %{
 5449     int vector_len = Assembler::AVX_128bit;
 5450     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5451     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5452     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5453     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5454     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5455   %}
 5456   ins_pipe( pipe_slow );
 5457 %}
 5458 
 5459 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5460   predicate(UseAVX > 0 && n->is_reduction());
 5461   match(Set dst (MaxD a b));
 5462   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5463 
 5464   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5465   ins_encode %{
 5466     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5467                     false /*min*/, false /*single*/);
 5468   %}
 5469   ins_pipe( pipe_slow );
 5470 %}
 5471 
 5472 // min = java.lang.Math.min(float a, float b)
 5473 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5474   predicate(UseAVX > 0 && !n->is_reduction());
 5475   match(Set dst (MinF a b));
 5476   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5477   format %{
 5478      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5479      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5480      "vminss           $tmp,$atmp,$btmp           \n\t"
 5481      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5482      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5483   %}
 5484   ins_encode %{
 5485     int vector_len = Assembler::AVX_128bit;
 5486     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5487     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5488     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5489     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5490     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5491   %}
 5492   ins_pipe( pipe_slow );
 5493 %}
 5494 
 5495 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5496   predicate(UseAVX > 0 && n->is_reduction());
 5497   match(Set dst (MinF a b));
 5498   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5499 
 5500   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5501   ins_encode %{
 5502     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5503                     true /*min*/, true /*single*/);
 5504   %}
 5505   ins_pipe( pipe_slow );
 5506 %}
 5507 
 5508 // min = java.lang.Math.min(double a, double b)
 5509 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5510   predicate(UseAVX > 0 && !n->is_reduction());
 5511   match(Set dst (MinD a b));
 5512   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5513   format %{
 5514      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5515      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5516      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5517      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5518      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5519   %}
 5520   ins_encode %{
 5521     int vector_len = Assembler::AVX_128bit;
 5522     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5523     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5524     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5525     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5526     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5527   %}
 5528   ins_pipe( pipe_slow );
 5529 %}
 5530 
 5531 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5532   predicate(UseAVX > 0 && n->is_reduction());
 5533   match(Set dst (MinD a b));
 5534   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5535 
 5536   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5537   ins_encode %{
 5538     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5539                     true /*min*/, false /*single*/);
 5540   %}
 5541   ins_pipe( pipe_slow );
 5542 %}
 5543 
 5544 // Load Effective Address
 5545 instruct leaP8(rRegP dst, indOffset8 mem)
 5546 %{
 5547   match(Set dst mem);
 5548 
 5549   ins_cost(110); // XXX
 5550   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5551   ins_encode %{
 5552     __ leaq($dst$$Register, $mem$$Address);
 5553   %}
 5554   ins_pipe(ialu_reg_reg_fat);
 5555 %}
 5556 
 5557 instruct leaP32(rRegP dst, indOffset32 mem)
 5558 %{
 5559   match(Set dst mem);
 5560 
 5561   ins_cost(110);
 5562   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5563   ins_encode %{
 5564     __ leaq($dst$$Register, $mem$$Address);
 5565   %}
 5566   ins_pipe(ialu_reg_reg_fat);
 5567 %}
 5568 
 5569 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5570 %{
 5571   match(Set dst mem);
 5572 
 5573   ins_cost(110);
 5574   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5575   ins_encode %{
 5576     __ leaq($dst$$Register, $mem$$Address);
 5577   %}
 5578   ins_pipe(ialu_reg_reg_fat);
 5579 %}
 5580 
 5581 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5582 %{
 5583   match(Set dst mem);
 5584 
 5585   ins_cost(110);
 5586   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5587   ins_encode %{
 5588     __ leaq($dst$$Register, $mem$$Address);
 5589   %}
 5590   ins_pipe(ialu_reg_reg_fat);
 5591 %}
 5592 
 5593 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5594 %{
 5595   match(Set dst mem);
 5596 
 5597   ins_cost(110);
 5598   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5599   ins_encode %{
 5600     __ leaq($dst$$Register, $mem$$Address);
 5601   %}
 5602   ins_pipe(ialu_reg_reg_fat);
 5603 %}
 5604 
 5605 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5606 %{
 5607   match(Set dst mem);
 5608 
 5609   ins_cost(110);
 5610   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5611   ins_encode %{
 5612     __ leaq($dst$$Register, $mem$$Address);
 5613   %}
 5614   ins_pipe(ialu_reg_reg_fat);
 5615 %}
 5616 
 5617 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5618 %{
 5619   match(Set dst mem);
 5620 
 5621   ins_cost(110);
 5622   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5623   ins_encode %{
 5624     __ leaq($dst$$Register, $mem$$Address);
 5625   %}
 5626   ins_pipe(ialu_reg_reg_fat);
 5627 %}
 5628 
 5629 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5630 %{
 5631   match(Set dst mem);
 5632 
 5633   ins_cost(110);
 5634   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5635   ins_encode %{
 5636     __ leaq($dst$$Register, $mem$$Address);
 5637   %}
 5638   ins_pipe(ialu_reg_reg_fat);
 5639 %}
 5640 
 5641 // Load Effective Address which uses Narrow (32-bits) oop
 5642 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5643 %{
 5644   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5645   match(Set dst mem);
 5646 
 5647   ins_cost(110);
 5648   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5649   ins_encode %{
 5650     __ leaq($dst$$Register, $mem$$Address);
 5651   %}
 5652   ins_pipe(ialu_reg_reg_fat);
 5653 %}
 5654 
 5655 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5656 %{
 5657   predicate(CompressedOops::shift() == 0);
 5658   match(Set dst mem);
 5659 
 5660   ins_cost(110); // XXX
 5661   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5662   ins_encode %{
 5663     __ leaq($dst$$Register, $mem$$Address);
 5664   %}
 5665   ins_pipe(ialu_reg_reg_fat);
 5666 %}
 5667 
 5668 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5669 %{
 5670   predicate(CompressedOops::shift() == 0);
 5671   match(Set dst mem);
 5672 
 5673   ins_cost(110);
 5674   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5675   ins_encode %{
 5676     __ leaq($dst$$Register, $mem$$Address);
 5677   %}
 5678   ins_pipe(ialu_reg_reg_fat);
 5679 %}
 5680 
 5681 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5682 %{
 5683   predicate(CompressedOops::shift() == 0);
 5684   match(Set dst mem);
 5685 
 5686   ins_cost(110);
 5687   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5688   ins_encode %{
 5689     __ leaq($dst$$Register, $mem$$Address);
 5690   %}
 5691   ins_pipe(ialu_reg_reg_fat);
 5692 %}
 5693 
 5694 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5695 %{
 5696   predicate(CompressedOops::shift() == 0);
 5697   match(Set dst mem);
 5698 
 5699   ins_cost(110);
 5700   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5701   ins_encode %{
 5702     __ leaq($dst$$Register, $mem$$Address);
 5703   %}
 5704   ins_pipe(ialu_reg_reg_fat);
 5705 %}
 5706 
 5707 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5708 %{
 5709   predicate(CompressedOops::shift() == 0);
 5710   match(Set dst mem);
 5711 
 5712   ins_cost(110);
 5713   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5714   ins_encode %{
 5715     __ leaq($dst$$Register, $mem$$Address);
 5716   %}
 5717   ins_pipe(ialu_reg_reg_fat);
 5718 %}
 5719 
 5720 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5721 %{
 5722   predicate(CompressedOops::shift() == 0);
 5723   match(Set dst mem);
 5724 
 5725   ins_cost(110);
 5726   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5727   ins_encode %{
 5728     __ leaq($dst$$Register, $mem$$Address);
 5729   %}
 5730   ins_pipe(ialu_reg_reg_fat);
 5731 %}
 5732 
 5733 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5734 %{
 5735   predicate(CompressedOops::shift() == 0);
 5736   match(Set dst mem);
 5737 
 5738   ins_cost(110);
 5739   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5740   ins_encode %{
 5741     __ leaq($dst$$Register, $mem$$Address);
 5742   %}
 5743   ins_pipe(ialu_reg_reg_fat);
 5744 %}
 5745 
 5746 instruct loadConI(rRegI dst, immI src)
 5747 %{
 5748   match(Set dst src);
 5749 
 5750   format %{ "movl    $dst, $src\t# int" %}
 5751   ins_encode %{
 5752     __ movl($dst$$Register, $src$$constant);
 5753   %}
 5754   ins_pipe(ialu_reg_fat); // XXX
 5755 %}
 5756 
 5757 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5758 %{
 5759   match(Set dst src);
 5760   effect(KILL cr);
 5761 
 5762   ins_cost(50);
 5763   format %{ "xorl    $dst, $dst\t# int" %}
 5764   ins_encode %{
 5765     __ xorl($dst$$Register, $dst$$Register);
 5766   %}
 5767   ins_pipe(ialu_reg);
 5768 %}
 5769 
 5770 instruct loadConL(rRegL dst, immL src)
 5771 %{
 5772   match(Set dst src);
 5773 
 5774   ins_cost(150);
 5775   format %{ "movq    $dst, $src\t# long" %}
 5776   ins_encode %{
 5777     __ mov64($dst$$Register, $src$$constant);
 5778   %}
 5779   ins_pipe(ialu_reg);
 5780 %}
 5781 
 5782 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5783 %{
 5784   match(Set dst src);
 5785   effect(KILL cr);
 5786 
 5787   ins_cost(50);
 5788   format %{ "xorl    $dst, $dst\t# long" %}
 5789   ins_encode %{
 5790     __ xorl($dst$$Register, $dst$$Register);
 5791   %}
 5792   ins_pipe(ialu_reg); // XXX
 5793 %}
 5794 
 5795 instruct loadConUL32(rRegL dst, immUL32 src)
 5796 %{
 5797   match(Set dst src);
 5798 
 5799   ins_cost(60);
 5800   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5801   ins_encode %{
 5802     __ movl($dst$$Register, $src$$constant);
 5803   %}
 5804   ins_pipe(ialu_reg);
 5805 %}
 5806 
 5807 instruct loadConL32(rRegL dst, immL32 src)
 5808 %{
 5809   match(Set dst src);
 5810 
 5811   ins_cost(70);
 5812   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5813   ins_encode %{
 5814     __ movq($dst$$Register, $src$$constant);
 5815   %}
 5816   ins_pipe(ialu_reg);
 5817 %}
 5818 
 5819 instruct loadConP(rRegP dst, immP con) %{
 5820   match(Set dst con);
 5821 
 5822   format %{ "movq    $dst, $con\t# ptr" %}
 5823   ins_encode %{
 5824     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5825   %}
 5826   ins_pipe(ialu_reg_fat); // XXX
 5827 %}
 5828 
 5829 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5830 %{
 5831   match(Set dst src);
 5832   effect(KILL cr);
 5833 
 5834   ins_cost(50);
 5835   format %{ "xorl    $dst, $dst\t# ptr" %}
 5836   ins_encode %{
 5837     __ xorl($dst$$Register, $dst$$Register);
 5838   %}
 5839   ins_pipe(ialu_reg);
 5840 %}
 5841 
 5842 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5843 %{
 5844   match(Set dst src);
 5845   effect(KILL cr);
 5846 
 5847   ins_cost(60);
 5848   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5849   ins_encode %{
 5850     __ movl($dst$$Register, $src$$constant);
 5851   %}
 5852   ins_pipe(ialu_reg);
 5853 %}
 5854 
 5855 instruct loadConF(regF dst, immF con) %{
 5856   match(Set dst con);
 5857   ins_cost(125);
 5858   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5859   ins_encode %{
 5860     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5861   %}
 5862   ins_pipe(pipe_slow);
 5863 %}
 5864 
 5865 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5866   match(Set dst src);
 5867   effect(KILL cr);
 5868   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5869   ins_encode %{
 5870     __ xorq($dst$$Register, $dst$$Register);
 5871   %}
 5872   ins_pipe(ialu_reg);
 5873 %}
 5874 
 5875 instruct loadConN(rRegN dst, immN src) %{
 5876   match(Set dst src);
 5877 
 5878   ins_cost(125);
 5879   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5880   ins_encode %{
 5881     address con = (address)$src$$constant;
 5882     if (con == NULL) {
 5883       ShouldNotReachHere();
 5884     } else {
 5885       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5886     }
 5887   %}
 5888   ins_pipe(ialu_reg_fat); // XXX
 5889 %}
 5890 
 5891 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5892   match(Set dst src);
 5893 
 5894   ins_cost(125);
 5895   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5896   ins_encode %{
 5897     address con = (address)$src$$constant;
 5898     if (con == NULL) {
 5899       ShouldNotReachHere();
 5900     } else {
 5901       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5902     }
 5903   %}
 5904   ins_pipe(ialu_reg_fat); // XXX
 5905 %}
 5906 
 5907 instruct loadConF0(regF dst, immF0 src)
 5908 %{
 5909   match(Set dst src);
 5910   ins_cost(100);
 5911 
 5912   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5913   ins_encode %{
 5914     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5915   %}
 5916   ins_pipe(pipe_slow);
 5917 %}
 5918 
 5919 // Use the same format since predicate() can not be used here.
 5920 instruct loadConD(regD dst, immD con) %{
 5921   match(Set dst con);
 5922   ins_cost(125);
 5923   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5924   ins_encode %{
 5925     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5926   %}
 5927   ins_pipe(pipe_slow);
 5928 %}
 5929 
 5930 instruct loadConD0(regD dst, immD0 src)
 5931 %{
 5932   match(Set dst src);
 5933   ins_cost(100);
 5934 
 5935   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5936   ins_encode %{
 5937     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5938   %}
 5939   ins_pipe(pipe_slow);
 5940 %}
 5941 
 5942 instruct loadSSI(rRegI dst, stackSlotI src)
 5943 %{
 5944   match(Set dst src);
 5945 
 5946   ins_cost(125);
 5947   format %{ "movl    $dst, $src\t# int stk" %}
 5948   opcode(0x8B);
 5949   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5950   ins_pipe(ialu_reg_mem);
 5951 %}
 5952 
 5953 instruct loadSSL(rRegL dst, stackSlotL src)
 5954 %{
 5955   match(Set dst src);
 5956 
 5957   ins_cost(125);
 5958   format %{ "movq    $dst, $src\t# long stk" %}
 5959   opcode(0x8B);
 5960   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5961   ins_pipe(ialu_reg_mem);
 5962 %}
 5963 
 5964 instruct loadSSP(rRegP dst, stackSlotP src)
 5965 %{
 5966   match(Set dst src);
 5967 
 5968   ins_cost(125);
 5969   format %{ "movq    $dst, $src\t# ptr stk" %}
 5970   opcode(0x8B);
 5971   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5972   ins_pipe(ialu_reg_mem);
 5973 %}
 5974 
 5975 instruct loadSSF(regF dst, stackSlotF src)
 5976 %{
 5977   match(Set dst src);
 5978 
 5979   ins_cost(125);
 5980   format %{ "movss   $dst, $src\t# float stk" %}
 5981   ins_encode %{
 5982     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5983   %}
 5984   ins_pipe(pipe_slow); // XXX
 5985 %}
 5986 
 5987 // Use the same format since predicate() can not be used here.
 5988 instruct loadSSD(regD dst, stackSlotD src)
 5989 %{
 5990   match(Set dst src);
 5991 
 5992   ins_cost(125);
 5993   format %{ "movsd   $dst, $src\t# double stk" %}
 5994   ins_encode  %{
 5995     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5996   %}
 5997   ins_pipe(pipe_slow); // XXX
 5998 %}
 5999 
 6000 // Prefetch instructions for allocation.
 6001 // Must be safe to execute with invalid address (cannot fault).
 6002 
 6003 instruct prefetchAlloc( memory mem ) %{
 6004   predicate(AllocatePrefetchInstr==3);
 6005   match(PrefetchAllocation mem);
 6006   ins_cost(125);
 6007 
 6008   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 6009   ins_encode %{
 6010     __ prefetchw($mem$$Address);
 6011   %}
 6012   ins_pipe(ialu_mem);
 6013 %}
 6014 
 6015 instruct prefetchAllocNTA( memory mem ) %{
 6016   predicate(AllocatePrefetchInstr==0);
 6017   match(PrefetchAllocation mem);
 6018   ins_cost(125);
 6019 
 6020   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 6021   ins_encode %{
 6022     __ prefetchnta($mem$$Address);
 6023   %}
 6024   ins_pipe(ialu_mem);
 6025 %}
 6026 
 6027 instruct prefetchAllocT0( memory mem ) %{
 6028   predicate(AllocatePrefetchInstr==1);
 6029   match(PrefetchAllocation mem);
 6030   ins_cost(125);
 6031 
 6032   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 6033   ins_encode %{
 6034     __ prefetcht0($mem$$Address);
 6035   %}
 6036   ins_pipe(ialu_mem);
 6037 %}
 6038 
 6039 instruct prefetchAllocT2( memory mem ) %{
 6040   predicate(AllocatePrefetchInstr==2);
 6041   match(PrefetchAllocation mem);
 6042   ins_cost(125);
 6043 
 6044   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6045   ins_encode %{
 6046     __ prefetcht2($mem$$Address);
 6047   %}
 6048   ins_pipe(ialu_mem);
 6049 %}
 6050 
 6051 //----------Store Instructions-------------------------------------------------
 6052 
 6053 // Store Byte
 6054 instruct storeB(memory mem, rRegI src)
 6055 %{
 6056   match(Set mem (StoreB mem src));
 6057 
 6058   ins_cost(125); // XXX
 6059   format %{ "movb    $mem, $src\t# byte" %}
 6060   ins_encode %{
 6061     __ movb($mem$$Address, $src$$Register);
 6062   %}
 6063   ins_pipe(ialu_mem_reg);
 6064 %}
 6065 
 6066 // Store Char/Short
 6067 instruct storeC(memory mem, rRegI src)
 6068 %{
 6069   match(Set mem (StoreC mem src));
 6070 
 6071   ins_cost(125); // XXX
 6072   format %{ "movw    $mem, $src\t# char/short" %}
 6073   ins_encode %{
 6074     __ movw($mem$$Address, $src$$Register);
 6075   %}
 6076   ins_pipe(ialu_mem_reg);
 6077 %}
 6078 
 6079 // Store Integer
 6080 instruct storeI(memory mem, rRegI src)
 6081 %{
 6082   match(Set mem (StoreI mem src));
 6083 
 6084   ins_cost(125); // XXX
 6085   format %{ "movl    $mem, $src\t# int" %}
 6086   ins_encode %{
 6087     __ movl($mem$$Address, $src$$Register);
 6088   %}
 6089   ins_pipe(ialu_mem_reg);
 6090 %}
 6091 
 6092 // Store Long
 6093 instruct storeL(memory mem, rRegL src)
 6094 %{
 6095   match(Set mem (StoreL mem src));
 6096 
 6097   ins_cost(125); // XXX
 6098   format %{ "movq    $mem, $src\t# long" %}
 6099   ins_encode %{
 6100     __ movq($mem$$Address, $src$$Register);
 6101   %}
 6102   ins_pipe(ialu_mem_reg); // XXX
 6103 %}
 6104 
 6105 // Store Pointer
 6106 instruct storeP(memory mem, any_RegP src)
 6107 %{
 6108   predicate(n->as_Store()->barrier_data() == 0);
 6109   match(Set mem (StoreP mem src));
 6110 
 6111   ins_cost(125); // XXX
 6112   format %{ "movq    $mem, $src\t# ptr" %}
 6113   ins_encode %{
 6114     __ movq($mem$$Address, $src$$Register);
 6115   %}
 6116   ins_pipe(ialu_mem_reg);
 6117 %}
 6118 
 6119 instruct storeImmP0(memory mem, immP0 zero)
 6120 %{
 6121   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && n->as_Store()->barrier_data() == 0);
 6122   match(Set mem (StoreP mem zero));
 6123 
 6124   ins_cost(125); // XXX
 6125   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6126   ins_encode %{
 6127     __ movq($mem$$Address, r12);
 6128   %}
 6129   ins_pipe(ialu_mem_reg);
 6130 %}
 6131 
 6132 // Store NULL Pointer, mark word, or other simple pointer constant.
 6133 instruct storeImmP(memory mem, immP31 src)
 6134 %{
 6135   predicate(n->as_Store()->barrier_data() == 0);
 6136   match(Set mem (StoreP mem src));
 6137 
 6138   ins_cost(150); // XXX
 6139   format %{ "movq    $mem, $src\t# ptr" %}
 6140   ins_encode %{
 6141     __ movq($mem$$Address, $src$$constant);
 6142   %}
 6143   ins_pipe(ialu_mem_imm);
 6144 %}
 6145 
 6146 // Store Compressed Pointer
 6147 instruct storeN(memory mem, rRegN src)
 6148 %{
 6149   match(Set mem (StoreN mem src));
 6150 
 6151   ins_cost(125); // XXX
 6152   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6153   ins_encode %{
 6154     __ movl($mem$$Address, $src$$Register);
 6155   %}
 6156   ins_pipe(ialu_mem_reg);
 6157 %}
 6158 
 6159 instruct storeNKlass(memory mem, rRegN src)
 6160 %{
 6161   match(Set mem (StoreNKlass mem src));
 6162 
 6163   ins_cost(125); // XXX
 6164   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6165   ins_encode %{
 6166     __ movl($mem$$Address, $src$$Register);
 6167   %}
 6168   ins_pipe(ialu_mem_reg);
 6169 %}
 6170 
 6171 instruct storeImmN0(memory mem, immN0 zero)
 6172 %{
 6173   predicate(CompressedOops::base() == NULL);
 6174   match(Set mem (StoreN mem zero));
 6175 
 6176   ins_cost(125); // XXX
 6177   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6178   ins_encode %{
 6179     __ movl($mem$$Address, r12);
 6180   %}
 6181   ins_pipe(ialu_mem_reg);
 6182 %}
 6183 
 6184 instruct storeImmN(memory mem, immN src)
 6185 %{
 6186   match(Set mem (StoreN mem src));
 6187 
 6188   ins_cost(150); // XXX
 6189   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6190   ins_encode %{
 6191     address con = (address)$src$$constant;
 6192     if (con == NULL) {
 6193       __ movl($mem$$Address, 0);
 6194     } else {
 6195       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6196     }
 6197   %}
 6198   ins_pipe(ialu_mem_imm);
 6199 %}
 6200 
 6201 instruct storeImmNKlass(memory mem, immNKlass src)
 6202 %{
 6203   match(Set mem (StoreNKlass mem src));
 6204 
 6205   ins_cost(150); // XXX
 6206   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6207   ins_encode %{
 6208     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6209   %}
 6210   ins_pipe(ialu_mem_imm);
 6211 %}
 6212 
 6213 // Store Integer Immediate
 6214 instruct storeImmI0(memory mem, immI_0 zero)
 6215 %{
 6216   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6217   match(Set mem (StoreI mem zero));
 6218 
 6219   ins_cost(125); // XXX
 6220   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6221   ins_encode %{
 6222     __ movl($mem$$Address, r12);
 6223   %}
 6224   ins_pipe(ialu_mem_reg);
 6225 %}
 6226 
 6227 instruct storeImmI(memory mem, immI src)
 6228 %{
 6229   match(Set mem (StoreI mem src));
 6230 
 6231   ins_cost(150);
 6232   format %{ "movl    $mem, $src\t# int" %}
 6233   ins_encode %{
 6234     __ movl($mem$$Address, $src$$constant);
 6235   %}
 6236   ins_pipe(ialu_mem_imm);
 6237 %}
 6238 
 6239 // Store Long Immediate
 6240 instruct storeImmL0(memory mem, immL0 zero)
 6241 %{
 6242   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6243   match(Set mem (StoreL mem zero));
 6244 
 6245   ins_cost(125); // XXX
 6246   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6247   ins_encode %{
 6248     __ movq($mem$$Address, r12);
 6249   %}
 6250   ins_pipe(ialu_mem_reg);
 6251 %}
 6252 
 6253 instruct storeImmL(memory mem, immL32 src)
 6254 %{
 6255   match(Set mem (StoreL mem src));
 6256 
 6257   ins_cost(150);
 6258   format %{ "movq    $mem, $src\t# long" %}
 6259   ins_encode %{
 6260     __ movq($mem$$Address, $src$$constant);
 6261   %}
 6262   ins_pipe(ialu_mem_imm);
 6263 %}
 6264 
 6265 // Store Short/Char Immediate
 6266 instruct storeImmC0(memory mem, immI_0 zero)
 6267 %{
 6268   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6269   match(Set mem (StoreC mem zero));
 6270 
 6271   ins_cost(125); // XXX
 6272   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6273   ins_encode %{
 6274     __ movw($mem$$Address, r12);
 6275   %}
 6276   ins_pipe(ialu_mem_reg);
 6277 %}
 6278 
 6279 instruct storeImmI16(memory mem, immI16 src)
 6280 %{
 6281   predicate(UseStoreImmI16);
 6282   match(Set mem (StoreC mem src));
 6283 
 6284   ins_cost(150);
 6285   format %{ "movw    $mem, $src\t# short/char" %}
 6286   ins_encode %{
 6287     __ movw($mem$$Address, $src$$constant);
 6288   %}
 6289   ins_pipe(ialu_mem_imm);
 6290 %}
 6291 
 6292 // Store Byte Immediate
 6293 instruct storeImmB0(memory mem, immI_0 zero)
 6294 %{
 6295   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6296   match(Set mem (StoreB mem zero));
 6297 
 6298   ins_cost(125); // XXX
 6299   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6300   ins_encode %{
 6301     __ movb($mem$$Address, r12);
 6302   %}
 6303   ins_pipe(ialu_mem_reg);
 6304 %}
 6305 
 6306 instruct storeImmB(memory mem, immI8 src)
 6307 %{
 6308   match(Set mem (StoreB mem src));
 6309 
 6310   ins_cost(150); // XXX
 6311   format %{ "movb    $mem, $src\t# byte" %}
 6312   ins_encode %{
 6313     __ movb($mem$$Address, $src$$constant);
 6314   %}
 6315   ins_pipe(ialu_mem_imm);
 6316 %}
 6317 
 6318 // Store CMS card-mark Immediate
 6319 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6320 %{
 6321   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6322   match(Set mem (StoreCM mem zero));
 6323 
 6324   ins_cost(125); // XXX
 6325   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6326   ins_encode %{
 6327     __ movb($mem$$Address, r12);
 6328   %}
 6329   ins_pipe(ialu_mem_reg);
 6330 %}
 6331 
 6332 instruct storeImmCM0(memory mem, immI_0 src)
 6333 %{
 6334   match(Set mem (StoreCM mem src));
 6335 
 6336   ins_cost(150); // XXX
 6337   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6338   ins_encode %{
 6339     __ movb($mem$$Address, $src$$constant);
 6340   %}
 6341   ins_pipe(ialu_mem_imm);
 6342 %}
 6343 
 6344 // Store Float
 6345 instruct storeF(memory mem, regF src)
 6346 %{
 6347   match(Set mem (StoreF mem src));
 6348 
 6349   ins_cost(95); // XXX
 6350   format %{ "movss   $mem, $src\t# float" %}
 6351   ins_encode %{
 6352     __ movflt($mem$$Address, $src$$XMMRegister);
 6353   %}
 6354   ins_pipe(pipe_slow); // XXX
 6355 %}
 6356 
 6357 // Store immediate Float value (it is faster than store from XMM register)
 6358 instruct storeF0(memory mem, immF0 zero)
 6359 %{
 6360   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6361   match(Set mem (StoreF mem zero));
 6362 
 6363   ins_cost(25); // XXX
 6364   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6365   ins_encode %{
 6366     __ movl($mem$$Address, r12);
 6367   %}
 6368   ins_pipe(ialu_mem_reg);
 6369 %}
 6370 
 6371 instruct storeF_imm(memory mem, immF src)
 6372 %{
 6373   match(Set mem (StoreF mem src));
 6374 
 6375   ins_cost(50);
 6376   format %{ "movl    $mem, $src\t# float" %}
 6377   ins_encode %{
 6378     __ movl($mem$$Address, jint_cast($src$$constant));
 6379   %}
 6380   ins_pipe(ialu_mem_imm);
 6381 %}
 6382 
 6383 // Store Double
 6384 instruct storeD(memory mem, regD src)
 6385 %{
 6386   match(Set mem (StoreD mem src));
 6387 
 6388   ins_cost(95); // XXX
 6389   format %{ "movsd   $mem, $src\t# double" %}
 6390   ins_encode %{
 6391     __ movdbl($mem$$Address, $src$$XMMRegister);
 6392   %}
 6393   ins_pipe(pipe_slow); // XXX
 6394 %}
 6395 
 6396 // Store immediate double 0.0 (it is faster than store from XMM register)
 6397 instruct storeD0_imm(memory mem, immD0 src)
 6398 %{
 6399   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6400   match(Set mem (StoreD mem src));
 6401 
 6402   ins_cost(50);
 6403   format %{ "movq    $mem, $src\t# double 0." %}
 6404   ins_encode %{
 6405     __ movq($mem$$Address, $src$$constant);
 6406   %}
 6407   ins_pipe(ialu_mem_imm);
 6408 %}
 6409 
 6410 instruct storeD0(memory mem, immD0 zero)
 6411 %{
 6412   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6413   match(Set mem (StoreD mem zero));
 6414 
 6415   ins_cost(25); // XXX
 6416   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6417   ins_encode %{
 6418     __ movq($mem$$Address, r12);
 6419   %}
 6420   ins_pipe(ialu_mem_reg);
 6421 %}
 6422 
 6423 instruct storeSSI(stackSlotI dst, rRegI src)
 6424 %{
 6425   match(Set dst src);
 6426 
 6427   ins_cost(100);
 6428   format %{ "movl    $dst, $src\t# int stk" %}
 6429   opcode(0x89);
 6430   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6431   ins_pipe( ialu_mem_reg );
 6432 %}
 6433 
 6434 instruct storeSSL(stackSlotL dst, rRegL src)
 6435 %{
 6436   match(Set dst src);
 6437 
 6438   ins_cost(100);
 6439   format %{ "movq    $dst, $src\t# long stk" %}
 6440   opcode(0x89);
 6441   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6442   ins_pipe(ialu_mem_reg);
 6443 %}
 6444 
 6445 instruct storeSSP(stackSlotP dst, rRegP src)
 6446 %{
 6447   match(Set dst src);
 6448 
 6449   ins_cost(100);
 6450   format %{ "movq    $dst, $src\t# ptr stk" %}
 6451   opcode(0x89);
 6452   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6453   ins_pipe(ialu_mem_reg);
 6454 %}
 6455 
 6456 instruct storeSSF(stackSlotF dst, regF src)
 6457 %{
 6458   match(Set dst src);
 6459 
 6460   ins_cost(95); // XXX
 6461   format %{ "movss   $dst, $src\t# float stk" %}
 6462   ins_encode %{
 6463     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6464   %}
 6465   ins_pipe(pipe_slow); // XXX
 6466 %}
 6467 
 6468 instruct storeSSD(stackSlotD dst, regD src)
 6469 %{
 6470   match(Set dst src);
 6471 
 6472   ins_cost(95); // XXX
 6473   format %{ "movsd   $dst, $src\t# double stk" %}
 6474   ins_encode %{
 6475     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6476   %}
 6477   ins_pipe(pipe_slow); // XXX
 6478 %}
 6479 
 6480 instruct cacheWB(indirect addr)
 6481 %{
 6482   predicate(VM_Version::supports_data_cache_line_flush());
 6483   match(CacheWB addr);
 6484 
 6485   ins_cost(100);
 6486   format %{"cache wb $addr" %}
 6487   ins_encode %{
 6488     assert($addr->index_position() < 0, "should be");
 6489     assert($addr$$disp == 0, "should be");
 6490     __ cache_wb(Address($addr$$base$$Register, 0));
 6491   %}
 6492   ins_pipe(pipe_slow); // XXX
 6493 %}
 6494 
 6495 instruct cacheWBPreSync()
 6496 %{
 6497   predicate(VM_Version::supports_data_cache_line_flush());
 6498   match(CacheWBPreSync);
 6499 
 6500   ins_cost(100);
 6501   format %{"cache wb presync" %}
 6502   ins_encode %{
 6503     __ cache_wbsync(true);
 6504   %}
 6505   ins_pipe(pipe_slow); // XXX
 6506 %}
 6507 
 6508 instruct cacheWBPostSync()
 6509 %{
 6510   predicate(VM_Version::supports_data_cache_line_flush());
 6511   match(CacheWBPostSync);
 6512 
 6513   ins_cost(100);
 6514   format %{"cache wb postsync" %}
 6515   ins_encode %{
 6516     __ cache_wbsync(false);
 6517   %}
 6518   ins_pipe(pipe_slow); // XXX
 6519 %}
 6520 
 6521 //----------BSWAP Instructions-------------------------------------------------
 6522 instruct bytes_reverse_int(rRegI dst) %{
 6523   match(Set dst (ReverseBytesI dst));
 6524 
 6525   format %{ "bswapl  $dst" %}
 6526   ins_encode %{
 6527     __ bswapl($dst$$Register);
 6528   %}
 6529   ins_pipe( ialu_reg );
 6530 %}
 6531 
 6532 instruct bytes_reverse_long(rRegL dst) %{
 6533   match(Set dst (ReverseBytesL dst));
 6534 
 6535   format %{ "bswapq  $dst" %}
 6536   ins_encode %{
 6537     __ bswapq($dst$$Register);
 6538   %}
 6539   ins_pipe( ialu_reg);
 6540 %}
 6541 
 6542 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6543   match(Set dst (ReverseBytesUS dst));
 6544   effect(KILL cr);
 6545 
 6546   format %{ "bswapl  $dst\n\t"
 6547             "shrl    $dst,16\n\t" %}
 6548   ins_encode %{
 6549     __ bswapl($dst$$Register);
 6550     __ shrl($dst$$Register, 16);
 6551   %}
 6552   ins_pipe( ialu_reg );
 6553 %}
 6554 
 6555 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6556   match(Set dst (ReverseBytesS dst));
 6557   effect(KILL cr);
 6558 
 6559   format %{ "bswapl  $dst\n\t"
 6560             "sar     $dst,16\n\t" %}
 6561   ins_encode %{
 6562     __ bswapl($dst$$Register);
 6563     __ sarl($dst$$Register, 16);
 6564   %}
 6565   ins_pipe( ialu_reg );
 6566 %}
 6567 
 6568 //---------- Zeros Count Instructions ------------------------------------------
 6569 
 6570 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6571   predicate(UseCountLeadingZerosInstruction);
 6572   match(Set dst (CountLeadingZerosI src));
 6573   effect(KILL cr);
 6574 
 6575   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6576   ins_encode %{
 6577     __ lzcntl($dst$$Register, $src$$Register);
 6578   %}
 6579   ins_pipe(ialu_reg);
 6580 %}
 6581 
 6582 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6583   predicate(UseCountLeadingZerosInstruction);
 6584   match(Set dst (CountLeadingZerosI (LoadI src)));
 6585   effect(KILL cr);
 6586   ins_cost(175);
 6587   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6588   ins_encode %{
 6589     __ lzcntl($dst$$Register, $src$$Address);
 6590   %}
 6591   ins_pipe(ialu_reg_mem);
 6592 %}
 6593 
 6594 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6595   predicate(!UseCountLeadingZerosInstruction);
 6596   match(Set dst (CountLeadingZerosI src));
 6597   effect(KILL cr);
 6598 
 6599   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6600             "jnz     skip\n\t"
 6601             "movl    $dst, -1\n"
 6602       "skip:\n\t"
 6603             "negl    $dst\n\t"
 6604             "addl    $dst, 31" %}
 6605   ins_encode %{
 6606     Register Rdst = $dst$$Register;
 6607     Register Rsrc = $src$$Register;
 6608     Label skip;
 6609     __ bsrl(Rdst, Rsrc);
 6610     __ jccb(Assembler::notZero, skip);
 6611     __ movl(Rdst, -1);
 6612     __ bind(skip);
 6613     __ negl(Rdst);
 6614     __ addl(Rdst, BitsPerInt - 1);
 6615   %}
 6616   ins_pipe(ialu_reg);
 6617 %}
 6618 
 6619 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6620   predicate(UseCountLeadingZerosInstruction);
 6621   match(Set dst (CountLeadingZerosL src));
 6622   effect(KILL cr);
 6623 
 6624   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6625   ins_encode %{
 6626     __ lzcntq($dst$$Register, $src$$Register);
 6627   %}
 6628   ins_pipe(ialu_reg);
 6629 %}
 6630 
 6631 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6632   predicate(UseCountLeadingZerosInstruction);
 6633   match(Set dst (CountLeadingZerosL (LoadL src)));
 6634   effect(KILL cr);
 6635   ins_cost(175);
 6636   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6637   ins_encode %{
 6638     __ lzcntq($dst$$Register, $src$$Address);
 6639   %}
 6640   ins_pipe(ialu_reg_mem);
 6641 %}
 6642 
 6643 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6644   predicate(!UseCountLeadingZerosInstruction);
 6645   match(Set dst (CountLeadingZerosL src));
 6646   effect(KILL cr);
 6647 
 6648   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6649             "jnz     skip\n\t"
 6650             "movl    $dst, -1\n"
 6651       "skip:\n\t"
 6652             "negl    $dst\n\t"
 6653             "addl    $dst, 63" %}
 6654   ins_encode %{
 6655     Register Rdst = $dst$$Register;
 6656     Register Rsrc = $src$$Register;
 6657     Label skip;
 6658     __ bsrq(Rdst, Rsrc);
 6659     __ jccb(Assembler::notZero, skip);
 6660     __ movl(Rdst, -1);
 6661     __ bind(skip);
 6662     __ negl(Rdst);
 6663     __ addl(Rdst, BitsPerLong - 1);
 6664   %}
 6665   ins_pipe(ialu_reg);
 6666 %}
 6667 
 6668 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6669   predicate(UseCountTrailingZerosInstruction);
 6670   match(Set dst (CountTrailingZerosI src));
 6671   effect(KILL cr);
 6672 
 6673   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6674   ins_encode %{
 6675     __ tzcntl($dst$$Register, $src$$Register);
 6676   %}
 6677   ins_pipe(ialu_reg);
 6678 %}
 6679 
 6680 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6681   predicate(UseCountTrailingZerosInstruction);
 6682   match(Set dst (CountTrailingZerosI (LoadI src)));
 6683   effect(KILL cr);
 6684   ins_cost(175);
 6685   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6686   ins_encode %{
 6687     __ tzcntl($dst$$Register, $src$$Address);
 6688   %}
 6689   ins_pipe(ialu_reg_mem);
 6690 %}
 6691 
 6692 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6693   predicate(!UseCountTrailingZerosInstruction);
 6694   match(Set dst (CountTrailingZerosI src));
 6695   effect(KILL cr);
 6696 
 6697   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6698             "jnz     done\n\t"
 6699             "movl    $dst, 32\n"
 6700       "done:" %}
 6701   ins_encode %{
 6702     Register Rdst = $dst$$Register;
 6703     Label done;
 6704     __ bsfl(Rdst, $src$$Register);
 6705     __ jccb(Assembler::notZero, done);
 6706     __ movl(Rdst, BitsPerInt);
 6707     __ bind(done);
 6708   %}
 6709   ins_pipe(ialu_reg);
 6710 %}
 6711 
 6712 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6713   predicate(UseCountTrailingZerosInstruction);
 6714   match(Set dst (CountTrailingZerosL src));
 6715   effect(KILL cr);
 6716 
 6717   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6718   ins_encode %{
 6719     __ tzcntq($dst$$Register, $src$$Register);
 6720   %}
 6721   ins_pipe(ialu_reg);
 6722 %}
 6723 
 6724 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6725   predicate(UseCountTrailingZerosInstruction);
 6726   match(Set dst (CountTrailingZerosL (LoadL src)));
 6727   effect(KILL cr);
 6728   ins_cost(175);
 6729   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6730   ins_encode %{
 6731     __ tzcntq($dst$$Register, $src$$Address);
 6732   %}
 6733   ins_pipe(ialu_reg_mem);
 6734 %}
 6735 
 6736 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6737   predicate(!UseCountTrailingZerosInstruction);
 6738   match(Set dst (CountTrailingZerosL src));
 6739   effect(KILL cr);
 6740 
 6741   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6742             "jnz     done\n\t"
 6743             "movl    $dst, 64\n"
 6744       "done:" %}
 6745   ins_encode %{
 6746     Register Rdst = $dst$$Register;
 6747     Label done;
 6748     __ bsfq(Rdst, $src$$Register);
 6749     __ jccb(Assembler::notZero, done);
 6750     __ movl(Rdst, BitsPerLong);
 6751     __ bind(done);
 6752   %}
 6753   ins_pipe(ialu_reg);
 6754 %}
 6755 
 6756 //--------------- Reverse Operation Instructions ----------------
 6757 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 6758   predicate(!VM_Version::supports_gfni());
 6759   match(Set dst (ReverseI src));
 6760   effect(TEMP dst, TEMP rtmp, KILL cr);
 6761   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 6762   ins_encode %{
 6763     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 6764   %}
 6765   ins_pipe( ialu_reg );
 6766 %}
 6767 
 6768 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6769   predicate(VM_Version::supports_gfni());
 6770   match(Set dst (ReverseI src));
 6771   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6772   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6773   ins_encode %{
 6774     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 6775   %}
 6776   ins_pipe( ialu_reg );
 6777 %}
 6778 
 6779 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 6780   predicate(!VM_Version::supports_gfni());
 6781   match(Set dst (ReverseL src));
 6782   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 6783   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 6784   ins_encode %{
 6785     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 6786   %}
 6787   ins_pipe( ialu_reg );
 6788 %}
 6789 
 6790 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6791   predicate(VM_Version::supports_gfni());
 6792   match(Set dst (ReverseL src));
 6793   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6794   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6795   ins_encode %{
 6796     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 6797   %}
 6798   ins_pipe( ialu_reg );
 6799 %}
 6800 
 6801 //---------- Population Count Instructions -------------------------------------
 6802 
 6803 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6804   predicate(UsePopCountInstruction);
 6805   match(Set dst (PopCountI src));
 6806   effect(KILL cr);
 6807 
 6808   format %{ "popcnt  $dst, $src" %}
 6809   ins_encode %{
 6810     __ popcntl($dst$$Register, $src$$Register);
 6811   %}
 6812   ins_pipe(ialu_reg);
 6813 %}
 6814 
 6815 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6816   predicate(UsePopCountInstruction);
 6817   match(Set dst (PopCountI (LoadI mem)));
 6818   effect(KILL cr);
 6819 
 6820   format %{ "popcnt  $dst, $mem" %}
 6821   ins_encode %{
 6822     __ popcntl($dst$$Register, $mem$$Address);
 6823   %}
 6824   ins_pipe(ialu_reg);
 6825 %}
 6826 
 6827 // Note: Long.bitCount(long) returns an int.
 6828 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6829   predicate(UsePopCountInstruction);
 6830   match(Set dst (PopCountL src));
 6831   effect(KILL cr);
 6832 
 6833   format %{ "popcnt  $dst, $src" %}
 6834   ins_encode %{
 6835     __ popcntq($dst$$Register, $src$$Register);
 6836   %}
 6837   ins_pipe(ialu_reg);
 6838 %}
 6839 
 6840 // Note: Long.bitCount(long) returns an int.
 6841 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6842   predicate(UsePopCountInstruction);
 6843   match(Set dst (PopCountL (LoadL mem)));
 6844   effect(KILL cr);
 6845 
 6846   format %{ "popcnt  $dst, $mem" %}
 6847   ins_encode %{
 6848     __ popcntq($dst$$Register, $mem$$Address);
 6849   %}
 6850   ins_pipe(ialu_reg);
 6851 %}
 6852 
 6853 
 6854 //----------MemBar Instructions-----------------------------------------------
 6855 // Memory barrier flavors
 6856 
 6857 instruct membar_acquire()
 6858 %{
 6859   match(MemBarAcquire);
 6860   match(LoadFence);
 6861   ins_cost(0);
 6862 
 6863   size(0);
 6864   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6865   ins_encode();
 6866   ins_pipe(empty);
 6867 %}
 6868 
 6869 instruct membar_acquire_lock()
 6870 %{
 6871   match(MemBarAcquireLock);
 6872   ins_cost(0);
 6873 
 6874   size(0);
 6875   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6876   ins_encode();
 6877   ins_pipe(empty);
 6878 %}
 6879 
 6880 instruct membar_release()
 6881 %{
 6882   match(MemBarRelease);
 6883   match(StoreFence);
 6884   ins_cost(0);
 6885 
 6886   size(0);
 6887   format %{ "MEMBAR-release ! (empty encoding)" %}
 6888   ins_encode();
 6889   ins_pipe(empty);
 6890 %}
 6891 
 6892 instruct membar_release_lock()
 6893 %{
 6894   match(MemBarReleaseLock);
 6895   ins_cost(0);
 6896 
 6897   size(0);
 6898   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6899   ins_encode();
 6900   ins_pipe(empty);
 6901 %}
 6902 
 6903 instruct membar_volatile(rFlagsReg cr) %{
 6904   match(MemBarVolatile);
 6905   effect(KILL cr);
 6906   ins_cost(400);
 6907 
 6908   format %{
 6909     $$template
 6910     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6911   %}
 6912   ins_encode %{
 6913     __ membar(Assembler::StoreLoad);
 6914   %}
 6915   ins_pipe(pipe_slow);
 6916 %}
 6917 
 6918 instruct unnecessary_membar_volatile()
 6919 %{
 6920   match(MemBarVolatile);
 6921   predicate(Matcher::post_store_load_barrier(n));
 6922   ins_cost(0);
 6923 
 6924   size(0);
 6925   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6926   ins_encode();
 6927   ins_pipe(empty);
 6928 %}
 6929 
 6930 instruct membar_storestore() %{
 6931   match(MemBarStoreStore);
 6932   match(StoreStoreFence);
 6933   ins_cost(0);
 6934 
 6935   size(0);
 6936   format %{ "MEMBAR-storestore (empty encoding)" %}
 6937   ins_encode( );
 6938   ins_pipe(empty);
 6939 %}
 6940 
 6941 //----------Move Instructions--------------------------------------------------
 6942 
 6943 instruct castX2P(rRegP dst, rRegL src)
 6944 %{
 6945   match(Set dst (CastX2P src));
 6946 
 6947   format %{ "movq    $dst, $src\t# long->ptr" %}
 6948   ins_encode %{
 6949     if ($dst$$reg != $src$$reg) {
 6950       __ movptr($dst$$Register, $src$$Register);
 6951     }
 6952   %}
 6953   ins_pipe(ialu_reg_reg); // XXX
 6954 %}
 6955 
 6956 instruct castP2X(rRegL dst, rRegP src)
 6957 %{
 6958   match(Set dst (CastP2X src));
 6959 
 6960   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6961   ins_encode %{
 6962     if ($dst$$reg != $src$$reg) {
 6963       __ movptr($dst$$Register, $src$$Register);
 6964     }
 6965   %}
 6966   ins_pipe(ialu_reg_reg); // XXX
 6967 %}
 6968 
 6969 // Convert oop into int for vectors alignment masking
 6970 instruct convP2I(rRegI dst, rRegP src)
 6971 %{
 6972   match(Set dst (ConvL2I (CastP2X src)));
 6973 
 6974   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6975   ins_encode %{
 6976     __ movl($dst$$Register, $src$$Register);
 6977   %}
 6978   ins_pipe(ialu_reg_reg); // XXX
 6979 %}
 6980 
 6981 // Convert compressed oop into int for vectors alignment masking
 6982 // in case of 32bit oops (heap < 4Gb).
 6983 instruct convN2I(rRegI dst, rRegN src)
 6984 %{
 6985   predicate(CompressedOops::shift() == 0);
 6986   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6987 
 6988   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6989   ins_encode %{
 6990     __ movl($dst$$Register, $src$$Register);
 6991   %}
 6992   ins_pipe(ialu_reg_reg); // XXX
 6993 %}
 6994 
 6995 // Convert oop pointer into compressed form
 6996 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6997   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6998   match(Set dst (EncodeP src));
 6999   effect(KILL cr);
 7000   format %{ "encode_heap_oop $dst,$src" %}
 7001   ins_encode %{
 7002     Register s = $src$$Register;
 7003     Register d = $dst$$Register;
 7004     if (s != d) {
 7005       __ movq(d, s);
 7006     }
 7007     __ encode_heap_oop(d);
 7008   %}
 7009   ins_pipe(ialu_reg_long);
 7010 %}
 7011 
 7012 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 7013   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 7014   match(Set dst (EncodeP src));
 7015   effect(KILL cr);
 7016   format %{ "encode_heap_oop_not_null $dst,$src" %}
 7017   ins_encode %{
 7018     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 7019   %}
 7020   ins_pipe(ialu_reg_long);
 7021 %}
 7022 
 7023 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 7024   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 7025             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 7026   match(Set dst (DecodeN src));
 7027   effect(KILL cr);
 7028   format %{ "decode_heap_oop $dst,$src" %}
 7029   ins_encode %{
 7030     Register s = $src$$Register;
 7031     Register d = $dst$$Register;
 7032     if (s != d) {
 7033       __ movq(d, s);
 7034     }
 7035     __ decode_heap_oop(d);
 7036   %}
 7037   ins_pipe(ialu_reg_long);
 7038 %}
 7039 
 7040 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7041   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 7042             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 7043   match(Set dst (DecodeN src));
 7044   effect(KILL cr);
 7045   format %{ "decode_heap_oop_not_null $dst,$src" %}
 7046   ins_encode %{
 7047     Register s = $src$$Register;
 7048     Register d = $dst$$Register;
 7049     if (s != d) {
 7050       __ decode_heap_oop_not_null(d, s);
 7051     } else {
 7052       __ decode_heap_oop_not_null(d);
 7053     }
 7054   %}
 7055   ins_pipe(ialu_reg_long);
 7056 %}
 7057 
 7058 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 7059   match(Set dst (EncodePKlass src));
 7060   effect(TEMP dst, KILL cr);
 7061   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 7062   ins_encode %{
 7063     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7064   %}
 7065   ins_pipe(ialu_reg_long);
 7066 %}
 7067 
 7068 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7069   match(Set dst (DecodeNKlass src));
 7070   effect(TEMP dst, KILL cr);
 7071   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 7072   ins_encode %{
 7073     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7074   %}
 7075   ins_pipe(ialu_reg_long);
 7076 %}
 7077 
 7078 //----------Conditional Move---------------------------------------------------
 7079 // Jump
 7080 // dummy instruction for generating temp registers
 7081 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 7082   match(Jump (LShiftL switch_val shift));
 7083   ins_cost(350);
 7084   predicate(false);
 7085   effect(TEMP dest);
 7086 
 7087   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7088             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 7089   ins_encode %{
 7090     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7091     // to do that and the compiler is using that register as one it can allocate.
 7092     // So we build it all by hand.
 7093     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 7094     // ArrayAddress dispatch(table, index);
 7095     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 7096     __ lea($dest$$Register, $constantaddress);
 7097     __ jmp(dispatch);
 7098   %}
 7099   ins_pipe(pipe_jmp);
 7100 %}
 7101 
 7102 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 7103   match(Jump (AddL (LShiftL switch_val shift) offset));
 7104   ins_cost(350);
 7105   effect(TEMP dest);
 7106 
 7107   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7108             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 7109   ins_encode %{
 7110     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7111     // to do that and the compiler is using that register as one it can allocate.
 7112     // So we build it all by hand.
 7113     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7114     // ArrayAddress dispatch(table, index);
 7115     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7116     __ lea($dest$$Register, $constantaddress);
 7117     __ jmp(dispatch);
 7118   %}
 7119   ins_pipe(pipe_jmp);
 7120 %}
 7121 
 7122 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 7123   match(Jump switch_val);
 7124   ins_cost(350);
 7125   effect(TEMP dest);
 7126 
 7127   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7128             "jmp     [$dest + $switch_val]\n\t" %}
 7129   ins_encode %{
 7130     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7131     // to do that and the compiler is using that register as one it can allocate.
 7132     // So we build it all by hand.
 7133     // Address index(noreg, switch_reg, Address::times_1);
 7134     // ArrayAddress dispatch(table, index);
 7135     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7136     __ lea($dest$$Register, $constantaddress);
 7137     __ jmp(dispatch);
 7138   %}
 7139   ins_pipe(pipe_jmp);
 7140 %}
 7141 
 7142 // Conditional move
 7143 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7144 %{
 7145   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7146   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7147 
 7148   ins_cost(100); // XXX
 7149   format %{ "setbn$cop $dst\t# signed, int" %}
 7150   ins_encode %{
 7151     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7152     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7153   %}
 7154   ins_pipe(ialu_reg);
 7155 %}
 7156 
 7157 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7158 %{
 7159   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7160 
 7161   ins_cost(200); // XXX
 7162   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7163   ins_encode %{
 7164     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7165   %}
 7166   ins_pipe(pipe_cmov_reg);
 7167 %}
 7168 
 7169 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7170 %{
 7171   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7172   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7173 
 7174   ins_cost(100); // XXX
 7175   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7176   ins_encode %{
 7177     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7178     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7179   %}
 7180   ins_pipe(ialu_reg);
 7181 %}
 7182 
 7183 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7184   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7185 
 7186   ins_cost(200); // XXX
 7187   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7188   ins_encode %{
 7189     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7190   %}
 7191   ins_pipe(pipe_cmov_reg);
 7192 %}
 7193 
 7194 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7195 %{
 7196   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7197   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7198 
 7199   ins_cost(100); // XXX
 7200   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7201   ins_encode %{
 7202     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7203     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7204   %}
 7205   ins_pipe(ialu_reg);
 7206 %}
 7207 
 7208 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7209   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7210   ins_cost(200);
 7211   expand %{
 7212     cmovI_regU(cop, cr, dst, src);
 7213   %}
 7214 %}
 7215 
 7216 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7217   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7218   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7219 
 7220   ins_cost(200); // XXX
 7221   format %{ "cmovpl  $dst, $src\n\t"
 7222             "cmovnel $dst, $src" %}
 7223   ins_encode %{
 7224     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7225     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7226   %}
 7227   ins_pipe(pipe_cmov_reg);
 7228 %}
 7229 
 7230 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7231 // inputs of the CMove
 7232 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7233   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7234   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7235 
 7236   ins_cost(200); // XXX
 7237   format %{ "cmovpl  $dst, $src\n\t"
 7238             "cmovnel $dst, $src" %}
 7239   ins_encode %{
 7240     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7241     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7242   %}
 7243   ins_pipe(pipe_cmov_reg);
 7244 %}
 7245 
 7246 // Conditional move
 7247 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7248   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7249 
 7250   ins_cost(250); // XXX
 7251   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7252   ins_encode %{
 7253     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7254   %}
 7255   ins_pipe(pipe_cmov_mem);
 7256 %}
 7257 
 7258 // Conditional move
 7259 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7260 %{
 7261   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7262 
 7263   ins_cost(250); // XXX
 7264   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7265   ins_encode %{
 7266     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7267   %}
 7268   ins_pipe(pipe_cmov_mem);
 7269 %}
 7270 
 7271 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7272   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7273   ins_cost(250);
 7274   expand %{
 7275     cmovI_memU(cop, cr, dst, src);
 7276   %}
 7277 %}
 7278 
 7279 // Conditional move
 7280 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7281 %{
 7282   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7283 
 7284   ins_cost(200); // XXX
 7285   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7286   ins_encode %{
 7287     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7288   %}
 7289   ins_pipe(pipe_cmov_reg);
 7290 %}
 7291 
 7292 // Conditional move
 7293 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7294 %{
 7295   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7296 
 7297   ins_cost(200); // XXX
 7298   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7299   ins_encode %{
 7300     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7301   %}
 7302   ins_pipe(pipe_cmov_reg);
 7303 %}
 7304 
 7305 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7306   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7307   ins_cost(200);
 7308   expand %{
 7309     cmovN_regU(cop, cr, dst, src);
 7310   %}
 7311 %}
 7312 
 7313 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7314   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7315   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7316 
 7317   ins_cost(200); // XXX
 7318   format %{ "cmovpl  $dst, $src\n\t"
 7319             "cmovnel $dst, $src" %}
 7320   ins_encode %{
 7321     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7322     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7323   %}
 7324   ins_pipe(pipe_cmov_reg);
 7325 %}
 7326 
 7327 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7328 // inputs of the CMove
 7329 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7330   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7331   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 7332 
 7333   ins_cost(200); // XXX
 7334   format %{ "cmovpl  $dst, $src\n\t"
 7335             "cmovnel $dst, $src" %}
 7336   ins_encode %{
 7337     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7338     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7339   %}
 7340   ins_pipe(pipe_cmov_reg);
 7341 %}
 7342 
 7343 // Conditional move
 7344 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7345 %{
 7346   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7347 
 7348   ins_cost(200); // XXX
 7349   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7350   ins_encode %{
 7351     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7352   %}
 7353   ins_pipe(pipe_cmov_reg);  // XXX
 7354 %}
 7355 
 7356 // Conditional move
 7357 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7358 %{
 7359   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7360 
 7361   ins_cost(200); // XXX
 7362   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7363   ins_encode %{
 7364     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7365   %}
 7366   ins_pipe(pipe_cmov_reg); // XXX
 7367 %}
 7368 
 7369 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7370   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7371   ins_cost(200);
 7372   expand %{
 7373     cmovP_regU(cop, cr, dst, src);
 7374   %}
 7375 %}
 7376 
 7377 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7378   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7379   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7380 
 7381   ins_cost(200); // XXX
 7382   format %{ "cmovpq  $dst, $src\n\t"
 7383             "cmovneq $dst, $src" %}
 7384   ins_encode %{
 7385     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7386     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7387   %}
 7388   ins_pipe(pipe_cmov_reg);
 7389 %}
 7390 
 7391 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7392 // inputs of the CMove
 7393 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7394   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7395   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 7396 
 7397   ins_cost(200); // XXX
 7398   format %{ "cmovpq  $dst, $src\n\t"
 7399             "cmovneq $dst, $src" %}
 7400   ins_encode %{
 7401     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7402     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7403   %}
 7404   ins_pipe(pipe_cmov_reg);
 7405 %}
 7406 
 7407 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7408 // correctly meets the two pointer arguments; one is an incoming
 7409 // register but the other is a memory operand.  ALSO appears to
 7410 // be buggy with implicit null checks.
 7411 //
 7412 //// Conditional move
 7413 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7414 //%{
 7415 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7416 //  ins_cost(250);
 7417 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7418 //  opcode(0x0F,0x40);
 7419 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7420 //  ins_pipe( pipe_cmov_mem );
 7421 //%}
 7422 //
 7423 //// Conditional move
 7424 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7425 //%{
 7426 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7427 //  ins_cost(250);
 7428 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7429 //  opcode(0x0F,0x40);
 7430 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7431 //  ins_pipe( pipe_cmov_mem );
 7432 //%}
 7433 
 7434 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7435 %{
 7436   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7437   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7438 
 7439   ins_cost(100); // XXX
 7440   format %{ "setbn$cop $dst\t# signed, long" %}
 7441   ins_encode %{
 7442     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7443     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7444   %}
 7445   ins_pipe(ialu_reg);
 7446 %}
 7447 
 7448 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7449 %{
 7450   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7451 
 7452   ins_cost(200); // XXX
 7453   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7454   ins_encode %{
 7455     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7456   %}
 7457   ins_pipe(pipe_cmov_reg);  // XXX
 7458 %}
 7459 
 7460 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7461 %{
 7462   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7463 
 7464   ins_cost(200); // XXX
 7465   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7466   ins_encode %{
 7467     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7468   %}
 7469   ins_pipe(pipe_cmov_mem);  // XXX
 7470 %}
 7471 
 7472 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7473 %{
 7474   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7475   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7476 
 7477   ins_cost(100); // XXX
 7478   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7479   ins_encode %{
 7480     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7481     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7482   %}
 7483   ins_pipe(ialu_reg);
 7484 %}
 7485 
 7486 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7487 %{
 7488   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7489 
 7490   ins_cost(200); // XXX
 7491   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7492   ins_encode %{
 7493     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7494   %}
 7495   ins_pipe(pipe_cmov_reg); // XXX
 7496 %}
 7497 
 7498 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7499 %{
 7500   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7501   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7502 
 7503   ins_cost(100); // XXX
 7504   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7505   ins_encode %{
 7506     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7507     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7508   %}
 7509   ins_pipe(ialu_reg);
 7510 %}
 7511 
 7512 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7513   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7514   ins_cost(200);
 7515   expand %{
 7516     cmovL_regU(cop, cr, dst, src);
 7517   %}
 7518 %}
 7519 
 7520 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7521   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7522   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7523 
 7524   ins_cost(200); // XXX
 7525   format %{ "cmovpq  $dst, $src\n\t"
 7526             "cmovneq $dst, $src" %}
 7527   ins_encode %{
 7528     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7529     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7530   %}
 7531   ins_pipe(pipe_cmov_reg);
 7532 %}
 7533 
 7534 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7535 // inputs of the CMove
 7536 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7537   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7538   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7539 
 7540   ins_cost(200); // XXX
 7541   format %{ "cmovpq  $dst, $src\n\t"
 7542             "cmovneq $dst, $src" %}
 7543   ins_encode %{
 7544     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7545     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7546   %}
 7547   ins_pipe(pipe_cmov_reg);
 7548 %}
 7549 
 7550 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7551 %{
 7552   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7553 
 7554   ins_cost(200); // XXX
 7555   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7556   ins_encode %{
 7557     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7558   %}
 7559   ins_pipe(pipe_cmov_mem); // XXX
 7560 %}
 7561 
 7562 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7563   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7564   ins_cost(200);
 7565   expand %{
 7566     cmovL_memU(cop, cr, dst, src);
 7567   %}
 7568 %}
 7569 
 7570 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7571 %{
 7572   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7573 
 7574   ins_cost(200); // XXX
 7575   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7576             "movss     $dst, $src\n"
 7577     "skip:" %}
 7578   ins_encode %{
 7579     Label Lskip;
 7580     // Invert sense of branch from sense of CMOV
 7581     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7582     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7583     __ bind(Lskip);
 7584   %}
 7585   ins_pipe(pipe_slow);
 7586 %}
 7587 
 7588 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7589 // %{
 7590 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7591 
 7592 //   ins_cost(200); // XXX
 7593 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7594 //             "movss     $dst, $src\n"
 7595 //     "skip:" %}
 7596 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7597 //   ins_pipe(pipe_slow);
 7598 // %}
 7599 
 7600 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7601 %{
 7602   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7603 
 7604   ins_cost(200); // XXX
 7605   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7606             "movss     $dst, $src\n"
 7607     "skip:" %}
 7608   ins_encode %{
 7609     Label Lskip;
 7610     // Invert sense of branch from sense of CMOV
 7611     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7612     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7613     __ bind(Lskip);
 7614   %}
 7615   ins_pipe(pipe_slow);
 7616 %}
 7617 
 7618 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7619   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7620   ins_cost(200);
 7621   expand %{
 7622     cmovF_regU(cop, cr, dst, src);
 7623   %}
 7624 %}
 7625 
 7626 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7627 %{
 7628   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7629 
 7630   ins_cost(200); // XXX
 7631   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7632             "movsd     $dst, $src\n"
 7633     "skip:" %}
 7634   ins_encode %{
 7635     Label Lskip;
 7636     // Invert sense of branch from sense of CMOV
 7637     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7638     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7639     __ bind(Lskip);
 7640   %}
 7641   ins_pipe(pipe_slow);
 7642 %}
 7643 
 7644 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7645 %{
 7646   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7647 
 7648   ins_cost(200); // XXX
 7649   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7650             "movsd     $dst, $src\n"
 7651     "skip:" %}
 7652   ins_encode %{
 7653     Label Lskip;
 7654     // Invert sense of branch from sense of CMOV
 7655     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7656     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7657     __ bind(Lskip);
 7658   %}
 7659   ins_pipe(pipe_slow);
 7660 %}
 7661 
 7662 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7663   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7664   ins_cost(200);
 7665   expand %{
 7666     cmovD_regU(cop, cr, dst, src);
 7667   %}
 7668 %}
 7669 
 7670 //----------Arithmetic Instructions--------------------------------------------
 7671 //----------Addition Instructions----------------------------------------------
 7672 
 7673 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7674 %{
 7675   match(Set dst (AddI dst src));
 7676   effect(KILL cr);
 7677 
 7678   format %{ "addl    $dst, $src\t# int" %}
 7679   ins_encode %{
 7680     __ addl($dst$$Register, $src$$Register);
 7681   %}
 7682   ins_pipe(ialu_reg_reg);
 7683 %}
 7684 
 7685 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7686 %{
 7687   match(Set dst (AddI dst src));
 7688   effect(KILL cr);
 7689 
 7690   format %{ "addl    $dst, $src\t# int" %}
 7691   ins_encode %{
 7692     __ addl($dst$$Register, $src$$constant);
 7693   %}
 7694   ins_pipe( ialu_reg );
 7695 %}
 7696 
 7697 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7698 %{
 7699   match(Set dst (AddI dst (LoadI src)));
 7700   effect(KILL cr);
 7701 
 7702   ins_cost(150); // XXX
 7703   format %{ "addl    $dst, $src\t# int" %}
 7704   ins_encode %{
 7705     __ addl($dst$$Register, $src$$Address);
 7706   %}
 7707   ins_pipe(ialu_reg_mem);
 7708 %}
 7709 
 7710 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7711 %{
 7712   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7713   effect(KILL cr);
 7714 
 7715   ins_cost(150); // XXX
 7716   format %{ "addl    $dst, $src\t# int" %}
 7717   ins_encode %{
 7718     __ addl($dst$$Address, $src$$Register);
 7719   %}
 7720   ins_pipe(ialu_mem_reg);
 7721 %}
 7722 
 7723 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7724 %{
 7725   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7726   effect(KILL cr);
 7727 
 7728   ins_cost(125); // XXX
 7729   format %{ "addl    $dst, $src\t# int" %}
 7730   ins_encode %{
 7731     __ addl($dst$$Address, $src$$constant);
 7732   %}
 7733   ins_pipe(ialu_mem_imm);
 7734 %}
 7735 
 7736 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7737 %{
 7738   predicate(UseIncDec);
 7739   match(Set dst (AddI dst src));
 7740   effect(KILL cr);
 7741 
 7742   format %{ "incl    $dst\t# int" %}
 7743   ins_encode %{
 7744     __ incrementl($dst$$Register);
 7745   %}
 7746   ins_pipe(ialu_reg);
 7747 %}
 7748 
 7749 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7750 %{
 7751   predicate(UseIncDec);
 7752   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7753   effect(KILL cr);
 7754 
 7755   ins_cost(125); // XXX
 7756   format %{ "incl    $dst\t# int" %}
 7757   ins_encode %{
 7758     __ incrementl($dst$$Address);
 7759   %}
 7760   ins_pipe(ialu_mem_imm);
 7761 %}
 7762 
 7763 // XXX why does that use AddI
 7764 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7765 %{
 7766   predicate(UseIncDec);
 7767   match(Set dst (AddI dst src));
 7768   effect(KILL cr);
 7769 
 7770   format %{ "decl    $dst\t# int" %}
 7771   ins_encode %{
 7772     __ decrementl($dst$$Register);
 7773   %}
 7774   ins_pipe(ialu_reg);
 7775 %}
 7776 
 7777 // XXX why does that use AddI
 7778 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7779 %{
 7780   predicate(UseIncDec);
 7781   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7782   effect(KILL cr);
 7783 
 7784   ins_cost(125); // XXX
 7785   format %{ "decl    $dst\t# int" %}
 7786   ins_encode %{
 7787     __ decrementl($dst$$Address);
 7788   %}
 7789   ins_pipe(ialu_mem_imm);
 7790 %}
 7791 
 7792 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7793 %{
 7794   predicate(VM_Version::supports_fast_2op_lea());
 7795   match(Set dst (AddI (LShiftI index scale) disp));
 7796 
 7797   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7798   ins_encode %{
 7799     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7800     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7801   %}
 7802   ins_pipe(ialu_reg_reg);
 7803 %}
 7804 
 7805 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7806 %{
 7807   predicate(VM_Version::supports_fast_3op_lea());
 7808   match(Set dst (AddI (AddI base index) disp));
 7809 
 7810   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7811   ins_encode %{
 7812     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7813   %}
 7814   ins_pipe(ialu_reg_reg);
 7815 %}
 7816 
 7817 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7818 %{
 7819   predicate(VM_Version::supports_fast_2op_lea());
 7820   match(Set dst (AddI base (LShiftI index scale)));
 7821 
 7822   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7823   ins_encode %{
 7824     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7825     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7826   %}
 7827   ins_pipe(ialu_reg_reg);
 7828 %}
 7829 
 7830 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7831 %{
 7832   predicate(VM_Version::supports_fast_3op_lea());
 7833   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7834 
 7835   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7836   ins_encode %{
 7837     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7838     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7839   %}
 7840   ins_pipe(ialu_reg_reg);
 7841 %}
 7842 
 7843 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7844 %{
 7845   match(Set dst (AddL dst src));
 7846   effect(KILL cr);
 7847 
 7848   format %{ "addq    $dst, $src\t# long" %}
 7849   ins_encode %{
 7850     __ addq($dst$$Register, $src$$Register);
 7851   %}
 7852   ins_pipe(ialu_reg_reg);
 7853 %}
 7854 
 7855 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7856 %{
 7857   match(Set dst (AddL dst src));
 7858   effect(KILL cr);
 7859 
 7860   format %{ "addq    $dst, $src\t# long" %}
 7861   ins_encode %{
 7862     __ addq($dst$$Register, $src$$constant);
 7863   %}
 7864   ins_pipe( ialu_reg );
 7865 %}
 7866 
 7867 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7868 %{
 7869   match(Set dst (AddL dst (LoadL src)));
 7870   effect(KILL cr);
 7871 
 7872   ins_cost(150); // XXX
 7873   format %{ "addq    $dst, $src\t# long" %}
 7874   ins_encode %{
 7875     __ addq($dst$$Register, $src$$Address);
 7876   %}
 7877   ins_pipe(ialu_reg_mem);
 7878 %}
 7879 
 7880 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7881 %{
 7882   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7883   effect(KILL cr);
 7884 
 7885   ins_cost(150); // XXX
 7886   format %{ "addq    $dst, $src\t# long" %}
 7887   ins_encode %{
 7888     __ addq($dst$$Address, $src$$Register);
 7889   %}
 7890   ins_pipe(ialu_mem_reg);
 7891 %}
 7892 
 7893 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7894 %{
 7895   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7896   effect(KILL cr);
 7897 
 7898   ins_cost(125); // XXX
 7899   format %{ "addq    $dst, $src\t# long" %}
 7900   ins_encode %{
 7901     __ addq($dst$$Address, $src$$constant);
 7902   %}
 7903   ins_pipe(ialu_mem_imm);
 7904 %}
 7905 
 7906 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7907 %{
 7908   predicate(UseIncDec);
 7909   match(Set dst (AddL dst src));
 7910   effect(KILL cr);
 7911 
 7912   format %{ "incq    $dst\t# long" %}
 7913   ins_encode %{
 7914     __ incrementq($dst$$Register);
 7915   %}
 7916   ins_pipe(ialu_reg);
 7917 %}
 7918 
 7919 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7920 %{
 7921   predicate(UseIncDec);
 7922   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7923   effect(KILL cr);
 7924 
 7925   ins_cost(125); // XXX
 7926   format %{ "incq    $dst\t# long" %}
 7927   ins_encode %{
 7928     __ incrementq($dst$$Address);
 7929   %}
 7930   ins_pipe(ialu_mem_imm);
 7931 %}
 7932 
 7933 // XXX why does that use AddL
 7934 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7935 %{
 7936   predicate(UseIncDec);
 7937   match(Set dst (AddL dst src));
 7938   effect(KILL cr);
 7939 
 7940   format %{ "decq    $dst\t# long" %}
 7941   ins_encode %{
 7942     __ decrementq($dst$$Register);
 7943   %}
 7944   ins_pipe(ialu_reg);
 7945 %}
 7946 
 7947 // XXX why does that use AddL
 7948 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7949 %{
 7950   predicate(UseIncDec);
 7951   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7952   effect(KILL cr);
 7953 
 7954   ins_cost(125); // XXX
 7955   format %{ "decq    $dst\t# long" %}
 7956   ins_encode %{
 7957     __ decrementq($dst$$Address);
 7958   %}
 7959   ins_pipe(ialu_mem_imm);
 7960 %}
 7961 
 7962 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7963 %{
 7964   predicate(VM_Version::supports_fast_2op_lea());
 7965   match(Set dst (AddL (LShiftL index scale) disp));
 7966 
 7967   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7968   ins_encode %{
 7969     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7970     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7971   %}
 7972   ins_pipe(ialu_reg_reg);
 7973 %}
 7974 
 7975 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7976 %{
 7977   predicate(VM_Version::supports_fast_3op_lea());
 7978   match(Set dst (AddL (AddL base index) disp));
 7979 
 7980   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7981   ins_encode %{
 7982     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7983   %}
 7984   ins_pipe(ialu_reg_reg);
 7985 %}
 7986 
 7987 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7988 %{
 7989   predicate(VM_Version::supports_fast_2op_lea());
 7990   match(Set dst (AddL base (LShiftL index scale)));
 7991 
 7992   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7993   ins_encode %{
 7994     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7995     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7996   %}
 7997   ins_pipe(ialu_reg_reg);
 7998 %}
 7999 
 8000 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 8001 %{
 8002   predicate(VM_Version::supports_fast_3op_lea());
 8003   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 8004 
 8005   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 8006   ins_encode %{
 8007     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 8008     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 8009   %}
 8010   ins_pipe(ialu_reg_reg);
 8011 %}
 8012 
 8013 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 8014 %{
 8015   match(Set dst (AddP dst src));
 8016   effect(KILL cr);
 8017 
 8018   format %{ "addq    $dst, $src\t# ptr" %}
 8019   ins_encode %{
 8020     __ addq($dst$$Register, $src$$Register);
 8021   %}
 8022   ins_pipe(ialu_reg_reg);
 8023 %}
 8024 
 8025 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 8026 %{
 8027   match(Set dst (AddP dst src));
 8028   effect(KILL cr);
 8029 
 8030   format %{ "addq    $dst, $src\t# ptr" %}
 8031   ins_encode %{
 8032     __ addq($dst$$Register, $src$$constant);
 8033   %}
 8034   ins_pipe( ialu_reg );
 8035 %}
 8036 
 8037 // XXX addP mem ops ????
 8038 
 8039 instruct checkCastPP(rRegP dst)
 8040 %{
 8041   match(Set dst (CheckCastPP dst));
 8042 
 8043   size(0);
 8044   format %{ "# checkcastPP of $dst" %}
 8045   ins_encode(/* empty encoding */);
 8046   ins_pipe(empty);
 8047 %}
 8048 
 8049 instruct castPP(rRegP dst)
 8050 %{
 8051   match(Set dst (CastPP dst));
 8052 
 8053   size(0);
 8054   format %{ "# castPP of $dst" %}
 8055   ins_encode(/* empty encoding */);
 8056   ins_pipe(empty);
 8057 %}
 8058 
 8059 instruct castII(rRegI dst)
 8060 %{
 8061   match(Set dst (CastII dst));
 8062 
 8063   size(0);
 8064   format %{ "# castII of $dst" %}
 8065   ins_encode(/* empty encoding */);
 8066   ins_cost(0);
 8067   ins_pipe(empty);
 8068 %}
 8069 
 8070 instruct castLL(rRegL dst)
 8071 %{
 8072   match(Set dst (CastLL dst));
 8073 
 8074   size(0);
 8075   format %{ "# castLL of $dst" %}
 8076   ins_encode(/* empty encoding */);
 8077   ins_cost(0);
 8078   ins_pipe(empty);
 8079 %}
 8080 
 8081 instruct castFF(regF dst)
 8082 %{
 8083   match(Set dst (CastFF dst));
 8084 
 8085   size(0);
 8086   format %{ "# castFF of $dst" %}
 8087   ins_encode(/* empty encoding */);
 8088   ins_cost(0);
 8089   ins_pipe(empty);
 8090 %}
 8091 
 8092 instruct castDD(regD dst)
 8093 %{
 8094   match(Set dst (CastDD dst));
 8095 
 8096   size(0);
 8097   format %{ "# castDD of $dst" %}
 8098   ins_encode(/* empty encoding */);
 8099   ins_cost(0);
 8100   ins_pipe(empty);
 8101 %}
 8102 
 8103 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 8104 instruct compareAndSwapP(rRegI res,
 8105                          memory mem_ptr,
 8106                          rax_RegP oldval, rRegP newval,
 8107                          rFlagsReg cr)
 8108 %{
 8109   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8110   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 8111   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 8112   effect(KILL cr, KILL oldval);
 8113 
 8114   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8115             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8116             "sete    $res\n\t"
 8117             "movzbl  $res, $res" %}
 8118   ins_encode %{
 8119     __ lock();
 8120     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8121     __ sete($res$$Register);
 8122     __ movzbl($res$$Register, $res$$Register);
 8123   %}
 8124   ins_pipe( pipe_cmpxchg );
 8125 %}
 8126 
 8127 instruct compareAndSwapL(rRegI res,
 8128                          memory mem_ptr,
 8129                          rax_RegL oldval, rRegL newval,
 8130                          rFlagsReg cr)
 8131 %{
 8132   predicate(VM_Version::supports_cx8());
 8133   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 8134   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 8135   effect(KILL cr, KILL oldval);
 8136 
 8137   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8138             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8139             "sete    $res\n\t"
 8140             "movzbl  $res, $res" %}
 8141   ins_encode %{
 8142     __ lock();
 8143     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8144     __ sete($res$$Register);
 8145     __ movzbl($res$$Register, $res$$Register);
 8146   %}
 8147   ins_pipe( pipe_cmpxchg );
 8148 %}
 8149 
 8150 instruct compareAndSwapI(rRegI res,
 8151                          memory mem_ptr,
 8152                          rax_RegI oldval, rRegI newval,
 8153                          rFlagsReg cr)
 8154 %{
 8155   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 8156   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 8157   effect(KILL cr, KILL oldval);
 8158 
 8159   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8160             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8161             "sete    $res\n\t"
 8162             "movzbl  $res, $res" %}
 8163   ins_encode %{
 8164     __ lock();
 8165     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8166     __ sete($res$$Register);
 8167     __ movzbl($res$$Register, $res$$Register);
 8168   %}
 8169   ins_pipe( pipe_cmpxchg );
 8170 %}
 8171 
 8172 instruct compareAndSwapB(rRegI res,
 8173                          memory mem_ptr,
 8174                          rax_RegI oldval, rRegI newval,
 8175                          rFlagsReg cr)
 8176 %{
 8177   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 8178   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 8179   effect(KILL cr, KILL oldval);
 8180 
 8181   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8182             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8183             "sete    $res\n\t"
 8184             "movzbl  $res, $res" %}
 8185   ins_encode %{
 8186     __ lock();
 8187     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8188     __ sete($res$$Register);
 8189     __ movzbl($res$$Register, $res$$Register);
 8190   %}
 8191   ins_pipe( pipe_cmpxchg );
 8192 %}
 8193 
 8194 instruct compareAndSwapS(rRegI res,
 8195                          memory mem_ptr,
 8196                          rax_RegI oldval, rRegI newval,
 8197                          rFlagsReg cr)
 8198 %{
 8199   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 8200   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 8201   effect(KILL cr, KILL oldval);
 8202 
 8203   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8204             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8205             "sete    $res\n\t"
 8206             "movzbl  $res, $res" %}
 8207   ins_encode %{
 8208     __ lock();
 8209     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8210     __ sete($res$$Register);
 8211     __ movzbl($res$$Register, $res$$Register);
 8212   %}
 8213   ins_pipe( pipe_cmpxchg );
 8214 %}
 8215 
 8216 instruct compareAndSwapN(rRegI res,
 8217                           memory mem_ptr,
 8218                           rax_RegN oldval, rRegN newval,
 8219                           rFlagsReg cr) %{
 8220   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 8221   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 8222   effect(KILL cr, KILL oldval);
 8223 
 8224   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8225             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8226             "sete    $res\n\t"
 8227             "movzbl  $res, $res" %}
 8228   ins_encode %{
 8229     __ lock();
 8230     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8231     __ sete($res$$Register);
 8232     __ movzbl($res$$Register, $res$$Register);
 8233   %}
 8234   ins_pipe( pipe_cmpxchg );
 8235 %}
 8236 
 8237 instruct compareAndExchangeB(
 8238                          memory mem_ptr,
 8239                          rax_RegI oldval, rRegI newval,
 8240                          rFlagsReg cr)
 8241 %{
 8242   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 8243   effect(KILL cr);
 8244 
 8245   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8246             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8247   ins_encode %{
 8248     __ lock();
 8249     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8250   %}
 8251   ins_pipe( pipe_cmpxchg );
 8252 %}
 8253 
 8254 instruct compareAndExchangeS(
 8255                          memory mem_ptr,
 8256                          rax_RegI oldval, rRegI newval,
 8257                          rFlagsReg cr)
 8258 %{
 8259   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 8260   effect(KILL cr);
 8261 
 8262   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8263             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8264   ins_encode %{
 8265     __ lock();
 8266     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8267   %}
 8268   ins_pipe( pipe_cmpxchg );
 8269 %}
 8270 
 8271 instruct compareAndExchangeI(
 8272                          memory mem_ptr,
 8273                          rax_RegI oldval, rRegI newval,
 8274                          rFlagsReg cr)
 8275 %{
 8276   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 8277   effect(KILL cr);
 8278 
 8279   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8280             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8281   ins_encode %{
 8282     __ lock();
 8283     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8284   %}
 8285   ins_pipe( pipe_cmpxchg );
 8286 %}
 8287 
 8288 instruct compareAndExchangeL(
 8289                          memory mem_ptr,
 8290                          rax_RegL oldval, rRegL newval,
 8291                          rFlagsReg cr)
 8292 %{
 8293   predicate(VM_Version::supports_cx8());
 8294   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 8295   effect(KILL cr);
 8296 
 8297   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8298             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8299   ins_encode %{
 8300     __ lock();
 8301     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8302   %}
 8303   ins_pipe( pipe_cmpxchg );
 8304 %}
 8305 
 8306 instruct compareAndExchangeN(
 8307                           memory mem_ptr,
 8308                           rax_RegN oldval, rRegN newval,
 8309                           rFlagsReg cr) %{
 8310   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 8311   effect(KILL cr);
 8312 
 8313   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8314             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8315   ins_encode %{
 8316     __ lock();
 8317     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8318   %}
 8319   ins_pipe( pipe_cmpxchg );
 8320 %}
 8321 
 8322 instruct compareAndExchangeP(
 8323                          memory mem_ptr,
 8324                          rax_RegP oldval, rRegP newval,
 8325                          rFlagsReg cr)
 8326 %{
 8327   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8328   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 8329   effect(KILL cr);
 8330 
 8331   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8332             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8333   ins_encode %{
 8334     __ lock();
 8335     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8336   %}
 8337   ins_pipe( pipe_cmpxchg );
 8338 %}
 8339 
 8340 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8341   predicate(n->as_LoadStore()->result_not_used());
 8342   match(Set dummy (GetAndAddB mem add));
 8343   effect(KILL cr);
 8344   format %{ "ADDB  [$mem],$add" %}
 8345   ins_encode %{
 8346     __ lock();
 8347     __ addb($mem$$Address, $add$$constant);
 8348   %}
 8349   ins_pipe( pipe_cmpxchg );
 8350 %}
 8351 
 8352 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8353   match(Set newval (GetAndAddB mem newval));
 8354   effect(KILL cr);
 8355   format %{ "XADDB  [$mem],$newval" %}
 8356   ins_encode %{
 8357     __ lock();
 8358     __ xaddb($mem$$Address, $newval$$Register);
 8359   %}
 8360   ins_pipe( pipe_cmpxchg );
 8361 %}
 8362 
 8363 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8364   predicate(n->as_LoadStore()->result_not_used());
 8365   match(Set dummy (GetAndAddS mem add));
 8366   effect(KILL cr);
 8367   format %{ "ADDW  [$mem],$add" %}
 8368   ins_encode %{
 8369     __ lock();
 8370     __ addw($mem$$Address, $add$$constant);
 8371   %}
 8372   ins_pipe( pipe_cmpxchg );
 8373 %}
 8374 
 8375 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8376   match(Set newval (GetAndAddS mem newval));
 8377   effect(KILL cr);
 8378   format %{ "XADDW  [$mem],$newval" %}
 8379   ins_encode %{
 8380     __ lock();
 8381     __ xaddw($mem$$Address, $newval$$Register);
 8382   %}
 8383   ins_pipe( pipe_cmpxchg );
 8384 %}
 8385 
 8386 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8387   predicate(n->as_LoadStore()->result_not_used());
 8388   match(Set dummy (GetAndAddI mem add));
 8389   effect(KILL cr);
 8390   format %{ "ADDL  [$mem],$add" %}
 8391   ins_encode %{
 8392     __ lock();
 8393     __ addl($mem$$Address, $add$$constant);
 8394   %}
 8395   ins_pipe( pipe_cmpxchg );
 8396 %}
 8397 
 8398 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8399   match(Set newval (GetAndAddI mem newval));
 8400   effect(KILL cr);
 8401   format %{ "XADDL  [$mem],$newval" %}
 8402   ins_encode %{
 8403     __ lock();
 8404     __ xaddl($mem$$Address, $newval$$Register);
 8405   %}
 8406   ins_pipe( pipe_cmpxchg );
 8407 %}
 8408 
 8409 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8410   predicate(n->as_LoadStore()->result_not_used());
 8411   match(Set dummy (GetAndAddL mem add));
 8412   effect(KILL cr);
 8413   format %{ "ADDQ  [$mem],$add" %}
 8414   ins_encode %{
 8415     __ lock();
 8416     __ addq($mem$$Address, $add$$constant);
 8417   %}
 8418   ins_pipe( pipe_cmpxchg );
 8419 %}
 8420 
 8421 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8422   match(Set newval (GetAndAddL mem newval));
 8423   effect(KILL cr);
 8424   format %{ "XADDQ  [$mem],$newval" %}
 8425   ins_encode %{
 8426     __ lock();
 8427     __ xaddq($mem$$Address, $newval$$Register);
 8428   %}
 8429   ins_pipe( pipe_cmpxchg );
 8430 %}
 8431 
 8432 instruct xchgB( memory mem, rRegI newval) %{
 8433   match(Set newval (GetAndSetB mem newval));
 8434   format %{ "XCHGB  $newval,[$mem]" %}
 8435   ins_encode %{
 8436     __ xchgb($newval$$Register, $mem$$Address);
 8437   %}
 8438   ins_pipe( pipe_cmpxchg );
 8439 %}
 8440 
 8441 instruct xchgS( memory mem, rRegI newval) %{
 8442   match(Set newval (GetAndSetS mem newval));
 8443   format %{ "XCHGW  $newval,[$mem]" %}
 8444   ins_encode %{
 8445     __ xchgw($newval$$Register, $mem$$Address);
 8446   %}
 8447   ins_pipe( pipe_cmpxchg );
 8448 %}
 8449 
 8450 instruct xchgI( memory mem, rRegI newval) %{
 8451   match(Set newval (GetAndSetI mem newval));
 8452   format %{ "XCHGL  $newval,[$mem]" %}
 8453   ins_encode %{
 8454     __ xchgl($newval$$Register, $mem$$Address);
 8455   %}
 8456   ins_pipe( pipe_cmpxchg );
 8457 %}
 8458 
 8459 instruct xchgL( memory mem, rRegL newval) %{
 8460   match(Set newval (GetAndSetL mem newval));
 8461   format %{ "XCHGL  $newval,[$mem]" %}
 8462   ins_encode %{
 8463     __ xchgq($newval$$Register, $mem$$Address);
 8464   %}
 8465   ins_pipe( pipe_cmpxchg );
 8466 %}
 8467 
 8468 instruct xchgP( memory mem, rRegP newval) %{
 8469   match(Set newval (GetAndSetP mem newval));
 8470   predicate(n->as_LoadStore()->barrier_data() == 0);
 8471   format %{ "XCHGQ  $newval,[$mem]" %}
 8472   ins_encode %{
 8473     __ xchgq($newval$$Register, $mem$$Address);
 8474   %}
 8475   ins_pipe( pipe_cmpxchg );
 8476 %}
 8477 
 8478 instruct xchgN( memory mem, rRegN newval) %{
 8479   match(Set newval (GetAndSetN mem newval));
 8480   format %{ "XCHGL  $newval,$mem]" %}
 8481   ins_encode %{
 8482     __ xchgl($newval$$Register, $mem$$Address);
 8483   %}
 8484   ins_pipe( pipe_cmpxchg );
 8485 %}
 8486 
 8487 //----------Abs Instructions-------------------------------------------
 8488 
 8489 // Integer Absolute Instructions
 8490 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8491 %{
 8492   match(Set dst (AbsI src));
 8493   effect(TEMP dst, TEMP tmp, KILL cr);
 8494   format %{ "movl $tmp, $src\n\t"
 8495             "sarl $tmp, 31\n\t"
 8496             "movl $dst, $src\n\t"
 8497             "xorl $dst, $tmp\n\t"
 8498             "subl $dst, $tmp\n"
 8499           %}
 8500   ins_encode %{
 8501     __ movl($tmp$$Register, $src$$Register);
 8502     __ sarl($tmp$$Register, 31);
 8503     __ movl($dst$$Register, $src$$Register);
 8504     __ xorl($dst$$Register, $tmp$$Register);
 8505     __ subl($dst$$Register, $tmp$$Register);
 8506   %}
 8507 
 8508   ins_pipe(ialu_reg_reg);
 8509 %}
 8510 
 8511 // Long Absolute Instructions
 8512 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8513 %{
 8514   match(Set dst (AbsL src));
 8515   effect(TEMP dst, TEMP tmp, KILL cr);
 8516   format %{ "movq $tmp, $src\n\t"
 8517             "sarq $tmp, 63\n\t"
 8518             "movq $dst, $src\n\t"
 8519             "xorq $dst, $tmp\n\t"
 8520             "subq $dst, $tmp\n"
 8521           %}
 8522   ins_encode %{
 8523     __ movq($tmp$$Register, $src$$Register);
 8524     __ sarq($tmp$$Register, 63);
 8525     __ movq($dst$$Register, $src$$Register);
 8526     __ xorq($dst$$Register, $tmp$$Register);
 8527     __ subq($dst$$Register, $tmp$$Register);
 8528   %}
 8529 
 8530   ins_pipe(ialu_reg_reg);
 8531 %}
 8532 
 8533 //----------Subtraction Instructions-------------------------------------------
 8534 
 8535 // Integer Subtraction Instructions
 8536 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8537 %{
 8538   match(Set dst (SubI dst src));
 8539   effect(KILL cr);
 8540 
 8541   format %{ "subl    $dst, $src\t# int" %}
 8542   ins_encode %{
 8543     __ subl($dst$$Register, $src$$Register);
 8544   %}
 8545   ins_pipe(ialu_reg_reg);
 8546 %}
 8547 
 8548 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8549 %{
 8550   match(Set dst (SubI dst (LoadI src)));
 8551   effect(KILL cr);
 8552 
 8553   ins_cost(150);
 8554   format %{ "subl    $dst, $src\t# int" %}
 8555   ins_encode %{
 8556     __ subl($dst$$Register, $src$$Address);
 8557   %}
 8558   ins_pipe(ialu_reg_mem);
 8559 %}
 8560 
 8561 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8562 %{
 8563   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8564   effect(KILL cr);
 8565 
 8566   ins_cost(150);
 8567   format %{ "subl    $dst, $src\t# int" %}
 8568   ins_encode %{
 8569     __ subl($dst$$Address, $src$$Register);
 8570   %}
 8571   ins_pipe(ialu_mem_reg);
 8572 %}
 8573 
 8574 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8575 %{
 8576   match(Set dst (SubL dst src));
 8577   effect(KILL cr);
 8578 
 8579   format %{ "subq    $dst, $src\t# long" %}
 8580   ins_encode %{
 8581     __ subq($dst$$Register, $src$$Register);
 8582   %}
 8583   ins_pipe(ialu_reg_reg);
 8584 %}
 8585 
 8586 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8587 %{
 8588   match(Set dst (SubL dst (LoadL src)));
 8589   effect(KILL cr);
 8590 
 8591   ins_cost(150);
 8592   format %{ "subq    $dst, $src\t# long" %}
 8593   ins_encode %{
 8594     __ subq($dst$$Register, $src$$Address);
 8595   %}
 8596   ins_pipe(ialu_reg_mem);
 8597 %}
 8598 
 8599 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8600 %{
 8601   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8602   effect(KILL cr);
 8603 
 8604   ins_cost(150);
 8605   format %{ "subq    $dst, $src\t# long" %}
 8606   ins_encode %{
 8607     __ subq($dst$$Address, $src$$Register);
 8608   %}
 8609   ins_pipe(ialu_mem_reg);
 8610 %}
 8611 
 8612 // Subtract from a pointer
 8613 // XXX hmpf???
 8614 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8615 %{
 8616   match(Set dst (AddP dst (SubI zero src)));
 8617   effect(KILL cr);
 8618 
 8619   format %{ "subq    $dst, $src\t# ptr - int" %}
 8620   opcode(0x2B);
 8621   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8622   ins_pipe(ialu_reg_reg);
 8623 %}
 8624 
 8625 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8626 %{
 8627   match(Set dst (SubI zero dst));
 8628   effect(KILL cr);
 8629 
 8630   format %{ "negl    $dst\t# int" %}
 8631   ins_encode %{
 8632     __ negl($dst$$Register);
 8633   %}
 8634   ins_pipe(ialu_reg);
 8635 %}
 8636 
 8637 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8638 %{
 8639   match(Set dst (NegI dst));
 8640   effect(KILL cr);
 8641 
 8642   format %{ "negl    $dst\t# int" %}
 8643   ins_encode %{
 8644     __ negl($dst$$Register);
 8645   %}
 8646   ins_pipe(ialu_reg);
 8647 %}
 8648 
 8649 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8650 %{
 8651   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8652   effect(KILL cr);
 8653 
 8654   format %{ "negl    $dst\t# int" %}
 8655   ins_encode %{
 8656     __ negl($dst$$Address);
 8657   %}
 8658   ins_pipe(ialu_reg);
 8659 %}
 8660 
 8661 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8662 %{
 8663   match(Set dst (SubL zero dst));
 8664   effect(KILL cr);
 8665 
 8666   format %{ "negq    $dst\t# long" %}
 8667   ins_encode %{
 8668     __ negq($dst$$Register);
 8669   %}
 8670   ins_pipe(ialu_reg);
 8671 %}
 8672 
 8673 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8674 %{
 8675   match(Set dst (NegL dst));
 8676   effect(KILL cr);
 8677 
 8678   format %{ "negq    $dst\t# int" %}
 8679   ins_encode %{
 8680     __ negq($dst$$Register);
 8681   %}
 8682   ins_pipe(ialu_reg);
 8683 %}
 8684 
 8685 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8686 %{
 8687   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8688   effect(KILL cr);
 8689 
 8690   format %{ "negq    $dst\t# long" %}
 8691   ins_encode %{
 8692     __ negq($dst$$Address);
 8693   %}
 8694   ins_pipe(ialu_reg);
 8695 %}
 8696 
 8697 //----------Multiplication/Division Instructions-------------------------------
 8698 // Integer Multiplication Instructions
 8699 // Multiply Register
 8700 
 8701 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8702 %{
 8703   match(Set dst (MulI dst src));
 8704   effect(KILL cr);
 8705 
 8706   ins_cost(300);
 8707   format %{ "imull   $dst, $src\t# int" %}
 8708   ins_encode %{
 8709     __ imull($dst$$Register, $src$$Register);
 8710   %}
 8711   ins_pipe(ialu_reg_reg_alu0);
 8712 %}
 8713 
 8714 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8715 %{
 8716   match(Set dst (MulI src imm));
 8717   effect(KILL cr);
 8718 
 8719   ins_cost(300);
 8720   format %{ "imull   $dst, $src, $imm\t# int" %}
 8721   ins_encode %{
 8722     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8723   %}
 8724   ins_pipe(ialu_reg_reg_alu0);
 8725 %}
 8726 
 8727 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8728 %{
 8729   match(Set dst (MulI dst (LoadI src)));
 8730   effect(KILL cr);
 8731 
 8732   ins_cost(350);
 8733   format %{ "imull   $dst, $src\t# int" %}
 8734   ins_encode %{
 8735     __ imull($dst$$Register, $src$$Address);
 8736   %}
 8737   ins_pipe(ialu_reg_mem_alu0);
 8738 %}
 8739 
 8740 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8741 %{
 8742   match(Set dst (MulI (LoadI src) imm));
 8743   effect(KILL cr);
 8744 
 8745   ins_cost(300);
 8746   format %{ "imull   $dst, $src, $imm\t# int" %}
 8747   ins_encode %{
 8748     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8749   %}
 8750   ins_pipe(ialu_reg_mem_alu0);
 8751 %}
 8752 
 8753 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8754 %{
 8755   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8756   effect(KILL cr, KILL src2);
 8757 
 8758   expand %{ mulI_rReg(dst, src1, cr);
 8759            mulI_rReg(src2, src3, cr);
 8760            addI_rReg(dst, src2, cr); %}
 8761 %}
 8762 
 8763 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8764 %{
 8765   match(Set dst (MulL dst src));
 8766   effect(KILL cr);
 8767 
 8768   ins_cost(300);
 8769   format %{ "imulq   $dst, $src\t# long" %}
 8770   ins_encode %{
 8771     __ imulq($dst$$Register, $src$$Register);
 8772   %}
 8773   ins_pipe(ialu_reg_reg_alu0);
 8774 %}
 8775 
 8776 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8777 %{
 8778   match(Set dst (MulL src imm));
 8779   effect(KILL cr);
 8780 
 8781   ins_cost(300);
 8782   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8783   ins_encode %{
 8784     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8785   %}
 8786   ins_pipe(ialu_reg_reg_alu0);
 8787 %}
 8788 
 8789 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8790 %{
 8791   match(Set dst (MulL dst (LoadL src)));
 8792   effect(KILL cr);
 8793 
 8794   ins_cost(350);
 8795   format %{ "imulq   $dst, $src\t# long" %}
 8796   ins_encode %{
 8797     __ imulq($dst$$Register, $src$$Address);
 8798   %}
 8799   ins_pipe(ialu_reg_mem_alu0);
 8800 %}
 8801 
 8802 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8803 %{
 8804   match(Set dst (MulL (LoadL src) imm));
 8805   effect(KILL cr);
 8806 
 8807   ins_cost(300);
 8808   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8809   ins_encode %{
 8810     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8811   %}
 8812   ins_pipe(ialu_reg_mem_alu0);
 8813 %}
 8814 
 8815 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8816 %{
 8817   match(Set dst (MulHiL src rax));
 8818   effect(USE_KILL rax, KILL cr);
 8819 
 8820   ins_cost(300);
 8821   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8822   ins_encode %{
 8823     __ imulq($src$$Register);
 8824   %}
 8825   ins_pipe(ialu_reg_reg_alu0);
 8826 %}
 8827 
 8828 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8829 %{
 8830   match(Set dst (UMulHiL src rax));
 8831   effect(USE_KILL rax, KILL cr);
 8832 
 8833   ins_cost(300);
 8834   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8835   ins_encode %{
 8836     __ mulq($src$$Register);
 8837   %}
 8838   ins_pipe(ialu_reg_reg_alu0);
 8839 %}
 8840 
 8841 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8842                    rFlagsReg cr)
 8843 %{
 8844   match(Set rax (DivI rax div));
 8845   effect(KILL rdx, KILL cr);
 8846 
 8847   ins_cost(30*100+10*100); // XXX
 8848   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8849             "jne,s   normal\n\t"
 8850             "xorl    rdx, rdx\n\t"
 8851             "cmpl    $div, -1\n\t"
 8852             "je,s    done\n"
 8853     "normal: cdql\n\t"
 8854             "idivl   $div\n"
 8855     "done:"        %}
 8856   ins_encode(cdql_enc(div));
 8857   ins_pipe(ialu_reg_reg_alu0);
 8858 %}
 8859 
 8860 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8861                    rFlagsReg cr)
 8862 %{
 8863   match(Set rax (DivL rax div));
 8864   effect(KILL rdx, KILL cr);
 8865 
 8866   ins_cost(30*100+10*100); // XXX
 8867   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8868             "cmpq    rax, rdx\n\t"
 8869             "jne,s   normal\n\t"
 8870             "xorl    rdx, rdx\n\t"
 8871             "cmpq    $div, -1\n\t"
 8872             "je,s    done\n"
 8873     "normal: cdqq\n\t"
 8874             "idivq   $div\n"
 8875     "done:"        %}
 8876   ins_encode(cdqq_enc(div));
 8877   ins_pipe(ialu_reg_reg_alu0);
 8878 %}
 8879 
 8880 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8881 %{
 8882   match(Set rax (UDivI rax div));
 8883   effect(KILL rdx, KILL cr);
 8884 
 8885   ins_cost(300);
 8886   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8887   ins_encode %{
 8888     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8889   %}
 8890   ins_pipe(ialu_reg_reg_alu0);
 8891 %}
 8892 
 8893 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8894 %{
 8895   match(Set rax (UDivL rax div));
 8896   effect(KILL rdx, KILL cr);
 8897 
 8898   ins_cost(300);
 8899   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8900   ins_encode %{
 8901      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8902   %}
 8903   ins_pipe(ialu_reg_reg_alu0);
 8904 %}
 8905 
 8906 // Integer DIVMOD with Register, both quotient and mod results
 8907 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8908                              rFlagsReg cr)
 8909 %{
 8910   match(DivModI rax div);
 8911   effect(KILL cr);
 8912 
 8913   ins_cost(30*100+10*100); // XXX
 8914   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8915             "jne,s   normal\n\t"
 8916             "xorl    rdx, rdx\n\t"
 8917             "cmpl    $div, -1\n\t"
 8918             "je,s    done\n"
 8919     "normal: cdql\n\t"
 8920             "idivl   $div\n"
 8921     "done:"        %}
 8922   ins_encode(cdql_enc(div));
 8923   ins_pipe(pipe_slow);
 8924 %}
 8925 
 8926 // Long DIVMOD with Register, both quotient and mod results
 8927 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8928                              rFlagsReg cr)
 8929 %{
 8930   match(DivModL rax div);
 8931   effect(KILL cr);
 8932 
 8933   ins_cost(30*100+10*100); // XXX
 8934   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8935             "cmpq    rax, rdx\n\t"
 8936             "jne,s   normal\n\t"
 8937             "xorl    rdx, rdx\n\t"
 8938             "cmpq    $div, -1\n\t"
 8939             "je,s    done\n"
 8940     "normal: cdqq\n\t"
 8941             "idivq   $div\n"
 8942     "done:"        %}
 8943   ins_encode(cdqq_enc(div));
 8944   ins_pipe(pipe_slow);
 8945 %}
 8946 
 8947 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8948 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8949                               no_rax_rdx_RegI div, rFlagsReg cr)
 8950 %{
 8951   match(UDivModI rax div);
 8952   effect(TEMP tmp, KILL cr);
 8953 
 8954   ins_cost(300);
 8955   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8956             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8957           %}
 8958   ins_encode %{
 8959     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8960   %}
 8961   ins_pipe(pipe_slow);
 8962 %}
 8963 
 8964 // Unsigned long DIVMOD with Register, both quotient and mod results
 8965 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8966                               no_rax_rdx_RegL div, rFlagsReg cr)
 8967 %{
 8968   match(UDivModL rax div);
 8969   effect(TEMP tmp, KILL cr);
 8970 
 8971   ins_cost(300);
 8972   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8973             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8974           %}
 8975   ins_encode %{
 8976     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8977   %}
 8978   ins_pipe(pipe_slow);
 8979 %}
 8980 
 8981 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8982                    rFlagsReg cr)
 8983 %{
 8984   match(Set rdx (ModI rax div));
 8985   effect(KILL rax, KILL cr);
 8986 
 8987   ins_cost(300); // XXX
 8988   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8989             "jne,s   normal\n\t"
 8990             "xorl    rdx, rdx\n\t"
 8991             "cmpl    $div, -1\n\t"
 8992             "je,s    done\n"
 8993     "normal: cdql\n\t"
 8994             "idivl   $div\n"
 8995     "done:"        %}
 8996   ins_encode(cdql_enc(div));
 8997   ins_pipe(ialu_reg_reg_alu0);
 8998 %}
 8999 
 9000 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 9001                    rFlagsReg cr)
 9002 %{
 9003   match(Set rdx (ModL rax div));
 9004   effect(KILL rax, KILL cr);
 9005 
 9006   ins_cost(300); // XXX
 9007   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 9008             "cmpq    rax, rdx\n\t"
 9009             "jne,s   normal\n\t"
 9010             "xorl    rdx, rdx\n\t"
 9011             "cmpq    $div, -1\n\t"
 9012             "je,s    done\n"
 9013     "normal: cdqq\n\t"
 9014             "idivq   $div\n"
 9015     "done:"        %}
 9016   ins_encode(cdqq_enc(div));
 9017   ins_pipe(ialu_reg_reg_alu0);
 9018 %}
 9019 
 9020 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 9021 %{
 9022   match(Set rdx (UModI rax div));
 9023   effect(KILL rax, KILL cr);
 9024 
 9025   ins_cost(300);
 9026   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 9027   ins_encode %{
 9028     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 9029   %}
 9030   ins_pipe(ialu_reg_reg_alu0);
 9031 %}
 9032 
 9033 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 9034 %{
 9035   match(Set rdx (UModL rax div));
 9036   effect(KILL rax, KILL cr);
 9037 
 9038   ins_cost(300);
 9039   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9040   ins_encode %{
 9041     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9042   %}
 9043   ins_pipe(ialu_reg_reg_alu0);
 9044 %}
 9045 
 9046 // Integer Shift Instructions
 9047 // Shift Left by one, two, three
 9048 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 9049 %{
 9050   match(Set dst (LShiftI dst shift));
 9051   effect(KILL cr);
 9052 
 9053   format %{ "sall    $dst, $shift" %}
 9054   ins_encode %{
 9055     __ sall($dst$$Register, $shift$$constant);
 9056   %}
 9057   ins_pipe(ialu_reg);
 9058 %}
 9059 
 9060 // Shift Left by 8-bit immediate
 9061 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9062 %{
 9063   match(Set dst (LShiftI dst shift));
 9064   effect(KILL cr);
 9065 
 9066   format %{ "sall    $dst, $shift" %}
 9067   ins_encode %{
 9068     __ sall($dst$$Register, $shift$$constant);
 9069   %}
 9070   ins_pipe(ialu_reg);
 9071 %}
 9072 
 9073 // Shift Left by 8-bit immediate
 9074 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9075 %{
 9076   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9077   effect(KILL cr);
 9078 
 9079   format %{ "sall    $dst, $shift" %}
 9080   ins_encode %{
 9081     __ sall($dst$$Address, $shift$$constant);
 9082   %}
 9083   ins_pipe(ialu_mem_imm);
 9084 %}
 9085 
 9086 // Shift Left by variable
 9087 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9088 %{
 9089   predicate(!VM_Version::supports_bmi2());
 9090   match(Set dst (LShiftI dst shift));
 9091   effect(KILL cr);
 9092 
 9093   format %{ "sall    $dst, $shift" %}
 9094   ins_encode %{
 9095     __ sall($dst$$Register);
 9096   %}
 9097   ins_pipe(ialu_reg_reg);
 9098 %}
 9099 
 9100 // Shift Left by variable
 9101 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9102 %{
 9103   predicate(!VM_Version::supports_bmi2());
 9104   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9105   effect(KILL cr);
 9106 
 9107   format %{ "sall    $dst, $shift" %}
 9108   ins_encode %{
 9109     __ sall($dst$$Address);
 9110   %}
 9111   ins_pipe(ialu_mem_reg);
 9112 %}
 9113 
 9114 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9115 %{
 9116   predicate(VM_Version::supports_bmi2());
 9117   match(Set dst (LShiftI src shift));
 9118 
 9119   format %{ "shlxl   $dst, $src, $shift" %}
 9120   ins_encode %{
 9121     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9122   %}
 9123   ins_pipe(ialu_reg_reg);
 9124 %}
 9125 
 9126 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9127 %{
 9128   predicate(VM_Version::supports_bmi2());
 9129   match(Set dst (LShiftI (LoadI src) shift));
 9130   ins_cost(175);
 9131   format %{ "shlxl   $dst, $src, $shift" %}
 9132   ins_encode %{
 9133     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9134   %}
 9135   ins_pipe(ialu_reg_mem);
 9136 %}
 9137 
 9138 // Arithmetic Shift Right by 8-bit immediate
 9139 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9140 %{
 9141   match(Set dst (RShiftI dst shift));
 9142   effect(KILL cr);
 9143 
 9144   format %{ "sarl    $dst, $shift" %}
 9145   ins_encode %{
 9146     __ sarl($dst$$Register, $shift$$constant);
 9147   %}
 9148   ins_pipe(ialu_mem_imm);
 9149 %}
 9150 
 9151 // Arithmetic Shift Right by 8-bit immediate
 9152 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9153 %{
 9154   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9155   effect(KILL cr);
 9156 
 9157   format %{ "sarl    $dst, $shift" %}
 9158   ins_encode %{
 9159     __ sarl($dst$$Address, $shift$$constant);
 9160   %}
 9161   ins_pipe(ialu_mem_imm);
 9162 %}
 9163 
 9164 // Arithmetic Shift Right by variable
 9165 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9166 %{
 9167   predicate(!VM_Version::supports_bmi2());
 9168   match(Set dst (RShiftI dst shift));
 9169   effect(KILL cr);
 9170   format %{ "sarl    $dst, $shift" %}
 9171   ins_encode %{
 9172     __ sarl($dst$$Register);
 9173   %}
 9174   ins_pipe(ialu_reg_reg);
 9175 %}
 9176 
 9177 // Arithmetic Shift Right by variable
 9178 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9179 %{
 9180   predicate(!VM_Version::supports_bmi2());
 9181   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9182   effect(KILL cr);
 9183 
 9184   format %{ "sarl    $dst, $shift" %}
 9185   ins_encode %{
 9186     __ sarl($dst$$Address);
 9187   %}
 9188   ins_pipe(ialu_mem_reg);
 9189 %}
 9190 
 9191 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9192 %{
 9193   predicate(VM_Version::supports_bmi2());
 9194   match(Set dst (RShiftI src shift));
 9195 
 9196   format %{ "sarxl   $dst, $src, $shift" %}
 9197   ins_encode %{
 9198     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9199   %}
 9200   ins_pipe(ialu_reg_reg);
 9201 %}
 9202 
 9203 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9204 %{
 9205   predicate(VM_Version::supports_bmi2());
 9206   match(Set dst (RShiftI (LoadI src) shift));
 9207   ins_cost(175);
 9208   format %{ "sarxl   $dst, $src, $shift" %}
 9209   ins_encode %{
 9210     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9211   %}
 9212   ins_pipe(ialu_reg_mem);
 9213 %}
 9214 
 9215 // Logical Shift Right by 8-bit immediate
 9216 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9217 %{
 9218   match(Set dst (URShiftI dst shift));
 9219   effect(KILL cr);
 9220 
 9221   format %{ "shrl    $dst, $shift" %}
 9222   ins_encode %{
 9223     __ shrl($dst$$Register, $shift$$constant);
 9224   %}
 9225   ins_pipe(ialu_reg);
 9226 %}
 9227 
 9228 // Logical Shift Right by 8-bit immediate
 9229 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9230 %{
 9231   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9232   effect(KILL cr);
 9233 
 9234   format %{ "shrl    $dst, $shift" %}
 9235   ins_encode %{
 9236     __ shrl($dst$$Address, $shift$$constant);
 9237   %}
 9238   ins_pipe(ialu_mem_imm);
 9239 %}
 9240 
 9241 // Logical Shift Right by variable
 9242 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9243 %{
 9244   predicate(!VM_Version::supports_bmi2());
 9245   match(Set dst (URShiftI dst shift));
 9246   effect(KILL cr);
 9247 
 9248   format %{ "shrl    $dst, $shift" %}
 9249   ins_encode %{
 9250     __ shrl($dst$$Register);
 9251   %}
 9252   ins_pipe(ialu_reg_reg);
 9253 %}
 9254 
 9255 // Logical Shift Right by variable
 9256 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9257 %{
 9258   predicate(!VM_Version::supports_bmi2());
 9259   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9260   effect(KILL cr);
 9261 
 9262   format %{ "shrl    $dst, $shift" %}
 9263   ins_encode %{
 9264     __ shrl($dst$$Address);
 9265   %}
 9266   ins_pipe(ialu_mem_reg);
 9267 %}
 9268 
 9269 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9270 %{
 9271   predicate(VM_Version::supports_bmi2());
 9272   match(Set dst (URShiftI src shift));
 9273 
 9274   format %{ "shrxl   $dst, $src, $shift" %}
 9275   ins_encode %{
 9276     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9277   %}
 9278   ins_pipe(ialu_reg_reg);
 9279 %}
 9280 
 9281 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9282 %{
 9283   predicate(VM_Version::supports_bmi2());
 9284   match(Set dst (URShiftI (LoadI src) shift));
 9285   ins_cost(175);
 9286   format %{ "shrxl   $dst, $src, $shift" %}
 9287   ins_encode %{
 9288     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9289   %}
 9290   ins_pipe(ialu_reg_mem);
 9291 %}
 9292 
 9293 // Long Shift Instructions
 9294 // Shift Left by one, two, three
 9295 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9296 %{
 9297   match(Set dst (LShiftL dst shift));
 9298   effect(KILL cr);
 9299 
 9300   format %{ "salq    $dst, $shift" %}
 9301   ins_encode %{
 9302     __ salq($dst$$Register, $shift$$constant);
 9303   %}
 9304   ins_pipe(ialu_reg);
 9305 %}
 9306 
 9307 // Shift Left by 8-bit immediate
 9308 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9309 %{
 9310   match(Set dst (LShiftL dst shift));
 9311   effect(KILL cr);
 9312 
 9313   format %{ "salq    $dst, $shift" %}
 9314   ins_encode %{
 9315     __ salq($dst$$Register, $shift$$constant);
 9316   %}
 9317   ins_pipe(ialu_reg);
 9318 %}
 9319 
 9320 // Shift Left by 8-bit immediate
 9321 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9322 %{
 9323   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9324   effect(KILL cr);
 9325 
 9326   format %{ "salq    $dst, $shift" %}
 9327   ins_encode %{
 9328     __ salq($dst$$Address, $shift$$constant);
 9329   %}
 9330   ins_pipe(ialu_mem_imm);
 9331 %}
 9332 
 9333 // Shift Left by variable
 9334 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9335 %{
 9336   predicate(!VM_Version::supports_bmi2());
 9337   match(Set dst (LShiftL dst shift));
 9338   effect(KILL cr);
 9339 
 9340   format %{ "salq    $dst, $shift" %}
 9341   ins_encode %{
 9342     __ salq($dst$$Register);
 9343   %}
 9344   ins_pipe(ialu_reg_reg);
 9345 %}
 9346 
 9347 // Shift Left by variable
 9348 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9349 %{
 9350   predicate(!VM_Version::supports_bmi2());
 9351   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9352   effect(KILL cr);
 9353 
 9354   format %{ "salq    $dst, $shift" %}
 9355   ins_encode %{
 9356     __ salq($dst$$Address);
 9357   %}
 9358   ins_pipe(ialu_mem_reg);
 9359 %}
 9360 
 9361 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9362 %{
 9363   predicate(VM_Version::supports_bmi2());
 9364   match(Set dst (LShiftL src shift));
 9365 
 9366   format %{ "shlxq   $dst, $src, $shift" %}
 9367   ins_encode %{
 9368     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9369   %}
 9370   ins_pipe(ialu_reg_reg);
 9371 %}
 9372 
 9373 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9374 %{
 9375   predicate(VM_Version::supports_bmi2());
 9376   match(Set dst (LShiftL (LoadL src) shift));
 9377   ins_cost(175);
 9378   format %{ "shlxq   $dst, $src, $shift" %}
 9379   ins_encode %{
 9380     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9381   %}
 9382   ins_pipe(ialu_reg_mem);
 9383 %}
 9384 
 9385 // Arithmetic Shift Right by 8-bit immediate
 9386 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9387 %{
 9388   match(Set dst (RShiftL dst shift));
 9389   effect(KILL cr);
 9390 
 9391   format %{ "sarq    $dst, $shift" %}
 9392   ins_encode %{
 9393     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9394   %}
 9395   ins_pipe(ialu_mem_imm);
 9396 %}
 9397 
 9398 // Arithmetic Shift Right by 8-bit immediate
 9399 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9400 %{
 9401   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9402   effect(KILL cr);
 9403 
 9404   format %{ "sarq    $dst, $shift" %}
 9405   ins_encode %{
 9406     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9407   %}
 9408   ins_pipe(ialu_mem_imm);
 9409 %}
 9410 
 9411 // Arithmetic Shift Right by variable
 9412 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9413 %{
 9414   predicate(!VM_Version::supports_bmi2());
 9415   match(Set dst (RShiftL dst shift));
 9416   effect(KILL cr);
 9417 
 9418   format %{ "sarq    $dst, $shift" %}
 9419   ins_encode %{
 9420     __ sarq($dst$$Register);
 9421   %}
 9422   ins_pipe(ialu_reg_reg);
 9423 %}
 9424 
 9425 // Arithmetic Shift Right by variable
 9426 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9427 %{
 9428   predicate(!VM_Version::supports_bmi2());
 9429   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9430   effect(KILL cr);
 9431 
 9432   format %{ "sarq    $dst, $shift" %}
 9433   ins_encode %{
 9434     __ sarq($dst$$Address);
 9435   %}
 9436   ins_pipe(ialu_mem_reg);
 9437 %}
 9438 
 9439 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9440 %{
 9441   predicate(VM_Version::supports_bmi2());
 9442   match(Set dst (RShiftL src shift));
 9443 
 9444   format %{ "sarxq   $dst, $src, $shift" %}
 9445   ins_encode %{
 9446     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9447   %}
 9448   ins_pipe(ialu_reg_reg);
 9449 %}
 9450 
 9451 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9452 %{
 9453   predicate(VM_Version::supports_bmi2());
 9454   match(Set dst (RShiftL (LoadL src) shift));
 9455   ins_cost(175);
 9456   format %{ "sarxq   $dst, $src, $shift" %}
 9457   ins_encode %{
 9458     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9459   %}
 9460   ins_pipe(ialu_reg_mem);
 9461 %}
 9462 
 9463 // Logical Shift Right by 8-bit immediate
 9464 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9465 %{
 9466   match(Set dst (URShiftL dst shift));
 9467   effect(KILL cr);
 9468 
 9469   format %{ "shrq    $dst, $shift" %}
 9470   ins_encode %{
 9471     __ shrq($dst$$Register, $shift$$constant);
 9472   %}
 9473   ins_pipe(ialu_reg);
 9474 %}
 9475 
 9476 // Logical Shift Right by 8-bit immediate
 9477 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9478 %{
 9479   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9480   effect(KILL cr);
 9481 
 9482   format %{ "shrq    $dst, $shift" %}
 9483   ins_encode %{
 9484     __ shrq($dst$$Address, $shift$$constant);
 9485   %}
 9486   ins_pipe(ialu_mem_imm);
 9487 %}
 9488 
 9489 // Logical Shift Right by variable
 9490 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9491 %{
 9492   predicate(!VM_Version::supports_bmi2());
 9493   match(Set dst (URShiftL dst shift));
 9494   effect(KILL cr);
 9495 
 9496   format %{ "shrq    $dst, $shift" %}
 9497   ins_encode %{
 9498     __ shrq($dst$$Register);
 9499   %}
 9500   ins_pipe(ialu_reg_reg);
 9501 %}
 9502 
 9503 // Logical Shift Right by variable
 9504 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9505 %{
 9506   predicate(!VM_Version::supports_bmi2());
 9507   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9508   effect(KILL cr);
 9509 
 9510   format %{ "shrq    $dst, $shift" %}
 9511   ins_encode %{
 9512     __ shrq($dst$$Address);
 9513   %}
 9514   ins_pipe(ialu_mem_reg);
 9515 %}
 9516 
 9517 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9518 %{
 9519   predicate(VM_Version::supports_bmi2());
 9520   match(Set dst (URShiftL src shift));
 9521 
 9522   format %{ "shrxq   $dst, $src, $shift" %}
 9523   ins_encode %{
 9524     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9525   %}
 9526   ins_pipe(ialu_reg_reg);
 9527 %}
 9528 
 9529 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9530 %{
 9531   predicate(VM_Version::supports_bmi2());
 9532   match(Set dst (URShiftL (LoadL src) shift));
 9533   ins_cost(175);
 9534   format %{ "shrxq   $dst, $src, $shift" %}
 9535   ins_encode %{
 9536     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9537   %}
 9538   ins_pipe(ialu_reg_mem);
 9539 %}
 9540 
 9541 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9542 // This idiom is used by the compiler for the i2b bytecode.
 9543 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9544 %{
 9545   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9546 
 9547   format %{ "movsbl  $dst, $src\t# i2b" %}
 9548   ins_encode %{
 9549     __ movsbl($dst$$Register, $src$$Register);
 9550   %}
 9551   ins_pipe(ialu_reg_reg);
 9552 %}
 9553 
 9554 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9555 // This idiom is used by the compiler the i2s bytecode.
 9556 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9557 %{
 9558   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9559 
 9560   format %{ "movswl  $dst, $src\t# i2s" %}
 9561   ins_encode %{
 9562     __ movswl($dst$$Register, $src$$Register);
 9563   %}
 9564   ins_pipe(ialu_reg_reg);
 9565 %}
 9566 
 9567 // ROL/ROR instructions
 9568 
 9569 // Rotate left by constant.
 9570 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9571 %{
 9572   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9573   match(Set dst (RotateLeft dst shift));
 9574   effect(KILL cr);
 9575   format %{ "roll    $dst, $shift" %}
 9576   ins_encode %{
 9577     __ roll($dst$$Register, $shift$$constant);
 9578   %}
 9579   ins_pipe(ialu_reg);
 9580 %}
 9581 
 9582 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9583 %{
 9584   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9585   match(Set dst (RotateLeft src shift));
 9586   format %{ "rolxl   $dst, $src, $shift" %}
 9587   ins_encode %{
 9588     int shift = 32 - ($shift$$constant & 31);
 9589     __ rorxl($dst$$Register, $src$$Register, shift);
 9590   %}
 9591   ins_pipe(ialu_reg_reg);
 9592 %}
 9593 
 9594 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9595 %{
 9596   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9597   match(Set dst (RotateLeft (LoadI src) shift));
 9598   ins_cost(175);
 9599   format %{ "rolxl   $dst, $src, $shift" %}
 9600   ins_encode %{
 9601     int shift = 32 - ($shift$$constant & 31);
 9602     __ rorxl($dst$$Register, $src$$Address, shift);
 9603   %}
 9604   ins_pipe(ialu_reg_mem);
 9605 %}
 9606 
 9607 // Rotate Left by variable
 9608 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9609 %{
 9610   predicate(n->bottom_type()->basic_type() == T_INT);
 9611   match(Set dst (RotateLeft dst shift));
 9612   effect(KILL cr);
 9613   format %{ "roll    $dst, $shift" %}
 9614   ins_encode %{
 9615     __ roll($dst$$Register);
 9616   %}
 9617   ins_pipe(ialu_reg_reg);
 9618 %}
 9619 
 9620 // Rotate Right by constant.
 9621 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9622 %{
 9623   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9624   match(Set dst (RotateRight dst shift));
 9625   effect(KILL cr);
 9626   format %{ "rorl    $dst, $shift" %}
 9627   ins_encode %{
 9628     __ rorl($dst$$Register, $shift$$constant);
 9629   %}
 9630   ins_pipe(ialu_reg);
 9631 %}
 9632 
 9633 // Rotate Right by constant.
 9634 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9635 %{
 9636   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9637   match(Set dst (RotateRight src shift));
 9638   format %{ "rorxl   $dst, $src, $shift" %}
 9639   ins_encode %{
 9640     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9641   %}
 9642   ins_pipe(ialu_reg_reg);
 9643 %}
 9644 
 9645 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9646 %{
 9647   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9648   match(Set dst (RotateRight (LoadI src) shift));
 9649   ins_cost(175);
 9650   format %{ "rorxl   $dst, $src, $shift" %}
 9651   ins_encode %{
 9652     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9653   %}
 9654   ins_pipe(ialu_reg_mem);
 9655 %}
 9656 
 9657 // Rotate Right by variable
 9658 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9659 %{
 9660   predicate(n->bottom_type()->basic_type() == T_INT);
 9661   match(Set dst (RotateRight dst shift));
 9662   effect(KILL cr);
 9663   format %{ "rorl    $dst, $shift" %}
 9664   ins_encode %{
 9665     __ rorl($dst$$Register);
 9666   %}
 9667   ins_pipe(ialu_reg_reg);
 9668 %}
 9669 
 9670 // Rotate Left by constant.
 9671 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9672 %{
 9673   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9674   match(Set dst (RotateLeft dst shift));
 9675   effect(KILL cr);
 9676   format %{ "rolq    $dst, $shift" %}
 9677   ins_encode %{
 9678     __ rolq($dst$$Register, $shift$$constant);
 9679   %}
 9680   ins_pipe(ialu_reg);
 9681 %}
 9682 
 9683 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9684 %{
 9685   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9686   match(Set dst (RotateLeft src shift));
 9687   format %{ "rolxq   $dst, $src, $shift" %}
 9688   ins_encode %{
 9689     int shift = 64 - ($shift$$constant & 63);
 9690     __ rorxq($dst$$Register, $src$$Register, shift);
 9691   %}
 9692   ins_pipe(ialu_reg_reg);
 9693 %}
 9694 
 9695 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9696 %{
 9697   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9698   match(Set dst (RotateLeft (LoadL src) shift));
 9699   ins_cost(175);
 9700   format %{ "rolxq   $dst, $src, $shift" %}
 9701   ins_encode %{
 9702     int shift = 64 - ($shift$$constant & 63);
 9703     __ rorxq($dst$$Register, $src$$Address, shift);
 9704   %}
 9705   ins_pipe(ialu_reg_mem);
 9706 %}
 9707 
 9708 // Rotate Left by variable
 9709 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9710 %{
 9711   predicate(n->bottom_type()->basic_type() == T_LONG);
 9712   match(Set dst (RotateLeft dst shift));
 9713   effect(KILL cr);
 9714   format %{ "rolq    $dst, $shift" %}
 9715   ins_encode %{
 9716     __ rolq($dst$$Register);
 9717   %}
 9718   ins_pipe(ialu_reg_reg);
 9719 %}
 9720 
 9721 // Rotate Right by constant.
 9722 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9723 %{
 9724   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9725   match(Set dst (RotateRight dst shift));
 9726   effect(KILL cr);
 9727   format %{ "rorq    $dst, $shift" %}
 9728   ins_encode %{
 9729     __ rorq($dst$$Register, $shift$$constant);
 9730   %}
 9731   ins_pipe(ialu_reg);
 9732 %}
 9733 
 9734 // Rotate Right by constant
 9735 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9736 %{
 9737   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9738   match(Set dst (RotateRight src shift));
 9739   format %{ "rorxq   $dst, $src, $shift" %}
 9740   ins_encode %{
 9741     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9742   %}
 9743   ins_pipe(ialu_reg_reg);
 9744 %}
 9745 
 9746 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9747 %{
 9748   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9749   match(Set dst (RotateRight (LoadL src) shift));
 9750   ins_cost(175);
 9751   format %{ "rorxq   $dst, $src, $shift" %}
 9752   ins_encode %{
 9753     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9754   %}
 9755   ins_pipe(ialu_reg_mem);
 9756 %}
 9757 
 9758 // Rotate Right by variable
 9759 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9760 %{
 9761   predicate(n->bottom_type()->basic_type() == T_LONG);
 9762   match(Set dst (RotateRight dst shift));
 9763   effect(KILL cr);
 9764   format %{ "rorq    $dst, $shift" %}
 9765   ins_encode %{
 9766     __ rorq($dst$$Register);
 9767   %}
 9768   ins_pipe(ialu_reg_reg);
 9769 %}
 9770 
 9771 //----------------------------- CompressBits/ExpandBits ------------------------
 9772 
 9773 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9774   predicate(n->bottom_type()->isa_long());
 9775   match(Set dst (CompressBits src mask));
 9776   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9777   ins_encode %{
 9778     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9779   %}
 9780   ins_pipe( pipe_slow );
 9781 %}
 9782 
 9783 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9784   predicate(n->bottom_type()->isa_long());
 9785   match(Set dst (ExpandBits src mask));
 9786   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9787   ins_encode %{
 9788     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 9789   %}
 9790   ins_pipe( pipe_slow );
 9791 %}
 9792 
 9793 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9794   predicate(n->bottom_type()->isa_long());
 9795   match(Set dst (CompressBits src (LoadL mask)));
 9796   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9797   ins_encode %{
 9798     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 9799   %}
 9800   ins_pipe( pipe_slow );
 9801 %}
 9802 
 9803 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9804   predicate(n->bottom_type()->isa_long());
 9805   match(Set dst (ExpandBits src (LoadL mask)));
 9806   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9807   ins_encode %{
 9808     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 9809   %}
 9810   ins_pipe( pipe_slow );
 9811 %}
 9812 
 9813 
 9814 // Logical Instructions
 9815 
 9816 // Integer Logical Instructions
 9817 
 9818 // And Instructions
 9819 // And Register with Register
 9820 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9821 %{
 9822   match(Set dst (AndI dst src));
 9823   effect(KILL cr);
 9824 
 9825   format %{ "andl    $dst, $src\t# int" %}
 9826   ins_encode %{
 9827     __ andl($dst$$Register, $src$$Register);
 9828   %}
 9829   ins_pipe(ialu_reg_reg);
 9830 %}
 9831 
 9832 // And Register with Immediate 255
 9833 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 9834 %{
 9835   match(Set dst (AndI src mask));
 9836 
 9837   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 9838   ins_encode %{
 9839     __ movzbl($dst$$Register, $src$$Register);
 9840   %}
 9841   ins_pipe(ialu_reg);
 9842 %}
 9843 
 9844 // And Register with Immediate 255 and promote to long
 9845 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9846 %{
 9847   match(Set dst (ConvI2L (AndI src mask)));
 9848 
 9849   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9850   ins_encode %{
 9851     __ movzbl($dst$$Register, $src$$Register);
 9852   %}
 9853   ins_pipe(ialu_reg);
 9854 %}
 9855 
 9856 // And Register with Immediate 65535
 9857 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 9858 %{
 9859   match(Set dst (AndI src mask));
 9860 
 9861   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 9862   ins_encode %{
 9863     __ movzwl($dst$$Register, $src$$Register);
 9864   %}
 9865   ins_pipe(ialu_reg);
 9866 %}
 9867 
 9868 // And Register with Immediate 65535 and promote to long
 9869 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9870 %{
 9871   match(Set dst (ConvI2L (AndI src mask)));
 9872 
 9873   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9874   ins_encode %{
 9875     __ movzwl($dst$$Register, $src$$Register);
 9876   %}
 9877   ins_pipe(ialu_reg);
 9878 %}
 9879 
 9880 // Can skip int2long conversions after AND with small bitmask
 9881 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9882 %{
 9883   predicate(VM_Version::supports_bmi2());
 9884   ins_cost(125);
 9885   effect(TEMP tmp, KILL cr);
 9886   match(Set dst (ConvI2L (AndI src mask)));
 9887   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9888   ins_encode %{
 9889     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9890     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9891   %}
 9892   ins_pipe(ialu_reg_reg);
 9893 %}
 9894 
 9895 // And Register with Immediate
 9896 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9897 %{
 9898   match(Set dst (AndI dst src));
 9899   effect(KILL cr);
 9900 
 9901   format %{ "andl    $dst, $src\t# int" %}
 9902   ins_encode %{
 9903     __ andl($dst$$Register, $src$$constant);
 9904   %}
 9905   ins_pipe(ialu_reg);
 9906 %}
 9907 
 9908 // And Register with Memory
 9909 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9910 %{
 9911   match(Set dst (AndI dst (LoadI src)));
 9912   effect(KILL cr);
 9913 
 9914   ins_cost(150);
 9915   format %{ "andl    $dst, $src\t# int" %}
 9916   ins_encode %{
 9917     __ andl($dst$$Register, $src$$Address);
 9918   %}
 9919   ins_pipe(ialu_reg_mem);
 9920 %}
 9921 
 9922 // And Memory with Register
 9923 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9924 %{
 9925   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9926   effect(KILL cr);
 9927 
 9928   ins_cost(150);
 9929   format %{ "andb    $dst, $src\t# byte" %}
 9930   ins_encode %{
 9931     __ andb($dst$$Address, $src$$Register);
 9932   %}
 9933   ins_pipe(ialu_mem_reg);
 9934 %}
 9935 
 9936 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9937 %{
 9938   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9939   effect(KILL cr);
 9940 
 9941   ins_cost(150);
 9942   format %{ "andl    $dst, $src\t# int" %}
 9943   ins_encode %{
 9944     __ andl($dst$$Address, $src$$Register);
 9945   %}
 9946   ins_pipe(ialu_mem_reg);
 9947 %}
 9948 
 9949 // And Memory with Immediate
 9950 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9951 %{
 9952   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9953   effect(KILL cr);
 9954 
 9955   ins_cost(125);
 9956   format %{ "andl    $dst, $src\t# int" %}
 9957   ins_encode %{
 9958     __ andl($dst$$Address, $src$$constant);
 9959   %}
 9960   ins_pipe(ialu_mem_imm);
 9961 %}
 9962 
 9963 // BMI1 instructions
 9964 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9965   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9966   predicate(UseBMI1Instructions);
 9967   effect(KILL cr);
 9968 
 9969   ins_cost(125);
 9970   format %{ "andnl  $dst, $src1, $src2" %}
 9971 
 9972   ins_encode %{
 9973     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9974   %}
 9975   ins_pipe(ialu_reg_mem);
 9976 %}
 9977 
 9978 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9979   match(Set dst (AndI (XorI src1 minus_1) src2));
 9980   predicate(UseBMI1Instructions);
 9981   effect(KILL cr);
 9982 
 9983   format %{ "andnl  $dst, $src1, $src2" %}
 9984 
 9985   ins_encode %{
 9986     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9987   %}
 9988   ins_pipe(ialu_reg);
 9989 %}
 9990 
 9991 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9992   match(Set dst (AndI (SubI imm_zero src) src));
 9993   predicate(UseBMI1Instructions);
 9994   effect(KILL cr);
 9995 
 9996   format %{ "blsil  $dst, $src" %}
 9997 
 9998   ins_encode %{
 9999     __ blsil($dst$$Register, $src$$Register);
10000   %}
10001   ins_pipe(ialu_reg);
10002 %}
10003 
10004 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10005   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10006   predicate(UseBMI1Instructions);
10007   effect(KILL cr);
10008 
10009   ins_cost(125);
10010   format %{ "blsil  $dst, $src" %}
10011 
10012   ins_encode %{
10013     __ blsil($dst$$Register, $src$$Address);
10014   %}
10015   ins_pipe(ialu_reg_mem);
10016 %}
10017 
10018 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10019 %{
10020   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10021   predicate(UseBMI1Instructions);
10022   effect(KILL cr);
10023 
10024   ins_cost(125);
10025   format %{ "blsmskl $dst, $src" %}
10026 
10027   ins_encode %{
10028     __ blsmskl($dst$$Register, $src$$Address);
10029   %}
10030   ins_pipe(ialu_reg_mem);
10031 %}
10032 
10033 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10034 %{
10035   match(Set dst (XorI (AddI src minus_1) src));
10036   predicate(UseBMI1Instructions);
10037   effect(KILL cr);
10038 
10039   format %{ "blsmskl $dst, $src" %}
10040 
10041   ins_encode %{
10042     __ blsmskl($dst$$Register, $src$$Register);
10043   %}
10044 
10045   ins_pipe(ialu_reg);
10046 %}
10047 
10048 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10049 %{
10050   match(Set dst (AndI (AddI src minus_1) src) );
10051   predicate(UseBMI1Instructions);
10052   effect(KILL cr);
10053 
10054   format %{ "blsrl  $dst, $src" %}
10055 
10056   ins_encode %{
10057     __ blsrl($dst$$Register, $src$$Register);
10058   %}
10059 
10060   ins_pipe(ialu_reg_mem);
10061 %}
10062 
10063 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10064 %{
10065   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10066   predicate(UseBMI1Instructions);
10067   effect(KILL cr);
10068 
10069   ins_cost(125);
10070   format %{ "blsrl  $dst, $src" %}
10071 
10072   ins_encode %{
10073     __ blsrl($dst$$Register, $src$$Address);
10074   %}
10075 
10076   ins_pipe(ialu_reg);
10077 %}
10078 
10079 // Or Instructions
10080 // Or Register with Register
10081 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10082 %{
10083   match(Set dst (OrI dst src));
10084   effect(KILL cr);
10085 
10086   format %{ "orl     $dst, $src\t# int" %}
10087   ins_encode %{
10088     __ orl($dst$$Register, $src$$Register);
10089   %}
10090   ins_pipe(ialu_reg_reg);
10091 %}
10092 
10093 // Or Register with Immediate
10094 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10095 %{
10096   match(Set dst (OrI dst src));
10097   effect(KILL cr);
10098 
10099   format %{ "orl     $dst, $src\t# int" %}
10100   ins_encode %{
10101     __ orl($dst$$Register, $src$$constant);
10102   %}
10103   ins_pipe(ialu_reg);
10104 %}
10105 
10106 // Or Register with Memory
10107 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10108 %{
10109   match(Set dst (OrI dst (LoadI src)));
10110   effect(KILL cr);
10111 
10112   ins_cost(150);
10113   format %{ "orl     $dst, $src\t# int" %}
10114   ins_encode %{
10115     __ orl($dst$$Register, $src$$Address);
10116   %}
10117   ins_pipe(ialu_reg_mem);
10118 %}
10119 
10120 // Or Memory with Register
10121 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10122 %{
10123   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10124   effect(KILL cr);
10125 
10126   ins_cost(150);
10127   format %{ "orb    $dst, $src\t# byte" %}
10128   ins_encode %{
10129     __ orb($dst$$Address, $src$$Register);
10130   %}
10131   ins_pipe(ialu_mem_reg);
10132 %}
10133 
10134 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10135 %{
10136   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10137   effect(KILL cr);
10138 
10139   ins_cost(150);
10140   format %{ "orl     $dst, $src\t# int" %}
10141   ins_encode %{
10142     __ orl($dst$$Address, $src$$Register);
10143   %}
10144   ins_pipe(ialu_mem_reg);
10145 %}
10146 
10147 // Or Memory with Immediate
10148 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10149 %{
10150   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10151   effect(KILL cr);
10152 
10153   ins_cost(125);
10154   format %{ "orl     $dst, $src\t# int" %}
10155   ins_encode %{
10156     __ orl($dst$$Address, $src$$constant);
10157   %}
10158   ins_pipe(ialu_mem_imm);
10159 %}
10160 
10161 // Xor Instructions
10162 // Xor Register with Register
10163 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10164 %{
10165   match(Set dst (XorI dst src));
10166   effect(KILL cr);
10167 
10168   format %{ "xorl    $dst, $src\t# int" %}
10169   ins_encode %{
10170     __ xorl($dst$$Register, $src$$Register);
10171   %}
10172   ins_pipe(ialu_reg_reg);
10173 %}
10174 
10175 // Xor Register with Immediate -1
10176 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
10177   match(Set dst (XorI dst imm));
10178 
10179   format %{ "not    $dst" %}
10180   ins_encode %{
10181      __ notl($dst$$Register);
10182   %}
10183   ins_pipe(ialu_reg);
10184 %}
10185 
10186 // Xor Register with Immediate
10187 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10188 %{
10189   match(Set dst (XorI dst src));
10190   effect(KILL cr);
10191 
10192   format %{ "xorl    $dst, $src\t# int" %}
10193   ins_encode %{
10194     __ xorl($dst$$Register, $src$$constant);
10195   %}
10196   ins_pipe(ialu_reg);
10197 %}
10198 
10199 // Xor Register with Memory
10200 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10201 %{
10202   match(Set dst (XorI dst (LoadI src)));
10203   effect(KILL cr);
10204 
10205   ins_cost(150);
10206   format %{ "xorl    $dst, $src\t# int" %}
10207   ins_encode %{
10208     __ xorl($dst$$Register, $src$$Address);
10209   %}
10210   ins_pipe(ialu_reg_mem);
10211 %}
10212 
10213 // Xor Memory with Register
10214 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10215 %{
10216   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10217   effect(KILL cr);
10218 
10219   ins_cost(150);
10220   format %{ "xorb    $dst, $src\t# byte" %}
10221   ins_encode %{
10222     __ xorb($dst$$Address, $src$$Register);
10223   %}
10224   ins_pipe(ialu_mem_reg);
10225 %}
10226 
10227 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10228 %{
10229   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10230   effect(KILL cr);
10231 
10232   ins_cost(150);
10233   format %{ "xorl    $dst, $src\t# int" %}
10234   ins_encode %{
10235     __ xorl($dst$$Address, $src$$Register);
10236   %}
10237   ins_pipe(ialu_mem_reg);
10238 %}
10239 
10240 // Xor Memory with Immediate
10241 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10242 %{
10243   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10244   effect(KILL cr);
10245 
10246   ins_cost(125);
10247   format %{ "xorl    $dst, $src\t# int" %}
10248   ins_encode %{
10249     __ xorl($dst$$Address, $src$$constant);
10250   %}
10251   ins_pipe(ialu_mem_imm);
10252 %}
10253 
10254 
10255 // Long Logical Instructions
10256 
10257 // And Instructions
10258 // And Register with Register
10259 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10260 %{
10261   match(Set dst (AndL dst src));
10262   effect(KILL cr);
10263 
10264   format %{ "andq    $dst, $src\t# long" %}
10265   ins_encode %{
10266     __ andq($dst$$Register, $src$$Register);
10267   %}
10268   ins_pipe(ialu_reg_reg);
10269 %}
10270 
10271 // And Register with Immediate 255
10272 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10273 %{
10274   match(Set dst (AndL src mask));
10275 
10276   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10277   ins_encode %{
10278     // movzbl zeroes out the upper 32-bit and does not need REX.W
10279     __ movzbl($dst$$Register, $src$$Register);
10280   %}
10281   ins_pipe(ialu_reg);
10282 %}
10283 
10284 // And Register with Immediate 65535
10285 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10286 %{
10287   match(Set dst (AndL src mask));
10288 
10289   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10290   ins_encode %{
10291     // movzwl zeroes out the upper 32-bit and does not need REX.W
10292     __ movzwl($dst$$Register, $src$$Register);
10293   %}
10294   ins_pipe(ialu_reg);
10295 %}
10296 
10297 // And Register with Immediate
10298 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10299 %{
10300   match(Set dst (AndL dst src));
10301   effect(KILL cr);
10302 
10303   format %{ "andq    $dst, $src\t# long" %}
10304   ins_encode %{
10305     __ andq($dst$$Register, $src$$constant);
10306   %}
10307   ins_pipe(ialu_reg);
10308 %}
10309 
10310 // And Register with Memory
10311 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10312 %{
10313   match(Set dst (AndL dst (LoadL src)));
10314   effect(KILL cr);
10315 
10316   ins_cost(150);
10317   format %{ "andq    $dst, $src\t# long" %}
10318   ins_encode %{
10319     __ andq($dst$$Register, $src$$Address);
10320   %}
10321   ins_pipe(ialu_reg_mem);
10322 %}
10323 
10324 // And Memory with Register
10325 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10326 %{
10327   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10328   effect(KILL cr);
10329 
10330   ins_cost(150);
10331   format %{ "andq    $dst, $src\t# long" %}
10332   ins_encode %{
10333     __ andq($dst$$Address, $src$$Register);
10334   %}
10335   ins_pipe(ialu_mem_reg);
10336 %}
10337 
10338 // And Memory with Immediate
10339 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10340 %{
10341   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10342   effect(KILL cr);
10343 
10344   ins_cost(125);
10345   format %{ "andq    $dst, $src\t# long" %}
10346   ins_encode %{
10347     __ andq($dst$$Address, $src$$constant);
10348   %}
10349   ins_pipe(ialu_mem_imm);
10350 %}
10351 
10352 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10353 %{
10354   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10355   // because AND/OR works well enough for 8/32-bit values.
10356   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10357 
10358   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10359   effect(KILL cr);
10360 
10361   ins_cost(125);
10362   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10363   ins_encode %{
10364     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10365   %}
10366   ins_pipe(ialu_mem_imm);
10367 %}
10368 
10369 // BMI1 instructions
10370 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10371   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10372   predicate(UseBMI1Instructions);
10373   effect(KILL cr);
10374 
10375   ins_cost(125);
10376   format %{ "andnq  $dst, $src1, $src2" %}
10377 
10378   ins_encode %{
10379     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10380   %}
10381   ins_pipe(ialu_reg_mem);
10382 %}
10383 
10384 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10385   match(Set dst (AndL (XorL src1 minus_1) src2));
10386   predicate(UseBMI1Instructions);
10387   effect(KILL cr);
10388 
10389   format %{ "andnq  $dst, $src1, $src2" %}
10390 
10391   ins_encode %{
10392   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10393   %}
10394   ins_pipe(ialu_reg_mem);
10395 %}
10396 
10397 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10398   match(Set dst (AndL (SubL imm_zero src) src));
10399   predicate(UseBMI1Instructions);
10400   effect(KILL cr);
10401 
10402   format %{ "blsiq  $dst, $src" %}
10403 
10404   ins_encode %{
10405     __ blsiq($dst$$Register, $src$$Register);
10406   %}
10407   ins_pipe(ialu_reg);
10408 %}
10409 
10410 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10411   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10412   predicate(UseBMI1Instructions);
10413   effect(KILL cr);
10414 
10415   ins_cost(125);
10416   format %{ "blsiq  $dst, $src" %}
10417 
10418   ins_encode %{
10419     __ blsiq($dst$$Register, $src$$Address);
10420   %}
10421   ins_pipe(ialu_reg_mem);
10422 %}
10423 
10424 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10425 %{
10426   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10427   predicate(UseBMI1Instructions);
10428   effect(KILL cr);
10429 
10430   ins_cost(125);
10431   format %{ "blsmskq $dst, $src" %}
10432 
10433   ins_encode %{
10434     __ blsmskq($dst$$Register, $src$$Address);
10435   %}
10436   ins_pipe(ialu_reg_mem);
10437 %}
10438 
10439 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10440 %{
10441   match(Set dst (XorL (AddL src minus_1) src));
10442   predicate(UseBMI1Instructions);
10443   effect(KILL cr);
10444 
10445   format %{ "blsmskq $dst, $src" %}
10446 
10447   ins_encode %{
10448     __ blsmskq($dst$$Register, $src$$Register);
10449   %}
10450 
10451   ins_pipe(ialu_reg);
10452 %}
10453 
10454 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10455 %{
10456   match(Set dst (AndL (AddL src minus_1) src) );
10457   predicate(UseBMI1Instructions);
10458   effect(KILL cr);
10459 
10460   format %{ "blsrq  $dst, $src" %}
10461 
10462   ins_encode %{
10463     __ blsrq($dst$$Register, $src$$Register);
10464   %}
10465 
10466   ins_pipe(ialu_reg);
10467 %}
10468 
10469 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10470 %{
10471   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10472   predicate(UseBMI1Instructions);
10473   effect(KILL cr);
10474 
10475   ins_cost(125);
10476   format %{ "blsrq  $dst, $src" %}
10477 
10478   ins_encode %{
10479     __ blsrq($dst$$Register, $src$$Address);
10480   %}
10481 
10482   ins_pipe(ialu_reg);
10483 %}
10484 
10485 // Or Instructions
10486 // Or Register with Register
10487 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10488 %{
10489   match(Set dst (OrL dst src));
10490   effect(KILL cr);
10491 
10492   format %{ "orq     $dst, $src\t# long" %}
10493   ins_encode %{
10494     __ orq($dst$$Register, $src$$Register);
10495   %}
10496   ins_pipe(ialu_reg_reg);
10497 %}
10498 
10499 // Use any_RegP to match R15 (TLS register) without spilling.
10500 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10501   match(Set dst (OrL dst (CastP2X src)));
10502   effect(KILL cr);
10503 
10504   format %{ "orq     $dst, $src\t# long" %}
10505   ins_encode %{
10506     __ orq($dst$$Register, $src$$Register);
10507   %}
10508   ins_pipe(ialu_reg_reg);
10509 %}
10510 
10511 
10512 // Or Register with Immediate
10513 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10514 %{
10515   match(Set dst (OrL dst src));
10516   effect(KILL cr);
10517 
10518   format %{ "orq     $dst, $src\t# long" %}
10519   ins_encode %{
10520     __ orq($dst$$Register, $src$$constant);
10521   %}
10522   ins_pipe(ialu_reg);
10523 %}
10524 
10525 // Or Register with Memory
10526 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10527 %{
10528   match(Set dst (OrL dst (LoadL src)));
10529   effect(KILL cr);
10530 
10531   ins_cost(150);
10532   format %{ "orq     $dst, $src\t# long" %}
10533   ins_encode %{
10534     __ orq($dst$$Register, $src$$Address);
10535   %}
10536   ins_pipe(ialu_reg_mem);
10537 %}
10538 
10539 // Or Memory with Register
10540 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10541 %{
10542   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10543   effect(KILL cr);
10544 
10545   ins_cost(150);
10546   format %{ "orq     $dst, $src\t# long" %}
10547   ins_encode %{
10548     __ orq($dst$$Address, $src$$Register);
10549   %}
10550   ins_pipe(ialu_mem_reg);
10551 %}
10552 
10553 // Or Memory with Immediate
10554 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10555 %{
10556   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10557   effect(KILL cr);
10558 
10559   ins_cost(125);
10560   format %{ "orq     $dst, $src\t# long" %}
10561   ins_encode %{
10562     __ orq($dst$$Address, $src$$constant);
10563   %}
10564   ins_pipe(ialu_mem_imm);
10565 %}
10566 
10567 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10568 %{
10569   // con should be a pure 64-bit power of 2 immediate
10570   // because AND/OR works well enough for 8/32-bit values.
10571   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10572 
10573   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10574   effect(KILL cr);
10575 
10576   ins_cost(125);
10577   format %{ "btsq    $dst, log2($con)\t# long" %}
10578   ins_encode %{
10579     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10580   %}
10581   ins_pipe(ialu_mem_imm);
10582 %}
10583 
10584 // Xor Instructions
10585 // Xor Register with Register
10586 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10587 %{
10588   match(Set dst (XorL dst src));
10589   effect(KILL cr);
10590 
10591   format %{ "xorq    $dst, $src\t# long" %}
10592   ins_encode %{
10593     __ xorq($dst$$Register, $src$$Register);
10594   %}
10595   ins_pipe(ialu_reg_reg);
10596 %}
10597 
10598 // Xor Register with Immediate -1
10599 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10600   match(Set dst (XorL dst imm));
10601 
10602   format %{ "notq   $dst" %}
10603   ins_encode %{
10604      __ notq($dst$$Register);
10605   %}
10606   ins_pipe(ialu_reg);
10607 %}
10608 
10609 // Xor Register with Immediate
10610 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10611 %{
10612   match(Set dst (XorL dst src));
10613   effect(KILL cr);
10614 
10615   format %{ "xorq    $dst, $src\t# long" %}
10616   ins_encode %{
10617     __ xorq($dst$$Register, $src$$constant);
10618   %}
10619   ins_pipe(ialu_reg);
10620 %}
10621 
10622 // Xor Register with Memory
10623 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10624 %{
10625   match(Set dst (XorL dst (LoadL src)));
10626   effect(KILL cr);
10627 
10628   ins_cost(150);
10629   format %{ "xorq    $dst, $src\t# long" %}
10630   ins_encode %{
10631     __ xorq($dst$$Register, $src$$Address);
10632   %}
10633   ins_pipe(ialu_reg_mem);
10634 %}
10635 
10636 // Xor Memory with Register
10637 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10638 %{
10639   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10640   effect(KILL cr);
10641 
10642   ins_cost(150);
10643   format %{ "xorq    $dst, $src\t# long" %}
10644   ins_encode %{
10645     __ xorq($dst$$Address, $src$$Register);
10646   %}
10647   ins_pipe(ialu_mem_reg);
10648 %}
10649 
10650 // Xor Memory with Immediate
10651 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10652 %{
10653   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10654   effect(KILL cr);
10655 
10656   ins_cost(125);
10657   format %{ "xorq    $dst, $src\t# long" %}
10658   ins_encode %{
10659     __ xorq($dst$$Address, $src$$constant);
10660   %}
10661   ins_pipe(ialu_mem_imm);
10662 %}
10663 
10664 // Convert Int to Boolean
10665 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10666 %{
10667   match(Set dst (Conv2B src));
10668   effect(KILL cr);
10669 
10670   format %{ "testl   $src, $src\t# ci2b\n\t"
10671             "setnz   $dst\n\t"
10672             "movzbl  $dst, $dst" %}
10673   ins_encode %{
10674     __ testl($src$$Register, $src$$Register);
10675     __ set_byte_if_not_zero($dst$$Register);
10676     __ movzbl($dst$$Register, $dst$$Register);
10677   %}
10678   ins_pipe(pipe_slow); // XXX
10679 %}
10680 
10681 // Convert Pointer to Boolean
10682 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10683 %{
10684   match(Set dst (Conv2B src));
10685   effect(KILL cr);
10686 
10687   format %{ "testq   $src, $src\t# cp2b\n\t"
10688             "setnz   $dst\n\t"
10689             "movzbl  $dst, $dst" %}
10690   ins_encode %{
10691     __ testq($src$$Register, $src$$Register);
10692     __ set_byte_if_not_zero($dst$$Register);
10693     __ movzbl($dst$$Register, $dst$$Register);
10694   %}
10695   ins_pipe(pipe_slow); // XXX
10696 %}
10697 
10698 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10699 %{
10700   match(Set dst (CmpLTMask p q));
10701   effect(KILL cr);
10702 
10703   ins_cost(400);
10704   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10705             "setlt   $dst\n\t"
10706             "movzbl  $dst, $dst\n\t"
10707             "negl    $dst" %}
10708   ins_encode %{
10709     __ cmpl($p$$Register, $q$$Register);
10710     __ setl($dst$$Register);
10711     __ movzbl($dst$$Register, $dst$$Register);
10712     __ negl($dst$$Register);
10713   %}
10714   ins_pipe(pipe_slow);
10715 %}
10716 
10717 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10718 %{
10719   match(Set dst (CmpLTMask dst zero));
10720   effect(KILL cr);
10721 
10722   ins_cost(100);
10723   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10724   ins_encode %{
10725     __ sarl($dst$$Register, 31);
10726   %}
10727   ins_pipe(ialu_reg);
10728 %}
10729 
10730 /* Better to save a register than avoid a branch */
10731 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10732 %{
10733   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10734   effect(KILL cr);
10735   ins_cost(300);
10736   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10737             "jge     done\n\t"
10738             "addl    $p,$y\n"
10739             "done:   " %}
10740   ins_encode %{
10741     Register Rp = $p$$Register;
10742     Register Rq = $q$$Register;
10743     Register Ry = $y$$Register;
10744     Label done;
10745     __ subl(Rp, Rq);
10746     __ jccb(Assembler::greaterEqual, done);
10747     __ addl(Rp, Ry);
10748     __ bind(done);
10749   %}
10750   ins_pipe(pipe_cmplt);
10751 %}
10752 
10753 /* Better to save a register than avoid a branch */
10754 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10755 %{
10756   match(Set y (AndI (CmpLTMask p q) y));
10757   effect(KILL cr);
10758 
10759   ins_cost(300);
10760 
10761   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10762             "jlt     done\n\t"
10763             "xorl    $y, $y\n"
10764             "done:   " %}
10765   ins_encode %{
10766     Register Rp = $p$$Register;
10767     Register Rq = $q$$Register;
10768     Register Ry = $y$$Register;
10769     Label done;
10770     __ cmpl(Rp, Rq);
10771     __ jccb(Assembler::less, done);
10772     __ xorl(Ry, Ry);
10773     __ bind(done);
10774   %}
10775   ins_pipe(pipe_cmplt);
10776 %}
10777 
10778 
10779 //---------- FP Instructions------------------------------------------------
10780 
10781 // Really expensive, avoid
10782 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10783 %{
10784   match(Set cr (CmpF src1 src2));
10785 
10786   ins_cost(500);
10787   format %{ "ucomiss $src1, $src2\n\t"
10788             "jnp,s   exit\n\t"
10789             "pushfq\t# saw NaN, set CF\n\t"
10790             "andq    [rsp], #0xffffff2b\n\t"
10791             "popfq\n"
10792     "exit:" %}
10793   ins_encode %{
10794     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10795     emit_cmpfp_fixup(_masm);
10796   %}
10797   ins_pipe(pipe_slow);
10798 %}
10799 
10800 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10801   match(Set cr (CmpF src1 src2));
10802 
10803   ins_cost(100);
10804   format %{ "ucomiss $src1, $src2" %}
10805   ins_encode %{
10806     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10807   %}
10808   ins_pipe(pipe_slow);
10809 %}
10810 
10811 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10812   match(Set cr (CmpF src1 (LoadF src2)));
10813 
10814   ins_cost(100);
10815   format %{ "ucomiss $src1, $src2" %}
10816   ins_encode %{
10817     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10818   %}
10819   ins_pipe(pipe_slow);
10820 %}
10821 
10822 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10823   match(Set cr (CmpF src con));
10824   ins_cost(100);
10825   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10826   ins_encode %{
10827     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10828   %}
10829   ins_pipe(pipe_slow);
10830 %}
10831 
10832 // Really expensive, avoid
10833 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10834 %{
10835   match(Set cr (CmpD src1 src2));
10836 
10837   ins_cost(500);
10838   format %{ "ucomisd $src1, $src2\n\t"
10839             "jnp,s   exit\n\t"
10840             "pushfq\t# saw NaN, set CF\n\t"
10841             "andq    [rsp], #0xffffff2b\n\t"
10842             "popfq\n"
10843     "exit:" %}
10844   ins_encode %{
10845     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10846     emit_cmpfp_fixup(_masm);
10847   %}
10848   ins_pipe(pipe_slow);
10849 %}
10850 
10851 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10852   match(Set cr (CmpD src1 src2));
10853 
10854   ins_cost(100);
10855   format %{ "ucomisd $src1, $src2 test" %}
10856   ins_encode %{
10857     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10858   %}
10859   ins_pipe(pipe_slow);
10860 %}
10861 
10862 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10863   match(Set cr (CmpD src1 (LoadD src2)));
10864 
10865   ins_cost(100);
10866   format %{ "ucomisd $src1, $src2" %}
10867   ins_encode %{
10868     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10869   %}
10870   ins_pipe(pipe_slow);
10871 %}
10872 
10873 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10874   match(Set cr (CmpD src con));
10875   ins_cost(100);
10876   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10877   ins_encode %{
10878     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10879   %}
10880   ins_pipe(pipe_slow);
10881 %}
10882 
10883 // Compare into -1,0,1
10884 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10885 %{
10886   match(Set dst (CmpF3 src1 src2));
10887   effect(KILL cr);
10888 
10889   ins_cost(275);
10890   format %{ "ucomiss $src1, $src2\n\t"
10891             "movl    $dst, #-1\n\t"
10892             "jp,s    done\n\t"
10893             "jb,s    done\n\t"
10894             "setne   $dst\n\t"
10895             "movzbl  $dst, $dst\n"
10896     "done:" %}
10897   ins_encode %{
10898     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10899     emit_cmpfp3(_masm, $dst$$Register);
10900   %}
10901   ins_pipe(pipe_slow);
10902 %}
10903 
10904 // Compare into -1,0,1
10905 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10906 %{
10907   match(Set dst (CmpF3 src1 (LoadF src2)));
10908   effect(KILL cr);
10909 
10910   ins_cost(275);
10911   format %{ "ucomiss $src1, $src2\n\t"
10912             "movl    $dst, #-1\n\t"
10913             "jp,s    done\n\t"
10914             "jb,s    done\n\t"
10915             "setne   $dst\n\t"
10916             "movzbl  $dst, $dst\n"
10917     "done:" %}
10918   ins_encode %{
10919     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10920     emit_cmpfp3(_masm, $dst$$Register);
10921   %}
10922   ins_pipe(pipe_slow);
10923 %}
10924 
10925 // Compare into -1,0,1
10926 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10927   match(Set dst (CmpF3 src con));
10928   effect(KILL cr);
10929 
10930   ins_cost(275);
10931   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10932             "movl    $dst, #-1\n\t"
10933             "jp,s    done\n\t"
10934             "jb,s    done\n\t"
10935             "setne   $dst\n\t"
10936             "movzbl  $dst, $dst\n"
10937     "done:" %}
10938   ins_encode %{
10939     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10940     emit_cmpfp3(_masm, $dst$$Register);
10941   %}
10942   ins_pipe(pipe_slow);
10943 %}
10944 
10945 // Compare into -1,0,1
10946 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10947 %{
10948   match(Set dst (CmpD3 src1 src2));
10949   effect(KILL cr);
10950 
10951   ins_cost(275);
10952   format %{ "ucomisd $src1, $src2\n\t"
10953             "movl    $dst, #-1\n\t"
10954             "jp,s    done\n\t"
10955             "jb,s    done\n\t"
10956             "setne   $dst\n\t"
10957             "movzbl  $dst, $dst\n"
10958     "done:" %}
10959   ins_encode %{
10960     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10961     emit_cmpfp3(_masm, $dst$$Register);
10962   %}
10963   ins_pipe(pipe_slow);
10964 %}
10965 
10966 // Compare into -1,0,1
10967 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10968 %{
10969   match(Set dst (CmpD3 src1 (LoadD src2)));
10970   effect(KILL cr);
10971 
10972   ins_cost(275);
10973   format %{ "ucomisd $src1, $src2\n\t"
10974             "movl    $dst, #-1\n\t"
10975             "jp,s    done\n\t"
10976             "jb,s    done\n\t"
10977             "setne   $dst\n\t"
10978             "movzbl  $dst, $dst\n"
10979     "done:" %}
10980   ins_encode %{
10981     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10982     emit_cmpfp3(_masm, $dst$$Register);
10983   %}
10984   ins_pipe(pipe_slow);
10985 %}
10986 
10987 // Compare into -1,0,1
10988 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10989   match(Set dst (CmpD3 src con));
10990   effect(KILL cr);
10991 
10992   ins_cost(275);
10993   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10994             "movl    $dst, #-1\n\t"
10995             "jp,s    done\n\t"
10996             "jb,s    done\n\t"
10997             "setne   $dst\n\t"
10998             "movzbl  $dst, $dst\n"
10999     "done:" %}
11000   ins_encode %{
11001     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11002     emit_cmpfp3(_masm, $dst$$Register);
11003   %}
11004   ins_pipe(pipe_slow);
11005 %}
11006 
11007 //----------Arithmetic Conversion Instructions---------------------------------
11008 
11009 instruct convF2D_reg_reg(regD dst, regF src)
11010 %{
11011   match(Set dst (ConvF2D src));
11012 
11013   format %{ "cvtss2sd $dst, $src" %}
11014   ins_encode %{
11015     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11016   %}
11017   ins_pipe(pipe_slow); // XXX
11018 %}
11019 
11020 instruct convF2D_reg_mem(regD dst, memory src)
11021 %{
11022   match(Set dst (ConvF2D (LoadF src)));
11023 
11024   format %{ "cvtss2sd $dst, $src" %}
11025   ins_encode %{
11026     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11027   %}
11028   ins_pipe(pipe_slow); // XXX
11029 %}
11030 
11031 instruct convD2F_reg_reg(regF dst, regD src)
11032 %{
11033   match(Set dst (ConvD2F src));
11034 
11035   format %{ "cvtsd2ss $dst, $src" %}
11036   ins_encode %{
11037     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11038   %}
11039   ins_pipe(pipe_slow); // XXX
11040 %}
11041 
11042 instruct convD2F_reg_mem(regF dst, memory src)
11043 %{
11044   match(Set dst (ConvD2F (LoadD src)));
11045 
11046   format %{ "cvtsd2ss $dst, $src" %}
11047   ins_encode %{
11048     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11049   %}
11050   ins_pipe(pipe_slow); // XXX
11051 %}
11052 
11053 // XXX do mem variants
11054 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11055 %{
11056   match(Set dst (ConvF2I src));
11057   effect(KILL cr);
11058   format %{ "convert_f2i $dst,$src" %}
11059   ins_encode %{
11060     __ convert_f2i($dst$$Register, $src$$XMMRegister);
11061   %}
11062   ins_pipe(pipe_slow);
11063 %}
11064 
11065 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11066 %{
11067   match(Set dst (ConvF2L src));
11068   effect(KILL cr);
11069   format %{ "convert_f2l $dst,$src"%}
11070   ins_encode %{
11071     __ convert_f2l($dst$$Register, $src$$XMMRegister);
11072   %}
11073   ins_pipe(pipe_slow);
11074 %}
11075 
11076 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11077 %{
11078   match(Set dst (ConvD2I src));
11079   effect(KILL cr);
11080   format %{ "convert_d2i $dst,$src"%}
11081   ins_encode %{
11082     __ convert_d2i($dst$$Register, $src$$XMMRegister);
11083   %}
11084   ins_pipe(pipe_slow);
11085 %}
11086 
11087 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11088 %{
11089   match(Set dst (ConvD2L src));
11090   effect(KILL cr);
11091   format %{ "convert_d2l $dst,$src"%}
11092   ins_encode %{
11093     __ convert_d2l($dst$$Register, $src$$XMMRegister);
11094   %}
11095   ins_pipe(pipe_slow);
11096 %}
11097 
11098 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11099 %{
11100   match(Set dst (RoundD src));
11101   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11102   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11103   ins_encode %{
11104     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11105   %}
11106   ins_pipe(pipe_slow);
11107 %}
11108 
11109 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11110 %{
11111   match(Set dst (RoundF src));
11112   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11113   format %{ "round_float $dst,$src" %}
11114   ins_encode %{
11115     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11116   %}
11117   ins_pipe(pipe_slow);
11118 %}
11119 
11120 instruct convI2F_reg_reg(regF dst, rRegI src)
11121 %{
11122   predicate(!UseXmmI2F);
11123   match(Set dst (ConvI2F src));
11124 
11125   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11126   ins_encode %{
11127     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11128   %}
11129   ins_pipe(pipe_slow); // XXX
11130 %}
11131 
11132 instruct convI2F_reg_mem(regF dst, memory src)
11133 %{
11134   match(Set dst (ConvI2F (LoadI src)));
11135 
11136   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11137   ins_encode %{
11138     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11139   %}
11140   ins_pipe(pipe_slow); // XXX
11141 %}
11142 
11143 instruct convI2D_reg_reg(regD dst, rRegI src)
11144 %{
11145   predicate(!UseXmmI2D);
11146   match(Set dst (ConvI2D src));
11147 
11148   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11149   ins_encode %{
11150     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11151   %}
11152   ins_pipe(pipe_slow); // XXX
11153 %}
11154 
11155 instruct convI2D_reg_mem(regD dst, memory src)
11156 %{
11157   match(Set dst (ConvI2D (LoadI src)));
11158 
11159   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11160   ins_encode %{
11161     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11162   %}
11163   ins_pipe(pipe_slow); // XXX
11164 %}
11165 
11166 instruct convXI2F_reg(regF dst, rRegI src)
11167 %{
11168   predicate(UseXmmI2F);
11169   match(Set dst (ConvI2F src));
11170 
11171   format %{ "movdl $dst, $src\n\t"
11172             "cvtdq2psl $dst, $dst\t# i2f" %}
11173   ins_encode %{
11174     __ movdl($dst$$XMMRegister, $src$$Register);
11175     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11176   %}
11177   ins_pipe(pipe_slow); // XXX
11178 %}
11179 
11180 instruct convXI2D_reg(regD dst, rRegI src)
11181 %{
11182   predicate(UseXmmI2D);
11183   match(Set dst (ConvI2D src));
11184 
11185   format %{ "movdl $dst, $src\n\t"
11186             "cvtdq2pdl $dst, $dst\t# i2d" %}
11187   ins_encode %{
11188     __ movdl($dst$$XMMRegister, $src$$Register);
11189     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11190   %}
11191   ins_pipe(pipe_slow); // XXX
11192 %}
11193 
11194 instruct convL2F_reg_reg(regF dst, rRegL src)
11195 %{
11196   match(Set dst (ConvL2F src));
11197 
11198   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11199   ins_encode %{
11200     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11201   %}
11202   ins_pipe(pipe_slow); // XXX
11203 %}
11204 
11205 instruct convL2F_reg_mem(regF dst, memory src)
11206 %{
11207   match(Set dst (ConvL2F (LoadL src)));
11208 
11209   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11210   ins_encode %{
11211     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11212   %}
11213   ins_pipe(pipe_slow); // XXX
11214 %}
11215 
11216 instruct convL2D_reg_reg(regD dst, rRegL src)
11217 %{
11218   match(Set dst (ConvL2D src));
11219 
11220   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11221   ins_encode %{
11222     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11223   %}
11224   ins_pipe(pipe_slow); // XXX
11225 %}
11226 
11227 instruct convL2D_reg_mem(regD dst, memory src)
11228 %{
11229   match(Set dst (ConvL2D (LoadL src)));
11230 
11231   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11232   ins_encode %{
11233     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11234   %}
11235   ins_pipe(pipe_slow); // XXX
11236 %}
11237 
11238 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11239 %{
11240   match(Set dst (ConvI2L src));
11241 
11242   ins_cost(125);
11243   format %{ "movslq  $dst, $src\t# i2l" %}
11244   ins_encode %{
11245     __ movslq($dst$$Register, $src$$Register);
11246   %}
11247   ins_pipe(ialu_reg_reg);
11248 %}
11249 
11250 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11251 // %{
11252 //   match(Set dst (ConvI2L src));
11253 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11254 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11255 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11256 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11257 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11258 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11259 
11260 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11261 //   ins_encode(enc_copy(dst, src));
11262 // //   opcode(0x63); // needs REX.W
11263 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11264 //   ins_pipe(ialu_reg_reg);
11265 // %}
11266 
11267 // Zero-extend convert int to long
11268 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11269 %{
11270   match(Set dst (AndL (ConvI2L src) mask));
11271 
11272   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11273   ins_encode %{
11274     if ($dst$$reg != $src$$reg) {
11275       __ movl($dst$$Register, $src$$Register);
11276     }
11277   %}
11278   ins_pipe(ialu_reg_reg);
11279 %}
11280 
11281 // Zero-extend convert int to long
11282 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11283 %{
11284   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11285 
11286   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11287   ins_encode %{
11288     __ movl($dst$$Register, $src$$Address);
11289   %}
11290   ins_pipe(ialu_reg_mem);
11291 %}
11292 
11293 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11294 %{
11295   match(Set dst (AndL src mask));
11296 
11297   format %{ "movl    $dst, $src\t# zero-extend long" %}
11298   ins_encode %{
11299     __ movl($dst$$Register, $src$$Register);
11300   %}
11301   ins_pipe(ialu_reg_reg);
11302 %}
11303 
11304 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11305 %{
11306   match(Set dst (ConvL2I src));
11307 
11308   format %{ "movl    $dst, $src\t# l2i" %}
11309   ins_encode %{
11310     __ movl($dst$$Register, $src$$Register);
11311   %}
11312   ins_pipe(ialu_reg_reg);
11313 %}
11314 
11315 
11316 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11317   match(Set dst (MoveF2I src));
11318   effect(DEF dst, USE src);
11319 
11320   ins_cost(125);
11321   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11322   ins_encode %{
11323     __ movl($dst$$Register, Address(rsp, $src$$disp));
11324   %}
11325   ins_pipe(ialu_reg_mem);
11326 %}
11327 
11328 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11329   match(Set dst (MoveI2F src));
11330   effect(DEF dst, USE src);
11331 
11332   ins_cost(125);
11333   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11334   ins_encode %{
11335     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11336   %}
11337   ins_pipe(pipe_slow);
11338 %}
11339 
11340 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11341   match(Set dst (MoveD2L src));
11342   effect(DEF dst, USE src);
11343 
11344   ins_cost(125);
11345   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11346   ins_encode %{
11347     __ movq($dst$$Register, Address(rsp, $src$$disp));
11348   %}
11349   ins_pipe(ialu_reg_mem);
11350 %}
11351 
11352 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11353   predicate(!UseXmmLoadAndClearUpper);
11354   match(Set dst (MoveL2D src));
11355   effect(DEF dst, USE src);
11356 
11357   ins_cost(125);
11358   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11359   ins_encode %{
11360     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11361   %}
11362   ins_pipe(pipe_slow);
11363 %}
11364 
11365 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11366   predicate(UseXmmLoadAndClearUpper);
11367   match(Set dst (MoveL2D src));
11368   effect(DEF dst, USE src);
11369 
11370   ins_cost(125);
11371   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11372   ins_encode %{
11373     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11374   %}
11375   ins_pipe(pipe_slow);
11376 %}
11377 
11378 
11379 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11380   match(Set dst (MoveF2I src));
11381   effect(DEF dst, USE src);
11382 
11383   ins_cost(95); // XXX
11384   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11385   ins_encode %{
11386     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11387   %}
11388   ins_pipe(pipe_slow);
11389 %}
11390 
11391 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11392   match(Set dst (MoveI2F src));
11393   effect(DEF dst, USE src);
11394 
11395   ins_cost(100);
11396   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11397   ins_encode %{
11398     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11399   %}
11400   ins_pipe( ialu_mem_reg );
11401 %}
11402 
11403 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11404   match(Set dst (MoveD2L src));
11405   effect(DEF dst, USE src);
11406 
11407   ins_cost(95); // XXX
11408   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11409   ins_encode %{
11410     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11411   %}
11412   ins_pipe(pipe_slow);
11413 %}
11414 
11415 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11416   match(Set dst (MoveL2D src));
11417   effect(DEF dst, USE src);
11418 
11419   ins_cost(100);
11420   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11421   ins_encode %{
11422     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11423   %}
11424   ins_pipe(ialu_mem_reg);
11425 %}
11426 
11427 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11428   match(Set dst (MoveF2I src));
11429   effect(DEF dst, USE src);
11430   ins_cost(85);
11431   format %{ "movd    $dst,$src\t# MoveF2I" %}
11432   ins_encode %{
11433     __ movdl($dst$$Register, $src$$XMMRegister);
11434   %}
11435   ins_pipe( pipe_slow );
11436 %}
11437 
11438 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11439   match(Set dst (MoveD2L src));
11440   effect(DEF dst, USE src);
11441   ins_cost(85);
11442   format %{ "movd    $dst,$src\t# MoveD2L" %}
11443   ins_encode %{
11444     __ movdq($dst$$Register, $src$$XMMRegister);
11445   %}
11446   ins_pipe( pipe_slow );
11447 %}
11448 
11449 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11450   match(Set dst (MoveI2F src));
11451   effect(DEF dst, USE src);
11452   ins_cost(100);
11453   format %{ "movd    $dst,$src\t# MoveI2F" %}
11454   ins_encode %{
11455     __ movdl($dst$$XMMRegister, $src$$Register);
11456   %}
11457   ins_pipe( pipe_slow );
11458 %}
11459 
11460 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11461   match(Set dst (MoveL2D src));
11462   effect(DEF dst, USE src);
11463   ins_cost(100);
11464   format %{ "movd    $dst,$src\t# MoveL2D" %}
11465   ins_encode %{
11466      __ movdq($dst$$XMMRegister, $src$$Register);
11467   %}
11468   ins_pipe( pipe_slow );
11469 %}
11470 
11471 // Fast clearing of an array
11472 // Small ClearArray non-AVX512.
11473 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11474                   Universe dummy, rFlagsReg cr)
11475 %{
11476   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11477   match(Set dummy (ClearArray cnt base));
11478   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11479 
11480   format %{ $$template
11481     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11482     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11483     $$emit$$"jg      LARGE\n\t"
11484     $$emit$$"dec     rcx\n\t"
11485     $$emit$$"js      DONE\t# Zero length\n\t"
11486     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11487     $$emit$$"dec     rcx\n\t"
11488     $$emit$$"jge     LOOP\n\t"
11489     $$emit$$"jmp     DONE\n\t"
11490     $$emit$$"# LARGE:\n\t"
11491     if (UseFastStosb) {
11492        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11493        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11494     } else if (UseXMMForObjInit) {
11495        $$emit$$"mov     rdi,rax\n\t"
11496        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11497        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11498        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11499        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11500        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11501        $$emit$$"add     0x40,rax\n\t"
11502        $$emit$$"# L_zero_64_bytes:\n\t"
11503        $$emit$$"sub     0x8,rcx\n\t"
11504        $$emit$$"jge     L_loop\n\t"
11505        $$emit$$"add     0x4,rcx\n\t"
11506        $$emit$$"jl      L_tail\n\t"
11507        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11508        $$emit$$"add     0x20,rax\n\t"
11509        $$emit$$"sub     0x4,rcx\n\t"
11510        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11511        $$emit$$"add     0x4,rcx\n\t"
11512        $$emit$$"jle     L_end\n\t"
11513        $$emit$$"dec     rcx\n\t"
11514        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11515        $$emit$$"vmovq   xmm0,(rax)\n\t"
11516        $$emit$$"add     0x8,rax\n\t"
11517        $$emit$$"dec     rcx\n\t"
11518        $$emit$$"jge     L_sloop\n\t"
11519        $$emit$$"# L_end:\n\t"
11520     } else {
11521        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11522     }
11523     $$emit$$"# DONE"
11524   %}
11525   ins_encode %{
11526     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11527                  $tmp$$XMMRegister, false, knoreg);
11528   %}
11529   ins_pipe(pipe_slow);
11530 %}
11531 
11532 // Small ClearArray AVX512 non-constant length.
11533 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11534                        Universe dummy, rFlagsReg cr)
11535 %{
11536   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11537   match(Set dummy (ClearArray cnt base));
11538   ins_cost(125);
11539   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11540 
11541   format %{ $$template
11542     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11543     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11544     $$emit$$"jg      LARGE\n\t"
11545     $$emit$$"dec     rcx\n\t"
11546     $$emit$$"js      DONE\t# Zero length\n\t"
11547     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11548     $$emit$$"dec     rcx\n\t"
11549     $$emit$$"jge     LOOP\n\t"
11550     $$emit$$"jmp     DONE\n\t"
11551     $$emit$$"# LARGE:\n\t"
11552     if (UseFastStosb) {
11553        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11554        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11555     } else if (UseXMMForObjInit) {
11556        $$emit$$"mov     rdi,rax\n\t"
11557        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11558        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11559        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11560        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11561        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11562        $$emit$$"add     0x40,rax\n\t"
11563        $$emit$$"# L_zero_64_bytes:\n\t"
11564        $$emit$$"sub     0x8,rcx\n\t"
11565        $$emit$$"jge     L_loop\n\t"
11566        $$emit$$"add     0x4,rcx\n\t"
11567        $$emit$$"jl      L_tail\n\t"
11568        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11569        $$emit$$"add     0x20,rax\n\t"
11570        $$emit$$"sub     0x4,rcx\n\t"
11571        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11572        $$emit$$"add     0x4,rcx\n\t"
11573        $$emit$$"jle     L_end\n\t"
11574        $$emit$$"dec     rcx\n\t"
11575        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11576        $$emit$$"vmovq   xmm0,(rax)\n\t"
11577        $$emit$$"add     0x8,rax\n\t"
11578        $$emit$$"dec     rcx\n\t"
11579        $$emit$$"jge     L_sloop\n\t"
11580        $$emit$$"# L_end:\n\t"
11581     } else {
11582        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11583     }
11584     $$emit$$"# DONE"
11585   %}
11586   ins_encode %{
11587     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11588                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11589   %}
11590   ins_pipe(pipe_slow);
11591 %}
11592 
11593 // Large ClearArray non-AVX512.
11594 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11595                         Universe dummy, rFlagsReg cr)
11596 %{
11597   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11598   match(Set dummy (ClearArray cnt base));
11599   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11600 
11601   format %{ $$template
11602     if (UseFastStosb) {
11603        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11604        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11605        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11606     } else if (UseXMMForObjInit) {
11607        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11608        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11609        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11610        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11611        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11612        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11613        $$emit$$"add     0x40,rax\n\t"
11614        $$emit$$"# L_zero_64_bytes:\n\t"
11615        $$emit$$"sub     0x8,rcx\n\t"
11616        $$emit$$"jge     L_loop\n\t"
11617        $$emit$$"add     0x4,rcx\n\t"
11618        $$emit$$"jl      L_tail\n\t"
11619        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11620        $$emit$$"add     0x20,rax\n\t"
11621        $$emit$$"sub     0x4,rcx\n\t"
11622        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11623        $$emit$$"add     0x4,rcx\n\t"
11624        $$emit$$"jle     L_end\n\t"
11625        $$emit$$"dec     rcx\n\t"
11626        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11627        $$emit$$"vmovq   xmm0,(rax)\n\t"
11628        $$emit$$"add     0x8,rax\n\t"
11629        $$emit$$"dec     rcx\n\t"
11630        $$emit$$"jge     L_sloop\n\t"
11631        $$emit$$"# L_end:\n\t"
11632     } else {
11633        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11634        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11635     }
11636   %}
11637   ins_encode %{
11638     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11639                  $tmp$$XMMRegister, true, knoreg);
11640   %}
11641   ins_pipe(pipe_slow);
11642 %}
11643 
11644 // Large ClearArray AVX512.
11645 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11646                              Universe dummy, rFlagsReg cr)
11647 %{
11648   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11649   match(Set dummy (ClearArray cnt base));
11650   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11651 
11652   format %{ $$template
11653     if (UseFastStosb) {
11654        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11655        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11656        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11657     } else if (UseXMMForObjInit) {
11658        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11659        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11660        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11661        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11662        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11663        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11664        $$emit$$"add     0x40,rax\n\t"
11665        $$emit$$"# L_zero_64_bytes:\n\t"
11666        $$emit$$"sub     0x8,rcx\n\t"
11667        $$emit$$"jge     L_loop\n\t"
11668        $$emit$$"add     0x4,rcx\n\t"
11669        $$emit$$"jl      L_tail\n\t"
11670        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11671        $$emit$$"add     0x20,rax\n\t"
11672        $$emit$$"sub     0x4,rcx\n\t"
11673        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11674        $$emit$$"add     0x4,rcx\n\t"
11675        $$emit$$"jle     L_end\n\t"
11676        $$emit$$"dec     rcx\n\t"
11677        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11678        $$emit$$"vmovq   xmm0,(rax)\n\t"
11679        $$emit$$"add     0x8,rax\n\t"
11680        $$emit$$"dec     rcx\n\t"
11681        $$emit$$"jge     L_sloop\n\t"
11682        $$emit$$"# L_end:\n\t"
11683     } else {
11684        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11685        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11686     }
11687   %}
11688   ins_encode %{
11689     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11690                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11691   %}
11692   ins_pipe(pipe_slow);
11693 %}
11694 
11695 // Small ClearArray AVX512 constant length.
11696 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11697 %{
11698   predicate(!((ClearArrayNode*)n)->is_large() &&
11699               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11700   match(Set dummy (ClearArray cnt base));
11701   ins_cost(100);
11702   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11703   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11704   ins_encode %{
11705    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11706   %}
11707   ins_pipe(pipe_slow);
11708 %}
11709 
11710 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11711                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11712 %{
11713   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11714   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11715   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11716 
11717   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11718   ins_encode %{
11719     __ string_compare($str1$$Register, $str2$$Register,
11720                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11721                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11722   %}
11723   ins_pipe( pipe_slow );
11724 %}
11725 
11726 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11727                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11728 %{
11729   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11730   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11731   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11732 
11733   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11734   ins_encode %{
11735     __ string_compare($str1$$Register, $str2$$Register,
11736                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11737                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11738   %}
11739   ins_pipe( pipe_slow );
11740 %}
11741 
11742 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11743                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11744 %{
11745   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11746   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11747   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11748 
11749   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11750   ins_encode %{
11751     __ string_compare($str1$$Register, $str2$$Register,
11752                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11753                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11754   %}
11755   ins_pipe( pipe_slow );
11756 %}
11757 
11758 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11759                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11760 %{
11761   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11762   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11763   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11764 
11765   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11766   ins_encode %{
11767     __ string_compare($str1$$Register, $str2$$Register,
11768                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11769                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11770   %}
11771   ins_pipe( pipe_slow );
11772 %}
11773 
11774 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11775                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11776 %{
11777   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11778   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11779   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11780 
11781   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11782   ins_encode %{
11783     __ string_compare($str1$$Register, $str2$$Register,
11784                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11785                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11786   %}
11787   ins_pipe( pipe_slow );
11788 %}
11789 
11790 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11791                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11792 %{
11793   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11794   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11795   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11796 
11797   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11798   ins_encode %{
11799     __ string_compare($str1$$Register, $str2$$Register,
11800                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11801                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11802   %}
11803   ins_pipe( pipe_slow );
11804 %}
11805 
11806 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11807                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11808 %{
11809   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11810   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11811   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11812 
11813   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11814   ins_encode %{
11815     __ string_compare($str2$$Register, $str1$$Register,
11816                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11817                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11818   %}
11819   ins_pipe( pipe_slow );
11820 %}
11821 
11822 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11823                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11824 %{
11825   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11826   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11827   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11828 
11829   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11830   ins_encode %{
11831     __ string_compare($str2$$Register, $str1$$Register,
11832                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11833                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11834   %}
11835   ins_pipe( pipe_slow );
11836 %}
11837 
11838 // fast search of substring with known size.
11839 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11840                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11841 %{
11842   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11843   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11844   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11845 
11846   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11847   ins_encode %{
11848     int icnt2 = (int)$int_cnt2$$constant;
11849     if (icnt2 >= 16) {
11850       // IndexOf for constant substrings with size >= 16 elements
11851       // which don't need to be loaded through stack.
11852       __ string_indexofC8($str1$$Register, $str2$$Register,
11853                           $cnt1$$Register, $cnt2$$Register,
11854                           icnt2, $result$$Register,
11855                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11856     } else {
11857       // Small strings are loaded through stack if they cross page boundary.
11858       __ string_indexof($str1$$Register, $str2$$Register,
11859                         $cnt1$$Register, $cnt2$$Register,
11860                         icnt2, $result$$Register,
11861                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11862     }
11863   %}
11864   ins_pipe( pipe_slow );
11865 %}
11866 
11867 // fast search of substring with known size.
11868 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11869                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11870 %{
11871   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11872   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11873   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11874 
11875   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11876   ins_encode %{
11877     int icnt2 = (int)$int_cnt2$$constant;
11878     if (icnt2 >= 8) {
11879       // IndexOf for constant substrings with size >= 8 elements
11880       // which don't need to be loaded through stack.
11881       __ string_indexofC8($str1$$Register, $str2$$Register,
11882                           $cnt1$$Register, $cnt2$$Register,
11883                           icnt2, $result$$Register,
11884                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11885     } else {
11886       // Small strings are loaded through stack if they cross page boundary.
11887       __ string_indexof($str1$$Register, $str2$$Register,
11888                         $cnt1$$Register, $cnt2$$Register,
11889                         icnt2, $result$$Register,
11890                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11891     }
11892   %}
11893   ins_pipe( pipe_slow );
11894 %}
11895 
11896 // fast search of substring with known size.
11897 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11898                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11899 %{
11900   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11901   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11902   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11903 
11904   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11905   ins_encode %{
11906     int icnt2 = (int)$int_cnt2$$constant;
11907     if (icnt2 >= 8) {
11908       // IndexOf for constant substrings with size >= 8 elements
11909       // which don't need to be loaded through stack.
11910       __ string_indexofC8($str1$$Register, $str2$$Register,
11911                           $cnt1$$Register, $cnt2$$Register,
11912                           icnt2, $result$$Register,
11913                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11914     } else {
11915       // Small strings are loaded through stack if they cross page boundary.
11916       __ string_indexof($str1$$Register, $str2$$Register,
11917                         $cnt1$$Register, $cnt2$$Register,
11918                         icnt2, $result$$Register,
11919                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11920     }
11921   %}
11922   ins_pipe( pipe_slow );
11923 %}
11924 
11925 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11926                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11927 %{
11928   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11929   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11930   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11931 
11932   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11933   ins_encode %{
11934     __ string_indexof($str1$$Register, $str2$$Register,
11935                       $cnt1$$Register, $cnt2$$Register,
11936                       (-1), $result$$Register,
11937                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11938   %}
11939   ins_pipe( pipe_slow );
11940 %}
11941 
11942 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11943                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11944 %{
11945   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11946   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11947   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11948 
11949   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11950   ins_encode %{
11951     __ string_indexof($str1$$Register, $str2$$Register,
11952                       $cnt1$$Register, $cnt2$$Register,
11953                       (-1), $result$$Register,
11954                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11955   %}
11956   ins_pipe( pipe_slow );
11957 %}
11958 
11959 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11960                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11961 %{
11962   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11963   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11964   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11965 
11966   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11967   ins_encode %{
11968     __ string_indexof($str1$$Register, $str2$$Register,
11969                       $cnt1$$Register, $cnt2$$Register,
11970                       (-1), $result$$Register,
11971                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11972   %}
11973   ins_pipe( pipe_slow );
11974 %}
11975 
11976 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11977                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11978 %{
11979   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11980   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11981   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11982   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11983   ins_encode %{
11984     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11985                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11986   %}
11987   ins_pipe( pipe_slow );
11988 %}
11989 
11990 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11991                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11992 %{
11993   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11994   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11995   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11996   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11997   ins_encode %{
11998     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11999                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12000   %}
12001   ins_pipe( pipe_slow );
12002 %}
12003 
12004 // fast string equals
12005 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12006                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12007 %{
12008   predicate(!VM_Version::supports_avx512vlbw());
12009   match(Set result (StrEquals (Binary str1 str2) cnt));
12010   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12011 
12012   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12013   ins_encode %{
12014     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12015                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12016                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12017   %}
12018   ins_pipe( pipe_slow );
12019 %}
12020 
12021 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12022                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12023 %{
12024   predicate(VM_Version::supports_avx512vlbw());
12025   match(Set result (StrEquals (Binary str1 str2) cnt));
12026   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12027 
12028   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12029   ins_encode %{
12030     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12031                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12032                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12033   %}
12034   ins_pipe( pipe_slow );
12035 %}
12036 
12037 // fast array equals
12038 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12039                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12040 %{
12041   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12042   match(Set result (AryEq ary1 ary2));
12043   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12044 
12045   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12046   ins_encode %{
12047     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12048                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12049                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12050   %}
12051   ins_pipe( pipe_slow );
12052 %}
12053 
12054 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12055                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12056 %{
12057   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12058   match(Set result (AryEq ary1 ary2));
12059   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12060 
12061   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12062   ins_encode %{
12063     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12064                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12065                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12066   %}
12067   ins_pipe( pipe_slow );
12068 %}
12069 
12070 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12071                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12072 %{
12073   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12074   match(Set result (AryEq ary1 ary2));
12075   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12076 
12077   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12078   ins_encode %{
12079     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12080                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12081                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12082   %}
12083   ins_pipe( pipe_slow );
12084 %}
12085 
12086 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12087                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12088 %{
12089   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12090   match(Set result (AryEq ary1 ary2));
12091   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12092 
12093   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12094   ins_encode %{
12095     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12096                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12097                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12098   %}
12099   ins_pipe( pipe_slow );
12100 %}
12101 
12102 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
12103                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
12104                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
12105                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
12106                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
12107 %{
12108   predicate(UseAVX >= 2);
12109   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
12110   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
12111          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
12112          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
12113          USE basic_type, KILL cr);
12114 
12115   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
12116   ins_encode %{
12117     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
12118                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
12119                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
12120                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
12121                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
12122                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
12123                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
12124   %}
12125   ins_pipe( pipe_slow );
12126 %}
12127 
12128 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12129                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12130 %{
12131   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12132   match(Set result (CountPositives ary1 len));
12133   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12134 
12135   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12136   ins_encode %{
12137     __ count_positives($ary1$$Register, $len$$Register,
12138                        $result$$Register, $tmp3$$Register,
12139                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12140   %}
12141   ins_pipe( pipe_slow );
12142 %}
12143 
12144 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12145                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12146 %{
12147   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12148   match(Set result (CountPositives ary1 len));
12149   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12150 
12151   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12152   ins_encode %{
12153     __ count_positives($ary1$$Register, $len$$Register,
12154                        $result$$Register, $tmp3$$Register,
12155                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12156   %}
12157   ins_pipe( pipe_slow );
12158 %}
12159 
12160 // fast char[] to byte[] compression
12161 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12162                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12163   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12164   match(Set result (StrCompressedCopy src (Binary dst len)));
12165   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12166          USE_KILL len, KILL tmp5, KILL cr);
12167 
12168   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12169   ins_encode %{
12170     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12171                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12172                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12173                            knoreg, knoreg);
12174   %}
12175   ins_pipe( pipe_slow );
12176 %}
12177 
12178 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12179                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12180   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12181   match(Set result (StrCompressedCopy src (Binary dst len)));
12182   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12183          USE_KILL len, KILL tmp5, KILL cr);
12184 
12185   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12186   ins_encode %{
12187     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12188                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12189                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12190                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12191   %}
12192   ins_pipe( pipe_slow );
12193 %}
12194 // fast byte[] to char[] inflation
12195 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12196                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12197   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12198   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12199   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12200 
12201   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12202   ins_encode %{
12203     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12204                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12205   %}
12206   ins_pipe( pipe_slow );
12207 %}
12208 
12209 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12210                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12211   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12212   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12213   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12214 
12215   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12216   ins_encode %{
12217     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12218                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12219   %}
12220   ins_pipe( pipe_slow );
12221 %}
12222 
12223 // encode char[] to byte[] in ISO_8859_1
12224 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12225                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12226                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12227   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12228   match(Set result (EncodeISOArray src (Binary dst len)));
12229   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12230 
12231   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12232   ins_encode %{
12233     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12234                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12235                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12236   %}
12237   ins_pipe( pipe_slow );
12238 %}
12239 
12240 // encode char[] to byte[] in ASCII
12241 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12242                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12243                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12244   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12245   match(Set result (EncodeISOArray src (Binary dst len)));
12246   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12247 
12248   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12249   ins_encode %{
12250     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12251                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12252                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12253   %}
12254   ins_pipe( pipe_slow );
12255 %}
12256 
12257 //----------Overflow Math Instructions-----------------------------------------
12258 
12259 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12260 %{
12261   match(Set cr (OverflowAddI op1 op2));
12262   effect(DEF cr, USE_KILL op1, USE op2);
12263 
12264   format %{ "addl    $op1, $op2\t# overflow check int" %}
12265 
12266   ins_encode %{
12267     __ addl($op1$$Register, $op2$$Register);
12268   %}
12269   ins_pipe(ialu_reg_reg);
12270 %}
12271 
12272 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12273 %{
12274   match(Set cr (OverflowAddI op1 op2));
12275   effect(DEF cr, USE_KILL op1, USE op2);
12276 
12277   format %{ "addl    $op1, $op2\t# overflow check int" %}
12278 
12279   ins_encode %{
12280     __ addl($op1$$Register, $op2$$constant);
12281   %}
12282   ins_pipe(ialu_reg_reg);
12283 %}
12284 
12285 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12286 %{
12287   match(Set cr (OverflowAddL op1 op2));
12288   effect(DEF cr, USE_KILL op1, USE op2);
12289 
12290   format %{ "addq    $op1, $op2\t# overflow check long" %}
12291   ins_encode %{
12292     __ addq($op1$$Register, $op2$$Register);
12293   %}
12294   ins_pipe(ialu_reg_reg);
12295 %}
12296 
12297 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12298 %{
12299   match(Set cr (OverflowAddL op1 op2));
12300   effect(DEF cr, USE_KILL op1, USE op2);
12301 
12302   format %{ "addq    $op1, $op2\t# overflow check long" %}
12303   ins_encode %{
12304     __ addq($op1$$Register, $op2$$constant);
12305   %}
12306   ins_pipe(ialu_reg_reg);
12307 %}
12308 
12309 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12310 %{
12311   match(Set cr (OverflowSubI op1 op2));
12312 
12313   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12314   ins_encode %{
12315     __ cmpl($op1$$Register, $op2$$Register);
12316   %}
12317   ins_pipe(ialu_reg_reg);
12318 %}
12319 
12320 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12321 %{
12322   match(Set cr (OverflowSubI op1 op2));
12323 
12324   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12325   ins_encode %{
12326     __ cmpl($op1$$Register, $op2$$constant);
12327   %}
12328   ins_pipe(ialu_reg_reg);
12329 %}
12330 
12331 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12332 %{
12333   match(Set cr (OverflowSubL op1 op2));
12334 
12335   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12336   ins_encode %{
12337     __ cmpq($op1$$Register, $op2$$Register);
12338   %}
12339   ins_pipe(ialu_reg_reg);
12340 %}
12341 
12342 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12343 %{
12344   match(Set cr (OverflowSubL op1 op2));
12345 
12346   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12347   ins_encode %{
12348     __ cmpq($op1$$Register, $op2$$constant);
12349   %}
12350   ins_pipe(ialu_reg_reg);
12351 %}
12352 
12353 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12354 %{
12355   match(Set cr (OverflowSubI zero op2));
12356   effect(DEF cr, USE_KILL op2);
12357 
12358   format %{ "negl    $op2\t# overflow check int" %}
12359   ins_encode %{
12360     __ negl($op2$$Register);
12361   %}
12362   ins_pipe(ialu_reg_reg);
12363 %}
12364 
12365 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12366 %{
12367   match(Set cr (OverflowSubL zero op2));
12368   effect(DEF cr, USE_KILL op2);
12369 
12370   format %{ "negq    $op2\t# overflow check long" %}
12371   ins_encode %{
12372     __ negq($op2$$Register);
12373   %}
12374   ins_pipe(ialu_reg_reg);
12375 %}
12376 
12377 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12378 %{
12379   match(Set cr (OverflowMulI op1 op2));
12380   effect(DEF cr, USE_KILL op1, USE op2);
12381 
12382   format %{ "imull    $op1, $op2\t# overflow check int" %}
12383   ins_encode %{
12384     __ imull($op1$$Register, $op2$$Register);
12385   %}
12386   ins_pipe(ialu_reg_reg_alu0);
12387 %}
12388 
12389 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12390 %{
12391   match(Set cr (OverflowMulI op1 op2));
12392   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12393 
12394   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12395   ins_encode %{
12396     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12397   %}
12398   ins_pipe(ialu_reg_reg_alu0);
12399 %}
12400 
12401 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12402 %{
12403   match(Set cr (OverflowMulL op1 op2));
12404   effect(DEF cr, USE_KILL op1, USE op2);
12405 
12406   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12407   ins_encode %{
12408     __ imulq($op1$$Register, $op2$$Register);
12409   %}
12410   ins_pipe(ialu_reg_reg_alu0);
12411 %}
12412 
12413 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12414 %{
12415   match(Set cr (OverflowMulL op1 op2));
12416   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12417 
12418   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12419   ins_encode %{
12420     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12421   %}
12422   ins_pipe(ialu_reg_reg_alu0);
12423 %}
12424 
12425 
12426 //----------Control Flow Instructions------------------------------------------
12427 // Signed compare Instructions
12428 
12429 // XXX more variants!!
12430 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12431 %{
12432   match(Set cr (CmpI op1 op2));
12433   effect(DEF cr, USE op1, USE op2);
12434 
12435   format %{ "cmpl    $op1, $op2" %}
12436   ins_encode %{
12437     __ cmpl($op1$$Register, $op2$$Register);
12438   %}
12439   ins_pipe(ialu_cr_reg_reg);
12440 %}
12441 
12442 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12443 %{
12444   match(Set cr (CmpI op1 op2));
12445 
12446   format %{ "cmpl    $op1, $op2" %}
12447   ins_encode %{
12448     __ cmpl($op1$$Register, $op2$$constant);
12449   %}
12450   ins_pipe(ialu_cr_reg_imm);
12451 %}
12452 
12453 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12454 %{
12455   match(Set cr (CmpI op1 (LoadI op2)));
12456 
12457   ins_cost(500); // XXX
12458   format %{ "cmpl    $op1, $op2" %}
12459   ins_encode %{
12460     __ cmpl($op1$$Register, $op2$$Address);
12461   %}
12462   ins_pipe(ialu_cr_reg_mem);
12463 %}
12464 
12465 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12466 %{
12467   match(Set cr (CmpI src zero));
12468 
12469   format %{ "testl   $src, $src" %}
12470   ins_encode %{
12471     __ testl($src$$Register, $src$$Register);
12472   %}
12473   ins_pipe(ialu_cr_reg_imm);
12474 %}
12475 
12476 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12477 %{
12478   match(Set cr (CmpI (AndI src con) zero));
12479 
12480   format %{ "testl   $src, $con" %}
12481   ins_encode %{
12482     __ testl($src$$Register, $con$$constant);
12483   %}
12484   ins_pipe(ialu_cr_reg_imm);
12485 %}
12486 
12487 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12488 %{
12489   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12490 
12491   format %{ "testl   $src, $mem" %}
12492   ins_encode %{
12493     __ testl($src$$Register, $mem$$Address);
12494   %}
12495   ins_pipe(ialu_cr_reg_mem);
12496 %}
12497 
12498 // Unsigned compare Instructions; really, same as signed except they
12499 // produce an rFlagsRegU instead of rFlagsReg.
12500 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12501 %{
12502   match(Set cr (CmpU op1 op2));
12503 
12504   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12505   ins_encode %{
12506     __ cmpl($op1$$Register, $op2$$Register);
12507   %}
12508   ins_pipe(ialu_cr_reg_reg);
12509 %}
12510 
12511 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12512 %{
12513   match(Set cr (CmpU op1 op2));
12514 
12515   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12516   ins_encode %{
12517     __ cmpl($op1$$Register, $op2$$constant);
12518   %}
12519   ins_pipe(ialu_cr_reg_imm);
12520 %}
12521 
12522 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12523 %{
12524   match(Set cr (CmpU op1 (LoadI op2)));
12525 
12526   ins_cost(500); // XXX
12527   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12528   ins_encode %{
12529     __ cmpl($op1$$Register, $op2$$Address);
12530   %}
12531   ins_pipe(ialu_cr_reg_mem);
12532 %}
12533 
12534 // // // Cisc-spilled version of cmpU_rReg
12535 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12536 // //%{
12537 // //  match(Set cr (CmpU (LoadI op1) op2));
12538 // //
12539 // //  format %{ "CMPu   $op1,$op2" %}
12540 // //  ins_cost(500);
12541 // //  opcode(0x39);  /* Opcode 39 /r */
12542 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12543 // //%}
12544 
12545 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12546 %{
12547   match(Set cr (CmpU src zero));
12548 
12549   format %{ "testl   $src, $src\t# unsigned" %}
12550   ins_encode %{
12551     __ testl($src$$Register, $src$$Register);
12552   %}
12553   ins_pipe(ialu_cr_reg_imm);
12554 %}
12555 
12556 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12557 %{
12558   match(Set cr (CmpP op1 op2));
12559 
12560   format %{ "cmpq    $op1, $op2\t# ptr" %}
12561   ins_encode %{
12562     __ cmpq($op1$$Register, $op2$$Register);
12563   %}
12564   ins_pipe(ialu_cr_reg_reg);
12565 %}
12566 
12567 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12568 %{
12569   match(Set cr (CmpP op1 (LoadP op2)));
12570   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12571 
12572   ins_cost(500); // XXX
12573   format %{ "cmpq    $op1, $op2\t# ptr" %}
12574   ins_encode %{
12575     __ cmpq($op1$$Register, $op2$$Address);
12576   %}
12577   ins_pipe(ialu_cr_reg_mem);
12578 %}
12579 
12580 // // // Cisc-spilled version of cmpP_rReg
12581 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12582 // //%{
12583 // //  match(Set cr (CmpP (LoadP op1) op2));
12584 // //
12585 // //  format %{ "CMPu   $op1,$op2" %}
12586 // //  ins_cost(500);
12587 // //  opcode(0x39);  /* Opcode 39 /r */
12588 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12589 // //%}
12590 
12591 // XXX this is generalized by compP_rReg_mem???
12592 // Compare raw pointer (used in out-of-heap check).
12593 // Only works because non-oop pointers must be raw pointers
12594 // and raw pointers have no anti-dependencies.
12595 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12596 %{
12597   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12598             n->in(2)->as_Load()->barrier_data() == 0);
12599   match(Set cr (CmpP op1 (LoadP op2)));
12600 
12601   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12602   ins_encode %{
12603     __ cmpq($op1$$Register, $op2$$Address);
12604   %}
12605   ins_pipe(ialu_cr_reg_mem);
12606 %}
12607 
12608 // This will generate a signed flags result. This should be OK since
12609 // any compare to a zero should be eq/neq.
12610 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12611 %{
12612   match(Set cr (CmpP src zero));
12613 
12614   format %{ "testq   $src, $src\t# ptr" %}
12615   ins_encode %{
12616     __ testq($src$$Register, $src$$Register);
12617   %}
12618   ins_pipe(ialu_cr_reg_imm);
12619 %}
12620 
12621 // This will generate a signed flags result. This should be OK since
12622 // any compare to a zero should be eq/neq.
12623 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12624 %{
12625   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12626             n->in(1)->as_Load()->barrier_data() == 0);
12627   match(Set cr (CmpP (LoadP op) zero));
12628 
12629   ins_cost(500); // XXX
12630   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12631   ins_encode %{
12632     __ testq($op$$Address, 0xFFFFFFFF);
12633   %}
12634   ins_pipe(ialu_cr_reg_imm);
12635 %}
12636 
12637 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12638 %{
12639   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12640             n->in(1)->as_Load()->barrier_data() == 0);
12641   match(Set cr (CmpP (LoadP mem) zero));
12642 
12643   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12644   ins_encode %{
12645     __ cmpq(r12, $mem$$Address);
12646   %}
12647   ins_pipe(ialu_cr_reg_mem);
12648 %}
12649 
12650 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12651 %{
12652   match(Set cr (CmpN op1 op2));
12653 
12654   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12655   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12656   ins_pipe(ialu_cr_reg_reg);
12657 %}
12658 
12659 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12660 %{
12661   match(Set cr (CmpN src (LoadN mem)));
12662 
12663   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12664   ins_encode %{
12665     __ cmpl($src$$Register, $mem$$Address);
12666   %}
12667   ins_pipe(ialu_cr_reg_mem);
12668 %}
12669 
12670 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12671   match(Set cr (CmpN op1 op2));
12672 
12673   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12674   ins_encode %{
12675     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12676   %}
12677   ins_pipe(ialu_cr_reg_imm);
12678 %}
12679 
12680 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12681 %{
12682   match(Set cr (CmpN src (LoadN mem)));
12683 
12684   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12685   ins_encode %{
12686     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12687   %}
12688   ins_pipe(ialu_cr_reg_mem);
12689 %}
12690 
12691 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12692   match(Set cr (CmpN op1 op2));
12693 
12694   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12695   ins_encode %{
12696     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12697   %}
12698   ins_pipe(ialu_cr_reg_imm);
12699 %}
12700 
12701 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12702 %{
12703   predicate(!UseCompactObjectHeaders);
12704   match(Set cr (CmpN src (LoadNKlass mem)));
12705 
12706   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12707   ins_encode %{
12708     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12709   %}
12710   ins_pipe(ialu_cr_reg_mem);
12711 %}
12712 
12713 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12714   match(Set cr (CmpN src zero));
12715 
12716   format %{ "testl   $src, $src\t# compressed ptr" %}
12717   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12718   ins_pipe(ialu_cr_reg_imm);
12719 %}
12720 
12721 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12722 %{
12723   predicate(CompressedOops::base() != NULL);
12724   match(Set cr (CmpN (LoadN mem) zero));
12725 
12726   ins_cost(500); // XXX
12727   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12728   ins_encode %{
12729     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12730   %}
12731   ins_pipe(ialu_cr_reg_mem);
12732 %}
12733 
12734 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12735 %{
12736   predicate(CompressedOops::base() == NULL);
12737   match(Set cr (CmpN (LoadN mem) zero));
12738 
12739   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12740   ins_encode %{
12741     __ cmpl(r12, $mem$$Address);
12742   %}
12743   ins_pipe(ialu_cr_reg_mem);
12744 %}
12745 
12746 // Yanked all unsigned pointer compare operations.
12747 // Pointer compares are done with CmpP which is already unsigned.
12748 
12749 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12750 %{
12751   match(Set cr (CmpL op1 op2));
12752 
12753   format %{ "cmpq    $op1, $op2" %}
12754   ins_encode %{
12755     __ cmpq($op1$$Register, $op2$$Register);
12756   %}
12757   ins_pipe(ialu_cr_reg_reg);
12758 %}
12759 
12760 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12761 %{
12762   match(Set cr (CmpL op1 op2));
12763 
12764   format %{ "cmpq    $op1, $op2" %}
12765   ins_encode %{
12766     __ cmpq($op1$$Register, $op2$$constant);
12767   %}
12768   ins_pipe(ialu_cr_reg_imm);
12769 %}
12770 
12771 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12772 %{
12773   match(Set cr (CmpL op1 (LoadL op2)));
12774 
12775   format %{ "cmpq    $op1, $op2" %}
12776   ins_encode %{
12777     __ cmpq($op1$$Register, $op2$$Address);
12778   %}
12779   ins_pipe(ialu_cr_reg_mem);
12780 %}
12781 
12782 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12783 %{
12784   match(Set cr (CmpL src zero));
12785 
12786   format %{ "testq   $src, $src" %}
12787   ins_encode %{
12788     __ testq($src$$Register, $src$$Register);
12789   %}
12790   ins_pipe(ialu_cr_reg_imm);
12791 %}
12792 
12793 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12794 %{
12795   match(Set cr (CmpL (AndL src con) zero));
12796 
12797   format %{ "testq   $src, $con\t# long" %}
12798   ins_encode %{
12799     __ testq($src$$Register, $con$$constant);
12800   %}
12801   ins_pipe(ialu_cr_reg_imm);
12802 %}
12803 
12804 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12805 %{
12806   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12807 
12808   format %{ "testq   $src, $mem" %}
12809   ins_encode %{
12810     __ testq($src$$Register, $mem$$Address);
12811   %}
12812   ins_pipe(ialu_cr_reg_mem);
12813 %}
12814 
12815 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12816 %{
12817   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12818 
12819   format %{ "testq   $src, $mem" %}
12820   ins_encode %{
12821     __ testq($src$$Register, $mem$$Address);
12822   %}
12823   ins_pipe(ialu_cr_reg_mem);
12824 %}
12825 
12826 // Manifest a CmpU result in an integer register.  Very painful.
12827 // This is the test to avoid.
12828 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
12829 %{
12830   match(Set dst (CmpU3 src1 src2));
12831   effect(KILL flags);
12832 
12833   ins_cost(275); // XXX
12834   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
12835             "movl    $dst, -1\n\t"
12836             "jb,u    done\n\t"
12837             "setne   $dst\n\t"
12838             "movzbl  $dst, $dst\n\t"
12839     "done:" %}
12840   ins_encode %{
12841     Label done;
12842     __ cmpl($src1$$Register, $src2$$Register);
12843     __ movl($dst$$Register, -1);
12844     __ jccb(Assembler::below, done);
12845     __ setne($dst$$Register);
12846     __ movzbl($dst$$Register, $dst$$Register);
12847     __ bind(done);
12848   %}
12849   ins_pipe(pipe_slow);
12850 %}
12851 
12852 // Manifest a CmpL result in an integer register.  Very painful.
12853 // This is the test to avoid.
12854 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12855 %{
12856   match(Set dst (CmpL3 src1 src2));
12857   effect(KILL flags);
12858 
12859   ins_cost(275); // XXX
12860   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12861             "movl    $dst, -1\n\t"
12862             "jl,s    done\n\t"
12863             "setne   $dst\n\t"
12864             "movzbl  $dst, $dst\n\t"
12865     "done:" %}
12866   ins_encode %{
12867     Label done;
12868     __ cmpq($src1$$Register, $src2$$Register);
12869     __ movl($dst$$Register, -1);
12870     __ jccb(Assembler::less, done);
12871     __ setne($dst$$Register);
12872     __ movzbl($dst$$Register, $dst$$Register);
12873     __ bind(done);
12874   %}
12875   ins_pipe(pipe_slow);
12876 %}
12877 
12878 // Manifest a CmpUL result in an integer register.  Very painful.
12879 // This is the test to avoid.
12880 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12881 %{
12882   match(Set dst (CmpUL3 src1 src2));
12883   effect(KILL flags);
12884 
12885   ins_cost(275); // XXX
12886   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12887             "movl    $dst, -1\n\t"
12888             "jb,u    done\n\t"
12889             "setne   $dst\n\t"
12890             "movzbl  $dst, $dst\n\t"
12891     "done:" %}
12892   ins_encode %{
12893     Label done;
12894     __ cmpq($src1$$Register, $src2$$Register);
12895     __ movl($dst$$Register, -1);
12896     __ jccb(Assembler::below, done);
12897     __ setne($dst$$Register);
12898     __ movzbl($dst$$Register, $dst$$Register);
12899     __ bind(done);
12900   %}
12901   ins_pipe(pipe_slow);
12902 %}
12903 
12904 // Unsigned long compare Instructions; really, same as signed long except they
12905 // produce an rFlagsRegU instead of rFlagsReg.
12906 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12907 %{
12908   match(Set cr (CmpUL op1 op2));
12909 
12910   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12911   ins_encode %{
12912     __ cmpq($op1$$Register, $op2$$Register);
12913   %}
12914   ins_pipe(ialu_cr_reg_reg);
12915 %}
12916 
12917 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12918 %{
12919   match(Set cr (CmpUL op1 op2));
12920 
12921   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12922   ins_encode %{
12923     __ cmpq($op1$$Register, $op2$$constant);
12924   %}
12925   ins_pipe(ialu_cr_reg_imm);
12926 %}
12927 
12928 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12929 %{
12930   match(Set cr (CmpUL op1 (LoadL op2)));
12931 
12932   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12933   ins_encode %{
12934     __ cmpq($op1$$Register, $op2$$Address);
12935   %}
12936   ins_pipe(ialu_cr_reg_mem);
12937 %}
12938 
12939 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12940 %{
12941   match(Set cr (CmpUL src zero));
12942 
12943   format %{ "testq   $src, $src\t# unsigned" %}
12944   ins_encode %{
12945     __ testq($src$$Register, $src$$Register);
12946   %}
12947   ins_pipe(ialu_cr_reg_imm);
12948 %}
12949 
12950 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12951 %{
12952   match(Set cr (CmpI (LoadB mem) imm));
12953 
12954   ins_cost(125);
12955   format %{ "cmpb    $mem, $imm" %}
12956   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12957   ins_pipe(ialu_cr_reg_mem);
12958 %}
12959 
12960 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12961 %{
12962   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12963 
12964   ins_cost(125);
12965   format %{ "testb   $mem, $imm\t# ubyte" %}
12966   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12967   ins_pipe(ialu_cr_reg_mem);
12968 %}
12969 
12970 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12971 %{
12972   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12973 
12974   ins_cost(125);
12975   format %{ "testb   $mem, $imm\t# byte" %}
12976   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12977   ins_pipe(ialu_cr_reg_mem);
12978 %}
12979 
12980 //----------Max and Min--------------------------------------------------------
12981 // Min Instructions
12982 
12983 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12984 %{
12985   effect(USE_DEF dst, USE src, USE cr);
12986 
12987   format %{ "cmovlgt $dst, $src\t# min" %}
12988   ins_encode %{
12989     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12990   %}
12991   ins_pipe(pipe_cmov_reg);
12992 %}
12993 
12994 
12995 instruct minI_rReg(rRegI dst, rRegI src)
12996 %{
12997   match(Set dst (MinI dst src));
12998 
12999   ins_cost(200);
13000   expand %{
13001     rFlagsReg cr;
13002     compI_rReg(cr, dst, src);
13003     cmovI_reg_g(dst, src, cr);
13004   %}
13005 %}
13006 
13007 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13008 %{
13009   effect(USE_DEF dst, USE src, USE cr);
13010 
13011   format %{ "cmovllt $dst, $src\t# max" %}
13012   ins_encode %{
13013     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13014   %}
13015   ins_pipe(pipe_cmov_reg);
13016 %}
13017 
13018 
13019 instruct maxI_rReg(rRegI dst, rRegI src)
13020 %{
13021   match(Set dst (MaxI dst src));
13022 
13023   ins_cost(200);
13024   expand %{
13025     rFlagsReg cr;
13026     compI_rReg(cr, dst, src);
13027     cmovI_reg_l(dst, src, cr);
13028   %}
13029 %}
13030 
13031 // ============================================================================
13032 // Branch Instructions
13033 
13034 // Jump Direct - Label defines a relative address from JMP+1
13035 instruct jmpDir(label labl)
13036 %{
13037   match(Goto);
13038   effect(USE labl);
13039 
13040   ins_cost(300);
13041   format %{ "jmp     $labl" %}
13042   size(5);
13043   ins_encode %{
13044     Label* L = $labl$$label;
13045     __ jmp(*L, false); // Always long jump
13046   %}
13047   ins_pipe(pipe_jmp);
13048 %}
13049 
13050 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13051 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13052 %{
13053   match(If cop cr);
13054   effect(USE labl);
13055 
13056   ins_cost(300);
13057   format %{ "j$cop     $labl" %}
13058   size(6);
13059   ins_encode %{
13060     Label* L = $labl$$label;
13061     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13062   %}
13063   ins_pipe(pipe_jcc);
13064 %}
13065 
13066 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13067 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13068 %{
13069   match(CountedLoopEnd cop cr);
13070   effect(USE labl);
13071 
13072   ins_cost(300);
13073   format %{ "j$cop     $labl\t# loop end" %}
13074   size(6);
13075   ins_encode %{
13076     Label* L = $labl$$label;
13077     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13078   %}
13079   ins_pipe(pipe_jcc);
13080 %}
13081 
13082 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13083 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13084   match(CountedLoopEnd cop cmp);
13085   effect(USE labl);
13086 
13087   ins_cost(300);
13088   format %{ "j$cop,u   $labl\t# loop end" %}
13089   size(6);
13090   ins_encode %{
13091     Label* L = $labl$$label;
13092     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13093   %}
13094   ins_pipe(pipe_jcc);
13095 %}
13096 
13097 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13098   match(CountedLoopEnd cop cmp);
13099   effect(USE labl);
13100 
13101   ins_cost(200);
13102   format %{ "j$cop,u   $labl\t# loop end" %}
13103   size(6);
13104   ins_encode %{
13105     Label* L = $labl$$label;
13106     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13107   %}
13108   ins_pipe(pipe_jcc);
13109 %}
13110 
13111 // Jump Direct Conditional - using unsigned comparison
13112 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13113   match(If cop cmp);
13114   effect(USE labl);
13115 
13116   ins_cost(300);
13117   format %{ "j$cop,u   $labl" %}
13118   size(6);
13119   ins_encode %{
13120     Label* L = $labl$$label;
13121     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13122   %}
13123   ins_pipe(pipe_jcc);
13124 %}
13125 
13126 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13127   match(If cop cmp);
13128   effect(USE labl);
13129 
13130   ins_cost(200);
13131   format %{ "j$cop,u   $labl" %}
13132   size(6);
13133   ins_encode %{
13134     Label* L = $labl$$label;
13135     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13136   %}
13137   ins_pipe(pipe_jcc);
13138 %}
13139 
13140 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13141   match(If cop cmp);
13142   effect(USE labl);
13143 
13144   ins_cost(200);
13145   format %{ $$template
13146     if ($cop$$cmpcode == Assembler::notEqual) {
13147       $$emit$$"jp,u    $labl\n\t"
13148       $$emit$$"j$cop,u   $labl"
13149     } else {
13150       $$emit$$"jp,u    done\n\t"
13151       $$emit$$"j$cop,u   $labl\n\t"
13152       $$emit$$"done:"
13153     }
13154   %}
13155   ins_encode %{
13156     Label* l = $labl$$label;
13157     if ($cop$$cmpcode == Assembler::notEqual) {
13158       __ jcc(Assembler::parity, *l, false);
13159       __ jcc(Assembler::notEqual, *l, false);
13160     } else if ($cop$$cmpcode == Assembler::equal) {
13161       Label done;
13162       __ jccb(Assembler::parity, done);
13163       __ jcc(Assembler::equal, *l, false);
13164       __ bind(done);
13165     } else {
13166        ShouldNotReachHere();
13167     }
13168   %}
13169   ins_pipe(pipe_jcc);
13170 %}
13171 
13172 // ============================================================================
13173 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13174 // superklass array for an instance of the superklass.  Set a hidden
13175 // internal cache on a hit (cache is checked with exposed code in
13176 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13177 // encoding ALSO sets flags.
13178 
13179 instruct partialSubtypeCheck(rdi_RegP result,
13180                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13181                              rFlagsReg cr)
13182 %{
13183   match(Set result (PartialSubtypeCheck sub super));
13184   effect(KILL rcx, KILL cr);
13185 
13186   ins_cost(1100);  // slightly larger than the next version
13187   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13188             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13189             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13190             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13191             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13192             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13193             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13194     "miss:\t" %}
13195 
13196   opcode(0x1); // Force a XOR of RDI
13197   ins_encode(enc_PartialSubtypeCheck());
13198   ins_pipe(pipe_slow);
13199 %}
13200 
13201 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
13202                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13203                                      immP0 zero,
13204                                      rdi_RegP result)
13205 %{
13206   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13207   effect(KILL rcx, KILL result);
13208 
13209   ins_cost(1000);
13210   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13211             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13212             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13213             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
13214             "jne,s   miss\t\t# Missed: flags nz\n\t"
13215             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13216     "miss:\t" %}
13217 
13218   opcode(0x0); // No need to XOR RDI
13219   ins_encode(enc_PartialSubtypeCheck());
13220   ins_pipe(pipe_slow);
13221 %}
13222 
13223 // ============================================================================
13224 // Branch Instructions -- short offset versions
13225 //
13226 // These instructions are used to replace jumps of a long offset (the default
13227 // match) with jumps of a shorter offset.  These instructions are all tagged
13228 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13229 // match rules in general matching.  Instead, the ADLC generates a conversion
13230 // method in the MachNode which can be used to do in-place replacement of the
13231 // long variant with the shorter variant.  The compiler will determine if a
13232 // branch can be taken by the is_short_branch_offset() predicate in the machine
13233 // specific code section of the file.
13234 
13235 // Jump Direct - Label defines a relative address from JMP+1
13236 instruct jmpDir_short(label labl) %{
13237   match(Goto);
13238   effect(USE labl);
13239 
13240   ins_cost(300);
13241   format %{ "jmp,s   $labl" %}
13242   size(2);
13243   ins_encode %{
13244     Label* L = $labl$$label;
13245     __ jmpb(*L);
13246   %}
13247   ins_pipe(pipe_jmp);
13248   ins_short_branch(1);
13249 %}
13250 
13251 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13252 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13253   match(If cop cr);
13254   effect(USE labl);
13255 
13256   ins_cost(300);
13257   format %{ "j$cop,s   $labl" %}
13258   size(2);
13259   ins_encode %{
13260     Label* L = $labl$$label;
13261     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13262   %}
13263   ins_pipe(pipe_jcc);
13264   ins_short_branch(1);
13265 %}
13266 
13267 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13268 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13269   match(CountedLoopEnd cop cr);
13270   effect(USE labl);
13271 
13272   ins_cost(300);
13273   format %{ "j$cop,s   $labl\t# loop end" %}
13274   size(2);
13275   ins_encode %{
13276     Label* L = $labl$$label;
13277     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13278   %}
13279   ins_pipe(pipe_jcc);
13280   ins_short_branch(1);
13281 %}
13282 
13283 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13284 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13285   match(CountedLoopEnd cop cmp);
13286   effect(USE labl);
13287 
13288   ins_cost(300);
13289   format %{ "j$cop,us  $labl\t# loop end" %}
13290   size(2);
13291   ins_encode %{
13292     Label* L = $labl$$label;
13293     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13294   %}
13295   ins_pipe(pipe_jcc);
13296   ins_short_branch(1);
13297 %}
13298 
13299 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13300   match(CountedLoopEnd cop cmp);
13301   effect(USE labl);
13302 
13303   ins_cost(300);
13304   format %{ "j$cop,us  $labl\t# loop end" %}
13305   size(2);
13306   ins_encode %{
13307     Label* L = $labl$$label;
13308     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13309   %}
13310   ins_pipe(pipe_jcc);
13311   ins_short_branch(1);
13312 %}
13313 
13314 // Jump Direct Conditional - using unsigned comparison
13315 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13316   match(If cop cmp);
13317   effect(USE labl);
13318 
13319   ins_cost(300);
13320   format %{ "j$cop,us  $labl" %}
13321   size(2);
13322   ins_encode %{
13323     Label* L = $labl$$label;
13324     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13325   %}
13326   ins_pipe(pipe_jcc);
13327   ins_short_branch(1);
13328 %}
13329 
13330 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13331   match(If cop cmp);
13332   effect(USE labl);
13333 
13334   ins_cost(300);
13335   format %{ "j$cop,us  $labl" %}
13336   size(2);
13337   ins_encode %{
13338     Label* L = $labl$$label;
13339     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13340   %}
13341   ins_pipe(pipe_jcc);
13342   ins_short_branch(1);
13343 %}
13344 
13345 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13346   match(If cop cmp);
13347   effect(USE labl);
13348 
13349   ins_cost(300);
13350   format %{ $$template
13351     if ($cop$$cmpcode == Assembler::notEqual) {
13352       $$emit$$"jp,u,s  $labl\n\t"
13353       $$emit$$"j$cop,u,s  $labl"
13354     } else {
13355       $$emit$$"jp,u,s  done\n\t"
13356       $$emit$$"j$cop,u,s  $labl\n\t"
13357       $$emit$$"done:"
13358     }
13359   %}
13360   size(4);
13361   ins_encode %{
13362     Label* l = $labl$$label;
13363     if ($cop$$cmpcode == Assembler::notEqual) {
13364       __ jccb(Assembler::parity, *l);
13365       __ jccb(Assembler::notEqual, *l);
13366     } else if ($cop$$cmpcode == Assembler::equal) {
13367       Label done;
13368       __ jccb(Assembler::parity, done);
13369       __ jccb(Assembler::equal, *l);
13370       __ bind(done);
13371     } else {
13372        ShouldNotReachHere();
13373     }
13374   %}
13375   ins_pipe(pipe_jcc);
13376   ins_short_branch(1);
13377 %}
13378 
13379 // ============================================================================
13380 // inlined locking and unlocking
13381 
13382 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
13383   predicate(Compile::current()->use_rtm());
13384   match(Set cr (FastLock object box));
13385   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
13386   ins_cost(300);
13387   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13388   ins_encode %{
13389     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13390                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
13391                  _rtm_counters, _stack_rtm_counters,
13392                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13393                  true, ra_->C->profile_rtm());
13394   %}
13395   ins_pipe(pipe_slow);
13396 %}
13397 
13398 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
13399   predicate(!Compile::current()->use_rtm());
13400   match(Set cr (FastLock object box));
13401   effect(TEMP tmp, TEMP scr, USE_KILL box);
13402   ins_cost(300);
13403   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13404   ins_encode %{
13405     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13406                  $scr$$Register, noreg, noreg, r15_thread, NULL, NULL, NULL, false, false);
13407   %}
13408   ins_pipe(pipe_slow);
13409 %}
13410 
13411 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13412   match(Set cr (FastUnlock object box));
13413   effect(TEMP tmp, USE_KILL box);
13414   ins_cost(300);
13415   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13416   ins_encode %{
13417     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13418   %}
13419   ins_pipe(pipe_slow);
13420 %}
13421 
13422 
13423 // ============================================================================
13424 // Safepoint Instructions
13425 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13426 %{
13427   match(SafePoint poll);
13428   effect(KILL cr, USE poll);
13429 
13430   format %{ "testl   rax, [$poll]\t"
13431             "# Safepoint: poll for GC" %}
13432   ins_cost(125);
13433   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13434   ins_encode %{
13435     __ relocate(relocInfo::poll_type);
13436     address pre_pc = __ pc();
13437     __ testl(rax, Address($poll$$Register, 0));
13438     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13439   %}
13440   ins_pipe(ialu_reg_mem);
13441 %}
13442 
13443 instruct mask_all_evexL(kReg dst, rRegL src) %{
13444   match(Set dst (MaskAll src));
13445   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13446   ins_encode %{
13447     int mask_len = Matcher::vector_length(this);
13448     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13449   %}
13450   ins_pipe( pipe_slow );
13451 %}
13452 
13453 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13454   predicate(Matcher::vector_length(n) > 32);
13455   match(Set dst (MaskAll src));
13456   effect(TEMP tmp);
13457   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13458   ins_encode %{
13459     int mask_len = Matcher::vector_length(this);
13460     __ movslq($tmp$$Register, $src$$Register);
13461     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13462   %}
13463   ins_pipe( pipe_slow );
13464 %}
13465 
13466 // ============================================================================
13467 // Procedure Call/Return Instructions
13468 // Call Java Static Instruction
13469 // Note: If this code changes, the corresponding ret_addr_offset() and
13470 //       compute_padding() functions will have to be adjusted.
13471 instruct CallStaticJavaDirect(method meth) %{
13472   match(CallStaticJava);
13473   effect(USE meth);
13474 
13475   ins_cost(300);
13476   format %{ "call,static " %}
13477   opcode(0xE8); /* E8 cd */
13478   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13479   ins_pipe(pipe_slow);
13480   ins_alignment(4);
13481 %}
13482 
13483 // Call Java Dynamic Instruction
13484 // Note: If this code changes, the corresponding ret_addr_offset() and
13485 //       compute_padding() functions will have to be adjusted.
13486 instruct CallDynamicJavaDirect(method meth)
13487 %{
13488   match(CallDynamicJava);
13489   effect(USE meth);
13490 
13491   ins_cost(300);
13492   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13493             "call,dynamic " %}
13494   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13495   ins_pipe(pipe_slow);
13496   ins_alignment(4);
13497 %}
13498 
13499 // Call Runtime Instruction
13500 instruct CallRuntimeDirect(method meth)
13501 %{
13502   match(CallRuntime);
13503   effect(USE meth);
13504 
13505   ins_cost(300);
13506   format %{ "call,runtime " %}
13507   ins_encode(clear_avx, Java_To_Runtime(meth));
13508   ins_pipe(pipe_slow);
13509 %}
13510 
13511 // Call runtime without safepoint
13512 instruct CallLeafDirect(method meth)
13513 %{
13514   match(CallLeaf);
13515   effect(USE meth);
13516 
13517   ins_cost(300);
13518   format %{ "call_leaf,runtime " %}
13519   ins_encode(clear_avx, Java_To_Runtime(meth));
13520   ins_pipe(pipe_slow);
13521 %}
13522 
13523 // Call runtime without safepoint and with vector arguments
13524 instruct CallLeafDirectVector(method meth)
13525 %{
13526   match(CallLeafVector);
13527   effect(USE meth);
13528 
13529   ins_cost(300);
13530   format %{ "call_leaf,vector " %}
13531   ins_encode(Java_To_Runtime(meth));
13532   ins_pipe(pipe_slow);
13533 %}
13534 
13535 // Call runtime without safepoint
13536 instruct CallLeafNoFPDirect(method meth)
13537 %{
13538   match(CallLeafNoFP);
13539   effect(USE meth);
13540 
13541   ins_cost(300);
13542   format %{ "call_leaf_nofp,runtime " %}
13543   ins_encode(clear_avx, Java_To_Runtime(meth));
13544   ins_pipe(pipe_slow);
13545 %}
13546 
13547 // Return Instruction
13548 // Remove the return address & jump to it.
13549 // Notice: We always emit a nop after a ret to make sure there is room
13550 // for safepoint patching
13551 instruct Ret()
13552 %{
13553   match(Return);
13554 
13555   format %{ "ret" %}
13556   ins_encode %{
13557     __ ret(0);
13558   %}
13559   ins_pipe(pipe_jmp);
13560 %}
13561 
13562 // Tail Call; Jump from runtime stub to Java code.
13563 // Also known as an 'interprocedural jump'.
13564 // Target of jump will eventually return to caller.
13565 // TailJump below removes the return address.
13566 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13567 %{
13568   match(TailCall jump_target method_ptr);
13569 
13570   ins_cost(300);
13571   format %{ "jmp     $jump_target\t# rbx holds method" %}
13572   ins_encode %{
13573     __ jmp($jump_target$$Register);
13574   %}
13575   ins_pipe(pipe_jmp);
13576 %}
13577 
13578 // Tail Jump; remove the return address; jump to target.
13579 // TailCall above leaves the return address around.
13580 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13581 %{
13582   match(TailJump jump_target ex_oop);
13583 
13584   ins_cost(300);
13585   format %{ "popq    rdx\t# pop return address\n\t"
13586             "jmp     $jump_target" %}
13587   ins_encode %{
13588     __ popq(as_Register(RDX_enc));
13589     __ jmp($jump_target$$Register);
13590   %}
13591   ins_pipe(pipe_jmp);
13592 %}
13593 
13594 // Create exception oop: created by stack-crawling runtime code.
13595 // Created exception is now available to this handler, and is setup
13596 // just prior to jumping to this handler.  No code emitted.
13597 instruct CreateException(rax_RegP ex_oop)
13598 %{
13599   match(Set ex_oop (CreateEx));
13600 
13601   size(0);
13602   // use the following format syntax
13603   format %{ "# exception oop is in rax; no code emitted" %}
13604   ins_encode();
13605   ins_pipe(empty);
13606 %}
13607 
13608 // Rethrow exception:
13609 // The exception oop will come in the first argument position.
13610 // Then JUMP (not call) to the rethrow stub code.
13611 instruct RethrowException()
13612 %{
13613   match(Rethrow);
13614 
13615   // use the following format syntax
13616   format %{ "jmp     rethrow_stub" %}
13617   ins_encode(enc_rethrow);
13618   ins_pipe(pipe_jmp);
13619 %}
13620 
13621 // ============================================================================
13622 // This name is KNOWN by the ADLC and cannot be changed.
13623 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13624 // for this guy.
13625 instruct tlsLoadP(r15_RegP dst) %{
13626   match(Set dst (ThreadLocal));
13627   effect(DEF dst);
13628 
13629   size(0);
13630   format %{ "# TLS is in R15" %}
13631   ins_encode( /*empty encoding*/ );
13632   ins_pipe(ialu_reg_reg);
13633 %}
13634 
13635 
13636 //----------PEEPHOLE RULES-----------------------------------------------------
13637 // These must follow all instruction definitions as they use the names
13638 // defined in the instructions definitions.
13639 //
13640 // peeppredicate ( rule_predicate );
13641 // // the predicate unless which the peephole rule will be ignored
13642 //
13643 // peepmatch ( root_instr_name [preceding_instruction]* );
13644 //
13645 // peepprocedure ( procedure_name );
13646 // // provide a procedure name to perform the optimization, the procedure should
13647 // // reside in the architecture dependent peephole file, the method has the
13648 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
13649 // // with the arguments being the basic block, the current node index inside the
13650 // // block, the register allocator, the functions upon invoked return a new node
13651 // // defined in peepreplace, and the rules of the nodes appearing in the
13652 // // corresponding peepmatch, the function return true if successful, else
13653 // // return false
13654 //
13655 // peepconstraint %{
13656 // (instruction_number.operand_name relational_op instruction_number.operand_name
13657 //  [, ...] );
13658 // // instruction numbers are zero-based using left to right order in peepmatch
13659 //
13660 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13661 // // provide an instruction_number.operand_name for each operand that appears
13662 // // in the replacement instruction's match rule
13663 //
13664 // ---------VM FLAGS---------------------------------------------------------
13665 //
13666 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13667 //
13668 // Each peephole rule is given an identifying number starting with zero and
13669 // increasing by one in the order seen by the parser.  An individual peephole
13670 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13671 // on the command-line.
13672 //
13673 // ---------CURRENT LIMITATIONS----------------------------------------------
13674 //
13675 // Only transformations inside a basic block (do we need more for peephole)
13676 //
13677 // ---------EXAMPLE----------------------------------------------------------
13678 //
13679 // // pertinent parts of existing instructions in architecture description
13680 // instruct movI(rRegI dst, rRegI src)
13681 // %{
13682 //   match(Set dst (CopyI src));
13683 // %}
13684 //
13685 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13686 // %{
13687 //   match(Set dst (AddI dst src));
13688 //   effect(KILL cr);
13689 // %}
13690 //
13691 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
13692 // %{
13693 //   match(Set dst (AddI dst src));
13694 // %}
13695 //
13696 // 1. Simple replacement
13697 // - Only match adjacent instructions in same basic block
13698 // - Only equality constraints
13699 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
13700 // - Only one replacement instruction
13701 //
13702 // // Change (inc mov) to lea
13703 // peephole %{
13704 //   // lea should only be emitted when beneficial
13705 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
13706 //   // increment preceded by register-register move
13707 //   peepmatch ( incI_rReg movI );
13708 //   // require that the destination register of the increment
13709 //   // match the destination register of the move
13710 //   peepconstraint ( 0.dst == 1.dst );
13711 //   // construct a replacement instruction that sets
13712 //   // the destination to ( move's source register + one )
13713 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13714 // %}
13715 //
13716 // 2. Procedural replacement
13717 // - More flexible finding relevent nodes
13718 // - More flexible constraints
13719 // - More flexible transformations
13720 // - May utilise architecture-dependent API more effectively
13721 // - Currently only one replacement instruction due to adlc parsing capabilities
13722 //
13723 // // Change (inc mov) to lea
13724 // peephole %{
13725 //   // lea should only be emitted when beneficial
13726 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
13727 //   // the rule numbers of these nodes inside are passed into the function below
13728 //   peepmatch ( incI_rReg movI );
13729 //   // the method that takes the responsibility of transformation
13730 //   peepprocedure ( inc_mov_to_lea );
13731 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
13732 //   // node is passed into the function above
13733 //   peepreplace ( leaI_rReg_immI() );
13734 // %}
13735 
13736 // These instructions is not matched by the matcher but used by the peephole
13737 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
13738 %{
13739   predicate(false);
13740   match(Set dst (AddI src1 src2));
13741   format %{ "leal    $dst, [$src1 + $src2]" %}
13742   ins_encode %{
13743     Register dst = $dst$$Register;
13744     Register src1 = $src1$$Register;
13745     Register src2 = $src2$$Register;
13746     if (src1 != rbp && src1 != r13) {
13747       __ leal(dst, Address(src1, src2, Address::times_1));
13748     } else {
13749       assert(src2 != rbp && src2 != r13, "");
13750       __ leal(dst, Address(src2, src1, Address::times_1));
13751     }
13752   %}
13753   ins_pipe(ialu_reg_reg);
13754 %}
13755 
13756 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
13757 %{
13758   predicate(false);
13759   match(Set dst (AddI src1 src2));
13760   format %{ "leal    $dst, [$src1 + $src2]" %}
13761   ins_encode %{
13762     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
13763   %}
13764   ins_pipe(ialu_reg_reg);
13765 %}
13766 
13767 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
13768 %{
13769   predicate(false);
13770   match(Set dst (LShiftI src shift));
13771   format %{ "leal    $dst, [$src << $shift]" %}
13772   ins_encode %{
13773     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
13774     Register src = $src$$Register;
13775     if (scale == Address::times_2 && src != rbp && src != r13) {
13776       __ leal($dst$$Register, Address(src, src, Address::times_1));
13777     } else {
13778       __ leal($dst$$Register, Address(noreg, src, scale));
13779     }
13780   %}
13781   ins_pipe(ialu_reg_reg);
13782 %}
13783 
13784 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
13785 %{
13786   predicate(false);
13787   match(Set dst (AddL src1 src2));
13788   format %{ "leaq    $dst, [$src1 + $src2]" %}
13789   ins_encode %{
13790     Register dst = $dst$$Register;
13791     Register src1 = $src1$$Register;
13792     Register src2 = $src2$$Register;
13793     if (src1 != rbp && src1 != r13) {
13794       __ leaq(dst, Address(src1, src2, Address::times_1));
13795     } else {
13796       assert(src2 != rbp && src2 != r13, "");
13797       __ leaq(dst, Address(src2, src1, Address::times_1));
13798     }
13799   %}
13800   ins_pipe(ialu_reg_reg);
13801 %}
13802 
13803 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
13804 %{
13805   predicate(false);
13806   match(Set dst (AddL src1 src2));
13807   format %{ "leaq    $dst, [$src1 + $src2]" %}
13808   ins_encode %{
13809     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
13810   %}
13811   ins_pipe(ialu_reg_reg);
13812 %}
13813 
13814 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
13815 %{
13816   predicate(false);
13817   match(Set dst (LShiftL src shift));
13818   format %{ "leaq    $dst, [$src << $shift]" %}
13819   ins_encode %{
13820     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
13821     Register src = $src$$Register;
13822     if (scale == Address::times_2 && src != rbp && src != r13) {
13823       __ leaq($dst$$Register, Address(src, src, Address::times_1));
13824     } else {
13825       __ leaq($dst$$Register, Address(noreg, src, scale));
13826     }
13827   %}
13828   ins_pipe(ialu_reg_reg);
13829 %}
13830 
13831 peephole
13832 %{
13833   peeppredicate(VM_Version::supports_fast_2op_lea());
13834   peepmatch (addI_rReg);
13835   peepprocedure (lea_coalesce_reg);
13836   peepreplace (leaI_rReg_rReg_peep());
13837 %}
13838 
13839 peephole
13840 %{
13841   peeppredicate(VM_Version::supports_fast_2op_lea());
13842   peepmatch (addI_rReg_imm);
13843   peepprocedure (lea_coalesce_imm);
13844   peepreplace (leaI_rReg_immI_peep());
13845 %}
13846 
13847 peephole
13848 %{
13849   peeppredicate(VM_Version::supports_fast_2op_lea());
13850   peepmatch (incI_rReg);
13851   peepprocedure (lea_coalesce_imm);
13852   peepreplace (leaI_rReg_immI_peep());
13853 %}
13854 
13855 peephole
13856 %{
13857   peeppredicate(VM_Version::supports_fast_2op_lea());
13858   peepmatch (decI_rReg);
13859   peepprocedure (lea_coalesce_imm);
13860   peepreplace (leaI_rReg_immI_peep());
13861 %}
13862 
13863 peephole
13864 %{
13865   peeppredicate(VM_Version::supports_fast_2op_lea());
13866   peepmatch (salI_rReg_immI2);
13867   peepprocedure (lea_coalesce_imm);
13868   peepreplace (leaI_rReg_immI2_peep());
13869 %}
13870 
13871 peephole
13872 %{
13873   peeppredicate(VM_Version::supports_fast_2op_lea());
13874   peepmatch (addL_rReg);
13875   peepprocedure (lea_coalesce_reg);
13876   peepreplace (leaL_rReg_rReg_peep());
13877 %}
13878 
13879 peephole
13880 %{
13881   peeppredicate(VM_Version::supports_fast_2op_lea());
13882   peepmatch (addL_rReg_imm);
13883   peepprocedure (lea_coalesce_imm);
13884   peepreplace (leaL_rReg_immL32_peep());
13885 %}
13886 
13887 peephole
13888 %{
13889   peeppredicate(VM_Version::supports_fast_2op_lea());
13890   peepmatch (incL_rReg);
13891   peepprocedure (lea_coalesce_imm);
13892   peepreplace (leaL_rReg_immL32_peep());
13893 %}
13894 
13895 peephole
13896 %{
13897   peeppredicate(VM_Version::supports_fast_2op_lea());
13898   peepmatch (decL_rReg);
13899   peepprocedure (lea_coalesce_imm);
13900   peepreplace (leaL_rReg_immL32_peep());
13901 %}
13902 
13903 peephole
13904 %{
13905   peeppredicate(VM_Version::supports_fast_2op_lea());
13906   peepmatch (salL_rReg_immI2);
13907   peepprocedure (lea_coalesce_imm);
13908   peepreplace (leaL_rReg_immI2_peep());
13909 %}
13910 
13911 //----------SMARTSPILL RULES---------------------------------------------------
13912 // These must follow all instruction definitions as they use the names
13913 // defined in the instructions definitions.