1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 source_hpp %{
  323 
  324 extern RegMask _ANY_REG_mask;
  325 extern RegMask _PTR_REG_mask;
  326 extern RegMask _PTR_REG_NO_RBP_mask;
  327 extern RegMask _PTR_NO_RAX_REG_mask;
  328 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  329 extern RegMask _LONG_REG_mask;
  330 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  331 extern RegMask _LONG_NO_RCX_REG_mask;
  332 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  333 extern RegMask _INT_REG_mask;
  334 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  335 extern RegMask _INT_NO_RCX_REG_mask;
  336 extern RegMask _INT_NO_RBP_R13_REG_mask;
  337 extern RegMask _FLOAT_REG_mask;
  338 
  339 extern RegMask _STACK_OR_PTR_REG_mask;
  340 extern RegMask _STACK_OR_LONG_REG_mask;
  341 extern RegMask _STACK_OR_LONG_NO_RAX_RDX_REG_mask;
  342 extern RegMask _STACK_OR_INT_REG_mask;
  343 extern RegMask _STACK_OR_INT_NO_RAX_RDX_REG_mask;
  344 
  345 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  346 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  347 inline const RegMask& STACK_OR_LONG_NO_RAX_RDX_REG_mask() {
  348   return _STACK_OR_LONG_NO_RAX_RDX_REG_mask;
  349 }
  350 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  351 inline const RegMask& STACK_OR_INT_NO_RAX_RDX_REG_mask() {
  352   return _STACK_OR_INT_NO_RAX_RDX_REG_mask;
  353 }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_LONG_NO_RAX_RDX_REG_mask;
  380 RegMask _STACK_OR_INT_REG_mask;
  381 RegMask _STACK_OR_INT_NO_RAX_RDX_REG_mask;
  382 
  383 static bool need_r12_heapbase() {
  384   return UseCompressedOops;
  385 }
  386 
  387 void reg_mask_init() {
  388   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  389   // We derive a number of subsets from it.
  390   _ANY_REG_mask = _ALL_REG_mask;
  391 
  392   if (PreserveFramePointer) {
  393     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  394     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  395   }
  396   if (need_r12_heapbase()) {
  397     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  398     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  399   }
  400 
  401   _PTR_REG_mask = _ANY_REG_mask;
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  404   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  405   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  406 
  407   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  408   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  409 
  410   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  411   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  412   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  413 
  414   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  415   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  416   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  417 
  418   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  419   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  420   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  421 
  422   _LONG_REG_mask = _PTR_REG_mask;
  423   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  424   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  425 
  426   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  429   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  430   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  431 
  432   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  433   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  434   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  435 
  436   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  439   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  440   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  441 
  442   _STACK_OR_LONG_NO_RAX_RDX_REG_mask = _LONG_NO_RAX_RDX_REG_mask;
  443   _STACK_OR_LONG_NO_RAX_RDX_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  444 
  445   _INT_REG_mask = _ALL_INT_REG_mask;
  446   if (PreserveFramePointer) {
  447     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  448   }
  449   if (need_r12_heapbase()) {
  450     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  451   }
  452 
  453   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  454   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  455 
  456   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  457   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  458   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  459 
  460   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  461   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  462 
  463   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  464   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  465   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  466 
  467   _STACK_OR_INT_NO_RAX_RDX_REG_mask = _INT_NO_RAX_RDX_REG_mask;
  468   _STACK_OR_INT_NO_RAX_RDX_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  469 
  470   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  471   // from the float_reg_legacy/float_reg_evex register class.
  472   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  473 }
  474 
  475 static bool generate_vzeroupper(Compile* C) {
  476   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  477 }
  478 
  479 static int clear_avx_size() {
  480   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  481 }
  482 
  483 // !!!!! Special hack to get all types of calls to specify the byte offset
  484 //       from the start of the call to the point where the return address
  485 //       will point.
  486 int MachCallStaticJavaNode::ret_addr_offset()
  487 {
  488   int offset = 5; // 5 bytes from start of call to where return address points
  489   offset += clear_avx_size();
  490   return offset;
  491 }
  492 
  493 int MachCallDynamicJavaNode::ret_addr_offset()
  494 {
  495   int offset = 15; // 15 bytes from start of call to where return address points
  496   offset += clear_avx_size();
  497   return offset;
  498 }
  499 
  500 int MachCallRuntimeNode::ret_addr_offset() {
  501   int offset = 13; // movq r10,#addr; callq (r10)
  502   if (this->ideal_Opcode() != Op_CallLeafVector) {
  503     offset += clear_avx_size();
  504   }
  505   return offset;
  506 }
  507 
  508 int MachCallNativeNode::ret_addr_offset() {
  509   int offset = 13; // movq r10,#addr; callq (r10)
  510   offset += clear_avx_size();
  511   return offset;
  512 }
  513 //
  514 // Compute padding required for nodes which need alignment
  515 //
  516 
  517 // The address of the call instruction needs to be 4-byte aligned to
  518 // ensure that it does not span a cache line so that it can be patched.
  519 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  520 {
  521   current_offset += clear_avx_size(); // skip vzeroupper
  522   current_offset += 1; // skip call opcode byte
  523   return align_up(current_offset, alignment_required()) - current_offset;
  524 }
  525 
  526 // The address of the call instruction needs to be 4-byte aligned to
  527 // ensure that it does not span a cache line so that it can be patched.
  528 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  529 {
  530   current_offset += clear_avx_size(); // skip vzeroupper
  531   current_offset += 11; // skip movq instruction + call opcode byte
  532   return align_up(current_offset, alignment_required()) - current_offset;
  533 }
  534 
  535 // EMIT_RM()
  536 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  537   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  538   cbuf.insts()->emit_int8(c);
  539 }
  540 
  541 // EMIT_CC()
  542 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  543   unsigned char c = (unsigned char) (f1 | f2);
  544   cbuf.insts()->emit_int8(c);
  545 }
  546 
  547 // EMIT_OPCODE()
  548 void emit_opcode(CodeBuffer &cbuf, int code) {
  549   cbuf.insts()->emit_int8((unsigned char) code);
  550 }
  551 
  552 // EMIT_OPCODE() w/ relocation information
  553 void emit_opcode(CodeBuffer &cbuf,
  554                  int code, relocInfo::relocType reloc, int offset, int format)
  555 {
  556   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  557   emit_opcode(cbuf, code);
  558 }
  559 
  560 // EMIT_D8()
  561 void emit_d8(CodeBuffer &cbuf, int d8) {
  562   cbuf.insts()->emit_int8((unsigned char) d8);
  563 }
  564 
  565 // EMIT_D16()
  566 void emit_d16(CodeBuffer &cbuf, int d16) {
  567   cbuf.insts()->emit_int16(d16);
  568 }
  569 
  570 // EMIT_D32()
  571 void emit_d32(CodeBuffer &cbuf, int d32) {
  572   cbuf.insts()->emit_int32(d32);
  573 }
  574 
  575 // EMIT_D64()
  576 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  577   cbuf.insts()->emit_int64(d64);
  578 }
  579 
  580 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  581 void emit_d32_reloc(CodeBuffer& cbuf,
  582                     int d32,
  583                     relocInfo::relocType reloc,
  584                     int format)
  585 {
  586   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  587   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  588   cbuf.insts()->emit_int32(d32);
  589 }
  590 
  591 // emit 32 bit value and construct relocation entry from RelocationHolder
  592 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  593 #ifdef ASSERT
  594   if (rspec.reloc()->type() == relocInfo::oop_type &&
  595       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  596     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  597     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  598   }
  599 #endif
  600   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  601   cbuf.insts()->emit_int32(d32);
  602 }
  603 
  604 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  605   address next_ip = cbuf.insts_end() + 4;
  606   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  607                  external_word_Relocation::spec(addr),
  608                  RELOC_DISP32);
  609 }
  610 
  611 
  612 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  613 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  614   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  615   cbuf.insts()->emit_int64(d64);
  616 }
  617 
  618 // emit 64 bit value and construct relocation entry from RelocationHolder
  619 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  620 #ifdef ASSERT
  621   if (rspec.reloc()->type() == relocInfo::oop_type &&
  622       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  623     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  624     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  625   }
  626 #endif
  627   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  628   cbuf.insts()->emit_int64(d64);
  629 }
  630 
  631 // Access stack slot for load or store
  632 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  633 {
  634   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  635   if (-0x80 <= disp && disp < 0x80) {
  636     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  637     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  638     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  639   } else {
  640     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  641     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  642     emit_d32(cbuf, disp);     // Displacement // R/M byte
  643   }
  644 }
  645 
  646    // rRegI ereg, memory mem) %{    // emit_reg_mem
  647 void encode_RegMem(CodeBuffer &cbuf,
  648                    int reg,
  649                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  650 {
  651   assert(disp_reloc == relocInfo::none, "cannot have disp");
  652   int regenc = reg & 7;
  653   int baseenc = base & 7;
  654   int indexenc = index & 7;
  655 
  656   // There is no index & no scale, use form without SIB byte
  657   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  658     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  659     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  660       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  661     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  662       // If 8-bit displacement, mode 0x1
  663       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  664       emit_d8(cbuf, disp);
  665     } else {
  666       // If 32-bit displacement
  667       if (base == -1) { // Special flag for absolute address
  668         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  669         if (disp_reloc != relocInfo::none) {
  670           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  671         } else {
  672           emit_d32(cbuf, disp);
  673         }
  674       } else {
  675         // Normal base + offset
  676         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  677         if (disp_reloc != relocInfo::none) {
  678           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  679         } else {
  680           emit_d32(cbuf, disp);
  681         }
  682       }
  683     }
  684   } else {
  685     // Else, encode with the SIB byte
  686     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  687     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  688       // If no displacement
  689       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  690       emit_rm(cbuf, scale, indexenc, baseenc);
  691     } else {
  692       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  693         // If 8-bit displacement, mode 0x1
  694         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  695         emit_rm(cbuf, scale, indexenc, baseenc);
  696         emit_d8(cbuf, disp);
  697       } else {
  698         // If 32-bit displacement
  699         if (base == 0x04 ) {
  700           emit_rm(cbuf, 0x2, regenc, 0x4);
  701           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  702         } else {
  703           emit_rm(cbuf, 0x2, regenc, 0x4);
  704           emit_rm(cbuf, scale, indexenc, baseenc); // *
  705         }
  706         if (disp_reloc != relocInfo::none) {
  707           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  708         } else {
  709           emit_d32(cbuf, disp);
  710         }
  711       }
  712     }
  713   }
  714 }
  715 
  716 // This could be in MacroAssembler but it's fairly C2 specific
  717 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  718   Label exit;
  719   __ jccb(Assembler::noParity, exit);
  720   __ pushf();
  721   //
  722   // comiss/ucomiss instructions set ZF,PF,CF flags and
  723   // zero OF,AF,SF for NaN values.
  724   // Fixup flags by zeroing ZF,PF so that compare of NaN
  725   // values returns 'less than' result (CF is set).
  726   // Leave the rest of flags unchanged.
  727   //
  728   //    7 6 5 4 3 2 1 0
  729   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  730   //    0 0 1 0 1 0 1 1   (0x2B)
  731   //
  732   __ andq(Address(rsp, 0), 0xffffff2b);
  733   __ popf();
  734   __ bind(exit);
  735 }
  736 
  737 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  738   Label done;
  739   __ movl(dst, -1);
  740   __ jcc(Assembler::parity, done);
  741   __ jcc(Assembler::below, done);
  742   __ setb(Assembler::notEqual, dst);
  743   __ movzbl(dst, dst);
  744   __ bind(done);
  745 }
  746 
  747 // Math.min()    # Math.max()
  748 // --------------------------
  749 // ucomis[s/d]   #
  750 // ja   -> b     # a
  751 // jp   -> NaN   # NaN
  752 // jb   -> a     # b
  753 // je            #
  754 // |-jz -> a | b # a & b
  755 // |    -> a     #
  756 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  757                      XMMRegister a, XMMRegister b,
  758                      XMMRegister xmmt, Register rt,
  759                      bool min, bool single) {
  760 
  761   Label nan, zero, below, above, done;
  762 
  763   if (single)
  764     __ ucomiss(a, b);
  765   else
  766     __ ucomisd(a, b);
  767 
  768   if (dst->encoding() != (min ? b : a)->encoding())
  769     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  770   else
  771     __ jccb(Assembler::above, done);
  772 
  773   __ jccb(Assembler::parity, nan);  // PF=1
  774   __ jccb(Assembler::below, below); // CF=1
  775 
  776   // equal
  777   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  778   if (single) {
  779     __ ucomiss(a, xmmt);
  780     __ jccb(Assembler::equal, zero);
  781 
  782     __ movflt(dst, a);
  783     __ jmp(done);
  784   }
  785   else {
  786     __ ucomisd(a, xmmt);
  787     __ jccb(Assembler::equal, zero);
  788 
  789     __ movdbl(dst, a);
  790     __ jmp(done);
  791   }
  792 
  793   __ bind(zero);
  794   if (min)
  795     __ vpor(dst, a, b, Assembler::AVX_128bit);
  796   else
  797     __ vpand(dst, a, b, Assembler::AVX_128bit);
  798 
  799   __ jmp(done);
  800 
  801   __ bind(above);
  802   if (single)
  803     __ movflt(dst, min ? b : a);
  804   else
  805     __ movdbl(dst, min ? b : a);
  806 
  807   __ jmp(done);
  808 
  809   __ bind(nan);
  810   if (single) {
  811     __ movl(rt, 0x7fc00000); // Float.NaN
  812     __ movdl(dst, rt);
  813   }
  814   else {
  815     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  816     __ movdq(dst, rt);
  817   }
  818   __ jmp(done);
  819 
  820   __ bind(below);
  821   if (single)
  822     __ movflt(dst, min ? a : b);
  823   else
  824     __ movdbl(dst, min ? a : b);
  825 
  826   __ bind(done);
  827 }
  828 
  829 //=============================================================================
  830 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  831 
  832 int ConstantTable::calculate_table_base_offset() const {
  833   return 0;  // absolute addressing, no offset
  834 }
  835 
  836 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  837 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  838   ShouldNotReachHere();
  839 }
  840 
  841 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  842   // Empty encoding
  843 }
  844 
  845 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  846   return 0;
  847 }
  848 
  849 #ifndef PRODUCT
  850 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  851   st->print("# MachConstantBaseNode (empty encoding)");
  852 }
  853 #endif
  854 
  855 
  856 //=============================================================================
  857 #ifndef PRODUCT
  858 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  859   Compile* C = ra_->C;
  860 
  861   int framesize = C->output()->frame_size_in_bytes();
  862   int bangsize = C->output()->bang_size_in_bytes();
  863   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  864   // Remove wordSize for return addr which is already pushed.
  865   framesize -= wordSize;
  866 
  867   if (C->output()->need_stack_bang(bangsize)) {
  868     framesize -= wordSize;
  869     st->print("# stack bang (%d bytes)", bangsize);
  870     st->print("\n\t");
  871     st->print("pushq   rbp\t# Save rbp");
  872     if (PreserveFramePointer) {
  873         st->print("\n\t");
  874         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  875     }
  876     if (framesize) {
  877       st->print("\n\t");
  878       st->print("subq    rsp, #%d\t# Create frame",framesize);
  879     }
  880   } else {
  881     st->print("subq    rsp, #%d\t# Create frame",framesize);
  882     st->print("\n\t");
  883     framesize -= wordSize;
  884     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  885     if (PreserveFramePointer) {
  886       st->print("\n\t");
  887       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  888       if (framesize > 0) {
  889         st->print("\n\t");
  890         st->print("addq    rbp, #%d", framesize);
  891       }
  892     }
  893   }
  894 
  895   if (VerifyStackAtCalls) {
  896     st->print("\n\t");
  897     framesize -= wordSize;
  898     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  899 #ifdef ASSERT
  900     st->print("\n\t");
  901     st->print("# stack alignment check");
  902 #endif
  903   }
  904   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  905     st->print("\n\t");
  906     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  907     st->print("\n\t");
  908     st->print("je      fast_entry\t");
  909     st->print("\n\t");
  910     st->print("call    #nmethod_entry_barrier_stub\t");
  911     st->print("\n\tfast_entry:");
  912   }
  913   st->cr();
  914 }
  915 #endif
  916 
  917 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  918   Compile* C = ra_->C;
  919   MacroAssembler _masm(&cbuf);
  920 
  921   int framesize = C->output()->frame_size_in_bytes();
  922   int bangsize = C->output()->bang_size_in_bytes();
  923 
  924   if (C->clinit_barrier_on_entry()) {
  925     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  926     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  927 
  928     Label L_skip_barrier;
  929     Register klass = rscratch1;
  930 
  931     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  932     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  933 
  934     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  935 
  936     __ bind(L_skip_barrier);
  937   }
  938 
  939   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  940 
  941   C->output()->set_frame_complete(cbuf.insts_size());
  942 
  943   if (C->has_mach_constant_base_node()) {
  944     // NOTE: We set the table base offset here because users might be
  945     // emitted before MachConstantBaseNode.
  946     ConstantTable& constant_table = C->output()->constant_table();
  947     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  948   }
  949 }
  950 
  951 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  952 {
  953   return MachNode::size(ra_); // too many variables; just compute it
  954                               // the hard way
  955 }
  956 
  957 int MachPrologNode::reloc() const
  958 {
  959   return 0; // a large enough number
  960 }
  961 
  962 //=============================================================================
  963 #ifndef PRODUCT
  964 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  965 {
  966   Compile* C = ra_->C;
  967   if (generate_vzeroupper(C)) {
  968     st->print("vzeroupper");
  969     st->cr(); st->print("\t");
  970   }
  971 
  972   int framesize = C->output()->frame_size_in_bytes();
  973   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  974   // Remove word for return adr already pushed
  975   // and RBP
  976   framesize -= 2*wordSize;
  977 
  978   if (framesize) {
  979     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  980     st->print("\t");
  981   }
  982 
  983   st->print_cr("popq    rbp");
  984   if (do_polling() && C->is_method_compilation()) {
  985     st->print("\t");
  986     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  987                  "ja       #safepoint_stub\t"
  988                  "# Safepoint: poll for GC");
  989   }
  990 }
  991 #endif
  992 
  993 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  994 {
  995   Compile* C = ra_->C;
  996   MacroAssembler _masm(&cbuf);
  997 
  998   if (generate_vzeroupper(C)) {
  999     // Clear upper bits of YMM registers when current compiled code uses
 1000     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1001     __ vzeroupper();
 1002   }
 1003 
 1004   int framesize = C->output()->frame_size_in_bytes();
 1005   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 1006   // Remove word for return adr already pushed
 1007   // and RBP
 1008   framesize -= 2*wordSize;
 1009 
 1010   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 1011 
 1012   if (framesize) {
 1013     emit_opcode(cbuf, Assembler::REX_W);
 1014     if (framesize < 0x80) {
 1015       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 1016       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 1017       emit_d8(cbuf, framesize);
 1018     } else {
 1019       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 1020       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 1021       emit_d32(cbuf, framesize);
 1022     }
 1023   }
 1024 
 1025   // popq rbp
 1026   emit_opcode(cbuf, 0x58 | RBP_enc);
 1027 
 1028   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1029     __ reserved_stack_check();
 1030   }
 1031 
 1032   if (do_polling() && C->is_method_compilation()) {
 1033     MacroAssembler _masm(&cbuf);
 1034     Label dummy_label;
 1035     Label* code_stub = &dummy_label;
 1036     if (!C->output()->in_scratch_emit_size()) {
 1037       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1038     }
 1039     __ relocate(relocInfo::poll_return_type);
 1040     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1041   }
 1042 }
 1043 
 1044 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1045 {
 1046   return MachNode::size(ra_); // too many variables; just compute it
 1047                               // the hard way
 1048 }
 1049 
 1050 int MachEpilogNode::reloc() const
 1051 {
 1052   return 2; // a large enough number
 1053 }
 1054 
 1055 const Pipeline* MachEpilogNode::pipeline() const
 1056 {
 1057   return MachNode::pipeline_class();
 1058 }
 1059 
 1060 //=============================================================================
 1061 
 1062 enum RC {
 1063   rc_bad,
 1064   rc_int,
 1065   rc_kreg,
 1066   rc_float,
 1067   rc_stack
 1068 };
 1069 
 1070 static enum RC rc_class(OptoReg::Name reg)
 1071 {
 1072   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1073 
 1074   if (OptoReg::is_stack(reg)) return rc_stack;
 1075 
 1076   VMReg r = OptoReg::as_VMReg(reg);
 1077 
 1078   if (r->is_Register()) return rc_int;
 1079 
 1080   if (r->is_KRegister()) return rc_kreg;
 1081 
 1082   assert(r->is_XMMRegister(), "must be");
 1083   return rc_float;
 1084 }
 1085 
 1086 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1087 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1088                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1089 
 1090 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1091                      int stack_offset, int reg, uint ireg, outputStream* st);
 1092 
 1093 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1094                                       int dst_offset, uint ireg, outputStream* st) {
 1095   if (cbuf) {
 1096     MacroAssembler _masm(cbuf);
 1097     switch (ireg) {
 1098     case Op_VecS:
 1099       __ movq(Address(rsp, -8), rax);
 1100       __ movl(rax, Address(rsp, src_offset));
 1101       __ movl(Address(rsp, dst_offset), rax);
 1102       __ movq(rax, Address(rsp, -8));
 1103       break;
 1104     case Op_VecD:
 1105       __ pushq(Address(rsp, src_offset));
 1106       __ popq (Address(rsp, dst_offset));
 1107       break;
 1108     case Op_VecX:
 1109       __ pushq(Address(rsp, src_offset));
 1110       __ popq (Address(rsp, dst_offset));
 1111       __ pushq(Address(rsp, src_offset+8));
 1112       __ popq (Address(rsp, dst_offset+8));
 1113       break;
 1114     case Op_VecY:
 1115       __ vmovdqu(Address(rsp, -32), xmm0);
 1116       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1117       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1118       __ vmovdqu(xmm0, Address(rsp, -32));
 1119       break;
 1120     case Op_VecZ:
 1121       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1122       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1123       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1124       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1125       break;
 1126     default:
 1127       ShouldNotReachHere();
 1128     }
 1129 #ifndef PRODUCT
 1130   } else {
 1131     switch (ireg) {
 1132     case Op_VecS:
 1133       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1134                 "movl    rax, [rsp + #%d]\n\t"
 1135                 "movl    [rsp + #%d], rax\n\t"
 1136                 "movq    rax, [rsp - #8]",
 1137                 src_offset, dst_offset);
 1138       break;
 1139     case Op_VecD:
 1140       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1141                 "popq    [rsp + #%d]",
 1142                 src_offset, dst_offset);
 1143       break;
 1144      case Op_VecX:
 1145       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1146                 "popq    [rsp + #%d]\n\t"
 1147                 "pushq   [rsp + #%d]\n\t"
 1148                 "popq    [rsp + #%d]",
 1149                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1150       break;
 1151     case Op_VecY:
 1152       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1153                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1154                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1155                 "vmovdqu xmm0, [rsp - #32]",
 1156                 src_offset, dst_offset);
 1157       break;
 1158     case Op_VecZ:
 1159       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1160                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1161                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1162                 "vmovdqu xmm0, [rsp - #64]",
 1163                 src_offset, dst_offset);
 1164       break;
 1165     default:
 1166       ShouldNotReachHere();
 1167     }
 1168 #endif
 1169   }
 1170 }
 1171 
 1172 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1173                                        PhaseRegAlloc* ra_,
 1174                                        bool do_size,
 1175                                        outputStream* st) const {
 1176   assert(cbuf != NULL || st  != NULL, "sanity");
 1177   // Get registers to move
 1178   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1179   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1180   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1181   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1182 
 1183   enum RC src_second_rc = rc_class(src_second);
 1184   enum RC src_first_rc = rc_class(src_first);
 1185   enum RC dst_second_rc = rc_class(dst_second);
 1186   enum RC dst_first_rc = rc_class(dst_first);
 1187 
 1188   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1189          "must move at least 1 register" );
 1190 
 1191   if (src_first == dst_first && src_second == dst_second) {
 1192     // Self copy, no move
 1193     return 0;
 1194   }
 1195   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1196     uint ireg = ideal_reg();
 1197     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1198     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1199     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1200       // mem -> mem
 1201       int src_offset = ra_->reg2offset(src_first);
 1202       int dst_offset = ra_->reg2offset(dst_first);
 1203       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1204     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1205       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1206     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1207       int stack_offset = ra_->reg2offset(dst_first);
 1208       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1209     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1210       int stack_offset = ra_->reg2offset(src_first);
 1211       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1212     } else {
 1213       ShouldNotReachHere();
 1214     }
 1215     return 0;
 1216   }
 1217   if (src_first_rc == rc_stack) {
 1218     // mem ->
 1219     if (dst_first_rc == rc_stack) {
 1220       // mem -> mem
 1221       assert(src_second != dst_first, "overlap");
 1222       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1223           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1224         // 64-bit
 1225         int src_offset = ra_->reg2offset(src_first);
 1226         int dst_offset = ra_->reg2offset(dst_first);
 1227         if (cbuf) {
 1228           MacroAssembler _masm(cbuf);
 1229           __ pushq(Address(rsp, src_offset));
 1230           __ popq (Address(rsp, dst_offset));
 1231 #ifndef PRODUCT
 1232         } else {
 1233           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1234                     "popq    [rsp + #%d]",
 1235                      src_offset, dst_offset);
 1236 #endif
 1237         }
 1238       } else {
 1239         // 32-bit
 1240         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1241         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1242         // No pushl/popl, so:
 1243         int src_offset = ra_->reg2offset(src_first);
 1244         int dst_offset = ra_->reg2offset(dst_first);
 1245         if (cbuf) {
 1246           MacroAssembler _masm(cbuf);
 1247           __ movq(Address(rsp, -8), rax);
 1248           __ movl(rax, Address(rsp, src_offset));
 1249           __ movl(Address(rsp, dst_offset), rax);
 1250           __ movq(rax, Address(rsp, -8));
 1251 #ifndef PRODUCT
 1252         } else {
 1253           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1254                     "movl    rax, [rsp + #%d]\n\t"
 1255                     "movl    [rsp + #%d], rax\n\t"
 1256                     "movq    rax, [rsp - #8]",
 1257                      src_offset, dst_offset);
 1258 #endif
 1259         }
 1260       }
 1261       return 0;
 1262     } else if (dst_first_rc == rc_int) {
 1263       // mem -> gpr
 1264       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1265           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1266         // 64-bit
 1267         int offset = ra_->reg2offset(src_first);
 1268         if (cbuf) {
 1269           MacroAssembler _masm(cbuf);
 1270           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1271 #ifndef PRODUCT
 1272         } else {
 1273           st->print("movq    %s, [rsp + #%d]\t# spill",
 1274                      Matcher::regName[dst_first],
 1275                      offset);
 1276 #endif
 1277         }
 1278       } else {
 1279         // 32-bit
 1280         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1281         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1282         int offset = ra_->reg2offset(src_first);
 1283         if (cbuf) {
 1284           MacroAssembler _masm(cbuf);
 1285           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1286 #ifndef PRODUCT
 1287         } else {
 1288           st->print("movl    %s, [rsp + #%d]\t# spill",
 1289                      Matcher::regName[dst_first],
 1290                      offset);
 1291 #endif
 1292         }
 1293       }
 1294       return 0;
 1295     } else if (dst_first_rc == rc_float) {
 1296       // mem-> xmm
 1297       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1298           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1299         // 64-bit
 1300         int offset = ra_->reg2offset(src_first);
 1301         if (cbuf) {
 1302           MacroAssembler _masm(cbuf);
 1303           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1304 #ifndef PRODUCT
 1305         } else {
 1306           st->print("%s  %s, [rsp + #%d]\t# spill",
 1307                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1308                      Matcher::regName[dst_first],
 1309                      offset);
 1310 #endif
 1311         }
 1312       } else {
 1313         // 32-bit
 1314         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1315         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1316         int offset = ra_->reg2offset(src_first);
 1317         if (cbuf) {
 1318           MacroAssembler _masm(cbuf);
 1319           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1320 #ifndef PRODUCT
 1321         } else {
 1322           st->print("movss   %s, [rsp + #%d]\t# spill",
 1323                      Matcher::regName[dst_first],
 1324                      offset);
 1325 #endif
 1326         }
 1327       }
 1328       return 0;
 1329     } else if (dst_first_rc == rc_kreg) {
 1330       // mem -> kreg
 1331       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1332           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1333         // 64-bit
 1334         int offset = ra_->reg2offset(src_first);
 1335         if (cbuf) {
 1336           MacroAssembler _masm(cbuf);
 1337           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1338 #ifndef PRODUCT
 1339         } else {
 1340           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1341                      Matcher::regName[dst_first],
 1342                      offset);
 1343 #endif
 1344         }
 1345       }
 1346       return 0;
 1347     }
 1348   } else if (src_first_rc == rc_int) {
 1349     // gpr ->
 1350     if (dst_first_rc == rc_stack) {
 1351       // gpr -> mem
 1352       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1353           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1354         // 64-bit
 1355         int offset = ra_->reg2offset(dst_first);
 1356         if (cbuf) {
 1357           MacroAssembler _masm(cbuf);
 1358           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1359 #ifndef PRODUCT
 1360         } else {
 1361           st->print("movq    [rsp + #%d], %s\t# spill",
 1362                      offset,
 1363                      Matcher::regName[src_first]);
 1364 #endif
 1365         }
 1366       } else {
 1367         // 32-bit
 1368         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1369         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1370         int offset = ra_->reg2offset(dst_first);
 1371         if (cbuf) {
 1372           MacroAssembler _masm(cbuf);
 1373           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1374 #ifndef PRODUCT
 1375         } else {
 1376           st->print("movl    [rsp + #%d], %s\t# spill",
 1377                      offset,
 1378                      Matcher::regName[src_first]);
 1379 #endif
 1380         }
 1381       }
 1382       return 0;
 1383     } else if (dst_first_rc == rc_int) {
 1384       // gpr -> gpr
 1385       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1386           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1387         // 64-bit
 1388         if (cbuf) {
 1389           MacroAssembler _masm(cbuf);
 1390           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1391                   as_Register(Matcher::_regEncode[src_first]));
 1392 #ifndef PRODUCT
 1393         } else {
 1394           st->print("movq    %s, %s\t# spill",
 1395                      Matcher::regName[dst_first],
 1396                      Matcher::regName[src_first]);
 1397 #endif
 1398         }
 1399         return 0;
 1400       } else {
 1401         // 32-bit
 1402         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1403         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1404         if (cbuf) {
 1405           MacroAssembler _masm(cbuf);
 1406           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1407                   as_Register(Matcher::_regEncode[src_first]));
 1408 #ifndef PRODUCT
 1409         } else {
 1410           st->print("movl    %s, %s\t# spill",
 1411                      Matcher::regName[dst_first],
 1412                      Matcher::regName[src_first]);
 1413 #endif
 1414         }
 1415         return 0;
 1416       }
 1417     } else if (dst_first_rc == rc_float) {
 1418       // gpr -> xmm
 1419       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1420           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1421         // 64-bit
 1422         if (cbuf) {
 1423           MacroAssembler _masm(cbuf);
 1424           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1425 #ifndef PRODUCT
 1426         } else {
 1427           st->print("movdq   %s, %s\t# spill",
 1428                      Matcher::regName[dst_first],
 1429                      Matcher::regName[src_first]);
 1430 #endif
 1431         }
 1432       } else {
 1433         // 32-bit
 1434         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1435         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1436         if (cbuf) {
 1437           MacroAssembler _masm(cbuf);
 1438           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1439 #ifndef PRODUCT
 1440         } else {
 1441           st->print("movdl   %s, %s\t# spill",
 1442                      Matcher::regName[dst_first],
 1443                      Matcher::regName[src_first]);
 1444 #endif
 1445         }
 1446       }
 1447       return 0;
 1448     } else if (dst_first_rc == rc_kreg) {
 1449       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1450           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1451         // 64-bit
 1452         if (cbuf) {
 1453           MacroAssembler _masm(cbuf);
 1454           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1455   #ifndef PRODUCT
 1456         } else {
 1457            st->print("kmovq   %s, %s\t# spill",
 1458                        Matcher::regName[dst_first],
 1459                        Matcher::regName[src_first]);
 1460   #endif
 1461         }
 1462       }
 1463       Unimplemented();
 1464       return 0;
 1465     }
 1466   } else if (src_first_rc == rc_float) {
 1467     // xmm ->
 1468     if (dst_first_rc == rc_stack) {
 1469       // xmm -> mem
 1470       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1471           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1472         // 64-bit
 1473         int offset = ra_->reg2offset(dst_first);
 1474         if (cbuf) {
 1475           MacroAssembler _masm(cbuf);
 1476           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1477 #ifndef PRODUCT
 1478         } else {
 1479           st->print("movsd   [rsp + #%d], %s\t# spill",
 1480                      offset,
 1481                      Matcher::regName[src_first]);
 1482 #endif
 1483         }
 1484       } else {
 1485         // 32-bit
 1486         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1487         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1488         int offset = ra_->reg2offset(dst_first);
 1489         if (cbuf) {
 1490           MacroAssembler _masm(cbuf);
 1491           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1492 #ifndef PRODUCT
 1493         } else {
 1494           st->print("movss   [rsp + #%d], %s\t# spill",
 1495                      offset,
 1496                      Matcher::regName[src_first]);
 1497 #endif
 1498         }
 1499       }
 1500       return 0;
 1501     } else if (dst_first_rc == rc_int) {
 1502       // xmm -> gpr
 1503       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1504           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1505         // 64-bit
 1506         if (cbuf) {
 1507           MacroAssembler _masm(cbuf);
 1508           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1509 #ifndef PRODUCT
 1510         } else {
 1511           st->print("movdq   %s, %s\t# spill",
 1512                      Matcher::regName[dst_first],
 1513                      Matcher::regName[src_first]);
 1514 #endif
 1515         }
 1516       } else {
 1517         // 32-bit
 1518         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1519         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1520         if (cbuf) {
 1521           MacroAssembler _masm(cbuf);
 1522           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1523 #ifndef PRODUCT
 1524         } else {
 1525           st->print("movdl   %s, %s\t# spill",
 1526                      Matcher::regName[dst_first],
 1527                      Matcher::regName[src_first]);
 1528 #endif
 1529         }
 1530       }
 1531       return 0;
 1532     } else if (dst_first_rc == rc_float) {
 1533       // xmm -> xmm
 1534       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1535           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1536         // 64-bit
 1537         if (cbuf) {
 1538           MacroAssembler _masm(cbuf);
 1539           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1540 #ifndef PRODUCT
 1541         } else {
 1542           st->print("%s  %s, %s\t# spill",
 1543                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1544                      Matcher::regName[dst_first],
 1545                      Matcher::regName[src_first]);
 1546 #endif
 1547         }
 1548       } else {
 1549         // 32-bit
 1550         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1551         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1552         if (cbuf) {
 1553           MacroAssembler _masm(cbuf);
 1554           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1555 #ifndef PRODUCT
 1556         } else {
 1557           st->print("%s  %s, %s\t# spill",
 1558                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1559                      Matcher::regName[dst_first],
 1560                      Matcher::regName[src_first]);
 1561 #endif
 1562         }
 1563       }
 1564       return 0;
 1565     } else if (dst_first_rc == rc_kreg) {
 1566       assert(false, "Illegal spilling");
 1567       return 0;
 1568     }
 1569   } else if (src_first_rc == rc_kreg) {
 1570     if (dst_first_rc == rc_stack) {
 1571       // mem -> kreg
 1572       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1573           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1574         // 64-bit
 1575         int offset = ra_->reg2offset(dst_first);
 1576         if (cbuf) {
 1577           MacroAssembler _masm(cbuf);
 1578           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1579 #ifndef PRODUCT
 1580         } else {
 1581           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1582                      offset,
 1583                      Matcher::regName[src_first]);
 1584 #endif
 1585         }
 1586       }
 1587       return 0;
 1588     } else if (dst_first_rc == rc_int) {
 1589       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1590           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1591         // 64-bit
 1592         if (cbuf) {
 1593           MacroAssembler _masm(cbuf);
 1594           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1595 #ifndef PRODUCT
 1596         } else {
 1597          st->print("kmovq   %s, %s\t# spill",
 1598                      Matcher::regName[dst_first],
 1599                      Matcher::regName[src_first]);
 1600 #endif
 1601         }
 1602       }
 1603       Unimplemented();
 1604       return 0;
 1605     } else if (dst_first_rc == rc_kreg) {
 1606       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1607           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1608         // 64-bit
 1609         if (cbuf) {
 1610           MacroAssembler _masm(cbuf);
 1611           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1612 #ifndef PRODUCT
 1613         } else {
 1614          st->print("kmovq   %s, %s\t# spill",
 1615                      Matcher::regName[dst_first],
 1616                      Matcher::regName[src_first]);
 1617 #endif
 1618         }
 1619       }
 1620       return 0;
 1621     } else if (dst_first_rc == rc_float) {
 1622       assert(false, "Illegal spill");
 1623       return 0;
 1624     }
 1625   }
 1626 
 1627   assert(0," foo ");
 1628   Unimplemented();
 1629   return 0;
 1630 }
 1631 
 1632 #ifndef PRODUCT
 1633 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1634   implementation(NULL, ra_, false, st);
 1635 }
 1636 #endif
 1637 
 1638 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1639   implementation(&cbuf, ra_, false, NULL);
 1640 }
 1641 
 1642 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1643   return MachNode::size(ra_);
 1644 }
 1645 
 1646 //=============================================================================
 1647 #ifndef PRODUCT
 1648 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1649 {
 1650   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1651   int reg = ra_->get_reg_first(this);
 1652   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1653             Matcher::regName[reg], offset);
 1654 }
 1655 #endif
 1656 
 1657 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1658 {
 1659   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1660   int reg = ra_->get_encode(this);
 1661   if (offset >= 0x80) {
 1662     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1663     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1664     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1665     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1666     emit_d32(cbuf, offset);
 1667   } else {
 1668     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1669     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1670     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1671     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1672     emit_d8(cbuf, offset);
 1673   }
 1674 }
 1675 
 1676 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1677 {
 1678   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1679   return (offset < 0x80) ? 5 : 8; // REX
 1680 }
 1681 
 1682 //=============================================================================
 1683 #ifndef PRODUCT
 1684 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1685 {
 1686   if (UseCompressedClassPointers) {
 1687     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1688     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1689     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1690   } else {
 1691     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1692                  "# Inline cache check");
 1693   }
 1694   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1695   st->print_cr("\tnop\t# nops to align entry point");
 1696 }
 1697 #endif
 1698 
 1699 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1700 {
 1701   MacroAssembler masm(&cbuf);
 1702   uint insts_size = cbuf.insts_size();
 1703   if (UseCompressedClassPointers) {
 1704     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1705     masm.cmpptr(rax, rscratch1);
 1706   } else {
 1707     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1708   }
 1709 
 1710   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1711 
 1712   /* WARNING these NOPs are critical so that verified entry point is properly
 1713      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1714   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1715   if (OptoBreakpoint) {
 1716     // Leave space for int3
 1717     nops_cnt -= 1;
 1718   }
 1719   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1720   if (nops_cnt > 0)
 1721     masm.nop(nops_cnt);
 1722 }
 1723 
 1724 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1725 {
 1726   return MachNode::size(ra_); // too many variables; just compute it
 1727                               // the hard way
 1728 }
 1729 
 1730 
 1731 //=============================================================================
 1732 
 1733 const bool Matcher::supports_vector_calling_convention(void) {
 1734   if (EnableVectorSupport && UseVectorStubs) {
 1735     return true;
 1736   }
 1737   return false;
 1738 }
 1739 
 1740 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1741   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1742   int lo = XMM0_num;
 1743   int hi = XMM0b_num;
 1744   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1745   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1746   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1747   return OptoRegPair(hi, lo);
 1748 }
 1749 
 1750 // Is this branch offset short enough that a short branch can be used?
 1751 //
 1752 // NOTE: If the platform does not provide any short branch variants, then
 1753 //       this method should return false for offset 0.
 1754 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1755   // The passed offset is relative to address of the branch.
 1756   // On 86 a branch displacement is calculated relative to address
 1757   // of a next instruction.
 1758   offset -= br_size;
 1759 
 1760   // the short version of jmpConUCF2 contains multiple branches,
 1761   // making the reach slightly less
 1762   if (rule == jmpConUCF2_rule)
 1763     return (-126 <= offset && offset <= 125);
 1764   return (-128 <= offset && offset <= 127);
 1765 }
 1766 
 1767 // Return whether or not this register is ever used as an argument.
 1768 // This function is used on startup to build the trampoline stubs in
 1769 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1770 // call in the trampoline, and arguments in those registers not be
 1771 // available to the callee.
 1772 bool Matcher::can_be_java_arg(int reg)
 1773 {
 1774   return
 1775     reg ==  RDI_num || reg == RDI_H_num ||
 1776     reg ==  RSI_num || reg == RSI_H_num ||
 1777     reg ==  RDX_num || reg == RDX_H_num ||
 1778     reg ==  RCX_num || reg == RCX_H_num ||
 1779     reg ==   R8_num || reg ==  R8_H_num ||
 1780     reg ==   R9_num || reg ==  R9_H_num ||
 1781     reg ==  R12_num || reg == R12_H_num ||
 1782     reg == XMM0_num || reg == XMM0b_num ||
 1783     reg == XMM1_num || reg == XMM1b_num ||
 1784     reg == XMM2_num || reg == XMM2b_num ||
 1785     reg == XMM3_num || reg == XMM3b_num ||
 1786     reg == XMM4_num || reg == XMM4b_num ||
 1787     reg == XMM5_num || reg == XMM5b_num ||
 1788     reg == XMM6_num || reg == XMM6b_num ||
 1789     reg == XMM7_num || reg == XMM7b_num;
 1790 }
 1791 
 1792 bool Matcher::is_spillable_arg(int reg)
 1793 {
 1794   return can_be_java_arg(reg);
 1795 }
 1796 
 1797 uint Matcher::int_pressure_limit()
 1798 {
 1799   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1800 }
 1801 
 1802 uint Matcher::float_pressure_limit()
 1803 {
 1804   // After experiment around with different values, the following default threshold
 1805   // works best for LCM's register pressure scheduling on x64.
 1806   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1807   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1808   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1809 }
 1810 
 1811 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1812   // In 64 bit mode a code which use multiply when
 1813   // devisor is constant is faster than hardware
 1814   // DIV instruction (it uses MulHiL).
 1815   return false;
 1816 }
 1817 
 1818 // Register for DIVI projection of divmodI
 1819 RegMask Matcher::divI_proj_mask() {
 1820   return INT_RAX_REG_mask();
 1821 }
 1822 
 1823 // Register for MODI projection of divmodI
 1824 RegMask Matcher::modI_proj_mask() {
 1825   return INT_RDX_REG_mask();
 1826 }
 1827 
 1828 // Register for DIVL projection of divmodL
 1829 RegMask Matcher::divL_proj_mask() {
 1830   return LONG_RAX_REG_mask();
 1831 }
 1832 
 1833 // Register for MODL projection of divmodL
 1834 RegMask Matcher::modL_proj_mask() {
 1835   return LONG_RDX_REG_mask();
 1836 }
 1837 
 1838 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1839 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1840     return NO_REG_mask();
 1841 }
 1842 
 1843 %}
 1844 
 1845 //----------ENCODING BLOCK-----------------------------------------------------
 1846 // This block specifies the encoding classes used by the compiler to
 1847 // output byte streams.  Encoding classes are parameterized macros
 1848 // used by Machine Instruction Nodes in order to generate the bit
 1849 // encoding of the instruction.  Operands specify their base encoding
 1850 // interface with the interface keyword.  There are currently
 1851 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1852 // COND_INTER.  REG_INTER causes an operand to generate a function
 1853 // which returns its register number when queried.  CONST_INTER causes
 1854 // an operand to generate a function which returns the value of the
 1855 // constant when queried.  MEMORY_INTER causes an operand to generate
 1856 // four functions which return the Base Register, the Index Register,
 1857 // the Scale Value, and the Offset Value of the operand when queried.
 1858 // COND_INTER causes an operand to generate six functions which return
 1859 // the encoding code (ie - encoding bits for the instruction)
 1860 // associated with each basic boolean condition for a conditional
 1861 // instruction.
 1862 //
 1863 // Instructions specify two basic values for encoding.  Again, a
 1864 // function is available to check if the constant displacement is an
 1865 // oop. They use the ins_encode keyword to specify their encoding
 1866 // classes (which must be a sequence of enc_class names, and their
 1867 // parameters, specified in the encoding block), and they use the
 1868 // opcode keyword to specify, in order, their primary, secondary, and
 1869 // tertiary opcode.  Only the opcode sections which a particular
 1870 // instruction needs for encoding need to be specified.
 1871 encode %{
 1872   // Build emit functions for each basic byte or larger field in the
 1873   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1874   // from C++ code in the enc_class source block.  Emit functions will
 1875   // live in the main source block for now.  In future, we can
 1876   // generalize this by adding a syntax that specifies the sizes of
 1877   // fields in an order, so that the adlc can build the emit functions
 1878   // automagically
 1879 
 1880   // Emit primary opcode
 1881   enc_class OpcP
 1882   %{
 1883     emit_opcode(cbuf, $primary);
 1884   %}
 1885 
 1886   // Emit secondary opcode
 1887   enc_class OpcS
 1888   %{
 1889     emit_opcode(cbuf, $secondary);
 1890   %}
 1891 
 1892   // Emit tertiary opcode
 1893   enc_class OpcT
 1894   %{
 1895     emit_opcode(cbuf, $tertiary);
 1896   %}
 1897 
 1898   // Emit opcode directly
 1899   enc_class Opcode(immI d8)
 1900   %{
 1901     emit_opcode(cbuf, $d8$$constant);
 1902   %}
 1903 
 1904   // Emit size prefix
 1905   enc_class SizePrefix
 1906   %{
 1907     emit_opcode(cbuf, 0x66);
 1908   %}
 1909 
 1910   enc_class reg(rRegI reg)
 1911   %{
 1912     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1913   %}
 1914 
 1915   enc_class reg_reg(rRegI dst, rRegI src)
 1916   %{
 1917     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1918   %}
 1919 
 1920   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1921   %{
 1922     emit_opcode(cbuf, $opcode$$constant);
 1923     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1924   %}
 1925 
 1926   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 1927   enc_class OpcSE(immI imm)
 1928   %{
 1929     // Emit primary opcode and set sign-extend bit
 1930     // Check for 8-bit immediate, and set sign extend bit in opcode
 1931     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 1932       emit_opcode(cbuf, $primary | 0x02);
 1933     } else {
 1934       // 32-bit immediate
 1935       emit_opcode(cbuf, $primary);
 1936     }
 1937   %}
 1938 
 1939   enc_class OpcSErm(rRegI dst, immI imm)
 1940   %{
 1941     // OpcSEr/m
 1942     int dstenc = $dst$$reg;
 1943     if (dstenc >= 8) {
 1944       emit_opcode(cbuf, Assembler::REX_B);
 1945       dstenc -= 8;
 1946     }
 1947     // Emit primary opcode and set sign-extend bit
 1948     // Check for 8-bit immediate, and set sign extend bit in opcode
 1949     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 1950       emit_opcode(cbuf, $primary | 0x02);
 1951     } else {
 1952       // 32-bit immediate
 1953       emit_opcode(cbuf, $primary);
 1954     }
 1955     // Emit r/m byte with secondary opcode, after primary opcode.
 1956     emit_rm(cbuf, 0x3, $secondary, dstenc);
 1957   %}
 1958 
 1959   enc_class OpcSErm_wide(rRegL dst, immI imm)
 1960   %{
 1961     // OpcSEr/m
 1962     int dstenc = $dst$$reg;
 1963     if (dstenc < 8) {
 1964       emit_opcode(cbuf, Assembler::REX_W);
 1965     } else {
 1966       emit_opcode(cbuf, Assembler::REX_WB);
 1967       dstenc -= 8;
 1968     }
 1969     // Emit primary opcode and set sign-extend bit
 1970     // Check for 8-bit immediate, and set sign extend bit in opcode
 1971     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 1972       emit_opcode(cbuf, $primary | 0x02);
 1973     } else {
 1974       // 32-bit immediate
 1975       emit_opcode(cbuf, $primary);
 1976     }
 1977     // Emit r/m byte with secondary opcode, after primary opcode.
 1978     emit_rm(cbuf, 0x3, $secondary, dstenc);
 1979   %}
 1980 
 1981   enc_class Con8or32(immI imm)
 1982   %{
 1983     // Check for 8-bit immediate, and set sign extend bit in opcode
 1984     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 1985       $$$emit8$imm$$constant;
 1986     } else {
 1987       // 32-bit immediate
 1988       $$$emit32$imm$$constant;
 1989     }
 1990   %}
 1991 
 1992   enc_class opc2_reg(rRegI dst)
 1993   %{
 1994     // BSWAP
 1995     emit_cc(cbuf, $secondary, $dst$$reg);
 1996   %}
 1997 
 1998   enc_class opc3_reg(rRegI dst)
 1999   %{
 2000     // BSWAP
 2001     emit_cc(cbuf, $tertiary, $dst$$reg);
 2002   %}
 2003 
 2004   enc_class reg_opc(rRegI div)
 2005   %{
 2006     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2007     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2008   %}
 2009 
 2010   enc_class enc_cmov(cmpOp cop)
 2011   %{
 2012     // CMOV
 2013     $$$emit8$primary;
 2014     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2015   %}
 2016 
 2017   enc_class enc_PartialSubtypeCheck()
 2018   %{
 2019     Register Rrdi = as_Register(RDI_enc); // result register
 2020     Register Rrax = as_Register(RAX_enc); // super class
 2021     Register Rrcx = as_Register(RCX_enc); // killed
 2022     Register Rrsi = as_Register(RSI_enc); // sub class
 2023     Label miss;
 2024     const bool set_cond_codes = true;
 2025 
 2026     MacroAssembler _masm(&cbuf);
 2027     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2028                                      NULL, &miss,
 2029                                      /*set_cond_codes:*/ true);
 2030     if ($primary) {
 2031       __ xorptr(Rrdi, Rrdi);
 2032     }
 2033     __ bind(miss);
 2034   %}
 2035 
 2036   enc_class clear_avx %{
 2037     debug_only(int off0 = cbuf.insts_size());
 2038     if (generate_vzeroupper(Compile::current())) {
 2039       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2040       // Clear upper bits of YMM registers when current compiled code uses
 2041       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2042       MacroAssembler _masm(&cbuf);
 2043       __ vzeroupper();
 2044     }
 2045     debug_only(int off1 = cbuf.insts_size());
 2046     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2047   %}
 2048 
 2049   enc_class Java_To_Runtime(method meth) %{
 2050     // No relocation needed
 2051     MacroAssembler _masm(&cbuf);
 2052     __ mov64(r10, (int64_t) $meth$$method);
 2053     __ call(r10);
 2054   %}
 2055 
 2056   enc_class Java_To_Interpreter(method meth)
 2057   %{
 2058     // CALL Java_To_Interpreter
 2059     // This is the instruction starting address for relocation info.
 2060     cbuf.set_insts_mark();
 2061     $$$emit8$primary;
 2062     // CALL directly to the runtime
 2063     emit_d32_reloc(cbuf,
 2064                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2065                    runtime_call_Relocation::spec(),
 2066                    RELOC_DISP32);
 2067   %}
 2068 
 2069   enc_class Java_Static_Call(method meth)
 2070   %{
 2071     // JAVA STATIC CALL
 2072     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2073     // determine who we intended to call.
 2074     cbuf.set_insts_mark();
 2075     $$$emit8$primary;
 2076 
 2077     if (!_method) {
 2078       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2079                      runtime_call_Relocation::spec(),
 2080                      RELOC_DISP32);
 2081     } else {
 2082       int method_index = resolved_method_index(cbuf);
 2083       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2084                                                   : static_call_Relocation::spec(method_index);
 2085       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2086                      rspec, RELOC_DISP32);
 2087       // Emit stubs for static call.
 2088       address mark = cbuf.insts_mark();
 2089       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2090       if (stub == NULL) {
 2091         ciEnv::current()->record_failure("CodeCache is full");
 2092         return;
 2093       }
 2094     }
 2095   %}
 2096 
 2097   enc_class Java_Dynamic_Call(method meth) %{
 2098     MacroAssembler _masm(&cbuf);
 2099     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2100   %}
 2101 
 2102   enc_class Java_Compiled_Call(method meth)
 2103   %{
 2104     // JAVA COMPILED CALL
 2105     int disp = in_bytes(Method:: from_compiled_offset());
 2106 
 2107     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
 2108     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
 2109 
 2110     // callq *disp(%rax)
 2111     cbuf.set_insts_mark();
 2112     $$$emit8$primary;
 2113     if (disp < 0x80) {
 2114       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
 2115       emit_d8(cbuf, disp); // Displacement
 2116     } else {
 2117       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
 2118       emit_d32(cbuf, disp); // Displacement
 2119     }
 2120   %}
 2121 
 2122   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2123   %{
 2124     // SAL, SAR, SHR
 2125     int dstenc = $dst$$reg;
 2126     if (dstenc >= 8) {
 2127       emit_opcode(cbuf, Assembler::REX_B);
 2128       dstenc -= 8;
 2129     }
 2130     $$$emit8$primary;
 2131     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2132     $$$emit8$shift$$constant;
 2133   %}
 2134 
 2135   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2136   %{
 2137     // SAL, SAR, SHR
 2138     int dstenc = $dst$$reg;
 2139     if (dstenc < 8) {
 2140       emit_opcode(cbuf, Assembler::REX_W);
 2141     } else {
 2142       emit_opcode(cbuf, Assembler::REX_WB);
 2143       dstenc -= 8;
 2144     }
 2145     $$$emit8$primary;
 2146     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2147     $$$emit8$shift$$constant;
 2148   %}
 2149 
 2150   enc_class load_immI(rRegI dst, immI src)
 2151   %{
 2152     int dstenc = $dst$$reg;
 2153     if (dstenc >= 8) {
 2154       emit_opcode(cbuf, Assembler::REX_B);
 2155       dstenc -= 8;
 2156     }
 2157     emit_opcode(cbuf, 0xB8 | dstenc);
 2158     $$$emit32$src$$constant;
 2159   %}
 2160 
 2161   enc_class load_immL(rRegL dst, immL src)
 2162   %{
 2163     int dstenc = $dst$$reg;
 2164     if (dstenc < 8) {
 2165       emit_opcode(cbuf, Assembler::REX_W);
 2166     } else {
 2167       emit_opcode(cbuf, Assembler::REX_WB);
 2168       dstenc -= 8;
 2169     }
 2170     emit_opcode(cbuf, 0xB8 | dstenc);
 2171     emit_d64(cbuf, $src$$constant);
 2172   %}
 2173 
 2174   enc_class load_immUL32(rRegL dst, immUL32 src)
 2175   %{
 2176     // same as load_immI, but this time we care about zeroes in the high word
 2177     int dstenc = $dst$$reg;
 2178     if (dstenc >= 8) {
 2179       emit_opcode(cbuf, Assembler::REX_B);
 2180       dstenc -= 8;
 2181     }
 2182     emit_opcode(cbuf, 0xB8 | dstenc);
 2183     $$$emit32$src$$constant;
 2184   %}
 2185 
 2186   enc_class load_immL32(rRegL dst, immL32 src)
 2187   %{
 2188     int dstenc = $dst$$reg;
 2189     if (dstenc < 8) {
 2190       emit_opcode(cbuf, Assembler::REX_W);
 2191     } else {
 2192       emit_opcode(cbuf, Assembler::REX_WB);
 2193       dstenc -= 8;
 2194     }
 2195     emit_opcode(cbuf, 0xC7);
 2196     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2197     $$$emit32$src$$constant;
 2198   %}
 2199 
 2200   enc_class load_immP31(rRegP dst, immP32 src)
 2201   %{
 2202     // same as load_immI, but this time we care about zeroes in the high word
 2203     int dstenc = $dst$$reg;
 2204     if (dstenc >= 8) {
 2205       emit_opcode(cbuf, Assembler::REX_B);
 2206       dstenc -= 8;
 2207     }
 2208     emit_opcode(cbuf, 0xB8 | dstenc);
 2209     $$$emit32$src$$constant;
 2210   %}
 2211 
 2212   enc_class load_immP(rRegP dst, immP src)
 2213   %{
 2214     int dstenc = $dst$$reg;
 2215     if (dstenc < 8) {
 2216       emit_opcode(cbuf, Assembler::REX_W);
 2217     } else {
 2218       emit_opcode(cbuf, Assembler::REX_WB);
 2219       dstenc -= 8;
 2220     }
 2221     emit_opcode(cbuf, 0xB8 | dstenc);
 2222     // This next line should be generated from ADLC
 2223     if ($src->constant_reloc() != relocInfo::none) {
 2224       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2225     } else {
 2226       emit_d64(cbuf, $src$$constant);
 2227     }
 2228   %}
 2229 
 2230   enc_class Con32(immI src)
 2231   %{
 2232     // Output immediate
 2233     $$$emit32$src$$constant;
 2234   %}
 2235 
 2236   enc_class Con32F_as_bits(immF src)
 2237   %{
 2238     // Output Float immediate bits
 2239     jfloat jf = $src$$constant;
 2240     jint jf_as_bits = jint_cast(jf);
 2241     emit_d32(cbuf, jf_as_bits);
 2242   %}
 2243 
 2244   enc_class Con16(immI src)
 2245   %{
 2246     // Output immediate
 2247     $$$emit16$src$$constant;
 2248   %}
 2249 
 2250   // How is this different from Con32??? XXX
 2251   enc_class Con_d32(immI src)
 2252   %{
 2253     emit_d32(cbuf,$src$$constant);
 2254   %}
 2255 
 2256   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2257     // Output immediate memory reference
 2258     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2259     emit_d32(cbuf, 0x00);
 2260   %}
 2261 
 2262   enc_class lock_prefix()
 2263   %{
 2264     emit_opcode(cbuf, 0xF0); // lock
 2265   %}
 2266 
 2267   enc_class REX_mem(memory mem)
 2268   %{
 2269     if ($mem$$base >= 8) {
 2270       if ($mem$$index < 8) {
 2271         emit_opcode(cbuf, Assembler::REX_B);
 2272       } else {
 2273         emit_opcode(cbuf, Assembler::REX_XB);
 2274       }
 2275     } else {
 2276       if ($mem$$index >= 8) {
 2277         emit_opcode(cbuf, Assembler::REX_X);
 2278       }
 2279     }
 2280   %}
 2281 
 2282   enc_class REX_mem_wide(memory mem)
 2283   %{
 2284     if ($mem$$base >= 8) {
 2285       if ($mem$$index < 8) {
 2286         emit_opcode(cbuf, Assembler::REX_WB);
 2287       } else {
 2288         emit_opcode(cbuf, Assembler::REX_WXB);
 2289       }
 2290     } else {
 2291       if ($mem$$index < 8) {
 2292         emit_opcode(cbuf, Assembler::REX_W);
 2293       } else {
 2294         emit_opcode(cbuf, Assembler::REX_WX);
 2295       }
 2296     }
 2297   %}
 2298 
 2299   // for byte regs
 2300   enc_class REX_breg(rRegI reg)
 2301   %{
 2302     if ($reg$$reg >= 4) {
 2303       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2304     }
 2305   %}
 2306 
 2307   // for byte regs
 2308   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2309   %{
 2310     if ($dst$$reg < 8) {
 2311       if ($src$$reg >= 4) {
 2312         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2313       }
 2314     } else {
 2315       if ($src$$reg < 8) {
 2316         emit_opcode(cbuf, Assembler::REX_R);
 2317       } else {
 2318         emit_opcode(cbuf, Assembler::REX_RB);
 2319       }
 2320     }
 2321   %}
 2322 
 2323   // for byte regs
 2324   enc_class REX_breg_mem(rRegI reg, memory mem)
 2325   %{
 2326     if ($reg$$reg < 8) {
 2327       if ($mem$$base < 8) {
 2328         if ($mem$$index >= 8) {
 2329           emit_opcode(cbuf, Assembler::REX_X);
 2330         } else if ($reg$$reg >= 4) {
 2331           emit_opcode(cbuf, Assembler::REX);
 2332         }
 2333       } else {
 2334         if ($mem$$index < 8) {
 2335           emit_opcode(cbuf, Assembler::REX_B);
 2336         } else {
 2337           emit_opcode(cbuf, Assembler::REX_XB);
 2338         }
 2339       }
 2340     } else {
 2341       if ($mem$$base < 8) {
 2342         if ($mem$$index < 8) {
 2343           emit_opcode(cbuf, Assembler::REX_R);
 2344         } else {
 2345           emit_opcode(cbuf, Assembler::REX_RX);
 2346         }
 2347       } else {
 2348         if ($mem$$index < 8) {
 2349           emit_opcode(cbuf, Assembler::REX_RB);
 2350         } else {
 2351           emit_opcode(cbuf, Assembler::REX_RXB);
 2352         }
 2353       }
 2354     }
 2355   %}
 2356 
 2357   enc_class REX_reg(rRegI reg)
 2358   %{
 2359     if ($reg$$reg >= 8) {
 2360       emit_opcode(cbuf, Assembler::REX_B);
 2361     }
 2362   %}
 2363 
 2364   enc_class REX_reg_wide(rRegI reg)
 2365   %{
 2366     if ($reg$$reg < 8) {
 2367       emit_opcode(cbuf, Assembler::REX_W);
 2368     } else {
 2369       emit_opcode(cbuf, Assembler::REX_WB);
 2370     }
 2371   %}
 2372 
 2373   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2374   %{
 2375     if ($dst$$reg < 8) {
 2376       if ($src$$reg >= 8) {
 2377         emit_opcode(cbuf, Assembler::REX_B);
 2378       }
 2379     } else {
 2380       if ($src$$reg < 8) {
 2381         emit_opcode(cbuf, Assembler::REX_R);
 2382       } else {
 2383         emit_opcode(cbuf, Assembler::REX_RB);
 2384       }
 2385     }
 2386   %}
 2387 
 2388   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2389   %{
 2390     if ($dst$$reg < 8) {
 2391       if ($src$$reg < 8) {
 2392         emit_opcode(cbuf, Assembler::REX_W);
 2393       } else {
 2394         emit_opcode(cbuf, Assembler::REX_WB);
 2395       }
 2396     } else {
 2397       if ($src$$reg < 8) {
 2398         emit_opcode(cbuf, Assembler::REX_WR);
 2399       } else {
 2400         emit_opcode(cbuf, Assembler::REX_WRB);
 2401       }
 2402     }
 2403   %}
 2404 
 2405   enc_class REX_reg_mem(rRegI reg, memory mem)
 2406   %{
 2407     if ($reg$$reg < 8) {
 2408       if ($mem$$base < 8) {
 2409         if ($mem$$index >= 8) {
 2410           emit_opcode(cbuf, Assembler::REX_X);
 2411         }
 2412       } else {
 2413         if ($mem$$index < 8) {
 2414           emit_opcode(cbuf, Assembler::REX_B);
 2415         } else {
 2416           emit_opcode(cbuf, Assembler::REX_XB);
 2417         }
 2418       }
 2419     } else {
 2420       if ($mem$$base < 8) {
 2421         if ($mem$$index < 8) {
 2422           emit_opcode(cbuf, Assembler::REX_R);
 2423         } else {
 2424           emit_opcode(cbuf, Assembler::REX_RX);
 2425         }
 2426       } else {
 2427         if ($mem$$index < 8) {
 2428           emit_opcode(cbuf, Assembler::REX_RB);
 2429         } else {
 2430           emit_opcode(cbuf, Assembler::REX_RXB);
 2431         }
 2432       }
 2433     }
 2434   %}
 2435 
 2436   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2437   %{
 2438     if ($reg$$reg < 8) {
 2439       if ($mem$$base < 8) {
 2440         if ($mem$$index < 8) {
 2441           emit_opcode(cbuf, Assembler::REX_W);
 2442         } else {
 2443           emit_opcode(cbuf, Assembler::REX_WX);
 2444         }
 2445       } else {
 2446         if ($mem$$index < 8) {
 2447           emit_opcode(cbuf, Assembler::REX_WB);
 2448         } else {
 2449           emit_opcode(cbuf, Assembler::REX_WXB);
 2450         }
 2451       }
 2452     } else {
 2453       if ($mem$$base < 8) {
 2454         if ($mem$$index < 8) {
 2455           emit_opcode(cbuf, Assembler::REX_WR);
 2456         } else {
 2457           emit_opcode(cbuf, Assembler::REX_WRX);
 2458         }
 2459       } else {
 2460         if ($mem$$index < 8) {
 2461           emit_opcode(cbuf, Assembler::REX_WRB);
 2462         } else {
 2463           emit_opcode(cbuf, Assembler::REX_WRXB);
 2464         }
 2465       }
 2466     }
 2467   %}
 2468 
 2469   enc_class reg_mem(rRegI ereg, memory mem)
 2470   %{
 2471     // High registers handle in encode_RegMem
 2472     int reg = $ereg$$reg;
 2473     int base = $mem$$base;
 2474     int index = $mem$$index;
 2475     int scale = $mem$$scale;
 2476     int disp = $mem$$disp;
 2477     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2478 
 2479     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2480   %}
 2481 
 2482   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2483   %{
 2484     int rm_byte_opcode = $rm_opcode$$constant;
 2485 
 2486     // High registers handle in encode_RegMem
 2487     int base = $mem$$base;
 2488     int index = $mem$$index;
 2489     int scale = $mem$$scale;
 2490     int displace = $mem$$disp;
 2491 
 2492     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2493                                             // working with static
 2494                                             // globals
 2495     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2496                   disp_reloc);
 2497   %}
 2498 
 2499   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2500   %{
 2501     int reg_encoding = $dst$$reg;
 2502     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2503     int index        = 0x04;            // 0x04 indicates no index
 2504     int scale        = 0x00;            // 0x00 indicates no scale
 2505     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2506     relocInfo::relocType disp_reloc = relocInfo::none;
 2507     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2508                   disp_reloc);
 2509   %}
 2510 
 2511   enc_class neg_reg(rRegI dst)
 2512   %{
 2513     int dstenc = $dst$$reg;
 2514     if (dstenc >= 8) {
 2515       emit_opcode(cbuf, Assembler::REX_B);
 2516       dstenc -= 8;
 2517     }
 2518     // NEG $dst
 2519     emit_opcode(cbuf, 0xF7);
 2520     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2521   %}
 2522 
 2523   enc_class neg_reg_wide(rRegI dst)
 2524   %{
 2525     int dstenc = $dst$$reg;
 2526     if (dstenc < 8) {
 2527       emit_opcode(cbuf, Assembler::REX_W);
 2528     } else {
 2529       emit_opcode(cbuf, Assembler::REX_WB);
 2530       dstenc -= 8;
 2531     }
 2532     // NEG $dst
 2533     emit_opcode(cbuf, 0xF7);
 2534     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2535   %}
 2536 
 2537   enc_class setLT_reg(rRegI dst)
 2538   %{
 2539     int dstenc = $dst$$reg;
 2540     if (dstenc >= 8) {
 2541       emit_opcode(cbuf, Assembler::REX_B);
 2542       dstenc -= 8;
 2543     } else if (dstenc >= 4) {
 2544       emit_opcode(cbuf, Assembler::REX);
 2545     }
 2546     // SETLT $dst
 2547     emit_opcode(cbuf, 0x0F);
 2548     emit_opcode(cbuf, 0x9C);
 2549     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2550   %}
 2551 
 2552   enc_class setNZ_reg(rRegI dst)
 2553   %{
 2554     int dstenc = $dst$$reg;
 2555     if (dstenc >= 8) {
 2556       emit_opcode(cbuf, Assembler::REX_B);
 2557       dstenc -= 8;
 2558     } else if (dstenc >= 4) {
 2559       emit_opcode(cbuf, Assembler::REX);
 2560     }
 2561     // SETNZ $dst
 2562     emit_opcode(cbuf, 0x0F);
 2563     emit_opcode(cbuf, 0x95);
 2564     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2565   %}
 2566 
 2567 
 2568   // Compare the lonogs and set -1, 0, or 1 into dst
 2569   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2570   %{
 2571     int src1enc = $src1$$reg;
 2572     int src2enc = $src2$$reg;
 2573     int dstenc = $dst$$reg;
 2574 
 2575     // cmpq $src1, $src2
 2576     if (src1enc < 8) {
 2577       if (src2enc < 8) {
 2578         emit_opcode(cbuf, Assembler::REX_W);
 2579       } else {
 2580         emit_opcode(cbuf, Assembler::REX_WB);
 2581       }
 2582     } else {
 2583       if (src2enc < 8) {
 2584         emit_opcode(cbuf, Assembler::REX_WR);
 2585       } else {
 2586         emit_opcode(cbuf, Assembler::REX_WRB);
 2587       }
 2588     }
 2589     emit_opcode(cbuf, 0x3B);
 2590     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2591 
 2592     // movl $dst, -1
 2593     if (dstenc >= 8) {
 2594       emit_opcode(cbuf, Assembler::REX_B);
 2595     }
 2596     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2597     emit_d32(cbuf, -1);
 2598 
 2599     // jl,s done
 2600     emit_opcode(cbuf, 0x7C);
 2601     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2602 
 2603     // setne $dst
 2604     if (dstenc >= 4) {
 2605       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2606     }
 2607     emit_opcode(cbuf, 0x0F);
 2608     emit_opcode(cbuf, 0x95);
 2609     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2610 
 2611     // movzbl $dst, $dst
 2612     if (dstenc >= 4) {
 2613       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2614     }
 2615     emit_opcode(cbuf, 0x0F);
 2616     emit_opcode(cbuf, 0xB6);
 2617     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2618   %}
 2619 
 2620   enc_class Push_ResultXD(regD dst) %{
 2621     MacroAssembler _masm(&cbuf);
 2622     __ fstp_d(Address(rsp, 0));
 2623     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2624     __ addptr(rsp, 8);
 2625   %}
 2626 
 2627   enc_class Push_SrcXD(regD src) %{
 2628     MacroAssembler _masm(&cbuf);
 2629     __ subptr(rsp, 8);
 2630     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2631     __ fld_d(Address(rsp, 0));
 2632   %}
 2633 
 2634 
 2635   enc_class enc_rethrow()
 2636   %{
 2637     cbuf.set_insts_mark();
 2638     emit_opcode(cbuf, 0xE9); // jmp entry
 2639     emit_d32_reloc(cbuf,
 2640                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2641                    runtime_call_Relocation::spec(),
 2642                    RELOC_DISP32);
 2643   %}
 2644 
 2645 %}
 2646 
 2647 
 2648 
 2649 //----------FRAME--------------------------------------------------------------
 2650 // Definition of frame structure and management information.
 2651 //
 2652 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2653 //                             |   (to get allocators register number
 2654 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2655 //  r   CALLER     |        |
 2656 //  o     |        +--------+      pad to even-align allocators stack-slot
 2657 //  w     V        |  pad0  |        numbers; owned by CALLER
 2658 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2659 //  h     ^        |   in   |  5
 2660 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2661 //  |     |        |        |  3
 2662 //  |     |        +--------+
 2663 //  V     |        | old out|      Empty on Intel, window on Sparc
 2664 //        |    old |preserve|      Must be even aligned.
 2665 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2666 //        |        |   in   |  3   area for Intel ret address
 2667 //     Owned by    |preserve|      Empty on Sparc.
 2668 //       SELF      +--------+
 2669 //        |        |  pad2  |  2   pad to align old SP
 2670 //        |        +--------+  1
 2671 //        |        | locks  |  0
 2672 //        |        +--------+----> OptoReg::stack0(), even aligned
 2673 //        |        |  pad1  | 11   pad to align new SP
 2674 //        |        +--------+
 2675 //        |        |        | 10
 2676 //        |        | spills |  9   spills
 2677 //        V        |        |  8   (pad0 slot for callee)
 2678 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2679 //        ^        |  out   |  7
 2680 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2681 //     Owned by    +--------+
 2682 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2683 //        |    new |preserve|      Must be even-aligned.
 2684 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2685 //        |        |        |
 2686 //
 2687 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2688 //         known from SELF's arguments and the Java calling convention.
 2689 //         Region 6-7 is determined per call site.
 2690 // Note 2: If the calling convention leaves holes in the incoming argument
 2691 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2692 //         are owned by the CALLEE.  Holes should not be necessary in the
 2693 //         incoming area, as the Java calling convention is completely under
 2694 //         the control of the AD file.  Doubles can be sorted and packed to
 2695 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2696 //         varargs C calling conventions.
 2697 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2698 //         even aligned with pad0 as needed.
 2699 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2700 //         region 6-11 is even aligned; it may be padded out more so that
 2701 //         the region from SP to FP meets the minimum stack alignment.
 2702 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2703 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2704 //         SP meets the minimum alignment.
 2705 
 2706 frame
 2707 %{
 2708   // These three registers define part of the calling convention
 2709   // between compiled code and the interpreter.
 2710   inline_cache_reg(RAX);                // Inline Cache Register
 2711 
 2712   // Optional: name the operand used by cisc-spilling to access
 2713   // [stack_pointer + offset]
 2714   cisc_spilling_operand_name(indOffset32);
 2715 
 2716   // Number of stack slots consumed by locking an object
 2717   sync_stack_slots(2);
 2718 
 2719   // Compiled code's Frame Pointer
 2720   frame_pointer(RSP);
 2721 
 2722   // Interpreter stores its frame pointer in a register which is
 2723   // stored to the stack by I2CAdaptors.
 2724   // I2CAdaptors convert from interpreted java to compiled java.
 2725   interpreter_frame_pointer(RBP);
 2726 
 2727   // Stack alignment requirement
 2728   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2729 
 2730   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2731   // for calls to C.  Supports the var-args backing area for register parms.
 2732   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2733 
 2734   // The after-PROLOG location of the return address.  Location of
 2735   // return address specifies a type (REG or STACK) and a number
 2736   // representing the register number (i.e. - use a register name) or
 2737   // stack slot.
 2738   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2739   // Otherwise, it is above the locks and verification slot and alignment word
 2740   return_addr(STACK - 2 +
 2741               align_up((Compile::current()->in_preserve_stack_slots() +
 2742                         Compile::current()->fixed_slots()),
 2743                        stack_alignment_in_slots()));
 2744 
 2745   // Location of compiled Java return values.  Same as C for now.
 2746   return_value
 2747   %{
 2748     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2749            "only return normal values");
 2750 
 2751     static const int lo[Op_RegL + 1] = {
 2752       0,
 2753       0,
 2754       RAX_num,  // Op_RegN
 2755       RAX_num,  // Op_RegI
 2756       RAX_num,  // Op_RegP
 2757       XMM0_num, // Op_RegF
 2758       XMM0_num, // Op_RegD
 2759       RAX_num   // Op_RegL
 2760     };
 2761     static const int hi[Op_RegL + 1] = {
 2762       0,
 2763       0,
 2764       OptoReg::Bad, // Op_RegN
 2765       OptoReg::Bad, // Op_RegI
 2766       RAX_H_num,    // Op_RegP
 2767       OptoReg::Bad, // Op_RegF
 2768       XMM0b_num,    // Op_RegD
 2769       RAX_H_num     // Op_RegL
 2770     };
 2771     // Excluded flags and vector registers.
 2772     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2773     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2774   %}
 2775 %}
 2776 
 2777 //----------ATTRIBUTES---------------------------------------------------------
 2778 //----------Operand Attributes-------------------------------------------------
 2779 op_attrib op_cost(0);        // Required cost attribute
 2780 
 2781 //----------Instruction Attributes---------------------------------------------
 2782 ins_attrib ins_cost(100);       // Required cost attribute
 2783 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2784 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2785                                 // a non-matching short branch variant
 2786                                 // of some long branch?
 2787 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2788                                 // be a power of 2) specifies the
 2789                                 // alignment that some part of the
 2790                                 // instruction (not necessarily the
 2791                                 // start) requires.  If > 1, a
 2792                                 // compute_padding() function must be
 2793                                 // provided for the instruction
 2794 
 2795 //----------OPERANDS-----------------------------------------------------------
 2796 // Operand definitions must precede instruction definitions for correct parsing
 2797 // in the ADLC because operands constitute user defined types which are used in
 2798 // instruction definitions.
 2799 
 2800 //----------Simple Operands----------------------------------------------------
 2801 // Immediate Operands
 2802 // Integer Immediate
 2803 operand immI()
 2804 %{
 2805   match(ConI);
 2806 
 2807   op_cost(10);
 2808   format %{ %}
 2809   interface(CONST_INTER);
 2810 %}
 2811 
 2812 // Constant for test vs zero
 2813 operand immI_0()
 2814 %{
 2815   predicate(n->get_int() == 0);
 2816   match(ConI);
 2817 
 2818   op_cost(0);
 2819   format %{ %}
 2820   interface(CONST_INTER);
 2821 %}
 2822 
 2823 // Constant for increment
 2824 operand immI_1()
 2825 %{
 2826   predicate(n->get_int() == 1);
 2827   match(ConI);
 2828 
 2829   op_cost(0);
 2830   format %{ %}
 2831   interface(CONST_INTER);
 2832 %}
 2833 
 2834 // Constant for decrement
 2835 operand immI_M1()
 2836 %{
 2837   predicate(n->get_int() == -1);
 2838   match(ConI);
 2839 
 2840   op_cost(0);
 2841   format %{ %}
 2842   interface(CONST_INTER);
 2843 %}
 2844 
 2845 operand immI_2()
 2846 %{
 2847   predicate(n->get_int() == 2);
 2848   match(ConI);
 2849 
 2850   op_cost(0);
 2851   format %{ %}
 2852   interface(CONST_INTER);
 2853 %}
 2854 
 2855 operand immI_4()
 2856 %{
 2857   predicate(n->get_int() == 4);
 2858   match(ConI);
 2859 
 2860   op_cost(0);
 2861   format %{ %}
 2862   interface(CONST_INTER);
 2863 %}
 2864 
 2865 operand immI_8()
 2866 %{
 2867   predicate(n->get_int() == 8);
 2868   match(ConI);
 2869 
 2870   op_cost(0);
 2871   format %{ %}
 2872   interface(CONST_INTER);
 2873 %}
 2874 
 2875 // Valid scale values for addressing modes
 2876 operand immI2()
 2877 %{
 2878   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2879   match(ConI);
 2880 
 2881   format %{ %}
 2882   interface(CONST_INTER);
 2883 %}
 2884 
 2885 operand immU7()
 2886 %{
 2887   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2888   match(ConI);
 2889 
 2890   op_cost(5);
 2891   format %{ %}
 2892   interface(CONST_INTER);
 2893 %}
 2894 
 2895 operand immI8()
 2896 %{
 2897   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2898   match(ConI);
 2899 
 2900   op_cost(5);
 2901   format %{ %}
 2902   interface(CONST_INTER);
 2903 %}
 2904 
 2905 operand immU8()
 2906 %{
 2907   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2908   match(ConI);
 2909 
 2910   op_cost(5);
 2911   format %{ %}
 2912   interface(CONST_INTER);
 2913 %}
 2914 
 2915 operand immI16()
 2916 %{
 2917   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2918   match(ConI);
 2919 
 2920   op_cost(10);
 2921   format %{ %}
 2922   interface(CONST_INTER);
 2923 %}
 2924 
 2925 // Int Immediate non-negative
 2926 operand immU31()
 2927 %{
 2928   predicate(n->get_int() >= 0);
 2929   match(ConI);
 2930 
 2931   op_cost(0);
 2932   format %{ %}
 2933   interface(CONST_INTER);
 2934 %}
 2935 
 2936 // Constant for long shifts
 2937 operand immI_32()
 2938 %{
 2939   predicate( n->get_int() == 32 );
 2940   match(ConI);
 2941 
 2942   op_cost(0);
 2943   format %{ %}
 2944   interface(CONST_INTER);
 2945 %}
 2946 
 2947 // Constant for long shifts
 2948 operand immI_64()
 2949 %{
 2950   predicate( n->get_int() == 64 );
 2951   match(ConI);
 2952 
 2953   op_cost(0);
 2954   format %{ %}
 2955   interface(CONST_INTER);
 2956 %}
 2957 
 2958 // Pointer Immediate
 2959 operand immP()
 2960 %{
 2961   match(ConP);
 2962 
 2963   op_cost(10);
 2964   format %{ %}
 2965   interface(CONST_INTER);
 2966 %}
 2967 
 2968 // NULL Pointer Immediate
 2969 operand immP0()
 2970 %{
 2971   predicate(n->get_ptr() == 0);
 2972   match(ConP);
 2973 
 2974   op_cost(5);
 2975   format %{ %}
 2976   interface(CONST_INTER);
 2977 %}
 2978 
 2979 // Pointer Immediate
 2980 operand immN() %{
 2981   match(ConN);
 2982 
 2983   op_cost(10);
 2984   format %{ %}
 2985   interface(CONST_INTER);
 2986 %}
 2987 
 2988 operand immNKlass() %{
 2989   match(ConNKlass);
 2990 
 2991   op_cost(10);
 2992   format %{ %}
 2993   interface(CONST_INTER);
 2994 %}
 2995 
 2996 // NULL Pointer Immediate
 2997 operand immN0() %{
 2998   predicate(n->get_narrowcon() == 0);
 2999   match(ConN);
 3000 
 3001   op_cost(5);
 3002   format %{ %}
 3003   interface(CONST_INTER);
 3004 %}
 3005 
 3006 operand immP31()
 3007 %{
 3008   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3009             && (n->get_ptr() >> 31) == 0);
 3010   match(ConP);
 3011 
 3012   op_cost(5);
 3013   format %{ %}
 3014   interface(CONST_INTER);
 3015 %}
 3016 
 3017 
 3018 // Long Immediate
 3019 operand immL()
 3020 %{
 3021   match(ConL);
 3022 
 3023   op_cost(20);
 3024   format %{ %}
 3025   interface(CONST_INTER);
 3026 %}
 3027 
 3028 // Long Immediate 8-bit
 3029 operand immL8()
 3030 %{
 3031   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3032   match(ConL);
 3033 
 3034   op_cost(5);
 3035   format %{ %}
 3036   interface(CONST_INTER);
 3037 %}
 3038 
 3039 // Long Immediate 32-bit unsigned
 3040 operand immUL32()
 3041 %{
 3042   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3043   match(ConL);
 3044 
 3045   op_cost(10);
 3046   format %{ %}
 3047   interface(CONST_INTER);
 3048 %}
 3049 
 3050 // Long Immediate 32-bit signed
 3051 operand immL32()
 3052 %{
 3053   predicate(n->get_long() == (int) (n->get_long()));
 3054   match(ConL);
 3055 
 3056   op_cost(15);
 3057   format %{ %}
 3058   interface(CONST_INTER);
 3059 %}
 3060 
 3061 operand immL_Pow2()
 3062 %{
 3063   predicate(is_power_of_2((julong)n->get_long()));
 3064   match(ConL);
 3065 
 3066   op_cost(15);
 3067   format %{ %}
 3068   interface(CONST_INTER);
 3069 %}
 3070 
 3071 operand immL_NotPow2()
 3072 %{
 3073   predicate(is_power_of_2((julong)~n->get_long()));
 3074   match(ConL);
 3075 
 3076   op_cost(15);
 3077   format %{ %}
 3078   interface(CONST_INTER);
 3079 %}
 3080 
 3081 // Long Immediate zero
 3082 operand immL0()
 3083 %{
 3084   predicate(n->get_long() == 0L);
 3085   match(ConL);
 3086 
 3087   op_cost(10);
 3088   format %{ %}
 3089   interface(CONST_INTER);
 3090 %}
 3091 
 3092 // Constant for increment
 3093 operand immL1()
 3094 %{
 3095   predicate(n->get_long() == 1);
 3096   match(ConL);
 3097 
 3098   format %{ %}
 3099   interface(CONST_INTER);
 3100 %}
 3101 
 3102 // Constant for decrement
 3103 operand immL_M1()
 3104 %{
 3105   predicate(n->get_long() == -1);
 3106   match(ConL);
 3107 
 3108   format %{ %}
 3109   interface(CONST_INTER);
 3110 %}
 3111 
 3112 // Long Immediate: the value 10
 3113 operand immL10()
 3114 %{
 3115   predicate(n->get_long() == 10);
 3116   match(ConL);
 3117 
 3118   format %{ %}
 3119   interface(CONST_INTER);
 3120 %}
 3121 
 3122 // Long immediate from 0 to 127.
 3123 // Used for a shorter form of long mul by 10.
 3124 operand immL_127()
 3125 %{
 3126   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3127   match(ConL);
 3128 
 3129   op_cost(10);
 3130   format %{ %}
 3131   interface(CONST_INTER);
 3132 %}
 3133 
 3134 // Long Immediate: low 32-bit mask
 3135 operand immL_32bits()
 3136 %{
 3137   predicate(n->get_long() == 0xFFFFFFFFL);
 3138   match(ConL);
 3139   op_cost(20);
 3140 
 3141   format %{ %}
 3142   interface(CONST_INTER);
 3143 %}
 3144 
 3145 // Int Immediate: 2^n-1, positive
 3146 operand immI_Pow2M1()
 3147 %{
 3148   predicate((n->get_int() > 0)
 3149             && is_power_of_2(n->get_int() + 1));
 3150   match(ConI);
 3151 
 3152   op_cost(20);
 3153   format %{ %}
 3154   interface(CONST_INTER);
 3155 %}
 3156 
 3157 // Float Immediate zero
 3158 operand immF0()
 3159 %{
 3160   predicate(jint_cast(n->getf()) == 0);
 3161   match(ConF);
 3162 
 3163   op_cost(5);
 3164   format %{ %}
 3165   interface(CONST_INTER);
 3166 %}
 3167 
 3168 // Float Immediate
 3169 operand immF()
 3170 %{
 3171   match(ConF);
 3172 
 3173   op_cost(15);
 3174   format %{ %}
 3175   interface(CONST_INTER);
 3176 %}
 3177 
 3178 // Double Immediate zero
 3179 operand immD0()
 3180 %{
 3181   predicate(jlong_cast(n->getd()) == 0);
 3182   match(ConD);
 3183 
 3184   op_cost(5);
 3185   format %{ %}
 3186   interface(CONST_INTER);
 3187 %}
 3188 
 3189 // Double Immediate
 3190 operand immD()
 3191 %{
 3192   match(ConD);
 3193 
 3194   op_cost(15);
 3195   format %{ %}
 3196   interface(CONST_INTER);
 3197 %}
 3198 
 3199 // Immediates for special shifts (sign extend)
 3200 
 3201 // Constants for increment
 3202 operand immI_16()
 3203 %{
 3204   predicate(n->get_int() == 16);
 3205   match(ConI);
 3206 
 3207   format %{ %}
 3208   interface(CONST_INTER);
 3209 %}
 3210 
 3211 operand immI_24()
 3212 %{
 3213   predicate(n->get_int() == 24);
 3214   match(ConI);
 3215 
 3216   format %{ %}
 3217   interface(CONST_INTER);
 3218 %}
 3219 
 3220 // Constant for byte-wide masking
 3221 operand immI_255()
 3222 %{
 3223   predicate(n->get_int() == 255);
 3224   match(ConI);
 3225 
 3226   format %{ %}
 3227   interface(CONST_INTER);
 3228 %}
 3229 
 3230 // Constant for short-wide masking
 3231 operand immI_65535()
 3232 %{
 3233   predicate(n->get_int() == 65535);
 3234   match(ConI);
 3235 
 3236   format %{ %}
 3237   interface(CONST_INTER);
 3238 %}
 3239 
 3240 // Constant for byte-wide masking
 3241 operand immL_255()
 3242 %{
 3243   predicate(n->get_long() == 255);
 3244   match(ConL);
 3245 
 3246   format %{ %}
 3247   interface(CONST_INTER);
 3248 %}
 3249 
 3250 // Constant for short-wide masking
 3251 operand immL_65535()
 3252 %{
 3253   predicate(n->get_long() == 65535);
 3254   match(ConL);
 3255 
 3256   format %{ %}
 3257   interface(CONST_INTER);
 3258 %}
 3259 
 3260 operand kReg()
 3261 %{
 3262   constraint(ALLOC_IN_RC(vectmask_reg));
 3263   match(RegVectMask);
 3264   format %{%}
 3265   interface(REG_INTER);
 3266 %}
 3267 
 3268 operand kReg_K1()
 3269 %{
 3270   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3271   match(RegVectMask);
 3272   format %{%}
 3273   interface(REG_INTER);
 3274 %}
 3275 
 3276 operand kReg_K2()
 3277 %{
 3278   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3279   match(RegVectMask);
 3280   format %{%}
 3281   interface(REG_INTER);
 3282 %}
 3283 
 3284 // Special Registers
 3285 operand kReg_K3()
 3286 %{
 3287   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3288   match(RegVectMask);
 3289   format %{%}
 3290   interface(REG_INTER);
 3291 %}
 3292 
 3293 operand kReg_K4()
 3294 %{
 3295   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3296   match(RegVectMask);
 3297   format %{%}
 3298   interface(REG_INTER);
 3299 %}
 3300 
 3301 operand kReg_K5()
 3302 %{
 3303   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3304   match(RegVectMask);
 3305   format %{%}
 3306   interface(REG_INTER);
 3307 %}
 3308 
 3309 operand kReg_K6()
 3310 %{
 3311   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3312   match(RegVectMask);
 3313   format %{%}
 3314   interface(REG_INTER);
 3315 %}
 3316 
 3317 // Special Registers
 3318 operand kReg_K7()
 3319 %{
 3320   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3321   match(RegVectMask);
 3322   format %{%}
 3323   interface(REG_INTER);
 3324 %}
 3325 
 3326 // Register Operands
 3327 // Integer Register
 3328 operand rRegI()
 3329 %{
 3330   constraint(ALLOC_IN_RC(int_reg));
 3331   match(RegI);
 3332 
 3333   match(rax_RegI);
 3334   match(rbx_RegI);
 3335   match(rcx_RegI);
 3336   match(rdx_RegI);
 3337   match(rdi_RegI);
 3338 
 3339   format %{ %}
 3340   interface(REG_INTER);
 3341 %}
 3342 
 3343 // Special Registers
 3344 operand rax_RegI()
 3345 %{
 3346   constraint(ALLOC_IN_RC(int_rax_reg));
 3347   match(RegI);
 3348   match(rRegI);
 3349 
 3350   format %{ "RAX" %}
 3351   interface(REG_INTER);
 3352 %}
 3353 
 3354 // Special Registers
 3355 operand rbx_RegI()
 3356 %{
 3357   constraint(ALLOC_IN_RC(int_rbx_reg));
 3358   match(RegI);
 3359   match(rRegI);
 3360 
 3361   format %{ "RBX" %}
 3362   interface(REG_INTER);
 3363 %}
 3364 
 3365 operand rcx_RegI()
 3366 %{
 3367   constraint(ALLOC_IN_RC(int_rcx_reg));
 3368   match(RegI);
 3369   match(rRegI);
 3370 
 3371   format %{ "RCX" %}
 3372   interface(REG_INTER);
 3373 %}
 3374 
 3375 operand rdx_RegI()
 3376 %{
 3377   constraint(ALLOC_IN_RC(int_rdx_reg));
 3378   match(RegI);
 3379   match(rRegI);
 3380 
 3381   format %{ "RDX" %}
 3382   interface(REG_INTER);
 3383 %}
 3384 
 3385 operand rdi_RegI()
 3386 %{
 3387   constraint(ALLOC_IN_RC(int_rdi_reg));
 3388   match(RegI);
 3389   match(rRegI);
 3390 
 3391   format %{ "RDI" %}
 3392   interface(REG_INTER);
 3393 %}
 3394 
 3395 operand no_rax_rdx_RegI()
 3396 %{
 3397   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3398   match(RegI);
 3399   match(rbx_RegI);
 3400   match(rcx_RegI);
 3401   match(rdi_RegI);
 3402 
 3403   format %{ %}
 3404   interface(REG_INTER);
 3405 %}
 3406 
 3407 operand no_rbp_r13_RegI()
 3408 %{
 3409   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3410   match(RegI);
 3411   match(rRegI);
 3412   match(rax_RegI);
 3413   match(rbx_RegI);
 3414   match(rcx_RegI);
 3415   match(rdx_RegI);
 3416   match(rdi_RegI);
 3417 
 3418   format %{ %}
 3419   interface(REG_INTER);
 3420 %}
 3421 
 3422 // Pointer Register
 3423 operand any_RegP()
 3424 %{
 3425   constraint(ALLOC_IN_RC(any_reg));
 3426   match(RegP);
 3427   match(rax_RegP);
 3428   match(rbx_RegP);
 3429   match(rdi_RegP);
 3430   match(rsi_RegP);
 3431   match(rbp_RegP);
 3432   match(r15_RegP);
 3433   match(rRegP);
 3434 
 3435   format %{ %}
 3436   interface(REG_INTER);
 3437 %}
 3438 
 3439 operand rRegP()
 3440 %{
 3441   constraint(ALLOC_IN_RC(ptr_reg));
 3442   match(RegP);
 3443   match(rax_RegP);
 3444   match(rbx_RegP);
 3445   match(rdi_RegP);
 3446   match(rsi_RegP);
 3447   match(rbp_RegP);  // See Q&A below about
 3448   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3449 
 3450   format %{ %}
 3451   interface(REG_INTER);
 3452 %}
 3453 
 3454 operand rRegN() %{
 3455   constraint(ALLOC_IN_RC(int_reg));
 3456   match(RegN);
 3457 
 3458   format %{ %}
 3459   interface(REG_INTER);
 3460 %}
 3461 
 3462 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3463 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3464 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3465 // The output of an instruction is controlled by the allocator, which respects
 3466 // register class masks, not match rules.  Unless an instruction mentions
 3467 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3468 // by the allocator as an input.
 3469 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3470 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3471 // result, RBP is not included in the output of the instruction either.
 3472 
 3473 operand no_rax_RegP()
 3474 %{
 3475   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3476   match(RegP);
 3477   match(rbx_RegP);
 3478   match(rsi_RegP);
 3479   match(rdi_RegP);
 3480 
 3481   format %{ %}
 3482   interface(REG_INTER);
 3483 %}
 3484 
 3485 // This operand is not allowed to use RBP even if
 3486 // RBP is not used to hold the frame pointer.
 3487 operand no_rbp_RegP()
 3488 %{
 3489   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3490   match(RegP);
 3491   match(rbx_RegP);
 3492   match(rsi_RegP);
 3493   match(rdi_RegP);
 3494 
 3495   format %{ %}
 3496   interface(REG_INTER);
 3497 %}
 3498 
 3499 operand no_rax_rbx_RegP()
 3500 %{
 3501   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3502   match(RegP);
 3503   match(rsi_RegP);
 3504   match(rdi_RegP);
 3505 
 3506   format %{ %}
 3507   interface(REG_INTER);
 3508 %}
 3509 
 3510 // Special Registers
 3511 // Return a pointer value
 3512 operand rax_RegP()
 3513 %{
 3514   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3515   match(RegP);
 3516   match(rRegP);
 3517 
 3518   format %{ %}
 3519   interface(REG_INTER);
 3520 %}
 3521 
 3522 // Special Registers
 3523 // Return a compressed pointer value
 3524 operand rax_RegN()
 3525 %{
 3526   constraint(ALLOC_IN_RC(int_rax_reg));
 3527   match(RegN);
 3528   match(rRegN);
 3529 
 3530   format %{ %}
 3531   interface(REG_INTER);
 3532 %}
 3533 
 3534 // Used in AtomicAdd
 3535 operand rbx_RegP()
 3536 %{
 3537   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3538   match(RegP);
 3539   match(rRegP);
 3540 
 3541   format %{ %}
 3542   interface(REG_INTER);
 3543 %}
 3544 
 3545 operand rsi_RegP()
 3546 %{
 3547   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3548   match(RegP);
 3549   match(rRegP);
 3550 
 3551   format %{ %}
 3552   interface(REG_INTER);
 3553 %}
 3554 
 3555 operand rbp_RegP()
 3556 %{
 3557   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3558   match(RegP);
 3559   match(rRegP);
 3560 
 3561   format %{ %}
 3562   interface(REG_INTER);
 3563 %}
 3564 
 3565 // Used in rep stosq
 3566 operand rdi_RegP()
 3567 %{
 3568   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3569   match(RegP);
 3570   match(rRegP);
 3571 
 3572   format %{ %}
 3573   interface(REG_INTER);
 3574 %}
 3575 
 3576 operand r15_RegP()
 3577 %{
 3578   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3579   match(RegP);
 3580   match(rRegP);
 3581 
 3582   format %{ %}
 3583   interface(REG_INTER);
 3584 %}
 3585 
 3586 operand rRegL()
 3587 %{
 3588   constraint(ALLOC_IN_RC(long_reg));
 3589   match(RegL);
 3590   match(rax_RegL);
 3591   match(rdx_RegL);
 3592 
 3593   format %{ %}
 3594   interface(REG_INTER);
 3595 %}
 3596 
 3597 // Special Registers
 3598 operand no_rax_rdx_RegL()
 3599 %{
 3600   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3601   match(RegL);
 3602   match(rRegL);
 3603 
 3604   format %{ %}
 3605   interface(REG_INTER);
 3606 %}
 3607 
 3608 operand no_rax_RegL()
 3609 %{
 3610   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3611   match(RegL);
 3612   match(rRegL);
 3613   match(rdx_RegL);
 3614 
 3615   format %{ %}
 3616   interface(REG_INTER);
 3617 %}
 3618 
 3619 operand rax_RegL()
 3620 %{
 3621   constraint(ALLOC_IN_RC(long_rax_reg));
 3622   match(RegL);
 3623   match(rRegL);
 3624 
 3625   format %{ "RAX" %}
 3626   interface(REG_INTER);
 3627 %}
 3628 
 3629 operand rcx_RegL()
 3630 %{
 3631   constraint(ALLOC_IN_RC(long_rcx_reg));
 3632   match(RegL);
 3633   match(rRegL);
 3634 
 3635   format %{ %}
 3636   interface(REG_INTER);
 3637 %}
 3638 
 3639 operand rdx_RegL()
 3640 %{
 3641   constraint(ALLOC_IN_RC(long_rdx_reg));
 3642   match(RegL);
 3643   match(rRegL);
 3644 
 3645   format %{ %}
 3646   interface(REG_INTER);
 3647 %}
 3648 
 3649 operand no_rbp_r13_RegL()
 3650 %{
 3651   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3652   match(RegL);
 3653   match(rRegL);
 3654   match(rax_RegL);
 3655   match(rcx_RegL);
 3656   match(rdx_RegL);
 3657 
 3658   format %{ %}
 3659   interface(REG_INTER);
 3660 %}
 3661 
 3662 // Flags register, used as output of compare instructions
 3663 operand rFlagsReg()
 3664 %{
 3665   constraint(ALLOC_IN_RC(int_flags));
 3666   match(RegFlags);
 3667 
 3668   format %{ "RFLAGS" %}
 3669   interface(REG_INTER);
 3670 %}
 3671 
 3672 // Flags register, used as output of FLOATING POINT compare instructions
 3673 operand rFlagsRegU()
 3674 %{
 3675   constraint(ALLOC_IN_RC(int_flags));
 3676   match(RegFlags);
 3677 
 3678   format %{ "RFLAGS_U" %}
 3679   interface(REG_INTER);
 3680 %}
 3681 
 3682 operand rFlagsRegUCF() %{
 3683   constraint(ALLOC_IN_RC(int_flags));
 3684   match(RegFlags);
 3685   predicate(false);
 3686 
 3687   format %{ "RFLAGS_U_CF" %}
 3688   interface(REG_INTER);
 3689 %}
 3690 
 3691 // Float register operands
 3692 operand regF() %{
 3693    constraint(ALLOC_IN_RC(float_reg));
 3694    match(RegF);
 3695 
 3696    format %{ %}
 3697    interface(REG_INTER);
 3698 %}
 3699 
 3700 // Float register operands
 3701 operand legRegF() %{
 3702    constraint(ALLOC_IN_RC(float_reg_legacy));
 3703    match(RegF);
 3704 
 3705    format %{ %}
 3706    interface(REG_INTER);
 3707 %}
 3708 
 3709 // Float register operands
 3710 operand vlRegF() %{
 3711    constraint(ALLOC_IN_RC(float_reg_vl));
 3712    match(RegF);
 3713 
 3714    format %{ %}
 3715    interface(REG_INTER);
 3716 %}
 3717 
 3718 // Double register operands
 3719 operand regD() %{
 3720    constraint(ALLOC_IN_RC(double_reg));
 3721    match(RegD);
 3722 
 3723    format %{ %}
 3724    interface(REG_INTER);
 3725 %}
 3726 
 3727 // Double register operands
 3728 operand legRegD() %{
 3729    constraint(ALLOC_IN_RC(double_reg_legacy));
 3730    match(RegD);
 3731 
 3732    format %{ %}
 3733    interface(REG_INTER);
 3734 %}
 3735 
 3736 // Double register operands
 3737 operand vlRegD() %{
 3738    constraint(ALLOC_IN_RC(double_reg_vl));
 3739    match(RegD);
 3740 
 3741    format %{ %}
 3742    interface(REG_INTER);
 3743 %}
 3744 
 3745 //----------Memory Operands----------------------------------------------------
 3746 // Direct Memory Operand
 3747 // operand direct(immP addr)
 3748 // %{
 3749 //   match(addr);
 3750 
 3751 //   format %{ "[$addr]" %}
 3752 //   interface(MEMORY_INTER) %{
 3753 //     base(0xFFFFFFFF);
 3754 //     index(0x4);
 3755 //     scale(0x0);
 3756 //     disp($addr);
 3757 //   %}
 3758 // %}
 3759 
 3760 // Indirect Memory Operand
 3761 operand indirect(any_RegP reg)
 3762 %{
 3763   constraint(ALLOC_IN_RC(ptr_reg));
 3764   match(reg);
 3765 
 3766   format %{ "[$reg]" %}
 3767   interface(MEMORY_INTER) %{
 3768     base($reg);
 3769     index(0x4);
 3770     scale(0x0);
 3771     disp(0x0);
 3772   %}
 3773 %}
 3774 
 3775 // Indirect Memory Plus Short Offset Operand
 3776 operand indOffset8(any_RegP reg, immL8 off)
 3777 %{
 3778   constraint(ALLOC_IN_RC(ptr_reg));
 3779   match(AddP reg off);
 3780 
 3781   format %{ "[$reg + $off (8-bit)]" %}
 3782   interface(MEMORY_INTER) %{
 3783     base($reg);
 3784     index(0x4);
 3785     scale(0x0);
 3786     disp($off);
 3787   %}
 3788 %}
 3789 
 3790 // Indirect Memory Plus Long Offset Operand
 3791 operand indOffset32(any_RegP reg, immL32 off)
 3792 %{
 3793   constraint(ALLOC_IN_RC(ptr_reg));
 3794   match(AddP reg off);
 3795 
 3796   format %{ "[$reg + $off (32-bit)]" %}
 3797   interface(MEMORY_INTER) %{
 3798     base($reg);
 3799     index(0x4);
 3800     scale(0x0);
 3801     disp($off);
 3802   %}
 3803 %}
 3804 
 3805 // Indirect Memory Plus Index Register Plus Offset Operand
 3806 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3807 %{
 3808   constraint(ALLOC_IN_RC(ptr_reg));
 3809   match(AddP (AddP reg lreg) off);
 3810 
 3811   op_cost(10);
 3812   format %{"[$reg + $off + $lreg]" %}
 3813   interface(MEMORY_INTER) %{
 3814     base($reg);
 3815     index($lreg);
 3816     scale(0x0);
 3817     disp($off);
 3818   %}
 3819 %}
 3820 
 3821 // Indirect Memory Plus Index Register Plus Offset Operand
 3822 operand indIndex(any_RegP reg, rRegL lreg)
 3823 %{
 3824   constraint(ALLOC_IN_RC(ptr_reg));
 3825   match(AddP reg lreg);
 3826 
 3827   op_cost(10);
 3828   format %{"[$reg + $lreg]" %}
 3829   interface(MEMORY_INTER) %{
 3830     base($reg);
 3831     index($lreg);
 3832     scale(0x0);
 3833     disp(0x0);
 3834   %}
 3835 %}
 3836 
 3837 // Indirect Memory Times Scale Plus Index Register
 3838 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3839 %{
 3840   constraint(ALLOC_IN_RC(ptr_reg));
 3841   match(AddP reg (LShiftL lreg scale));
 3842 
 3843   op_cost(10);
 3844   format %{"[$reg + $lreg << $scale]" %}
 3845   interface(MEMORY_INTER) %{
 3846     base($reg);
 3847     index($lreg);
 3848     scale($scale);
 3849     disp(0x0);
 3850   %}
 3851 %}
 3852 
 3853 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3854 %{
 3855   constraint(ALLOC_IN_RC(ptr_reg));
 3856   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3857   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3858 
 3859   op_cost(10);
 3860   format %{"[$reg + pos $idx << $scale]" %}
 3861   interface(MEMORY_INTER) %{
 3862     base($reg);
 3863     index($idx);
 3864     scale($scale);
 3865     disp(0x0);
 3866   %}
 3867 %}
 3868 
 3869 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3870 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3871 %{
 3872   constraint(ALLOC_IN_RC(ptr_reg));
 3873   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3874 
 3875   op_cost(10);
 3876   format %{"[$reg + $off + $lreg << $scale]" %}
 3877   interface(MEMORY_INTER) %{
 3878     base($reg);
 3879     index($lreg);
 3880     scale($scale);
 3881     disp($off);
 3882   %}
 3883 %}
 3884 
 3885 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3886 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3887 %{
 3888   constraint(ALLOC_IN_RC(ptr_reg));
 3889   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3890   match(AddP (AddP reg (ConvI2L idx)) off);
 3891 
 3892   op_cost(10);
 3893   format %{"[$reg + $off + $idx]" %}
 3894   interface(MEMORY_INTER) %{
 3895     base($reg);
 3896     index($idx);
 3897     scale(0x0);
 3898     disp($off);
 3899   %}
 3900 %}
 3901 
 3902 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3903 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3904 %{
 3905   constraint(ALLOC_IN_RC(ptr_reg));
 3906   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3907   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3908 
 3909   op_cost(10);
 3910   format %{"[$reg + $off + $idx << $scale]" %}
 3911   interface(MEMORY_INTER) %{
 3912     base($reg);
 3913     index($idx);
 3914     scale($scale);
 3915     disp($off);
 3916   %}
 3917 %}
 3918 
 3919 // Indirect Narrow Oop Plus Offset Operand
 3920 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3921 // we can't free r12 even with CompressedOops::base() == NULL.
 3922 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3923   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3924   constraint(ALLOC_IN_RC(ptr_reg));
 3925   match(AddP (DecodeN reg) off);
 3926 
 3927   op_cost(10);
 3928   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3929   interface(MEMORY_INTER) %{
 3930     base(0xc); // R12
 3931     index($reg);
 3932     scale(0x3);
 3933     disp($off);
 3934   %}
 3935 %}
 3936 
 3937 // Indirect Memory Operand
 3938 operand indirectNarrow(rRegN reg)
 3939 %{
 3940   predicate(CompressedOops::shift() == 0);
 3941   constraint(ALLOC_IN_RC(ptr_reg));
 3942   match(DecodeN reg);
 3943 
 3944   format %{ "[$reg]" %}
 3945   interface(MEMORY_INTER) %{
 3946     base($reg);
 3947     index(0x4);
 3948     scale(0x0);
 3949     disp(0x0);
 3950   %}
 3951 %}
 3952 
 3953 // Indirect Memory Plus Short Offset Operand
 3954 operand indOffset8Narrow(rRegN reg, immL8 off)
 3955 %{
 3956   predicate(CompressedOops::shift() == 0);
 3957   constraint(ALLOC_IN_RC(ptr_reg));
 3958   match(AddP (DecodeN reg) off);
 3959 
 3960   format %{ "[$reg + $off (8-bit)]" %}
 3961   interface(MEMORY_INTER) %{
 3962     base($reg);
 3963     index(0x4);
 3964     scale(0x0);
 3965     disp($off);
 3966   %}
 3967 %}
 3968 
 3969 // Indirect Memory Plus Long Offset Operand
 3970 operand indOffset32Narrow(rRegN reg, immL32 off)
 3971 %{
 3972   predicate(CompressedOops::shift() == 0);
 3973   constraint(ALLOC_IN_RC(ptr_reg));
 3974   match(AddP (DecodeN reg) off);
 3975 
 3976   format %{ "[$reg + $off (32-bit)]" %}
 3977   interface(MEMORY_INTER) %{
 3978     base($reg);
 3979     index(0x4);
 3980     scale(0x0);
 3981     disp($off);
 3982   %}
 3983 %}
 3984 
 3985 // Indirect Memory Plus Index Register Plus Offset Operand
 3986 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3987 %{
 3988   predicate(CompressedOops::shift() == 0);
 3989   constraint(ALLOC_IN_RC(ptr_reg));
 3990   match(AddP (AddP (DecodeN reg) lreg) off);
 3991 
 3992   op_cost(10);
 3993   format %{"[$reg + $off + $lreg]" %}
 3994   interface(MEMORY_INTER) %{
 3995     base($reg);
 3996     index($lreg);
 3997     scale(0x0);
 3998     disp($off);
 3999   %}
 4000 %}
 4001 
 4002 // Indirect Memory Plus Index Register Plus Offset Operand
 4003 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4004 %{
 4005   predicate(CompressedOops::shift() == 0);
 4006   constraint(ALLOC_IN_RC(ptr_reg));
 4007   match(AddP (DecodeN reg) lreg);
 4008 
 4009   op_cost(10);
 4010   format %{"[$reg + $lreg]" %}
 4011   interface(MEMORY_INTER) %{
 4012     base($reg);
 4013     index($lreg);
 4014     scale(0x0);
 4015     disp(0x0);
 4016   %}
 4017 %}
 4018 
 4019 // Indirect Memory Times Scale Plus Index Register
 4020 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4021 %{
 4022   predicate(CompressedOops::shift() == 0);
 4023   constraint(ALLOC_IN_RC(ptr_reg));
 4024   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4025 
 4026   op_cost(10);
 4027   format %{"[$reg + $lreg << $scale]" %}
 4028   interface(MEMORY_INTER) %{
 4029     base($reg);
 4030     index($lreg);
 4031     scale($scale);
 4032     disp(0x0);
 4033   %}
 4034 %}
 4035 
 4036 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4037 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4038 %{
 4039   predicate(CompressedOops::shift() == 0);
 4040   constraint(ALLOC_IN_RC(ptr_reg));
 4041   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4042 
 4043   op_cost(10);
 4044   format %{"[$reg + $off + $lreg << $scale]" %}
 4045   interface(MEMORY_INTER) %{
 4046     base($reg);
 4047     index($lreg);
 4048     scale($scale);
 4049     disp($off);
 4050   %}
 4051 %}
 4052 
 4053 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4054 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4055 %{
 4056   constraint(ALLOC_IN_RC(ptr_reg));
 4057   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4058   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4059 
 4060   op_cost(10);
 4061   format %{"[$reg + $off + $idx]" %}
 4062   interface(MEMORY_INTER) %{
 4063     base($reg);
 4064     index($idx);
 4065     scale(0x0);
 4066     disp($off);
 4067   %}
 4068 %}
 4069 
 4070 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4071 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4072 %{
 4073   constraint(ALLOC_IN_RC(ptr_reg));
 4074   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4075   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4076 
 4077   op_cost(10);
 4078   format %{"[$reg + $off + $idx << $scale]" %}
 4079   interface(MEMORY_INTER) %{
 4080     base($reg);
 4081     index($idx);
 4082     scale($scale);
 4083     disp($off);
 4084   %}
 4085 %}
 4086 
 4087 //----------Special Memory Operands--------------------------------------------
 4088 // Stack Slot Operand - This operand is used for loading and storing temporary
 4089 //                      values on the stack where a match requires a value to
 4090 //                      flow through memory.
 4091 operand stackSlotP(sRegP reg)
 4092 %{
 4093   constraint(ALLOC_IN_RC(stack_slots));
 4094   // No match rule because this operand is only generated in matching
 4095 
 4096   format %{ "[$reg]" %}
 4097   interface(MEMORY_INTER) %{
 4098     base(0x4);   // RSP
 4099     index(0x4);  // No Index
 4100     scale(0x0);  // No Scale
 4101     disp($reg);  // Stack Offset
 4102   %}
 4103 %}
 4104 
 4105 operand stackSlotI(sRegI reg)
 4106 %{
 4107   constraint(ALLOC_IN_RC(stack_slots));
 4108   // No match rule because this operand is only generated in matching
 4109 
 4110   format %{ "[$reg]" %}
 4111   interface(MEMORY_INTER) %{
 4112     base(0x4);   // RSP
 4113     index(0x4);  // No Index
 4114     scale(0x0);  // No Scale
 4115     disp($reg);  // Stack Offset
 4116   %}
 4117 %}
 4118 
 4119 operand stackSlotF(sRegF reg)
 4120 %{
 4121   constraint(ALLOC_IN_RC(stack_slots));
 4122   // No match rule because this operand is only generated in matching
 4123 
 4124   format %{ "[$reg]" %}
 4125   interface(MEMORY_INTER) %{
 4126     base(0x4);   // RSP
 4127     index(0x4);  // No Index
 4128     scale(0x0);  // No Scale
 4129     disp($reg);  // Stack Offset
 4130   %}
 4131 %}
 4132 
 4133 operand stackSlotD(sRegD reg)
 4134 %{
 4135   constraint(ALLOC_IN_RC(stack_slots));
 4136   // No match rule because this operand is only generated in matching
 4137 
 4138   format %{ "[$reg]" %}
 4139   interface(MEMORY_INTER) %{
 4140     base(0x4);   // RSP
 4141     index(0x4);  // No Index
 4142     scale(0x0);  // No Scale
 4143     disp($reg);  // Stack Offset
 4144   %}
 4145 %}
 4146 operand stackSlotL(sRegL reg)
 4147 %{
 4148   constraint(ALLOC_IN_RC(stack_slots));
 4149   // No match rule because this operand is only generated in matching
 4150 
 4151   format %{ "[$reg]" %}
 4152   interface(MEMORY_INTER) %{
 4153     base(0x4);   // RSP
 4154     index(0x4);  // No Index
 4155     scale(0x0);  // No Scale
 4156     disp($reg);  // Stack Offset
 4157   %}
 4158 %}
 4159 
 4160 //----------Conditional Branch Operands----------------------------------------
 4161 // Comparison Op  - This is the operation of the comparison, and is limited to
 4162 //                  the following set of codes:
 4163 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4164 //
 4165 // Other attributes of the comparison, such as unsignedness, are specified
 4166 // by the comparison instruction that sets a condition code flags register.
 4167 // That result is represented by a flags operand whose subtype is appropriate
 4168 // to the unsignedness (etc.) of the comparison.
 4169 //
 4170 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4171 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4172 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4173 
 4174 // Comparison Code
 4175 operand cmpOp()
 4176 %{
 4177   match(Bool);
 4178 
 4179   format %{ "" %}
 4180   interface(COND_INTER) %{
 4181     equal(0x4, "e");
 4182     not_equal(0x5, "ne");
 4183     less(0xC, "l");
 4184     greater_equal(0xD, "ge");
 4185     less_equal(0xE, "le");
 4186     greater(0xF, "g");
 4187     overflow(0x0, "o");
 4188     no_overflow(0x1, "no");
 4189   %}
 4190 %}
 4191 
 4192 // Comparison Code, unsigned compare.  Used by FP also, with
 4193 // C2 (unordered) turned into GT or LT already.  The other bits
 4194 // C0 and C3 are turned into Carry & Zero flags.
 4195 operand cmpOpU()
 4196 %{
 4197   match(Bool);
 4198 
 4199   format %{ "" %}
 4200   interface(COND_INTER) %{
 4201     equal(0x4, "e");
 4202     not_equal(0x5, "ne");
 4203     less(0x2, "b");
 4204     greater_equal(0x3, "nb");
 4205     less_equal(0x6, "be");
 4206     greater(0x7, "nbe");
 4207     overflow(0x0, "o");
 4208     no_overflow(0x1, "no");
 4209   %}
 4210 %}
 4211 
 4212 
 4213 // Floating comparisons that don't require any fixup for the unordered case
 4214 operand cmpOpUCF() %{
 4215   match(Bool);
 4216   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4217             n->as_Bool()->_test._test == BoolTest::ge ||
 4218             n->as_Bool()->_test._test == BoolTest::le ||
 4219             n->as_Bool()->_test._test == BoolTest::gt);
 4220   format %{ "" %}
 4221   interface(COND_INTER) %{
 4222     equal(0x4, "e");
 4223     not_equal(0x5, "ne");
 4224     less(0x2, "b");
 4225     greater_equal(0x3, "nb");
 4226     less_equal(0x6, "be");
 4227     greater(0x7, "nbe");
 4228     overflow(0x0, "o");
 4229     no_overflow(0x1, "no");
 4230   %}
 4231 %}
 4232 
 4233 
 4234 // Floating comparisons that can be fixed up with extra conditional jumps
 4235 operand cmpOpUCF2() %{
 4236   match(Bool);
 4237   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
 4238             n->as_Bool()->_test._test == BoolTest::eq);
 4239   format %{ "" %}
 4240   interface(COND_INTER) %{
 4241     equal(0x4, "e");
 4242     not_equal(0x5, "ne");
 4243     less(0x2, "b");
 4244     greater_equal(0x3, "nb");
 4245     less_equal(0x6, "be");
 4246     greater(0x7, "nbe");
 4247     overflow(0x0, "o");
 4248     no_overflow(0x1, "no");
 4249   %}
 4250 %}
 4251 
 4252 //----------OPERAND CLASSES----------------------------------------------------
 4253 // Operand Classes are groups of operands that are used as to simplify
 4254 // instruction definitions by not requiring the AD writer to specify separate
 4255 // instructions for every form of operand when the instruction accepts
 4256 // multiple operand types with the same basic encoding and format.  The classic
 4257 // case of this is memory operands.
 4258 
 4259 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4260                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4261                indCompressedOopOffset,
 4262                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4263                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4264                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4265 
 4266 //----------PIPELINE-----------------------------------------------------------
 4267 // Rules which define the behavior of the target architectures pipeline.
 4268 pipeline %{
 4269 
 4270 //----------ATTRIBUTES---------------------------------------------------------
 4271 attributes %{
 4272   variable_size_instructions;        // Fixed size instructions
 4273   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4274   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4275   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4276   instruction_fetch_units = 1;       // of 16 bytes
 4277 
 4278   // List of nop instructions
 4279   nops( MachNop );
 4280 %}
 4281 
 4282 //----------RESOURCES----------------------------------------------------------
 4283 // Resources are the functional units available to the machine
 4284 
 4285 // Generic P2/P3 pipeline
 4286 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4287 // 3 instructions decoded per cycle.
 4288 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4289 // 3 ALU op, only ALU0 handles mul instructions.
 4290 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4291            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4292            BR, FPU,
 4293            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4294 
 4295 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4296 // Pipeline Description specifies the stages in the machine's pipeline
 4297 
 4298 // Generic P2/P3 pipeline
 4299 pipe_desc(S0, S1, S2, S3, S4, S5);
 4300 
 4301 //----------PIPELINE CLASSES---------------------------------------------------
 4302 // Pipeline Classes describe the stages in which input and output are
 4303 // referenced by the hardware pipeline.
 4304 
 4305 // Naming convention: ialu or fpu
 4306 // Then: _reg
 4307 // Then: _reg if there is a 2nd register
 4308 // Then: _long if it's a pair of instructions implementing a long
 4309 // Then: _fat if it requires the big decoder
 4310 //   Or: _mem if it requires the big decoder and a memory unit.
 4311 
 4312 // Integer ALU reg operation
 4313 pipe_class ialu_reg(rRegI dst)
 4314 %{
 4315     single_instruction;
 4316     dst    : S4(write);
 4317     dst    : S3(read);
 4318     DECODE : S0;        // any decoder
 4319     ALU    : S3;        // any alu
 4320 %}
 4321 
 4322 // Long ALU reg operation
 4323 pipe_class ialu_reg_long(rRegL dst)
 4324 %{
 4325     instruction_count(2);
 4326     dst    : S4(write);
 4327     dst    : S3(read);
 4328     DECODE : S0(2);     // any 2 decoders
 4329     ALU    : S3(2);     // both alus
 4330 %}
 4331 
 4332 // Integer ALU reg operation using big decoder
 4333 pipe_class ialu_reg_fat(rRegI dst)
 4334 %{
 4335     single_instruction;
 4336     dst    : S4(write);
 4337     dst    : S3(read);
 4338     D0     : S0;        // big decoder only
 4339     ALU    : S3;        // any alu
 4340 %}
 4341 
 4342 // Integer ALU reg-reg operation
 4343 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4344 %{
 4345     single_instruction;
 4346     dst    : S4(write);
 4347     src    : S3(read);
 4348     DECODE : S0;        // any decoder
 4349     ALU    : S3;        // any alu
 4350 %}
 4351 
 4352 // Integer ALU reg-reg operation
 4353 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4354 %{
 4355     single_instruction;
 4356     dst    : S4(write);
 4357     src    : S3(read);
 4358     D0     : S0;        // big decoder only
 4359     ALU    : S3;        // any alu
 4360 %}
 4361 
 4362 // Integer ALU reg-mem operation
 4363 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4364 %{
 4365     single_instruction;
 4366     dst    : S5(write);
 4367     mem    : S3(read);
 4368     D0     : S0;        // big decoder only
 4369     ALU    : S4;        // any alu
 4370     MEM    : S3;        // any mem
 4371 %}
 4372 
 4373 // Integer mem operation (prefetch)
 4374 pipe_class ialu_mem(memory mem)
 4375 %{
 4376     single_instruction;
 4377     mem    : S3(read);
 4378     D0     : S0;        // big decoder only
 4379     MEM    : S3;        // any mem
 4380 %}
 4381 
 4382 // Integer Store to Memory
 4383 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4384 %{
 4385     single_instruction;
 4386     mem    : S3(read);
 4387     src    : S5(read);
 4388     D0     : S0;        // big decoder only
 4389     ALU    : S4;        // any alu
 4390     MEM    : S3;
 4391 %}
 4392 
 4393 // // Long Store to Memory
 4394 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4395 // %{
 4396 //     instruction_count(2);
 4397 //     mem    : S3(read);
 4398 //     src    : S5(read);
 4399 //     D0     : S0(2);          // big decoder only; twice
 4400 //     ALU    : S4(2);     // any 2 alus
 4401 //     MEM    : S3(2);  // Both mems
 4402 // %}
 4403 
 4404 // Integer Store to Memory
 4405 pipe_class ialu_mem_imm(memory mem)
 4406 %{
 4407     single_instruction;
 4408     mem    : S3(read);
 4409     D0     : S0;        // big decoder only
 4410     ALU    : S4;        // any alu
 4411     MEM    : S3;
 4412 %}
 4413 
 4414 // Integer ALU0 reg-reg operation
 4415 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4416 %{
 4417     single_instruction;
 4418     dst    : S4(write);
 4419     src    : S3(read);
 4420     D0     : S0;        // Big decoder only
 4421     ALU0   : S3;        // only alu0
 4422 %}
 4423 
 4424 // Integer ALU0 reg-mem operation
 4425 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4426 %{
 4427     single_instruction;
 4428     dst    : S5(write);
 4429     mem    : S3(read);
 4430     D0     : S0;        // big decoder only
 4431     ALU0   : S4;        // ALU0 only
 4432     MEM    : S3;        // any mem
 4433 %}
 4434 
 4435 // Integer ALU reg-reg operation
 4436 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4437 %{
 4438     single_instruction;
 4439     cr     : S4(write);
 4440     src1   : S3(read);
 4441     src2   : S3(read);
 4442     DECODE : S0;        // any decoder
 4443     ALU    : S3;        // any alu
 4444 %}
 4445 
 4446 // Integer ALU reg-imm operation
 4447 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4448 %{
 4449     single_instruction;
 4450     cr     : S4(write);
 4451     src1   : S3(read);
 4452     DECODE : S0;        // any decoder
 4453     ALU    : S3;        // any alu
 4454 %}
 4455 
 4456 // Integer ALU reg-mem operation
 4457 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4458 %{
 4459     single_instruction;
 4460     cr     : S4(write);
 4461     src1   : S3(read);
 4462     src2   : S3(read);
 4463     D0     : S0;        // big decoder only
 4464     ALU    : S4;        // any alu
 4465     MEM    : S3;
 4466 %}
 4467 
 4468 // Conditional move reg-reg
 4469 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4470 %{
 4471     instruction_count(4);
 4472     y      : S4(read);
 4473     q      : S3(read);
 4474     p      : S3(read);
 4475     DECODE : S0(4);     // any decoder
 4476 %}
 4477 
 4478 // Conditional move reg-reg
 4479 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4480 %{
 4481     single_instruction;
 4482     dst    : S4(write);
 4483     src    : S3(read);
 4484     cr     : S3(read);
 4485     DECODE : S0;        // any decoder
 4486 %}
 4487 
 4488 // Conditional move reg-mem
 4489 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4490 %{
 4491     single_instruction;
 4492     dst    : S4(write);
 4493     src    : S3(read);
 4494     cr     : S3(read);
 4495     DECODE : S0;        // any decoder
 4496     MEM    : S3;
 4497 %}
 4498 
 4499 // Conditional move reg-reg long
 4500 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4501 %{
 4502     single_instruction;
 4503     dst    : S4(write);
 4504     src    : S3(read);
 4505     cr     : S3(read);
 4506     DECODE : S0(2);     // any 2 decoders
 4507 %}
 4508 
 4509 // XXX
 4510 // // Conditional move double reg-reg
 4511 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4512 // %{
 4513 //     single_instruction;
 4514 //     dst    : S4(write);
 4515 //     src    : S3(read);
 4516 //     cr     : S3(read);
 4517 //     DECODE : S0;     // any decoder
 4518 // %}
 4519 
 4520 // Float reg-reg operation
 4521 pipe_class fpu_reg(regD dst)
 4522 %{
 4523     instruction_count(2);
 4524     dst    : S3(read);
 4525     DECODE : S0(2);     // any 2 decoders
 4526     FPU    : S3;
 4527 %}
 4528 
 4529 // Float reg-reg operation
 4530 pipe_class fpu_reg_reg(regD dst, regD src)
 4531 %{
 4532     instruction_count(2);
 4533     dst    : S4(write);
 4534     src    : S3(read);
 4535     DECODE : S0(2);     // any 2 decoders
 4536     FPU    : S3;
 4537 %}
 4538 
 4539 // Float reg-reg operation
 4540 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4541 %{
 4542     instruction_count(3);
 4543     dst    : S4(write);
 4544     src1   : S3(read);
 4545     src2   : S3(read);
 4546     DECODE : S0(3);     // any 3 decoders
 4547     FPU    : S3(2);
 4548 %}
 4549 
 4550 // Float reg-reg operation
 4551 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4552 %{
 4553     instruction_count(4);
 4554     dst    : S4(write);
 4555     src1   : S3(read);
 4556     src2   : S3(read);
 4557     src3   : S3(read);
 4558     DECODE : S0(4);     // any 3 decoders
 4559     FPU    : S3(2);
 4560 %}
 4561 
 4562 // Float reg-reg operation
 4563 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4564 %{
 4565     instruction_count(4);
 4566     dst    : S4(write);
 4567     src1   : S3(read);
 4568     src2   : S3(read);
 4569     src3   : S3(read);
 4570     DECODE : S1(3);     // any 3 decoders
 4571     D0     : S0;        // Big decoder only
 4572     FPU    : S3(2);
 4573     MEM    : S3;
 4574 %}
 4575 
 4576 // Float reg-mem operation
 4577 pipe_class fpu_reg_mem(regD dst, memory mem)
 4578 %{
 4579     instruction_count(2);
 4580     dst    : S5(write);
 4581     mem    : S3(read);
 4582     D0     : S0;        // big decoder only
 4583     DECODE : S1;        // any decoder for FPU POP
 4584     FPU    : S4;
 4585     MEM    : S3;        // any mem
 4586 %}
 4587 
 4588 // Float reg-mem operation
 4589 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4590 %{
 4591     instruction_count(3);
 4592     dst    : S5(write);
 4593     src1   : S3(read);
 4594     mem    : S3(read);
 4595     D0     : S0;        // big decoder only
 4596     DECODE : S1(2);     // any decoder for FPU POP
 4597     FPU    : S4;
 4598     MEM    : S3;        // any mem
 4599 %}
 4600 
 4601 // Float mem-reg operation
 4602 pipe_class fpu_mem_reg(memory mem, regD src)
 4603 %{
 4604     instruction_count(2);
 4605     src    : S5(read);
 4606     mem    : S3(read);
 4607     DECODE : S0;        // any decoder for FPU PUSH
 4608     D0     : S1;        // big decoder only
 4609     FPU    : S4;
 4610     MEM    : S3;        // any mem
 4611 %}
 4612 
 4613 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4614 %{
 4615     instruction_count(3);
 4616     src1   : S3(read);
 4617     src2   : S3(read);
 4618     mem    : S3(read);
 4619     DECODE : S0(2);     // any decoder for FPU PUSH
 4620     D0     : S1;        // big decoder only
 4621     FPU    : S4;
 4622     MEM    : S3;        // any mem
 4623 %}
 4624 
 4625 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4626 %{
 4627     instruction_count(3);
 4628     src1   : S3(read);
 4629     src2   : S3(read);
 4630     mem    : S4(read);
 4631     DECODE : S0;        // any decoder for FPU PUSH
 4632     D0     : S0(2);     // big decoder only
 4633     FPU    : S4;
 4634     MEM    : S3(2);     // any mem
 4635 %}
 4636 
 4637 pipe_class fpu_mem_mem(memory dst, memory src1)
 4638 %{
 4639     instruction_count(2);
 4640     src1   : S3(read);
 4641     dst    : S4(read);
 4642     D0     : S0(2);     // big decoder only
 4643     MEM    : S3(2);     // any mem
 4644 %}
 4645 
 4646 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4647 %{
 4648     instruction_count(3);
 4649     src1   : S3(read);
 4650     src2   : S3(read);
 4651     dst    : S4(read);
 4652     D0     : S0(3);     // big decoder only
 4653     FPU    : S4;
 4654     MEM    : S3(3);     // any mem
 4655 %}
 4656 
 4657 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4658 %{
 4659     instruction_count(3);
 4660     src1   : S4(read);
 4661     mem    : S4(read);
 4662     DECODE : S0;        // any decoder for FPU PUSH
 4663     D0     : S0(2);     // big decoder only
 4664     FPU    : S4;
 4665     MEM    : S3(2);     // any mem
 4666 %}
 4667 
 4668 // Float load constant
 4669 pipe_class fpu_reg_con(regD dst)
 4670 %{
 4671     instruction_count(2);
 4672     dst    : S5(write);
 4673     D0     : S0;        // big decoder only for the load
 4674     DECODE : S1;        // any decoder for FPU POP
 4675     FPU    : S4;
 4676     MEM    : S3;        // any mem
 4677 %}
 4678 
 4679 // Float load constant
 4680 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4681 %{
 4682     instruction_count(3);
 4683     dst    : S5(write);
 4684     src    : S3(read);
 4685     D0     : S0;        // big decoder only for the load
 4686     DECODE : S1(2);     // any decoder for FPU POP
 4687     FPU    : S4;
 4688     MEM    : S3;        // any mem
 4689 %}
 4690 
 4691 // UnConditional branch
 4692 pipe_class pipe_jmp(label labl)
 4693 %{
 4694     single_instruction;
 4695     BR   : S3;
 4696 %}
 4697 
 4698 // Conditional branch
 4699 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4700 %{
 4701     single_instruction;
 4702     cr    : S1(read);
 4703     BR    : S3;
 4704 %}
 4705 
 4706 // Allocation idiom
 4707 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4708 %{
 4709     instruction_count(1); force_serialization;
 4710     fixed_latency(6);
 4711     heap_ptr : S3(read);
 4712     DECODE   : S0(3);
 4713     D0       : S2;
 4714     MEM      : S3;
 4715     ALU      : S3(2);
 4716     dst      : S5(write);
 4717     BR       : S5;
 4718 %}
 4719 
 4720 // Generic big/slow expanded idiom
 4721 pipe_class pipe_slow()
 4722 %{
 4723     instruction_count(10); multiple_bundles; force_serialization;
 4724     fixed_latency(100);
 4725     D0  : S0(2);
 4726     MEM : S3(2);
 4727 %}
 4728 
 4729 // The real do-nothing guy
 4730 pipe_class empty()
 4731 %{
 4732     instruction_count(0);
 4733 %}
 4734 
 4735 // Define the class for the Nop node
 4736 define
 4737 %{
 4738    MachNop = empty;
 4739 %}
 4740 
 4741 %}
 4742 
 4743 //----------INSTRUCTIONS-------------------------------------------------------
 4744 //
 4745 // match      -- States which machine-independent subtree may be replaced
 4746 //               by this instruction.
 4747 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4748 //               selection to identify a minimum cost tree of machine
 4749 //               instructions that matches a tree of machine-independent
 4750 //               instructions.
 4751 // format     -- A string providing the disassembly for this instruction.
 4752 //               The value of an instruction's operand may be inserted
 4753 //               by referring to it with a '$' prefix.
 4754 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4755 //               to within an encode class as $primary, $secondary, and $tertiary
 4756 //               rrspectively.  The primary opcode is commonly used to
 4757 //               indicate the type of machine instruction, while secondary
 4758 //               and tertiary are often used for prefix options or addressing
 4759 //               modes.
 4760 // ins_encode -- A list of encode classes with parameters. The encode class
 4761 //               name must have been defined in an 'enc_class' specification
 4762 //               in the encode section of the architecture description.
 4763 
 4764 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4765 // Load Float
 4766 instruct MoveF2VL(vlRegF dst, regF src) %{
 4767   match(Set dst src);
 4768   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4769   ins_encode %{
 4770     ShouldNotReachHere();
 4771   %}
 4772   ins_pipe( fpu_reg_reg );
 4773 %}
 4774 
 4775 // Load Float
 4776 instruct MoveF2LEG(legRegF dst, regF src) %{
 4777   match(Set dst src);
 4778   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4779   ins_encode %{
 4780     ShouldNotReachHere();
 4781   %}
 4782   ins_pipe( fpu_reg_reg );
 4783 %}
 4784 
 4785 // Load Float
 4786 instruct MoveVL2F(regF dst, vlRegF src) %{
 4787   match(Set dst src);
 4788   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4789   ins_encode %{
 4790     ShouldNotReachHere();
 4791   %}
 4792   ins_pipe( fpu_reg_reg );
 4793 %}
 4794 
 4795 // Load Float
 4796 instruct MoveLEG2F(regF dst, legRegF src) %{
 4797   match(Set dst src);
 4798   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4799   ins_encode %{
 4800     ShouldNotReachHere();
 4801   %}
 4802   ins_pipe( fpu_reg_reg );
 4803 %}
 4804 
 4805 // Load Double
 4806 instruct MoveD2VL(vlRegD dst, regD src) %{
 4807   match(Set dst src);
 4808   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4809   ins_encode %{
 4810     ShouldNotReachHere();
 4811   %}
 4812   ins_pipe( fpu_reg_reg );
 4813 %}
 4814 
 4815 // Load Double
 4816 instruct MoveD2LEG(legRegD dst, regD src) %{
 4817   match(Set dst src);
 4818   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4819   ins_encode %{
 4820     ShouldNotReachHere();
 4821   %}
 4822   ins_pipe( fpu_reg_reg );
 4823 %}
 4824 
 4825 // Load Double
 4826 instruct MoveVL2D(regD dst, vlRegD src) %{
 4827   match(Set dst src);
 4828   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4829   ins_encode %{
 4830     ShouldNotReachHere();
 4831   %}
 4832   ins_pipe( fpu_reg_reg );
 4833 %}
 4834 
 4835 // Load Double
 4836 instruct MoveLEG2D(regD dst, legRegD src) %{
 4837   match(Set dst src);
 4838   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4839   ins_encode %{
 4840     ShouldNotReachHere();
 4841   %}
 4842   ins_pipe( fpu_reg_reg );
 4843 %}
 4844 
 4845 //----------Load/Store/Move Instructions---------------------------------------
 4846 //----------Load Instructions--------------------------------------------------
 4847 
 4848 // Load Byte (8 bit signed)
 4849 instruct loadB(rRegI dst, memory mem)
 4850 %{
 4851   match(Set dst (LoadB mem));
 4852 
 4853   ins_cost(125);
 4854   format %{ "movsbl  $dst, $mem\t# byte" %}
 4855 
 4856   ins_encode %{
 4857     __ movsbl($dst$$Register, $mem$$Address);
 4858   %}
 4859 
 4860   ins_pipe(ialu_reg_mem);
 4861 %}
 4862 
 4863 // Load Byte (8 bit signed) into Long Register
 4864 instruct loadB2L(rRegL dst, memory mem)
 4865 %{
 4866   match(Set dst (ConvI2L (LoadB mem)));
 4867 
 4868   ins_cost(125);
 4869   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4870 
 4871   ins_encode %{
 4872     __ movsbq($dst$$Register, $mem$$Address);
 4873   %}
 4874 
 4875   ins_pipe(ialu_reg_mem);
 4876 %}
 4877 
 4878 // Load Unsigned Byte (8 bit UNsigned)
 4879 instruct loadUB(rRegI dst, memory mem)
 4880 %{
 4881   match(Set dst (LoadUB mem));
 4882 
 4883   ins_cost(125);
 4884   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4885 
 4886   ins_encode %{
 4887     __ movzbl($dst$$Register, $mem$$Address);
 4888   %}
 4889 
 4890   ins_pipe(ialu_reg_mem);
 4891 %}
 4892 
 4893 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4894 instruct loadUB2L(rRegL dst, memory mem)
 4895 %{
 4896   match(Set dst (ConvI2L (LoadUB mem)));
 4897 
 4898   ins_cost(125);
 4899   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4900 
 4901   ins_encode %{
 4902     __ movzbq($dst$$Register, $mem$$Address);
 4903   %}
 4904 
 4905   ins_pipe(ialu_reg_mem);
 4906 %}
 4907 
 4908 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4909 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4910   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4911   effect(KILL cr);
 4912 
 4913   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4914             "andl    $dst, right_n_bits($mask, 8)" %}
 4915   ins_encode %{
 4916     Register Rdst = $dst$$Register;
 4917     __ movzbq(Rdst, $mem$$Address);
 4918     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4919   %}
 4920   ins_pipe(ialu_reg_mem);
 4921 %}
 4922 
 4923 // Load Short (16 bit signed)
 4924 instruct loadS(rRegI dst, memory mem)
 4925 %{
 4926   match(Set dst (LoadS mem));
 4927 
 4928   ins_cost(125);
 4929   format %{ "movswl $dst, $mem\t# short" %}
 4930 
 4931   ins_encode %{
 4932     __ movswl($dst$$Register, $mem$$Address);
 4933   %}
 4934 
 4935   ins_pipe(ialu_reg_mem);
 4936 %}
 4937 
 4938 // Load Short (16 bit signed) to Byte (8 bit signed)
 4939 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4940   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4941 
 4942   ins_cost(125);
 4943   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4944   ins_encode %{
 4945     __ movsbl($dst$$Register, $mem$$Address);
 4946   %}
 4947   ins_pipe(ialu_reg_mem);
 4948 %}
 4949 
 4950 // Load Short (16 bit signed) into Long Register
 4951 instruct loadS2L(rRegL dst, memory mem)
 4952 %{
 4953   match(Set dst (ConvI2L (LoadS mem)));
 4954 
 4955   ins_cost(125);
 4956   format %{ "movswq $dst, $mem\t# short -> long" %}
 4957 
 4958   ins_encode %{
 4959     __ movswq($dst$$Register, $mem$$Address);
 4960   %}
 4961 
 4962   ins_pipe(ialu_reg_mem);
 4963 %}
 4964 
 4965 // Load Unsigned Short/Char (16 bit UNsigned)
 4966 instruct loadUS(rRegI dst, memory mem)
 4967 %{
 4968   match(Set dst (LoadUS mem));
 4969 
 4970   ins_cost(125);
 4971   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4972 
 4973   ins_encode %{
 4974     __ movzwl($dst$$Register, $mem$$Address);
 4975   %}
 4976 
 4977   ins_pipe(ialu_reg_mem);
 4978 %}
 4979 
 4980 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4981 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4982   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4983 
 4984   ins_cost(125);
 4985   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4986   ins_encode %{
 4987     __ movsbl($dst$$Register, $mem$$Address);
 4988   %}
 4989   ins_pipe(ialu_reg_mem);
 4990 %}
 4991 
 4992 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4993 instruct loadUS2L(rRegL dst, memory mem)
 4994 %{
 4995   match(Set dst (ConvI2L (LoadUS mem)));
 4996 
 4997   ins_cost(125);
 4998   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4999 
 5000   ins_encode %{
 5001     __ movzwq($dst$$Register, $mem$$Address);
 5002   %}
 5003 
 5004   ins_pipe(ialu_reg_mem);
 5005 %}
 5006 
 5007 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5008 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5009   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5010 
 5011   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5012   ins_encode %{
 5013     __ movzbq($dst$$Register, $mem$$Address);
 5014   %}
 5015   ins_pipe(ialu_reg_mem);
 5016 %}
 5017 
 5018 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5019 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5020   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5021   effect(KILL cr);
 5022 
 5023   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5024             "andl    $dst, right_n_bits($mask, 16)" %}
 5025   ins_encode %{
 5026     Register Rdst = $dst$$Register;
 5027     __ movzwq(Rdst, $mem$$Address);
 5028     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5029   %}
 5030   ins_pipe(ialu_reg_mem);
 5031 %}
 5032 
 5033 // Load Integer
 5034 instruct loadI(rRegI dst, memory mem)
 5035 %{
 5036   match(Set dst (LoadI mem));
 5037 
 5038   ins_cost(125);
 5039   format %{ "movl    $dst, $mem\t# int" %}
 5040 
 5041   ins_encode %{
 5042     __ movl($dst$$Register, $mem$$Address);
 5043   %}
 5044 
 5045   ins_pipe(ialu_reg_mem);
 5046 %}
 5047 
 5048 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5049 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5050   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5051 
 5052   ins_cost(125);
 5053   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5054   ins_encode %{
 5055     __ movsbl($dst$$Register, $mem$$Address);
 5056   %}
 5057   ins_pipe(ialu_reg_mem);
 5058 %}
 5059 
 5060 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5061 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5062   match(Set dst (AndI (LoadI mem) mask));
 5063 
 5064   ins_cost(125);
 5065   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5066   ins_encode %{
 5067     __ movzbl($dst$$Register, $mem$$Address);
 5068   %}
 5069   ins_pipe(ialu_reg_mem);
 5070 %}
 5071 
 5072 // Load Integer (32 bit signed) to Short (16 bit signed)
 5073 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5074   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5075 
 5076   ins_cost(125);
 5077   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5078   ins_encode %{
 5079     __ movswl($dst$$Register, $mem$$Address);
 5080   %}
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5085 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5086   match(Set dst (AndI (LoadI mem) mask));
 5087 
 5088   ins_cost(125);
 5089   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5090   ins_encode %{
 5091     __ movzwl($dst$$Register, $mem$$Address);
 5092   %}
 5093   ins_pipe(ialu_reg_mem);
 5094 %}
 5095 
 5096 // Load Integer into Long Register
 5097 instruct loadI2L(rRegL dst, memory mem)
 5098 %{
 5099   match(Set dst (ConvI2L (LoadI mem)));
 5100 
 5101   ins_cost(125);
 5102   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5103 
 5104   ins_encode %{
 5105     __ movslq($dst$$Register, $mem$$Address);
 5106   %}
 5107 
 5108   ins_pipe(ialu_reg_mem);
 5109 %}
 5110 
 5111 // Load Integer with mask 0xFF into Long Register
 5112 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5113   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5114 
 5115   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5116   ins_encode %{
 5117     __ movzbq($dst$$Register, $mem$$Address);
 5118   %}
 5119   ins_pipe(ialu_reg_mem);
 5120 %}
 5121 
 5122 // Load Integer with mask 0xFFFF into Long Register
 5123 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5124   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5125 
 5126   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5127   ins_encode %{
 5128     __ movzwq($dst$$Register, $mem$$Address);
 5129   %}
 5130   ins_pipe(ialu_reg_mem);
 5131 %}
 5132 
 5133 // Load Integer with a 31-bit mask into Long Register
 5134 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5135   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5136   effect(KILL cr);
 5137 
 5138   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5139             "andl    $dst, $mask" %}
 5140   ins_encode %{
 5141     Register Rdst = $dst$$Register;
 5142     __ movl(Rdst, $mem$$Address);
 5143     __ andl(Rdst, $mask$$constant);
 5144   %}
 5145   ins_pipe(ialu_reg_mem);
 5146 %}
 5147 
 5148 // Load Unsigned Integer into Long Register
 5149 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5150 %{
 5151   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5152 
 5153   ins_cost(125);
 5154   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5155 
 5156   ins_encode %{
 5157     __ movl($dst$$Register, $mem$$Address);
 5158   %}
 5159 
 5160   ins_pipe(ialu_reg_mem);
 5161 %}
 5162 
 5163 // Load Long
 5164 instruct loadL(rRegL dst, memory mem)
 5165 %{
 5166   match(Set dst (LoadL mem));
 5167 
 5168   ins_cost(125);
 5169   format %{ "movq    $dst, $mem\t# long" %}
 5170 
 5171   ins_encode %{
 5172     __ movq($dst$$Register, $mem$$Address);
 5173   %}
 5174 
 5175   ins_pipe(ialu_reg_mem); // XXX
 5176 %}
 5177 
 5178 // Load Range
 5179 instruct loadRange(rRegI dst, memory mem)
 5180 %{
 5181   match(Set dst (LoadRange mem));
 5182 
 5183   ins_cost(125); // XXX
 5184   format %{ "movl    $dst, $mem\t# range" %}
 5185   ins_encode %{
 5186     __ movl($dst$$Register, $mem$$Address);
 5187   %}
 5188   ins_pipe(ialu_reg_mem);
 5189 %}
 5190 
 5191 // Load Pointer
 5192 instruct loadP(rRegP dst, memory mem)
 5193 %{
 5194   match(Set dst (LoadP mem));
 5195   predicate(n->as_Load()->barrier_data() == 0);
 5196 
 5197   ins_cost(125); // XXX
 5198   format %{ "movq    $dst, $mem\t# ptr" %}
 5199   ins_encode %{
 5200     __ movq($dst$$Register, $mem$$Address);
 5201   %}
 5202   ins_pipe(ialu_reg_mem); // XXX
 5203 %}
 5204 
 5205 // Load Compressed Pointer
 5206 instruct loadN(rRegN dst, memory mem)
 5207 %{
 5208    match(Set dst (LoadN mem));
 5209 
 5210    ins_cost(125); // XXX
 5211    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5212    ins_encode %{
 5213      __ movl($dst$$Register, $mem$$Address);
 5214    %}
 5215    ins_pipe(ialu_reg_mem); // XXX
 5216 %}
 5217 
 5218 
 5219 // Load Klass Pointer
 5220 instruct loadKlass(rRegP dst, memory mem)
 5221 %{
 5222   match(Set dst (LoadKlass mem));
 5223 
 5224   ins_cost(125); // XXX
 5225   format %{ "movq    $dst, $mem\t# class" %}
 5226   ins_encode %{
 5227     __ movq($dst$$Register, $mem$$Address);
 5228   %}
 5229   ins_pipe(ialu_reg_mem); // XXX
 5230 %}
 5231 
 5232 // Load narrow Klass Pointer
 5233 instruct loadNKlass(rRegN dst, memory mem)
 5234 %{
 5235   match(Set dst (LoadNKlass mem));
 5236 
 5237   ins_cost(125); // XXX
 5238   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5239   ins_encode %{
 5240     __ movl($dst$$Register, $mem$$Address);
 5241   %}
 5242   ins_pipe(ialu_reg_mem); // XXX
 5243 %}
 5244 
 5245 // Load Float
 5246 instruct loadF(regF dst, memory mem)
 5247 %{
 5248   match(Set dst (LoadF mem));
 5249 
 5250   ins_cost(145); // XXX
 5251   format %{ "movss   $dst, $mem\t# float" %}
 5252   ins_encode %{
 5253     __ movflt($dst$$XMMRegister, $mem$$Address);
 5254   %}
 5255   ins_pipe(pipe_slow); // XXX
 5256 %}
 5257 
 5258 // Load Double
 5259 instruct loadD_partial(regD dst, memory mem)
 5260 %{
 5261   predicate(!UseXmmLoadAndClearUpper);
 5262   match(Set dst (LoadD mem));
 5263 
 5264   ins_cost(145); // XXX
 5265   format %{ "movlpd  $dst, $mem\t# double" %}
 5266   ins_encode %{
 5267     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5268   %}
 5269   ins_pipe(pipe_slow); // XXX
 5270 %}
 5271 
 5272 instruct loadD(regD dst, memory mem)
 5273 %{
 5274   predicate(UseXmmLoadAndClearUpper);
 5275   match(Set dst (LoadD mem));
 5276 
 5277   ins_cost(145); // XXX
 5278   format %{ "movsd   $dst, $mem\t# double" %}
 5279   ins_encode %{
 5280     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5281   %}
 5282   ins_pipe(pipe_slow); // XXX
 5283 %}
 5284 
 5285 
 5286 // Following pseudo code describes the algorithm for max[FD]:
 5287 // Min algorithm is on similar lines
 5288 //  btmp = (b < +0.0) ? a : b
 5289 //  atmp = (b < +0.0) ? b : a
 5290 //  Tmp  = Max_Float(atmp , btmp)
 5291 //  Res  = (atmp == NaN) ? atmp : Tmp
 5292 
 5293 // max = java.lang.Math.max(float a, float b)
 5294 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5295   predicate(UseAVX > 0 && !n->is_reduction());
 5296   match(Set dst (MaxF a b));
 5297   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5298   format %{
 5299      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5300      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5301      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5302      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5303      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5304   %}
 5305   ins_encode %{
 5306     int vector_len = Assembler::AVX_128bit;
 5307     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5308     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5309     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5310     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5311     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5312  %}
 5313   ins_pipe( pipe_slow );
 5314 %}
 5315 
 5316 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5317   predicate(UseAVX > 0 && n->is_reduction());
 5318   match(Set dst (MaxF a b));
 5319   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5320 
 5321   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5322   ins_encode %{
 5323     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5324                     false /*min*/, true /*single*/);
 5325   %}
 5326   ins_pipe( pipe_slow );
 5327 %}
 5328 
 5329 // max = java.lang.Math.max(double a, double b)
 5330 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5331   predicate(UseAVX > 0 && !n->is_reduction());
 5332   match(Set dst (MaxD a b));
 5333   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5334   format %{
 5335      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5336      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5337      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5338      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5339      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5340   %}
 5341   ins_encode %{
 5342     int vector_len = Assembler::AVX_128bit;
 5343     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5344     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5345     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5346     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5347     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5348   %}
 5349   ins_pipe( pipe_slow );
 5350 %}
 5351 
 5352 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5353   predicate(UseAVX > 0 && n->is_reduction());
 5354   match(Set dst (MaxD a b));
 5355   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5356 
 5357   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5358   ins_encode %{
 5359     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5360                     false /*min*/, false /*single*/);
 5361   %}
 5362   ins_pipe( pipe_slow );
 5363 %}
 5364 
 5365 // min = java.lang.Math.min(float a, float b)
 5366 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5367   predicate(UseAVX > 0 && !n->is_reduction());
 5368   match(Set dst (MinF a b));
 5369   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5370   format %{
 5371      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5372      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5373      "vminss           $tmp,$atmp,$btmp           \n\t"
 5374      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5375      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5376   %}
 5377   ins_encode %{
 5378     int vector_len = Assembler::AVX_128bit;
 5379     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5380     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5381     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5382     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5383     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5384   %}
 5385   ins_pipe( pipe_slow );
 5386 %}
 5387 
 5388 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5389   predicate(UseAVX > 0 && n->is_reduction());
 5390   match(Set dst (MinF a b));
 5391   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5392 
 5393   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5394   ins_encode %{
 5395     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5396                     true /*min*/, true /*single*/);
 5397   %}
 5398   ins_pipe( pipe_slow );
 5399 %}
 5400 
 5401 // min = java.lang.Math.min(double a, double b)
 5402 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5403   predicate(UseAVX > 0 && !n->is_reduction());
 5404   match(Set dst (MinD a b));
 5405   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5406   format %{
 5407      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5408      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5409      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5410      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5411      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5412   %}
 5413   ins_encode %{
 5414     int vector_len = Assembler::AVX_128bit;
 5415     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5416     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5417     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5418     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5419     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5420   %}
 5421   ins_pipe( pipe_slow );
 5422 %}
 5423 
 5424 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5425   predicate(UseAVX > 0 && n->is_reduction());
 5426   match(Set dst (MinD a b));
 5427   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5428 
 5429   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5430   ins_encode %{
 5431     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5432                     true /*min*/, false /*single*/);
 5433   %}
 5434   ins_pipe( pipe_slow );
 5435 %}
 5436 
 5437 // Load Effective Address
 5438 instruct leaP8(rRegP dst, indOffset8 mem)
 5439 %{
 5440   match(Set dst mem);
 5441 
 5442   ins_cost(110); // XXX
 5443   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5444   ins_encode %{
 5445     __ leaq($dst$$Register, $mem$$Address);
 5446   %}
 5447   ins_pipe(ialu_reg_reg_fat);
 5448 %}
 5449 
 5450 instruct leaP32(rRegP dst, indOffset32 mem)
 5451 %{
 5452   match(Set dst mem);
 5453 
 5454   ins_cost(110);
 5455   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5456   ins_encode %{
 5457     __ leaq($dst$$Register, $mem$$Address);
 5458   %}
 5459   ins_pipe(ialu_reg_reg_fat);
 5460 %}
 5461 
 5462 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5463 %{
 5464   match(Set dst mem);
 5465 
 5466   ins_cost(110);
 5467   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5468   ins_encode %{
 5469     __ leaq($dst$$Register, $mem$$Address);
 5470   %}
 5471   ins_pipe(ialu_reg_reg_fat);
 5472 %}
 5473 
 5474 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5475 %{
 5476   match(Set dst mem);
 5477 
 5478   ins_cost(110);
 5479   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5480   ins_encode %{
 5481     __ leaq($dst$$Register, $mem$$Address);
 5482   %}
 5483   ins_pipe(ialu_reg_reg_fat);
 5484 %}
 5485 
 5486 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5487 %{
 5488   match(Set dst mem);
 5489 
 5490   ins_cost(110);
 5491   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5492   ins_encode %{
 5493     __ leaq($dst$$Register, $mem$$Address);
 5494   %}
 5495   ins_pipe(ialu_reg_reg_fat);
 5496 %}
 5497 
 5498 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5499 %{
 5500   match(Set dst mem);
 5501 
 5502   ins_cost(110);
 5503   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5504   ins_encode %{
 5505     __ leaq($dst$$Register, $mem$$Address);
 5506   %}
 5507   ins_pipe(ialu_reg_reg_fat);
 5508 %}
 5509 
 5510 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5511 %{
 5512   match(Set dst mem);
 5513 
 5514   ins_cost(110);
 5515   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5516   ins_encode %{
 5517     __ leaq($dst$$Register, $mem$$Address);
 5518   %}
 5519   ins_pipe(ialu_reg_reg_fat);
 5520 %}
 5521 
 5522 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5523 %{
 5524   match(Set dst mem);
 5525 
 5526   ins_cost(110);
 5527   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5528   ins_encode %{
 5529     __ leaq($dst$$Register, $mem$$Address);
 5530   %}
 5531   ins_pipe(ialu_reg_reg_fat);
 5532 %}
 5533 
 5534 // Load Effective Address which uses Narrow (32-bits) oop
 5535 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5536 %{
 5537   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5538   match(Set dst mem);
 5539 
 5540   ins_cost(110);
 5541   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5542   ins_encode %{
 5543     __ leaq($dst$$Register, $mem$$Address);
 5544   %}
 5545   ins_pipe(ialu_reg_reg_fat);
 5546 %}
 5547 
 5548 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5549 %{
 5550   predicate(CompressedOops::shift() == 0);
 5551   match(Set dst mem);
 5552 
 5553   ins_cost(110); // XXX
 5554   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5555   ins_encode %{
 5556     __ leaq($dst$$Register, $mem$$Address);
 5557   %}
 5558   ins_pipe(ialu_reg_reg_fat);
 5559 %}
 5560 
 5561 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5562 %{
 5563   predicate(CompressedOops::shift() == 0);
 5564   match(Set dst mem);
 5565 
 5566   ins_cost(110);
 5567   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5568   ins_encode %{
 5569     __ leaq($dst$$Register, $mem$$Address);
 5570   %}
 5571   ins_pipe(ialu_reg_reg_fat);
 5572 %}
 5573 
 5574 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5575 %{
 5576   predicate(CompressedOops::shift() == 0);
 5577   match(Set dst mem);
 5578 
 5579   ins_cost(110);
 5580   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5581   ins_encode %{
 5582     __ leaq($dst$$Register, $mem$$Address);
 5583   %}
 5584   ins_pipe(ialu_reg_reg_fat);
 5585 %}
 5586 
 5587 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5588 %{
 5589   predicate(CompressedOops::shift() == 0);
 5590   match(Set dst mem);
 5591 
 5592   ins_cost(110);
 5593   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5594   ins_encode %{
 5595     __ leaq($dst$$Register, $mem$$Address);
 5596   %}
 5597   ins_pipe(ialu_reg_reg_fat);
 5598 %}
 5599 
 5600 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5601 %{
 5602   predicate(CompressedOops::shift() == 0);
 5603   match(Set dst mem);
 5604 
 5605   ins_cost(110);
 5606   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5607   ins_encode %{
 5608     __ leaq($dst$$Register, $mem$$Address);
 5609   %}
 5610   ins_pipe(ialu_reg_reg_fat);
 5611 %}
 5612 
 5613 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5614 %{
 5615   predicate(CompressedOops::shift() == 0);
 5616   match(Set dst mem);
 5617 
 5618   ins_cost(110);
 5619   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5620   ins_encode %{
 5621     __ leaq($dst$$Register, $mem$$Address);
 5622   %}
 5623   ins_pipe(ialu_reg_reg_fat);
 5624 %}
 5625 
 5626 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5627 %{
 5628   predicate(CompressedOops::shift() == 0);
 5629   match(Set dst mem);
 5630 
 5631   ins_cost(110);
 5632   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5633   ins_encode %{
 5634     __ leaq($dst$$Register, $mem$$Address);
 5635   %}
 5636   ins_pipe(ialu_reg_reg_fat);
 5637 %}
 5638 
 5639 instruct loadConI(rRegI dst, immI src)
 5640 %{
 5641   match(Set dst src);
 5642 
 5643   format %{ "movl    $dst, $src\t# int" %}
 5644   ins_encode %{
 5645     __ movl($dst$$Register, $src$$constant);
 5646   %}
 5647   ins_pipe(ialu_reg_fat); // XXX
 5648 %}
 5649 
 5650 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5651 %{
 5652   match(Set dst src);
 5653   effect(KILL cr);
 5654 
 5655   ins_cost(50);
 5656   format %{ "xorl    $dst, $dst\t# int" %}
 5657   ins_encode %{
 5658     __ xorl($dst$$Register, $dst$$Register);
 5659   %}
 5660   ins_pipe(ialu_reg);
 5661 %}
 5662 
 5663 instruct loadConL(rRegL dst, immL src)
 5664 %{
 5665   match(Set dst src);
 5666 
 5667   ins_cost(150);
 5668   format %{ "movq    $dst, $src\t# long" %}
 5669   ins_encode %{
 5670     __ mov64($dst$$Register, $src$$constant);
 5671   %}
 5672   ins_pipe(ialu_reg);
 5673 %}
 5674 
 5675 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5676 %{
 5677   match(Set dst src);
 5678   effect(KILL cr);
 5679 
 5680   ins_cost(50);
 5681   format %{ "xorl    $dst, $dst\t# long" %}
 5682   ins_encode %{
 5683     __ xorl($dst$$Register, $dst$$Register);
 5684   %}
 5685   ins_pipe(ialu_reg); // XXX
 5686 %}
 5687 
 5688 instruct loadConUL32(rRegL dst, immUL32 src)
 5689 %{
 5690   match(Set dst src);
 5691 
 5692   ins_cost(60);
 5693   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5694   ins_encode %{
 5695     __ movl($dst$$Register, $src$$constant);
 5696   %}
 5697   ins_pipe(ialu_reg);
 5698 %}
 5699 
 5700 instruct loadConL32(rRegL dst, immL32 src)
 5701 %{
 5702   match(Set dst src);
 5703 
 5704   ins_cost(70);
 5705   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5706   ins_encode %{
 5707     __ movq($dst$$Register, $src$$constant);
 5708   %}
 5709   ins_pipe(ialu_reg);
 5710 %}
 5711 
 5712 instruct loadConP(rRegP dst, immP con) %{
 5713   match(Set dst con);
 5714 
 5715   format %{ "movq    $dst, $con\t# ptr" %}
 5716   ins_encode %{
 5717     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5718   %}
 5719   ins_pipe(ialu_reg_fat); // XXX
 5720 %}
 5721 
 5722 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5723 %{
 5724   match(Set dst src);
 5725   effect(KILL cr);
 5726 
 5727   ins_cost(50);
 5728   format %{ "xorl    $dst, $dst\t# ptr" %}
 5729   ins_encode %{
 5730     __ xorl($dst$$Register, $dst$$Register);
 5731   %}
 5732   ins_pipe(ialu_reg);
 5733 %}
 5734 
 5735 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5736 %{
 5737   match(Set dst src);
 5738   effect(KILL cr);
 5739 
 5740   ins_cost(60);
 5741   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5742   ins_encode %{
 5743     __ movl($dst$$Register, $src$$constant);
 5744   %}
 5745   ins_pipe(ialu_reg);
 5746 %}
 5747 
 5748 instruct loadConF(regF dst, immF con) %{
 5749   match(Set dst con);
 5750   ins_cost(125);
 5751   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5752   ins_encode %{
 5753     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5754   %}
 5755   ins_pipe(pipe_slow);
 5756 %}
 5757 
 5758 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5759   match(Set dst src);
 5760   effect(KILL cr);
 5761   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5762   ins_encode %{
 5763     __ xorq($dst$$Register, $dst$$Register);
 5764   %}
 5765   ins_pipe(ialu_reg);
 5766 %}
 5767 
 5768 instruct loadConN(rRegN dst, immN src) %{
 5769   match(Set dst src);
 5770 
 5771   ins_cost(125);
 5772   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5773   ins_encode %{
 5774     address con = (address)$src$$constant;
 5775     if (con == NULL) {
 5776       ShouldNotReachHere();
 5777     } else {
 5778       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5779     }
 5780   %}
 5781   ins_pipe(ialu_reg_fat); // XXX
 5782 %}
 5783 
 5784 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5785   match(Set dst src);
 5786 
 5787   ins_cost(125);
 5788   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5789   ins_encode %{
 5790     address con = (address)$src$$constant;
 5791     if (con == NULL) {
 5792       ShouldNotReachHere();
 5793     } else {
 5794       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5795     }
 5796   %}
 5797   ins_pipe(ialu_reg_fat); // XXX
 5798 %}
 5799 
 5800 instruct loadConF0(regF dst, immF0 src)
 5801 %{
 5802   match(Set dst src);
 5803   ins_cost(100);
 5804 
 5805   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5806   ins_encode %{
 5807     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5808   %}
 5809   ins_pipe(pipe_slow);
 5810 %}
 5811 
 5812 // Use the same format since predicate() can not be used here.
 5813 instruct loadConD(regD dst, immD con) %{
 5814   match(Set dst con);
 5815   ins_cost(125);
 5816   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5817   ins_encode %{
 5818     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5819   %}
 5820   ins_pipe(pipe_slow);
 5821 %}
 5822 
 5823 instruct loadConD0(regD dst, immD0 src)
 5824 %{
 5825   match(Set dst src);
 5826   ins_cost(100);
 5827 
 5828   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5829   ins_encode %{
 5830     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5831   %}
 5832   ins_pipe(pipe_slow);
 5833 %}
 5834 
 5835 instruct loadSSI(rRegI dst, stackSlotI src)
 5836 %{
 5837   match(Set dst src);
 5838 
 5839   ins_cost(125);
 5840   format %{ "movl    $dst, $src\t# int stk" %}
 5841   opcode(0x8B);
 5842   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5843   ins_pipe(ialu_reg_mem);
 5844 %}
 5845 
 5846 instruct loadSSL(rRegL dst, stackSlotL src)
 5847 %{
 5848   match(Set dst src);
 5849 
 5850   ins_cost(125);
 5851   format %{ "movq    $dst, $src\t# long stk" %}
 5852   opcode(0x8B);
 5853   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5854   ins_pipe(ialu_reg_mem);
 5855 %}
 5856 
 5857 instruct loadSSP(rRegP dst, stackSlotP src)
 5858 %{
 5859   match(Set dst src);
 5860 
 5861   ins_cost(125);
 5862   format %{ "movq    $dst, $src\t# ptr stk" %}
 5863   opcode(0x8B);
 5864   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5865   ins_pipe(ialu_reg_mem);
 5866 %}
 5867 
 5868 instruct loadSSF(regF dst, stackSlotF src)
 5869 %{
 5870   match(Set dst src);
 5871 
 5872   ins_cost(125);
 5873   format %{ "movss   $dst, $src\t# float stk" %}
 5874   ins_encode %{
 5875     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5876   %}
 5877   ins_pipe(pipe_slow); // XXX
 5878 %}
 5879 
 5880 // Use the same format since predicate() can not be used here.
 5881 instruct loadSSD(regD dst, stackSlotD src)
 5882 %{
 5883   match(Set dst src);
 5884 
 5885   ins_cost(125);
 5886   format %{ "movsd   $dst, $src\t# double stk" %}
 5887   ins_encode  %{
 5888     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5889   %}
 5890   ins_pipe(pipe_slow); // XXX
 5891 %}
 5892 
 5893 // Prefetch instructions for allocation.
 5894 // Must be safe to execute with invalid address (cannot fault).
 5895 
 5896 instruct prefetchAlloc( memory mem ) %{
 5897   predicate(AllocatePrefetchInstr==3);
 5898   match(PrefetchAllocation mem);
 5899   ins_cost(125);
 5900 
 5901   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5902   ins_encode %{
 5903     __ prefetchw($mem$$Address);
 5904   %}
 5905   ins_pipe(ialu_mem);
 5906 %}
 5907 
 5908 instruct prefetchAllocNTA( memory mem ) %{
 5909   predicate(AllocatePrefetchInstr==0);
 5910   match(PrefetchAllocation mem);
 5911   ins_cost(125);
 5912 
 5913   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5914   ins_encode %{
 5915     __ prefetchnta($mem$$Address);
 5916   %}
 5917   ins_pipe(ialu_mem);
 5918 %}
 5919 
 5920 instruct prefetchAllocT0( memory mem ) %{
 5921   predicate(AllocatePrefetchInstr==1);
 5922   match(PrefetchAllocation mem);
 5923   ins_cost(125);
 5924 
 5925   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5926   ins_encode %{
 5927     __ prefetcht0($mem$$Address);
 5928   %}
 5929   ins_pipe(ialu_mem);
 5930 %}
 5931 
 5932 instruct prefetchAllocT2( memory mem ) %{
 5933   predicate(AllocatePrefetchInstr==2);
 5934   match(PrefetchAllocation mem);
 5935   ins_cost(125);
 5936 
 5937   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5938   ins_encode %{
 5939     __ prefetcht2($mem$$Address);
 5940   %}
 5941   ins_pipe(ialu_mem);
 5942 %}
 5943 
 5944 //----------Store Instructions-------------------------------------------------
 5945 
 5946 // Store Byte
 5947 instruct storeB(memory mem, rRegI src)
 5948 %{
 5949   match(Set mem (StoreB mem src));
 5950 
 5951   ins_cost(125); // XXX
 5952   format %{ "movb    $mem, $src\t# byte" %}
 5953   ins_encode %{
 5954     __ movb($mem$$Address, $src$$Register);
 5955   %}
 5956   ins_pipe(ialu_mem_reg);
 5957 %}
 5958 
 5959 // Store Char/Short
 5960 instruct storeC(memory mem, rRegI src)
 5961 %{
 5962   match(Set mem (StoreC mem src));
 5963 
 5964   ins_cost(125); // XXX
 5965   format %{ "movw    $mem, $src\t# char/short" %}
 5966   ins_encode %{
 5967     __ movw($mem$$Address, $src$$Register);
 5968   %}
 5969   ins_pipe(ialu_mem_reg);
 5970 %}
 5971 
 5972 // Store Integer
 5973 instruct storeI(memory mem, rRegI src)
 5974 %{
 5975   match(Set mem (StoreI mem src));
 5976 
 5977   ins_cost(125); // XXX
 5978   format %{ "movl    $mem, $src\t# int" %}
 5979   ins_encode %{
 5980     __ movl($mem$$Address, $src$$Register);
 5981   %}
 5982   ins_pipe(ialu_mem_reg);
 5983 %}
 5984 
 5985 // Store Long
 5986 instruct storeL(memory mem, rRegL src)
 5987 %{
 5988   match(Set mem (StoreL mem src));
 5989 
 5990   ins_cost(125); // XXX
 5991   format %{ "movq    $mem, $src\t# long" %}
 5992   ins_encode %{
 5993     __ movq($mem$$Address, $src$$Register);
 5994   %}
 5995   ins_pipe(ialu_mem_reg); // XXX
 5996 %}
 5997 
 5998 // Store Pointer
 5999 instruct storeP(memory mem, any_RegP src)
 6000 %{
 6001   match(Set mem (StoreP mem src));
 6002 
 6003   ins_cost(125); // XXX
 6004   format %{ "movq    $mem, $src\t# ptr" %}
 6005   ins_encode %{
 6006     __ movq($mem$$Address, $src$$Register);
 6007   %}
 6008   ins_pipe(ialu_mem_reg);
 6009 %}
 6010 
 6011 instruct storeImmP0(memory mem, immP0 zero)
 6012 %{
 6013   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6014   match(Set mem (StoreP mem zero));
 6015 
 6016   ins_cost(125); // XXX
 6017   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6018   ins_encode %{
 6019     __ movq($mem$$Address, r12);
 6020   %}
 6021   ins_pipe(ialu_mem_reg);
 6022 %}
 6023 
 6024 // Store NULL Pointer, mark word, or other simple pointer constant.
 6025 instruct storeImmP(memory mem, immP31 src)
 6026 %{
 6027   match(Set mem (StoreP mem src));
 6028 
 6029   ins_cost(150); // XXX
 6030   format %{ "movq    $mem, $src\t# ptr" %}
 6031   ins_encode %{
 6032     __ movq($mem$$Address, $src$$constant);
 6033   %}
 6034   ins_pipe(ialu_mem_imm);
 6035 %}
 6036 
 6037 // Store Compressed Pointer
 6038 instruct storeN(memory mem, rRegN src)
 6039 %{
 6040   match(Set mem (StoreN mem src));
 6041 
 6042   ins_cost(125); // XXX
 6043   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6044   ins_encode %{
 6045     __ movl($mem$$Address, $src$$Register);
 6046   %}
 6047   ins_pipe(ialu_mem_reg);
 6048 %}
 6049 
 6050 instruct storeNKlass(memory mem, rRegN src)
 6051 %{
 6052   match(Set mem (StoreNKlass mem src));
 6053 
 6054   ins_cost(125); // XXX
 6055   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6056   ins_encode %{
 6057     __ movl($mem$$Address, $src$$Register);
 6058   %}
 6059   ins_pipe(ialu_mem_reg);
 6060 %}
 6061 
 6062 instruct storeImmN0(memory mem, immN0 zero)
 6063 %{
 6064   predicate(CompressedOops::base() == NULL);
 6065   match(Set mem (StoreN mem zero));
 6066 
 6067   ins_cost(125); // XXX
 6068   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6069   ins_encode %{
 6070     __ movl($mem$$Address, r12);
 6071   %}
 6072   ins_pipe(ialu_mem_reg);
 6073 %}
 6074 
 6075 instruct storeImmN(memory mem, immN src)
 6076 %{
 6077   match(Set mem (StoreN mem src));
 6078 
 6079   ins_cost(150); // XXX
 6080   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6081   ins_encode %{
 6082     address con = (address)$src$$constant;
 6083     if (con == NULL) {
 6084       __ movl($mem$$Address, (int32_t)0);
 6085     } else {
 6086       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6087     }
 6088   %}
 6089   ins_pipe(ialu_mem_imm);
 6090 %}
 6091 
 6092 instruct storeImmNKlass(memory mem, immNKlass src)
 6093 %{
 6094   match(Set mem (StoreNKlass mem src));
 6095 
 6096   ins_cost(150); // XXX
 6097   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6098   ins_encode %{
 6099     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6100   %}
 6101   ins_pipe(ialu_mem_imm);
 6102 %}
 6103 
 6104 // Store Integer Immediate
 6105 instruct storeImmI0(memory mem, immI_0 zero)
 6106 %{
 6107   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6108   match(Set mem (StoreI mem zero));
 6109 
 6110   ins_cost(125); // XXX
 6111   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6112   ins_encode %{
 6113     __ movl($mem$$Address, r12);
 6114   %}
 6115   ins_pipe(ialu_mem_reg);
 6116 %}
 6117 
 6118 instruct storeImmI(memory mem, immI src)
 6119 %{
 6120   match(Set mem (StoreI mem src));
 6121 
 6122   ins_cost(150);
 6123   format %{ "movl    $mem, $src\t# int" %}
 6124   ins_encode %{
 6125     __ movl($mem$$Address, $src$$constant);
 6126   %}
 6127   ins_pipe(ialu_mem_imm);
 6128 %}
 6129 
 6130 // Store Long Immediate
 6131 instruct storeImmL0(memory mem, immL0 zero)
 6132 %{
 6133   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6134   match(Set mem (StoreL mem zero));
 6135 
 6136   ins_cost(125); // XXX
 6137   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6138   ins_encode %{
 6139     __ movq($mem$$Address, r12);
 6140   %}
 6141   ins_pipe(ialu_mem_reg);
 6142 %}
 6143 
 6144 instruct storeImmL(memory mem, immL32 src)
 6145 %{
 6146   match(Set mem (StoreL mem src));
 6147 
 6148   ins_cost(150);
 6149   format %{ "movq    $mem, $src\t# long" %}
 6150   ins_encode %{
 6151     __ movq($mem$$Address, $src$$constant);
 6152   %}
 6153   ins_pipe(ialu_mem_imm);
 6154 %}
 6155 
 6156 // Store Short/Char Immediate
 6157 instruct storeImmC0(memory mem, immI_0 zero)
 6158 %{
 6159   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6160   match(Set mem (StoreC mem zero));
 6161 
 6162   ins_cost(125); // XXX
 6163   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6164   ins_encode %{
 6165     __ movw($mem$$Address, r12);
 6166   %}
 6167   ins_pipe(ialu_mem_reg);
 6168 %}
 6169 
 6170 instruct storeImmI16(memory mem, immI16 src)
 6171 %{
 6172   predicate(UseStoreImmI16);
 6173   match(Set mem (StoreC mem src));
 6174 
 6175   ins_cost(150);
 6176   format %{ "movw    $mem, $src\t# short/char" %}
 6177   ins_encode %{
 6178     __ movw($mem$$Address, $src$$constant);
 6179   %}
 6180   ins_pipe(ialu_mem_imm);
 6181 %}
 6182 
 6183 // Store Byte Immediate
 6184 instruct storeImmB0(memory mem, immI_0 zero)
 6185 %{
 6186   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6187   match(Set mem (StoreB mem zero));
 6188 
 6189   ins_cost(125); // XXX
 6190   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6191   ins_encode %{
 6192     __ movb($mem$$Address, r12);
 6193   %}
 6194   ins_pipe(ialu_mem_reg);
 6195 %}
 6196 
 6197 instruct storeImmB(memory mem, immI8 src)
 6198 %{
 6199   match(Set mem (StoreB mem src));
 6200 
 6201   ins_cost(150); // XXX
 6202   format %{ "movb    $mem, $src\t# byte" %}
 6203   ins_encode %{
 6204     __ movb($mem$$Address, $src$$constant);
 6205   %}
 6206   ins_pipe(ialu_mem_imm);
 6207 %}
 6208 
 6209 // Store CMS card-mark Immediate
 6210 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6211 %{
 6212   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6213   match(Set mem (StoreCM mem zero));
 6214 
 6215   ins_cost(125); // XXX
 6216   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6217   ins_encode %{
 6218     __ movb($mem$$Address, r12);
 6219   %}
 6220   ins_pipe(ialu_mem_reg);
 6221 %}
 6222 
 6223 instruct storeImmCM0(memory mem, immI_0 src)
 6224 %{
 6225   match(Set mem (StoreCM mem src));
 6226 
 6227   ins_cost(150); // XXX
 6228   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6229   ins_encode %{
 6230     __ movb($mem$$Address, $src$$constant);
 6231   %}
 6232   ins_pipe(ialu_mem_imm);
 6233 %}
 6234 
 6235 // Store Float
 6236 instruct storeF(memory mem, regF src)
 6237 %{
 6238   match(Set mem (StoreF mem src));
 6239 
 6240   ins_cost(95); // XXX
 6241   format %{ "movss   $mem, $src\t# float" %}
 6242   ins_encode %{
 6243     __ movflt($mem$$Address, $src$$XMMRegister);
 6244   %}
 6245   ins_pipe(pipe_slow); // XXX
 6246 %}
 6247 
 6248 // Store immediate Float value (it is faster than store from XMM register)
 6249 instruct storeF0(memory mem, immF0 zero)
 6250 %{
 6251   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6252   match(Set mem (StoreF mem zero));
 6253 
 6254   ins_cost(25); // XXX
 6255   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6256   ins_encode %{
 6257     __ movl($mem$$Address, r12);
 6258   %}
 6259   ins_pipe(ialu_mem_reg);
 6260 %}
 6261 
 6262 instruct storeF_imm(memory mem, immF src)
 6263 %{
 6264   match(Set mem (StoreF mem src));
 6265 
 6266   ins_cost(50);
 6267   format %{ "movl    $mem, $src\t# float" %}
 6268   ins_encode %{
 6269     __ movl($mem$$Address, jint_cast($src$$constant));
 6270   %}
 6271   ins_pipe(ialu_mem_imm);
 6272 %}
 6273 
 6274 // Store Double
 6275 instruct storeD(memory mem, regD src)
 6276 %{
 6277   match(Set mem (StoreD mem src));
 6278 
 6279   ins_cost(95); // XXX
 6280   format %{ "movsd   $mem, $src\t# double" %}
 6281   ins_encode %{
 6282     __ movdbl($mem$$Address, $src$$XMMRegister);
 6283   %}
 6284   ins_pipe(pipe_slow); // XXX
 6285 %}
 6286 
 6287 // Store immediate double 0.0 (it is faster than store from XMM register)
 6288 instruct storeD0_imm(memory mem, immD0 src)
 6289 %{
 6290   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6291   match(Set mem (StoreD mem src));
 6292 
 6293   ins_cost(50);
 6294   format %{ "movq    $mem, $src\t# double 0." %}
 6295   ins_encode %{
 6296     __ movq($mem$$Address, $src$$constant);
 6297   %}
 6298   ins_pipe(ialu_mem_imm);
 6299 %}
 6300 
 6301 instruct storeD0(memory mem, immD0 zero)
 6302 %{
 6303   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6304   match(Set mem (StoreD mem zero));
 6305 
 6306   ins_cost(25); // XXX
 6307   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6308   ins_encode %{
 6309     __ movq($mem$$Address, r12);
 6310   %}
 6311   ins_pipe(ialu_mem_reg);
 6312 %}
 6313 
 6314 instruct storeSSI(stackSlotI dst, rRegI src)
 6315 %{
 6316   match(Set dst src);
 6317 
 6318   ins_cost(100);
 6319   format %{ "movl    $dst, $src\t# int stk" %}
 6320   opcode(0x89);
 6321   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6322   ins_pipe( ialu_mem_reg );
 6323 %}
 6324 
 6325 instruct storeSSL(stackSlotL dst, rRegL src)
 6326 %{
 6327   match(Set dst src);
 6328 
 6329   ins_cost(100);
 6330   format %{ "movq    $dst, $src\t# long stk" %}
 6331   opcode(0x89);
 6332   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6333   ins_pipe(ialu_mem_reg);
 6334 %}
 6335 
 6336 instruct storeSSP(stackSlotP dst, rRegP src)
 6337 %{
 6338   match(Set dst src);
 6339 
 6340   ins_cost(100);
 6341   format %{ "movq    $dst, $src\t# ptr stk" %}
 6342   opcode(0x89);
 6343   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6344   ins_pipe(ialu_mem_reg);
 6345 %}
 6346 
 6347 instruct storeSSF(stackSlotF dst, regF src)
 6348 %{
 6349   match(Set dst src);
 6350 
 6351   ins_cost(95); // XXX
 6352   format %{ "movss   $dst, $src\t# float stk" %}
 6353   ins_encode %{
 6354     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6355   %}
 6356   ins_pipe(pipe_slow); // XXX
 6357 %}
 6358 
 6359 instruct storeSSD(stackSlotD dst, regD src)
 6360 %{
 6361   match(Set dst src);
 6362 
 6363   ins_cost(95); // XXX
 6364   format %{ "movsd   $dst, $src\t# double stk" %}
 6365   ins_encode %{
 6366     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6367   %}
 6368   ins_pipe(pipe_slow); // XXX
 6369 %}
 6370 
 6371 instruct cacheWB(indirect addr)
 6372 %{
 6373   predicate(VM_Version::supports_data_cache_line_flush());
 6374   match(CacheWB addr);
 6375 
 6376   ins_cost(100);
 6377   format %{"cache wb $addr" %}
 6378   ins_encode %{
 6379     assert($addr->index_position() < 0, "should be");
 6380     assert($addr$$disp == 0, "should be");
 6381     __ cache_wb(Address($addr$$base$$Register, 0));
 6382   %}
 6383   ins_pipe(pipe_slow); // XXX
 6384 %}
 6385 
 6386 instruct cacheWBPreSync()
 6387 %{
 6388   predicate(VM_Version::supports_data_cache_line_flush());
 6389   match(CacheWBPreSync);
 6390 
 6391   ins_cost(100);
 6392   format %{"cache wb presync" %}
 6393   ins_encode %{
 6394     __ cache_wbsync(true);
 6395   %}
 6396   ins_pipe(pipe_slow); // XXX
 6397 %}
 6398 
 6399 instruct cacheWBPostSync()
 6400 %{
 6401   predicate(VM_Version::supports_data_cache_line_flush());
 6402   match(CacheWBPostSync);
 6403 
 6404   ins_cost(100);
 6405   format %{"cache wb postsync" %}
 6406   ins_encode %{
 6407     __ cache_wbsync(false);
 6408   %}
 6409   ins_pipe(pipe_slow); // XXX
 6410 %}
 6411 
 6412 //----------BSWAP Instructions-------------------------------------------------
 6413 instruct bytes_reverse_int(rRegI dst) %{
 6414   match(Set dst (ReverseBytesI dst));
 6415 
 6416   format %{ "bswapl  $dst" %}
 6417   ins_encode %{
 6418     __ bswapl($dst$$Register);
 6419   %}
 6420   ins_pipe( ialu_reg );
 6421 %}
 6422 
 6423 instruct bytes_reverse_long(rRegL dst) %{
 6424   match(Set dst (ReverseBytesL dst));
 6425 
 6426   format %{ "bswapq  $dst" %}
 6427   ins_encode %{
 6428     __ bswapq($dst$$Register);
 6429   %}
 6430   ins_pipe( ialu_reg);
 6431 %}
 6432 
 6433 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6434   match(Set dst (ReverseBytesUS dst));
 6435   effect(KILL cr);
 6436 
 6437   format %{ "bswapl  $dst\n\t"
 6438             "shrl    $dst,16\n\t" %}
 6439   ins_encode %{
 6440     __ bswapl($dst$$Register);
 6441     __ shrl($dst$$Register, 16);
 6442   %}
 6443   ins_pipe( ialu_reg );
 6444 %}
 6445 
 6446 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6447   match(Set dst (ReverseBytesS dst));
 6448   effect(KILL cr);
 6449 
 6450   format %{ "bswapl  $dst\n\t"
 6451             "sar     $dst,16\n\t" %}
 6452   ins_encode %{
 6453     __ bswapl($dst$$Register);
 6454     __ sarl($dst$$Register, 16);
 6455   %}
 6456   ins_pipe( ialu_reg );
 6457 %}
 6458 
 6459 //---------- Zeros Count Instructions ------------------------------------------
 6460 
 6461 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6462   predicate(UseCountLeadingZerosInstruction);
 6463   match(Set dst (CountLeadingZerosI src));
 6464   effect(KILL cr);
 6465 
 6466   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6467   ins_encode %{
 6468     __ lzcntl($dst$$Register, $src$$Register);
 6469   %}
 6470   ins_pipe(ialu_reg);
 6471 %}
 6472 
 6473 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6474   predicate(!UseCountLeadingZerosInstruction);
 6475   match(Set dst (CountLeadingZerosI src));
 6476   effect(KILL cr);
 6477 
 6478   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6479             "jnz     skip\n\t"
 6480             "movl    $dst, -1\n"
 6481       "skip:\n\t"
 6482             "negl    $dst\n\t"
 6483             "addl    $dst, 31" %}
 6484   ins_encode %{
 6485     Register Rdst = $dst$$Register;
 6486     Register Rsrc = $src$$Register;
 6487     Label skip;
 6488     __ bsrl(Rdst, Rsrc);
 6489     __ jccb(Assembler::notZero, skip);
 6490     __ movl(Rdst, -1);
 6491     __ bind(skip);
 6492     __ negl(Rdst);
 6493     __ addl(Rdst, BitsPerInt - 1);
 6494   %}
 6495   ins_pipe(ialu_reg);
 6496 %}
 6497 
 6498 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6499   predicate(UseCountLeadingZerosInstruction);
 6500   match(Set dst (CountLeadingZerosL src));
 6501   effect(KILL cr);
 6502 
 6503   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6504   ins_encode %{
 6505     __ lzcntq($dst$$Register, $src$$Register);
 6506   %}
 6507   ins_pipe(ialu_reg);
 6508 %}
 6509 
 6510 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6511   predicate(!UseCountLeadingZerosInstruction);
 6512   match(Set dst (CountLeadingZerosL src));
 6513   effect(KILL cr);
 6514 
 6515   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6516             "jnz     skip\n\t"
 6517             "movl    $dst, -1\n"
 6518       "skip:\n\t"
 6519             "negl    $dst\n\t"
 6520             "addl    $dst, 63" %}
 6521   ins_encode %{
 6522     Register Rdst = $dst$$Register;
 6523     Register Rsrc = $src$$Register;
 6524     Label skip;
 6525     __ bsrq(Rdst, Rsrc);
 6526     __ jccb(Assembler::notZero, skip);
 6527     __ movl(Rdst, -1);
 6528     __ bind(skip);
 6529     __ negl(Rdst);
 6530     __ addl(Rdst, BitsPerLong - 1);
 6531   %}
 6532   ins_pipe(ialu_reg);
 6533 %}
 6534 
 6535 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6536   predicate(UseCountTrailingZerosInstruction);
 6537   match(Set dst (CountTrailingZerosI src));
 6538   effect(KILL cr);
 6539 
 6540   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6541   ins_encode %{
 6542     __ tzcntl($dst$$Register, $src$$Register);
 6543   %}
 6544   ins_pipe(ialu_reg);
 6545 %}
 6546 
 6547 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6548   predicate(!UseCountTrailingZerosInstruction);
 6549   match(Set dst (CountTrailingZerosI src));
 6550   effect(KILL cr);
 6551 
 6552   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6553             "jnz     done\n\t"
 6554             "movl    $dst, 32\n"
 6555       "done:" %}
 6556   ins_encode %{
 6557     Register Rdst = $dst$$Register;
 6558     Label done;
 6559     __ bsfl(Rdst, $src$$Register);
 6560     __ jccb(Assembler::notZero, done);
 6561     __ movl(Rdst, BitsPerInt);
 6562     __ bind(done);
 6563   %}
 6564   ins_pipe(ialu_reg);
 6565 %}
 6566 
 6567 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6568   predicate(UseCountTrailingZerosInstruction);
 6569   match(Set dst (CountTrailingZerosL src));
 6570   effect(KILL cr);
 6571 
 6572   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6573   ins_encode %{
 6574     __ tzcntq($dst$$Register, $src$$Register);
 6575   %}
 6576   ins_pipe(ialu_reg);
 6577 %}
 6578 
 6579 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6580   predicate(!UseCountTrailingZerosInstruction);
 6581   match(Set dst (CountTrailingZerosL src));
 6582   effect(KILL cr);
 6583 
 6584   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6585             "jnz     done\n\t"
 6586             "movl    $dst, 64\n"
 6587       "done:" %}
 6588   ins_encode %{
 6589     Register Rdst = $dst$$Register;
 6590     Label done;
 6591     __ bsfq(Rdst, $src$$Register);
 6592     __ jccb(Assembler::notZero, done);
 6593     __ movl(Rdst, BitsPerLong);
 6594     __ bind(done);
 6595   %}
 6596   ins_pipe(ialu_reg);
 6597 %}
 6598 
 6599 
 6600 //---------- Population Count Instructions -------------------------------------
 6601 
 6602 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6603   predicate(UsePopCountInstruction);
 6604   match(Set dst (PopCountI src));
 6605   effect(KILL cr);
 6606 
 6607   format %{ "popcnt  $dst, $src" %}
 6608   ins_encode %{
 6609     __ popcntl($dst$$Register, $src$$Register);
 6610   %}
 6611   ins_pipe(ialu_reg);
 6612 %}
 6613 
 6614 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6615   predicate(UsePopCountInstruction);
 6616   match(Set dst (PopCountI (LoadI mem)));
 6617   effect(KILL cr);
 6618 
 6619   format %{ "popcnt  $dst, $mem" %}
 6620   ins_encode %{
 6621     __ popcntl($dst$$Register, $mem$$Address);
 6622   %}
 6623   ins_pipe(ialu_reg);
 6624 %}
 6625 
 6626 // Note: Long.bitCount(long) returns an int.
 6627 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6628   predicate(UsePopCountInstruction);
 6629   match(Set dst (PopCountL src));
 6630   effect(KILL cr);
 6631 
 6632   format %{ "popcnt  $dst, $src" %}
 6633   ins_encode %{
 6634     __ popcntq($dst$$Register, $src$$Register);
 6635   %}
 6636   ins_pipe(ialu_reg);
 6637 %}
 6638 
 6639 // Note: Long.bitCount(long) returns an int.
 6640 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6641   predicate(UsePopCountInstruction);
 6642   match(Set dst (PopCountL (LoadL mem)));
 6643   effect(KILL cr);
 6644 
 6645   format %{ "popcnt  $dst, $mem" %}
 6646   ins_encode %{
 6647     __ popcntq($dst$$Register, $mem$$Address);
 6648   %}
 6649   ins_pipe(ialu_reg);
 6650 %}
 6651 
 6652 
 6653 //----------MemBar Instructions-----------------------------------------------
 6654 // Memory barrier flavors
 6655 
 6656 instruct membar_acquire()
 6657 %{
 6658   match(MemBarAcquire);
 6659   match(LoadFence);
 6660   ins_cost(0);
 6661 
 6662   size(0);
 6663   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6664   ins_encode();
 6665   ins_pipe(empty);
 6666 %}
 6667 
 6668 instruct membar_acquire_lock()
 6669 %{
 6670   match(MemBarAcquireLock);
 6671   ins_cost(0);
 6672 
 6673   size(0);
 6674   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6675   ins_encode();
 6676   ins_pipe(empty);
 6677 %}
 6678 
 6679 instruct membar_release()
 6680 %{
 6681   match(MemBarRelease);
 6682   match(StoreFence);
 6683   ins_cost(0);
 6684 
 6685   size(0);
 6686   format %{ "MEMBAR-release ! (empty encoding)" %}
 6687   ins_encode();
 6688   ins_pipe(empty);
 6689 %}
 6690 
 6691 instruct membar_release_lock()
 6692 %{
 6693   match(MemBarReleaseLock);
 6694   ins_cost(0);
 6695 
 6696   size(0);
 6697   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6698   ins_encode();
 6699   ins_pipe(empty);
 6700 %}
 6701 
 6702 instruct membar_volatile(rFlagsReg cr) %{
 6703   match(MemBarVolatile);
 6704   effect(KILL cr);
 6705   ins_cost(400);
 6706 
 6707   format %{
 6708     $$template
 6709     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6710   %}
 6711   ins_encode %{
 6712     __ membar(Assembler::StoreLoad);
 6713   %}
 6714   ins_pipe(pipe_slow);
 6715 %}
 6716 
 6717 instruct unnecessary_membar_volatile()
 6718 %{
 6719   match(MemBarVolatile);
 6720   predicate(Matcher::post_store_load_barrier(n));
 6721   ins_cost(0);
 6722 
 6723   size(0);
 6724   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6725   ins_encode();
 6726   ins_pipe(empty);
 6727 %}
 6728 
 6729 instruct membar_storestore() %{
 6730   match(MemBarStoreStore);
 6731   match(StoreStoreFence);
 6732   ins_cost(0);
 6733 
 6734   size(0);
 6735   format %{ "MEMBAR-storestore (empty encoding)" %}
 6736   ins_encode( );
 6737   ins_pipe(empty);
 6738 %}
 6739 
 6740 //----------Move Instructions--------------------------------------------------
 6741 
 6742 instruct castX2P(rRegP dst, rRegL src)
 6743 %{
 6744   match(Set dst (CastX2P src));
 6745 
 6746   format %{ "movq    $dst, $src\t# long->ptr" %}
 6747   ins_encode %{
 6748     if ($dst$$reg != $src$$reg) {
 6749       __ movptr($dst$$Register, $src$$Register);
 6750     }
 6751   %}
 6752   ins_pipe(ialu_reg_reg); // XXX
 6753 %}
 6754 
 6755 instruct castP2X(rRegL dst, rRegP src)
 6756 %{
 6757   match(Set dst (CastP2X src));
 6758 
 6759   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6760   ins_encode %{
 6761     if ($dst$$reg != $src$$reg) {
 6762       __ movptr($dst$$Register, $src$$Register);
 6763     }
 6764   %}
 6765   ins_pipe(ialu_reg_reg); // XXX
 6766 %}
 6767 
 6768 // Convert oop into int for vectors alignment masking
 6769 instruct convP2I(rRegI dst, rRegP src)
 6770 %{
 6771   match(Set dst (ConvL2I (CastP2X src)));
 6772 
 6773   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6774   ins_encode %{
 6775     __ movl($dst$$Register, $src$$Register);
 6776   %}
 6777   ins_pipe(ialu_reg_reg); // XXX
 6778 %}
 6779 
 6780 // Convert compressed oop into int for vectors alignment masking
 6781 // in case of 32bit oops (heap < 4Gb).
 6782 instruct convN2I(rRegI dst, rRegN src)
 6783 %{
 6784   predicate(CompressedOops::shift() == 0);
 6785   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6786 
 6787   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6788   ins_encode %{
 6789     __ movl($dst$$Register, $src$$Register);
 6790   %}
 6791   ins_pipe(ialu_reg_reg); // XXX
 6792 %}
 6793 
 6794 // Convert oop pointer into compressed form
 6795 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6796   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6797   match(Set dst (EncodeP src));
 6798   effect(KILL cr);
 6799   format %{ "encode_heap_oop $dst,$src" %}
 6800   ins_encode %{
 6801     Register s = $src$$Register;
 6802     Register d = $dst$$Register;
 6803     if (s != d) {
 6804       __ movq(d, s);
 6805     }
 6806     __ encode_heap_oop(d);
 6807   %}
 6808   ins_pipe(ialu_reg_long);
 6809 %}
 6810 
 6811 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6812   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6813   match(Set dst (EncodeP src));
 6814   effect(KILL cr);
 6815   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6816   ins_encode %{
 6817     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6818   %}
 6819   ins_pipe(ialu_reg_long);
 6820 %}
 6821 
 6822 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6823   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6824             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6825   match(Set dst (DecodeN src));
 6826   effect(KILL cr);
 6827   format %{ "decode_heap_oop $dst,$src" %}
 6828   ins_encode %{
 6829     Register s = $src$$Register;
 6830     Register d = $dst$$Register;
 6831     if (s != d) {
 6832       __ movq(d, s);
 6833     }
 6834     __ decode_heap_oop(d);
 6835   %}
 6836   ins_pipe(ialu_reg_long);
 6837 %}
 6838 
 6839 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6840   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6841             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6842   match(Set dst (DecodeN src));
 6843   effect(KILL cr);
 6844   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6845   ins_encode %{
 6846     Register s = $src$$Register;
 6847     Register d = $dst$$Register;
 6848     if (s != d) {
 6849       __ decode_heap_oop_not_null(d, s);
 6850     } else {
 6851       __ decode_heap_oop_not_null(d);
 6852     }
 6853   %}
 6854   ins_pipe(ialu_reg_long);
 6855 %}
 6856 
 6857 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6858   match(Set dst (EncodePKlass src));
 6859   effect(TEMP dst, KILL cr);
 6860   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6861   ins_encode %{
 6862     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6863   %}
 6864   ins_pipe(ialu_reg_long);
 6865 %}
 6866 
 6867 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6868   match(Set dst (DecodeNKlass src));
 6869   effect(TEMP dst, KILL cr);
 6870   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6871   ins_encode %{
 6872     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6873   %}
 6874   ins_pipe(ialu_reg_long);
 6875 %}
 6876 
 6877 //----------Conditional Move---------------------------------------------------
 6878 // Jump
 6879 // dummy instruction for generating temp registers
 6880 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6881   match(Jump (LShiftL switch_val shift));
 6882   ins_cost(350);
 6883   predicate(false);
 6884   effect(TEMP dest);
 6885 
 6886   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6887             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6888   ins_encode %{
 6889     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6890     // to do that and the compiler is using that register as one it can allocate.
 6891     // So we build it all by hand.
 6892     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6893     // ArrayAddress dispatch(table, index);
 6894     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6895     __ lea($dest$$Register, $constantaddress);
 6896     __ jmp(dispatch);
 6897   %}
 6898   ins_pipe(pipe_jmp);
 6899 %}
 6900 
 6901 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6902   match(Jump (AddL (LShiftL switch_val shift) offset));
 6903   ins_cost(350);
 6904   effect(TEMP dest);
 6905 
 6906   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6907             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6908   ins_encode %{
 6909     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6910     // to do that and the compiler is using that register as one it can allocate.
 6911     // So we build it all by hand.
 6912     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6913     // ArrayAddress dispatch(table, index);
 6914     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6915     __ lea($dest$$Register, $constantaddress);
 6916     __ jmp(dispatch);
 6917   %}
 6918   ins_pipe(pipe_jmp);
 6919 %}
 6920 
 6921 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6922   match(Jump switch_val);
 6923   ins_cost(350);
 6924   effect(TEMP dest);
 6925 
 6926   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6927             "jmp     [$dest + $switch_val]\n\t" %}
 6928   ins_encode %{
 6929     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6930     // to do that and the compiler is using that register as one it can allocate.
 6931     // So we build it all by hand.
 6932     // Address index(noreg, switch_reg, Address::times_1);
 6933     // ArrayAddress dispatch(table, index);
 6934     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6935     __ lea($dest$$Register, $constantaddress);
 6936     __ jmp(dispatch);
 6937   %}
 6938   ins_pipe(pipe_jmp);
 6939 %}
 6940 
 6941 // Conditional move
 6942 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6943 %{
 6944   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6945 
 6946   ins_cost(200); // XXX
 6947   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6948   ins_encode %{
 6949     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6950   %}
 6951   ins_pipe(pipe_cmov_reg);
 6952 %}
 6953 
 6954 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6955   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6956 
 6957   ins_cost(200); // XXX
 6958   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6959   ins_encode %{
 6960     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6961   %}
 6962   ins_pipe(pipe_cmov_reg);
 6963 %}
 6964 
 6965 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6966   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6967   ins_cost(200);
 6968   expand %{
 6969     cmovI_regU(cop, cr, dst, src);
 6970   %}
 6971 %}
 6972 
 6973 // Conditional move
 6974 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6975   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6976 
 6977   ins_cost(250); // XXX
 6978   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6979   ins_encode %{
 6980     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6981   %}
 6982   ins_pipe(pipe_cmov_mem);
 6983 %}
 6984 
 6985 // Conditional move
 6986 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6987 %{
 6988   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6989 
 6990   ins_cost(250); // XXX
 6991   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6992   ins_encode %{
 6993     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6994   %}
 6995   ins_pipe(pipe_cmov_mem);
 6996 %}
 6997 
 6998 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6999   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7000   ins_cost(250);
 7001   expand %{
 7002     cmovI_memU(cop, cr, dst, src);
 7003   %}
 7004 %}
 7005 
 7006 // Conditional move
 7007 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7008 %{
 7009   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7010 
 7011   ins_cost(200); // XXX
 7012   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7013   ins_encode %{
 7014     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7015   %}
 7016   ins_pipe(pipe_cmov_reg);
 7017 %}
 7018 
 7019 // Conditional move
 7020 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7021 %{
 7022   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7023 
 7024   ins_cost(200); // XXX
 7025   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7026   ins_encode %{
 7027     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7028   %}
 7029   ins_pipe(pipe_cmov_reg);
 7030 %}
 7031 
 7032 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7033   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7034   ins_cost(200);
 7035   expand %{
 7036     cmovN_regU(cop, cr, dst, src);
 7037   %}
 7038 %}
 7039 
 7040 // Conditional move
 7041 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7042 %{
 7043   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7044 
 7045   ins_cost(200); // XXX
 7046   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7047   ins_encode %{
 7048     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7049   %}
 7050   ins_pipe(pipe_cmov_reg);  // XXX
 7051 %}
 7052 
 7053 // Conditional move
 7054 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7055 %{
 7056   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7057 
 7058   ins_cost(200); // XXX
 7059   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7060   ins_encode %{
 7061     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7062   %}
 7063   ins_pipe(pipe_cmov_reg); // XXX
 7064 %}
 7065 
 7066 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7067   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7068   ins_cost(200);
 7069   expand %{
 7070     cmovP_regU(cop, cr, dst, src);
 7071   %}
 7072 %}
 7073 
 7074 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7075 // correctly meets the two pointer arguments; one is an incoming
 7076 // register but the other is a memory operand.  ALSO appears to
 7077 // be buggy with implicit null checks.
 7078 //
 7079 //// Conditional move
 7080 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7081 //%{
 7082 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7083 //  ins_cost(250);
 7084 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7085 //  opcode(0x0F,0x40);
 7086 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7087 //  ins_pipe( pipe_cmov_mem );
 7088 //%}
 7089 //
 7090 //// Conditional move
 7091 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7092 //%{
 7093 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7094 //  ins_cost(250);
 7095 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7096 //  opcode(0x0F,0x40);
 7097 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7098 //  ins_pipe( pipe_cmov_mem );
 7099 //%}
 7100 
 7101 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7102 %{
 7103   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7104 
 7105   ins_cost(200); // XXX
 7106   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7107   ins_encode %{
 7108     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7109   %}
 7110   ins_pipe(pipe_cmov_reg);  // XXX
 7111 %}
 7112 
 7113 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7114 %{
 7115   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7116 
 7117   ins_cost(200); // XXX
 7118   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7119   ins_encode %{
 7120     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7121   %}
 7122   ins_pipe(pipe_cmov_mem);  // XXX
 7123 %}
 7124 
 7125 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7126 %{
 7127   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7128 
 7129   ins_cost(200); // XXX
 7130   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7131   ins_encode %{
 7132     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7133   %}
 7134   ins_pipe(pipe_cmov_reg); // XXX
 7135 %}
 7136 
 7137 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7138   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7139   ins_cost(200);
 7140   expand %{
 7141     cmovL_regU(cop, cr, dst, src);
 7142   %}
 7143 %}
 7144 
 7145 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7146 %{
 7147   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7148 
 7149   ins_cost(200); // XXX
 7150   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7151   ins_encode %{
 7152     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7153   %}
 7154   ins_pipe(pipe_cmov_mem); // XXX
 7155 %}
 7156 
 7157 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7158   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7159   ins_cost(200);
 7160   expand %{
 7161     cmovL_memU(cop, cr, dst, src);
 7162   %}
 7163 %}
 7164 
 7165 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7166 %{
 7167   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7168 
 7169   ins_cost(200); // XXX
 7170   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7171             "movss     $dst, $src\n"
 7172     "skip:" %}
 7173   ins_encode %{
 7174     Label Lskip;
 7175     // Invert sense of branch from sense of CMOV
 7176     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7177     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7178     __ bind(Lskip);
 7179   %}
 7180   ins_pipe(pipe_slow);
 7181 %}
 7182 
 7183 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7184 // %{
 7185 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7186 
 7187 //   ins_cost(200); // XXX
 7188 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7189 //             "movss     $dst, $src\n"
 7190 //     "skip:" %}
 7191 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7192 //   ins_pipe(pipe_slow);
 7193 // %}
 7194 
 7195 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7196 %{
 7197   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7198 
 7199   ins_cost(200); // XXX
 7200   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7201             "movss     $dst, $src\n"
 7202     "skip:" %}
 7203   ins_encode %{
 7204     Label Lskip;
 7205     // Invert sense of branch from sense of CMOV
 7206     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7207     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7208     __ bind(Lskip);
 7209   %}
 7210   ins_pipe(pipe_slow);
 7211 %}
 7212 
 7213 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7214   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7215   ins_cost(200);
 7216   expand %{
 7217     cmovF_regU(cop, cr, dst, src);
 7218   %}
 7219 %}
 7220 
 7221 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7222 %{
 7223   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7224 
 7225   ins_cost(200); // XXX
 7226   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7227             "movsd     $dst, $src\n"
 7228     "skip:" %}
 7229   ins_encode %{
 7230     Label Lskip;
 7231     // Invert sense of branch from sense of CMOV
 7232     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7233     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7234     __ bind(Lskip);
 7235   %}
 7236   ins_pipe(pipe_slow);
 7237 %}
 7238 
 7239 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7240 %{
 7241   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7242 
 7243   ins_cost(200); // XXX
 7244   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7245             "movsd     $dst, $src\n"
 7246     "skip:" %}
 7247   ins_encode %{
 7248     Label Lskip;
 7249     // Invert sense of branch from sense of CMOV
 7250     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7251     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7252     __ bind(Lskip);
 7253   %}
 7254   ins_pipe(pipe_slow);
 7255 %}
 7256 
 7257 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7258   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7259   ins_cost(200);
 7260   expand %{
 7261     cmovD_regU(cop, cr, dst, src);
 7262   %}
 7263 %}
 7264 
 7265 //----------Arithmetic Instructions--------------------------------------------
 7266 //----------Addition Instructions----------------------------------------------
 7267 
 7268 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7269 %{
 7270   match(Set dst (AddI dst src));
 7271   effect(KILL cr);
 7272 
 7273   format %{ "addl    $dst, $src\t# int" %}
 7274   ins_encode %{
 7275     __ addl($dst$$Register, $src$$Register);
 7276   %}
 7277   ins_pipe(ialu_reg_reg);
 7278 %}
 7279 
 7280 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7281 %{
 7282   match(Set dst (AddI dst src));
 7283   effect(KILL cr);
 7284 
 7285   format %{ "addl    $dst, $src\t# int" %}
 7286   ins_encode %{
 7287     __ addl($dst$$Register, $src$$constant);
 7288   %}
 7289   ins_pipe( ialu_reg );
 7290 %}
 7291 
 7292 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7293 %{
 7294   match(Set dst (AddI dst (LoadI src)));
 7295   effect(KILL cr);
 7296 
 7297   ins_cost(125); // XXX
 7298   format %{ "addl    $dst, $src\t# int" %}
 7299   ins_encode %{
 7300     __ addl($dst$$Register, $src$$Address);
 7301   %}
 7302   ins_pipe(ialu_reg_mem);
 7303 %}
 7304 
 7305 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7306 %{
 7307   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7308   effect(KILL cr);
 7309 
 7310   ins_cost(150); // XXX
 7311   format %{ "addl    $dst, $src\t# int" %}
 7312   ins_encode %{
 7313     __ addl($dst$$Address, $src$$Register);
 7314   %}
 7315   ins_pipe(ialu_mem_reg);
 7316 %}
 7317 
 7318 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7319 %{
 7320   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7321   effect(KILL cr);
 7322 
 7323   ins_cost(125); // XXX
 7324   format %{ "addl    $dst, $src\t# int" %}
 7325   ins_encode %{
 7326     __ addl($dst$$Address, $src$$constant);
 7327   %}
 7328   ins_pipe(ialu_mem_imm);
 7329 %}
 7330 
 7331 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7332 %{
 7333   predicate(UseIncDec);
 7334   match(Set dst (AddI dst src));
 7335   effect(KILL cr);
 7336 
 7337   format %{ "incl    $dst\t# int" %}
 7338   ins_encode %{
 7339     __ incrementl($dst$$Register);
 7340   %}
 7341   ins_pipe(ialu_reg);
 7342 %}
 7343 
 7344 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7345 %{
 7346   predicate(UseIncDec);
 7347   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7348   effect(KILL cr);
 7349 
 7350   ins_cost(125); // XXX
 7351   format %{ "incl    $dst\t# int" %}
 7352   ins_encode %{
 7353     __ incrementl($dst$$Address);
 7354   %}
 7355   ins_pipe(ialu_mem_imm);
 7356 %}
 7357 
 7358 // XXX why does that use AddI
 7359 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7360 %{
 7361   predicate(UseIncDec);
 7362   match(Set dst (AddI dst src));
 7363   effect(KILL cr);
 7364 
 7365   format %{ "decl    $dst\t# int" %}
 7366   ins_encode %{
 7367     __ decrementl($dst$$Register);
 7368   %}
 7369   ins_pipe(ialu_reg);
 7370 %}
 7371 
 7372 // XXX why does that use AddI
 7373 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7374 %{
 7375   predicate(UseIncDec);
 7376   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7377   effect(KILL cr);
 7378 
 7379   ins_cost(125); // XXX
 7380   format %{ "decl    $dst\t# int" %}
 7381   ins_encode %{
 7382     __ decrementl($dst$$Address);
 7383   %}
 7384   ins_pipe(ialu_mem_imm);
 7385 %}
 7386 
 7387 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7388 %{
 7389   predicate(VM_Version::supports_fast_2op_lea());
 7390   match(Set dst (AddI (LShiftI index scale) disp));
 7391 
 7392   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7393   ins_encode %{
 7394     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7395     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7396   %}
 7397   ins_pipe(ialu_reg_reg);
 7398 %}
 7399 
 7400 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7401 %{
 7402   predicate(VM_Version::supports_fast_3op_lea());
 7403   match(Set dst (AddI (AddI base index) disp));
 7404 
 7405   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7406   ins_encode %{
 7407     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7408   %}
 7409   ins_pipe(ialu_reg_reg);
 7410 %}
 7411 
 7412 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7413 %{
 7414   predicate(VM_Version::supports_fast_2op_lea());
 7415   match(Set dst (AddI base (LShiftI index scale)));
 7416 
 7417   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7418   ins_encode %{
 7419     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7420     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7421   %}
 7422   ins_pipe(ialu_reg_reg);
 7423 %}
 7424 
 7425 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7426 %{
 7427   predicate(VM_Version::supports_fast_3op_lea());
 7428   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7429 
 7430   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7431   ins_encode %{
 7432     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7433     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7434   %}
 7435   ins_pipe(ialu_reg_reg);
 7436 %}
 7437 
 7438 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7439 %{
 7440   match(Set dst (AddL dst src));
 7441   effect(KILL cr);
 7442 
 7443   format %{ "addq    $dst, $src\t# long" %}
 7444   ins_encode %{
 7445     __ addq($dst$$Register, $src$$Register);
 7446   %}
 7447   ins_pipe(ialu_reg_reg);
 7448 %}
 7449 
 7450 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7451 %{
 7452   match(Set dst (AddL dst src));
 7453   effect(KILL cr);
 7454 
 7455   format %{ "addq    $dst, $src\t# long" %}
 7456   ins_encode %{
 7457     __ addq($dst$$Register, $src$$constant);
 7458   %}
 7459   ins_pipe( ialu_reg );
 7460 %}
 7461 
 7462 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7463 %{
 7464   match(Set dst (AddL dst (LoadL src)));
 7465   effect(KILL cr);
 7466 
 7467   ins_cost(125); // XXX
 7468   format %{ "addq    $dst, $src\t# long" %}
 7469   ins_encode %{
 7470     __ addq($dst$$Register, $src$$Address);
 7471   %}
 7472   ins_pipe(ialu_reg_mem);
 7473 %}
 7474 
 7475 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7476 %{
 7477   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7478   effect(KILL cr);
 7479 
 7480   ins_cost(150); // XXX
 7481   format %{ "addq    $dst, $src\t# long" %}
 7482   ins_encode %{
 7483     __ addq($dst$$Address, $src$$Register);
 7484   %}
 7485   ins_pipe(ialu_mem_reg);
 7486 %}
 7487 
 7488 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7489 %{
 7490   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7491   effect(KILL cr);
 7492 
 7493   ins_cost(125); // XXX
 7494   format %{ "addq    $dst, $src\t# long" %}
 7495   ins_encode %{
 7496     __ addq($dst$$Address, $src$$constant);
 7497   %}
 7498   ins_pipe(ialu_mem_imm);
 7499 %}
 7500 
 7501 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7502 %{
 7503   predicate(UseIncDec);
 7504   match(Set dst (AddL dst src));
 7505   effect(KILL cr);
 7506 
 7507   format %{ "incq    $dst\t# long" %}
 7508   ins_encode %{
 7509     __ incrementq($dst$$Register);
 7510   %}
 7511   ins_pipe(ialu_reg);
 7512 %}
 7513 
 7514 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7515 %{
 7516   predicate(UseIncDec);
 7517   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7518   effect(KILL cr);
 7519 
 7520   ins_cost(125); // XXX
 7521   format %{ "incq    $dst\t# long" %}
 7522   ins_encode %{
 7523     __ incrementq($dst$$Address);
 7524   %}
 7525   ins_pipe(ialu_mem_imm);
 7526 %}
 7527 
 7528 // XXX why does that use AddL
 7529 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7530 %{
 7531   predicate(UseIncDec);
 7532   match(Set dst (AddL dst src));
 7533   effect(KILL cr);
 7534 
 7535   format %{ "decq    $dst\t# long" %}
 7536   ins_encode %{
 7537     __ decrementq($dst$$Register);
 7538   %}
 7539   ins_pipe(ialu_reg);
 7540 %}
 7541 
 7542 // XXX why does that use AddL
 7543 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7544 %{
 7545   predicate(UseIncDec);
 7546   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7547   effect(KILL cr);
 7548 
 7549   ins_cost(125); // XXX
 7550   format %{ "decq    $dst\t# long" %}
 7551   ins_encode %{
 7552     __ decrementq($dst$$Address);
 7553   %}
 7554   ins_pipe(ialu_mem_imm);
 7555 %}
 7556 
 7557 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7558 %{
 7559   predicate(VM_Version::supports_fast_2op_lea());
 7560   match(Set dst (AddL (LShiftL index scale) disp));
 7561 
 7562   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7563   ins_encode %{
 7564     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7565     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7566   %}
 7567   ins_pipe(ialu_reg_reg);
 7568 %}
 7569 
 7570 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7571 %{
 7572   predicate(VM_Version::supports_fast_3op_lea());
 7573   match(Set dst (AddL (AddL base index) disp));
 7574 
 7575   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7576   ins_encode %{
 7577     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7578   %}
 7579   ins_pipe(ialu_reg_reg);
 7580 %}
 7581 
 7582 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7583 %{
 7584   predicate(VM_Version::supports_fast_2op_lea());
 7585   match(Set dst (AddL base (LShiftL index scale)));
 7586 
 7587   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7588   ins_encode %{
 7589     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7590     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7591   %}
 7592   ins_pipe(ialu_reg_reg);
 7593 %}
 7594 
 7595 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7596 %{
 7597   predicate(VM_Version::supports_fast_3op_lea());
 7598   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7599 
 7600   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7601   ins_encode %{
 7602     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7603     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7604   %}
 7605   ins_pipe(ialu_reg_reg);
 7606 %}
 7607 
 7608 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7609 %{
 7610   match(Set dst (AddP dst src));
 7611   effect(KILL cr);
 7612 
 7613   format %{ "addq    $dst, $src\t# ptr" %}
 7614   ins_encode %{
 7615     __ addq($dst$$Register, $src$$Register);
 7616   %}
 7617   ins_pipe(ialu_reg_reg);
 7618 %}
 7619 
 7620 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7621 %{
 7622   match(Set dst (AddP dst src));
 7623   effect(KILL cr);
 7624 
 7625   format %{ "addq    $dst, $src\t# ptr" %}
 7626   ins_encode %{
 7627     __ addq($dst$$Register, $src$$constant);
 7628   %}
 7629   ins_pipe( ialu_reg );
 7630 %}
 7631 
 7632 // XXX addP mem ops ????
 7633 
 7634 instruct checkCastPP(rRegP dst)
 7635 %{
 7636   match(Set dst (CheckCastPP dst));
 7637 
 7638   size(0);
 7639   format %{ "# checkcastPP of $dst" %}
 7640   ins_encode(/* empty encoding */);
 7641   ins_pipe(empty);
 7642 %}
 7643 
 7644 instruct castPP(rRegP dst)
 7645 %{
 7646   match(Set dst (CastPP dst));
 7647 
 7648   size(0);
 7649   format %{ "# castPP of $dst" %}
 7650   ins_encode(/* empty encoding */);
 7651   ins_pipe(empty);
 7652 %}
 7653 
 7654 instruct castII(rRegI dst)
 7655 %{
 7656   match(Set dst (CastII dst));
 7657 
 7658   size(0);
 7659   format %{ "# castII of $dst" %}
 7660   ins_encode(/* empty encoding */);
 7661   ins_cost(0);
 7662   ins_pipe(empty);
 7663 %}
 7664 
 7665 instruct castLL(rRegL dst)
 7666 %{
 7667   match(Set dst (CastLL dst));
 7668 
 7669   size(0);
 7670   format %{ "# castLL of $dst" %}
 7671   ins_encode(/* empty encoding */);
 7672   ins_cost(0);
 7673   ins_pipe(empty);
 7674 %}
 7675 
 7676 instruct castFF(regF dst)
 7677 %{
 7678   match(Set dst (CastFF dst));
 7679 
 7680   size(0);
 7681   format %{ "# castFF of $dst" %}
 7682   ins_encode(/* empty encoding */);
 7683   ins_cost(0);
 7684   ins_pipe(empty);
 7685 %}
 7686 
 7687 instruct castDD(regD dst)
 7688 %{
 7689   match(Set dst (CastDD dst));
 7690 
 7691   size(0);
 7692   format %{ "# castDD of $dst" %}
 7693   ins_encode(/* empty encoding */);
 7694   ins_cost(0);
 7695   ins_pipe(empty);
 7696 %}
 7697 
 7698 // LoadP-locked same as a regular LoadP when used with compare-swap
 7699 instruct loadPLocked(rRegP dst, memory mem)
 7700 %{
 7701   match(Set dst (LoadPLocked mem));
 7702 
 7703   ins_cost(125); // XXX
 7704   format %{ "movq    $dst, $mem\t# ptr locked" %}
 7705   ins_encode %{
 7706     __ movq($dst$$Register, $mem$$Address);
 7707   %}
 7708   ins_pipe(ialu_reg_mem); // XXX
 7709 %}
 7710 
 7711 // Conditional-store of the updated heap-top.
 7712 // Used during allocation of the shared heap.
 7713 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
 7714 
 7715 instruct storePConditional(memory heap_top_ptr,
 7716                            rax_RegP oldval, rRegP newval,
 7717                            rFlagsReg cr)
 7718 %{
 7719   predicate(n->as_LoadStore()->barrier_data() == 0);
 7720   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 7721 
 7722   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
 7723             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
 7724   ins_encode %{
 7725     __ lock();
 7726     __ cmpxchgq($newval$$Register, $heap_top_ptr$$Address);
 7727   %}
 7728   ins_pipe(pipe_cmpxchg);
 7729 %}
 7730 
 7731 // Conditional-store of an int value.
 7732 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7733 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
 7734 %{
 7735   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 7736   effect(KILL oldval);
 7737 
 7738   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7739   opcode(0x0F, 0xB1);
 7740   ins_encode(lock_prefix,
 7741              REX_reg_mem(newval, mem),
 7742              OpcP, OpcS,
 7743              reg_mem(newval, mem));
 7744   ins_pipe(pipe_cmpxchg);
 7745 %}
 7746 
 7747 // Conditional-store of a long value.
 7748 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7749 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
 7750 %{
 7751   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 7752   effect(KILL oldval);
 7753 
 7754   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7755   ins_encode %{
 7756     __ lock();
 7757     __ cmpxchgq($newval$$Register, $mem$$Address);
 7758   %}
 7759   ins_pipe(pipe_cmpxchg);
 7760 %}
 7761 
 7762 
 7763 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7764 instruct compareAndSwapP(rRegI res,
 7765                          memory mem_ptr,
 7766                          rax_RegP oldval, rRegP newval,
 7767                          rFlagsReg cr)
 7768 %{
 7769   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7770   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7771   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7772   effect(KILL cr, KILL oldval);
 7773 
 7774   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7775             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7776             "sete    $res\n\t"
 7777             "movzbl  $res, $res" %}
 7778   ins_encode %{
 7779     __ lock();
 7780     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7781     __ sete($res$$Register);
 7782     __ movzbl($res$$Register, $res$$Register);
 7783   %}
 7784   ins_pipe( pipe_cmpxchg );
 7785 %}
 7786 
 7787 instruct compareAndSwapL(rRegI res,
 7788                          memory mem_ptr,
 7789                          rax_RegL oldval, rRegL newval,
 7790                          rFlagsReg cr)
 7791 %{
 7792   predicate(VM_Version::supports_cx8());
 7793   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7794   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7795   effect(KILL cr, KILL oldval);
 7796 
 7797   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7798             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7799             "sete    $res\n\t"
 7800             "movzbl  $res, $res" %}
 7801   ins_encode %{
 7802     __ lock();
 7803     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7804     __ sete($res$$Register);
 7805     __ movzbl($res$$Register, $res$$Register);
 7806   %}
 7807   ins_pipe( pipe_cmpxchg );
 7808 %}
 7809 
 7810 instruct compareAndSwapI(rRegI res,
 7811                          memory mem_ptr,
 7812                          rax_RegI oldval, rRegI newval,
 7813                          rFlagsReg cr)
 7814 %{
 7815   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7816   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7817   effect(KILL cr, KILL oldval);
 7818 
 7819   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7820             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7821             "sete    $res\n\t"
 7822             "movzbl  $res, $res" %}
 7823   ins_encode %{
 7824     __ lock();
 7825     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7826     __ sete($res$$Register);
 7827     __ movzbl($res$$Register, $res$$Register);
 7828   %}
 7829   ins_pipe( pipe_cmpxchg );
 7830 %}
 7831 
 7832 instruct compareAndSwapB(rRegI res,
 7833                          memory mem_ptr,
 7834                          rax_RegI oldval, rRegI newval,
 7835                          rFlagsReg cr)
 7836 %{
 7837   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7838   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7839   effect(KILL cr, KILL oldval);
 7840 
 7841   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7842             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7843             "sete    $res\n\t"
 7844             "movzbl  $res, $res" %}
 7845   ins_encode %{
 7846     __ lock();
 7847     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7848     __ sete($res$$Register);
 7849     __ movzbl($res$$Register, $res$$Register);
 7850   %}
 7851   ins_pipe( pipe_cmpxchg );
 7852 %}
 7853 
 7854 instruct compareAndSwapS(rRegI res,
 7855                          memory mem_ptr,
 7856                          rax_RegI oldval, rRegI newval,
 7857                          rFlagsReg cr)
 7858 %{
 7859   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7860   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7861   effect(KILL cr, KILL oldval);
 7862 
 7863   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7864             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7865             "sete    $res\n\t"
 7866             "movzbl  $res, $res" %}
 7867   ins_encode %{
 7868     __ lock();
 7869     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7870     __ sete($res$$Register);
 7871     __ movzbl($res$$Register, $res$$Register);
 7872   %}
 7873   ins_pipe( pipe_cmpxchg );
 7874 %}
 7875 
 7876 instruct compareAndSwapN(rRegI res,
 7877                           memory mem_ptr,
 7878                           rax_RegN oldval, rRegN newval,
 7879                           rFlagsReg cr) %{
 7880   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7881   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7882   effect(KILL cr, KILL oldval);
 7883 
 7884   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7885             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7886             "sete    $res\n\t"
 7887             "movzbl  $res, $res" %}
 7888   ins_encode %{
 7889     __ lock();
 7890     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7891     __ sete($res$$Register);
 7892     __ movzbl($res$$Register, $res$$Register);
 7893   %}
 7894   ins_pipe( pipe_cmpxchg );
 7895 %}
 7896 
 7897 instruct compareAndExchangeB(
 7898                          memory mem_ptr,
 7899                          rax_RegI oldval, rRegI newval,
 7900                          rFlagsReg cr)
 7901 %{
 7902   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7903   effect(KILL cr);
 7904 
 7905   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7906             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7907   ins_encode %{
 7908     __ lock();
 7909     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7910   %}
 7911   ins_pipe( pipe_cmpxchg );
 7912 %}
 7913 
 7914 instruct compareAndExchangeS(
 7915                          memory mem_ptr,
 7916                          rax_RegI oldval, rRegI newval,
 7917                          rFlagsReg cr)
 7918 %{
 7919   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7920   effect(KILL cr);
 7921 
 7922   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7923             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7924   ins_encode %{
 7925     __ lock();
 7926     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7927   %}
 7928   ins_pipe( pipe_cmpxchg );
 7929 %}
 7930 
 7931 instruct compareAndExchangeI(
 7932                          memory mem_ptr,
 7933                          rax_RegI oldval, rRegI newval,
 7934                          rFlagsReg cr)
 7935 %{
 7936   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7937   effect(KILL cr);
 7938 
 7939   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7940             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7941   ins_encode %{
 7942     __ lock();
 7943     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7944   %}
 7945   ins_pipe( pipe_cmpxchg );
 7946 %}
 7947 
 7948 instruct compareAndExchangeL(
 7949                          memory mem_ptr,
 7950                          rax_RegL oldval, rRegL newval,
 7951                          rFlagsReg cr)
 7952 %{
 7953   predicate(VM_Version::supports_cx8());
 7954   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7955   effect(KILL cr);
 7956 
 7957   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7958             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7959   ins_encode %{
 7960     __ lock();
 7961     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7962   %}
 7963   ins_pipe( pipe_cmpxchg );
 7964 %}
 7965 
 7966 instruct compareAndExchangeN(
 7967                           memory mem_ptr,
 7968                           rax_RegN oldval, rRegN newval,
 7969                           rFlagsReg cr) %{
 7970   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7971   effect(KILL cr);
 7972 
 7973   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7974             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7975   ins_encode %{
 7976     __ lock();
 7977     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7978   %}
 7979   ins_pipe( pipe_cmpxchg );
 7980 %}
 7981 
 7982 instruct compareAndExchangeP(
 7983                          memory mem_ptr,
 7984                          rax_RegP oldval, rRegP newval,
 7985                          rFlagsReg cr)
 7986 %{
 7987   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7988   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7989   effect(KILL cr);
 7990 
 7991   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7992             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7993   ins_encode %{
 7994     __ lock();
 7995     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7996   %}
 7997   ins_pipe( pipe_cmpxchg );
 7998 %}
 7999 
 8000 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8001   predicate(n->as_LoadStore()->result_not_used());
 8002   match(Set dummy (GetAndAddB mem add));
 8003   effect(KILL cr);
 8004   format %{ "ADDB  [$mem],$add" %}
 8005   ins_encode %{
 8006     __ lock();
 8007     __ addb($mem$$Address, $add$$constant);
 8008   %}
 8009   ins_pipe( pipe_cmpxchg );
 8010 %}
 8011 
 8012 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8013   match(Set newval (GetAndAddB mem newval));
 8014   effect(KILL cr);
 8015   format %{ "XADDB  [$mem],$newval" %}
 8016   ins_encode %{
 8017     __ lock();
 8018     __ xaddb($mem$$Address, $newval$$Register);
 8019   %}
 8020   ins_pipe( pipe_cmpxchg );
 8021 %}
 8022 
 8023 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8024   predicate(n->as_LoadStore()->result_not_used());
 8025   match(Set dummy (GetAndAddS mem add));
 8026   effect(KILL cr);
 8027   format %{ "ADDW  [$mem],$add" %}
 8028   ins_encode %{
 8029     __ lock();
 8030     __ addw($mem$$Address, $add$$constant);
 8031   %}
 8032   ins_pipe( pipe_cmpxchg );
 8033 %}
 8034 
 8035 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8036   match(Set newval (GetAndAddS mem newval));
 8037   effect(KILL cr);
 8038   format %{ "XADDW  [$mem],$newval" %}
 8039   ins_encode %{
 8040     __ lock();
 8041     __ xaddw($mem$$Address, $newval$$Register);
 8042   %}
 8043   ins_pipe( pipe_cmpxchg );
 8044 %}
 8045 
 8046 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8047   predicate(n->as_LoadStore()->result_not_used());
 8048   match(Set dummy (GetAndAddI mem add));
 8049   effect(KILL cr);
 8050   format %{ "ADDL  [$mem],$add" %}
 8051   ins_encode %{
 8052     __ lock();
 8053     __ addl($mem$$Address, $add$$constant);
 8054   %}
 8055   ins_pipe( pipe_cmpxchg );
 8056 %}
 8057 
 8058 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8059   match(Set newval (GetAndAddI mem newval));
 8060   effect(KILL cr);
 8061   format %{ "XADDL  [$mem],$newval" %}
 8062   ins_encode %{
 8063     __ lock();
 8064     __ xaddl($mem$$Address, $newval$$Register);
 8065   %}
 8066   ins_pipe( pipe_cmpxchg );
 8067 %}
 8068 
 8069 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8070   predicate(n->as_LoadStore()->result_not_used());
 8071   match(Set dummy (GetAndAddL mem add));
 8072   effect(KILL cr);
 8073   format %{ "ADDQ  [$mem],$add" %}
 8074   ins_encode %{
 8075     __ lock();
 8076     __ addq($mem$$Address, $add$$constant);
 8077   %}
 8078   ins_pipe( pipe_cmpxchg );
 8079 %}
 8080 
 8081 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8082   match(Set newval (GetAndAddL mem newval));
 8083   effect(KILL cr);
 8084   format %{ "XADDQ  [$mem],$newval" %}
 8085   ins_encode %{
 8086     __ lock();
 8087     __ xaddq($mem$$Address, $newval$$Register);
 8088   %}
 8089   ins_pipe( pipe_cmpxchg );
 8090 %}
 8091 
 8092 instruct xchgB( memory mem, rRegI newval) %{
 8093   match(Set newval (GetAndSetB mem newval));
 8094   format %{ "XCHGB  $newval,[$mem]" %}
 8095   ins_encode %{
 8096     __ xchgb($newval$$Register, $mem$$Address);
 8097   %}
 8098   ins_pipe( pipe_cmpxchg );
 8099 %}
 8100 
 8101 instruct xchgS( memory mem, rRegI newval) %{
 8102   match(Set newval (GetAndSetS mem newval));
 8103   format %{ "XCHGW  $newval,[$mem]" %}
 8104   ins_encode %{
 8105     __ xchgw($newval$$Register, $mem$$Address);
 8106   %}
 8107   ins_pipe( pipe_cmpxchg );
 8108 %}
 8109 
 8110 instruct xchgI( memory mem, rRegI newval) %{
 8111   match(Set newval (GetAndSetI mem newval));
 8112   format %{ "XCHGL  $newval,[$mem]" %}
 8113   ins_encode %{
 8114     __ xchgl($newval$$Register, $mem$$Address);
 8115   %}
 8116   ins_pipe( pipe_cmpxchg );
 8117 %}
 8118 
 8119 instruct xchgL( memory mem, rRegL newval) %{
 8120   match(Set newval (GetAndSetL mem newval));
 8121   format %{ "XCHGL  $newval,[$mem]" %}
 8122   ins_encode %{
 8123     __ xchgq($newval$$Register, $mem$$Address);
 8124   %}
 8125   ins_pipe( pipe_cmpxchg );
 8126 %}
 8127 
 8128 instruct xchgP( memory mem, rRegP newval) %{
 8129   match(Set newval (GetAndSetP mem newval));
 8130   predicate(n->as_LoadStore()->barrier_data() == 0);
 8131   format %{ "XCHGQ  $newval,[$mem]" %}
 8132   ins_encode %{
 8133     __ xchgq($newval$$Register, $mem$$Address);
 8134   %}
 8135   ins_pipe( pipe_cmpxchg );
 8136 %}
 8137 
 8138 instruct xchgN( memory mem, rRegN newval) %{
 8139   match(Set newval (GetAndSetN mem newval));
 8140   format %{ "XCHGL  $newval,$mem]" %}
 8141   ins_encode %{
 8142     __ xchgl($newval$$Register, $mem$$Address);
 8143   %}
 8144   ins_pipe( pipe_cmpxchg );
 8145 %}
 8146 
 8147 //----------Abs Instructions-------------------------------------------
 8148 
 8149 // Integer Absolute Instructions
 8150 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8151 %{
 8152   match(Set dst (AbsI src));
 8153   effect(TEMP dst, TEMP tmp, KILL cr);
 8154   format %{ "movl $tmp, $src\n\t"
 8155             "sarl $tmp, 31\n\t"
 8156             "movl $dst, $src\n\t"
 8157             "xorl $dst, $tmp\n\t"
 8158             "subl $dst, $tmp\n"
 8159           %}
 8160   ins_encode %{
 8161     __ movl($tmp$$Register, $src$$Register);
 8162     __ sarl($tmp$$Register, 31);
 8163     __ movl($dst$$Register, $src$$Register);
 8164     __ xorl($dst$$Register, $tmp$$Register);
 8165     __ subl($dst$$Register, $tmp$$Register);
 8166   %}
 8167 
 8168   ins_pipe(ialu_reg_reg);
 8169 %}
 8170 
 8171 // Long Absolute Instructions
 8172 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8173 %{
 8174   match(Set dst (AbsL src));
 8175   effect(TEMP dst, TEMP tmp, KILL cr);
 8176   format %{ "movq $tmp, $src\n\t"
 8177             "sarq $tmp, 63\n\t"
 8178             "movq $dst, $src\n\t"
 8179             "xorq $dst, $tmp\n\t"
 8180             "subq $dst, $tmp\n"
 8181           %}
 8182   ins_encode %{
 8183     __ movq($tmp$$Register, $src$$Register);
 8184     __ sarq($tmp$$Register, 63);
 8185     __ movq($dst$$Register, $src$$Register);
 8186     __ xorq($dst$$Register, $tmp$$Register);
 8187     __ subq($dst$$Register, $tmp$$Register);
 8188   %}
 8189 
 8190   ins_pipe(ialu_reg_reg);
 8191 %}
 8192 
 8193 //----------Subtraction Instructions-------------------------------------------
 8194 
 8195 // Integer Subtraction Instructions
 8196 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8197 %{
 8198   match(Set dst (SubI dst src));
 8199   effect(KILL cr);
 8200 
 8201   format %{ "subl    $dst, $src\t# int" %}
 8202   ins_encode %{
 8203     __ subl($dst$$Register, $src$$Register);
 8204   %}
 8205   ins_pipe(ialu_reg_reg);
 8206 %}
 8207 
 8208 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8209 %{
 8210   match(Set dst (SubI dst src));
 8211   effect(KILL cr);
 8212 
 8213   format %{ "subl    $dst, $src\t# int" %}
 8214   ins_encode %{
 8215     __ subl($dst$$Register, $src$$constant);
 8216   %}
 8217   ins_pipe(ialu_reg);
 8218 %}
 8219 
 8220 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8221 %{
 8222   match(Set dst (SubI dst (LoadI src)));
 8223   effect(KILL cr);
 8224 
 8225   ins_cost(125);
 8226   format %{ "subl    $dst, $src\t# int" %}
 8227   ins_encode %{
 8228     __ subl($dst$$Register, $src$$Address);
 8229   %}
 8230   ins_pipe(ialu_reg_mem);
 8231 %}
 8232 
 8233 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8234 %{
 8235   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8236   effect(KILL cr);
 8237 
 8238   ins_cost(150);
 8239   format %{ "subl    $dst, $src\t# int" %}
 8240   ins_encode %{
 8241     __ subl($dst$$Address, $src$$Register);
 8242   %}
 8243   ins_pipe(ialu_mem_reg);
 8244 %}
 8245 
 8246 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8247 %{
 8248   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8249   effect(KILL cr);
 8250 
 8251   ins_cost(125); // XXX
 8252   format %{ "subl    $dst, $src\t# int" %}
 8253   ins_encode %{
 8254     __ subl($dst$$Address, $src$$constant);
 8255   %}
 8256   ins_pipe(ialu_mem_imm);
 8257 %}
 8258 
 8259 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8260 %{
 8261   match(Set dst (SubL dst src));
 8262   effect(KILL cr);
 8263 
 8264   format %{ "subq    $dst, $src\t# long" %}
 8265   ins_encode %{
 8266     __ subq($dst$$Register, $src$$Register);
 8267   %}
 8268   ins_pipe(ialu_reg_reg);
 8269 %}
 8270 
 8271 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8272 %{
 8273   match(Set dst (SubL dst src));
 8274   effect(KILL cr);
 8275 
 8276   format %{ "subq    $dst, $src\t# long" %}
 8277   ins_encode %{
 8278     __ subq($dst$$Register, $src$$constant);
 8279   %}
 8280   ins_pipe(ialu_reg);
 8281 %}
 8282 
 8283 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8284 %{
 8285   match(Set dst (SubL dst (LoadL src)));
 8286   effect(KILL cr);
 8287 
 8288   ins_cost(125);
 8289   format %{ "subq    $dst, $src\t# long" %}
 8290   ins_encode %{
 8291     __ subq($dst$$Register, $src$$Address);
 8292   %}
 8293   ins_pipe(ialu_reg_mem);
 8294 %}
 8295 
 8296 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8297 %{
 8298   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8299   effect(KILL cr);
 8300 
 8301   ins_cost(150);
 8302   format %{ "subq    $dst, $src\t# long" %}
 8303   ins_encode %{
 8304     __ subq($dst$$Address, $src$$Register);
 8305   %}
 8306   ins_pipe(ialu_mem_reg);
 8307 %}
 8308 
 8309 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8310 %{
 8311   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8312   effect(KILL cr);
 8313 
 8314   ins_cost(125); // XXX
 8315   format %{ "subq    $dst, $src\t# long" %}
 8316   ins_encode %{
 8317     __ subq($dst$$Address, $src$$constant);
 8318   %}
 8319   ins_pipe(ialu_mem_imm);
 8320 %}
 8321 
 8322 // Subtract from a pointer
 8323 // XXX hmpf???
 8324 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8325 %{
 8326   match(Set dst (AddP dst (SubI zero src)));
 8327   effect(KILL cr);
 8328 
 8329   format %{ "subq    $dst, $src\t# ptr - int" %}
 8330   opcode(0x2B);
 8331   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8332   ins_pipe(ialu_reg_reg);
 8333 %}
 8334 
 8335 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8336 %{
 8337   match(Set dst (SubI zero dst));
 8338   effect(KILL cr);
 8339 
 8340   format %{ "negl    $dst\t# int" %}
 8341   ins_encode %{
 8342     __ negl($dst$$Register);
 8343   %}
 8344   ins_pipe(ialu_reg);
 8345 %}
 8346 
 8347 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8348 %{
 8349   match(Set dst (NegI dst));
 8350   effect(KILL cr);
 8351 
 8352   format %{ "negl    $dst\t# int" %}
 8353   ins_encode %{
 8354     __ negl($dst$$Register);
 8355   %}
 8356   ins_pipe(ialu_reg);
 8357 %}
 8358 
 8359 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8360 %{
 8361   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8362   effect(KILL cr);
 8363 
 8364   format %{ "negl    $dst\t# int" %}
 8365   ins_encode %{
 8366     __ negl($dst$$Address);
 8367   %}
 8368   ins_pipe(ialu_reg);
 8369 %}
 8370 
 8371 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8372 %{
 8373   match(Set dst (SubL zero dst));
 8374   effect(KILL cr);
 8375 
 8376   format %{ "negq    $dst\t# long" %}
 8377   ins_encode %{
 8378     __ negq($dst$$Register);
 8379   %}
 8380   ins_pipe(ialu_reg);
 8381 %}
 8382 
 8383 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8384 %{
 8385   match(Set dst (NegL dst));
 8386   effect(KILL cr);
 8387 
 8388   format %{ "negq    $dst\t# int" %}
 8389   ins_encode %{
 8390     __ negq($dst$$Register);
 8391   %}
 8392   ins_pipe(ialu_reg);
 8393 %}
 8394 
 8395 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8396 %{
 8397   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8398   effect(KILL cr);
 8399 
 8400   format %{ "negq    $dst\t# long" %}
 8401   ins_encode %{
 8402     __ negq($dst$$Address);
 8403   %}
 8404   ins_pipe(ialu_reg);
 8405 %}
 8406 
 8407 //----------Multiplication/Division Instructions-------------------------------
 8408 // Integer Multiplication Instructions
 8409 // Multiply Register
 8410 
 8411 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8412 %{
 8413   match(Set dst (MulI dst src));
 8414   effect(KILL cr);
 8415 
 8416   ins_cost(300);
 8417   format %{ "imull   $dst, $src\t# int" %}
 8418   ins_encode %{
 8419     __ imull($dst$$Register, $src$$Register);
 8420   %}
 8421   ins_pipe(ialu_reg_reg_alu0);
 8422 %}
 8423 
 8424 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8425 %{
 8426   match(Set dst (MulI src imm));
 8427   effect(KILL cr);
 8428 
 8429   ins_cost(300);
 8430   format %{ "imull   $dst, $src, $imm\t# int" %}
 8431   ins_encode %{
 8432     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8433   %}
 8434   ins_pipe(ialu_reg_reg_alu0);
 8435 %}
 8436 
 8437 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8438 %{
 8439   match(Set dst (MulI dst (LoadI src)));
 8440   effect(KILL cr);
 8441 
 8442   ins_cost(350);
 8443   format %{ "imull   $dst, $src\t# int" %}
 8444   ins_encode %{
 8445     __ imull($dst$$Register, $src$$Address);
 8446   %}
 8447   ins_pipe(ialu_reg_mem_alu0);
 8448 %}
 8449 
 8450 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8451 %{
 8452   match(Set dst (MulI (LoadI src) imm));
 8453   effect(KILL cr);
 8454 
 8455   ins_cost(300);
 8456   format %{ "imull   $dst, $src, $imm\t# int" %}
 8457   ins_encode %{
 8458     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8459   %}
 8460   ins_pipe(ialu_reg_mem_alu0);
 8461 %}
 8462 
 8463 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8464 %{
 8465   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8466   effect(KILL cr, KILL src2);
 8467 
 8468   expand %{ mulI_rReg(dst, src1, cr);
 8469            mulI_rReg(src2, src3, cr);
 8470            addI_rReg(dst, src2, cr); %}
 8471 %}
 8472 
 8473 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8474 %{
 8475   match(Set dst (MulL dst src));
 8476   effect(KILL cr);
 8477 
 8478   ins_cost(300);
 8479   format %{ "imulq   $dst, $src\t# long" %}
 8480   ins_encode %{
 8481     __ imulq($dst$$Register, $src$$Register);
 8482   %}
 8483   ins_pipe(ialu_reg_reg_alu0);
 8484 %}
 8485 
 8486 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8487 %{
 8488   match(Set dst (MulL src imm));
 8489   effect(KILL cr);
 8490 
 8491   ins_cost(300);
 8492   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8493   ins_encode %{
 8494     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8495   %}
 8496   ins_pipe(ialu_reg_reg_alu0);
 8497 %}
 8498 
 8499 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8500 %{
 8501   match(Set dst (MulL dst (LoadL src)));
 8502   effect(KILL cr);
 8503 
 8504   ins_cost(350);
 8505   format %{ "imulq   $dst, $src\t# long" %}
 8506   ins_encode %{
 8507     __ imulq($dst$$Register, $src$$Address);
 8508   %}
 8509   ins_pipe(ialu_reg_mem_alu0);
 8510 %}
 8511 
 8512 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8513 %{
 8514   match(Set dst (MulL (LoadL src) imm));
 8515   effect(KILL cr);
 8516 
 8517   ins_cost(300);
 8518   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8519   ins_encode %{
 8520     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8521   %}
 8522   ins_pipe(ialu_reg_mem_alu0);
 8523 %}
 8524 
 8525 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8526 %{
 8527   match(Set dst (MulHiL src rax));
 8528   effect(USE_KILL rax, KILL cr);
 8529 
 8530   ins_cost(300);
 8531   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8532   ins_encode %{
 8533     __ imulq($src$$Register);
 8534   %}
 8535   ins_pipe(ialu_reg_reg_alu0);
 8536 %}
 8537 
 8538 instruct umulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8539 %{
 8540   match(Set dst (UMulHiL src rax));
 8541   effect(USE_KILL rax, KILL cr);
 8542 
 8543   ins_cost(300);
 8544   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8545   ins_encode %{
 8546     __ mulq($src$$Register);
 8547   %}
 8548   ins_pipe(ialu_reg_reg_alu0);
 8549 %}
 8550 
 8551 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8552                    rFlagsReg cr)
 8553 %{
 8554   match(Set rax (NoOvfDivI rax div));
 8555   effect(KILL rdx, KILL cr);
 8556 
 8557   ins_cost(500);
 8558   format %{ "cdql\n\t"
 8559             "idivl   $div" %}
 8560   ins_encode %{
 8561     __ cdql();
 8562     __ idivl($div$$Register);
 8563   %}
 8564   ins_pipe(pipe_slow);
 8565 %}
 8566 
 8567 instruct divI_mem(rax_RegI rax, rdx_RegI rdx, memory div,
 8568                    rFlagsReg cr)
 8569 %{
 8570   match(Set rax (NoOvfDivI rax (LoadI div)));
 8571   effect(KILL rdx, KILL cr);
 8572 
 8573   ins_cost(575);
 8574   format %{ "cdql\n\t"
 8575             "idivl   $div" %}
 8576   ins_encode %{
 8577     __ cdql();
 8578     __ idivl($div$$Address);
 8579   %}
 8580   ins_pipe(pipe_slow);
 8581 %}
 8582 
 8583 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8584                    rFlagsReg cr)
 8585 %{
 8586   match(Set rax (NoOvfDivL rax div));
 8587   effect(KILL rdx, KILL cr);
 8588 
 8589   ins_cost(500);
 8590   format %{ "cdqq\n\t"
 8591             "idivq   $div" %}
 8592   ins_encode %{
 8593     __ cdqq();
 8594     __ idivq($div$$Register);
 8595   %}
 8596   ins_pipe(pipe_slow);
 8597 %}
 8598 
 8599 instruct divL_mem(rax_RegL rax, rdx_RegL rdx, memory div,
 8600                    rFlagsReg cr)
 8601 %{
 8602   match(Set rax (NoOvfDivL rax (LoadL div)));
 8603   effect(KILL rdx, KILL cr);
 8604 
 8605   ins_cost(575);
 8606   format %{ "cdqq\n\t"
 8607             "idivq   $div" %}
 8608   ins_encode %{
 8609     __ cdqq();
 8610     __ idivq($div$$Address);
 8611   %}
 8612   ins_pipe(pipe_slow);
 8613 %}
 8614 
 8615 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8616 %{
 8617   match(Set rax (UDivI rax div));
 8618   effect(KILL rdx, KILL cr);
 8619 
 8620   ins_cost(300);
 8621   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8622   ins_encode %{
 8623     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8624   %}
 8625   ins_pipe(ialu_reg_reg_alu0);
 8626 %}
 8627 
 8628 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8629 %{
 8630   match(Set rax (UDivL rax div));
 8631   effect(KILL rdx, KILL cr);
 8632 
 8633   ins_cost(300);
 8634   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8635   ins_encode %{
 8636      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8637   %}
 8638   ins_pipe(ialu_reg_reg_alu0);
 8639 %}
 8640 
 8641 // Integer DIVMOD with Register, both quotient and mod results
 8642 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8643                              rFlagsReg cr)
 8644 %{
 8645   match(NoOvfDivModI rax div);
 8646   effect(KILL cr);
 8647 
 8648   ins_cost(500);
 8649   format %{ "cdql\n\t"
 8650             "idivl   $div" %}
 8651   ins_encode %{
 8652     __ cdql();
 8653     __ idivl($div$$Register);
 8654   %}
 8655   ins_pipe(pipe_slow);
 8656 %}
 8657 
 8658 // Long DIVMOD with Register, both quotient and mod results
 8659 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8660                              rFlagsReg cr)
 8661 %{
 8662   match(NoOvfDivModL rax div);
 8663   effect(KILL cr);
 8664 
 8665   ins_cost(500);
 8666   format %{ "cdqq\n\t"
 8667             "idivq   $div" %}
 8668   ins_encode %{
 8669     __ cdqq();
 8670     __ idivq($div$$Register);
 8671   %}
 8672   ins_pipe(pipe_slow);
 8673 %}
 8674 
 8675 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8676 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8677                               no_rax_rdx_RegI div, rFlagsReg cr)
 8678 %{
 8679   match(UDivModI rax div);
 8680   effect(TEMP tmp, KILL cr);
 8681 
 8682   ins_cost(300);
 8683   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8684             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8685           %}
 8686   ins_encode %{
 8687     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8688   %}
 8689   ins_pipe(pipe_slow);
 8690 %}
 8691 
 8692 // Unsigned long DIVMOD with Register, both quotient and mod results
 8693 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8694                               no_rax_rdx_RegL div, rFlagsReg cr)
 8695 %{
 8696   match(UDivModL rax div);
 8697   effect(TEMP tmp, KILL cr);
 8698 
 8699   ins_cost(300);
 8700   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8701             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8702           %}
 8703   ins_encode %{
 8704     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8705   %}
 8706   ins_pipe(pipe_slow);
 8707 %}
 8708 
 8709 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8710                    rFlagsReg cr)
 8711 %{
 8712   match(Set rdx (NoOvfModI rax div));
 8713   effect(KILL rax, KILL cr);
 8714 
 8715   ins_cost(500);
 8716   format %{ "cdql\n\t"
 8717             "idivl   $div" %}
 8718   ins_encode %{
 8719     __ cdql();
 8720     __ idivl($div$$Register);
 8721   %}
 8722   ins_pipe(pipe_slow);
 8723 %}
 8724 
 8725 instruct modI_mem(rdx_RegI rdx, rax_RegI rax, memory div,
 8726                    rFlagsReg cr)
 8727 %{
 8728   match(Set rdx (NoOvfModI rax (LoadI div)));
 8729   effect(KILL rax, KILL cr);
 8730 
 8731   ins_cost(575);
 8732   format %{ "cdql\n\t"
 8733             "idivl   $div" %}
 8734   ins_encode %{
 8735     __ cdql();
 8736     __ idivl($div$$Address);
 8737   %}
 8738   ins_pipe(pipe_slow);
 8739 %}
 8740 
 8741 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8742                    rFlagsReg cr)
 8743 %{
 8744   match(Set rdx (NoOvfModL rax div));
 8745   effect(KILL rax, KILL cr);
 8746 
 8747   ins_cost(500);
 8748   format %{ "cdqq\n\t"
 8749             "idivq   $div" %}
 8750   ins_encode %{
 8751     __ cdqq();
 8752     __ idivq($div$$Register);
 8753   %}
 8754   ins_pipe(pipe_slow);
 8755 %}
 8756 
 8757 instruct modL_mem(rdx_RegL rdx, rax_RegL rax, memory div,
 8758                    rFlagsReg cr)
 8759 %{
 8760   match(Set rdx (NoOvfModL rax (LoadL div)));
 8761   effect(KILL rax, KILL cr);
 8762 
 8763   ins_cost(575);
 8764   format %{ "cdqq\n\t"
 8765             "idivq   $div" %}
 8766   ins_encode %{
 8767     __ cdqq();
 8768     __ idivq($div$$Address);
 8769   %}
 8770   ins_pipe(pipe_slow);
 8771 %}
 8772 
 8773 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8774 %{
 8775   match(Set rdx (UModI rax div));
 8776   effect(KILL rax, KILL cr);
 8777 
 8778   ins_cost(300);
 8779   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8780   ins_encode %{
 8781     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8782   %}
 8783   ins_pipe(ialu_reg_reg_alu0);
 8784 %}
 8785 
 8786 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8787 %{
 8788   match(Set rdx (UModL rax div));
 8789   effect(KILL rax, KILL cr);
 8790 
 8791   ins_cost(300);
 8792   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8793   ins_encode %{
 8794     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8795   %}
 8796   ins_pipe(ialu_reg_reg_alu0);
 8797 %}
 8798 
 8799 // Integer Shift Instructions
 8800 // Shift Left by one
 8801 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8802 %{
 8803   match(Set dst (LShiftI dst shift));
 8804   effect(KILL cr);
 8805 
 8806   format %{ "sall    $dst, $shift" %}
 8807   ins_encode %{
 8808     __ sall($dst$$Register, $shift$$constant);
 8809   %}
 8810   ins_pipe(ialu_reg);
 8811 %}
 8812 
 8813 // Shift Left by one
 8814 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8815 %{
 8816   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8817   effect(KILL cr);
 8818 
 8819   format %{ "sall    $dst, $shift\t" %}
 8820   ins_encode %{
 8821     __ sall($dst$$Address, $shift$$constant);
 8822   %}
 8823   ins_pipe(ialu_mem_imm);
 8824 %}
 8825 
 8826 // Shift Left by 8-bit immediate
 8827 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8828 %{
 8829   match(Set dst (LShiftI dst shift));
 8830   effect(KILL cr);
 8831 
 8832   format %{ "sall    $dst, $shift" %}
 8833   ins_encode %{
 8834     __ sall($dst$$Register, $shift$$constant);
 8835   %}
 8836   ins_pipe(ialu_reg);
 8837 %}
 8838 
 8839 // Shift Left by 8-bit immediate
 8840 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8841 %{
 8842   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8843   effect(KILL cr);
 8844 
 8845   format %{ "sall    $dst, $shift" %}
 8846   ins_encode %{
 8847     __ sall($dst$$Address, $shift$$constant);
 8848   %}
 8849   ins_pipe(ialu_mem_imm);
 8850 %}
 8851 
 8852 // Shift Left by variable
 8853 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8854 %{
 8855   match(Set dst (LShiftI dst shift));
 8856   effect(KILL cr);
 8857 
 8858   format %{ "sall    $dst, $shift" %}
 8859   ins_encode %{
 8860     __ sall($dst$$Register);
 8861   %}
 8862   ins_pipe(ialu_reg_reg);
 8863 %}
 8864 
 8865 // Shift Left by variable
 8866 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8867 %{
 8868   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8869   effect(KILL cr);
 8870 
 8871   format %{ "sall    $dst, $shift" %}
 8872   ins_encode %{
 8873     __ sall($dst$$Address);
 8874   %}
 8875   ins_pipe(ialu_mem_reg);
 8876 %}
 8877 
 8878 // Arithmetic shift right by one
 8879 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8880 %{
 8881   match(Set dst (RShiftI dst shift));
 8882   effect(KILL cr);
 8883 
 8884   format %{ "sarl    $dst, $shift" %}
 8885   ins_encode %{
 8886     __ sarl($dst$$Register, $shift$$constant);
 8887   %}
 8888   ins_pipe(ialu_reg);
 8889 %}
 8890 
 8891 // Arithmetic shift right by one
 8892 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8893 %{
 8894   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8895   effect(KILL cr);
 8896 
 8897   format %{ "sarl    $dst, $shift" %}
 8898   ins_encode %{
 8899     __ sarl($dst$$Address, $shift$$constant);
 8900   %}
 8901   ins_pipe(ialu_mem_imm);
 8902 %}
 8903 
 8904 // Arithmetic Shift Right by 8-bit immediate
 8905 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8906 %{
 8907   match(Set dst (RShiftI dst shift));
 8908   effect(KILL cr);
 8909 
 8910   format %{ "sarl    $dst, $shift" %}
 8911   ins_encode %{
 8912     __ sarl($dst$$Register, $shift$$constant);
 8913   %}
 8914   ins_pipe(ialu_mem_imm);
 8915 %}
 8916 
 8917 // Arithmetic Shift Right by 8-bit immediate
 8918 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8919 %{
 8920   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8921   effect(KILL cr);
 8922 
 8923   format %{ "sarl    $dst, $shift" %}
 8924   ins_encode %{
 8925     __ sarl($dst$$Address, $shift$$constant);
 8926   %}
 8927   ins_pipe(ialu_mem_imm);
 8928 %}
 8929 
 8930 // Arithmetic Shift Right by variable
 8931 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8932 %{
 8933   match(Set dst (RShiftI dst shift));
 8934   effect(KILL cr);
 8935   format %{ "sarl    $dst, $shift" %}
 8936   ins_encode %{
 8937     __ sarl($dst$$Register);
 8938   %}
 8939   ins_pipe(ialu_reg_reg);
 8940 %}
 8941 
 8942 // Arithmetic Shift Right by variable
 8943 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8944 %{
 8945   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8946   effect(KILL cr);
 8947 
 8948   format %{ "sarl    $dst, $shift" %}
 8949   ins_encode %{
 8950     __ sarl($dst$$Address);
 8951   %}
 8952   ins_pipe(ialu_mem_reg);
 8953 %}
 8954 
 8955 // Logical shift right by one
 8956 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8957 %{
 8958   match(Set dst (URShiftI dst shift));
 8959   effect(KILL cr);
 8960 
 8961   format %{ "shrl    $dst, $shift" %}
 8962   ins_encode %{
 8963     __ shrl($dst$$Register, $shift$$constant);
 8964   %}
 8965   ins_pipe(ialu_reg);
 8966 %}
 8967 
 8968 // Logical shift right by one
 8969 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8970 %{
 8971   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8972   effect(KILL cr);
 8973 
 8974   format %{ "shrl    $dst, $shift" %}
 8975   ins_encode %{
 8976     __ shrl($dst$$Address, $shift$$constant);
 8977   %}
 8978   ins_pipe(ialu_mem_imm);
 8979 %}
 8980 
 8981 // Logical Shift Right by 8-bit immediate
 8982 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8983 %{
 8984   match(Set dst (URShiftI dst shift));
 8985   effect(KILL cr);
 8986 
 8987   format %{ "shrl    $dst, $shift" %}
 8988   ins_encode %{
 8989     __ shrl($dst$$Register, $shift$$constant);
 8990   %}
 8991   ins_pipe(ialu_reg);
 8992 %}
 8993 
 8994 // Logical Shift Right by 8-bit immediate
 8995 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8996 %{
 8997   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8998   effect(KILL cr);
 8999 
 9000   format %{ "shrl    $dst, $shift" %}
 9001   ins_encode %{
 9002     __ shrl($dst$$Address, $shift$$constant);
 9003   %}
 9004   ins_pipe(ialu_mem_imm);
 9005 %}
 9006 
 9007 // Logical Shift Right by variable
 9008 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9009 %{
 9010   match(Set dst (URShiftI dst shift));
 9011   effect(KILL cr);
 9012 
 9013   format %{ "shrl    $dst, $shift" %}
 9014   ins_encode %{
 9015     __ shrl($dst$$Register);
 9016   %}
 9017   ins_pipe(ialu_reg_reg);
 9018 %}
 9019 
 9020 // Logical Shift Right by variable
 9021 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9022 %{
 9023   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9024   effect(KILL cr);
 9025 
 9026   format %{ "shrl    $dst, $shift" %}
 9027   ins_encode %{
 9028     __ shrl($dst$$Address);
 9029   %}
 9030   ins_pipe(ialu_mem_reg);
 9031 %}
 9032 
 9033 // Long Shift Instructions
 9034 // Shift Left by one
 9035 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9036 %{
 9037   match(Set dst (LShiftL dst shift));
 9038   effect(KILL cr);
 9039 
 9040   format %{ "salq    $dst, $shift" %}
 9041   ins_encode %{
 9042     __ salq($dst$$Register, $shift$$constant);
 9043   %}
 9044   ins_pipe(ialu_reg);
 9045 %}
 9046 
 9047 // Shift Left by one
 9048 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9049 %{
 9050   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9051   effect(KILL cr);
 9052 
 9053   format %{ "salq    $dst, $shift" %}
 9054   ins_encode %{
 9055     __ salq($dst$$Address, $shift$$constant);
 9056   %}
 9057   ins_pipe(ialu_mem_imm);
 9058 %}
 9059 
 9060 // Shift Left by 8-bit immediate
 9061 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9062 %{
 9063   match(Set dst (LShiftL dst shift));
 9064   effect(KILL cr);
 9065 
 9066   format %{ "salq    $dst, $shift" %}
 9067   ins_encode %{
 9068     __ salq($dst$$Register, $shift$$constant);
 9069   %}
 9070   ins_pipe(ialu_reg);
 9071 %}
 9072 
 9073 // Shift Left by 8-bit immediate
 9074 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9075 %{
 9076   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9077   effect(KILL cr);
 9078 
 9079   format %{ "salq    $dst, $shift" %}
 9080   ins_encode %{
 9081     __ salq($dst$$Address, $shift$$constant);
 9082   %}
 9083   ins_pipe(ialu_mem_imm);
 9084 %}
 9085 
 9086 // Shift Left by variable
 9087 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9088 %{
 9089   match(Set dst (LShiftL dst shift));
 9090   effect(KILL cr);
 9091 
 9092   format %{ "salq    $dst, $shift" %}
 9093   ins_encode %{
 9094     __ salq($dst$$Register);
 9095   %}
 9096   ins_pipe(ialu_reg_reg);
 9097 %}
 9098 
 9099 // Shift Left by variable
 9100 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9101 %{
 9102   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9103   effect(KILL cr);
 9104 
 9105   format %{ "salq    $dst, $shift" %}
 9106   ins_encode %{
 9107     __ salq($dst$$Address);
 9108   %}
 9109   ins_pipe(ialu_mem_reg);
 9110 %}
 9111 
 9112 // Arithmetic shift right by one
 9113 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9114 %{
 9115   match(Set dst (RShiftL dst shift));
 9116   effect(KILL cr);
 9117 
 9118   format %{ "sarq    $dst, $shift" %}
 9119   ins_encode %{
 9120     __ sarq($dst$$Register, $shift$$constant);
 9121   %}
 9122   ins_pipe(ialu_reg);
 9123 %}
 9124 
 9125 // Arithmetic shift right by one
 9126 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9127 %{
 9128   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9129   effect(KILL cr);
 9130 
 9131   format %{ "sarq    $dst, $shift" %}
 9132   ins_encode %{
 9133     __ sarq($dst$$Address, $shift$$constant);
 9134   %}
 9135   ins_pipe(ialu_mem_imm);
 9136 %}
 9137 
 9138 // Arithmetic Shift Right by 8-bit immediate
 9139 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9140 %{
 9141   match(Set dst (RShiftL dst shift));
 9142   effect(KILL cr);
 9143 
 9144   format %{ "sarq    $dst, $shift" %}
 9145   ins_encode %{
 9146     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9147   %}
 9148   ins_pipe(ialu_mem_imm);
 9149 %}
 9150 
 9151 // Arithmetic Shift Right by 8-bit immediate
 9152 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9153 %{
 9154   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9155   effect(KILL cr);
 9156 
 9157   format %{ "sarq    $dst, $shift" %}
 9158   ins_encode %{
 9159     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9160   %}
 9161   ins_pipe(ialu_mem_imm);
 9162 %}
 9163 
 9164 // Arithmetic Shift Right by variable
 9165 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9166 %{
 9167   match(Set dst (RShiftL dst shift));
 9168   effect(KILL cr);
 9169 
 9170   format %{ "sarq    $dst, $shift" %}
 9171   ins_encode %{
 9172     __ sarq($dst$$Register);
 9173   %}
 9174   ins_pipe(ialu_reg_reg);
 9175 %}
 9176 
 9177 // Arithmetic Shift Right by variable
 9178 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9179 %{
 9180   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9181   effect(KILL cr);
 9182 
 9183   format %{ "sarq    $dst, $shift" %}
 9184   ins_encode %{
 9185     __ sarq($dst$$Address);
 9186   %}
 9187   ins_pipe(ialu_mem_reg);
 9188 %}
 9189 
 9190 // Logical shift right by one
 9191 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9192 %{
 9193   match(Set dst (URShiftL dst shift));
 9194   effect(KILL cr);
 9195 
 9196   format %{ "shrq    $dst, $shift" %}
 9197   ins_encode %{
 9198     __ shrq($dst$$Register, $shift$$constant);
 9199   %}
 9200   ins_pipe(ialu_reg);
 9201 %}
 9202 
 9203 // Logical shift right by one
 9204 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9205 %{
 9206   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9207   effect(KILL cr);
 9208 
 9209   format %{ "shrq    $dst, $shift" %}
 9210   ins_encode %{
 9211     __ shrq($dst$$Address, $shift$$constant);
 9212   %}
 9213   ins_pipe(ialu_mem_imm);
 9214 %}
 9215 
 9216 // Logical Shift Right by 8-bit immediate
 9217 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9218 %{
 9219   match(Set dst (URShiftL dst shift));
 9220   effect(KILL cr);
 9221 
 9222   format %{ "shrq    $dst, $shift" %}
 9223   ins_encode %{
 9224     __ shrq($dst$$Register, $shift$$constant);
 9225   %}
 9226   ins_pipe(ialu_reg);
 9227 %}
 9228 
 9229 // Logical Shift Right by 8-bit immediate
 9230 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9231 %{
 9232   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9233   effect(KILL cr);
 9234 
 9235   format %{ "shrq    $dst, $shift" %}
 9236   ins_encode %{
 9237     __ shrq($dst$$Address, $shift$$constant);
 9238   %}
 9239   ins_pipe(ialu_mem_imm);
 9240 %}
 9241 
 9242 // Logical Shift Right by variable
 9243 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9244 %{
 9245   match(Set dst (URShiftL dst shift));
 9246   effect(KILL cr);
 9247 
 9248   format %{ "shrq    $dst, $shift" %}
 9249   ins_encode %{
 9250     __ shrq($dst$$Register);
 9251   %}
 9252   ins_pipe(ialu_reg_reg);
 9253 %}
 9254 
 9255 // Logical Shift Right by variable
 9256 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9257 %{
 9258   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9259   effect(KILL cr);
 9260 
 9261   format %{ "shrq    $dst, $shift" %}
 9262   ins_encode %{
 9263     __ shrq($dst$$Address);
 9264   %}
 9265   ins_pipe(ialu_mem_reg);
 9266 %}
 9267 
 9268 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9269 // This idiom is used by the compiler for the i2b bytecode.
 9270 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9271 %{
 9272   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9273 
 9274   format %{ "movsbl  $dst, $src\t# i2b" %}
 9275   ins_encode %{
 9276     __ movsbl($dst$$Register, $src$$Register);
 9277   %}
 9278   ins_pipe(ialu_reg_reg);
 9279 %}
 9280 
 9281 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9282 // This idiom is used by the compiler the i2s bytecode.
 9283 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9284 %{
 9285   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9286 
 9287   format %{ "movswl  $dst, $src\t# i2s" %}
 9288   ins_encode %{
 9289     __ movswl($dst$$Register, $src$$Register);
 9290   %}
 9291   ins_pipe(ialu_reg_reg);
 9292 %}
 9293 
 9294 // ROL/ROR instructions
 9295 
 9296 // Rotate left by constant.
 9297 instruct rolI_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9298 %{
 9299   predicate(n->bottom_type()->basic_type() == T_INT);
 9300   match(Set dst (RotateLeft dst shift));
 9301   effect(KILL cr);
 9302   format %{ "roll    $dst, $shift" %}
 9303   ins_encode %{
 9304     __ roll($dst$$Register, $shift$$constant);
 9305   %}
 9306   ins_pipe(ialu_reg);
 9307 %}
 9308 
 9309 // Rotate Left by variable
 9310 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9311 %{
 9312   predicate(n->bottom_type()->basic_type() == T_INT);
 9313   match(Set dst (RotateLeft dst shift));
 9314   effect(KILL cr);
 9315   format %{ "roll    $dst, $shift" %}
 9316   ins_encode %{
 9317     __ roll($dst$$Register);
 9318   %}
 9319   ins_pipe(ialu_reg_reg);
 9320 %}
 9321 
 9322 // Rotate Right by constant.
 9323 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9324 %{
 9325   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9326   match(Set dst (RotateRight dst shift));
 9327   effect(KILL cr);
 9328   format %{ "rorl    $dst, $shift" %}
 9329   ins_encode %{
 9330     __ rorl($dst$$Register, $shift$$constant);
 9331   %}
 9332   ins_pipe(ialu_reg);
 9333 %}
 9334 
 9335 // Rotate Right by constant.
 9336 instruct rorI_immI8(rRegI dst, immI8 shift)
 9337 %{
 9338   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9339   match(Set dst (RotateRight dst shift));
 9340   format %{ "rorxd     $dst, $shift" %}
 9341   ins_encode %{
 9342     __ rorxd($dst$$Register, $dst$$Register, $shift$$constant);
 9343   %}
 9344   ins_pipe(ialu_reg_reg);
 9345 %}
 9346 
 9347 // Rotate Right by variable
 9348 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9349 %{
 9350   predicate(n->bottom_type()->basic_type() == T_INT);
 9351   match(Set dst (RotateRight dst shift));
 9352   effect(KILL cr);
 9353   format %{ "rorl    $dst, $shift" %}
 9354   ins_encode %{
 9355     __ rorl($dst$$Register);
 9356   %}
 9357   ins_pipe(ialu_reg_reg);
 9358 %}
 9359 
 9360 
 9361 // Rotate Left by constant.
 9362 instruct rolL_immI8(rRegL dst, immI8 shift, rFlagsReg cr)
 9363 %{
 9364   predicate(n->bottom_type()->basic_type() == T_LONG);
 9365   match(Set dst (RotateLeft dst shift));
 9366   effect(KILL cr);
 9367   format %{ "rolq    $dst, $shift" %}
 9368   ins_encode %{
 9369     __ rolq($dst$$Register, $shift$$constant);
 9370   %}
 9371   ins_pipe(ialu_reg);
 9372 %}
 9373 
 9374 // Rotate Left by variable
 9375 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9376 %{
 9377   predicate(n->bottom_type()->basic_type() == T_LONG);
 9378   match(Set dst (RotateLeft dst shift));
 9379   effect(KILL cr);
 9380   format %{ "rolq    $dst, $shift" %}
 9381   ins_encode %{
 9382     __ rolq($dst$$Register);
 9383   %}
 9384   ins_pipe(ialu_reg_reg);
 9385 %}
 9386 
 9387 
 9388 // Rotate Right by constant.
 9389 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9390 %{
 9391   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9392   match(Set dst (RotateRight dst shift));
 9393   effect(KILL cr);
 9394   format %{ "rorq    $dst, $shift" %}
 9395   ins_encode %{
 9396     __ rorq($dst$$Register, $shift$$constant);
 9397   %}
 9398   ins_pipe(ialu_reg);
 9399 %}
 9400 
 9401 
 9402 // Rotate Right by constant
 9403 instruct rorL_immI8(rRegL dst, immI8 shift)
 9404 %{
 9405   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9406   match(Set dst (RotateRight dst shift));
 9407   format %{ "rorxq    $dst, $shift" %}
 9408   ins_encode %{
 9409     __ rorxq($dst$$Register, $dst$$Register, $shift$$constant);
 9410   %}
 9411   ins_pipe(ialu_reg_reg);
 9412 %}
 9413 
 9414 // Rotate Right by variable
 9415 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9416 %{
 9417   predicate(n->bottom_type()->basic_type() == T_LONG);
 9418   match(Set dst (RotateRight dst shift));
 9419   effect(KILL cr);
 9420   format %{ "rorq    $dst, $shift" %}
 9421   ins_encode %{
 9422     __ rorq($dst$$Register);
 9423   %}
 9424   ins_pipe(ialu_reg_reg);
 9425 %}
 9426 
 9427 
 9428 // Logical Instructions
 9429 
 9430 // Integer Logical Instructions
 9431 
 9432 // And Instructions
 9433 // And Register with Register
 9434 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9435 %{
 9436   match(Set dst (AndI dst src));
 9437   effect(KILL cr);
 9438 
 9439   format %{ "andl    $dst, $src\t# int" %}
 9440   ins_encode %{
 9441     __ andl($dst$$Register, $src$$Register);
 9442   %}
 9443   ins_pipe(ialu_reg_reg);
 9444 %}
 9445 
 9446 // And Register with Immediate 255
 9447 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
 9448 %{
 9449   match(Set dst (AndI dst src));
 9450 
 9451   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
 9452   ins_encode %{
 9453     __ movzbl($dst$$Register, $dst$$Register);
 9454   %}
 9455   ins_pipe(ialu_reg);
 9456 %}
 9457 
 9458 // And Register with Immediate 255 and promote to long
 9459 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9460 %{
 9461   match(Set dst (ConvI2L (AndI src mask)));
 9462 
 9463   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9464   ins_encode %{
 9465     __ movzbl($dst$$Register, $src$$Register);
 9466   %}
 9467   ins_pipe(ialu_reg);
 9468 %}
 9469 
 9470 // And Register with Immediate 65535
 9471 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
 9472 %{
 9473   match(Set dst (AndI dst src));
 9474 
 9475   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
 9476   ins_encode %{
 9477     __ movzwl($dst$$Register, $dst$$Register);
 9478   %}
 9479   ins_pipe(ialu_reg);
 9480 %}
 9481 
 9482 // And Register with Immediate 65535 and promote to long
 9483 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9484 %{
 9485   match(Set dst (ConvI2L (AndI src mask)));
 9486 
 9487   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9488   ins_encode %{
 9489     __ movzwl($dst$$Register, $src$$Register);
 9490   %}
 9491   ins_pipe(ialu_reg);
 9492 %}
 9493 
 9494 // Can skip int2long conversions after AND with small bitmask
 9495 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9496 %{
 9497   predicate(VM_Version::supports_bmi2());
 9498   ins_cost(125);
 9499   effect(TEMP tmp, KILL cr);
 9500   match(Set dst (ConvI2L (AndI src mask)));
 9501   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9502   ins_encode %{
 9503     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9504     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9505   %}
 9506   ins_pipe(ialu_reg_reg);
 9507 %}
 9508 
 9509 // And Register with Immediate
 9510 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9511 %{
 9512   match(Set dst (AndI dst src));
 9513   effect(KILL cr);
 9514 
 9515   format %{ "andl    $dst, $src\t# int" %}
 9516   ins_encode %{
 9517     __ andl($dst$$Register, $src$$constant);
 9518   %}
 9519   ins_pipe(ialu_reg);
 9520 %}
 9521 
 9522 // And Register with Memory
 9523 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9524 %{
 9525   match(Set dst (AndI dst (LoadI src)));
 9526   effect(KILL cr);
 9527 
 9528   ins_cost(125);
 9529   format %{ "andl    $dst, $src\t# int" %}
 9530   ins_encode %{
 9531     __ andl($dst$$Register, $src$$Address);
 9532   %}
 9533   ins_pipe(ialu_reg_mem);
 9534 %}
 9535 
 9536 // And Memory with Register
 9537 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9538 %{
 9539   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9540   effect(KILL cr);
 9541 
 9542   ins_cost(150);
 9543   format %{ "andb    $dst, $src\t# byte" %}
 9544   ins_encode %{
 9545     __ andb($dst$$Address, $src$$Register);
 9546   %}
 9547   ins_pipe(ialu_mem_reg);
 9548 %}
 9549 
 9550 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9551 %{
 9552   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9553   effect(KILL cr);
 9554 
 9555   ins_cost(150);
 9556   format %{ "andl    $dst, $src\t# int" %}
 9557   ins_encode %{
 9558     __ andl($dst$$Address, $src$$Register);
 9559   %}
 9560   ins_pipe(ialu_mem_reg);
 9561 %}
 9562 
 9563 // And Memory with Immediate
 9564 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9565 %{
 9566   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9567   effect(KILL cr);
 9568 
 9569   ins_cost(125);
 9570   format %{ "andl    $dst, $src\t# int" %}
 9571   ins_encode %{
 9572     __ andl($dst$$Address, $src$$constant);
 9573   %}
 9574   ins_pipe(ialu_mem_imm);
 9575 %}
 9576 
 9577 // BMI1 instructions
 9578 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9579   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9580   predicate(UseBMI1Instructions);
 9581   effect(KILL cr);
 9582 
 9583   ins_cost(125);
 9584   format %{ "andnl  $dst, $src1, $src2" %}
 9585 
 9586   ins_encode %{
 9587     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9588   %}
 9589   ins_pipe(ialu_reg_mem);
 9590 %}
 9591 
 9592 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9593   match(Set dst (AndI (XorI src1 minus_1) src2));
 9594   predicate(UseBMI1Instructions);
 9595   effect(KILL cr);
 9596 
 9597   format %{ "andnl  $dst, $src1, $src2" %}
 9598 
 9599   ins_encode %{
 9600     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9601   %}
 9602   ins_pipe(ialu_reg);
 9603 %}
 9604 
 9605 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9606   match(Set dst (AndI (SubI imm_zero src) src));
 9607   predicate(UseBMI1Instructions);
 9608   effect(KILL cr);
 9609 
 9610   format %{ "blsil  $dst, $src" %}
 9611 
 9612   ins_encode %{
 9613     __ blsil($dst$$Register, $src$$Register);
 9614   %}
 9615   ins_pipe(ialu_reg);
 9616 %}
 9617 
 9618 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9619   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9620   predicate(UseBMI1Instructions);
 9621   effect(KILL cr);
 9622 
 9623   ins_cost(125);
 9624   format %{ "blsil  $dst, $src" %}
 9625 
 9626   ins_encode %{
 9627     __ blsil($dst$$Register, $src$$Address);
 9628   %}
 9629   ins_pipe(ialu_reg_mem);
 9630 %}
 9631 
 9632 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9633 %{
 9634   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9635   predicate(UseBMI1Instructions);
 9636   effect(KILL cr);
 9637 
 9638   ins_cost(125);
 9639   format %{ "blsmskl $dst, $src" %}
 9640 
 9641   ins_encode %{
 9642     __ blsmskl($dst$$Register, $src$$Address);
 9643   %}
 9644   ins_pipe(ialu_reg_mem);
 9645 %}
 9646 
 9647 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9648 %{
 9649   match(Set dst (XorI (AddI src minus_1) src));
 9650   predicate(UseBMI1Instructions);
 9651   effect(KILL cr);
 9652 
 9653   format %{ "blsmskl $dst, $src" %}
 9654 
 9655   ins_encode %{
 9656     __ blsmskl($dst$$Register, $src$$Register);
 9657   %}
 9658 
 9659   ins_pipe(ialu_reg);
 9660 %}
 9661 
 9662 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9663 %{
 9664   match(Set dst (AndI (AddI src minus_1) src) );
 9665   predicate(UseBMI1Instructions);
 9666   effect(KILL cr);
 9667 
 9668   format %{ "blsrl  $dst, $src" %}
 9669 
 9670   ins_encode %{
 9671     __ blsrl($dst$$Register, $src$$Register);
 9672   %}
 9673 
 9674   ins_pipe(ialu_reg_mem);
 9675 %}
 9676 
 9677 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9678 %{
 9679   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9680   predicate(UseBMI1Instructions);
 9681   effect(KILL cr);
 9682 
 9683   ins_cost(125);
 9684   format %{ "blsrl  $dst, $src" %}
 9685 
 9686   ins_encode %{
 9687     __ blsrl($dst$$Register, $src$$Address);
 9688   %}
 9689 
 9690   ins_pipe(ialu_reg);
 9691 %}
 9692 
 9693 // Or Instructions
 9694 // Or Register with Register
 9695 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9696 %{
 9697   match(Set dst (OrI dst src));
 9698   effect(KILL cr);
 9699 
 9700   format %{ "orl     $dst, $src\t# int" %}
 9701   ins_encode %{
 9702     __ orl($dst$$Register, $src$$Register);
 9703   %}
 9704   ins_pipe(ialu_reg_reg);
 9705 %}
 9706 
 9707 // Or Register with Immediate
 9708 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9709 %{
 9710   match(Set dst (OrI dst src));
 9711   effect(KILL cr);
 9712 
 9713   format %{ "orl     $dst, $src\t# int" %}
 9714   ins_encode %{
 9715     __ orl($dst$$Register, $src$$constant);
 9716   %}
 9717   ins_pipe(ialu_reg);
 9718 %}
 9719 
 9720 // Or Register with Memory
 9721 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9722 %{
 9723   match(Set dst (OrI dst (LoadI src)));
 9724   effect(KILL cr);
 9725 
 9726   ins_cost(125);
 9727   format %{ "orl     $dst, $src\t# int" %}
 9728   ins_encode %{
 9729     __ orl($dst$$Register, $src$$Address);
 9730   %}
 9731   ins_pipe(ialu_reg_mem);
 9732 %}
 9733 
 9734 // Or Memory with Register
 9735 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9736 %{
 9737   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9738   effect(KILL cr);
 9739 
 9740   ins_cost(150);
 9741   format %{ "orb    $dst, $src\t# byte" %}
 9742   ins_encode %{
 9743     __ orb($dst$$Address, $src$$Register);
 9744   %}
 9745   ins_pipe(ialu_mem_reg);
 9746 %}
 9747 
 9748 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9749 %{
 9750   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9751   effect(KILL cr);
 9752 
 9753   ins_cost(150);
 9754   format %{ "orl     $dst, $src\t# int" %}
 9755   ins_encode %{
 9756     __ orl($dst$$Address, $src$$Register);
 9757   %}
 9758   ins_pipe(ialu_mem_reg);
 9759 %}
 9760 
 9761 // Or Memory with Immediate
 9762 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9763 %{
 9764   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9765   effect(KILL cr);
 9766 
 9767   ins_cost(125);
 9768   format %{ "orl     $dst, $src\t# int" %}
 9769   ins_encode %{
 9770     __ orl($dst$$Address, $src$$constant);
 9771   %}
 9772   ins_pipe(ialu_mem_imm);
 9773 %}
 9774 
 9775 // Xor Instructions
 9776 // Xor Register with Register
 9777 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9778 %{
 9779   match(Set dst (XorI dst src));
 9780   effect(KILL cr);
 9781 
 9782   format %{ "xorl    $dst, $src\t# int" %}
 9783   ins_encode %{
 9784     __ xorl($dst$$Register, $src$$Register);
 9785   %}
 9786   ins_pipe(ialu_reg_reg);
 9787 %}
 9788 
 9789 // Xor Register with Immediate -1
 9790 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9791   match(Set dst (XorI dst imm));
 9792 
 9793   format %{ "not    $dst" %}
 9794   ins_encode %{
 9795      __ notl($dst$$Register);
 9796   %}
 9797   ins_pipe(ialu_reg);
 9798 %}
 9799 
 9800 // Xor Register with Immediate
 9801 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9802 %{
 9803   match(Set dst (XorI dst src));
 9804   effect(KILL cr);
 9805 
 9806   format %{ "xorl    $dst, $src\t# int" %}
 9807   ins_encode %{
 9808     __ xorl($dst$$Register, $src$$constant);
 9809   %}
 9810   ins_pipe(ialu_reg);
 9811 %}
 9812 
 9813 // Xor Register with Memory
 9814 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9815 %{
 9816   match(Set dst (XorI dst (LoadI src)));
 9817   effect(KILL cr);
 9818 
 9819   ins_cost(125);
 9820   format %{ "xorl    $dst, $src\t# int" %}
 9821   ins_encode %{
 9822     __ xorl($dst$$Register, $src$$Address);
 9823   %}
 9824   ins_pipe(ialu_reg_mem);
 9825 %}
 9826 
 9827 // Xor Memory with Register
 9828 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9829 %{
 9830   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9831   effect(KILL cr);
 9832 
 9833   ins_cost(150);
 9834   format %{ "xorb    $dst, $src\t# byte" %}
 9835   ins_encode %{
 9836     __ xorb($dst$$Address, $src$$Register);
 9837   %}
 9838   ins_pipe(ialu_mem_reg);
 9839 %}
 9840 
 9841 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9842 %{
 9843   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9844   effect(KILL cr);
 9845 
 9846   ins_cost(150);
 9847   format %{ "xorl    $dst, $src\t# int" %}
 9848   ins_encode %{
 9849     __ xorl($dst$$Address, $src$$Register);
 9850   %}
 9851   ins_pipe(ialu_mem_reg);
 9852 %}
 9853 
 9854 // Xor Memory with Immediate
 9855 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9856 %{
 9857   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9858   effect(KILL cr);
 9859 
 9860   ins_cost(125);
 9861   format %{ "xorl    $dst, $src\t# int" %}
 9862   ins_encode %{
 9863     __ xorl($dst$$Address, $src$$constant);
 9864   %}
 9865   ins_pipe(ialu_mem_imm);
 9866 %}
 9867 
 9868 
 9869 // Long Logical Instructions
 9870 
 9871 // And Instructions
 9872 // And Register with Register
 9873 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9874 %{
 9875   match(Set dst (AndL dst src));
 9876   effect(KILL cr);
 9877 
 9878   format %{ "andq    $dst, $src\t# long" %}
 9879   ins_encode %{
 9880     __ andq($dst$$Register, $src$$Register);
 9881   %}
 9882   ins_pipe(ialu_reg_reg);
 9883 %}
 9884 
 9885 // And Register with Immediate 255
 9886 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
 9887 %{
 9888   match(Set dst (AndL dst src));
 9889 
 9890   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
 9891   ins_encode %{
 9892     __ movzbq($dst$$Register, $dst$$Register);
 9893   %}
 9894   ins_pipe(ialu_reg);
 9895 %}
 9896 
 9897 // And Register with Immediate 65535
 9898 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
 9899 %{
 9900   match(Set dst (AndL dst src));
 9901 
 9902   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
 9903   ins_encode %{
 9904     __ movzwq($dst$$Register, $dst$$Register);
 9905   %}
 9906   ins_pipe(ialu_reg);
 9907 %}
 9908 
 9909 // And Register with Immediate
 9910 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9911 %{
 9912   match(Set dst (AndL dst src));
 9913   effect(KILL cr);
 9914 
 9915   format %{ "andq    $dst, $src\t# long" %}
 9916   ins_encode %{
 9917     __ andq($dst$$Register, $src$$constant);
 9918   %}
 9919   ins_pipe(ialu_reg);
 9920 %}
 9921 
 9922 // And Register with Memory
 9923 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9924 %{
 9925   match(Set dst (AndL dst (LoadL src)));
 9926   effect(KILL cr);
 9927 
 9928   ins_cost(125);
 9929   format %{ "andq    $dst, $src\t# long" %}
 9930   ins_encode %{
 9931     __ andq($dst$$Register, $src$$Address);
 9932   %}
 9933   ins_pipe(ialu_reg_mem);
 9934 %}
 9935 
 9936 // And Memory with Register
 9937 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9938 %{
 9939   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9940   effect(KILL cr);
 9941 
 9942   ins_cost(150);
 9943   format %{ "andq    $dst, $src\t# long" %}
 9944   ins_encode %{
 9945     __ andq($dst$$Address, $src$$Register);
 9946   %}
 9947   ins_pipe(ialu_mem_reg);
 9948 %}
 9949 
 9950 // And Memory with Immediate
 9951 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9952 %{
 9953   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9954   effect(KILL cr);
 9955 
 9956   ins_cost(125);
 9957   format %{ "andq    $dst, $src\t# long" %}
 9958   ins_encode %{
 9959     __ andq($dst$$Address, $src$$constant);
 9960   %}
 9961   ins_pipe(ialu_mem_imm);
 9962 %}
 9963 
 9964 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9965 %{
 9966   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9967   // because AND/OR works well enough for 8/32-bit values.
 9968   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9969 
 9970   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9971   effect(KILL cr);
 9972 
 9973   ins_cost(125);
 9974   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9975   ins_encode %{
 9976     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9977   %}
 9978   ins_pipe(ialu_mem_imm);
 9979 %}
 9980 
 9981 // BMI1 instructions
 9982 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9983   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9984   predicate(UseBMI1Instructions);
 9985   effect(KILL cr);
 9986 
 9987   ins_cost(125);
 9988   format %{ "andnq  $dst, $src1, $src2" %}
 9989 
 9990   ins_encode %{
 9991     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9992   %}
 9993   ins_pipe(ialu_reg_mem);
 9994 %}
 9995 
 9996 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9997   match(Set dst (AndL (XorL src1 minus_1) src2));
 9998   predicate(UseBMI1Instructions);
 9999   effect(KILL cr);
10000 
10001   format %{ "andnq  $dst, $src1, $src2" %}
10002 
10003   ins_encode %{
10004   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10005   %}
10006   ins_pipe(ialu_reg_mem);
10007 %}
10008 
10009 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10010   match(Set dst (AndL (SubL imm_zero src) src));
10011   predicate(UseBMI1Instructions);
10012   effect(KILL cr);
10013 
10014   format %{ "blsiq  $dst, $src" %}
10015 
10016   ins_encode %{
10017     __ blsiq($dst$$Register, $src$$Register);
10018   %}
10019   ins_pipe(ialu_reg);
10020 %}
10021 
10022 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10023   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10024   predicate(UseBMI1Instructions);
10025   effect(KILL cr);
10026 
10027   ins_cost(125);
10028   format %{ "blsiq  $dst, $src" %}
10029 
10030   ins_encode %{
10031     __ blsiq($dst$$Register, $src$$Address);
10032   %}
10033   ins_pipe(ialu_reg_mem);
10034 %}
10035 
10036 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10037 %{
10038   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10039   predicate(UseBMI1Instructions);
10040   effect(KILL cr);
10041 
10042   ins_cost(125);
10043   format %{ "blsmskq $dst, $src" %}
10044 
10045   ins_encode %{
10046     __ blsmskq($dst$$Register, $src$$Address);
10047   %}
10048   ins_pipe(ialu_reg_mem);
10049 %}
10050 
10051 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10052 %{
10053   match(Set dst (XorL (AddL src minus_1) src));
10054   predicate(UseBMI1Instructions);
10055   effect(KILL cr);
10056 
10057   format %{ "blsmskq $dst, $src" %}
10058 
10059   ins_encode %{
10060     __ blsmskq($dst$$Register, $src$$Register);
10061   %}
10062 
10063   ins_pipe(ialu_reg);
10064 %}
10065 
10066 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10067 %{
10068   match(Set dst (AndL (AddL src minus_1) src) );
10069   predicate(UseBMI1Instructions);
10070   effect(KILL cr);
10071 
10072   format %{ "blsrq  $dst, $src" %}
10073 
10074   ins_encode %{
10075     __ blsrq($dst$$Register, $src$$Register);
10076   %}
10077 
10078   ins_pipe(ialu_reg);
10079 %}
10080 
10081 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10082 %{
10083   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10084   predicate(UseBMI1Instructions);
10085   effect(KILL cr);
10086 
10087   ins_cost(125);
10088   format %{ "blsrq  $dst, $src" %}
10089 
10090   ins_encode %{
10091     __ blsrq($dst$$Register, $src$$Address);
10092   %}
10093 
10094   ins_pipe(ialu_reg);
10095 %}
10096 
10097 // Or Instructions
10098 // Or Register with Register
10099 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10100 %{
10101   match(Set dst (OrL dst src));
10102   effect(KILL cr);
10103 
10104   format %{ "orq     $dst, $src\t# long" %}
10105   ins_encode %{
10106     __ orq($dst$$Register, $src$$Register);
10107   %}
10108   ins_pipe(ialu_reg_reg);
10109 %}
10110 
10111 // Use any_RegP to match R15 (TLS register) without spilling.
10112 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10113   match(Set dst (OrL dst (CastP2X src)));
10114   effect(KILL cr);
10115 
10116   format %{ "orq     $dst, $src\t# long" %}
10117   ins_encode %{
10118     __ orq($dst$$Register, $src$$Register);
10119   %}
10120   ins_pipe(ialu_reg_reg);
10121 %}
10122 
10123 
10124 // Or Register with Immediate
10125 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10126 %{
10127   match(Set dst (OrL dst src));
10128   effect(KILL cr);
10129 
10130   format %{ "orq     $dst, $src\t# long" %}
10131   ins_encode %{
10132     __ orq($dst$$Register, $src$$constant);
10133   %}
10134   ins_pipe(ialu_reg);
10135 %}
10136 
10137 // Or Register with Memory
10138 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10139 %{
10140   match(Set dst (OrL dst (LoadL src)));
10141   effect(KILL cr);
10142 
10143   ins_cost(125);
10144   format %{ "orq     $dst, $src\t# long" %}
10145   ins_encode %{
10146     __ orq($dst$$Register, $src$$Address);
10147   %}
10148   ins_pipe(ialu_reg_mem);
10149 %}
10150 
10151 // Or Memory with Register
10152 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10153 %{
10154   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10155   effect(KILL cr);
10156 
10157   ins_cost(150);
10158   format %{ "orq     $dst, $src\t# long" %}
10159   ins_encode %{
10160     __ orq($dst$$Address, $src$$Register);
10161   %}
10162   ins_pipe(ialu_mem_reg);
10163 %}
10164 
10165 // Or Memory with Immediate
10166 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10167 %{
10168   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10169   effect(KILL cr);
10170 
10171   ins_cost(125);
10172   format %{ "orq     $dst, $src\t# long" %}
10173   ins_encode %{
10174     __ orq($dst$$Address, $src$$constant);
10175   %}
10176   ins_pipe(ialu_mem_imm);
10177 %}
10178 
10179 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10180 %{
10181   // con should be a pure 64-bit power of 2 immediate
10182   // because AND/OR works well enough for 8/32-bit values.
10183   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10184 
10185   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10186   effect(KILL cr);
10187 
10188   ins_cost(125);
10189   format %{ "btsq    $dst, log2($con)\t# long" %}
10190   ins_encode %{
10191     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10192   %}
10193   ins_pipe(ialu_mem_imm);
10194 %}
10195 
10196 // Xor Instructions
10197 // Xor Register with Register
10198 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10199 %{
10200   match(Set dst (XorL dst src));
10201   effect(KILL cr);
10202 
10203   format %{ "xorq    $dst, $src\t# long" %}
10204   ins_encode %{
10205     __ xorq($dst$$Register, $src$$Register);
10206   %}
10207   ins_pipe(ialu_reg_reg);
10208 %}
10209 
10210 // Xor Register with Immediate -1
10211 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10212   match(Set dst (XorL dst imm));
10213 
10214   format %{ "notq   $dst" %}
10215   ins_encode %{
10216      __ notq($dst$$Register);
10217   %}
10218   ins_pipe(ialu_reg);
10219 %}
10220 
10221 // Xor Register with Immediate
10222 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10223 %{
10224   match(Set dst (XorL dst src));
10225   effect(KILL cr);
10226 
10227   format %{ "xorq    $dst, $src\t# long" %}
10228   ins_encode %{
10229     __ xorq($dst$$Register, $src$$constant);
10230   %}
10231   ins_pipe(ialu_reg);
10232 %}
10233 
10234 // Xor Register with Memory
10235 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10236 %{
10237   match(Set dst (XorL dst (LoadL src)));
10238   effect(KILL cr);
10239 
10240   ins_cost(125);
10241   format %{ "xorq    $dst, $src\t# long" %}
10242   ins_encode %{
10243     __ xorq($dst$$Register, $src$$Address);
10244   %}
10245   ins_pipe(ialu_reg_mem);
10246 %}
10247 
10248 // Xor Memory with Register
10249 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10250 %{
10251   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10252   effect(KILL cr);
10253 
10254   ins_cost(150);
10255   format %{ "xorq    $dst, $src\t# long" %}
10256   ins_encode %{
10257     __ xorq($dst$$Address, $src$$Register);
10258   %}
10259   ins_pipe(ialu_mem_reg);
10260 %}
10261 
10262 // Xor Memory with Immediate
10263 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10264 %{
10265   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10266   effect(KILL cr);
10267 
10268   ins_cost(125);
10269   format %{ "xorq    $dst, $src\t# long" %}
10270   ins_encode %{
10271     __ xorq($dst$$Address, $src$$constant);
10272   %}
10273   ins_pipe(ialu_mem_imm);
10274 %}
10275 
10276 // Convert Int to Boolean
10277 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10278 %{
10279   match(Set dst (Conv2B src));
10280   effect(KILL cr);
10281 
10282   format %{ "testl   $src, $src\t# ci2b\n\t"
10283             "setnz   $dst\n\t"
10284             "movzbl  $dst, $dst" %}
10285   ins_encode %{
10286     __ testl($src$$Register, $src$$Register);
10287     __ set_byte_if_not_zero($dst$$Register);
10288     __ movzbl($dst$$Register, $dst$$Register);
10289   %}
10290   ins_pipe(pipe_slow); // XXX
10291 %}
10292 
10293 // Convert Pointer to Boolean
10294 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10295 %{
10296   match(Set dst (Conv2B src));
10297   effect(KILL cr);
10298 
10299   format %{ "testq   $src, $src\t# cp2b\n\t"
10300             "setnz   $dst\n\t"
10301             "movzbl  $dst, $dst" %}
10302   ins_encode %{
10303     __ testq($src$$Register, $src$$Register);
10304     __ set_byte_if_not_zero($dst$$Register);
10305     __ movzbl($dst$$Register, $dst$$Register);
10306   %}
10307   ins_pipe(pipe_slow); // XXX
10308 %}
10309 
10310 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10311 %{
10312   match(Set dst (CmpLTMask p q));
10313   effect(KILL cr);
10314 
10315   ins_cost(400);
10316   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10317             "setlt   $dst\n\t"
10318             "movzbl  $dst, $dst\n\t"
10319             "negl    $dst" %}
10320   ins_encode %{
10321     __ cmpl($p$$Register, $q$$Register);
10322     __ setl($dst$$Register);
10323     __ movzbl($dst$$Register, $dst$$Register);
10324     __ negl($dst$$Register);
10325   %}
10326   ins_pipe(pipe_slow);
10327 %}
10328 
10329 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10330 %{
10331   match(Set dst (CmpLTMask dst zero));
10332   effect(KILL cr);
10333 
10334   ins_cost(100);
10335   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10336   ins_encode %{
10337     __ sarl($dst$$Register, 31);
10338   %}
10339   ins_pipe(ialu_reg);
10340 %}
10341 
10342 /* Better to save a register than avoid a branch */
10343 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10344 %{
10345   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10346   effect(KILL cr);
10347   ins_cost(300);
10348   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10349             "jge     done\n\t"
10350             "addl    $p,$y\n"
10351             "done:   " %}
10352   ins_encode %{
10353     Register Rp = $p$$Register;
10354     Register Rq = $q$$Register;
10355     Register Ry = $y$$Register;
10356     Label done;
10357     __ subl(Rp, Rq);
10358     __ jccb(Assembler::greaterEqual, done);
10359     __ addl(Rp, Ry);
10360     __ bind(done);
10361   %}
10362   ins_pipe(pipe_cmplt);
10363 %}
10364 
10365 /* Better to save a register than avoid a branch */
10366 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10367 %{
10368   match(Set y (AndI (CmpLTMask p q) y));
10369   effect(KILL cr);
10370 
10371   ins_cost(300);
10372 
10373   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10374             "jlt     done\n\t"
10375             "xorl    $y, $y\n"
10376             "done:   " %}
10377   ins_encode %{
10378     Register Rp = $p$$Register;
10379     Register Rq = $q$$Register;
10380     Register Ry = $y$$Register;
10381     Label done;
10382     __ cmpl(Rp, Rq);
10383     __ jccb(Assembler::less, done);
10384     __ xorl(Ry, Ry);
10385     __ bind(done);
10386   %}
10387   ins_pipe(pipe_cmplt);
10388 %}
10389 
10390 
10391 //---------- FP Instructions------------------------------------------------
10392 
10393 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10394 %{
10395   match(Set cr (CmpF src1 src2));
10396 
10397   ins_cost(145);
10398   format %{ "ucomiss $src1, $src2\n\t"
10399             "jnp,s   exit\n\t"
10400             "pushfq\t# saw NaN, set CF\n\t"
10401             "andq    [rsp], #0xffffff2b\n\t"
10402             "popfq\n"
10403     "exit:" %}
10404   ins_encode %{
10405     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10406     emit_cmpfp_fixup(_masm);
10407   %}
10408   ins_pipe(pipe_slow);
10409 %}
10410 
10411 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10412   match(Set cr (CmpF src1 src2));
10413 
10414   ins_cost(100);
10415   format %{ "ucomiss $src1, $src2" %}
10416   ins_encode %{
10417     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10418   %}
10419   ins_pipe(pipe_slow);
10420 %}
10421 
10422 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10423 %{
10424   match(Set cr (CmpF src1 (LoadF src2)));
10425 
10426   ins_cost(145);
10427   format %{ "ucomiss $src1, $src2\n\t"
10428             "jnp,s   exit\n\t"
10429             "pushfq\t# saw NaN, set CF\n\t"
10430             "andq    [rsp], #0xffffff2b\n\t"
10431             "popfq\n"
10432     "exit:" %}
10433   ins_encode %{
10434     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10435     emit_cmpfp_fixup(_masm);
10436   %}
10437   ins_pipe(pipe_slow);
10438 %}
10439 
10440 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10441   match(Set cr (CmpF src1 (LoadF src2)));
10442 
10443   ins_cost(100);
10444   format %{ "ucomiss $src1, $src2" %}
10445   ins_encode %{
10446     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10447   %}
10448   ins_pipe(pipe_slow);
10449 %}
10450 
10451 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10452   match(Set cr (CmpF src con));
10453 
10454   ins_cost(145);
10455   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10456             "jnp,s   exit\n\t"
10457             "pushfq\t# saw NaN, set CF\n\t"
10458             "andq    [rsp], #0xffffff2b\n\t"
10459             "popfq\n"
10460     "exit:" %}
10461   ins_encode %{
10462     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10463     emit_cmpfp_fixup(_masm);
10464   %}
10465   ins_pipe(pipe_slow);
10466 %}
10467 
10468 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10469   match(Set cr (CmpF src con));
10470   ins_cost(100);
10471   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10472   ins_encode %{
10473     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10474   %}
10475   ins_pipe(pipe_slow);
10476 %}
10477 
10478 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10479 %{
10480   match(Set cr (CmpD src1 src2));
10481 
10482   ins_cost(145);
10483   format %{ "ucomisd $src1, $src2\n\t"
10484             "jnp,s   exit\n\t"
10485             "pushfq\t# saw NaN, set CF\n\t"
10486             "andq    [rsp], #0xffffff2b\n\t"
10487             "popfq\n"
10488     "exit:" %}
10489   ins_encode %{
10490     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10491     emit_cmpfp_fixup(_masm);
10492   %}
10493   ins_pipe(pipe_slow);
10494 %}
10495 
10496 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10497   match(Set cr (CmpD src1 src2));
10498 
10499   ins_cost(100);
10500   format %{ "ucomisd $src1, $src2 test" %}
10501   ins_encode %{
10502     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10503   %}
10504   ins_pipe(pipe_slow);
10505 %}
10506 
10507 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10508 %{
10509   match(Set cr (CmpD src1 (LoadD src2)));
10510 
10511   ins_cost(145);
10512   format %{ "ucomisd $src1, $src2\n\t"
10513             "jnp,s   exit\n\t"
10514             "pushfq\t# saw NaN, set CF\n\t"
10515             "andq    [rsp], #0xffffff2b\n\t"
10516             "popfq\n"
10517     "exit:" %}
10518   ins_encode %{
10519     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10520     emit_cmpfp_fixup(_masm);
10521   %}
10522   ins_pipe(pipe_slow);
10523 %}
10524 
10525 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10526   match(Set cr (CmpD src1 (LoadD src2)));
10527 
10528   ins_cost(100);
10529   format %{ "ucomisd $src1, $src2" %}
10530   ins_encode %{
10531     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10532   %}
10533   ins_pipe(pipe_slow);
10534 %}
10535 
10536 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10537   match(Set cr (CmpD src con));
10538 
10539   ins_cost(145);
10540   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10541             "jnp,s   exit\n\t"
10542             "pushfq\t# saw NaN, set CF\n\t"
10543             "andq    [rsp], #0xffffff2b\n\t"
10544             "popfq\n"
10545     "exit:" %}
10546   ins_encode %{
10547     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10548     emit_cmpfp_fixup(_masm);
10549   %}
10550   ins_pipe(pipe_slow);
10551 %}
10552 
10553 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10554   match(Set cr (CmpD src con));
10555   ins_cost(100);
10556   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10557   ins_encode %{
10558     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10559   %}
10560   ins_pipe(pipe_slow);
10561 %}
10562 
10563 // Compare into -1,0,1
10564 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10565 %{
10566   match(Set dst (CmpF3 src1 src2));
10567   effect(KILL cr);
10568 
10569   ins_cost(275);
10570   format %{ "ucomiss $src1, $src2\n\t"
10571             "movl    $dst, #-1\n\t"
10572             "jp,s    done\n\t"
10573             "jb,s    done\n\t"
10574             "setne   $dst\n\t"
10575             "movzbl  $dst, $dst\n"
10576     "done:" %}
10577   ins_encode %{
10578     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10579     emit_cmpfp3(_masm, $dst$$Register);
10580   %}
10581   ins_pipe(pipe_slow);
10582 %}
10583 
10584 // Compare into -1,0,1
10585 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10586 %{
10587   match(Set dst (CmpF3 src1 (LoadF src2)));
10588   effect(KILL cr);
10589 
10590   ins_cost(275);
10591   format %{ "ucomiss $src1, $src2\n\t"
10592             "movl    $dst, #-1\n\t"
10593             "jp,s    done\n\t"
10594             "jb,s    done\n\t"
10595             "setne   $dst\n\t"
10596             "movzbl  $dst, $dst\n"
10597     "done:" %}
10598   ins_encode %{
10599     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10600     emit_cmpfp3(_masm, $dst$$Register);
10601   %}
10602   ins_pipe(pipe_slow);
10603 %}
10604 
10605 // Compare into -1,0,1
10606 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10607   match(Set dst (CmpF3 src con));
10608   effect(KILL cr);
10609 
10610   ins_cost(275);
10611   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10612             "movl    $dst, #-1\n\t"
10613             "jp,s    done\n\t"
10614             "jb,s    done\n\t"
10615             "setne   $dst\n\t"
10616             "movzbl  $dst, $dst\n"
10617     "done:" %}
10618   ins_encode %{
10619     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10620     emit_cmpfp3(_masm, $dst$$Register);
10621   %}
10622   ins_pipe(pipe_slow);
10623 %}
10624 
10625 // Compare into -1,0,1
10626 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10627 %{
10628   match(Set dst (CmpD3 src1 src2));
10629   effect(KILL cr);
10630 
10631   ins_cost(275);
10632   format %{ "ucomisd $src1, $src2\n\t"
10633             "movl    $dst, #-1\n\t"
10634             "jp,s    done\n\t"
10635             "jb,s    done\n\t"
10636             "setne   $dst\n\t"
10637             "movzbl  $dst, $dst\n"
10638     "done:" %}
10639   ins_encode %{
10640     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10641     emit_cmpfp3(_masm, $dst$$Register);
10642   %}
10643   ins_pipe(pipe_slow);
10644 %}
10645 
10646 // Compare into -1,0,1
10647 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10648 %{
10649   match(Set dst (CmpD3 src1 (LoadD src2)));
10650   effect(KILL cr);
10651 
10652   ins_cost(275);
10653   format %{ "ucomisd $src1, $src2\n\t"
10654             "movl    $dst, #-1\n\t"
10655             "jp,s    done\n\t"
10656             "jb,s    done\n\t"
10657             "setne   $dst\n\t"
10658             "movzbl  $dst, $dst\n"
10659     "done:" %}
10660   ins_encode %{
10661     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10662     emit_cmpfp3(_masm, $dst$$Register);
10663   %}
10664   ins_pipe(pipe_slow);
10665 %}
10666 
10667 // Compare into -1,0,1
10668 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10669   match(Set dst (CmpD3 src con));
10670   effect(KILL cr);
10671 
10672   ins_cost(275);
10673   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10674             "movl    $dst, #-1\n\t"
10675             "jp,s    done\n\t"
10676             "jb,s    done\n\t"
10677             "setne   $dst\n\t"
10678             "movzbl  $dst, $dst\n"
10679     "done:" %}
10680   ins_encode %{
10681     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10682     emit_cmpfp3(_masm, $dst$$Register);
10683   %}
10684   ins_pipe(pipe_slow);
10685 %}
10686 
10687 //----------Arithmetic Conversion Instructions---------------------------------
10688 
10689 instruct convF2D_reg_reg(regD dst, regF src)
10690 %{
10691   match(Set dst (ConvF2D src));
10692 
10693   format %{ "cvtss2sd $dst, $src" %}
10694   ins_encode %{
10695     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10696   %}
10697   ins_pipe(pipe_slow); // XXX
10698 %}
10699 
10700 instruct convF2D_reg_mem(regD dst, memory src)
10701 %{
10702   match(Set dst (ConvF2D (LoadF src)));
10703 
10704   format %{ "cvtss2sd $dst, $src" %}
10705   ins_encode %{
10706     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10707   %}
10708   ins_pipe(pipe_slow); // XXX
10709 %}
10710 
10711 instruct convD2F_reg_reg(regF dst, regD src)
10712 %{
10713   match(Set dst (ConvD2F src));
10714 
10715   format %{ "cvtsd2ss $dst, $src" %}
10716   ins_encode %{
10717     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10718   %}
10719   ins_pipe(pipe_slow); // XXX
10720 %}
10721 
10722 instruct convD2F_reg_mem(regF dst, memory src)
10723 %{
10724   match(Set dst (ConvD2F (LoadD src)));
10725 
10726   format %{ "cvtsd2ss $dst, $src" %}
10727   ins_encode %{
10728     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10729   %}
10730   ins_pipe(pipe_slow); // XXX
10731 %}
10732 
10733 // XXX do mem variants
10734 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10735 %{
10736   match(Set dst (ConvF2I src));
10737   effect(KILL cr);
10738   format %{ "convert_f2i $dst,$src" %}
10739   ins_encode %{
10740     __ convert_f2i($dst$$Register, $src$$XMMRegister);
10741   %}
10742   ins_pipe(pipe_slow);
10743 %}
10744 
10745 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10746 %{
10747   match(Set dst (ConvF2L src));
10748   effect(KILL cr);
10749   format %{ "convert_f2l $dst,$src"%}
10750   ins_encode %{
10751     __ convert_f2l($dst$$Register, $src$$XMMRegister);
10752   %}
10753   ins_pipe(pipe_slow);
10754 %}
10755 
10756 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10757 %{
10758   match(Set dst (ConvD2I src));
10759   effect(KILL cr);
10760   format %{ "convert_d2i $dst,$src"%}
10761   ins_encode %{
10762     __ convert_d2i($dst$$Register, $src$$XMMRegister);
10763   %}
10764   ins_pipe(pipe_slow);
10765 %}
10766 
10767 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10768 %{
10769   match(Set dst (ConvD2L src));
10770   effect(KILL cr);
10771   format %{ "convert_d2l $dst,$src"%}
10772   ins_encode %{
10773     __ convert_d2l($dst$$Register, $src$$XMMRegister);
10774   %}
10775   ins_pipe(pipe_slow);
10776 %}
10777 
10778 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10779 %{
10780   match(Set dst (RoundD src));
10781   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10782   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10783   ins_encode %{
10784     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10785   %}
10786   ins_pipe(pipe_slow);
10787 %}
10788 
10789 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10790 %{
10791   match(Set dst (RoundF src));
10792   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10793   format %{ "round_float $dst,$src" %}
10794   ins_encode %{
10795     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10796   %}
10797   ins_pipe(pipe_slow);
10798 %}
10799 
10800 instruct convI2F_reg_reg(regF dst, rRegI src)
10801 %{
10802   predicate(!UseXmmI2F);
10803   match(Set dst (ConvI2F src));
10804 
10805   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10806   ins_encode %{
10807     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10808   %}
10809   ins_pipe(pipe_slow); // XXX
10810 %}
10811 
10812 instruct convI2F_reg_mem(regF dst, memory src)
10813 %{
10814   match(Set dst (ConvI2F (LoadI src)));
10815 
10816   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10817   ins_encode %{
10818     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10819   %}
10820   ins_pipe(pipe_slow); // XXX
10821 %}
10822 
10823 instruct convI2D_reg_reg(regD dst, rRegI src)
10824 %{
10825   predicate(!UseXmmI2D);
10826   match(Set dst (ConvI2D src));
10827 
10828   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10829   ins_encode %{
10830     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10831   %}
10832   ins_pipe(pipe_slow); // XXX
10833 %}
10834 
10835 instruct convI2D_reg_mem(regD dst, memory src)
10836 %{
10837   match(Set dst (ConvI2D (LoadI src)));
10838 
10839   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10840   ins_encode %{
10841     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10842   %}
10843   ins_pipe(pipe_slow); // XXX
10844 %}
10845 
10846 instruct convXI2F_reg(regF dst, rRegI src)
10847 %{
10848   predicate(UseXmmI2F);
10849   match(Set dst (ConvI2F src));
10850 
10851   format %{ "movdl $dst, $src\n\t"
10852             "cvtdq2psl $dst, $dst\t# i2f" %}
10853   ins_encode %{
10854     __ movdl($dst$$XMMRegister, $src$$Register);
10855     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10856   %}
10857   ins_pipe(pipe_slow); // XXX
10858 %}
10859 
10860 instruct convXI2D_reg(regD dst, rRegI src)
10861 %{
10862   predicate(UseXmmI2D);
10863   match(Set dst (ConvI2D src));
10864 
10865   format %{ "movdl $dst, $src\n\t"
10866             "cvtdq2pdl $dst, $dst\t# i2d" %}
10867   ins_encode %{
10868     __ movdl($dst$$XMMRegister, $src$$Register);
10869     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10870   %}
10871   ins_pipe(pipe_slow); // XXX
10872 %}
10873 
10874 instruct convL2F_reg_reg(regF dst, rRegL src)
10875 %{
10876   match(Set dst (ConvL2F src));
10877 
10878   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10879   ins_encode %{
10880     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10881   %}
10882   ins_pipe(pipe_slow); // XXX
10883 %}
10884 
10885 instruct convL2F_reg_mem(regF dst, memory src)
10886 %{
10887   match(Set dst (ConvL2F (LoadL src)));
10888 
10889   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10890   ins_encode %{
10891     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10892   %}
10893   ins_pipe(pipe_slow); // XXX
10894 %}
10895 
10896 instruct convL2D_reg_reg(regD dst, rRegL src)
10897 %{
10898   match(Set dst (ConvL2D src));
10899 
10900   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10901   ins_encode %{
10902     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10903   %}
10904   ins_pipe(pipe_slow); // XXX
10905 %}
10906 
10907 instruct convL2D_reg_mem(regD dst, memory src)
10908 %{
10909   match(Set dst (ConvL2D (LoadL src)));
10910 
10911   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10912   ins_encode %{
10913     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10914   %}
10915   ins_pipe(pipe_slow); // XXX
10916 %}
10917 
10918 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10919 %{
10920   match(Set dst (ConvI2L src));
10921 
10922   ins_cost(125);
10923   format %{ "movslq  $dst, $src\t# i2l" %}
10924   ins_encode %{
10925     __ movslq($dst$$Register, $src$$Register);
10926   %}
10927   ins_pipe(ialu_reg_reg);
10928 %}
10929 
10930 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10931 // %{
10932 //   match(Set dst (ConvI2L src));
10933 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10934 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10935 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10936 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10937 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10938 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10939 
10940 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10941 //   ins_encode(enc_copy(dst, src));
10942 // //   opcode(0x63); // needs REX.W
10943 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10944 //   ins_pipe(ialu_reg_reg);
10945 // %}
10946 
10947 // Zero-extend convert int to long
10948 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10949 %{
10950   match(Set dst (AndL (ConvI2L src) mask));
10951 
10952   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10953   ins_encode %{
10954     if ($dst$$reg != $src$$reg) {
10955       __ movl($dst$$Register, $src$$Register);
10956     }
10957   %}
10958   ins_pipe(ialu_reg_reg);
10959 %}
10960 
10961 // Zero-extend convert int to long
10962 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10963 %{
10964   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10965 
10966   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10967   ins_encode %{
10968     __ movl($dst$$Register, $src$$Address);
10969   %}
10970   ins_pipe(ialu_reg_mem);
10971 %}
10972 
10973 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10974 %{
10975   match(Set dst (AndL src mask));
10976 
10977   format %{ "movl    $dst, $src\t# zero-extend long" %}
10978   ins_encode %{
10979     __ movl($dst$$Register, $src$$Register);
10980   %}
10981   ins_pipe(ialu_reg_reg);
10982 %}
10983 
10984 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10985 %{
10986   match(Set dst (ConvL2I src));
10987 
10988   format %{ "movl    $dst, $src\t# l2i" %}
10989   ins_encode %{
10990     __ movl($dst$$Register, $src$$Register);
10991   %}
10992   ins_pipe(ialu_reg_reg);
10993 %}
10994 
10995 
10996 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10997   match(Set dst (MoveF2I src));
10998   effect(DEF dst, USE src);
10999 
11000   ins_cost(125);
11001   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11002   ins_encode %{
11003     __ movl($dst$$Register, Address(rsp, $src$$disp));
11004   %}
11005   ins_pipe(ialu_reg_mem);
11006 %}
11007 
11008 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11009   match(Set dst (MoveI2F src));
11010   effect(DEF dst, USE src);
11011 
11012   ins_cost(125);
11013   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11014   ins_encode %{
11015     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11016   %}
11017   ins_pipe(pipe_slow);
11018 %}
11019 
11020 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11021   match(Set dst (MoveD2L src));
11022   effect(DEF dst, USE src);
11023 
11024   ins_cost(125);
11025   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11026   ins_encode %{
11027     __ movq($dst$$Register, Address(rsp, $src$$disp));
11028   %}
11029   ins_pipe(ialu_reg_mem);
11030 %}
11031 
11032 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11033   predicate(!UseXmmLoadAndClearUpper);
11034   match(Set dst (MoveL2D src));
11035   effect(DEF dst, USE src);
11036 
11037   ins_cost(125);
11038   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11039   ins_encode %{
11040     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11041   %}
11042   ins_pipe(pipe_slow);
11043 %}
11044 
11045 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11046   predicate(UseXmmLoadAndClearUpper);
11047   match(Set dst (MoveL2D src));
11048   effect(DEF dst, USE src);
11049 
11050   ins_cost(125);
11051   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11052   ins_encode %{
11053     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11054   %}
11055   ins_pipe(pipe_slow);
11056 %}
11057 
11058 
11059 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11060   match(Set dst (MoveF2I src));
11061   effect(DEF dst, USE src);
11062 
11063   ins_cost(95); // XXX
11064   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11065   ins_encode %{
11066     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11067   %}
11068   ins_pipe(pipe_slow);
11069 %}
11070 
11071 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11072   match(Set dst (MoveI2F src));
11073   effect(DEF dst, USE src);
11074 
11075   ins_cost(100);
11076   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11077   ins_encode %{
11078     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11079   %}
11080   ins_pipe( ialu_mem_reg );
11081 %}
11082 
11083 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11084   match(Set dst (MoveD2L src));
11085   effect(DEF dst, USE src);
11086 
11087   ins_cost(95); // XXX
11088   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11089   ins_encode %{
11090     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11091   %}
11092   ins_pipe(pipe_slow);
11093 %}
11094 
11095 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11096   match(Set dst (MoveL2D src));
11097   effect(DEF dst, USE src);
11098 
11099   ins_cost(100);
11100   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11101   ins_encode %{
11102     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11103   %}
11104   ins_pipe(ialu_mem_reg);
11105 %}
11106 
11107 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11108   match(Set dst (MoveF2I src));
11109   effect(DEF dst, USE src);
11110   ins_cost(85);
11111   format %{ "movd    $dst,$src\t# MoveF2I" %}
11112   ins_encode %{
11113     __ movdl($dst$$Register, $src$$XMMRegister);
11114   %}
11115   ins_pipe( pipe_slow );
11116 %}
11117 
11118 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11119   match(Set dst (MoveD2L src));
11120   effect(DEF dst, USE src);
11121   ins_cost(85);
11122   format %{ "movd    $dst,$src\t# MoveD2L" %}
11123   ins_encode %{
11124     __ movdq($dst$$Register, $src$$XMMRegister);
11125   %}
11126   ins_pipe( pipe_slow );
11127 %}
11128 
11129 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11130   match(Set dst (MoveI2F src));
11131   effect(DEF dst, USE src);
11132   ins_cost(100);
11133   format %{ "movd    $dst,$src\t# MoveI2F" %}
11134   ins_encode %{
11135     __ movdl($dst$$XMMRegister, $src$$Register);
11136   %}
11137   ins_pipe( pipe_slow );
11138 %}
11139 
11140 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11141   match(Set dst (MoveL2D src));
11142   effect(DEF dst, USE src);
11143   ins_cost(100);
11144   format %{ "movd    $dst,$src\t# MoveL2D" %}
11145   ins_encode %{
11146      __ movdq($dst$$XMMRegister, $src$$Register);
11147   %}
11148   ins_pipe( pipe_slow );
11149 %}
11150 
11151 // Fast clearing of an array
11152 // Small ClearArray non-AVX512.
11153 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11154                   Universe dummy, rFlagsReg cr)
11155 %{
11156   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11157   match(Set dummy (ClearArray cnt base));
11158   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11159 
11160   format %{ $$template
11161     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11162     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11163     $$emit$$"jg      LARGE\n\t"
11164     $$emit$$"dec     rcx\n\t"
11165     $$emit$$"js      DONE\t# Zero length\n\t"
11166     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11167     $$emit$$"dec     rcx\n\t"
11168     $$emit$$"jge     LOOP\n\t"
11169     $$emit$$"jmp     DONE\n\t"
11170     $$emit$$"# LARGE:\n\t"
11171     if (UseFastStosb) {
11172        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11173        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11174     } else if (UseXMMForObjInit) {
11175        $$emit$$"mov     rdi,rax\n\t"
11176        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11177        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11178        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11179        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11180        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11181        $$emit$$"add     0x40,rax\n\t"
11182        $$emit$$"# L_zero_64_bytes:\n\t"
11183        $$emit$$"sub     0x8,rcx\n\t"
11184        $$emit$$"jge     L_loop\n\t"
11185        $$emit$$"add     0x4,rcx\n\t"
11186        $$emit$$"jl      L_tail\n\t"
11187        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11188        $$emit$$"add     0x20,rax\n\t"
11189        $$emit$$"sub     0x4,rcx\n\t"
11190        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11191        $$emit$$"add     0x4,rcx\n\t"
11192        $$emit$$"jle     L_end\n\t"
11193        $$emit$$"dec     rcx\n\t"
11194        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11195        $$emit$$"vmovq   xmm0,(rax)\n\t"
11196        $$emit$$"add     0x8,rax\n\t"
11197        $$emit$$"dec     rcx\n\t"
11198        $$emit$$"jge     L_sloop\n\t"
11199        $$emit$$"# L_end:\n\t"
11200     } else {
11201        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11202     }
11203     $$emit$$"# DONE"
11204   %}
11205   ins_encode %{
11206     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11207                  $tmp$$XMMRegister, false, knoreg);
11208   %}
11209   ins_pipe(pipe_slow);
11210 %}
11211 
11212 // Small ClearArray AVX512 non-constant length.
11213 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11214                        Universe dummy, rFlagsReg cr)
11215 %{
11216   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11217   match(Set dummy (ClearArray cnt base));
11218   ins_cost(125);
11219   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11220 
11221   format %{ $$template
11222     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11223     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11224     $$emit$$"jg      LARGE\n\t"
11225     $$emit$$"dec     rcx\n\t"
11226     $$emit$$"js      DONE\t# Zero length\n\t"
11227     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11228     $$emit$$"dec     rcx\n\t"
11229     $$emit$$"jge     LOOP\n\t"
11230     $$emit$$"jmp     DONE\n\t"
11231     $$emit$$"# LARGE:\n\t"
11232     if (UseFastStosb) {
11233        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11234        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11235     } else if (UseXMMForObjInit) {
11236        $$emit$$"mov     rdi,rax\n\t"
11237        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11238        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11239        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11240        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11241        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11242        $$emit$$"add     0x40,rax\n\t"
11243        $$emit$$"# L_zero_64_bytes:\n\t"
11244        $$emit$$"sub     0x8,rcx\n\t"
11245        $$emit$$"jge     L_loop\n\t"
11246        $$emit$$"add     0x4,rcx\n\t"
11247        $$emit$$"jl      L_tail\n\t"
11248        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11249        $$emit$$"add     0x20,rax\n\t"
11250        $$emit$$"sub     0x4,rcx\n\t"
11251        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11252        $$emit$$"add     0x4,rcx\n\t"
11253        $$emit$$"jle     L_end\n\t"
11254        $$emit$$"dec     rcx\n\t"
11255        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11256        $$emit$$"vmovq   xmm0,(rax)\n\t"
11257        $$emit$$"add     0x8,rax\n\t"
11258        $$emit$$"dec     rcx\n\t"
11259        $$emit$$"jge     L_sloop\n\t"
11260        $$emit$$"# L_end:\n\t"
11261     } else {
11262        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11263     }
11264     $$emit$$"# DONE"
11265   %}
11266   ins_encode %{
11267     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11268                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11269   %}
11270   ins_pipe(pipe_slow);
11271 %}
11272 
11273 // Large ClearArray non-AVX512.
11274 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11275                         Universe dummy, rFlagsReg cr)
11276 %{
11277   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11278   match(Set dummy (ClearArray cnt base));
11279   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11280 
11281   format %{ $$template
11282     if (UseFastStosb) {
11283        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11284        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11285        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11286     } else if (UseXMMForObjInit) {
11287        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11288        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11289        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11290        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11291        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11292        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11293        $$emit$$"add     0x40,rax\n\t"
11294        $$emit$$"# L_zero_64_bytes:\n\t"
11295        $$emit$$"sub     0x8,rcx\n\t"
11296        $$emit$$"jge     L_loop\n\t"
11297        $$emit$$"add     0x4,rcx\n\t"
11298        $$emit$$"jl      L_tail\n\t"
11299        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11300        $$emit$$"add     0x20,rax\n\t"
11301        $$emit$$"sub     0x4,rcx\n\t"
11302        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11303        $$emit$$"add     0x4,rcx\n\t"
11304        $$emit$$"jle     L_end\n\t"
11305        $$emit$$"dec     rcx\n\t"
11306        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11307        $$emit$$"vmovq   xmm0,(rax)\n\t"
11308        $$emit$$"add     0x8,rax\n\t"
11309        $$emit$$"dec     rcx\n\t"
11310        $$emit$$"jge     L_sloop\n\t"
11311        $$emit$$"# L_end:\n\t"
11312     } else {
11313        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11314        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11315     }
11316   %}
11317   ins_encode %{
11318     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11319                  $tmp$$XMMRegister, true, knoreg);
11320   %}
11321   ins_pipe(pipe_slow);
11322 %}
11323 
11324 // Large ClearArray AVX512.
11325 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11326                              Universe dummy, rFlagsReg cr)
11327 %{
11328   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11329   match(Set dummy (ClearArray cnt base));
11330   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11331 
11332   format %{ $$template
11333     if (UseFastStosb) {
11334        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11335        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11336        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11337     } else if (UseXMMForObjInit) {
11338        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11339        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11340        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11341        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11342        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11343        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11344        $$emit$$"add     0x40,rax\n\t"
11345        $$emit$$"# L_zero_64_bytes:\n\t"
11346        $$emit$$"sub     0x8,rcx\n\t"
11347        $$emit$$"jge     L_loop\n\t"
11348        $$emit$$"add     0x4,rcx\n\t"
11349        $$emit$$"jl      L_tail\n\t"
11350        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11351        $$emit$$"add     0x20,rax\n\t"
11352        $$emit$$"sub     0x4,rcx\n\t"
11353        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11354        $$emit$$"add     0x4,rcx\n\t"
11355        $$emit$$"jle     L_end\n\t"
11356        $$emit$$"dec     rcx\n\t"
11357        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11358        $$emit$$"vmovq   xmm0,(rax)\n\t"
11359        $$emit$$"add     0x8,rax\n\t"
11360        $$emit$$"dec     rcx\n\t"
11361        $$emit$$"jge     L_sloop\n\t"
11362        $$emit$$"# L_end:\n\t"
11363     } else {
11364        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11365        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11366     }
11367   %}
11368   ins_encode %{
11369     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11370                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11371   %}
11372   ins_pipe(pipe_slow);
11373 %}
11374 
11375 // Small ClearArray AVX512 constant length.
11376 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11377 %{
11378   predicate(!((ClearArrayNode*)n)->is_large() &&
11379               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11380   match(Set dummy (ClearArray cnt base));
11381   ins_cost(100);
11382   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11383   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11384   ins_encode %{
11385    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11386   %}
11387   ins_pipe(pipe_slow);
11388 %}
11389 
11390 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11391                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11392 %{
11393   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11394   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11395   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11396 
11397   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11398   ins_encode %{
11399     __ string_compare($str1$$Register, $str2$$Register,
11400                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11401                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11402   %}
11403   ins_pipe( pipe_slow );
11404 %}
11405 
11406 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11407                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11408 %{
11409   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11410   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11411   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11412 
11413   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11414   ins_encode %{
11415     __ string_compare($str1$$Register, $str2$$Register,
11416                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11417                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11418   %}
11419   ins_pipe( pipe_slow );
11420 %}
11421 
11422 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11423                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11424 %{
11425   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11426   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11427   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11428 
11429   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11430   ins_encode %{
11431     __ string_compare($str1$$Register, $str2$$Register,
11432                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11433                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11434   %}
11435   ins_pipe( pipe_slow );
11436 %}
11437 
11438 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11439                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11440 %{
11441   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11442   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11443   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11444 
11445   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11446   ins_encode %{
11447     __ string_compare($str1$$Register, $str2$$Register,
11448                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11449                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11450   %}
11451   ins_pipe( pipe_slow );
11452 %}
11453 
11454 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11455                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11456 %{
11457   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11458   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11459   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11460 
11461   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11462   ins_encode %{
11463     __ string_compare($str1$$Register, $str2$$Register,
11464                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11465                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11466   %}
11467   ins_pipe( pipe_slow );
11468 %}
11469 
11470 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11471                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11472 %{
11473   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11474   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11475   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11476 
11477   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11478   ins_encode %{
11479     __ string_compare($str1$$Register, $str2$$Register,
11480                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11481                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11482   %}
11483   ins_pipe( pipe_slow );
11484 %}
11485 
11486 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11487                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11488 %{
11489   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11490   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11491   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11492 
11493   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11494   ins_encode %{
11495     __ string_compare($str2$$Register, $str1$$Register,
11496                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11497                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11498   %}
11499   ins_pipe( pipe_slow );
11500 %}
11501 
11502 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11503                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11504 %{
11505   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11506   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11507   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11508 
11509   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11510   ins_encode %{
11511     __ string_compare($str2$$Register, $str1$$Register,
11512                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11513                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11514   %}
11515   ins_pipe( pipe_slow );
11516 %}
11517 
11518 // fast search of substring with known size.
11519 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11520                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11521 %{
11522   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11523   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11524   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11525 
11526   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11527   ins_encode %{
11528     int icnt2 = (int)$int_cnt2$$constant;
11529     if (icnt2 >= 16) {
11530       // IndexOf for constant substrings with size >= 16 elements
11531       // which don't need to be loaded through stack.
11532       __ string_indexofC8($str1$$Register, $str2$$Register,
11533                           $cnt1$$Register, $cnt2$$Register,
11534                           icnt2, $result$$Register,
11535                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11536     } else {
11537       // Small strings are loaded through stack if they cross page boundary.
11538       __ string_indexof($str1$$Register, $str2$$Register,
11539                         $cnt1$$Register, $cnt2$$Register,
11540                         icnt2, $result$$Register,
11541                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11542     }
11543   %}
11544   ins_pipe( pipe_slow );
11545 %}
11546 
11547 // fast search of substring with known size.
11548 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11549                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11550 %{
11551   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11552   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11553   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11554 
11555   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11556   ins_encode %{
11557     int icnt2 = (int)$int_cnt2$$constant;
11558     if (icnt2 >= 8) {
11559       // IndexOf for constant substrings with size >= 8 elements
11560       // which don't need to be loaded through stack.
11561       __ string_indexofC8($str1$$Register, $str2$$Register,
11562                           $cnt1$$Register, $cnt2$$Register,
11563                           icnt2, $result$$Register,
11564                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11565     } else {
11566       // Small strings are loaded through stack if they cross page boundary.
11567       __ string_indexof($str1$$Register, $str2$$Register,
11568                         $cnt1$$Register, $cnt2$$Register,
11569                         icnt2, $result$$Register,
11570                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11571     }
11572   %}
11573   ins_pipe( pipe_slow );
11574 %}
11575 
11576 // fast search of substring with known size.
11577 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11578                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11579 %{
11580   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11581   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11582   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11583 
11584   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11585   ins_encode %{
11586     int icnt2 = (int)$int_cnt2$$constant;
11587     if (icnt2 >= 8) {
11588       // IndexOf for constant substrings with size >= 8 elements
11589       // which don't need to be loaded through stack.
11590       __ string_indexofC8($str1$$Register, $str2$$Register,
11591                           $cnt1$$Register, $cnt2$$Register,
11592                           icnt2, $result$$Register,
11593                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11594     } else {
11595       // Small strings are loaded through stack if they cross page boundary.
11596       __ string_indexof($str1$$Register, $str2$$Register,
11597                         $cnt1$$Register, $cnt2$$Register,
11598                         icnt2, $result$$Register,
11599                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11600     }
11601   %}
11602   ins_pipe( pipe_slow );
11603 %}
11604 
11605 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11606                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11607 %{
11608   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11609   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11610   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11611 
11612   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11613   ins_encode %{
11614     __ string_indexof($str1$$Register, $str2$$Register,
11615                       $cnt1$$Register, $cnt2$$Register,
11616                       (-1), $result$$Register,
11617                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11618   %}
11619   ins_pipe( pipe_slow );
11620 %}
11621 
11622 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11623                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11624 %{
11625   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11626   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11627   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11628 
11629   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11630   ins_encode %{
11631     __ string_indexof($str1$$Register, $str2$$Register,
11632                       $cnt1$$Register, $cnt2$$Register,
11633                       (-1), $result$$Register,
11634                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11635   %}
11636   ins_pipe( pipe_slow );
11637 %}
11638 
11639 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11640                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11641 %{
11642   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11643   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11644   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11645 
11646   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11647   ins_encode %{
11648     __ string_indexof($str1$$Register, $str2$$Register,
11649                       $cnt1$$Register, $cnt2$$Register,
11650                       (-1), $result$$Register,
11651                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11652   %}
11653   ins_pipe( pipe_slow );
11654 %}
11655 
11656 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11657                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11658 %{
11659   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11660   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11661   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11662   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11663   ins_encode %{
11664     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11665                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11666   %}
11667   ins_pipe( pipe_slow );
11668 %}
11669 
11670 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11671                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11672 %{
11673   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11674   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11675   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11676   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11677   ins_encode %{
11678     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11679                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11680   %}
11681   ins_pipe( pipe_slow );
11682 %}
11683 
11684 // fast string equals
11685 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11686                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11687 %{
11688   predicate(!VM_Version::supports_avx512vlbw());
11689   match(Set result (StrEquals (Binary str1 str2) cnt));
11690   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11691 
11692   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11693   ins_encode %{
11694     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11695                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11696                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11697   %}
11698   ins_pipe( pipe_slow );
11699 %}
11700 
11701 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11702                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11703 %{
11704   predicate(VM_Version::supports_avx512vlbw());
11705   match(Set result (StrEquals (Binary str1 str2) cnt));
11706   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11707 
11708   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11709   ins_encode %{
11710     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11711                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11712                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11713   %}
11714   ins_pipe( pipe_slow );
11715 %}
11716 
11717 // fast array equals
11718 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11719                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11720 %{
11721   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11722   match(Set result (AryEq ary1 ary2));
11723   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11724 
11725   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11726   ins_encode %{
11727     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11728                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11729                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11730   %}
11731   ins_pipe( pipe_slow );
11732 %}
11733 
11734 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11735                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11736 %{
11737   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11738   match(Set result (AryEq ary1 ary2));
11739   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11740 
11741   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11742   ins_encode %{
11743     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11744                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11745                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11746   %}
11747   ins_pipe( pipe_slow );
11748 %}
11749 
11750 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11751                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11752 %{
11753   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11754   match(Set result (AryEq ary1 ary2));
11755   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11756 
11757   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11758   ins_encode %{
11759     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11760                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11761                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11762   %}
11763   ins_pipe( pipe_slow );
11764 %}
11765 
11766 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11767                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11768 %{
11769   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11770   match(Set result (AryEq ary1 ary2));
11771   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11772 
11773   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11774   ins_encode %{
11775     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11776                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11777                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11778   %}
11779   ins_pipe( pipe_slow );
11780 %}
11781 
11782 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11783                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11784 %{
11785   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11786   match(Set result (CountPositives ary1 len));
11787   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11788 
11789   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11790   ins_encode %{
11791     __ count_positives($ary1$$Register, $len$$Register,
11792                        $result$$Register, $tmp3$$Register,
11793                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11794   %}
11795   ins_pipe( pipe_slow );
11796 %}
11797 
11798 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11799                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11800 %{
11801   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11802   match(Set result (CountPositives ary1 len));
11803   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11804 
11805   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11806   ins_encode %{
11807     __ count_positives($ary1$$Register, $len$$Register,
11808                        $result$$Register, $tmp3$$Register,
11809                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11810   %}
11811   ins_pipe( pipe_slow );
11812 %}
11813 
11814 // fast char[] to byte[] compression
11815 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11816                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11817   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11818   match(Set result (StrCompressedCopy src (Binary dst len)));
11819   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11820          USE_KILL len, KILL tmp5, KILL cr);
11821 
11822   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11823   ins_encode %{
11824     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11825                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11826                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11827                            knoreg, knoreg);
11828   %}
11829   ins_pipe( pipe_slow );
11830 %}
11831 
11832 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11833                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11834   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11835   match(Set result (StrCompressedCopy src (Binary dst len)));
11836   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11837          USE_KILL len, KILL tmp5, KILL cr);
11838 
11839   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11840   ins_encode %{
11841     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11842                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11843                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11844                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11845   %}
11846   ins_pipe( pipe_slow );
11847 %}
11848 // fast byte[] to char[] inflation
11849 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11850                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11851   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11852   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11853   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11854 
11855   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11856   ins_encode %{
11857     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11858                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11859   %}
11860   ins_pipe( pipe_slow );
11861 %}
11862 
11863 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11864                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11865   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11866   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11867   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11868 
11869   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11870   ins_encode %{
11871     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11872                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11873   %}
11874   ins_pipe( pipe_slow );
11875 %}
11876 
11877 // encode char[] to byte[] in ISO_8859_1
11878 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11879                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11880                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11881   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11882   match(Set result (EncodeISOArray src (Binary dst len)));
11883   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11884 
11885   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11886   ins_encode %{
11887     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11888                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11889                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11890   %}
11891   ins_pipe( pipe_slow );
11892 %}
11893 
11894 // encode char[] to byte[] in ASCII
11895 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11896                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11897                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11898   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11899   match(Set result (EncodeISOArray src (Binary dst len)));
11900   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11901 
11902   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11903   ins_encode %{
11904     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11905                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11906                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11907   %}
11908   ins_pipe( pipe_slow );
11909 %}
11910 
11911 //----------Overflow Math Instructions-----------------------------------------
11912 
11913 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11914 %{
11915   match(Set cr (OverflowAddI op1 op2));
11916   effect(DEF cr, USE_KILL op1, USE op2);
11917 
11918   format %{ "addl    $op1, $op2\t# overflow check int" %}
11919 
11920   ins_encode %{
11921     __ addl($op1$$Register, $op2$$Register);
11922   %}
11923   ins_pipe(ialu_reg_reg);
11924 %}
11925 
11926 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11927 %{
11928   match(Set cr (OverflowAddI op1 op2));
11929   effect(DEF cr, USE_KILL op1, USE op2);
11930 
11931   format %{ "addl    $op1, $op2\t# overflow check int" %}
11932 
11933   ins_encode %{
11934     __ addl($op1$$Register, $op2$$constant);
11935   %}
11936   ins_pipe(ialu_reg_reg);
11937 %}
11938 
11939 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11940 %{
11941   match(Set cr (OverflowAddL op1 op2));
11942   effect(DEF cr, USE_KILL op1, USE op2);
11943 
11944   format %{ "addq    $op1, $op2\t# overflow check long" %}
11945   ins_encode %{
11946     __ addq($op1$$Register, $op2$$Register);
11947   %}
11948   ins_pipe(ialu_reg_reg);
11949 %}
11950 
11951 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11952 %{
11953   match(Set cr (OverflowAddL op1 op2));
11954   effect(DEF cr, USE_KILL op1, USE op2);
11955 
11956   format %{ "addq    $op1, $op2\t# overflow check long" %}
11957   ins_encode %{
11958     __ addq($op1$$Register, $op2$$constant);
11959   %}
11960   ins_pipe(ialu_reg_reg);
11961 %}
11962 
11963 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11964 %{
11965   match(Set cr (OverflowSubI op1 op2));
11966 
11967   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11968   ins_encode %{
11969     __ cmpl($op1$$Register, $op2$$Register);
11970   %}
11971   ins_pipe(ialu_reg_reg);
11972 %}
11973 
11974 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11975 %{
11976   match(Set cr (OverflowSubI op1 op2));
11977 
11978   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11979   ins_encode %{
11980     __ cmpl($op1$$Register, $op2$$constant);
11981   %}
11982   ins_pipe(ialu_reg_reg);
11983 %}
11984 
11985 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11986 %{
11987   match(Set cr (OverflowSubL op1 op2));
11988 
11989   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11990   ins_encode %{
11991     __ cmpq($op1$$Register, $op2$$Register);
11992   %}
11993   ins_pipe(ialu_reg_reg);
11994 %}
11995 
11996 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11997 %{
11998   match(Set cr (OverflowSubL op1 op2));
11999 
12000   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12001   ins_encode %{
12002     __ cmpq($op1$$Register, $op2$$constant);
12003   %}
12004   ins_pipe(ialu_reg_reg);
12005 %}
12006 
12007 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12008 %{
12009   match(Set cr (OverflowSubI zero op2));
12010   effect(DEF cr, USE_KILL op2);
12011 
12012   format %{ "negl    $op2\t# overflow check int" %}
12013   ins_encode %{
12014     __ negl($op2$$Register);
12015   %}
12016   ins_pipe(ialu_reg_reg);
12017 %}
12018 
12019 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12020 %{
12021   match(Set cr (OverflowSubL zero op2));
12022   effect(DEF cr, USE_KILL op2);
12023 
12024   format %{ "negq    $op2\t# overflow check long" %}
12025   ins_encode %{
12026     __ negq($op2$$Register);
12027   %}
12028   ins_pipe(ialu_reg_reg);
12029 %}
12030 
12031 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12032 %{
12033   match(Set cr (OverflowMulI op1 op2));
12034   effect(DEF cr, USE_KILL op1, USE op2);
12035 
12036   format %{ "imull    $op1, $op2\t# overflow check int" %}
12037   ins_encode %{
12038     __ imull($op1$$Register, $op2$$Register);
12039   %}
12040   ins_pipe(ialu_reg_reg_alu0);
12041 %}
12042 
12043 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12044 %{
12045   match(Set cr (OverflowMulI op1 op2));
12046   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12047 
12048   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12049   ins_encode %{
12050     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12051   %}
12052   ins_pipe(ialu_reg_reg_alu0);
12053 %}
12054 
12055 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12056 %{
12057   match(Set cr (OverflowMulL op1 op2));
12058   effect(DEF cr, USE_KILL op1, USE op2);
12059 
12060   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12061   ins_encode %{
12062     __ imulq($op1$$Register, $op2$$Register);
12063   %}
12064   ins_pipe(ialu_reg_reg_alu0);
12065 %}
12066 
12067 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12068 %{
12069   match(Set cr (OverflowMulL op1 op2));
12070   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12071 
12072   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12073   ins_encode %{
12074     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12075   %}
12076   ins_pipe(ialu_reg_reg_alu0);
12077 %}
12078 
12079 
12080 //----------Control Flow Instructions------------------------------------------
12081 // Signed compare Instructions
12082 
12083 // XXX more variants!!
12084 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12085 %{
12086   match(Set cr (CmpI op1 op2));
12087   effect(DEF cr, USE op1, USE op2);
12088 
12089   format %{ "cmpl    $op1, $op2" %}
12090   ins_encode %{
12091     __ cmpl($op1$$Register, $op2$$Register);
12092   %}
12093   ins_pipe(ialu_cr_reg_reg);
12094 %}
12095 
12096 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12097 %{
12098   match(Set cr (CmpI op1 op2));
12099 
12100   format %{ "cmpl    $op1, $op2" %}
12101   ins_encode %{
12102     __ cmpl($op1$$Register, $op2$$constant);
12103   %}
12104   ins_pipe(ialu_cr_reg_imm);
12105 %}
12106 
12107 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12108 %{
12109   match(Set cr (CmpI op1 (LoadI op2)));
12110 
12111   ins_cost(500); // XXX
12112   format %{ "cmpl    $op1, $op2" %}
12113   ins_encode %{
12114     __ cmpl($op1$$Register, $op2$$Address);
12115   %}
12116   ins_pipe(ialu_cr_reg_mem);
12117 %}
12118 
12119 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12120 %{
12121   match(Set cr (CmpI src zero));
12122 
12123   format %{ "testl   $src, $src" %}
12124   ins_encode %{
12125     __ testl($src$$Register, $src$$Register);
12126   %}
12127   ins_pipe(ialu_cr_reg_imm);
12128 %}
12129 
12130 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12131 %{
12132   match(Set cr (CmpI (AndI src con) zero));
12133 
12134   format %{ "testl   $src, $con" %}
12135   ins_encode %{
12136     __ testl($src$$Register, $con$$constant);
12137   %}
12138   ins_pipe(ialu_cr_reg_imm);
12139 %}
12140 
12141 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12142 %{
12143   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12144 
12145   format %{ "testl   $src, $mem" %}
12146   ins_encode %{
12147     __ testl($src$$Register, $mem$$Address);
12148   %}
12149   ins_pipe(ialu_cr_reg_mem);
12150 %}
12151 
12152 // Unsigned compare Instructions; really, same as signed except they
12153 // produce an rFlagsRegU instead of rFlagsReg.
12154 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12155 %{
12156   match(Set cr (CmpU op1 op2));
12157 
12158   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12159   ins_encode %{
12160     __ cmpl($op1$$Register, $op2$$Register);
12161   %}
12162   ins_pipe(ialu_cr_reg_reg);
12163 %}
12164 
12165 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12166 %{
12167   match(Set cr (CmpU op1 op2));
12168 
12169   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12170   ins_encode %{
12171     __ cmpl($op1$$Register, $op2$$constant);
12172   %}
12173   ins_pipe(ialu_cr_reg_imm);
12174 %}
12175 
12176 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12177 %{
12178   match(Set cr (CmpU op1 (LoadI op2)));
12179 
12180   ins_cost(500); // XXX
12181   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12182   ins_encode %{
12183     __ cmpl($op1$$Register, $op2$$Address);
12184   %}
12185   ins_pipe(ialu_cr_reg_mem);
12186 %}
12187 
12188 // // // Cisc-spilled version of cmpU_rReg
12189 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12190 // //%{
12191 // //  match(Set cr (CmpU (LoadI op1) op2));
12192 // //
12193 // //  format %{ "CMPu   $op1,$op2" %}
12194 // //  ins_cost(500);
12195 // //  opcode(0x39);  /* Opcode 39 /r */
12196 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12197 // //%}
12198 
12199 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12200 %{
12201   match(Set cr (CmpU src zero));
12202 
12203   format %{ "testl   $src, $src\t# unsigned" %}
12204   ins_encode %{
12205     __ testl($src$$Register, $src$$Register);
12206   %}
12207   ins_pipe(ialu_cr_reg_imm);
12208 %}
12209 
12210 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12211 %{
12212   match(Set cr (CmpP op1 op2));
12213 
12214   format %{ "cmpq    $op1, $op2\t# ptr" %}
12215   ins_encode %{
12216     __ cmpq($op1$$Register, $op2$$Register);
12217   %}
12218   ins_pipe(ialu_cr_reg_reg);
12219 %}
12220 
12221 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12222 %{
12223   match(Set cr (CmpP op1 (LoadP op2)));
12224   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12225 
12226   ins_cost(500); // XXX
12227   format %{ "cmpq    $op1, $op2\t# ptr" %}
12228   ins_encode %{
12229     __ cmpq($op1$$Register, $op2$$Address);
12230   %}
12231   ins_pipe(ialu_cr_reg_mem);
12232 %}
12233 
12234 // // // Cisc-spilled version of cmpP_rReg
12235 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12236 // //%{
12237 // //  match(Set cr (CmpP (LoadP op1) op2));
12238 // //
12239 // //  format %{ "CMPu   $op1,$op2" %}
12240 // //  ins_cost(500);
12241 // //  opcode(0x39);  /* Opcode 39 /r */
12242 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12243 // //%}
12244 
12245 // XXX this is generalized by compP_rReg_mem???
12246 // Compare raw pointer (used in out-of-heap check).
12247 // Only works because non-oop pointers must be raw pointers
12248 // and raw pointers have no anti-dependencies.
12249 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12250 %{
12251   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12252             n->in(2)->as_Load()->barrier_data() == 0);
12253   match(Set cr (CmpP op1 (LoadP op2)));
12254 
12255   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12256   ins_encode %{
12257     __ cmpq($op1$$Register, $op2$$Address);
12258   %}
12259   ins_pipe(ialu_cr_reg_mem);
12260 %}
12261 
12262 // This will generate a signed flags result. This should be OK since
12263 // any compare to a zero should be eq/neq.
12264 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12265 %{
12266   match(Set cr (CmpP src zero));
12267 
12268   format %{ "testq   $src, $src\t# ptr" %}
12269   ins_encode %{
12270     __ testq($src$$Register, $src$$Register);
12271   %}
12272   ins_pipe(ialu_cr_reg_imm);
12273 %}
12274 
12275 // This will generate a signed flags result. This should be OK since
12276 // any compare to a zero should be eq/neq.
12277 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12278 %{
12279   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12280             n->in(1)->as_Load()->barrier_data() == 0);
12281   match(Set cr (CmpP (LoadP op) zero));
12282 
12283   ins_cost(500); // XXX
12284   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12285   ins_encode %{
12286     __ testq($op$$Address, 0xFFFFFFFF);
12287   %}
12288   ins_pipe(ialu_cr_reg_imm);
12289 %}
12290 
12291 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12292 %{
12293   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12294             n->in(1)->as_Load()->barrier_data() == 0);
12295   match(Set cr (CmpP (LoadP mem) zero));
12296 
12297   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12298   ins_encode %{
12299     __ cmpq(r12, $mem$$Address);
12300   %}
12301   ins_pipe(ialu_cr_reg_mem);
12302 %}
12303 
12304 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12305 %{
12306   match(Set cr (CmpN op1 op2));
12307 
12308   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12309   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12310   ins_pipe(ialu_cr_reg_reg);
12311 %}
12312 
12313 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12314 %{
12315   match(Set cr (CmpN src (LoadN mem)));
12316 
12317   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12318   ins_encode %{
12319     __ cmpl($src$$Register, $mem$$Address);
12320   %}
12321   ins_pipe(ialu_cr_reg_mem);
12322 %}
12323 
12324 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12325   match(Set cr (CmpN op1 op2));
12326 
12327   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12328   ins_encode %{
12329     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12330   %}
12331   ins_pipe(ialu_cr_reg_imm);
12332 %}
12333 
12334 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12335 %{
12336   match(Set cr (CmpN src (LoadN mem)));
12337 
12338   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12339   ins_encode %{
12340     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12341   %}
12342   ins_pipe(ialu_cr_reg_mem);
12343 %}
12344 
12345 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12346   match(Set cr (CmpN op1 op2));
12347 
12348   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12349   ins_encode %{
12350     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12351   %}
12352   ins_pipe(ialu_cr_reg_imm);
12353 %}
12354 
12355 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12356 %{
12357   match(Set cr (CmpN src (LoadNKlass mem)));
12358 
12359   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12360   ins_encode %{
12361     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12362   %}
12363   ins_pipe(ialu_cr_reg_mem);
12364 %}
12365 
12366 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12367   match(Set cr (CmpN src zero));
12368 
12369   format %{ "testl   $src, $src\t# compressed ptr" %}
12370   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12371   ins_pipe(ialu_cr_reg_imm);
12372 %}
12373 
12374 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12375 %{
12376   predicate(CompressedOops::base() != NULL);
12377   match(Set cr (CmpN (LoadN mem) zero));
12378 
12379   ins_cost(500); // XXX
12380   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12381   ins_encode %{
12382     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12383   %}
12384   ins_pipe(ialu_cr_reg_mem);
12385 %}
12386 
12387 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12388 %{
12389   predicate(CompressedOops::base() == NULL);
12390   match(Set cr (CmpN (LoadN mem) zero));
12391 
12392   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12393   ins_encode %{
12394     __ cmpl(r12, $mem$$Address);
12395   %}
12396   ins_pipe(ialu_cr_reg_mem);
12397 %}
12398 
12399 // Yanked all unsigned pointer compare operations.
12400 // Pointer compares are done with CmpP which is already unsigned.
12401 
12402 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12403 %{
12404   match(Set cr (CmpL op1 op2));
12405 
12406   format %{ "cmpq    $op1, $op2" %}
12407   ins_encode %{
12408     __ cmpq($op1$$Register, $op2$$Register);
12409   %}
12410   ins_pipe(ialu_cr_reg_reg);
12411 %}
12412 
12413 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12414 %{
12415   match(Set cr (CmpL op1 op2));
12416 
12417   format %{ "cmpq    $op1, $op2" %}
12418   ins_encode %{
12419     __ cmpq($op1$$Register, $op2$$constant);
12420   %}
12421   ins_pipe(ialu_cr_reg_imm);
12422 %}
12423 
12424 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12425 %{
12426   match(Set cr (CmpL op1 (LoadL op2)));
12427 
12428   format %{ "cmpq    $op1, $op2" %}
12429   ins_encode %{
12430     __ cmpq($op1$$Register, $op2$$Address);
12431   %}
12432   ins_pipe(ialu_cr_reg_mem);
12433 %}
12434 
12435 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12436 %{
12437   match(Set cr (CmpL src zero));
12438 
12439   format %{ "testq   $src, $src" %}
12440   ins_encode %{
12441     __ testq($src$$Register, $src$$Register);
12442   %}
12443   ins_pipe(ialu_cr_reg_imm);
12444 %}
12445 
12446 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12447 %{
12448   match(Set cr (CmpL (AndL src con) zero));
12449 
12450   format %{ "testq   $src, $con\t# long" %}
12451   ins_encode %{
12452     __ testq($src$$Register, $con$$constant);
12453   %}
12454   ins_pipe(ialu_cr_reg_imm);
12455 %}
12456 
12457 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12458 %{
12459   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12460 
12461   format %{ "testq   $src, $mem" %}
12462   ins_encode %{
12463     __ testq($src$$Register, $mem$$Address);
12464   %}
12465   ins_pipe(ialu_cr_reg_mem);
12466 %}
12467 
12468 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12469 %{
12470   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12471 
12472   format %{ "testq   $src, $mem" %}
12473   ins_encode %{
12474     __ testq($src$$Register, $mem$$Address);
12475   %}
12476   ins_pipe(ialu_cr_reg_mem);
12477 %}
12478 
12479 // Manifest a CmpL result in an integer register.  Very painful.
12480 // This is the test to avoid.
12481 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12482 %{
12483   match(Set dst (CmpL3 src1 src2));
12484   effect(KILL flags);
12485 
12486   ins_cost(275); // XXX
12487   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12488             "movl    $dst, -1\n\t"
12489             "jl,s    done\n\t"
12490             "setne   $dst\n\t"
12491             "movzbl  $dst, $dst\n\t"
12492     "done:" %}
12493   ins_encode %{
12494     Label done;
12495     __ cmpq($src1$$Register, $src2$$Register);
12496     __ movl($dst$$Register, -1);
12497     __ jccb(Assembler::less, done);
12498     __ setne($dst$$Register);
12499     __ movzbl($dst$$Register, $dst$$Register);
12500     __ bind(done);
12501   %}
12502   ins_pipe(pipe_slow);
12503 %}
12504 
12505 // Unsigned long compare Instructions; really, same as signed long except they
12506 // produce an rFlagsRegU instead of rFlagsReg.
12507 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12508 %{
12509   match(Set cr (CmpUL op1 op2));
12510 
12511   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12512   ins_encode %{
12513     __ cmpq($op1$$Register, $op2$$Register);
12514   %}
12515   ins_pipe(ialu_cr_reg_reg);
12516 %}
12517 
12518 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12519 %{
12520   match(Set cr (CmpUL op1 op2));
12521 
12522   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12523   ins_encode %{
12524     __ cmpq($op1$$Register, $op2$$constant);
12525   %}
12526   ins_pipe(ialu_cr_reg_imm);
12527 %}
12528 
12529 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12530 %{
12531   match(Set cr (CmpUL op1 (LoadL op2)));
12532 
12533   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12534   ins_encode %{
12535     __ cmpq($op1$$Register, $op2$$Address);
12536   %}
12537   ins_pipe(ialu_cr_reg_mem);
12538 %}
12539 
12540 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12541 %{
12542   match(Set cr (CmpUL src zero));
12543 
12544   format %{ "testq   $src, $src\t# unsigned" %}
12545   ins_encode %{
12546     __ testq($src$$Register, $src$$Register);
12547   %}
12548   ins_pipe(ialu_cr_reg_imm);
12549 %}
12550 
12551 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12552 %{
12553   match(Set cr (CmpI (LoadB mem) imm));
12554 
12555   ins_cost(125);
12556   format %{ "cmpb    $mem, $imm" %}
12557   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12558   ins_pipe(ialu_cr_reg_mem);
12559 %}
12560 
12561 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12562 %{
12563   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12564 
12565   ins_cost(125);
12566   format %{ "testb   $mem, $imm\t# ubyte" %}
12567   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12568   ins_pipe(ialu_cr_reg_mem);
12569 %}
12570 
12571 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12572 %{
12573   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12574 
12575   ins_cost(125);
12576   format %{ "testb   $mem, $imm\t# byte" %}
12577   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12578   ins_pipe(ialu_cr_reg_mem);
12579 %}
12580 
12581 //----------Max and Min--------------------------------------------------------
12582 // Min Instructions
12583 
12584 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12585 %{
12586   effect(USE_DEF dst, USE src, USE cr);
12587 
12588   format %{ "cmovlgt $dst, $src\t# min" %}
12589   ins_encode %{
12590     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12591   %}
12592   ins_pipe(pipe_cmov_reg);
12593 %}
12594 
12595 
12596 instruct minI_rReg(rRegI dst, rRegI src)
12597 %{
12598   match(Set dst (MinI dst src));
12599 
12600   ins_cost(200);
12601   expand %{
12602     rFlagsReg cr;
12603     compI_rReg(cr, dst, src);
12604     cmovI_reg_g(dst, src, cr);
12605   %}
12606 %}
12607 
12608 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12609 %{
12610   effect(USE_DEF dst, USE src, USE cr);
12611 
12612   format %{ "cmovllt $dst, $src\t# max" %}
12613   ins_encode %{
12614     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12615   %}
12616   ins_pipe(pipe_cmov_reg);
12617 %}
12618 
12619 
12620 instruct maxI_rReg(rRegI dst, rRegI src)
12621 %{
12622   match(Set dst (MaxI dst src));
12623 
12624   ins_cost(200);
12625   expand %{
12626     rFlagsReg cr;
12627     compI_rReg(cr, dst, src);
12628     cmovI_reg_l(dst, src, cr);
12629   %}
12630 %}
12631 
12632 // ============================================================================
12633 // Branch Instructions
12634 
12635 // Jump Direct - Label defines a relative address from JMP+1
12636 instruct jmpDir(label labl)
12637 %{
12638   match(Goto);
12639   effect(USE labl);
12640 
12641   ins_cost(300);
12642   format %{ "jmp     $labl" %}
12643   size(5);
12644   ins_encode %{
12645     Label* L = $labl$$label;
12646     __ jmp(*L, false); // Always long jump
12647   %}
12648   ins_pipe(pipe_jmp);
12649 %}
12650 
12651 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12652 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12653 %{
12654   match(If cop cr);
12655   effect(USE labl);
12656 
12657   ins_cost(300);
12658   format %{ "j$cop     $labl" %}
12659   size(6);
12660   ins_encode %{
12661     Label* L = $labl$$label;
12662     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12663   %}
12664   ins_pipe(pipe_jcc);
12665 %}
12666 
12667 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12668 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12669 %{
12670   match(CountedLoopEnd cop cr);
12671   effect(USE labl);
12672 
12673   ins_cost(300);
12674   format %{ "j$cop     $labl\t# loop end" %}
12675   size(6);
12676   ins_encode %{
12677     Label* L = $labl$$label;
12678     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12679   %}
12680   ins_pipe(pipe_jcc);
12681 %}
12682 
12683 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12684 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12685   match(CountedLoopEnd cop cmp);
12686   effect(USE labl);
12687 
12688   ins_cost(300);
12689   format %{ "j$cop,u   $labl\t# loop end" %}
12690   size(6);
12691   ins_encode %{
12692     Label* L = $labl$$label;
12693     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12694   %}
12695   ins_pipe(pipe_jcc);
12696 %}
12697 
12698 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12699   match(CountedLoopEnd cop cmp);
12700   effect(USE labl);
12701 
12702   ins_cost(200);
12703   format %{ "j$cop,u   $labl\t# loop end" %}
12704   size(6);
12705   ins_encode %{
12706     Label* L = $labl$$label;
12707     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12708   %}
12709   ins_pipe(pipe_jcc);
12710 %}
12711 
12712 // Jump Direct Conditional - using unsigned comparison
12713 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12714   match(If cop cmp);
12715   effect(USE labl);
12716 
12717   ins_cost(300);
12718   format %{ "j$cop,u   $labl" %}
12719   size(6);
12720   ins_encode %{
12721     Label* L = $labl$$label;
12722     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12723   %}
12724   ins_pipe(pipe_jcc);
12725 %}
12726 
12727 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12728   match(If cop cmp);
12729   effect(USE labl);
12730 
12731   ins_cost(200);
12732   format %{ "j$cop,u   $labl" %}
12733   size(6);
12734   ins_encode %{
12735     Label* L = $labl$$label;
12736     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12737   %}
12738   ins_pipe(pipe_jcc);
12739 %}
12740 
12741 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12742   match(If cop cmp);
12743   effect(USE labl);
12744 
12745   ins_cost(200);
12746   format %{ $$template
12747     if ($cop$$cmpcode == Assembler::notEqual) {
12748       $$emit$$"jp,u    $labl\n\t"
12749       $$emit$$"j$cop,u   $labl"
12750     } else {
12751       $$emit$$"jp,u    done\n\t"
12752       $$emit$$"j$cop,u   $labl\n\t"
12753       $$emit$$"done:"
12754     }
12755   %}
12756   ins_encode %{
12757     Label* l = $labl$$label;
12758     if ($cop$$cmpcode == Assembler::notEqual) {
12759       __ jcc(Assembler::parity, *l, false);
12760       __ jcc(Assembler::notEqual, *l, false);
12761     } else if ($cop$$cmpcode == Assembler::equal) {
12762       Label done;
12763       __ jccb(Assembler::parity, done);
12764       __ jcc(Assembler::equal, *l, false);
12765       __ bind(done);
12766     } else {
12767        ShouldNotReachHere();
12768     }
12769   %}
12770   ins_pipe(pipe_jcc);
12771 %}
12772 
12773 // ============================================================================
12774 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12775 // superklass array for an instance of the superklass.  Set a hidden
12776 // internal cache on a hit (cache is checked with exposed code in
12777 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12778 // encoding ALSO sets flags.
12779 
12780 instruct partialSubtypeCheck(rdi_RegP result,
12781                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12782                              rFlagsReg cr)
12783 %{
12784   match(Set result (PartialSubtypeCheck sub super));
12785   effect(KILL rcx, KILL cr);
12786 
12787   ins_cost(1100);  // slightly larger than the next version
12788   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12789             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12790             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12791             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12792             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12793             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12794             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12795     "miss:\t" %}
12796 
12797   opcode(0x1); // Force a XOR of RDI
12798   ins_encode(enc_PartialSubtypeCheck());
12799   ins_pipe(pipe_slow);
12800 %}
12801 
12802 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12803                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12804                                      immP0 zero,
12805                                      rdi_RegP result)
12806 %{
12807   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12808   effect(KILL rcx, KILL result);
12809 
12810   ins_cost(1000);
12811   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12812             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12813             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12814             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12815             "jne,s   miss\t\t# Missed: flags nz\n\t"
12816             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12817     "miss:\t" %}
12818 
12819   opcode(0x0); // No need to XOR RDI
12820   ins_encode(enc_PartialSubtypeCheck());
12821   ins_pipe(pipe_slow);
12822 %}
12823 
12824 // ============================================================================
12825 // Branch Instructions -- short offset versions
12826 //
12827 // These instructions are used to replace jumps of a long offset (the default
12828 // match) with jumps of a shorter offset.  These instructions are all tagged
12829 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12830 // match rules in general matching.  Instead, the ADLC generates a conversion
12831 // method in the MachNode which can be used to do in-place replacement of the
12832 // long variant with the shorter variant.  The compiler will determine if a
12833 // branch can be taken by the is_short_branch_offset() predicate in the machine
12834 // specific code section of the file.
12835 
12836 // Jump Direct - Label defines a relative address from JMP+1
12837 instruct jmpDir_short(label labl) %{
12838   match(Goto);
12839   effect(USE labl);
12840 
12841   ins_cost(300);
12842   format %{ "jmp,s   $labl" %}
12843   size(2);
12844   ins_encode %{
12845     Label* L = $labl$$label;
12846     __ jmpb(*L);
12847   %}
12848   ins_pipe(pipe_jmp);
12849   ins_short_branch(1);
12850 %}
12851 
12852 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12853 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12854   match(If cop cr);
12855   effect(USE labl);
12856 
12857   ins_cost(300);
12858   format %{ "j$cop,s   $labl" %}
12859   size(2);
12860   ins_encode %{
12861     Label* L = $labl$$label;
12862     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12863   %}
12864   ins_pipe(pipe_jcc);
12865   ins_short_branch(1);
12866 %}
12867 
12868 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12869 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12870   match(CountedLoopEnd cop cr);
12871   effect(USE labl);
12872 
12873   ins_cost(300);
12874   format %{ "j$cop,s   $labl\t# loop end" %}
12875   size(2);
12876   ins_encode %{
12877     Label* L = $labl$$label;
12878     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12879   %}
12880   ins_pipe(pipe_jcc);
12881   ins_short_branch(1);
12882 %}
12883 
12884 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12885 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12886   match(CountedLoopEnd cop cmp);
12887   effect(USE labl);
12888 
12889   ins_cost(300);
12890   format %{ "j$cop,us  $labl\t# loop end" %}
12891   size(2);
12892   ins_encode %{
12893     Label* L = $labl$$label;
12894     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12895   %}
12896   ins_pipe(pipe_jcc);
12897   ins_short_branch(1);
12898 %}
12899 
12900 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12901   match(CountedLoopEnd cop cmp);
12902   effect(USE labl);
12903 
12904   ins_cost(300);
12905   format %{ "j$cop,us  $labl\t# loop end" %}
12906   size(2);
12907   ins_encode %{
12908     Label* L = $labl$$label;
12909     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12910   %}
12911   ins_pipe(pipe_jcc);
12912   ins_short_branch(1);
12913 %}
12914 
12915 // Jump Direct Conditional - using unsigned comparison
12916 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12917   match(If cop cmp);
12918   effect(USE labl);
12919 
12920   ins_cost(300);
12921   format %{ "j$cop,us  $labl" %}
12922   size(2);
12923   ins_encode %{
12924     Label* L = $labl$$label;
12925     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12926   %}
12927   ins_pipe(pipe_jcc);
12928   ins_short_branch(1);
12929 %}
12930 
12931 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12932   match(If cop cmp);
12933   effect(USE labl);
12934 
12935   ins_cost(300);
12936   format %{ "j$cop,us  $labl" %}
12937   size(2);
12938   ins_encode %{
12939     Label* L = $labl$$label;
12940     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12941   %}
12942   ins_pipe(pipe_jcc);
12943   ins_short_branch(1);
12944 %}
12945 
12946 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12947   match(If cop cmp);
12948   effect(USE labl);
12949 
12950   ins_cost(300);
12951   format %{ $$template
12952     if ($cop$$cmpcode == Assembler::notEqual) {
12953       $$emit$$"jp,u,s  $labl\n\t"
12954       $$emit$$"j$cop,u,s  $labl"
12955     } else {
12956       $$emit$$"jp,u,s  done\n\t"
12957       $$emit$$"j$cop,u,s  $labl\n\t"
12958       $$emit$$"done:"
12959     }
12960   %}
12961   size(4);
12962   ins_encode %{
12963     Label* l = $labl$$label;
12964     if ($cop$$cmpcode == Assembler::notEqual) {
12965       __ jccb(Assembler::parity, *l);
12966       __ jccb(Assembler::notEqual, *l);
12967     } else if ($cop$$cmpcode == Assembler::equal) {
12968       Label done;
12969       __ jccb(Assembler::parity, done);
12970       __ jccb(Assembler::equal, *l);
12971       __ bind(done);
12972     } else {
12973        ShouldNotReachHere();
12974     }
12975   %}
12976   ins_pipe(pipe_jcc);
12977   ins_short_branch(1);
12978 %}
12979 
12980 // ============================================================================
12981 // inlined locking and unlocking
12982 
12983 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12984   predicate(Compile::current()->use_rtm());
12985   match(Set cr (FastLock object box));
12986   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12987   ins_cost(300);
12988   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12989   ins_encode %{
12990     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12991                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12992                  _rtm_counters, _stack_rtm_counters,
12993                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12994                  true, ra_->C->profile_rtm());
12995   %}
12996   ins_pipe(pipe_slow);
12997 %}
12998 
12999 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13000   predicate(!Compile::current()->use_rtm());
13001   match(Set cr (FastLock object box));
13002   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
13003   ins_cost(300);
13004   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13005   ins_encode %{
13006     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13007                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
13008   %}
13009   ins_pipe(pipe_slow);
13010 %}
13011 
13012 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13013   match(Set cr (FastUnlock object box));
13014   effect(TEMP tmp, USE_KILL box);
13015   ins_cost(300);
13016   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13017   ins_encode %{
13018     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13019   %}
13020   ins_pipe(pipe_slow);
13021 %}
13022 
13023 
13024 // ============================================================================
13025 // Safepoint Instructions
13026 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13027 %{
13028   match(SafePoint poll);
13029   effect(KILL cr, USE poll);
13030 
13031   format %{ "testl   rax, [$poll]\t"
13032             "# Safepoint: poll for GC" %}
13033   ins_cost(125);
13034   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13035   ins_encode %{
13036     __ relocate(relocInfo::poll_type);
13037     address pre_pc = __ pc();
13038     __ testl(rax, Address($poll$$Register, 0));
13039     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13040   %}
13041   ins_pipe(ialu_reg_mem);
13042 %}
13043 
13044 instruct mask_all_evexL(kReg dst, rRegL src) %{
13045   match(Set dst (MaskAll src));
13046   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13047   ins_encode %{
13048     int mask_len = Matcher::vector_length(this);
13049     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13050   %}
13051   ins_pipe( pipe_slow );
13052 %}
13053 
13054 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13055   predicate(Matcher::vector_length(n) > 32);
13056   match(Set dst (MaskAll src));
13057   effect(TEMP tmp);
13058   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13059   ins_encode %{
13060     int mask_len = Matcher::vector_length(this);
13061     __ movslq($tmp$$Register, $src$$Register);
13062     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13063   %}
13064   ins_pipe( pipe_slow );
13065 %}
13066 
13067 // ============================================================================
13068 // Procedure Call/Return Instructions
13069 // Call Java Static Instruction
13070 // Note: If this code changes, the corresponding ret_addr_offset() and
13071 //       compute_padding() functions will have to be adjusted.
13072 instruct CallStaticJavaDirect(method meth) %{
13073   match(CallStaticJava);
13074   effect(USE meth);
13075 
13076   ins_cost(300);
13077   format %{ "call,static " %}
13078   opcode(0xE8); /* E8 cd */
13079   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13080   ins_pipe(pipe_slow);
13081   ins_alignment(4);
13082 %}
13083 
13084 // Call Java Dynamic Instruction
13085 // Note: If this code changes, the corresponding ret_addr_offset() and
13086 //       compute_padding() functions will have to be adjusted.
13087 instruct CallDynamicJavaDirect(method meth)
13088 %{
13089   match(CallDynamicJava);
13090   effect(USE meth);
13091 
13092   ins_cost(300);
13093   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13094             "call,dynamic " %}
13095   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13096   ins_pipe(pipe_slow);
13097   ins_alignment(4);
13098 %}
13099 
13100 // Call Runtime Instruction
13101 instruct CallRuntimeDirect(method meth)
13102 %{
13103   match(CallRuntime);
13104   effect(USE meth);
13105 
13106   ins_cost(300);
13107   format %{ "call,runtime " %}
13108   ins_encode(clear_avx, Java_To_Runtime(meth));
13109   ins_pipe(pipe_slow);
13110 %}
13111 
13112 // Call runtime without safepoint
13113 instruct CallLeafDirect(method meth)
13114 %{
13115   match(CallLeaf);
13116   effect(USE meth);
13117 
13118   ins_cost(300);
13119   format %{ "call_leaf,runtime " %}
13120   ins_encode(clear_avx, Java_To_Runtime(meth));
13121   ins_pipe(pipe_slow);
13122 %}
13123 
13124 // Call runtime without safepoint and with vector arguments
13125 instruct CallLeafDirectVector(method meth)
13126 %{
13127   match(CallLeafVector);
13128   effect(USE meth);
13129 
13130   ins_cost(300);
13131   format %{ "call_leaf,vector " %}
13132   ins_encode(Java_To_Runtime(meth));
13133   ins_pipe(pipe_slow);
13134 %}
13135 
13136 //
13137 instruct CallNativeDirect(method meth)
13138 %{
13139   match(CallNative);
13140   effect(USE meth);
13141 
13142   ins_cost(300);
13143   format %{ "call_native " %}
13144   ins_encode(clear_avx, Java_To_Runtime(meth));
13145   ins_pipe(pipe_slow);
13146 %}
13147 
13148 // Call runtime without safepoint
13149 instruct CallLeafNoFPDirect(method meth)
13150 %{
13151   match(CallLeafNoFP);
13152   effect(USE meth);
13153 
13154   ins_cost(300);
13155   format %{ "call_leaf_nofp,runtime " %}
13156   ins_encode(clear_avx, Java_To_Runtime(meth));
13157   ins_pipe(pipe_slow);
13158 %}
13159 
13160 // Return Instruction
13161 // Remove the return address & jump to it.
13162 // Notice: We always emit a nop after a ret to make sure there is room
13163 // for safepoint patching
13164 instruct Ret()
13165 %{
13166   match(Return);
13167 
13168   format %{ "ret" %}
13169   ins_encode %{
13170     __ ret(0);
13171   %}
13172   ins_pipe(pipe_jmp);
13173 %}
13174 
13175 // Tail Call; Jump from runtime stub to Java code.
13176 // Also known as an 'interprocedural jump'.
13177 // Target of jump will eventually return to caller.
13178 // TailJump below removes the return address.
13179 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13180 %{
13181   match(TailCall jump_target method_ptr);
13182 
13183   ins_cost(300);
13184   format %{ "jmp     $jump_target\t# rbx holds method" %}
13185   ins_encode %{
13186     __ jmp($jump_target$$Register);
13187   %}
13188   ins_pipe(pipe_jmp);
13189 %}
13190 
13191 // Tail Jump; remove the return address; jump to target.
13192 // TailCall above leaves the return address around.
13193 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13194 %{
13195   match(TailJump jump_target ex_oop);
13196 
13197   ins_cost(300);
13198   format %{ "popq    rdx\t# pop return address\n\t"
13199             "jmp     $jump_target" %}
13200   ins_encode %{
13201     __ popq(as_Register(RDX_enc));
13202     __ jmp($jump_target$$Register);
13203   %}
13204   ins_pipe(pipe_jmp);
13205 %}
13206 
13207 // Create exception oop: created by stack-crawling runtime code.
13208 // Created exception is now available to this handler, and is setup
13209 // just prior to jumping to this handler.  No code emitted.
13210 instruct CreateException(rax_RegP ex_oop)
13211 %{
13212   match(Set ex_oop (CreateEx));
13213 
13214   size(0);
13215   // use the following format syntax
13216   format %{ "# exception oop is in rax; no code emitted" %}
13217   ins_encode();
13218   ins_pipe(empty);
13219 %}
13220 
13221 // Rethrow exception:
13222 // The exception oop will come in the first argument position.
13223 // Then JUMP (not call) to the rethrow stub code.
13224 instruct RethrowException()
13225 %{
13226   match(Rethrow);
13227 
13228   // use the following format syntax
13229   format %{ "jmp     rethrow_stub" %}
13230   ins_encode(enc_rethrow);
13231   ins_pipe(pipe_jmp);
13232 %}
13233 
13234 // ============================================================================
13235 // This name is KNOWN by the ADLC and cannot be changed.
13236 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13237 // for this guy.
13238 instruct tlsLoadP(r15_RegP dst) %{
13239   match(Set dst (ThreadLocal));
13240   effect(DEF dst);
13241 
13242   size(0);
13243   format %{ "# TLS is in R15" %}
13244   ins_encode( /*empty encoding*/ );
13245   ins_pipe(ialu_reg_reg);
13246 %}
13247 
13248 
13249 //----------PEEPHOLE RULES-----------------------------------------------------
13250 // These must follow all instruction definitions as they use the names
13251 // defined in the instructions definitions.
13252 //
13253 // peepmatch ( root_instr_name [preceding_instruction]* );
13254 //
13255 // peepconstraint %{
13256 // (instruction_number.operand_name relational_op instruction_number.operand_name
13257 //  [, ...] );
13258 // // instruction numbers are zero-based using left to right order in peepmatch
13259 //
13260 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13261 // // provide an instruction_number.operand_name for each operand that appears
13262 // // in the replacement instruction's match rule
13263 //
13264 // ---------VM FLAGS---------------------------------------------------------
13265 //
13266 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13267 //
13268 // Each peephole rule is given an identifying number starting with zero and
13269 // increasing by one in the order seen by the parser.  An individual peephole
13270 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13271 // on the command-line.
13272 //
13273 // ---------CURRENT LIMITATIONS----------------------------------------------
13274 //
13275 // Only match adjacent instructions in same basic block
13276 // Only equality constraints
13277 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13278 // Only one replacement instruction
13279 //
13280 // ---------EXAMPLE----------------------------------------------------------
13281 //
13282 // // pertinent parts of existing instructions in architecture description
13283 // instruct movI(rRegI dst, rRegI src)
13284 // %{
13285 //   match(Set dst (CopyI src));
13286 // %}
13287 //
13288 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13289 // %{
13290 //   match(Set dst (AddI dst src));
13291 //   effect(KILL cr);
13292 // %}
13293 //
13294 // // Change (inc mov) to lea
13295 // peephole %{
13296 //   // increment preceded by register-register move
13297 //   peepmatch ( incI_rReg movI );
13298 //   // require that the destination register of the increment
13299 //   // match the destination register of the move
13300 //   peepconstraint ( 0.dst == 1.dst );
13301 //   // construct a replacement instruction that sets
13302 //   // the destination to ( move's source register + one )
13303 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13304 // %}
13305 //
13306 
13307 // Implementation no longer uses movX instructions since
13308 // machine-independent system no longer uses CopyX nodes.
13309 //
13310 // peephole
13311 // %{
13312 //   peepmatch (incI_rReg movI);
13313 //   peepconstraint (0.dst == 1.dst);
13314 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13315 // %}
13316 
13317 // peephole
13318 // %{
13319 //   peepmatch (decI_rReg movI);
13320 //   peepconstraint (0.dst == 1.dst);
13321 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13322 // %}
13323 
13324 // peephole
13325 // %{
13326 //   peepmatch (addI_rReg_imm movI);
13327 //   peepconstraint (0.dst == 1.dst);
13328 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13329 // %}
13330 
13331 // peephole
13332 // %{
13333 //   peepmatch (incL_rReg movL);
13334 //   peepconstraint (0.dst == 1.dst);
13335 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13336 // %}
13337 
13338 // peephole
13339 // %{
13340 //   peepmatch (decL_rReg movL);
13341 //   peepconstraint (0.dst == 1.dst);
13342 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13343 // %}
13344 
13345 // peephole
13346 // %{
13347 //   peepmatch (addL_rReg_imm movL);
13348 //   peepconstraint (0.dst == 1.dst);
13349 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13350 // %}
13351 
13352 // peephole
13353 // %{
13354 //   peepmatch (addP_rReg_imm movP);
13355 //   peepconstraint (0.dst == 1.dst);
13356 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13357 // %}
13358 
13359 // // Change load of spilled value to only a spill
13360 // instruct storeI(memory mem, rRegI src)
13361 // %{
13362 //   match(Set mem (StoreI mem src));
13363 // %}
13364 //
13365 // instruct loadI(rRegI dst, memory mem)
13366 // %{
13367 //   match(Set dst (LoadI mem));
13368 // %}
13369 //
13370 
13371 peephole
13372 %{
13373   peepmatch (loadI storeI);
13374   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13375   peepreplace (storeI(1.mem 1.mem 1.src));
13376 %}
13377 
13378 peephole
13379 %{
13380   peepmatch (loadL storeL);
13381   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13382   peepreplace (storeL(1.mem 1.mem 1.src));
13383 %}
13384 
13385 //----------SMARTSPILL RULES---------------------------------------------------
13386 // These must follow all instruction definitions as they use the names
13387 // defined in the instructions definitions.