1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 source_hpp %{
  323 
  324 extern RegMask _ANY_REG_mask;
  325 extern RegMask _PTR_REG_mask;
  326 extern RegMask _PTR_REG_NO_RBP_mask;
  327 extern RegMask _PTR_NO_RAX_REG_mask;
  328 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  329 extern RegMask _LONG_REG_mask;
  330 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  331 extern RegMask _LONG_NO_RCX_REG_mask;
  332 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  333 extern RegMask _INT_REG_mask;
  334 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  335 extern RegMask _INT_NO_RCX_REG_mask;
  336 extern RegMask _INT_NO_RBP_R13_REG_mask;
  337 extern RegMask _FLOAT_REG_mask;
  338 
  339 extern RegMask _STACK_OR_PTR_REG_mask;
  340 extern RegMask _STACK_OR_LONG_REG_mask;
  341 extern RegMask _STACK_OR_LONG_NO_RAX_RDX_REG_mask;
  342 extern RegMask _STACK_OR_INT_REG_mask;
  343 extern RegMask _STACK_OR_INT_NO_RAX_RDX_REG_mask;
  344 
  345 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  346 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  347 inline const RegMask& STACK_OR_LONG_NO_RAX_RDX_REG_mask() {
  348   return _STACK_OR_LONG_NO_RAX_RDX_REG_mask;
  349 }
  350 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  351 inline const RegMask& STACK_OR_INT_NO_RAX_RDX_REG_mask() {
  352   return _STACK_OR_INT_NO_RAX_RDX_REG_mask;
  353 }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_LONG_NO_RAX_RDX_REG_mask;
  380 RegMask _STACK_OR_INT_REG_mask;
  381 RegMask _STACK_OR_INT_NO_RAX_RDX_REG_mask;
  382 
  383 static bool need_r12_heapbase() {
  384   return UseCompressedOops;
  385 }
  386 
  387 void reg_mask_init() {
  388   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  389   // We derive a number of subsets from it.
  390   _ANY_REG_mask = _ALL_REG_mask;
  391 
  392   if (PreserveFramePointer) {
  393     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  394     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  395   }
  396   if (need_r12_heapbase()) {
  397     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  398     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  399   }
  400 
  401   _PTR_REG_mask = _ANY_REG_mask;
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  404   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  405   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  406 
  407   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  408   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  409 
  410   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  411   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  412   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  413 
  414   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  415   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  416   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  417 
  418   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  419   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  420   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  421 
  422   _LONG_REG_mask = _PTR_REG_mask;
  423   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  424   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  425 
  426   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  429   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  430   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  431 
  432   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  433   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  434   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  435 
  436   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  439   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  440   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  441 
  442   _STACK_OR_LONG_NO_RAX_RDX_REG_mask = _LONG_NO_RAX_RDX_REG_mask;
  443   _STACK_OR_LONG_NO_RAX_RDX_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  444 
  445   _INT_REG_mask = _ALL_INT_REG_mask;
  446   if (PreserveFramePointer) {
  447     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  448   }
  449   if (need_r12_heapbase()) {
  450     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  451   }
  452 
  453   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  454   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  455 
  456   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  457   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  458   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  459 
  460   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  461   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  462 
  463   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  464   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  465   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  466 
  467   _STACK_OR_INT_NO_RAX_RDX_REG_mask = _INT_NO_RAX_RDX_REG_mask;
  468   _STACK_OR_INT_NO_RAX_RDX_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  469 
  470   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  471   // from the float_reg_legacy/float_reg_evex register class.
  472   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  473 }
  474 
  475 static bool generate_vzeroupper(Compile* C) {
  476   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  477 }
  478 
  479 static int clear_avx_size() {
  480   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  481 }
  482 
  483 // !!!!! Special hack to get all types of calls to specify the byte offset
  484 //       from the start of the call to the point where the return address
  485 //       will point.
  486 int MachCallStaticJavaNode::ret_addr_offset()
  487 {
  488   int offset = 5; // 5 bytes from start of call to where return address points
  489   offset += clear_avx_size();
  490   return offset;
  491 }
  492 
  493 int MachCallDynamicJavaNode::ret_addr_offset()
  494 {
  495   int offset = 15; // 15 bytes from start of call to where return address points
  496   offset += clear_avx_size();
  497   return offset;
  498 }
  499 
  500 int MachCallRuntimeNode::ret_addr_offset() {
  501   int offset = 13; // movq r10,#addr; callq (r10)
  502   if (this->ideal_Opcode() != Op_CallLeafVector) {
  503     offset += clear_avx_size();
  504   }
  505   return offset;
  506 }
  507 
  508 int MachCallNativeNode::ret_addr_offset() {
  509   int offset = 13; // movq r10,#addr; callq (r10)
  510   offset += clear_avx_size();
  511   return offset;
  512 }
  513 //
  514 // Compute padding required for nodes which need alignment
  515 //
  516 
  517 // The address of the call instruction needs to be 4-byte aligned to
  518 // ensure that it does not span a cache line so that it can be patched.
  519 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  520 {
  521   current_offset += clear_avx_size(); // skip vzeroupper
  522   current_offset += 1; // skip call opcode byte
  523   return align_up(current_offset, alignment_required()) - current_offset;
  524 }
  525 
  526 // The address of the call instruction needs to be 4-byte aligned to
  527 // ensure that it does not span a cache line so that it can be patched.
  528 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  529 {
  530   current_offset += clear_avx_size(); // skip vzeroupper
  531   current_offset += 11; // skip movq instruction + call opcode byte
  532   return align_up(current_offset, alignment_required()) - current_offset;
  533 }
  534 
  535 // EMIT_RM()
  536 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  537   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  538   cbuf.insts()->emit_int8(c);
  539 }
  540 
  541 // EMIT_CC()
  542 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  543   unsigned char c = (unsigned char) (f1 | f2);
  544   cbuf.insts()->emit_int8(c);
  545 }
  546 
  547 // EMIT_OPCODE()
  548 void emit_opcode(CodeBuffer &cbuf, int code) {
  549   cbuf.insts()->emit_int8((unsigned char) code);
  550 }
  551 
  552 // EMIT_OPCODE() w/ relocation information
  553 void emit_opcode(CodeBuffer &cbuf,
  554                  int code, relocInfo::relocType reloc, int offset, int format)
  555 {
  556   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  557   emit_opcode(cbuf, code);
  558 }
  559 
  560 // EMIT_D8()
  561 void emit_d8(CodeBuffer &cbuf, int d8) {
  562   cbuf.insts()->emit_int8((unsigned char) d8);
  563 }
  564 
  565 // EMIT_D16()
  566 void emit_d16(CodeBuffer &cbuf, int d16) {
  567   cbuf.insts()->emit_int16(d16);
  568 }
  569 
  570 // EMIT_D32()
  571 void emit_d32(CodeBuffer &cbuf, int d32) {
  572   cbuf.insts()->emit_int32(d32);
  573 }
  574 
  575 // EMIT_D64()
  576 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  577   cbuf.insts()->emit_int64(d64);
  578 }
  579 
  580 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  581 void emit_d32_reloc(CodeBuffer& cbuf,
  582                     int d32,
  583                     relocInfo::relocType reloc,
  584                     int format)
  585 {
  586   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  587   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  588   cbuf.insts()->emit_int32(d32);
  589 }
  590 
  591 // emit 32 bit value and construct relocation entry from RelocationHolder
  592 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  593 #ifdef ASSERT
  594   if (rspec.reloc()->type() == relocInfo::oop_type &&
  595       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  596     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  597     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  598   }
  599 #endif
  600   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  601   cbuf.insts()->emit_int32(d32);
  602 }
  603 
  604 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  605   address next_ip = cbuf.insts_end() + 4;
  606   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  607                  external_word_Relocation::spec(addr),
  608                  RELOC_DISP32);
  609 }
  610 
  611 
  612 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  613 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  614   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  615   cbuf.insts()->emit_int64(d64);
  616 }
  617 
  618 // emit 64 bit value and construct relocation entry from RelocationHolder
  619 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  620 #ifdef ASSERT
  621   if (rspec.reloc()->type() == relocInfo::oop_type &&
  622       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  623     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  624     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  625   }
  626 #endif
  627   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  628   cbuf.insts()->emit_int64(d64);
  629 }
  630 
  631 // Access stack slot for load or store
  632 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  633 {
  634   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  635   if (-0x80 <= disp && disp < 0x80) {
  636     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  637     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  638     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  639   } else {
  640     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  641     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  642     emit_d32(cbuf, disp);     // Displacement // R/M byte
  643   }
  644 }
  645 
  646    // rRegI ereg, memory mem) %{    // emit_reg_mem
  647 void encode_RegMem(CodeBuffer &cbuf,
  648                    int reg,
  649                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  650 {
  651   assert(disp_reloc == relocInfo::none, "cannot have disp");
  652   int regenc = reg & 7;
  653   int baseenc = base & 7;
  654   int indexenc = index & 7;
  655 
  656   // There is no index & no scale, use form without SIB byte
  657   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  658     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  659     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  660       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  661     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  662       // If 8-bit displacement, mode 0x1
  663       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  664       emit_d8(cbuf, disp);
  665     } else {
  666       // If 32-bit displacement
  667       if (base == -1) { // Special flag for absolute address
  668         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  669         if (disp_reloc != relocInfo::none) {
  670           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  671         } else {
  672           emit_d32(cbuf, disp);
  673         }
  674       } else {
  675         // Normal base + offset
  676         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  677         if (disp_reloc != relocInfo::none) {
  678           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  679         } else {
  680           emit_d32(cbuf, disp);
  681         }
  682       }
  683     }
  684   } else {
  685     // Else, encode with the SIB byte
  686     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  687     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  688       // If no displacement
  689       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  690       emit_rm(cbuf, scale, indexenc, baseenc);
  691     } else {
  692       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  693         // If 8-bit displacement, mode 0x1
  694         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  695         emit_rm(cbuf, scale, indexenc, baseenc);
  696         emit_d8(cbuf, disp);
  697       } else {
  698         // If 32-bit displacement
  699         if (base == 0x04 ) {
  700           emit_rm(cbuf, 0x2, regenc, 0x4);
  701           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  702         } else {
  703           emit_rm(cbuf, 0x2, regenc, 0x4);
  704           emit_rm(cbuf, scale, indexenc, baseenc); // *
  705         }
  706         if (disp_reloc != relocInfo::none) {
  707           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  708         } else {
  709           emit_d32(cbuf, disp);
  710         }
  711       }
  712     }
  713   }
  714 }
  715 
  716 // This could be in MacroAssembler but it's fairly C2 specific
  717 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  718   Label exit;
  719   __ jccb(Assembler::noParity, exit);
  720   __ pushf();
  721   //
  722   // comiss/ucomiss instructions set ZF,PF,CF flags and
  723   // zero OF,AF,SF for NaN values.
  724   // Fixup flags by zeroing ZF,PF so that compare of NaN
  725   // values returns 'less than' result (CF is set).
  726   // Leave the rest of flags unchanged.
  727   //
  728   //    7 6 5 4 3 2 1 0
  729   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  730   //    0 0 1 0 1 0 1 1   (0x2B)
  731   //
  732   __ andq(Address(rsp, 0), 0xffffff2b);
  733   __ popf();
  734   __ bind(exit);
  735 }
  736 
  737 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  738   Label done;
  739   __ movl(dst, -1);
  740   __ jcc(Assembler::parity, done);
  741   __ jcc(Assembler::below, done);
  742   __ setb(Assembler::notEqual, dst);
  743   __ movzbl(dst, dst);
  744   __ bind(done);
  745 }
  746 
  747 // Math.min()    # Math.max()
  748 // --------------------------
  749 // ucomis[s/d]   #
  750 // ja   -> b     # a
  751 // jp   -> NaN   # NaN
  752 // jb   -> a     # b
  753 // je            #
  754 // |-jz -> a | b # a & b
  755 // |    -> a     #
  756 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  757                      XMMRegister a, XMMRegister b,
  758                      XMMRegister xmmt, Register rt,
  759                      bool min, bool single) {
  760 
  761   Label nan, zero, below, above, done;
  762 
  763   if (single)
  764     __ ucomiss(a, b);
  765   else
  766     __ ucomisd(a, b);
  767 
  768   if (dst->encoding() != (min ? b : a)->encoding())
  769     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  770   else
  771     __ jccb(Assembler::above, done);
  772 
  773   __ jccb(Assembler::parity, nan);  // PF=1
  774   __ jccb(Assembler::below, below); // CF=1
  775 
  776   // equal
  777   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  778   if (single) {
  779     __ ucomiss(a, xmmt);
  780     __ jccb(Assembler::equal, zero);
  781 
  782     __ movflt(dst, a);
  783     __ jmp(done);
  784   }
  785   else {
  786     __ ucomisd(a, xmmt);
  787     __ jccb(Assembler::equal, zero);
  788 
  789     __ movdbl(dst, a);
  790     __ jmp(done);
  791   }
  792 
  793   __ bind(zero);
  794   if (min)
  795     __ vpor(dst, a, b, Assembler::AVX_128bit);
  796   else
  797     __ vpand(dst, a, b, Assembler::AVX_128bit);
  798 
  799   __ jmp(done);
  800 
  801   __ bind(above);
  802   if (single)
  803     __ movflt(dst, min ? b : a);
  804   else
  805     __ movdbl(dst, min ? b : a);
  806 
  807   __ jmp(done);
  808 
  809   __ bind(nan);
  810   if (single) {
  811     __ movl(rt, 0x7fc00000); // Float.NaN
  812     __ movdl(dst, rt);
  813   }
  814   else {
  815     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  816     __ movdq(dst, rt);
  817   }
  818   __ jmp(done);
  819 
  820   __ bind(below);
  821   if (single)
  822     __ movflt(dst, min ? a : b);
  823   else
  824     __ movdbl(dst, min ? a : b);
  825 
  826   __ bind(done);
  827 }
  828 
  829 //=============================================================================
  830 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  831 
  832 int ConstantTable::calculate_table_base_offset() const {
  833   return 0;  // absolute addressing, no offset
  834 }
  835 
  836 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  837 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  838   ShouldNotReachHere();
  839 }
  840 
  841 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  842   // Empty encoding
  843 }
  844 
  845 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  846   return 0;
  847 }
  848 
  849 #ifndef PRODUCT
  850 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  851   st->print("# MachConstantBaseNode (empty encoding)");
  852 }
  853 #endif
  854 
  855 
  856 //=============================================================================
  857 #ifndef PRODUCT
  858 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  859   Compile* C = ra_->C;
  860 
  861   int framesize = C->output()->frame_size_in_bytes();
  862   int bangsize = C->output()->bang_size_in_bytes();
  863   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  864   // Remove wordSize for return addr which is already pushed.
  865   framesize -= wordSize;
  866 
  867   if (C->output()->need_stack_bang(bangsize)) {
  868     framesize -= wordSize;
  869     st->print("# stack bang (%d bytes)", bangsize);
  870     st->print("\n\t");
  871     st->print("pushq   rbp\t# Save rbp");
  872     if (PreserveFramePointer) {
  873         st->print("\n\t");
  874         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  875     }
  876     if (framesize) {
  877       st->print("\n\t");
  878       st->print("subq    rsp, #%d\t# Create frame",framesize);
  879     }
  880   } else {
  881     st->print("subq    rsp, #%d\t# Create frame",framesize);
  882     st->print("\n\t");
  883     framesize -= wordSize;
  884     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  885     if (PreserveFramePointer) {
  886       st->print("\n\t");
  887       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  888       if (framesize > 0) {
  889         st->print("\n\t");
  890         st->print("addq    rbp, #%d", framesize);
  891       }
  892     }
  893   }
  894 
  895   if (VerifyStackAtCalls) {
  896     st->print("\n\t");
  897     framesize -= wordSize;
  898     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  899 #ifdef ASSERT
  900     st->print("\n\t");
  901     st->print("# stack alignment check");
  902 #endif
  903   }
  904   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  905     st->print("\n\t");
  906     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  907     st->print("\n\t");
  908     st->print("je      fast_entry\t");
  909     st->print("\n\t");
  910     st->print("call    #nmethod_entry_barrier_stub\t");
  911     st->print("\n\tfast_entry:");
  912   }
  913   st->cr();
  914 }
  915 #endif
  916 
  917 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  918   Compile* C = ra_->C;
  919   MacroAssembler _masm(&cbuf);
  920 
  921   int framesize = C->output()->frame_size_in_bytes();
  922   int bangsize = C->output()->bang_size_in_bytes();
  923 
  924   if (C->clinit_barrier_on_entry()) {
  925     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  926     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  927 
  928     Label L_skip_barrier;
  929     Register klass = rscratch1;
  930 
  931     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  932     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  933 
  934     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  935 
  936     __ bind(L_skip_barrier);
  937   }
  938 
  939   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  940 
  941   C->output()->set_frame_complete(cbuf.insts_size());
  942 
  943   if (C->has_mach_constant_base_node()) {
  944     // NOTE: We set the table base offset here because users might be
  945     // emitted before MachConstantBaseNode.
  946     ConstantTable& constant_table = C->output()->constant_table();
  947     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  948   }
  949 }
  950 
  951 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  952 {
  953   return MachNode::size(ra_); // too many variables; just compute it
  954                               // the hard way
  955 }
  956 
  957 int MachPrologNode::reloc() const
  958 {
  959   return 0; // a large enough number
  960 }
  961 
  962 //=============================================================================
  963 #ifndef PRODUCT
  964 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  965 {
  966   Compile* C = ra_->C;
  967   if (generate_vzeroupper(C)) {
  968     st->print("vzeroupper");
  969     st->cr(); st->print("\t");
  970   }
  971 
  972   int framesize = C->output()->frame_size_in_bytes();
  973   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  974   // Remove word for return adr already pushed
  975   // and RBP
  976   framesize -= 2*wordSize;
  977 
  978   if (framesize) {
  979     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  980     st->print("\t");
  981   }
  982 
  983   st->print_cr("popq    rbp");
  984   if (do_polling() && C->is_method_compilation()) {
  985     st->print("\t");
  986     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  987                  "ja       #safepoint_stub\t"
  988                  "# Safepoint: poll for GC");
  989   }
  990 }
  991 #endif
  992 
  993 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  994 {
  995   Compile* C = ra_->C;
  996   MacroAssembler _masm(&cbuf);
  997 
  998   if (generate_vzeroupper(C)) {
  999     // Clear upper bits of YMM registers when current compiled code uses
 1000     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1001     __ vzeroupper();
 1002   }
 1003 
 1004   int framesize = C->output()->frame_size_in_bytes();
 1005   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 1006   // Remove word for return adr already pushed
 1007   // and RBP
 1008   framesize -= 2*wordSize;
 1009 
 1010   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 1011 
 1012   if (framesize) {
 1013     emit_opcode(cbuf, Assembler::REX_W);
 1014     if (framesize < 0x80) {
 1015       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 1016       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 1017       emit_d8(cbuf, framesize);
 1018     } else {
 1019       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 1020       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 1021       emit_d32(cbuf, framesize);
 1022     }
 1023   }
 1024 
 1025   // popq rbp
 1026   emit_opcode(cbuf, 0x58 | RBP_enc);
 1027 
 1028   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1029     __ reserved_stack_check();
 1030   }
 1031 
 1032   if (do_polling() && C->is_method_compilation()) {
 1033     MacroAssembler _masm(&cbuf);
 1034     Label dummy_label;
 1035     Label* code_stub = &dummy_label;
 1036     if (!C->output()->in_scratch_emit_size()) {
 1037       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1038     }
 1039     __ relocate(relocInfo::poll_return_type);
 1040     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1041   }
 1042 }
 1043 
 1044 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1045 {
 1046   return MachNode::size(ra_); // too many variables; just compute it
 1047                               // the hard way
 1048 }
 1049 
 1050 int MachEpilogNode::reloc() const
 1051 {
 1052   return 2; // a large enough number
 1053 }
 1054 
 1055 const Pipeline* MachEpilogNode::pipeline() const
 1056 {
 1057   return MachNode::pipeline_class();
 1058 }
 1059 
 1060 //=============================================================================
 1061 
 1062 enum RC {
 1063   rc_bad,
 1064   rc_int,
 1065   rc_kreg,
 1066   rc_float,
 1067   rc_stack
 1068 };
 1069 
 1070 static enum RC rc_class(OptoReg::Name reg)
 1071 {
 1072   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1073 
 1074   if (OptoReg::is_stack(reg)) return rc_stack;
 1075 
 1076   VMReg r = OptoReg::as_VMReg(reg);
 1077 
 1078   if (r->is_Register()) return rc_int;
 1079 
 1080   if (r->is_KRegister()) return rc_kreg;
 1081 
 1082   assert(r->is_XMMRegister(), "must be");
 1083   return rc_float;
 1084 }
 1085 
 1086 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1087 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1088                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1089 
 1090 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1091                      int stack_offset, int reg, uint ireg, outputStream* st);
 1092 
 1093 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1094                                       int dst_offset, uint ireg, outputStream* st) {
 1095   if (cbuf) {
 1096     MacroAssembler _masm(cbuf);
 1097     switch (ireg) {
 1098     case Op_VecS:
 1099       __ movq(Address(rsp, -8), rax);
 1100       __ movl(rax, Address(rsp, src_offset));
 1101       __ movl(Address(rsp, dst_offset), rax);
 1102       __ movq(rax, Address(rsp, -8));
 1103       break;
 1104     case Op_VecD:
 1105       __ pushq(Address(rsp, src_offset));
 1106       __ popq (Address(rsp, dst_offset));
 1107       break;
 1108     case Op_VecX:
 1109       __ pushq(Address(rsp, src_offset));
 1110       __ popq (Address(rsp, dst_offset));
 1111       __ pushq(Address(rsp, src_offset+8));
 1112       __ popq (Address(rsp, dst_offset+8));
 1113       break;
 1114     case Op_VecY:
 1115       __ vmovdqu(Address(rsp, -32), xmm0);
 1116       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1117       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1118       __ vmovdqu(xmm0, Address(rsp, -32));
 1119       break;
 1120     case Op_VecZ:
 1121       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1122       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1123       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1124       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1125       break;
 1126     default:
 1127       ShouldNotReachHere();
 1128     }
 1129 #ifndef PRODUCT
 1130   } else {
 1131     switch (ireg) {
 1132     case Op_VecS:
 1133       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1134                 "movl    rax, [rsp + #%d]\n\t"
 1135                 "movl    [rsp + #%d], rax\n\t"
 1136                 "movq    rax, [rsp - #8]",
 1137                 src_offset, dst_offset);
 1138       break;
 1139     case Op_VecD:
 1140       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1141                 "popq    [rsp + #%d]",
 1142                 src_offset, dst_offset);
 1143       break;
 1144      case Op_VecX:
 1145       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1146                 "popq    [rsp + #%d]\n\t"
 1147                 "pushq   [rsp + #%d]\n\t"
 1148                 "popq    [rsp + #%d]",
 1149                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1150       break;
 1151     case Op_VecY:
 1152       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1153                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1154                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1155                 "vmovdqu xmm0, [rsp - #32]",
 1156                 src_offset, dst_offset);
 1157       break;
 1158     case Op_VecZ:
 1159       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1160                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1161                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1162                 "vmovdqu xmm0, [rsp - #64]",
 1163                 src_offset, dst_offset);
 1164       break;
 1165     default:
 1166       ShouldNotReachHere();
 1167     }
 1168 #endif
 1169   }
 1170 }
 1171 
 1172 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1173                                        PhaseRegAlloc* ra_,
 1174                                        bool do_size,
 1175                                        outputStream* st) const {
 1176   assert(cbuf != NULL || st  != NULL, "sanity");
 1177   // Get registers to move
 1178   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1179   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1180   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1181   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1182 
 1183   enum RC src_second_rc = rc_class(src_second);
 1184   enum RC src_first_rc = rc_class(src_first);
 1185   enum RC dst_second_rc = rc_class(dst_second);
 1186   enum RC dst_first_rc = rc_class(dst_first);
 1187 
 1188   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1189          "must move at least 1 register" );
 1190 
 1191   if (src_first == dst_first && src_second == dst_second) {
 1192     // Self copy, no move
 1193     return 0;
 1194   }
 1195   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1196     uint ireg = ideal_reg();
 1197     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1198     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1199     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1200       // mem -> mem
 1201       int src_offset = ra_->reg2offset(src_first);
 1202       int dst_offset = ra_->reg2offset(dst_first);
 1203       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1204     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1205       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1206     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1207       int stack_offset = ra_->reg2offset(dst_first);
 1208       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1209     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1210       int stack_offset = ra_->reg2offset(src_first);
 1211       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1212     } else {
 1213       ShouldNotReachHere();
 1214     }
 1215     return 0;
 1216   }
 1217   if (src_first_rc == rc_stack) {
 1218     // mem ->
 1219     if (dst_first_rc == rc_stack) {
 1220       // mem -> mem
 1221       assert(src_second != dst_first, "overlap");
 1222       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1223           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1224         // 64-bit
 1225         int src_offset = ra_->reg2offset(src_first);
 1226         int dst_offset = ra_->reg2offset(dst_first);
 1227         if (cbuf) {
 1228           MacroAssembler _masm(cbuf);
 1229           __ pushq(Address(rsp, src_offset));
 1230           __ popq (Address(rsp, dst_offset));
 1231 #ifndef PRODUCT
 1232         } else {
 1233           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1234                     "popq    [rsp + #%d]",
 1235                      src_offset, dst_offset);
 1236 #endif
 1237         }
 1238       } else {
 1239         // 32-bit
 1240         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1241         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1242         // No pushl/popl, so:
 1243         int src_offset = ra_->reg2offset(src_first);
 1244         int dst_offset = ra_->reg2offset(dst_first);
 1245         if (cbuf) {
 1246           MacroAssembler _masm(cbuf);
 1247           __ movq(Address(rsp, -8), rax);
 1248           __ movl(rax, Address(rsp, src_offset));
 1249           __ movl(Address(rsp, dst_offset), rax);
 1250           __ movq(rax, Address(rsp, -8));
 1251 #ifndef PRODUCT
 1252         } else {
 1253           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1254                     "movl    rax, [rsp + #%d]\n\t"
 1255                     "movl    [rsp + #%d], rax\n\t"
 1256                     "movq    rax, [rsp - #8]",
 1257                      src_offset, dst_offset);
 1258 #endif
 1259         }
 1260       }
 1261       return 0;
 1262     } else if (dst_first_rc == rc_int) {
 1263       // mem -> gpr
 1264       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1265           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1266         // 64-bit
 1267         int offset = ra_->reg2offset(src_first);
 1268         if (cbuf) {
 1269           MacroAssembler _masm(cbuf);
 1270           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1271 #ifndef PRODUCT
 1272         } else {
 1273           st->print("movq    %s, [rsp + #%d]\t# spill",
 1274                      Matcher::regName[dst_first],
 1275                      offset);
 1276 #endif
 1277         }
 1278       } else {
 1279         // 32-bit
 1280         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1281         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1282         int offset = ra_->reg2offset(src_first);
 1283         if (cbuf) {
 1284           MacroAssembler _masm(cbuf);
 1285           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1286 #ifndef PRODUCT
 1287         } else {
 1288           st->print("movl    %s, [rsp + #%d]\t# spill",
 1289                      Matcher::regName[dst_first],
 1290                      offset);
 1291 #endif
 1292         }
 1293       }
 1294       return 0;
 1295     } else if (dst_first_rc == rc_float) {
 1296       // mem-> xmm
 1297       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1298           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1299         // 64-bit
 1300         int offset = ra_->reg2offset(src_first);
 1301         if (cbuf) {
 1302           MacroAssembler _masm(cbuf);
 1303           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1304 #ifndef PRODUCT
 1305         } else {
 1306           st->print("%s  %s, [rsp + #%d]\t# spill",
 1307                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1308                      Matcher::regName[dst_first],
 1309                      offset);
 1310 #endif
 1311         }
 1312       } else {
 1313         // 32-bit
 1314         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1315         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1316         int offset = ra_->reg2offset(src_first);
 1317         if (cbuf) {
 1318           MacroAssembler _masm(cbuf);
 1319           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1320 #ifndef PRODUCT
 1321         } else {
 1322           st->print("movss   %s, [rsp + #%d]\t# spill",
 1323                      Matcher::regName[dst_first],
 1324                      offset);
 1325 #endif
 1326         }
 1327       }
 1328       return 0;
 1329     } else if (dst_first_rc == rc_kreg) {
 1330       // mem -> kreg
 1331       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1332           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1333         // 64-bit
 1334         int offset = ra_->reg2offset(src_first);
 1335         if (cbuf) {
 1336           MacroAssembler _masm(cbuf);
 1337           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1338 #ifndef PRODUCT
 1339         } else {
 1340           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1341                      Matcher::regName[dst_first],
 1342                      offset);
 1343 #endif
 1344         }
 1345       }
 1346       return 0;
 1347     }
 1348   } else if (src_first_rc == rc_int) {
 1349     // gpr ->
 1350     if (dst_first_rc == rc_stack) {
 1351       // gpr -> mem
 1352       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1353           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1354         // 64-bit
 1355         int offset = ra_->reg2offset(dst_first);
 1356         if (cbuf) {
 1357           MacroAssembler _masm(cbuf);
 1358           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1359 #ifndef PRODUCT
 1360         } else {
 1361           st->print("movq    [rsp + #%d], %s\t# spill",
 1362                      offset,
 1363                      Matcher::regName[src_first]);
 1364 #endif
 1365         }
 1366       } else {
 1367         // 32-bit
 1368         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1369         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1370         int offset = ra_->reg2offset(dst_first);
 1371         if (cbuf) {
 1372           MacroAssembler _masm(cbuf);
 1373           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1374 #ifndef PRODUCT
 1375         } else {
 1376           st->print("movl    [rsp + #%d], %s\t# spill",
 1377                      offset,
 1378                      Matcher::regName[src_first]);
 1379 #endif
 1380         }
 1381       }
 1382       return 0;
 1383     } else if (dst_first_rc == rc_int) {
 1384       // gpr -> gpr
 1385       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1386           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1387         // 64-bit
 1388         if (cbuf) {
 1389           MacroAssembler _masm(cbuf);
 1390           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1391                   as_Register(Matcher::_regEncode[src_first]));
 1392 #ifndef PRODUCT
 1393         } else {
 1394           st->print("movq    %s, %s\t# spill",
 1395                      Matcher::regName[dst_first],
 1396                      Matcher::regName[src_first]);
 1397 #endif
 1398         }
 1399         return 0;
 1400       } else {
 1401         // 32-bit
 1402         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1403         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1404         if (cbuf) {
 1405           MacroAssembler _masm(cbuf);
 1406           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1407                   as_Register(Matcher::_regEncode[src_first]));
 1408 #ifndef PRODUCT
 1409         } else {
 1410           st->print("movl    %s, %s\t# spill",
 1411                      Matcher::regName[dst_first],
 1412                      Matcher::regName[src_first]);
 1413 #endif
 1414         }
 1415         return 0;
 1416       }
 1417     } else if (dst_first_rc == rc_float) {
 1418       // gpr -> xmm
 1419       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1420           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1421         // 64-bit
 1422         if (cbuf) {
 1423           MacroAssembler _masm(cbuf);
 1424           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1425 #ifndef PRODUCT
 1426         } else {
 1427           st->print("movdq   %s, %s\t# spill",
 1428                      Matcher::regName[dst_first],
 1429                      Matcher::regName[src_first]);
 1430 #endif
 1431         }
 1432       } else {
 1433         // 32-bit
 1434         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1435         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1436         if (cbuf) {
 1437           MacroAssembler _masm(cbuf);
 1438           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1439 #ifndef PRODUCT
 1440         } else {
 1441           st->print("movdl   %s, %s\t# spill",
 1442                      Matcher::regName[dst_first],
 1443                      Matcher::regName[src_first]);
 1444 #endif
 1445         }
 1446       }
 1447       return 0;
 1448     } else if (dst_first_rc == rc_kreg) {
 1449       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1450           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1451         // 64-bit
 1452         if (cbuf) {
 1453           MacroAssembler _masm(cbuf);
 1454           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1455   #ifndef PRODUCT
 1456         } else {
 1457            st->print("kmovq   %s, %s\t# spill",
 1458                        Matcher::regName[dst_first],
 1459                        Matcher::regName[src_first]);
 1460   #endif
 1461         }
 1462       }
 1463       Unimplemented();
 1464       return 0;
 1465     }
 1466   } else if (src_first_rc == rc_float) {
 1467     // xmm ->
 1468     if (dst_first_rc == rc_stack) {
 1469       // xmm -> mem
 1470       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1471           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1472         // 64-bit
 1473         int offset = ra_->reg2offset(dst_first);
 1474         if (cbuf) {
 1475           MacroAssembler _masm(cbuf);
 1476           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1477 #ifndef PRODUCT
 1478         } else {
 1479           st->print("movsd   [rsp + #%d], %s\t# spill",
 1480                      offset,
 1481                      Matcher::regName[src_first]);
 1482 #endif
 1483         }
 1484       } else {
 1485         // 32-bit
 1486         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1487         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1488         int offset = ra_->reg2offset(dst_first);
 1489         if (cbuf) {
 1490           MacroAssembler _masm(cbuf);
 1491           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1492 #ifndef PRODUCT
 1493         } else {
 1494           st->print("movss   [rsp + #%d], %s\t# spill",
 1495                      offset,
 1496                      Matcher::regName[src_first]);
 1497 #endif
 1498         }
 1499       }
 1500       return 0;
 1501     } else if (dst_first_rc == rc_int) {
 1502       // xmm -> gpr
 1503       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1504           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1505         // 64-bit
 1506         if (cbuf) {
 1507           MacroAssembler _masm(cbuf);
 1508           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1509 #ifndef PRODUCT
 1510         } else {
 1511           st->print("movdq   %s, %s\t# spill",
 1512                      Matcher::regName[dst_first],
 1513                      Matcher::regName[src_first]);
 1514 #endif
 1515         }
 1516       } else {
 1517         // 32-bit
 1518         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1519         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1520         if (cbuf) {
 1521           MacroAssembler _masm(cbuf);
 1522           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1523 #ifndef PRODUCT
 1524         } else {
 1525           st->print("movdl   %s, %s\t# spill",
 1526                      Matcher::regName[dst_first],
 1527                      Matcher::regName[src_first]);
 1528 #endif
 1529         }
 1530       }
 1531       return 0;
 1532     } else if (dst_first_rc == rc_float) {
 1533       // xmm -> xmm
 1534       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1535           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1536         // 64-bit
 1537         if (cbuf) {
 1538           MacroAssembler _masm(cbuf);
 1539           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1540 #ifndef PRODUCT
 1541         } else {
 1542           st->print("%s  %s, %s\t# spill",
 1543                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1544                      Matcher::regName[dst_first],
 1545                      Matcher::regName[src_first]);
 1546 #endif
 1547         }
 1548       } else {
 1549         // 32-bit
 1550         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1551         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1552         if (cbuf) {
 1553           MacroAssembler _masm(cbuf);
 1554           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1555 #ifndef PRODUCT
 1556         } else {
 1557           st->print("%s  %s, %s\t# spill",
 1558                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1559                      Matcher::regName[dst_first],
 1560                      Matcher::regName[src_first]);
 1561 #endif
 1562         }
 1563       }
 1564       return 0;
 1565     } else if (dst_first_rc == rc_kreg) {
 1566       assert(false, "Illegal spilling");
 1567       return 0;
 1568     }
 1569   } else if (src_first_rc == rc_kreg) {
 1570     if (dst_first_rc == rc_stack) {
 1571       // mem -> kreg
 1572       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1573           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1574         // 64-bit
 1575         int offset = ra_->reg2offset(dst_first);
 1576         if (cbuf) {
 1577           MacroAssembler _masm(cbuf);
 1578           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1579 #ifndef PRODUCT
 1580         } else {
 1581           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1582                      offset,
 1583                      Matcher::regName[src_first]);
 1584 #endif
 1585         }
 1586       }
 1587       return 0;
 1588     } else if (dst_first_rc == rc_int) {
 1589       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1590           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1591         // 64-bit
 1592         if (cbuf) {
 1593           MacroAssembler _masm(cbuf);
 1594           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1595 #ifndef PRODUCT
 1596         } else {
 1597          st->print("kmovq   %s, %s\t# spill",
 1598                      Matcher::regName[dst_first],
 1599                      Matcher::regName[src_first]);
 1600 #endif
 1601         }
 1602       }
 1603       Unimplemented();
 1604       return 0;
 1605     } else if (dst_first_rc == rc_kreg) {
 1606       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1607           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1608         // 64-bit
 1609         if (cbuf) {
 1610           MacroAssembler _masm(cbuf);
 1611           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1612 #ifndef PRODUCT
 1613         } else {
 1614          st->print("kmovq   %s, %s\t# spill",
 1615                      Matcher::regName[dst_first],
 1616                      Matcher::regName[src_first]);
 1617 #endif
 1618         }
 1619       }
 1620       return 0;
 1621     } else if (dst_first_rc == rc_float) {
 1622       assert(false, "Illegal spill");
 1623       return 0;
 1624     }
 1625   }
 1626 
 1627   assert(0," foo ");
 1628   Unimplemented();
 1629   return 0;
 1630 }
 1631 
 1632 #ifndef PRODUCT
 1633 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1634   implementation(NULL, ra_, false, st);
 1635 }
 1636 #endif
 1637 
 1638 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1639   implementation(&cbuf, ra_, false, NULL);
 1640 }
 1641 
 1642 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1643   return MachNode::size(ra_);
 1644 }
 1645 
 1646 //=============================================================================
 1647 #ifndef PRODUCT
 1648 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1649 {
 1650   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1651   int reg = ra_->get_reg_first(this);
 1652   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1653             Matcher::regName[reg], offset);
 1654 }
 1655 #endif
 1656 
 1657 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1658 {
 1659   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1660   int reg = ra_->get_encode(this);
 1661   if (offset >= 0x80) {
 1662     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1663     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1664     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1665     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1666     emit_d32(cbuf, offset);
 1667   } else {
 1668     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1669     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1670     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1671     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1672     emit_d8(cbuf, offset);
 1673   }
 1674 }
 1675 
 1676 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1677 {
 1678   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1679   return (offset < 0x80) ? 5 : 8; // REX
 1680 }
 1681 
 1682 //=============================================================================
 1683 #ifndef PRODUCT
 1684 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1685 {
 1686   if (UseCompressedClassPointers) {
 1687     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1688     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1689     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1690   } else {
 1691     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1692                  "# Inline cache check");
 1693   }
 1694   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1695   st->print_cr("\tnop\t# nops to align entry point");
 1696 }
 1697 #endif
 1698 
 1699 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1700 {
 1701   MacroAssembler masm(&cbuf);
 1702   uint insts_size = cbuf.insts_size();
 1703   if (UseCompressedClassPointers) {
 1704     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1705     masm.cmpptr(rax, rscratch1);
 1706   } else {
 1707     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1708   }
 1709 
 1710   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1711 
 1712   /* WARNING these NOPs are critical so that verified entry point is properly
 1713      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1714   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1715   if (OptoBreakpoint) {
 1716     // Leave space for int3
 1717     nops_cnt -= 1;
 1718   }
 1719   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1720   if (nops_cnt > 0)
 1721     masm.nop(nops_cnt);
 1722 }
 1723 
 1724 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1725 {
 1726   return MachNode::size(ra_); // too many variables; just compute it
 1727                               // the hard way
 1728 }
 1729 
 1730 
 1731 //=============================================================================
 1732 
 1733 const bool Matcher::supports_vector_calling_convention(void) {
 1734   if (EnableVectorSupport && UseVectorStubs) {
 1735     return true;
 1736   }
 1737   return false;
 1738 }
 1739 
 1740 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1741   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1742   int lo = XMM0_num;
 1743   int hi = XMM0b_num;
 1744   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1745   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1746   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1747   return OptoRegPair(hi, lo);
 1748 }
 1749 
 1750 // Is this branch offset short enough that a short branch can be used?
 1751 //
 1752 // NOTE: If the platform does not provide any short branch variants, then
 1753 //       this method should return false for offset 0.
 1754 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1755   // The passed offset is relative to address of the branch.
 1756   // On 86 a branch displacement is calculated relative to address
 1757   // of a next instruction.
 1758   offset -= br_size;
 1759 
 1760   // the short version of jmpConUCF2 contains multiple branches,
 1761   // making the reach slightly less
 1762   if (rule == jmpConUCF2_rule)
 1763     return (-126 <= offset && offset <= 125);
 1764   return (-128 <= offset && offset <= 127);
 1765 }
 1766 
 1767 // Return whether or not this register is ever used as an argument.
 1768 // This function is used on startup to build the trampoline stubs in
 1769 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1770 // call in the trampoline, and arguments in those registers not be
 1771 // available to the callee.
 1772 bool Matcher::can_be_java_arg(int reg)
 1773 {
 1774   return
 1775     reg ==  RDI_num || reg == RDI_H_num ||
 1776     reg ==  RSI_num || reg == RSI_H_num ||
 1777     reg ==  RDX_num || reg == RDX_H_num ||
 1778     reg ==  RCX_num || reg == RCX_H_num ||
 1779     reg ==   R8_num || reg ==  R8_H_num ||
 1780     reg ==   R9_num || reg ==  R9_H_num ||
 1781     reg ==  R12_num || reg == R12_H_num ||
 1782     reg == XMM0_num || reg == XMM0b_num ||
 1783     reg == XMM1_num || reg == XMM1b_num ||
 1784     reg == XMM2_num || reg == XMM2b_num ||
 1785     reg == XMM3_num || reg == XMM3b_num ||
 1786     reg == XMM4_num || reg == XMM4b_num ||
 1787     reg == XMM5_num || reg == XMM5b_num ||
 1788     reg == XMM6_num || reg == XMM6b_num ||
 1789     reg == XMM7_num || reg == XMM7b_num;
 1790 }
 1791 
 1792 bool Matcher::is_spillable_arg(int reg)
 1793 {
 1794   return can_be_java_arg(reg);
 1795 }
 1796 
 1797 uint Matcher::int_pressure_limit()
 1798 {
 1799   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1800 }
 1801 
 1802 uint Matcher::float_pressure_limit()
 1803 {
 1804   // After experiment around with different values, the following default threshold
 1805   // works best for LCM's register pressure scheduling on x64.
 1806   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1807   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1808   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1809 }
 1810 
 1811 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1812   // In 64 bit mode a code which use multiply when
 1813   // devisor is constant is faster than hardware
 1814   // DIV instruction (it uses MulHiL).
 1815   return false;
 1816 }
 1817 
 1818 // Register for DIVI projection of divmodI
 1819 RegMask Matcher::divI_proj_mask() {
 1820   return INT_RAX_REG_mask();
 1821 }
 1822 
 1823 // Register for MODI projection of divmodI
 1824 RegMask Matcher::modI_proj_mask() {
 1825   return INT_RDX_REG_mask();
 1826 }
 1827 
 1828 // Register for DIVL projection of divmodL
 1829 RegMask Matcher::divL_proj_mask() {
 1830   return LONG_RAX_REG_mask();
 1831 }
 1832 
 1833 // Register for MODL projection of divmodL
 1834 RegMask Matcher::modL_proj_mask() {
 1835   return LONG_RDX_REG_mask();
 1836 }
 1837 
 1838 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1839 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1840     return NO_REG_mask();
 1841 }
 1842 
 1843 %}
 1844 
 1845 //----------ENCODING BLOCK-----------------------------------------------------
 1846 // This block specifies the encoding classes used by the compiler to
 1847 // output byte streams.  Encoding classes are parameterized macros
 1848 // used by Machine Instruction Nodes in order to generate the bit
 1849 // encoding of the instruction.  Operands specify their base encoding
 1850 // interface with the interface keyword.  There are currently
 1851 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1852 // COND_INTER.  REG_INTER causes an operand to generate a function
 1853 // which returns its register number when queried.  CONST_INTER causes
 1854 // an operand to generate a function which returns the value of the
 1855 // constant when queried.  MEMORY_INTER causes an operand to generate
 1856 // four functions which return the Base Register, the Index Register,
 1857 // the Scale Value, and the Offset Value of the operand when queried.
 1858 // COND_INTER causes an operand to generate six functions which return
 1859 // the encoding code (ie - encoding bits for the instruction)
 1860 // associated with each basic boolean condition for a conditional
 1861 // instruction.
 1862 //
 1863 // Instructions specify two basic values for encoding.  Again, a
 1864 // function is available to check if the constant displacement is an
 1865 // oop. They use the ins_encode keyword to specify their encoding
 1866 // classes (which must be a sequence of enc_class names, and their
 1867 // parameters, specified in the encoding block), and they use the
 1868 // opcode keyword to specify, in order, their primary, secondary, and
 1869 // tertiary opcode.  Only the opcode sections which a particular
 1870 // instruction needs for encoding need to be specified.
 1871 encode %{
 1872   // Build emit functions for each basic byte or larger field in the
 1873   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1874   // from C++ code in the enc_class source block.  Emit functions will
 1875   // live in the main source block for now.  In future, we can
 1876   // generalize this by adding a syntax that specifies the sizes of
 1877   // fields in an order, so that the adlc can build the emit functions
 1878   // automagically
 1879 
 1880   // Emit primary opcode
 1881   enc_class OpcP
 1882   %{
 1883     emit_opcode(cbuf, $primary);
 1884   %}
 1885 
 1886   // Emit secondary opcode
 1887   enc_class OpcS
 1888   %{
 1889     emit_opcode(cbuf, $secondary);
 1890   %}
 1891 
 1892   // Emit tertiary opcode
 1893   enc_class OpcT
 1894   %{
 1895     emit_opcode(cbuf, $tertiary);
 1896   %}
 1897 
 1898   // Emit opcode directly
 1899   enc_class Opcode(immI d8)
 1900   %{
 1901     emit_opcode(cbuf, $d8$$constant);
 1902   %}
 1903 
 1904   // Emit size prefix
 1905   enc_class SizePrefix
 1906   %{
 1907     emit_opcode(cbuf, 0x66);
 1908   %}
 1909 
 1910   enc_class reg(rRegI reg)
 1911   %{
 1912     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1913   %}
 1914 
 1915   enc_class reg_reg(rRegI dst, rRegI src)
 1916   %{
 1917     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1918   %}
 1919 
 1920   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1921   %{
 1922     emit_opcode(cbuf, $opcode$$constant);
 1923     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1924   %}
 1925 
 1926   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 1927   enc_class OpcSE(immI imm)
 1928   %{
 1929     // Emit primary opcode and set sign-extend bit
 1930     // Check for 8-bit immediate, and set sign extend bit in opcode
 1931     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 1932       emit_opcode(cbuf, $primary | 0x02);
 1933     } else {
 1934       // 32-bit immediate
 1935       emit_opcode(cbuf, $primary);
 1936     }
 1937   %}
 1938 
 1939   enc_class OpcSErm(rRegI dst, immI imm)
 1940   %{
 1941     // OpcSEr/m
 1942     int dstenc = $dst$$reg;
 1943     if (dstenc >= 8) {
 1944       emit_opcode(cbuf, Assembler::REX_B);
 1945       dstenc -= 8;
 1946     }
 1947     // Emit primary opcode and set sign-extend bit
 1948     // Check for 8-bit immediate, and set sign extend bit in opcode
 1949     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 1950       emit_opcode(cbuf, $primary | 0x02);
 1951     } else {
 1952       // 32-bit immediate
 1953       emit_opcode(cbuf, $primary);
 1954     }
 1955     // Emit r/m byte with secondary opcode, after primary opcode.
 1956     emit_rm(cbuf, 0x3, $secondary, dstenc);
 1957   %}
 1958 
 1959   enc_class OpcSErm_wide(rRegL dst, immI imm)
 1960   %{
 1961     // OpcSEr/m
 1962     int dstenc = $dst$$reg;
 1963     if (dstenc < 8) {
 1964       emit_opcode(cbuf, Assembler::REX_W);
 1965     } else {
 1966       emit_opcode(cbuf, Assembler::REX_WB);
 1967       dstenc -= 8;
 1968     }
 1969     // Emit primary opcode and set sign-extend bit
 1970     // Check for 8-bit immediate, and set sign extend bit in opcode
 1971     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 1972       emit_opcode(cbuf, $primary | 0x02);
 1973     } else {
 1974       // 32-bit immediate
 1975       emit_opcode(cbuf, $primary);
 1976     }
 1977     // Emit r/m byte with secondary opcode, after primary opcode.
 1978     emit_rm(cbuf, 0x3, $secondary, dstenc);
 1979   %}
 1980 
 1981   enc_class Con8or32(immI imm)
 1982   %{
 1983     // Check for 8-bit immediate, and set sign extend bit in opcode
 1984     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 1985       $$$emit8$imm$$constant;
 1986     } else {
 1987       // 32-bit immediate
 1988       $$$emit32$imm$$constant;
 1989     }
 1990   %}
 1991 
 1992   enc_class opc2_reg(rRegI dst)
 1993   %{
 1994     // BSWAP
 1995     emit_cc(cbuf, $secondary, $dst$$reg);
 1996   %}
 1997 
 1998   enc_class opc3_reg(rRegI dst)
 1999   %{
 2000     // BSWAP
 2001     emit_cc(cbuf, $tertiary, $dst$$reg);
 2002   %}
 2003 
 2004   enc_class reg_opc(rRegI div)
 2005   %{
 2006     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2007     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2008   %}
 2009 
 2010   enc_class enc_cmov(cmpOp cop)
 2011   %{
 2012     // CMOV
 2013     $$$emit8$primary;
 2014     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2015   %}
 2016 
 2017   enc_class enc_PartialSubtypeCheck()
 2018   %{
 2019     Register Rrdi = as_Register(RDI_enc); // result register
 2020     Register Rrax = as_Register(RAX_enc); // super class
 2021     Register Rrcx = as_Register(RCX_enc); // killed
 2022     Register Rrsi = as_Register(RSI_enc); // sub class
 2023     Label miss;
 2024     const bool set_cond_codes = true;
 2025 
 2026     MacroAssembler _masm(&cbuf);
 2027     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2028                                      NULL, &miss,
 2029                                      /*set_cond_codes:*/ true);
 2030     if ($primary) {
 2031       __ xorptr(Rrdi, Rrdi);
 2032     }
 2033     __ bind(miss);
 2034   %}
 2035 
 2036   enc_class clear_avx %{
 2037     debug_only(int off0 = cbuf.insts_size());
 2038     if (generate_vzeroupper(Compile::current())) {
 2039       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2040       // Clear upper bits of YMM registers when current compiled code uses
 2041       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2042       MacroAssembler _masm(&cbuf);
 2043       __ vzeroupper();
 2044     }
 2045     debug_only(int off1 = cbuf.insts_size());
 2046     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2047   %}
 2048 
 2049   enc_class Java_To_Runtime(method meth) %{
 2050     // No relocation needed
 2051     MacroAssembler _masm(&cbuf);
 2052     __ mov64(r10, (int64_t) $meth$$method);
 2053     __ call(r10);
 2054   %}
 2055 
 2056   enc_class Java_To_Interpreter(method meth)
 2057   %{
 2058     // CALL Java_To_Interpreter
 2059     // This is the instruction starting address for relocation info.
 2060     cbuf.set_insts_mark();
 2061     $$$emit8$primary;
 2062     // CALL directly to the runtime
 2063     emit_d32_reloc(cbuf,
 2064                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2065                    runtime_call_Relocation::spec(),
 2066                    RELOC_DISP32);
 2067   %}
 2068 
 2069   enc_class Java_Static_Call(method meth)
 2070   %{
 2071     // JAVA STATIC CALL
 2072     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2073     // determine who we intended to call.
 2074     cbuf.set_insts_mark();
 2075     $$$emit8$primary;
 2076 
 2077     if (!_method) {
 2078       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2079                      runtime_call_Relocation::spec(),
 2080                      RELOC_DISP32);
 2081     } else {
 2082       int method_index = resolved_method_index(cbuf);
 2083       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2084                                                   : static_call_Relocation::spec(method_index);
 2085       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2086                      rspec, RELOC_DISP32);
 2087       // Emit stubs for static call.
 2088       address mark = cbuf.insts_mark();
 2089       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2090       if (stub == NULL) {
 2091         ciEnv::current()->record_failure("CodeCache is full");
 2092         return;
 2093       }
 2094     }
 2095   %}
 2096 
 2097   enc_class Java_Dynamic_Call(method meth) %{
 2098     MacroAssembler _masm(&cbuf);
 2099     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2100   %}
 2101 
 2102   enc_class Java_Compiled_Call(method meth)
 2103   %{
 2104     // JAVA COMPILED CALL
 2105     int disp = in_bytes(Method:: from_compiled_offset());
 2106 
 2107     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
 2108     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
 2109 
 2110     // callq *disp(%rax)
 2111     cbuf.set_insts_mark();
 2112     $$$emit8$primary;
 2113     if (disp < 0x80) {
 2114       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
 2115       emit_d8(cbuf, disp); // Displacement
 2116     } else {
 2117       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
 2118       emit_d32(cbuf, disp); // Displacement
 2119     }
 2120   %}
 2121 
 2122   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2123   %{
 2124     // SAL, SAR, SHR
 2125     int dstenc = $dst$$reg;
 2126     if (dstenc >= 8) {
 2127       emit_opcode(cbuf, Assembler::REX_B);
 2128       dstenc -= 8;
 2129     }
 2130     $$$emit8$primary;
 2131     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2132     $$$emit8$shift$$constant;
 2133   %}
 2134 
 2135   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2136   %{
 2137     // SAL, SAR, SHR
 2138     int dstenc = $dst$$reg;
 2139     if (dstenc < 8) {
 2140       emit_opcode(cbuf, Assembler::REX_W);
 2141     } else {
 2142       emit_opcode(cbuf, Assembler::REX_WB);
 2143       dstenc -= 8;
 2144     }
 2145     $$$emit8$primary;
 2146     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2147     $$$emit8$shift$$constant;
 2148   %}
 2149 
 2150   enc_class load_immI(rRegI dst, immI src)
 2151   %{
 2152     int dstenc = $dst$$reg;
 2153     if (dstenc >= 8) {
 2154       emit_opcode(cbuf, Assembler::REX_B);
 2155       dstenc -= 8;
 2156     }
 2157     emit_opcode(cbuf, 0xB8 | dstenc);
 2158     $$$emit32$src$$constant;
 2159   %}
 2160 
 2161   enc_class load_immL(rRegL dst, immL src)
 2162   %{
 2163     int dstenc = $dst$$reg;
 2164     if (dstenc < 8) {
 2165       emit_opcode(cbuf, Assembler::REX_W);
 2166     } else {
 2167       emit_opcode(cbuf, Assembler::REX_WB);
 2168       dstenc -= 8;
 2169     }
 2170     emit_opcode(cbuf, 0xB8 | dstenc);
 2171     emit_d64(cbuf, $src$$constant);
 2172   %}
 2173 
 2174   enc_class load_immUL32(rRegL dst, immUL32 src)
 2175   %{
 2176     // same as load_immI, but this time we care about zeroes in the high word
 2177     int dstenc = $dst$$reg;
 2178     if (dstenc >= 8) {
 2179       emit_opcode(cbuf, Assembler::REX_B);
 2180       dstenc -= 8;
 2181     }
 2182     emit_opcode(cbuf, 0xB8 | dstenc);
 2183     $$$emit32$src$$constant;
 2184   %}
 2185 
 2186   enc_class load_immL32(rRegL dst, immL32 src)
 2187   %{
 2188     int dstenc = $dst$$reg;
 2189     if (dstenc < 8) {
 2190       emit_opcode(cbuf, Assembler::REX_W);
 2191     } else {
 2192       emit_opcode(cbuf, Assembler::REX_WB);
 2193       dstenc -= 8;
 2194     }
 2195     emit_opcode(cbuf, 0xC7);
 2196     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2197     $$$emit32$src$$constant;
 2198   %}
 2199 
 2200   enc_class load_immP31(rRegP dst, immP32 src)
 2201   %{
 2202     // same as load_immI, but this time we care about zeroes in the high word
 2203     int dstenc = $dst$$reg;
 2204     if (dstenc >= 8) {
 2205       emit_opcode(cbuf, Assembler::REX_B);
 2206       dstenc -= 8;
 2207     }
 2208     emit_opcode(cbuf, 0xB8 | dstenc);
 2209     $$$emit32$src$$constant;
 2210   %}
 2211 
 2212   enc_class load_immP(rRegP dst, immP src)
 2213   %{
 2214     int dstenc = $dst$$reg;
 2215     if (dstenc < 8) {
 2216       emit_opcode(cbuf, Assembler::REX_W);
 2217     } else {
 2218       emit_opcode(cbuf, Assembler::REX_WB);
 2219       dstenc -= 8;
 2220     }
 2221     emit_opcode(cbuf, 0xB8 | dstenc);
 2222     // This next line should be generated from ADLC
 2223     if ($src->constant_reloc() != relocInfo::none) {
 2224       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2225     } else {
 2226       emit_d64(cbuf, $src$$constant);
 2227     }
 2228   %}
 2229 
 2230   enc_class Con32(immI src)
 2231   %{
 2232     // Output immediate
 2233     $$$emit32$src$$constant;
 2234   %}
 2235 
 2236   enc_class Con32F_as_bits(immF src)
 2237   %{
 2238     // Output Float immediate bits
 2239     jfloat jf = $src$$constant;
 2240     jint jf_as_bits = jint_cast(jf);
 2241     emit_d32(cbuf, jf_as_bits);
 2242   %}
 2243 
 2244   enc_class Con16(immI src)
 2245   %{
 2246     // Output immediate
 2247     $$$emit16$src$$constant;
 2248   %}
 2249 
 2250   // How is this different from Con32??? XXX
 2251   enc_class Con_d32(immI src)
 2252   %{
 2253     emit_d32(cbuf,$src$$constant);
 2254   %}
 2255 
 2256   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2257     // Output immediate memory reference
 2258     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2259     emit_d32(cbuf, 0x00);
 2260   %}
 2261 
 2262   enc_class lock_prefix()
 2263   %{
 2264     emit_opcode(cbuf, 0xF0); // lock
 2265   %}
 2266 
 2267   enc_class REX_mem(memory mem)
 2268   %{
 2269     if ($mem$$base >= 8) {
 2270       if ($mem$$index < 8) {
 2271         emit_opcode(cbuf, Assembler::REX_B);
 2272       } else {
 2273         emit_opcode(cbuf, Assembler::REX_XB);
 2274       }
 2275     } else {
 2276       if ($mem$$index >= 8) {
 2277         emit_opcode(cbuf, Assembler::REX_X);
 2278       }
 2279     }
 2280   %}
 2281 
 2282   enc_class REX_mem_wide(memory mem)
 2283   %{
 2284     if ($mem$$base >= 8) {
 2285       if ($mem$$index < 8) {
 2286         emit_opcode(cbuf, Assembler::REX_WB);
 2287       } else {
 2288         emit_opcode(cbuf, Assembler::REX_WXB);
 2289       }
 2290     } else {
 2291       if ($mem$$index < 8) {
 2292         emit_opcode(cbuf, Assembler::REX_W);
 2293       } else {
 2294         emit_opcode(cbuf, Assembler::REX_WX);
 2295       }
 2296     }
 2297   %}
 2298 
 2299   // for byte regs
 2300   enc_class REX_breg(rRegI reg)
 2301   %{
 2302     if ($reg$$reg >= 4) {
 2303       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2304     }
 2305   %}
 2306 
 2307   // for byte regs
 2308   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2309   %{
 2310     if ($dst$$reg < 8) {
 2311       if ($src$$reg >= 4) {
 2312         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2313       }
 2314     } else {
 2315       if ($src$$reg < 8) {
 2316         emit_opcode(cbuf, Assembler::REX_R);
 2317       } else {
 2318         emit_opcode(cbuf, Assembler::REX_RB);
 2319       }
 2320     }
 2321   %}
 2322 
 2323   // for byte regs
 2324   enc_class REX_breg_mem(rRegI reg, memory mem)
 2325   %{
 2326     if ($reg$$reg < 8) {
 2327       if ($mem$$base < 8) {
 2328         if ($mem$$index >= 8) {
 2329           emit_opcode(cbuf, Assembler::REX_X);
 2330         } else if ($reg$$reg >= 4) {
 2331           emit_opcode(cbuf, Assembler::REX);
 2332         }
 2333       } else {
 2334         if ($mem$$index < 8) {
 2335           emit_opcode(cbuf, Assembler::REX_B);
 2336         } else {
 2337           emit_opcode(cbuf, Assembler::REX_XB);
 2338         }
 2339       }
 2340     } else {
 2341       if ($mem$$base < 8) {
 2342         if ($mem$$index < 8) {
 2343           emit_opcode(cbuf, Assembler::REX_R);
 2344         } else {
 2345           emit_opcode(cbuf, Assembler::REX_RX);
 2346         }
 2347       } else {
 2348         if ($mem$$index < 8) {
 2349           emit_opcode(cbuf, Assembler::REX_RB);
 2350         } else {
 2351           emit_opcode(cbuf, Assembler::REX_RXB);
 2352         }
 2353       }
 2354     }
 2355   %}
 2356 
 2357   enc_class REX_reg(rRegI reg)
 2358   %{
 2359     if ($reg$$reg >= 8) {
 2360       emit_opcode(cbuf, Assembler::REX_B);
 2361     }
 2362   %}
 2363 
 2364   enc_class REX_reg_wide(rRegI reg)
 2365   %{
 2366     if ($reg$$reg < 8) {
 2367       emit_opcode(cbuf, Assembler::REX_W);
 2368     } else {
 2369       emit_opcode(cbuf, Assembler::REX_WB);
 2370     }
 2371   %}
 2372 
 2373   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2374   %{
 2375     if ($dst$$reg < 8) {
 2376       if ($src$$reg >= 8) {
 2377         emit_opcode(cbuf, Assembler::REX_B);
 2378       }
 2379     } else {
 2380       if ($src$$reg < 8) {
 2381         emit_opcode(cbuf, Assembler::REX_R);
 2382       } else {
 2383         emit_opcode(cbuf, Assembler::REX_RB);
 2384       }
 2385     }
 2386   %}
 2387 
 2388   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2389   %{
 2390     if ($dst$$reg < 8) {
 2391       if ($src$$reg < 8) {
 2392         emit_opcode(cbuf, Assembler::REX_W);
 2393       } else {
 2394         emit_opcode(cbuf, Assembler::REX_WB);
 2395       }
 2396     } else {
 2397       if ($src$$reg < 8) {
 2398         emit_opcode(cbuf, Assembler::REX_WR);
 2399       } else {
 2400         emit_opcode(cbuf, Assembler::REX_WRB);
 2401       }
 2402     }
 2403   %}
 2404 
 2405   enc_class REX_reg_mem(rRegI reg, memory mem)
 2406   %{
 2407     if ($reg$$reg < 8) {
 2408       if ($mem$$base < 8) {
 2409         if ($mem$$index >= 8) {
 2410           emit_opcode(cbuf, Assembler::REX_X);
 2411         }
 2412       } else {
 2413         if ($mem$$index < 8) {
 2414           emit_opcode(cbuf, Assembler::REX_B);
 2415         } else {
 2416           emit_opcode(cbuf, Assembler::REX_XB);
 2417         }
 2418       }
 2419     } else {
 2420       if ($mem$$base < 8) {
 2421         if ($mem$$index < 8) {
 2422           emit_opcode(cbuf, Assembler::REX_R);
 2423         } else {
 2424           emit_opcode(cbuf, Assembler::REX_RX);
 2425         }
 2426       } else {
 2427         if ($mem$$index < 8) {
 2428           emit_opcode(cbuf, Assembler::REX_RB);
 2429         } else {
 2430           emit_opcode(cbuf, Assembler::REX_RXB);
 2431         }
 2432       }
 2433     }
 2434   %}
 2435 
 2436   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2437   %{
 2438     if ($reg$$reg < 8) {
 2439       if ($mem$$base < 8) {
 2440         if ($mem$$index < 8) {
 2441           emit_opcode(cbuf, Assembler::REX_W);
 2442         } else {
 2443           emit_opcode(cbuf, Assembler::REX_WX);
 2444         }
 2445       } else {
 2446         if ($mem$$index < 8) {
 2447           emit_opcode(cbuf, Assembler::REX_WB);
 2448         } else {
 2449           emit_opcode(cbuf, Assembler::REX_WXB);
 2450         }
 2451       }
 2452     } else {
 2453       if ($mem$$base < 8) {
 2454         if ($mem$$index < 8) {
 2455           emit_opcode(cbuf, Assembler::REX_WR);
 2456         } else {
 2457           emit_opcode(cbuf, Assembler::REX_WRX);
 2458         }
 2459       } else {
 2460         if ($mem$$index < 8) {
 2461           emit_opcode(cbuf, Assembler::REX_WRB);
 2462         } else {
 2463           emit_opcode(cbuf, Assembler::REX_WRXB);
 2464         }
 2465       }
 2466     }
 2467   %}
 2468 
 2469   enc_class reg_mem(rRegI ereg, memory mem)
 2470   %{
 2471     // High registers handle in encode_RegMem
 2472     int reg = $ereg$$reg;
 2473     int base = $mem$$base;
 2474     int index = $mem$$index;
 2475     int scale = $mem$$scale;
 2476     int disp = $mem$$disp;
 2477     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2478 
 2479     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2480   %}
 2481 
 2482   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2483   %{
 2484     int rm_byte_opcode = $rm_opcode$$constant;
 2485 
 2486     // High registers handle in encode_RegMem
 2487     int base = $mem$$base;
 2488     int index = $mem$$index;
 2489     int scale = $mem$$scale;
 2490     int displace = $mem$$disp;
 2491 
 2492     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2493                                             // working with static
 2494                                             // globals
 2495     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2496                   disp_reloc);
 2497   %}
 2498 
 2499   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2500   %{
 2501     int reg_encoding = $dst$$reg;
 2502     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2503     int index        = 0x04;            // 0x04 indicates no index
 2504     int scale        = 0x00;            // 0x00 indicates no scale
 2505     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2506     relocInfo::relocType disp_reloc = relocInfo::none;
 2507     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2508                   disp_reloc);
 2509   %}
 2510 
 2511   enc_class neg_reg(rRegI dst)
 2512   %{
 2513     int dstenc = $dst$$reg;
 2514     if (dstenc >= 8) {
 2515       emit_opcode(cbuf, Assembler::REX_B);
 2516       dstenc -= 8;
 2517     }
 2518     // NEG $dst
 2519     emit_opcode(cbuf, 0xF7);
 2520     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2521   %}
 2522 
 2523   enc_class neg_reg_wide(rRegI dst)
 2524   %{
 2525     int dstenc = $dst$$reg;
 2526     if (dstenc < 8) {
 2527       emit_opcode(cbuf, Assembler::REX_W);
 2528     } else {
 2529       emit_opcode(cbuf, Assembler::REX_WB);
 2530       dstenc -= 8;
 2531     }
 2532     // NEG $dst
 2533     emit_opcode(cbuf, 0xF7);
 2534     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2535   %}
 2536 
 2537   enc_class setLT_reg(rRegI dst)
 2538   %{
 2539     int dstenc = $dst$$reg;
 2540     if (dstenc >= 8) {
 2541       emit_opcode(cbuf, Assembler::REX_B);
 2542       dstenc -= 8;
 2543     } else if (dstenc >= 4) {
 2544       emit_opcode(cbuf, Assembler::REX);
 2545     }
 2546     // SETLT $dst
 2547     emit_opcode(cbuf, 0x0F);
 2548     emit_opcode(cbuf, 0x9C);
 2549     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2550   %}
 2551 
 2552   enc_class setNZ_reg(rRegI dst)
 2553   %{
 2554     int dstenc = $dst$$reg;
 2555     if (dstenc >= 8) {
 2556       emit_opcode(cbuf, Assembler::REX_B);
 2557       dstenc -= 8;
 2558     } else if (dstenc >= 4) {
 2559       emit_opcode(cbuf, Assembler::REX);
 2560     }
 2561     // SETNZ $dst
 2562     emit_opcode(cbuf, 0x0F);
 2563     emit_opcode(cbuf, 0x95);
 2564     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2565   %}
 2566 
 2567 
 2568   // Compare the lonogs and set -1, 0, or 1 into dst
 2569   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2570   %{
 2571     int src1enc = $src1$$reg;
 2572     int src2enc = $src2$$reg;
 2573     int dstenc = $dst$$reg;
 2574 
 2575     // cmpq $src1, $src2
 2576     if (src1enc < 8) {
 2577       if (src2enc < 8) {
 2578         emit_opcode(cbuf, Assembler::REX_W);
 2579       } else {
 2580         emit_opcode(cbuf, Assembler::REX_WB);
 2581       }
 2582     } else {
 2583       if (src2enc < 8) {
 2584         emit_opcode(cbuf, Assembler::REX_WR);
 2585       } else {
 2586         emit_opcode(cbuf, Assembler::REX_WRB);
 2587       }
 2588     }
 2589     emit_opcode(cbuf, 0x3B);
 2590     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2591 
 2592     // movl $dst, -1
 2593     if (dstenc >= 8) {
 2594       emit_opcode(cbuf, Assembler::REX_B);
 2595     }
 2596     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2597     emit_d32(cbuf, -1);
 2598 
 2599     // jl,s done
 2600     emit_opcode(cbuf, 0x7C);
 2601     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2602 
 2603     // setne $dst
 2604     if (dstenc >= 4) {
 2605       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2606     }
 2607     emit_opcode(cbuf, 0x0F);
 2608     emit_opcode(cbuf, 0x95);
 2609     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2610 
 2611     // movzbl $dst, $dst
 2612     if (dstenc >= 4) {
 2613       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2614     }
 2615     emit_opcode(cbuf, 0x0F);
 2616     emit_opcode(cbuf, 0xB6);
 2617     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2618   %}
 2619 
 2620   enc_class Push_ResultXD(regD dst) %{
 2621     MacroAssembler _masm(&cbuf);
 2622     __ fstp_d(Address(rsp, 0));
 2623     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2624     __ addptr(rsp, 8);
 2625   %}
 2626 
 2627   enc_class Push_SrcXD(regD src) %{
 2628     MacroAssembler _masm(&cbuf);
 2629     __ subptr(rsp, 8);
 2630     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2631     __ fld_d(Address(rsp, 0));
 2632   %}
 2633 
 2634 
 2635   enc_class enc_rethrow()
 2636   %{
 2637     cbuf.set_insts_mark();
 2638     emit_opcode(cbuf, 0xE9); // jmp entry
 2639     emit_d32_reloc(cbuf,
 2640                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2641                    runtime_call_Relocation::spec(),
 2642                    RELOC_DISP32);
 2643   %}
 2644 
 2645 %}
 2646 
 2647 
 2648 
 2649 //----------FRAME--------------------------------------------------------------
 2650 // Definition of frame structure and management information.
 2651 //
 2652 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2653 //                             |   (to get allocators register number
 2654 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2655 //  r   CALLER     |        |
 2656 //  o     |        +--------+      pad to even-align allocators stack-slot
 2657 //  w     V        |  pad0  |        numbers; owned by CALLER
 2658 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2659 //  h     ^        |   in   |  5
 2660 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2661 //  |     |        |        |  3
 2662 //  |     |        +--------+
 2663 //  V     |        | old out|      Empty on Intel, window on Sparc
 2664 //        |    old |preserve|      Must be even aligned.
 2665 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2666 //        |        |   in   |  3   area for Intel ret address
 2667 //     Owned by    |preserve|      Empty on Sparc.
 2668 //       SELF      +--------+
 2669 //        |        |  pad2  |  2   pad to align old SP
 2670 //        |        +--------+  1
 2671 //        |        | locks  |  0
 2672 //        |        +--------+----> OptoReg::stack0(), even aligned
 2673 //        |        |  pad1  | 11   pad to align new SP
 2674 //        |        +--------+
 2675 //        |        |        | 10
 2676 //        |        | spills |  9   spills
 2677 //        V        |        |  8   (pad0 slot for callee)
 2678 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2679 //        ^        |  out   |  7
 2680 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2681 //     Owned by    +--------+
 2682 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2683 //        |    new |preserve|      Must be even-aligned.
 2684 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2685 //        |        |        |
 2686 //
 2687 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2688 //         known from SELF's arguments and the Java calling convention.
 2689 //         Region 6-7 is determined per call site.
 2690 // Note 2: If the calling convention leaves holes in the incoming argument
 2691 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2692 //         are owned by the CALLEE.  Holes should not be necessary in the
 2693 //         incoming area, as the Java calling convention is completely under
 2694 //         the control of the AD file.  Doubles can be sorted and packed to
 2695 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2696 //         varargs C calling conventions.
 2697 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2698 //         even aligned with pad0 as needed.
 2699 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2700 //         region 6-11 is even aligned; it may be padded out more so that
 2701 //         the region from SP to FP meets the minimum stack alignment.
 2702 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2703 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2704 //         SP meets the minimum alignment.
 2705 
 2706 frame
 2707 %{
 2708   // These three registers define part of the calling convention
 2709   // between compiled code and the interpreter.
 2710   inline_cache_reg(RAX);                // Inline Cache Register
 2711 
 2712   // Optional: name the operand used by cisc-spilling to access
 2713   // [stack_pointer + offset]
 2714   cisc_spilling_operand_name(indOffset32);
 2715 
 2716   // Number of stack slots consumed by locking an object
 2717   sync_stack_slots(2);
 2718 
 2719   // Compiled code's Frame Pointer
 2720   frame_pointer(RSP);
 2721 
 2722   // Interpreter stores its frame pointer in a register which is
 2723   // stored to the stack by I2CAdaptors.
 2724   // I2CAdaptors convert from interpreted java to compiled java.
 2725   interpreter_frame_pointer(RBP);
 2726 
 2727   // Stack alignment requirement
 2728   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2729 
 2730   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2731   // for calls to C.  Supports the var-args backing area for register parms.
 2732   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2733 
 2734   // The after-PROLOG location of the return address.  Location of
 2735   // return address specifies a type (REG or STACK) and a number
 2736   // representing the register number (i.e. - use a register name) or
 2737   // stack slot.
 2738   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2739   // Otherwise, it is above the locks and verification slot and alignment word
 2740   return_addr(STACK - 2 +
 2741               align_up((Compile::current()->in_preserve_stack_slots() +
 2742                         Compile::current()->fixed_slots()),
 2743                        stack_alignment_in_slots()));
 2744 
 2745   // Location of compiled Java return values.  Same as C for now.
 2746   return_value
 2747   %{
 2748     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2749            "only return normal values");
 2750 
 2751     static const int lo[Op_RegL + 1] = {
 2752       0,
 2753       0,
 2754       RAX_num,  // Op_RegN
 2755       RAX_num,  // Op_RegI
 2756       RAX_num,  // Op_RegP
 2757       XMM0_num, // Op_RegF
 2758       XMM0_num, // Op_RegD
 2759       RAX_num   // Op_RegL
 2760     };
 2761     static const int hi[Op_RegL + 1] = {
 2762       0,
 2763       0,
 2764       OptoReg::Bad, // Op_RegN
 2765       OptoReg::Bad, // Op_RegI
 2766       RAX_H_num,    // Op_RegP
 2767       OptoReg::Bad, // Op_RegF
 2768       XMM0b_num,    // Op_RegD
 2769       RAX_H_num     // Op_RegL
 2770     };
 2771     // Excluded flags and vector registers.
 2772     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2773     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2774   %}
 2775 %}
 2776 
 2777 //----------ATTRIBUTES---------------------------------------------------------
 2778 //----------Operand Attributes-------------------------------------------------
 2779 op_attrib op_cost(0);        // Required cost attribute
 2780 
 2781 //----------Instruction Attributes---------------------------------------------
 2782 ins_attrib ins_cost(100);       // Required cost attribute
 2783 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2784 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2785                                 // a non-matching short branch variant
 2786                                 // of some long branch?
 2787 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2788                                 // be a power of 2) specifies the
 2789                                 // alignment that some part of the
 2790                                 // instruction (not necessarily the
 2791                                 // start) requires.  If > 1, a
 2792                                 // compute_padding() function must be
 2793                                 // provided for the instruction
 2794 
 2795 //----------OPERANDS-----------------------------------------------------------
 2796 // Operand definitions must precede instruction definitions for correct parsing
 2797 // in the ADLC because operands constitute user defined types which are used in
 2798 // instruction definitions.
 2799 
 2800 //----------Simple Operands----------------------------------------------------
 2801 // Immediate Operands
 2802 // Integer Immediate
 2803 operand immI()
 2804 %{
 2805   match(ConI);
 2806 
 2807   op_cost(10);
 2808   format %{ %}
 2809   interface(CONST_INTER);
 2810 %}
 2811 
 2812 // Constant for test vs zero
 2813 operand immI_0()
 2814 %{
 2815   predicate(n->get_int() == 0);
 2816   match(ConI);
 2817 
 2818   op_cost(0);
 2819   format %{ %}
 2820   interface(CONST_INTER);
 2821 %}
 2822 
 2823 // Constant for increment
 2824 operand immI_1()
 2825 %{
 2826   predicate(n->get_int() == 1);
 2827   match(ConI);
 2828 
 2829   op_cost(0);
 2830   format %{ %}
 2831   interface(CONST_INTER);
 2832 %}
 2833 
 2834 // Constant for decrement
 2835 operand immI_M1()
 2836 %{
 2837   predicate(n->get_int() == -1);
 2838   match(ConI);
 2839 
 2840   op_cost(0);
 2841   format %{ %}
 2842   interface(CONST_INTER);
 2843 %}
 2844 
 2845 operand immI_2()
 2846 %{
 2847   predicate(n->get_int() == 2);
 2848   match(ConI);
 2849 
 2850   op_cost(0);
 2851   format %{ %}
 2852   interface(CONST_INTER);
 2853 %}
 2854 
 2855 operand immI_4()
 2856 %{
 2857   predicate(n->get_int() == 4);
 2858   match(ConI);
 2859 
 2860   op_cost(0);
 2861   format %{ %}
 2862   interface(CONST_INTER);
 2863 %}
 2864 
 2865 operand immI_8()
 2866 %{
 2867   predicate(n->get_int() == 8);
 2868   match(ConI);
 2869 
 2870   op_cost(0);
 2871   format %{ %}
 2872   interface(CONST_INTER);
 2873 %}
 2874 
 2875 // Valid scale values for addressing modes
 2876 operand immI2()
 2877 %{
 2878   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2879   match(ConI);
 2880 
 2881   format %{ %}
 2882   interface(CONST_INTER);
 2883 %}
 2884 
 2885 operand immU7()
 2886 %{
 2887   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2888   match(ConI);
 2889 
 2890   op_cost(5);
 2891   format %{ %}
 2892   interface(CONST_INTER);
 2893 %}
 2894 
 2895 operand immI8()
 2896 %{
 2897   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2898   match(ConI);
 2899 
 2900   op_cost(5);
 2901   format %{ %}
 2902   interface(CONST_INTER);
 2903 %}
 2904 
 2905 operand immU8()
 2906 %{
 2907   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2908   match(ConI);
 2909 
 2910   op_cost(5);
 2911   format %{ %}
 2912   interface(CONST_INTER);
 2913 %}
 2914 
 2915 operand immI16()
 2916 %{
 2917   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2918   match(ConI);
 2919 
 2920   op_cost(10);
 2921   format %{ %}
 2922   interface(CONST_INTER);
 2923 %}
 2924 
 2925 // Int Immediate non-negative
 2926 operand immU31()
 2927 %{
 2928   predicate(n->get_int() >= 0);
 2929   match(ConI);
 2930 
 2931   op_cost(0);
 2932   format %{ %}
 2933   interface(CONST_INTER);
 2934 %}
 2935 
 2936 // Constant for long shifts
 2937 operand immI_32()
 2938 %{
 2939   predicate( n->get_int() == 32 );
 2940   match(ConI);
 2941 
 2942   op_cost(0);
 2943   format %{ %}
 2944   interface(CONST_INTER);
 2945 %}
 2946 
 2947 // Constant for long shifts
 2948 operand immI_64()
 2949 %{
 2950   predicate( n->get_int() == 64 );
 2951   match(ConI);
 2952 
 2953   op_cost(0);
 2954   format %{ %}
 2955   interface(CONST_INTER);
 2956 %}
 2957 
 2958 // Pointer Immediate
 2959 operand immP()
 2960 %{
 2961   match(ConP);
 2962 
 2963   op_cost(10);
 2964   format %{ %}
 2965   interface(CONST_INTER);
 2966 %}
 2967 
 2968 // NULL Pointer Immediate
 2969 operand immP0()
 2970 %{
 2971   predicate(n->get_ptr() == 0);
 2972   match(ConP);
 2973 
 2974   op_cost(5);
 2975   format %{ %}
 2976   interface(CONST_INTER);
 2977 %}
 2978 
 2979 // Pointer Immediate
 2980 operand immN() %{
 2981   match(ConN);
 2982 
 2983   op_cost(10);
 2984   format %{ %}
 2985   interface(CONST_INTER);
 2986 %}
 2987 
 2988 operand immNKlass() %{
 2989   match(ConNKlass);
 2990 
 2991   op_cost(10);
 2992   format %{ %}
 2993   interface(CONST_INTER);
 2994 %}
 2995 
 2996 // NULL Pointer Immediate
 2997 operand immN0() %{
 2998   predicate(n->get_narrowcon() == 0);
 2999   match(ConN);
 3000 
 3001   op_cost(5);
 3002   format %{ %}
 3003   interface(CONST_INTER);
 3004 %}
 3005 
 3006 operand immP31()
 3007 %{
 3008   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3009             && (n->get_ptr() >> 31) == 0);
 3010   match(ConP);
 3011 
 3012   op_cost(5);
 3013   format %{ %}
 3014   interface(CONST_INTER);
 3015 %}
 3016 
 3017 
 3018 // Long Immediate
 3019 operand immL()
 3020 %{
 3021   match(ConL);
 3022 
 3023   op_cost(20);
 3024   format %{ %}
 3025   interface(CONST_INTER);
 3026 %}
 3027 
 3028 // Long Immediate 8-bit
 3029 operand immL8()
 3030 %{
 3031   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3032   match(ConL);
 3033 
 3034   op_cost(5);
 3035   format %{ %}
 3036   interface(CONST_INTER);
 3037 %}
 3038 
 3039 // Long Immediate 32-bit unsigned
 3040 operand immUL32()
 3041 %{
 3042   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3043   match(ConL);
 3044 
 3045   op_cost(10);
 3046   format %{ %}
 3047   interface(CONST_INTER);
 3048 %}
 3049 
 3050 // Long Immediate 32-bit signed
 3051 operand immL32()
 3052 %{
 3053   predicate(n->get_long() == (int) (n->get_long()));
 3054   match(ConL);
 3055 
 3056   op_cost(15);
 3057   format %{ %}
 3058   interface(CONST_INTER);
 3059 %}
 3060 
 3061 operand immL_Pow2()
 3062 %{
 3063   predicate(is_power_of_2((julong)n->get_long()));
 3064   match(ConL);
 3065 
 3066   op_cost(15);
 3067   format %{ %}
 3068   interface(CONST_INTER);
 3069 %}
 3070 
 3071 operand immL_NotPow2()
 3072 %{
 3073   predicate(is_power_of_2((julong)~n->get_long()));
 3074   match(ConL);
 3075 
 3076   op_cost(15);
 3077   format %{ %}
 3078   interface(CONST_INTER);
 3079 %}
 3080 
 3081 // Long Immediate zero
 3082 operand immL0()
 3083 %{
 3084   predicate(n->get_long() == 0L);
 3085   match(ConL);
 3086 
 3087   op_cost(10);
 3088   format %{ %}
 3089   interface(CONST_INTER);
 3090 %}
 3091 
 3092 // Constant for increment
 3093 operand immL1()
 3094 %{
 3095   predicate(n->get_long() == 1);
 3096   match(ConL);
 3097 
 3098   format %{ %}
 3099   interface(CONST_INTER);
 3100 %}
 3101 
 3102 // Constant for decrement
 3103 operand immL_M1()
 3104 %{
 3105   predicate(n->get_long() == -1);
 3106   match(ConL);
 3107 
 3108   format %{ %}
 3109   interface(CONST_INTER);
 3110 %}
 3111 
 3112 // Long Immediate: the value 10
 3113 operand immL10()
 3114 %{
 3115   predicate(n->get_long() == 10);
 3116   match(ConL);
 3117 
 3118   format %{ %}
 3119   interface(CONST_INTER);
 3120 %}
 3121 
 3122 // Long immediate from 0 to 127.
 3123 // Used for a shorter form of long mul by 10.
 3124 operand immL_127()
 3125 %{
 3126   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3127   match(ConL);
 3128 
 3129   op_cost(10);
 3130   format %{ %}
 3131   interface(CONST_INTER);
 3132 %}
 3133 
 3134 // Long Immediate: low 32-bit mask
 3135 operand immL_32bits()
 3136 %{
 3137   predicate(n->get_long() == 0xFFFFFFFFL);
 3138   match(ConL);
 3139   op_cost(20);
 3140 
 3141   format %{ %}
 3142   interface(CONST_INTER);
 3143 %}
 3144 
 3145 // Int Immediate: 2^n-1, positive
 3146 operand immI_Pow2M1()
 3147 %{
 3148   predicate((n->get_int() > 0)
 3149             && is_power_of_2(n->get_int() + 1));
 3150   match(ConI);
 3151 
 3152   op_cost(20);
 3153   format %{ %}
 3154   interface(CONST_INTER);
 3155 %}
 3156 
 3157 // Float Immediate zero
 3158 operand immF0()
 3159 %{
 3160   predicate(jint_cast(n->getf()) == 0);
 3161   match(ConF);
 3162 
 3163   op_cost(5);
 3164   format %{ %}
 3165   interface(CONST_INTER);
 3166 %}
 3167 
 3168 // Float Immediate
 3169 operand immF()
 3170 %{
 3171   match(ConF);
 3172 
 3173   op_cost(15);
 3174   format %{ %}
 3175   interface(CONST_INTER);
 3176 %}
 3177 
 3178 // Double Immediate zero
 3179 operand immD0()
 3180 %{
 3181   predicate(jlong_cast(n->getd()) == 0);
 3182   match(ConD);
 3183 
 3184   op_cost(5);
 3185   format %{ %}
 3186   interface(CONST_INTER);
 3187 %}
 3188 
 3189 // Double Immediate
 3190 operand immD()
 3191 %{
 3192   match(ConD);
 3193 
 3194   op_cost(15);
 3195   format %{ %}
 3196   interface(CONST_INTER);
 3197 %}
 3198 
 3199 // Immediates for special shifts (sign extend)
 3200 
 3201 // Constants for increment
 3202 operand immI_16()
 3203 %{
 3204   predicate(n->get_int() == 16);
 3205   match(ConI);
 3206 
 3207   format %{ %}
 3208   interface(CONST_INTER);
 3209 %}
 3210 
 3211 operand immI_24()
 3212 %{
 3213   predicate(n->get_int() == 24);
 3214   match(ConI);
 3215 
 3216   format %{ %}
 3217   interface(CONST_INTER);
 3218 %}
 3219 
 3220 // Constant for byte-wide masking
 3221 operand immI_255()
 3222 %{
 3223   predicate(n->get_int() == 255);
 3224   match(ConI);
 3225 
 3226   format %{ %}
 3227   interface(CONST_INTER);
 3228 %}
 3229 
 3230 // Constant for short-wide masking
 3231 operand immI_65535()
 3232 %{
 3233   predicate(n->get_int() == 65535);
 3234   match(ConI);
 3235 
 3236   format %{ %}
 3237   interface(CONST_INTER);
 3238 %}
 3239 
 3240 // Constant for byte-wide masking
 3241 operand immL_255()
 3242 %{
 3243   predicate(n->get_long() == 255);
 3244   match(ConL);
 3245 
 3246   format %{ %}
 3247   interface(CONST_INTER);
 3248 %}
 3249 
 3250 // Constant for short-wide masking
 3251 operand immL_65535()
 3252 %{
 3253   predicate(n->get_long() == 65535);
 3254   match(ConL);
 3255 
 3256   format %{ %}
 3257   interface(CONST_INTER);
 3258 %}
 3259 
 3260 operand kReg()
 3261 %{
 3262   constraint(ALLOC_IN_RC(vectmask_reg));
 3263   match(RegVectMask);
 3264   format %{%}
 3265   interface(REG_INTER);
 3266 %}
 3267 
 3268 operand kReg_K1()
 3269 %{
 3270   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3271   match(RegVectMask);
 3272   format %{%}
 3273   interface(REG_INTER);
 3274 %}
 3275 
 3276 operand kReg_K2()
 3277 %{
 3278   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3279   match(RegVectMask);
 3280   format %{%}
 3281   interface(REG_INTER);
 3282 %}
 3283 
 3284 // Special Registers
 3285 operand kReg_K3()
 3286 %{
 3287   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3288   match(RegVectMask);
 3289   format %{%}
 3290   interface(REG_INTER);
 3291 %}
 3292 
 3293 operand kReg_K4()
 3294 %{
 3295   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3296   match(RegVectMask);
 3297   format %{%}
 3298   interface(REG_INTER);
 3299 %}
 3300 
 3301 operand kReg_K5()
 3302 %{
 3303   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3304   match(RegVectMask);
 3305   format %{%}
 3306   interface(REG_INTER);
 3307 %}
 3308 
 3309 operand kReg_K6()
 3310 %{
 3311   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3312   match(RegVectMask);
 3313   format %{%}
 3314   interface(REG_INTER);
 3315 %}
 3316 
 3317 // Special Registers
 3318 operand kReg_K7()
 3319 %{
 3320   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3321   match(RegVectMask);
 3322   format %{%}
 3323   interface(REG_INTER);
 3324 %}
 3325 
 3326 // Register Operands
 3327 // Integer Register
 3328 operand rRegI()
 3329 %{
 3330   constraint(ALLOC_IN_RC(int_reg));
 3331   match(RegI);
 3332 
 3333   match(rax_RegI);
 3334   match(rbx_RegI);
 3335   match(rcx_RegI);
 3336   match(rdx_RegI);
 3337   match(rdi_RegI);
 3338 
 3339   format %{ %}
 3340   interface(REG_INTER);
 3341 %}
 3342 
 3343 // Special Registers
 3344 operand rax_RegI()
 3345 %{
 3346   constraint(ALLOC_IN_RC(int_rax_reg));
 3347   match(RegI);
 3348   match(rRegI);
 3349 
 3350   format %{ "RAX" %}
 3351   interface(REG_INTER);
 3352 %}
 3353 
 3354 // Special Registers
 3355 operand rbx_RegI()
 3356 %{
 3357   constraint(ALLOC_IN_RC(int_rbx_reg));
 3358   match(RegI);
 3359   match(rRegI);
 3360 
 3361   format %{ "RBX" %}
 3362   interface(REG_INTER);
 3363 %}
 3364 
 3365 operand rcx_RegI()
 3366 %{
 3367   constraint(ALLOC_IN_RC(int_rcx_reg));
 3368   match(RegI);
 3369   match(rRegI);
 3370 
 3371   format %{ "RCX" %}
 3372   interface(REG_INTER);
 3373 %}
 3374 
 3375 operand rdx_RegI()
 3376 %{
 3377   constraint(ALLOC_IN_RC(int_rdx_reg));
 3378   match(RegI);
 3379   match(rRegI);
 3380 
 3381   format %{ "RDX" %}
 3382   interface(REG_INTER);
 3383 %}
 3384 
 3385 operand rdi_RegI()
 3386 %{
 3387   constraint(ALLOC_IN_RC(int_rdi_reg));
 3388   match(RegI);
 3389   match(rRegI);
 3390 
 3391   format %{ "RDI" %}
 3392   interface(REG_INTER);
 3393 %}
 3394 
 3395 operand no_rax_rdx_RegI()
 3396 %{
 3397   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3398   match(RegI);
 3399   match(rbx_RegI);
 3400   match(rcx_RegI);
 3401   match(rdi_RegI);
 3402 
 3403   format %{ %}
 3404   interface(REG_INTER);
 3405 %}
 3406 
 3407 operand no_rbp_r13_RegI()
 3408 %{
 3409   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3410   match(RegI);
 3411   match(rRegI);
 3412   match(rax_RegI);
 3413   match(rbx_RegI);
 3414   match(rcx_RegI);
 3415   match(rdx_RegI);
 3416   match(rdi_RegI);
 3417 
 3418   format %{ %}
 3419   interface(REG_INTER);
 3420 %}
 3421 
 3422 // Pointer Register
 3423 operand any_RegP()
 3424 %{
 3425   constraint(ALLOC_IN_RC(any_reg));
 3426   match(RegP);
 3427   match(rax_RegP);
 3428   match(rbx_RegP);
 3429   match(rdi_RegP);
 3430   match(rsi_RegP);
 3431   match(rbp_RegP);
 3432   match(r15_RegP);
 3433   match(rRegP);
 3434 
 3435   format %{ %}
 3436   interface(REG_INTER);
 3437 %}
 3438 
 3439 operand rRegP()
 3440 %{
 3441   constraint(ALLOC_IN_RC(ptr_reg));
 3442   match(RegP);
 3443   match(rax_RegP);
 3444   match(rbx_RegP);
 3445   match(rdi_RegP);
 3446   match(rsi_RegP);
 3447   match(rbp_RegP);  // See Q&A below about
 3448   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3449 
 3450   format %{ %}
 3451   interface(REG_INTER);
 3452 %}
 3453 
 3454 operand rRegN() %{
 3455   constraint(ALLOC_IN_RC(int_reg));
 3456   match(RegN);
 3457 
 3458   format %{ %}
 3459   interface(REG_INTER);
 3460 %}
 3461 
 3462 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3463 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3464 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3465 // The output of an instruction is controlled by the allocator, which respects
 3466 // register class masks, not match rules.  Unless an instruction mentions
 3467 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3468 // by the allocator as an input.
 3469 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3470 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3471 // result, RBP is not included in the output of the instruction either.
 3472 
 3473 operand no_rax_RegP()
 3474 %{
 3475   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3476   match(RegP);
 3477   match(rbx_RegP);
 3478   match(rsi_RegP);
 3479   match(rdi_RegP);
 3480 
 3481   format %{ %}
 3482   interface(REG_INTER);
 3483 %}
 3484 
 3485 // This operand is not allowed to use RBP even if
 3486 // RBP is not used to hold the frame pointer.
 3487 operand no_rbp_RegP()
 3488 %{
 3489   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3490   match(RegP);
 3491   match(rbx_RegP);
 3492   match(rsi_RegP);
 3493   match(rdi_RegP);
 3494 
 3495   format %{ %}
 3496   interface(REG_INTER);
 3497 %}
 3498 
 3499 operand no_rax_rbx_RegP()
 3500 %{
 3501   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3502   match(RegP);
 3503   match(rsi_RegP);
 3504   match(rdi_RegP);
 3505 
 3506   format %{ %}
 3507   interface(REG_INTER);
 3508 %}
 3509 
 3510 // Special Registers
 3511 // Return a pointer value
 3512 operand rax_RegP()
 3513 %{
 3514   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3515   match(RegP);
 3516   match(rRegP);
 3517 
 3518   format %{ %}
 3519   interface(REG_INTER);
 3520 %}
 3521 
 3522 // Special Registers
 3523 // Return a compressed pointer value
 3524 operand rax_RegN()
 3525 %{
 3526   constraint(ALLOC_IN_RC(int_rax_reg));
 3527   match(RegN);
 3528   match(rRegN);
 3529 
 3530   format %{ %}
 3531   interface(REG_INTER);
 3532 %}
 3533 
 3534 // Used in AtomicAdd
 3535 operand rbx_RegP()
 3536 %{
 3537   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3538   match(RegP);
 3539   match(rRegP);
 3540 
 3541   format %{ %}
 3542   interface(REG_INTER);
 3543 %}
 3544 
 3545 operand rsi_RegP()
 3546 %{
 3547   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3548   match(RegP);
 3549   match(rRegP);
 3550 
 3551   format %{ %}
 3552   interface(REG_INTER);
 3553 %}
 3554 
 3555 operand rbp_RegP()
 3556 %{
 3557   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3558   match(RegP);
 3559   match(rRegP);
 3560 
 3561   format %{ %}
 3562   interface(REG_INTER);
 3563 %}
 3564 
 3565 // Used in rep stosq
 3566 operand rdi_RegP()
 3567 %{
 3568   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3569   match(RegP);
 3570   match(rRegP);
 3571 
 3572   format %{ %}
 3573   interface(REG_INTER);
 3574 %}
 3575 
 3576 operand r15_RegP()
 3577 %{
 3578   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3579   match(RegP);
 3580   match(rRegP);
 3581 
 3582   format %{ %}
 3583   interface(REG_INTER);
 3584 %}
 3585 
 3586 operand rRegL()
 3587 %{
 3588   constraint(ALLOC_IN_RC(long_reg));
 3589   match(RegL);
 3590   match(rax_RegL);
 3591   match(rdx_RegL);
 3592 
 3593   format %{ %}
 3594   interface(REG_INTER);
 3595 %}
 3596 
 3597 // Special Registers
 3598 operand no_rax_rdx_RegL()
 3599 %{
 3600   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3601   match(RegL);
 3602   match(rRegL);
 3603 
 3604   format %{ %}
 3605   interface(REG_INTER);
 3606 %}
 3607 
 3608 operand no_rax_RegL()
 3609 %{
 3610   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3611   match(RegL);
 3612   match(rRegL);
 3613   match(rdx_RegL);
 3614 
 3615   format %{ %}
 3616   interface(REG_INTER);
 3617 %}
 3618 
 3619 operand rax_RegL()
 3620 %{
 3621   constraint(ALLOC_IN_RC(long_rax_reg));
 3622   match(RegL);
 3623   match(rRegL);
 3624 
 3625   format %{ "RAX" %}
 3626   interface(REG_INTER);
 3627 %}
 3628 
 3629 operand rcx_RegL()
 3630 %{
 3631   constraint(ALLOC_IN_RC(long_rcx_reg));
 3632   match(RegL);
 3633   match(rRegL);
 3634 
 3635   format %{ %}
 3636   interface(REG_INTER);
 3637 %}
 3638 
 3639 operand rdx_RegL()
 3640 %{
 3641   constraint(ALLOC_IN_RC(long_rdx_reg));
 3642   match(RegL);
 3643   match(rRegL);
 3644 
 3645   format %{ %}
 3646   interface(REG_INTER);
 3647 %}
 3648 
 3649 operand no_rbp_r13_RegL()
 3650 %{
 3651   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3652   match(RegL);
 3653   match(rRegL);
 3654   match(rax_RegL);
 3655   match(rcx_RegL);
 3656   match(rdx_RegL);
 3657 
 3658   format %{ %}
 3659   interface(REG_INTER);
 3660 %}
 3661 
 3662 // Flags register, used as output of compare instructions
 3663 operand rFlagsReg()
 3664 %{
 3665   constraint(ALLOC_IN_RC(int_flags));
 3666   match(RegFlags);
 3667 
 3668   format %{ "RFLAGS" %}
 3669   interface(REG_INTER);
 3670 %}
 3671 
 3672 // Flags register, used as output of FLOATING POINT compare instructions
 3673 operand rFlagsRegU()
 3674 %{
 3675   constraint(ALLOC_IN_RC(int_flags));
 3676   match(RegFlags);
 3677 
 3678   format %{ "RFLAGS_U" %}
 3679   interface(REG_INTER);
 3680 %}
 3681 
 3682 operand rFlagsRegUCF() %{
 3683   constraint(ALLOC_IN_RC(int_flags));
 3684   match(RegFlags);
 3685   predicate(false);
 3686 
 3687   format %{ "RFLAGS_U_CF" %}
 3688   interface(REG_INTER);
 3689 %}
 3690 
 3691 // Float register operands
 3692 operand regF() %{
 3693    constraint(ALLOC_IN_RC(float_reg));
 3694    match(RegF);
 3695 
 3696    format %{ %}
 3697    interface(REG_INTER);
 3698 %}
 3699 
 3700 // Float register operands
 3701 operand legRegF() %{
 3702    constraint(ALLOC_IN_RC(float_reg_legacy));
 3703    match(RegF);
 3704 
 3705    format %{ %}
 3706    interface(REG_INTER);
 3707 %}
 3708 
 3709 // Float register operands
 3710 operand vlRegF() %{
 3711    constraint(ALLOC_IN_RC(float_reg_vl));
 3712    match(RegF);
 3713 
 3714    format %{ %}
 3715    interface(REG_INTER);
 3716 %}
 3717 
 3718 // Double register operands
 3719 operand regD() %{
 3720    constraint(ALLOC_IN_RC(double_reg));
 3721    match(RegD);
 3722 
 3723    format %{ %}
 3724    interface(REG_INTER);
 3725 %}
 3726 
 3727 // Double register operands
 3728 operand legRegD() %{
 3729    constraint(ALLOC_IN_RC(double_reg_legacy));
 3730    match(RegD);
 3731 
 3732    format %{ %}
 3733    interface(REG_INTER);
 3734 %}
 3735 
 3736 // Double register operands
 3737 operand vlRegD() %{
 3738    constraint(ALLOC_IN_RC(double_reg_vl));
 3739    match(RegD);
 3740 
 3741    format %{ %}
 3742    interface(REG_INTER);
 3743 %}
 3744 
 3745 //----------Memory Operands----------------------------------------------------
 3746 // Direct Memory Operand
 3747 // operand direct(immP addr)
 3748 // %{
 3749 //   match(addr);
 3750 
 3751 //   format %{ "[$addr]" %}
 3752 //   interface(MEMORY_INTER) %{
 3753 //     base(0xFFFFFFFF);
 3754 //     index(0x4);
 3755 //     scale(0x0);
 3756 //     disp($addr);
 3757 //   %}
 3758 // %}
 3759 
 3760 // Indirect Memory Operand
 3761 operand indirect(any_RegP reg)
 3762 %{
 3763   constraint(ALLOC_IN_RC(ptr_reg));
 3764   match(reg);
 3765 
 3766   format %{ "[$reg]" %}
 3767   interface(MEMORY_INTER) %{
 3768     base($reg);
 3769     index(0x4);
 3770     scale(0x0);
 3771     disp(0x0);
 3772   %}
 3773 %}
 3774 
 3775 // Indirect Memory Plus Short Offset Operand
 3776 operand indOffset8(any_RegP reg, immL8 off)
 3777 %{
 3778   constraint(ALLOC_IN_RC(ptr_reg));
 3779   match(AddP reg off);
 3780 
 3781   format %{ "[$reg + $off (8-bit)]" %}
 3782   interface(MEMORY_INTER) %{
 3783     base($reg);
 3784     index(0x4);
 3785     scale(0x0);
 3786     disp($off);
 3787   %}
 3788 %}
 3789 
 3790 // Indirect Memory Plus Long Offset Operand
 3791 operand indOffset32(any_RegP reg, immL32 off)
 3792 %{
 3793   constraint(ALLOC_IN_RC(ptr_reg));
 3794   match(AddP reg off);
 3795 
 3796   format %{ "[$reg + $off (32-bit)]" %}
 3797   interface(MEMORY_INTER) %{
 3798     base($reg);
 3799     index(0x4);
 3800     scale(0x0);
 3801     disp($off);
 3802   %}
 3803 %}
 3804 
 3805 // Indirect Memory Plus Index Register Plus Offset Operand
 3806 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3807 %{
 3808   constraint(ALLOC_IN_RC(ptr_reg));
 3809   match(AddP (AddP reg lreg) off);
 3810 
 3811   op_cost(10);
 3812   format %{"[$reg + $off + $lreg]" %}
 3813   interface(MEMORY_INTER) %{
 3814     base($reg);
 3815     index($lreg);
 3816     scale(0x0);
 3817     disp($off);
 3818   %}
 3819 %}
 3820 
 3821 // Indirect Memory Plus Index Register Plus Offset Operand
 3822 operand indIndex(any_RegP reg, rRegL lreg)
 3823 %{
 3824   constraint(ALLOC_IN_RC(ptr_reg));
 3825   match(AddP reg lreg);
 3826 
 3827   op_cost(10);
 3828   format %{"[$reg + $lreg]" %}
 3829   interface(MEMORY_INTER) %{
 3830     base($reg);
 3831     index($lreg);
 3832     scale(0x0);
 3833     disp(0x0);
 3834   %}
 3835 %}
 3836 
 3837 // Indirect Memory Times Scale Plus Index Register
 3838 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3839 %{
 3840   constraint(ALLOC_IN_RC(ptr_reg));
 3841   match(AddP reg (LShiftL lreg scale));
 3842 
 3843   op_cost(10);
 3844   format %{"[$reg + $lreg << $scale]" %}
 3845   interface(MEMORY_INTER) %{
 3846     base($reg);
 3847     index($lreg);
 3848     scale($scale);
 3849     disp(0x0);
 3850   %}
 3851 %}
 3852 
 3853 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3854 %{
 3855   constraint(ALLOC_IN_RC(ptr_reg));
 3856   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3857   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3858 
 3859   op_cost(10);
 3860   format %{"[$reg + pos $idx << $scale]" %}
 3861   interface(MEMORY_INTER) %{
 3862     base($reg);
 3863     index($idx);
 3864     scale($scale);
 3865     disp(0x0);
 3866   %}
 3867 %}
 3868 
 3869 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3870 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3871 %{
 3872   constraint(ALLOC_IN_RC(ptr_reg));
 3873   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3874 
 3875   op_cost(10);
 3876   format %{"[$reg + $off + $lreg << $scale]" %}
 3877   interface(MEMORY_INTER) %{
 3878     base($reg);
 3879     index($lreg);
 3880     scale($scale);
 3881     disp($off);
 3882   %}
 3883 %}
 3884 
 3885 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3886 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3887 %{
 3888   constraint(ALLOC_IN_RC(ptr_reg));
 3889   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3890   match(AddP (AddP reg (ConvI2L idx)) off);
 3891 
 3892   op_cost(10);
 3893   format %{"[$reg + $off + $idx]" %}
 3894   interface(MEMORY_INTER) %{
 3895     base($reg);
 3896     index($idx);
 3897     scale(0x0);
 3898     disp($off);
 3899   %}
 3900 %}
 3901 
 3902 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3903 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3904 %{
 3905   constraint(ALLOC_IN_RC(ptr_reg));
 3906   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3907   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3908 
 3909   op_cost(10);
 3910   format %{"[$reg + $off + $idx << $scale]" %}
 3911   interface(MEMORY_INTER) %{
 3912     base($reg);
 3913     index($idx);
 3914     scale($scale);
 3915     disp($off);
 3916   %}
 3917 %}
 3918 
 3919 // Indirect Narrow Oop Plus Offset Operand
 3920 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3921 // we can't free r12 even with CompressedOops::base() == NULL.
 3922 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3923   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3924   constraint(ALLOC_IN_RC(ptr_reg));
 3925   match(AddP (DecodeN reg) off);
 3926 
 3927   op_cost(10);
 3928   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3929   interface(MEMORY_INTER) %{
 3930     base(0xc); // R12
 3931     index($reg);
 3932     scale(0x3);
 3933     disp($off);
 3934   %}
 3935 %}
 3936 
 3937 // Indirect Memory Operand
 3938 operand indirectNarrow(rRegN reg)
 3939 %{
 3940   predicate(CompressedOops::shift() == 0);
 3941   constraint(ALLOC_IN_RC(ptr_reg));
 3942   match(DecodeN reg);
 3943 
 3944   format %{ "[$reg]" %}
 3945   interface(MEMORY_INTER) %{
 3946     base($reg);
 3947     index(0x4);
 3948     scale(0x0);
 3949     disp(0x0);
 3950   %}
 3951 %}
 3952 
 3953 // Indirect Memory Plus Short Offset Operand
 3954 operand indOffset8Narrow(rRegN reg, immL8 off)
 3955 %{
 3956   predicate(CompressedOops::shift() == 0);
 3957   constraint(ALLOC_IN_RC(ptr_reg));
 3958   match(AddP (DecodeN reg) off);
 3959 
 3960   format %{ "[$reg + $off (8-bit)]" %}
 3961   interface(MEMORY_INTER) %{
 3962     base($reg);
 3963     index(0x4);
 3964     scale(0x0);
 3965     disp($off);
 3966   %}
 3967 %}
 3968 
 3969 // Indirect Memory Plus Long Offset Operand
 3970 operand indOffset32Narrow(rRegN reg, immL32 off)
 3971 %{
 3972   predicate(CompressedOops::shift() == 0);
 3973   constraint(ALLOC_IN_RC(ptr_reg));
 3974   match(AddP (DecodeN reg) off);
 3975 
 3976   format %{ "[$reg + $off (32-bit)]" %}
 3977   interface(MEMORY_INTER) %{
 3978     base($reg);
 3979     index(0x4);
 3980     scale(0x0);
 3981     disp($off);
 3982   %}
 3983 %}
 3984 
 3985 // Indirect Memory Plus Index Register Plus Offset Operand
 3986 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3987 %{
 3988   predicate(CompressedOops::shift() == 0);
 3989   constraint(ALLOC_IN_RC(ptr_reg));
 3990   match(AddP (AddP (DecodeN reg) lreg) off);
 3991 
 3992   op_cost(10);
 3993   format %{"[$reg + $off + $lreg]" %}
 3994   interface(MEMORY_INTER) %{
 3995     base($reg);
 3996     index($lreg);
 3997     scale(0x0);
 3998     disp($off);
 3999   %}
 4000 %}
 4001 
 4002 // Indirect Memory Plus Index Register Plus Offset Operand
 4003 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4004 %{
 4005   predicate(CompressedOops::shift() == 0);
 4006   constraint(ALLOC_IN_RC(ptr_reg));
 4007   match(AddP (DecodeN reg) lreg);
 4008 
 4009   op_cost(10);
 4010   format %{"[$reg + $lreg]" %}
 4011   interface(MEMORY_INTER) %{
 4012     base($reg);
 4013     index($lreg);
 4014     scale(0x0);
 4015     disp(0x0);
 4016   %}
 4017 %}
 4018 
 4019 // Indirect Memory Times Scale Plus Index Register
 4020 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4021 %{
 4022   predicate(CompressedOops::shift() == 0);
 4023   constraint(ALLOC_IN_RC(ptr_reg));
 4024   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4025 
 4026   op_cost(10);
 4027   format %{"[$reg + $lreg << $scale]" %}
 4028   interface(MEMORY_INTER) %{
 4029     base($reg);
 4030     index($lreg);
 4031     scale($scale);
 4032     disp(0x0);
 4033   %}
 4034 %}
 4035 
 4036 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4037 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4038 %{
 4039   predicate(CompressedOops::shift() == 0);
 4040   constraint(ALLOC_IN_RC(ptr_reg));
 4041   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4042 
 4043   op_cost(10);
 4044   format %{"[$reg + $off + $lreg << $scale]" %}
 4045   interface(MEMORY_INTER) %{
 4046     base($reg);
 4047     index($lreg);
 4048     scale($scale);
 4049     disp($off);
 4050   %}
 4051 %}
 4052 
 4053 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4054 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4055 %{
 4056   constraint(ALLOC_IN_RC(ptr_reg));
 4057   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4058   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4059 
 4060   op_cost(10);
 4061   format %{"[$reg + $off + $idx]" %}
 4062   interface(MEMORY_INTER) %{
 4063     base($reg);
 4064     index($idx);
 4065     scale(0x0);
 4066     disp($off);
 4067   %}
 4068 %}
 4069 
 4070 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4071 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4072 %{
 4073   constraint(ALLOC_IN_RC(ptr_reg));
 4074   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4075   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4076 
 4077   op_cost(10);
 4078   format %{"[$reg + $off + $idx << $scale]" %}
 4079   interface(MEMORY_INTER) %{
 4080     base($reg);
 4081     index($idx);
 4082     scale($scale);
 4083     disp($off);
 4084   %}
 4085 %}
 4086 
 4087 //----------Special Memory Operands--------------------------------------------
 4088 // Stack Slot Operand - This operand is used for loading and storing temporary
 4089 //                      values on the stack where a match requires a value to
 4090 //                      flow through memory.
 4091 operand stackSlotP(sRegP reg)
 4092 %{
 4093   constraint(ALLOC_IN_RC(stack_slots));
 4094   // No match rule because this operand is only generated in matching
 4095 
 4096   format %{ "[$reg]" %}
 4097   interface(MEMORY_INTER) %{
 4098     base(0x4);   // RSP
 4099     index(0x4);  // No Index
 4100     scale(0x0);  // No Scale
 4101     disp($reg);  // Stack Offset
 4102   %}
 4103 %}
 4104 
 4105 operand stackSlotI(sRegI reg)
 4106 %{
 4107   constraint(ALLOC_IN_RC(stack_slots));
 4108   // No match rule because this operand is only generated in matching
 4109 
 4110   format %{ "[$reg]" %}
 4111   interface(MEMORY_INTER) %{
 4112     base(0x4);   // RSP
 4113     index(0x4);  // No Index
 4114     scale(0x0);  // No Scale
 4115     disp($reg);  // Stack Offset
 4116   %}
 4117 %}
 4118 
 4119 operand stackSlotF(sRegF reg)
 4120 %{
 4121   constraint(ALLOC_IN_RC(stack_slots));
 4122   // No match rule because this operand is only generated in matching
 4123 
 4124   format %{ "[$reg]" %}
 4125   interface(MEMORY_INTER) %{
 4126     base(0x4);   // RSP
 4127     index(0x4);  // No Index
 4128     scale(0x0);  // No Scale
 4129     disp($reg);  // Stack Offset
 4130   %}
 4131 %}
 4132 
 4133 operand stackSlotD(sRegD reg)
 4134 %{
 4135   constraint(ALLOC_IN_RC(stack_slots));
 4136   // No match rule because this operand is only generated in matching
 4137 
 4138   format %{ "[$reg]" %}
 4139   interface(MEMORY_INTER) %{
 4140     base(0x4);   // RSP
 4141     index(0x4);  // No Index
 4142     scale(0x0);  // No Scale
 4143     disp($reg);  // Stack Offset
 4144   %}
 4145 %}
 4146 operand stackSlotL(sRegL reg)
 4147 %{
 4148   constraint(ALLOC_IN_RC(stack_slots));
 4149   // No match rule because this operand is only generated in matching
 4150 
 4151   format %{ "[$reg]" %}
 4152   interface(MEMORY_INTER) %{
 4153     base(0x4);   // RSP
 4154     index(0x4);  // No Index
 4155     scale(0x0);  // No Scale
 4156     disp($reg);  // Stack Offset
 4157   %}
 4158 %}
 4159 
 4160 //----------Conditional Branch Operands----------------------------------------
 4161 // Comparison Op  - This is the operation of the comparison, and is limited to
 4162 //                  the following set of codes:
 4163 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4164 //
 4165 // Other attributes of the comparison, such as unsignedness, are specified
 4166 // by the comparison instruction that sets a condition code flags register.
 4167 // That result is represented by a flags operand whose subtype is appropriate
 4168 // to the unsignedness (etc.) of the comparison.
 4169 //
 4170 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4171 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4172 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4173 
 4174 // Comparison Code
 4175 operand cmpOp()
 4176 %{
 4177   match(Bool);
 4178 
 4179   format %{ "" %}
 4180   interface(COND_INTER) %{
 4181     equal(0x4, "e");
 4182     not_equal(0x5, "ne");
 4183     less(0xC, "l");
 4184     greater_equal(0xD, "ge");
 4185     less_equal(0xE, "le");
 4186     greater(0xF, "g");
 4187     overflow(0x0, "o");
 4188     no_overflow(0x1, "no");
 4189   %}
 4190 %}
 4191 
 4192 // Comparison Code, unsigned compare.  Used by FP also, with
 4193 // C2 (unordered) turned into GT or LT already.  The other bits
 4194 // C0 and C3 are turned into Carry & Zero flags.
 4195 operand cmpOpU()
 4196 %{
 4197   match(Bool);
 4198 
 4199   format %{ "" %}
 4200   interface(COND_INTER) %{
 4201     equal(0x4, "e");
 4202     not_equal(0x5, "ne");
 4203     less(0x2, "b");
 4204     greater_equal(0x3, "nb");
 4205     less_equal(0x6, "be");
 4206     greater(0x7, "nbe");
 4207     overflow(0x0, "o");
 4208     no_overflow(0x1, "no");
 4209   %}
 4210 %}
 4211 
 4212 
 4213 // Floating comparisons that don't require any fixup for the unordered case
 4214 operand cmpOpUCF() %{
 4215   match(Bool);
 4216   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4217             n->as_Bool()->_test._test == BoolTest::ge ||
 4218             n->as_Bool()->_test._test == BoolTest::le ||
 4219             n->as_Bool()->_test._test == BoolTest::gt);
 4220   format %{ "" %}
 4221   interface(COND_INTER) %{
 4222     equal(0x4, "e");
 4223     not_equal(0x5, "ne");
 4224     less(0x2, "b");
 4225     greater_equal(0x3, "nb");
 4226     less_equal(0x6, "be");
 4227     greater(0x7, "nbe");
 4228     overflow(0x0, "o");
 4229     no_overflow(0x1, "no");
 4230   %}
 4231 %}
 4232 
 4233 
 4234 // Floating comparisons that can be fixed up with extra conditional jumps
 4235 operand cmpOpUCF2() %{
 4236   match(Bool);
 4237   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
 4238             n->as_Bool()->_test._test == BoolTest::eq);
 4239   format %{ "" %}
 4240   interface(COND_INTER) %{
 4241     equal(0x4, "e");
 4242     not_equal(0x5, "ne");
 4243     less(0x2, "b");
 4244     greater_equal(0x3, "nb");
 4245     less_equal(0x6, "be");
 4246     greater(0x7, "nbe");
 4247     overflow(0x0, "o");
 4248     no_overflow(0x1, "no");
 4249   %}
 4250 %}
 4251 
 4252 //----------OPERAND CLASSES----------------------------------------------------
 4253 // Operand Classes are groups of operands that are used as to simplify
 4254 // instruction definitions by not requiring the AD writer to specify separate
 4255 // instructions for every form of operand when the instruction accepts
 4256 // multiple operand types with the same basic encoding and format.  The classic
 4257 // case of this is memory operands.
 4258 
 4259 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4260                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4261                indCompressedOopOffset,
 4262                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4263                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4264                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4265 
 4266 //----------PIPELINE-----------------------------------------------------------
 4267 // Rules which define the behavior of the target architectures pipeline.
 4268 pipeline %{
 4269 
 4270 //----------ATTRIBUTES---------------------------------------------------------
 4271 attributes %{
 4272   variable_size_instructions;        // Fixed size instructions
 4273   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4274   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4275   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4276   instruction_fetch_units = 1;       // of 16 bytes
 4277 
 4278   // List of nop instructions
 4279   nops( MachNop );
 4280 %}
 4281 
 4282 //----------RESOURCES----------------------------------------------------------
 4283 // Resources are the functional units available to the machine
 4284 
 4285 // Generic P2/P3 pipeline
 4286 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4287 // 3 instructions decoded per cycle.
 4288 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4289 // 3 ALU op, only ALU0 handles mul instructions.
 4290 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4291            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4292            BR, FPU,
 4293            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4294 
 4295 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4296 // Pipeline Description specifies the stages in the machine's pipeline
 4297 
 4298 // Generic P2/P3 pipeline
 4299 pipe_desc(S0, S1, S2, S3, S4, S5);
 4300 
 4301 //----------PIPELINE CLASSES---------------------------------------------------
 4302 // Pipeline Classes describe the stages in which input and output are
 4303 // referenced by the hardware pipeline.
 4304 
 4305 // Naming convention: ialu or fpu
 4306 // Then: _reg
 4307 // Then: _reg if there is a 2nd register
 4308 // Then: _long if it's a pair of instructions implementing a long
 4309 // Then: _fat if it requires the big decoder
 4310 //   Or: _mem if it requires the big decoder and a memory unit.
 4311 
 4312 // Integer ALU reg operation
 4313 pipe_class ialu_reg(rRegI dst)
 4314 %{
 4315     single_instruction;
 4316     dst    : S4(write);
 4317     dst    : S3(read);
 4318     DECODE : S0;        // any decoder
 4319     ALU    : S3;        // any alu
 4320 %}
 4321 
 4322 // Long ALU reg operation
 4323 pipe_class ialu_reg_long(rRegL dst)
 4324 %{
 4325     instruction_count(2);
 4326     dst    : S4(write);
 4327     dst    : S3(read);
 4328     DECODE : S0(2);     // any 2 decoders
 4329     ALU    : S3(2);     // both alus
 4330 %}
 4331 
 4332 // Integer ALU reg operation using big decoder
 4333 pipe_class ialu_reg_fat(rRegI dst)
 4334 %{
 4335     single_instruction;
 4336     dst    : S4(write);
 4337     dst    : S3(read);
 4338     D0     : S0;        // big decoder only
 4339     ALU    : S3;        // any alu
 4340 %}
 4341 
 4342 // Integer ALU reg-reg operation
 4343 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4344 %{
 4345     single_instruction;
 4346     dst    : S4(write);
 4347     src    : S3(read);
 4348     DECODE : S0;        // any decoder
 4349     ALU    : S3;        // any alu
 4350 %}
 4351 
 4352 // Integer ALU reg-reg operation
 4353 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4354 %{
 4355     single_instruction;
 4356     dst    : S4(write);
 4357     src    : S3(read);
 4358     D0     : S0;        // big decoder only
 4359     ALU    : S3;        // any alu
 4360 %}
 4361 
 4362 // Integer ALU reg-mem operation
 4363 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4364 %{
 4365     single_instruction;
 4366     dst    : S5(write);
 4367     mem    : S3(read);
 4368     D0     : S0;        // big decoder only
 4369     ALU    : S4;        // any alu
 4370     MEM    : S3;        // any mem
 4371 %}
 4372 
 4373 // Integer mem operation (prefetch)
 4374 pipe_class ialu_mem(memory mem)
 4375 %{
 4376     single_instruction;
 4377     mem    : S3(read);
 4378     D0     : S0;        // big decoder only
 4379     MEM    : S3;        // any mem
 4380 %}
 4381 
 4382 // Integer Store to Memory
 4383 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4384 %{
 4385     single_instruction;
 4386     mem    : S3(read);
 4387     src    : S5(read);
 4388     D0     : S0;        // big decoder only
 4389     ALU    : S4;        // any alu
 4390     MEM    : S3;
 4391 %}
 4392 
 4393 // // Long Store to Memory
 4394 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4395 // %{
 4396 //     instruction_count(2);
 4397 //     mem    : S3(read);
 4398 //     src    : S5(read);
 4399 //     D0     : S0(2);          // big decoder only; twice
 4400 //     ALU    : S4(2);     // any 2 alus
 4401 //     MEM    : S3(2);  // Both mems
 4402 // %}
 4403 
 4404 // Integer Store to Memory
 4405 pipe_class ialu_mem_imm(memory mem)
 4406 %{
 4407     single_instruction;
 4408     mem    : S3(read);
 4409     D0     : S0;        // big decoder only
 4410     ALU    : S4;        // any alu
 4411     MEM    : S3;
 4412 %}
 4413 
 4414 // Integer ALU0 reg-reg operation
 4415 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4416 %{
 4417     single_instruction;
 4418     dst    : S4(write);
 4419     src    : S3(read);
 4420     D0     : S0;        // Big decoder only
 4421     ALU0   : S3;        // only alu0
 4422 %}
 4423 
 4424 // Integer ALU0 reg-mem operation
 4425 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4426 %{
 4427     single_instruction;
 4428     dst    : S5(write);
 4429     mem    : S3(read);
 4430     D0     : S0;        // big decoder only
 4431     ALU0   : S4;        // ALU0 only
 4432     MEM    : S3;        // any mem
 4433 %}
 4434 
 4435 // Integer ALU reg-reg operation
 4436 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4437 %{
 4438     single_instruction;
 4439     cr     : S4(write);
 4440     src1   : S3(read);
 4441     src2   : S3(read);
 4442     DECODE : S0;        // any decoder
 4443     ALU    : S3;        // any alu
 4444 %}
 4445 
 4446 // Integer ALU reg-imm operation
 4447 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4448 %{
 4449     single_instruction;
 4450     cr     : S4(write);
 4451     src1   : S3(read);
 4452     DECODE : S0;        // any decoder
 4453     ALU    : S3;        // any alu
 4454 %}
 4455 
 4456 // Integer ALU reg-mem operation
 4457 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4458 %{
 4459     single_instruction;
 4460     cr     : S4(write);
 4461     src1   : S3(read);
 4462     src2   : S3(read);
 4463     D0     : S0;        // big decoder only
 4464     ALU    : S4;        // any alu
 4465     MEM    : S3;
 4466 %}
 4467 
 4468 // Conditional move reg-reg
 4469 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4470 %{
 4471     instruction_count(4);
 4472     y      : S4(read);
 4473     q      : S3(read);
 4474     p      : S3(read);
 4475     DECODE : S0(4);     // any decoder
 4476 %}
 4477 
 4478 // Conditional move reg-reg
 4479 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4480 %{
 4481     single_instruction;
 4482     dst    : S4(write);
 4483     src    : S3(read);
 4484     cr     : S3(read);
 4485     DECODE : S0;        // any decoder
 4486 %}
 4487 
 4488 // Conditional move reg-mem
 4489 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4490 %{
 4491     single_instruction;
 4492     dst    : S4(write);
 4493     src    : S3(read);
 4494     cr     : S3(read);
 4495     DECODE : S0;        // any decoder
 4496     MEM    : S3;
 4497 %}
 4498 
 4499 // Conditional move reg-reg long
 4500 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4501 %{
 4502     single_instruction;
 4503     dst    : S4(write);
 4504     src    : S3(read);
 4505     cr     : S3(read);
 4506     DECODE : S0(2);     // any 2 decoders
 4507 %}
 4508 
 4509 // XXX
 4510 // // Conditional move double reg-reg
 4511 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4512 // %{
 4513 //     single_instruction;
 4514 //     dst    : S4(write);
 4515 //     src    : S3(read);
 4516 //     cr     : S3(read);
 4517 //     DECODE : S0;     // any decoder
 4518 // %}
 4519 
 4520 // Float reg-reg operation
 4521 pipe_class fpu_reg(regD dst)
 4522 %{
 4523     instruction_count(2);
 4524     dst    : S3(read);
 4525     DECODE : S0(2);     // any 2 decoders
 4526     FPU    : S3;
 4527 %}
 4528 
 4529 // Float reg-reg operation
 4530 pipe_class fpu_reg_reg(regD dst, regD src)
 4531 %{
 4532     instruction_count(2);
 4533     dst    : S4(write);
 4534     src    : S3(read);
 4535     DECODE : S0(2);     // any 2 decoders
 4536     FPU    : S3;
 4537 %}
 4538 
 4539 // Float reg-reg operation
 4540 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4541 %{
 4542     instruction_count(3);
 4543     dst    : S4(write);
 4544     src1   : S3(read);
 4545     src2   : S3(read);
 4546     DECODE : S0(3);     // any 3 decoders
 4547     FPU    : S3(2);
 4548 %}
 4549 
 4550 // Float reg-reg operation
 4551 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4552 %{
 4553     instruction_count(4);
 4554     dst    : S4(write);
 4555     src1   : S3(read);
 4556     src2   : S3(read);
 4557     src3   : S3(read);
 4558     DECODE : S0(4);     // any 3 decoders
 4559     FPU    : S3(2);
 4560 %}
 4561 
 4562 // Float reg-reg operation
 4563 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4564 %{
 4565     instruction_count(4);
 4566     dst    : S4(write);
 4567     src1   : S3(read);
 4568     src2   : S3(read);
 4569     src3   : S3(read);
 4570     DECODE : S1(3);     // any 3 decoders
 4571     D0     : S0;        // Big decoder only
 4572     FPU    : S3(2);
 4573     MEM    : S3;
 4574 %}
 4575 
 4576 // Float reg-mem operation
 4577 pipe_class fpu_reg_mem(regD dst, memory mem)
 4578 %{
 4579     instruction_count(2);
 4580     dst    : S5(write);
 4581     mem    : S3(read);
 4582     D0     : S0;        // big decoder only
 4583     DECODE : S1;        // any decoder for FPU POP
 4584     FPU    : S4;
 4585     MEM    : S3;        // any mem
 4586 %}
 4587 
 4588 // Float reg-mem operation
 4589 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4590 %{
 4591     instruction_count(3);
 4592     dst    : S5(write);
 4593     src1   : S3(read);
 4594     mem    : S3(read);
 4595     D0     : S0;        // big decoder only
 4596     DECODE : S1(2);     // any decoder for FPU POP
 4597     FPU    : S4;
 4598     MEM    : S3;        // any mem
 4599 %}
 4600 
 4601 // Float mem-reg operation
 4602 pipe_class fpu_mem_reg(memory mem, regD src)
 4603 %{
 4604     instruction_count(2);
 4605     src    : S5(read);
 4606     mem    : S3(read);
 4607     DECODE : S0;        // any decoder for FPU PUSH
 4608     D0     : S1;        // big decoder only
 4609     FPU    : S4;
 4610     MEM    : S3;        // any mem
 4611 %}
 4612 
 4613 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4614 %{
 4615     instruction_count(3);
 4616     src1   : S3(read);
 4617     src2   : S3(read);
 4618     mem    : S3(read);
 4619     DECODE : S0(2);     // any decoder for FPU PUSH
 4620     D0     : S1;        // big decoder only
 4621     FPU    : S4;
 4622     MEM    : S3;        // any mem
 4623 %}
 4624 
 4625 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4626 %{
 4627     instruction_count(3);
 4628     src1   : S3(read);
 4629     src2   : S3(read);
 4630     mem    : S4(read);
 4631     DECODE : S0;        // any decoder for FPU PUSH
 4632     D0     : S0(2);     // big decoder only
 4633     FPU    : S4;
 4634     MEM    : S3(2);     // any mem
 4635 %}
 4636 
 4637 pipe_class fpu_mem_mem(memory dst, memory src1)
 4638 %{
 4639     instruction_count(2);
 4640     src1   : S3(read);
 4641     dst    : S4(read);
 4642     D0     : S0(2);     // big decoder only
 4643     MEM    : S3(2);     // any mem
 4644 %}
 4645 
 4646 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4647 %{
 4648     instruction_count(3);
 4649     src1   : S3(read);
 4650     src2   : S3(read);
 4651     dst    : S4(read);
 4652     D0     : S0(3);     // big decoder only
 4653     FPU    : S4;
 4654     MEM    : S3(3);     // any mem
 4655 %}
 4656 
 4657 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4658 %{
 4659     instruction_count(3);
 4660     src1   : S4(read);
 4661     mem    : S4(read);
 4662     DECODE : S0;        // any decoder for FPU PUSH
 4663     D0     : S0(2);     // big decoder only
 4664     FPU    : S4;
 4665     MEM    : S3(2);     // any mem
 4666 %}
 4667 
 4668 // Float load constant
 4669 pipe_class fpu_reg_con(regD dst)
 4670 %{
 4671     instruction_count(2);
 4672     dst    : S5(write);
 4673     D0     : S0;        // big decoder only for the load
 4674     DECODE : S1;        // any decoder for FPU POP
 4675     FPU    : S4;
 4676     MEM    : S3;        // any mem
 4677 %}
 4678 
 4679 // Float load constant
 4680 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4681 %{
 4682     instruction_count(3);
 4683     dst    : S5(write);
 4684     src    : S3(read);
 4685     D0     : S0;        // big decoder only for the load
 4686     DECODE : S1(2);     // any decoder for FPU POP
 4687     FPU    : S4;
 4688     MEM    : S3;        // any mem
 4689 %}
 4690 
 4691 // UnConditional branch
 4692 pipe_class pipe_jmp(label labl)
 4693 %{
 4694     single_instruction;
 4695     BR   : S3;
 4696 %}
 4697 
 4698 // Conditional branch
 4699 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4700 %{
 4701     single_instruction;
 4702     cr    : S1(read);
 4703     BR    : S3;
 4704 %}
 4705 
 4706 // Allocation idiom
 4707 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4708 %{
 4709     instruction_count(1); force_serialization;
 4710     fixed_latency(6);
 4711     heap_ptr : S3(read);
 4712     DECODE   : S0(3);
 4713     D0       : S2;
 4714     MEM      : S3;
 4715     ALU      : S3(2);
 4716     dst      : S5(write);
 4717     BR       : S5;
 4718 %}
 4719 
 4720 // Generic big/slow expanded idiom
 4721 pipe_class pipe_slow()
 4722 %{
 4723     instruction_count(10); multiple_bundles; force_serialization;
 4724     fixed_latency(100);
 4725     D0  : S0(2);
 4726     MEM : S3(2);
 4727 %}
 4728 
 4729 // The real do-nothing guy
 4730 pipe_class empty()
 4731 %{
 4732     instruction_count(0);
 4733 %}
 4734 
 4735 // Define the class for the Nop node
 4736 define
 4737 %{
 4738    MachNop = empty;
 4739 %}
 4740 
 4741 %}
 4742 
 4743 //----------INSTRUCTIONS-------------------------------------------------------
 4744 //
 4745 // match      -- States which machine-independent subtree may be replaced
 4746 //               by this instruction.
 4747 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4748 //               selection to identify a minimum cost tree of machine
 4749 //               instructions that matches a tree of machine-independent
 4750 //               instructions.
 4751 // format     -- A string providing the disassembly for this instruction.
 4752 //               The value of an instruction's operand may be inserted
 4753 //               by referring to it with a '$' prefix.
 4754 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4755 //               to within an encode class as $primary, $secondary, and $tertiary
 4756 //               rrspectively.  The primary opcode is commonly used to
 4757 //               indicate the type of machine instruction, while secondary
 4758 //               and tertiary are often used for prefix options or addressing
 4759 //               modes.
 4760 // ins_encode -- A list of encode classes with parameters. The encode class
 4761 //               name must have been defined in an 'enc_class' specification
 4762 //               in the encode section of the architecture description.
 4763 
 4764 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4765 // Load Float
 4766 instruct MoveF2VL(vlRegF dst, regF src) %{
 4767   match(Set dst src);
 4768   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4769   ins_encode %{
 4770     ShouldNotReachHere();
 4771   %}
 4772   ins_pipe( fpu_reg_reg );
 4773 %}
 4774 
 4775 // Load Float
 4776 instruct MoveF2LEG(legRegF dst, regF src) %{
 4777   match(Set dst src);
 4778   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4779   ins_encode %{
 4780     ShouldNotReachHere();
 4781   %}
 4782   ins_pipe( fpu_reg_reg );
 4783 %}
 4784 
 4785 // Load Float
 4786 instruct MoveVL2F(regF dst, vlRegF src) %{
 4787   match(Set dst src);
 4788   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4789   ins_encode %{
 4790     ShouldNotReachHere();
 4791   %}
 4792   ins_pipe( fpu_reg_reg );
 4793 %}
 4794 
 4795 // Load Float
 4796 instruct MoveLEG2F(regF dst, legRegF src) %{
 4797   match(Set dst src);
 4798   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4799   ins_encode %{
 4800     ShouldNotReachHere();
 4801   %}
 4802   ins_pipe( fpu_reg_reg );
 4803 %}
 4804 
 4805 // Load Double
 4806 instruct MoveD2VL(vlRegD dst, regD src) %{
 4807   match(Set dst src);
 4808   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4809   ins_encode %{
 4810     ShouldNotReachHere();
 4811   %}
 4812   ins_pipe( fpu_reg_reg );
 4813 %}
 4814 
 4815 // Load Double
 4816 instruct MoveD2LEG(legRegD dst, regD src) %{
 4817   match(Set dst src);
 4818   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4819   ins_encode %{
 4820     ShouldNotReachHere();
 4821   %}
 4822   ins_pipe( fpu_reg_reg );
 4823 %}
 4824 
 4825 // Load Double
 4826 instruct MoveVL2D(regD dst, vlRegD src) %{
 4827   match(Set dst src);
 4828   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4829   ins_encode %{
 4830     ShouldNotReachHere();
 4831   %}
 4832   ins_pipe( fpu_reg_reg );
 4833 %}
 4834 
 4835 // Load Double
 4836 instruct MoveLEG2D(regD dst, legRegD src) %{
 4837   match(Set dst src);
 4838   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4839   ins_encode %{
 4840     ShouldNotReachHere();
 4841   %}
 4842   ins_pipe( fpu_reg_reg );
 4843 %}
 4844 
 4845 //----------Load/Store/Move Instructions---------------------------------------
 4846 //----------Load Instructions--------------------------------------------------
 4847 
 4848 // Load Byte (8 bit signed)
 4849 instruct loadB(rRegI dst, memory mem)
 4850 %{
 4851   match(Set dst (LoadB mem));
 4852 
 4853   ins_cost(125);
 4854   format %{ "movsbl  $dst, $mem\t# byte" %}
 4855 
 4856   ins_encode %{
 4857     __ movsbl($dst$$Register, $mem$$Address);
 4858   %}
 4859 
 4860   ins_pipe(ialu_reg_mem);
 4861 %}
 4862 
 4863 // Load Byte (8 bit signed) into Long Register
 4864 instruct loadB2L(rRegL dst, memory mem)
 4865 %{
 4866   match(Set dst (ConvI2L (LoadB mem)));
 4867 
 4868   ins_cost(125);
 4869   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4870 
 4871   ins_encode %{
 4872     __ movsbq($dst$$Register, $mem$$Address);
 4873   %}
 4874 
 4875   ins_pipe(ialu_reg_mem);
 4876 %}
 4877 
 4878 // Load Unsigned Byte (8 bit UNsigned)
 4879 instruct loadUB(rRegI dst, memory mem)
 4880 %{
 4881   match(Set dst (LoadUB mem));
 4882 
 4883   ins_cost(125);
 4884   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4885 
 4886   ins_encode %{
 4887     __ movzbl($dst$$Register, $mem$$Address);
 4888   %}
 4889 
 4890   ins_pipe(ialu_reg_mem);
 4891 %}
 4892 
 4893 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4894 instruct loadUB2L(rRegL dst, memory mem)
 4895 %{
 4896   match(Set dst (ConvI2L (LoadUB mem)));
 4897 
 4898   ins_cost(125);
 4899   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4900 
 4901   ins_encode %{
 4902     __ movzbq($dst$$Register, $mem$$Address);
 4903   %}
 4904 
 4905   ins_pipe(ialu_reg_mem);
 4906 %}
 4907 
 4908 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4909 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4910   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4911   effect(KILL cr);
 4912 
 4913   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4914             "andl    $dst, right_n_bits($mask, 8)" %}
 4915   ins_encode %{
 4916     Register Rdst = $dst$$Register;
 4917     __ movzbq(Rdst, $mem$$Address);
 4918     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4919   %}
 4920   ins_pipe(ialu_reg_mem);
 4921 %}
 4922 
 4923 // Load Short (16 bit signed)
 4924 instruct loadS(rRegI dst, memory mem)
 4925 %{
 4926   match(Set dst (LoadS mem));
 4927 
 4928   ins_cost(125);
 4929   format %{ "movswl $dst, $mem\t# short" %}
 4930 
 4931   ins_encode %{
 4932     __ movswl($dst$$Register, $mem$$Address);
 4933   %}
 4934 
 4935   ins_pipe(ialu_reg_mem);
 4936 %}
 4937 
 4938 // Load Short (16 bit signed) to Byte (8 bit signed)
 4939 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4940   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4941 
 4942   ins_cost(125);
 4943   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4944   ins_encode %{
 4945     __ movsbl($dst$$Register, $mem$$Address);
 4946   %}
 4947   ins_pipe(ialu_reg_mem);
 4948 %}
 4949 
 4950 // Load Short (16 bit signed) into Long Register
 4951 instruct loadS2L(rRegL dst, memory mem)
 4952 %{
 4953   match(Set dst (ConvI2L (LoadS mem)));
 4954 
 4955   ins_cost(125);
 4956   format %{ "movswq $dst, $mem\t# short -> long" %}
 4957 
 4958   ins_encode %{
 4959     __ movswq($dst$$Register, $mem$$Address);
 4960   %}
 4961 
 4962   ins_pipe(ialu_reg_mem);
 4963 %}
 4964 
 4965 // Load Unsigned Short/Char (16 bit UNsigned)
 4966 instruct loadUS(rRegI dst, memory mem)
 4967 %{
 4968   match(Set dst (LoadUS mem));
 4969 
 4970   ins_cost(125);
 4971   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4972 
 4973   ins_encode %{
 4974     __ movzwl($dst$$Register, $mem$$Address);
 4975   %}
 4976 
 4977   ins_pipe(ialu_reg_mem);
 4978 %}
 4979 
 4980 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4981 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4982   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4983 
 4984   ins_cost(125);
 4985   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4986   ins_encode %{
 4987     __ movsbl($dst$$Register, $mem$$Address);
 4988   %}
 4989   ins_pipe(ialu_reg_mem);
 4990 %}
 4991 
 4992 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4993 instruct loadUS2L(rRegL dst, memory mem)
 4994 %{
 4995   match(Set dst (ConvI2L (LoadUS mem)));
 4996 
 4997   ins_cost(125);
 4998   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4999 
 5000   ins_encode %{
 5001     __ movzwq($dst$$Register, $mem$$Address);
 5002   %}
 5003 
 5004   ins_pipe(ialu_reg_mem);
 5005 %}
 5006 
 5007 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5008 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5009   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5010 
 5011   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5012   ins_encode %{
 5013     __ movzbq($dst$$Register, $mem$$Address);
 5014   %}
 5015   ins_pipe(ialu_reg_mem);
 5016 %}
 5017 
 5018 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5019 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5020   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5021   effect(KILL cr);
 5022 
 5023   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5024             "andl    $dst, right_n_bits($mask, 16)" %}
 5025   ins_encode %{
 5026     Register Rdst = $dst$$Register;
 5027     __ movzwq(Rdst, $mem$$Address);
 5028     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5029   %}
 5030   ins_pipe(ialu_reg_mem);
 5031 %}
 5032 
 5033 // Load Integer
 5034 instruct loadI(rRegI dst, memory mem)
 5035 %{
 5036   match(Set dst (LoadI mem));
 5037 
 5038   ins_cost(125);
 5039   format %{ "movl    $dst, $mem\t# int" %}
 5040 
 5041   ins_encode %{
 5042     __ movl($dst$$Register, $mem$$Address);
 5043   %}
 5044 
 5045   ins_pipe(ialu_reg_mem);
 5046 %}
 5047 
 5048 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5049 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5050   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5051 
 5052   ins_cost(125);
 5053   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5054   ins_encode %{
 5055     __ movsbl($dst$$Register, $mem$$Address);
 5056   %}
 5057   ins_pipe(ialu_reg_mem);
 5058 %}
 5059 
 5060 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5061 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5062   match(Set dst (AndI (LoadI mem) mask));
 5063 
 5064   ins_cost(125);
 5065   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5066   ins_encode %{
 5067     __ movzbl($dst$$Register, $mem$$Address);
 5068   %}
 5069   ins_pipe(ialu_reg_mem);
 5070 %}
 5071 
 5072 // Load Integer (32 bit signed) to Short (16 bit signed)
 5073 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5074   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5075 
 5076   ins_cost(125);
 5077   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5078   ins_encode %{
 5079     __ movswl($dst$$Register, $mem$$Address);
 5080   %}
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5085 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5086   match(Set dst (AndI (LoadI mem) mask));
 5087 
 5088   ins_cost(125);
 5089   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5090   ins_encode %{
 5091     __ movzwl($dst$$Register, $mem$$Address);
 5092   %}
 5093   ins_pipe(ialu_reg_mem);
 5094 %}
 5095 
 5096 // Load Integer into Long Register
 5097 instruct loadI2L(rRegL dst, memory mem)
 5098 %{
 5099   match(Set dst (ConvI2L (LoadI mem)));
 5100 
 5101   ins_cost(125);
 5102   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5103 
 5104   ins_encode %{
 5105     __ movslq($dst$$Register, $mem$$Address);
 5106   %}
 5107 
 5108   ins_pipe(ialu_reg_mem);
 5109 %}
 5110 
 5111 // Load Integer with mask 0xFF into Long Register
 5112 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5113   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5114 
 5115   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5116   ins_encode %{
 5117     __ movzbq($dst$$Register, $mem$$Address);
 5118   %}
 5119   ins_pipe(ialu_reg_mem);
 5120 %}
 5121 
 5122 // Load Integer with mask 0xFFFF into Long Register
 5123 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5124   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5125 
 5126   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5127   ins_encode %{
 5128     __ movzwq($dst$$Register, $mem$$Address);
 5129   %}
 5130   ins_pipe(ialu_reg_mem);
 5131 %}
 5132 
 5133 // Load Integer with a 31-bit mask into Long Register
 5134 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5135   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5136   effect(KILL cr);
 5137 
 5138   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5139             "andl    $dst, $mask" %}
 5140   ins_encode %{
 5141     Register Rdst = $dst$$Register;
 5142     __ movl(Rdst, $mem$$Address);
 5143     __ andl(Rdst, $mask$$constant);
 5144   %}
 5145   ins_pipe(ialu_reg_mem);
 5146 %}
 5147 
 5148 // Load Unsigned Integer into Long Register
 5149 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5150 %{
 5151   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5152 
 5153   ins_cost(125);
 5154   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5155 
 5156   ins_encode %{
 5157     __ movl($dst$$Register, $mem$$Address);
 5158   %}
 5159 
 5160   ins_pipe(ialu_reg_mem);
 5161 %}
 5162 
 5163 // Load Long
 5164 instruct loadL(rRegL dst, memory mem)
 5165 %{
 5166   match(Set dst (LoadL mem));
 5167 
 5168   ins_cost(125);
 5169   format %{ "movq    $dst, $mem\t# long" %}
 5170 
 5171   ins_encode %{
 5172     __ movq($dst$$Register, $mem$$Address);
 5173   %}
 5174 
 5175   ins_pipe(ialu_reg_mem); // XXX
 5176 %}
 5177 
 5178 // Load Range
 5179 instruct loadRange(rRegI dst, memory mem)
 5180 %{
 5181   match(Set dst (LoadRange mem));
 5182 
 5183   ins_cost(125); // XXX
 5184   format %{ "movl    $dst, $mem\t# range" %}
 5185   ins_encode %{
 5186     __ movl($dst$$Register, $mem$$Address);
 5187   %}
 5188   ins_pipe(ialu_reg_mem);
 5189 %}
 5190 
 5191 // Load Pointer
 5192 instruct loadP(rRegP dst, memory mem)
 5193 %{
 5194   match(Set dst (LoadP mem));
 5195   predicate(n->as_Load()->barrier_data() == 0);
 5196 
 5197   ins_cost(125); // XXX
 5198   format %{ "movq    $dst, $mem\t# ptr" %}
 5199   ins_encode %{
 5200     __ movq($dst$$Register, $mem$$Address);
 5201   %}
 5202   ins_pipe(ialu_reg_mem); // XXX
 5203 %}
 5204 
 5205 // Load Compressed Pointer
 5206 instruct loadN(rRegN dst, memory mem)
 5207 %{
 5208    match(Set dst (LoadN mem));
 5209 
 5210    ins_cost(125); // XXX
 5211    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5212    ins_encode %{
 5213      __ movl($dst$$Register, $mem$$Address);
 5214    %}
 5215    ins_pipe(ialu_reg_mem); // XXX
 5216 %}
 5217 
 5218 
 5219 // Load Klass Pointer
 5220 instruct loadKlass(rRegP dst, memory mem)
 5221 %{
 5222   match(Set dst (LoadKlass mem));
 5223 
 5224   ins_cost(125); // XXX
 5225   format %{ "movq    $dst, $mem\t# class" %}
 5226   ins_encode %{
 5227     __ movq($dst$$Register, $mem$$Address);
 5228   %}
 5229   ins_pipe(ialu_reg_mem); // XXX
 5230 %}
 5231 
 5232 // Load narrow Klass Pointer
 5233 instruct loadNKlass(rRegN dst, indOffset8 mem, rFlagsReg cr)
 5234 %{
 5235   match(Set dst (LoadNKlass mem));
 5236   effect(TEMP_DEF dst, KILL cr);
 5237   ins_cost(125); // XXX
 5238   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5239   ins_encode %{
 5240     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset 4, but got: %d", $mem$$disp);
 5241     assert($mem$$index == 4, "expect no index register: %d", $mem$$index);
 5242     __ load_nklass($dst$$Register, $mem$$base$$Register);
 5243   %}
 5244   ins_pipe(pipe_slow); // XXX
 5245 %}
 5246 
 5247 // Load Float
 5248 instruct loadF(regF dst, memory mem)
 5249 %{
 5250   match(Set dst (LoadF mem));
 5251 
 5252   ins_cost(145); // XXX
 5253   format %{ "movss   $dst, $mem\t# float" %}
 5254   ins_encode %{
 5255     __ movflt($dst$$XMMRegister, $mem$$Address);
 5256   %}
 5257   ins_pipe(pipe_slow); // XXX
 5258 %}
 5259 
 5260 // Load Double
 5261 instruct loadD_partial(regD dst, memory mem)
 5262 %{
 5263   predicate(!UseXmmLoadAndClearUpper);
 5264   match(Set dst (LoadD mem));
 5265 
 5266   ins_cost(145); // XXX
 5267   format %{ "movlpd  $dst, $mem\t# double" %}
 5268   ins_encode %{
 5269     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5270   %}
 5271   ins_pipe(pipe_slow); // XXX
 5272 %}
 5273 
 5274 instruct loadD(regD dst, memory mem)
 5275 %{
 5276   predicate(UseXmmLoadAndClearUpper);
 5277   match(Set dst (LoadD mem));
 5278 
 5279   ins_cost(145); // XXX
 5280   format %{ "movsd   $dst, $mem\t# double" %}
 5281   ins_encode %{
 5282     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5283   %}
 5284   ins_pipe(pipe_slow); // XXX
 5285 %}
 5286 
 5287 
 5288 // Following pseudo code describes the algorithm for max[FD]:
 5289 // Min algorithm is on similar lines
 5290 //  btmp = (b < +0.0) ? a : b
 5291 //  atmp = (b < +0.0) ? b : a
 5292 //  Tmp  = Max_Float(atmp , btmp)
 5293 //  Res  = (atmp == NaN) ? atmp : Tmp
 5294 
 5295 // max = java.lang.Math.max(float a, float b)
 5296 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5297   predicate(UseAVX > 0 && !n->is_reduction());
 5298   match(Set dst (MaxF a b));
 5299   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5300   format %{
 5301      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5302      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5303      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5304      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5305      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5306   %}
 5307   ins_encode %{
 5308     int vector_len = Assembler::AVX_128bit;
 5309     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5310     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5311     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5312     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5313     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5314  %}
 5315   ins_pipe( pipe_slow );
 5316 %}
 5317 
 5318 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5319   predicate(UseAVX > 0 && n->is_reduction());
 5320   match(Set dst (MaxF a b));
 5321   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5322 
 5323   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5324   ins_encode %{
 5325     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5326                     false /*min*/, true /*single*/);
 5327   %}
 5328   ins_pipe( pipe_slow );
 5329 %}
 5330 
 5331 // max = java.lang.Math.max(double a, double b)
 5332 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5333   predicate(UseAVX > 0 && !n->is_reduction());
 5334   match(Set dst (MaxD a b));
 5335   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5336   format %{
 5337      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5338      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5339      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5340      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5341      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5342   %}
 5343   ins_encode %{
 5344     int vector_len = Assembler::AVX_128bit;
 5345     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5346     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5347     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5348     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5349     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5350   %}
 5351   ins_pipe( pipe_slow );
 5352 %}
 5353 
 5354 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5355   predicate(UseAVX > 0 && n->is_reduction());
 5356   match(Set dst (MaxD a b));
 5357   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5358 
 5359   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5360   ins_encode %{
 5361     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5362                     false /*min*/, false /*single*/);
 5363   %}
 5364   ins_pipe( pipe_slow );
 5365 %}
 5366 
 5367 // min = java.lang.Math.min(float a, float b)
 5368 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5369   predicate(UseAVX > 0 && !n->is_reduction());
 5370   match(Set dst (MinF a b));
 5371   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5372   format %{
 5373      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5374      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5375      "vminss           $tmp,$atmp,$btmp           \n\t"
 5376      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5377      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5378   %}
 5379   ins_encode %{
 5380     int vector_len = Assembler::AVX_128bit;
 5381     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5382     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5383     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5384     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5385     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5386   %}
 5387   ins_pipe( pipe_slow );
 5388 %}
 5389 
 5390 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5391   predicate(UseAVX > 0 && n->is_reduction());
 5392   match(Set dst (MinF a b));
 5393   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5394 
 5395   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5396   ins_encode %{
 5397     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5398                     true /*min*/, true /*single*/);
 5399   %}
 5400   ins_pipe( pipe_slow );
 5401 %}
 5402 
 5403 // min = java.lang.Math.min(double a, double b)
 5404 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5405   predicate(UseAVX > 0 && !n->is_reduction());
 5406   match(Set dst (MinD a b));
 5407   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5408   format %{
 5409      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5410      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5411      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5412      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5413      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5414   %}
 5415   ins_encode %{
 5416     int vector_len = Assembler::AVX_128bit;
 5417     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5418     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5419     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5420     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5421     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5422   %}
 5423   ins_pipe( pipe_slow );
 5424 %}
 5425 
 5426 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5427   predicate(UseAVX > 0 && n->is_reduction());
 5428   match(Set dst (MinD a b));
 5429   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5430 
 5431   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5432   ins_encode %{
 5433     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5434                     true /*min*/, false /*single*/);
 5435   %}
 5436   ins_pipe( pipe_slow );
 5437 %}
 5438 
 5439 // Load Effective Address
 5440 instruct leaP8(rRegP dst, indOffset8 mem)
 5441 %{
 5442   match(Set dst mem);
 5443 
 5444   ins_cost(110); // XXX
 5445   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5446   ins_encode %{
 5447     __ leaq($dst$$Register, $mem$$Address);
 5448   %}
 5449   ins_pipe(ialu_reg_reg_fat);
 5450 %}
 5451 
 5452 instruct leaP32(rRegP dst, indOffset32 mem)
 5453 %{
 5454   match(Set dst mem);
 5455 
 5456   ins_cost(110);
 5457   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5458   ins_encode %{
 5459     __ leaq($dst$$Register, $mem$$Address);
 5460   %}
 5461   ins_pipe(ialu_reg_reg_fat);
 5462 %}
 5463 
 5464 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5465 %{
 5466   match(Set dst mem);
 5467 
 5468   ins_cost(110);
 5469   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5470   ins_encode %{
 5471     __ leaq($dst$$Register, $mem$$Address);
 5472   %}
 5473   ins_pipe(ialu_reg_reg_fat);
 5474 %}
 5475 
 5476 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5477 %{
 5478   match(Set dst mem);
 5479 
 5480   ins_cost(110);
 5481   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5482   ins_encode %{
 5483     __ leaq($dst$$Register, $mem$$Address);
 5484   %}
 5485   ins_pipe(ialu_reg_reg_fat);
 5486 %}
 5487 
 5488 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5489 %{
 5490   match(Set dst mem);
 5491 
 5492   ins_cost(110);
 5493   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5494   ins_encode %{
 5495     __ leaq($dst$$Register, $mem$$Address);
 5496   %}
 5497   ins_pipe(ialu_reg_reg_fat);
 5498 %}
 5499 
 5500 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5501 %{
 5502   match(Set dst mem);
 5503 
 5504   ins_cost(110);
 5505   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5506   ins_encode %{
 5507     __ leaq($dst$$Register, $mem$$Address);
 5508   %}
 5509   ins_pipe(ialu_reg_reg_fat);
 5510 %}
 5511 
 5512 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5513 %{
 5514   match(Set dst mem);
 5515 
 5516   ins_cost(110);
 5517   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5518   ins_encode %{
 5519     __ leaq($dst$$Register, $mem$$Address);
 5520   %}
 5521   ins_pipe(ialu_reg_reg_fat);
 5522 %}
 5523 
 5524 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5525 %{
 5526   match(Set dst mem);
 5527 
 5528   ins_cost(110);
 5529   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5530   ins_encode %{
 5531     __ leaq($dst$$Register, $mem$$Address);
 5532   %}
 5533   ins_pipe(ialu_reg_reg_fat);
 5534 %}
 5535 
 5536 // Load Effective Address which uses Narrow (32-bits) oop
 5537 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5538 %{
 5539   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5540   match(Set dst mem);
 5541 
 5542   ins_cost(110);
 5543   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5544   ins_encode %{
 5545     __ leaq($dst$$Register, $mem$$Address);
 5546   %}
 5547   ins_pipe(ialu_reg_reg_fat);
 5548 %}
 5549 
 5550 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5551 %{
 5552   predicate(CompressedOops::shift() == 0);
 5553   match(Set dst mem);
 5554 
 5555   ins_cost(110); // XXX
 5556   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5557   ins_encode %{
 5558     __ leaq($dst$$Register, $mem$$Address);
 5559   %}
 5560   ins_pipe(ialu_reg_reg_fat);
 5561 %}
 5562 
 5563 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5564 %{
 5565   predicate(CompressedOops::shift() == 0);
 5566   match(Set dst mem);
 5567 
 5568   ins_cost(110);
 5569   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5570   ins_encode %{
 5571     __ leaq($dst$$Register, $mem$$Address);
 5572   %}
 5573   ins_pipe(ialu_reg_reg_fat);
 5574 %}
 5575 
 5576 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5577 %{
 5578   predicate(CompressedOops::shift() == 0);
 5579   match(Set dst mem);
 5580 
 5581   ins_cost(110);
 5582   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5583   ins_encode %{
 5584     __ leaq($dst$$Register, $mem$$Address);
 5585   %}
 5586   ins_pipe(ialu_reg_reg_fat);
 5587 %}
 5588 
 5589 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5590 %{
 5591   predicate(CompressedOops::shift() == 0);
 5592   match(Set dst mem);
 5593 
 5594   ins_cost(110);
 5595   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5596   ins_encode %{
 5597     __ leaq($dst$$Register, $mem$$Address);
 5598   %}
 5599   ins_pipe(ialu_reg_reg_fat);
 5600 %}
 5601 
 5602 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5603 %{
 5604   predicate(CompressedOops::shift() == 0);
 5605   match(Set dst mem);
 5606 
 5607   ins_cost(110);
 5608   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5609   ins_encode %{
 5610     __ leaq($dst$$Register, $mem$$Address);
 5611   %}
 5612   ins_pipe(ialu_reg_reg_fat);
 5613 %}
 5614 
 5615 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5616 %{
 5617   predicate(CompressedOops::shift() == 0);
 5618   match(Set dst mem);
 5619 
 5620   ins_cost(110);
 5621   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5622   ins_encode %{
 5623     __ leaq($dst$$Register, $mem$$Address);
 5624   %}
 5625   ins_pipe(ialu_reg_reg_fat);
 5626 %}
 5627 
 5628 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5629 %{
 5630   predicate(CompressedOops::shift() == 0);
 5631   match(Set dst mem);
 5632 
 5633   ins_cost(110);
 5634   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5635   ins_encode %{
 5636     __ leaq($dst$$Register, $mem$$Address);
 5637   %}
 5638   ins_pipe(ialu_reg_reg_fat);
 5639 %}
 5640 
 5641 instruct loadConI(rRegI dst, immI src)
 5642 %{
 5643   match(Set dst src);
 5644 
 5645   format %{ "movl    $dst, $src\t# int" %}
 5646   ins_encode %{
 5647     __ movl($dst$$Register, $src$$constant);
 5648   %}
 5649   ins_pipe(ialu_reg_fat); // XXX
 5650 %}
 5651 
 5652 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5653 %{
 5654   match(Set dst src);
 5655   effect(KILL cr);
 5656 
 5657   ins_cost(50);
 5658   format %{ "xorl    $dst, $dst\t# int" %}
 5659   ins_encode %{
 5660     __ xorl($dst$$Register, $dst$$Register);
 5661   %}
 5662   ins_pipe(ialu_reg);
 5663 %}
 5664 
 5665 instruct loadConL(rRegL dst, immL src)
 5666 %{
 5667   match(Set dst src);
 5668 
 5669   ins_cost(150);
 5670   format %{ "movq    $dst, $src\t# long" %}
 5671   ins_encode %{
 5672     __ mov64($dst$$Register, $src$$constant);
 5673   %}
 5674   ins_pipe(ialu_reg);
 5675 %}
 5676 
 5677 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5678 %{
 5679   match(Set dst src);
 5680   effect(KILL cr);
 5681 
 5682   ins_cost(50);
 5683   format %{ "xorl    $dst, $dst\t# long" %}
 5684   ins_encode %{
 5685     __ xorl($dst$$Register, $dst$$Register);
 5686   %}
 5687   ins_pipe(ialu_reg); // XXX
 5688 %}
 5689 
 5690 instruct loadConUL32(rRegL dst, immUL32 src)
 5691 %{
 5692   match(Set dst src);
 5693 
 5694   ins_cost(60);
 5695   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5696   ins_encode %{
 5697     __ movl($dst$$Register, $src$$constant);
 5698   %}
 5699   ins_pipe(ialu_reg);
 5700 %}
 5701 
 5702 instruct loadConL32(rRegL dst, immL32 src)
 5703 %{
 5704   match(Set dst src);
 5705 
 5706   ins_cost(70);
 5707   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5708   ins_encode %{
 5709     __ movq($dst$$Register, $src$$constant);
 5710   %}
 5711   ins_pipe(ialu_reg);
 5712 %}
 5713 
 5714 instruct loadConP(rRegP dst, immP con) %{
 5715   match(Set dst con);
 5716 
 5717   format %{ "movq    $dst, $con\t# ptr" %}
 5718   ins_encode %{
 5719     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5720   %}
 5721   ins_pipe(ialu_reg_fat); // XXX
 5722 %}
 5723 
 5724 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5725 %{
 5726   match(Set dst src);
 5727   effect(KILL cr);
 5728 
 5729   ins_cost(50);
 5730   format %{ "xorl    $dst, $dst\t# ptr" %}
 5731   ins_encode %{
 5732     __ xorl($dst$$Register, $dst$$Register);
 5733   %}
 5734   ins_pipe(ialu_reg);
 5735 %}
 5736 
 5737 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5738 %{
 5739   match(Set dst src);
 5740   effect(KILL cr);
 5741 
 5742   ins_cost(60);
 5743   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5744   ins_encode %{
 5745     __ movl($dst$$Register, $src$$constant);
 5746   %}
 5747   ins_pipe(ialu_reg);
 5748 %}
 5749 
 5750 instruct loadConF(regF dst, immF con) %{
 5751   match(Set dst con);
 5752   ins_cost(125);
 5753   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5754   ins_encode %{
 5755     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5756   %}
 5757   ins_pipe(pipe_slow);
 5758 %}
 5759 
 5760 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5761   match(Set dst src);
 5762   effect(KILL cr);
 5763   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5764   ins_encode %{
 5765     __ xorq($dst$$Register, $dst$$Register);
 5766   %}
 5767   ins_pipe(ialu_reg);
 5768 %}
 5769 
 5770 instruct loadConN(rRegN dst, immN src) %{
 5771   match(Set dst src);
 5772 
 5773   ins_cost(125);
 5774   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5775   ins_encode %{
 5776     address con = (address)$src$$constant;
 5777     if (con == NULL) {
 5778       ShouldNotReachHere();
 5779     } else {
 5780       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5781     }
 5782   %}
 5783   ins_pipe(ialu_reg_fat); // XXX
 5784 %}
 5785 
 5786 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5787   match(Set dst src);
 5788 
 5789   ins_cost(125);
 5790   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5791   ins_encode %{
 5792     address con = (address)$src$$constant;
 5793     if (con == NULL) {
 5794       ShouldNotReachHere();
 5795     } else {
 5796       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5797     }
 5798   %}
 5799   ins_pipe(ialu_reg_fat); // XXX
 5800 %}
 5801 
 5802 instruct loadConF0(regF dst, immF0 src)
 5803 %{
 5804   match(Set dst src);
 5805   ins_cost(100);
 5806 
 5807   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5808   ins_encode %{
 5809     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5810   %}
 5811   ins_pipe(pipe_slow);
 5812 %}
 5813 
 5814 // Use the same format since predicate() can not be used here.
 5815 instruct loadConD(regD dst, immD con) %{
 5816   match(Set dst con);
 5817   ins_cost(125);
 5818   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5819   ins_encode %{
 5820     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5821   %}
 5822   ins_pipe(pipe_slow);
 5823 %}
 5824 
 5825 instruct loadConD0(regD dst, immD0 src)
 5826 %{
 5827   match(Set dst src);
 5828   ins_cost(100);
 5829 
 5830   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5831   ins_encode %{
 5832     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5833   %}
 5834   ins_pipe(pipe_slow);
 5835 %}
 5836 
 5837 instruct loadSSI(rRegI dst, stackSlotI src)
 5838 %{
 5839   match(Set dst src);
 5840 
 5841   ins_cost(125);
 5842   format %{ "movl    $dst, $src\t# int stk" %}
 5843   opcode(0x8B);
 5844   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5845   ins_pipe(ialu_reg_mem);
 5846 %}
 5847 
 5848 instruct loadSSL(rRegL dst, stackSlotL src)
 5849 %{
 5850   match(Set dst src);
 5851 
 5852   ins_cost(125);
 5853   format %{ "movq    $dst, $src\t# long stk" %}
 5854   opcode(0x8B);
 5855   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5856   ins_pipe(ialu_reg_mem);
 5857 %}
 5858 
 5859 instruct loadSSP(rRegP dst, stackSlotP src)
 5860 %{
 5861   match(Set dst src);
 5862 
 5863   ins_cost(125);
 5864   format %{ "movq    $dst, $src\t# ptr stk" %}
 5865   opcode(0x8B);
 5866   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5867   ins_pipe(ialu_reg_mem);
 5868 %}
 5869 
 5870 instruct loadSSF(regF dst, stackSlotF src)
 5871 %{
 5872   match(Set dst src);
 5873 
 5874   ins_cost(125);
 5875   format %{ "movss   $dst, $src\t# float stk" %}
 5876   ins_encode %{
 5877     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5878   %}
 5879   ins_pipe(pipe_slow); // XXX
 5880 %}
 5881 
 5882 // Use the same format since predicate() can not be used here.
 5883 instruct loadSSD(regD dst, stackSlotD src)
 5884 %{
 5885   match(Set dst src);
 5886 
 5887   ins_cost(125);
 5888   format %{ "movsd   $dst, $src\t# double stk" %}
 5889   ins_encode  %{
 5890     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5891   %}
 5892   ins_pipe(pipe_slow); // XXX
 5893 %}
 5894 
 5895 // Prefetch instructions for allocation.
 5896 // Must be safe to execute with invalid address (cannot fault).
 5897 
 5898 instruct prefetchAlloc( memory mem ) %{
 5899   predicate(AllocatePrefetchInstr==3);
 5900   match(PrefetchAllocation mem);
 5901   ins_cost(125);
 5902 
 5903   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5904   ins_encode %{
 5905     __ prefetchw($mem$$Address);
 5906   %}
 5907   ins_pipe(ialu_mem);
 5908 %}
 5909 
 5910 instruct prefetchAllocNTA( memory mem ) %{
 5911   predicate(AllocatePrefetchInstr==0);
 5912   match(PrefetchAllocation mem);
 5913   ins_cost(125);
 5914 
 5915   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5916   ins_encode %{
 5917     __ prefetchnta($mem$$Address);
 5918   %}
 5919   ins_pipe(ialu_mem);
 5920 %}
 5921 
 5922 instruct prefetchAllocT0( memory mem ) %{
 5923   predicate(AllocatePrefetchInstr==1);
 5924   match(PrefetchAllocation mem);
 5925   ins_cost(125);
 5926 
 5927   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5928   ins_encode %{
 5929     __ prefetcht0($mem$$Address);
 5930   %}
 5931   ins_pipe(ialu_mem);
 5932 %}
 5933 
 5934 instruct prefetchAllocT2( memory mem ) %{
 5935   predicate(AllocatePrefetchInstr==2);
 5936   match(PrefetchAllocation mem);
 5937   ins_cost(125);
 5938 
 5939   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5940   ins_encode %{
 5941     __ prefetcht2($mem$$Address);
 5942   %}
 5943   ins_pipe(ialu_mem);
 5944 %}
 5945 
 5946 //----------Store Instructions-------------------------------------------------
 5947 
 5948 // Store Byte
 5949 instruct storeB(memory mem, rRegI src)
 5950 %{
 5951   match(Set mem (StoreB mem src));
 5952 
 5953   ins_cost(125); // XXX
 5954   format %{ "movb    $mem, $src\t# byte" %}
 5955   ins_encode %{
 5956     __ movb($mem$$Address, $src$$Register);
 5957   %}
 5958   ins_pipe(ialu_mem_reg);
 5959 %}
 5960 
 5961 // Store Char/Short
 5962 instruct storeC(memory mem, rRegI src)
 5963 %{
 5964   match(Set mem (StoreC mem src));
 5965 
 5966   ins_cost(125); // XXX
 5967   format %{ "movw    $mem, $src\t# char/short" %}
 5968   ins_encode %{
 5969     __ movw($mem$$Address, $src$$Register);
 5970   %}
 5971   ins_pipe(ialu_mem_reg);
 5972 %}
 5973 
 5974 // Store Integer
 5975 instruct storeI(memory mem, rRegI src)
 5976 %{
 5977   match(Set mem (StoreI mem src));
 5978 
 5979   ins_cost(125); // XXX
 5980   format %{ "movl    $mem, $src\t# int" %}
 5981   ins_encode %{
 5982     __ movl($mem$$Address, $src$$Register);
 5983   %}
 5984   ins_pipe(ialu_mem_reg);
 5985 %}
 5986 
 5987 // Store Long
 5988 instruct storeL(memory mem, rRegL src)
 5989 %{
 5990   match(Set mem (StoreL mem src));
 5991 
 5992   ins_cost(125); // XXX
 5993   format %{ "movq    $mem, $src\t# long" %}
 5994   ins_encode %{
 5995     __ movq($mem$$Address, $src$$Register);
 5996   %}
 5997   ins_pipe(ialu_mem_reg); // XXX
 5998 %}
 5999 
 6000 // Store Pointer
 6001 instruct storeP(memory mem, any_RegP src)
 6002 %{
 6003   match(Set mem (StoreP mem src));
 6004 
 6005   ins_cost(125); // XXX
 6006   format %{ "movq    $mem, $src\t# ptr" %}
 6007   ins_encode %{
 6008     __ movq($mem$$Address, $src$$Register);
 6009   %}
 6010   ins_pipe(ialu_mem_reg);
 6011 %}
 6012 
 6013 instruct storeImmP0(memory mem, immP0 zero)
 6014 %{
 6015   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6016   match(Set mem (StoreP mem zero));
 6017 
 6018   ins_cost(125); // XXX
 6019   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6020   ins_encode %{
 6021     __ movq($mem$$Address, r12);
 6022   %}
 6023   ins_pipe(ialu_mem_reg);
 6024 %}
 6025 
 6026 // Store NULL Pointer, mark word, or other simple pointer constant.
 6027 instruct storeImmP(memory mem, immP31 src)
 6028 %{
 6029   match(Set mem (StoreP mem src));
 6030 
 6031   ins_cost(150); // XXX
 6032   format %{ "movq    $mem, $src\t# ptr" %}
 6033   ins_encode %{
 6034     __ movq($mem$$Address, $src$$constant);
 6035   %}
 6036   ins_pipe(ialu_mem_imm);
 6037 %}
 6038 
 6039 // Store Compressed Pointer
 6040 instruct storeN(memory mem, rRegN src)
 6041 %{
 6042   match(Set mem (StoreN mem src));
 6043 
 6044   ins_cost(125); // XXX
 6045   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6046   ins_encode %{
 6047     __ movl($mem$$Address, $src$$Register);
 6048   %}
 6049   ins_pipe(ialu_mem_reg);
 6050 %}
 6051 
 6052 instruct storeNKlass(memory mem, rRegN src)
 6053 %{
 6054   match(Set mem (StoreNKlass mem src));
 6055 
 6056   ins_cost(125); // XXX
 6057   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6058   ins_encode %{
 6059     __ movl($mem$$Address, $src$$Register);
 6060   %}
 6061   ins_pipe(ialu_mem_reg);
 6062 %}
 6063 
 6064 instruct storeImmN0(memory mem, immN0 zero)
 6065 %{
 6066   predicate(CompressedOops::base() == NULL);
 6067   match(Set mem (StoreN mem zero));
 6068 
 6069   ins_cost(125); // XXX
 6070   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6071   ins_encode %{
 6072     __ movl($mem$$Address, r12);
 6073   %}
 6074   ins_pipe(ialu_mem_reg);
 6075 %}
 6076 
 6077 instruct storeImmN(memory mem, immN src)
 6078 %{
 6079   match(Set mem (StoreN mem src));
 6080 
 6081   ins_cost(150); // XXX
 6082   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6083   ins_encode %{
 6084     address con = (address)$src$$constant;
 6085     if (con == NULL) {
 6086       __ movl($mem$$Address, (int32_t)0);
 6087     } else {
 6088       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6089     }
 6090   %}
 6091   ins_pipe(ialu_mem_imm);
 6092 %}
 6093 
 6094 instruct storeImmNKlass(memory mem, immNKlass src)
 6095 %{
 6096   match(Set mem (StoreNKlass mem src));
 6097 
 6098   ins_cost(150); // XXX
 6099   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6100   ins_encode %{
 6101     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6102   %}
 6103   ins_pipe(ialu_mem_imm);
 6104 %}
 6105 
 6106 // Store Integer Immediate
 6107 instruct storeImmI0(memory mem, immI_0 zero)
 6108 %{
 6109   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6110   match(Set mem (StoreI mem zero));
 6111 
 6112   ins_cost(125); // XXX
 6113   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6114   ins_encode %{
 6115     __ movl($mem$$Address, r12);
 6116   %}
 6117   ins_pipe(ialu_mem_reg);
 6118 %}
 6119 
 6120 instruct storeImmI(memory mem, immI src)
 6121 %{
 6122   match(Set mem (StoreI mem src));
 6123 
 6124   ins_cost(150);
 6125   format %{ "movl    $mem, $src\t# int" %}
 6126   ins_encode %{
 6127     __ movl($mem$$Address, $src$$constant);
 6128   %}
 6129   ins_pipe(ialu_mem_imm);
 6130 %}
 6131 
 6132 // Store Long Immediate
 6133 instruct storeImmL0(memory mem, immL0 zero)
 6134 %{
 6135   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6136   match(Set mem (StoreL mem zero));
 6137 
 6138   ins_cost(125); // XXX
 6139   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6140   ins_encode %{
 6141     __ movq($mem$$Address, r12);
 6142   %}
 6143   ins_pipe(ialu_mem_reg);
 6144 %}
 6145 
 6146 instruct storeImmL(memory mem, immL32 src)
 6147 %{
 6148   match(Set mem (StoreL mem src));
 6149 
 6150   ins_cost(150);
 6151   format %{ "movq    $mem, $src\t# long" %}
 6152   ins_encode %{
 6153     __ movq($mem$$Address, $src$$constant);
 6154   %}
 6155   ins_pipe(ialu_mem_imm);
 6156 %}
 6157 
 6158 // Store Short/Char Immediate
 6159 instruct storeImmC0(memory mem, immI_0 zero)
 6160 %{
 6161   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6162   match(Set mem (StoreC mem zero));
 6163 
 6164   ins_cost(125); // XXX
 6165   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6166   ins_encode %{
 6167     __ movw($mem$$Address, r12);
 6168   %}
 6169   ins_pipe(ialu_mem_reg);
 6170 %}
 6171 
 6172 instruct storeImmI16(memory mem, immI16 src)
 6173 %{
 6174   predicate(UseStoreImmI16);
 6175   match(Set mem (StoreC mem src));
 6176 
 6177   ins_cost(150);
 6178   format %{ "movw    $mem, $src\t# short/char" %}
 6179   ins_encode %{
 6180     __ movw($mem$$Address, $src$$constant);
 6181   %}
 6182   ins_pipe(ialu_mem_imm);
 6183 %}
 6184 
 6185 // Store Byte Immediate
 6186 instruct storeImmB0(memory mem, immI_0 zero)
 6187 %{
 6188   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6189   match(Set mem (StoreB mem zero));
 6190 
 6191   ins_cost(125); // XXX
 6192   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6193   ins_encode %{
 6194     __ movb($mem$$Address, r12);
 6195   %}
 6196   ins_pipe(ialu_mem_reg);
 6197 %}
 6198 
 6199 instruct storeImmB(memory mem, immI8 src)
 6200 %{
 6201   match(Set mem (StoreB mem src));
 6202 
 6203   ins_cost(150); // XXX
 6204   format %{ "movb    $mem, $src\t# byte" %}
 6205   ins_encode %{
 6206     __ movb($mem$$Address, $src$$constant);
 6207   %}
 6208   ins_pipe(ialu_mem_imm);
 6209 %}
 6210 
 6211 // Store CMS card-mark Immediate
 6212 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6213 %{
 6214   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6215   match(Set mem (StoreCM mem zero));
 6216 
 6217   ins_cost(125); // XXX
 6218   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6219   ins_encode %{
 6220     __ movb($mem$$Address, r12);
 6221   %}
 6222   ins_pipe(ialu_mem_reg);
 6223 %}
 6224 
 6225 instruct storeImmCM0(memory mem, immI_0 src)
 6226 %{
 6227   match(Set mem (StoreCM mem src));
 6228 
 6229   ins_cost(150); // XXX
 6230   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6231   ins_encode %{
 6232     __ movb($mem$$Address, $src$$constant);
 6233   %}
 6234   ins_pipe(ialu_mem_imm);
 6235 %}
 6236 
 6237 // Store Float
 6238 instruct storeF(memory mem, regF src)
 6239 %{
 6240   match(Set mem (StoreF mem src));
 6241 
 6242   ins_cost(95); // XXX
 6243   format %{ "movss   $mem, $src\t# float" %}
 6244   ins_encode %{
 6245     __ movflt($mem$$Address, $src$$XMMRegister);
 6246   %}
 6247   ins_pipe(pipe_slow); // XXX
 6248 %}
 6249 
 6250 // Store immediate Float value (it is faster than store from XMM register)
 6251 instruct storeF0(memory mem, immF0 zero)
 6252 %{
 6253   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6254   match(Set mem (StoreF mem zero));
 6255 
 6256   ins_cost(25); // XXX
 6257   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6258   ins_encode %{
 6259     __ movl($mem$$Address, r12);
 6260   %}
 6261   ins_pipe(ialu_mem_reg);
 6262 %}
 6263 
 6264 instruct storeF_imm(memory mem, immF src)
 6265 %{
 6266   match(Set mem (StoreF mem src));
 6267 
 6268   ins_cost(50);
 6269   format %{ "movl    $mem, $src\t# float" %}
 6270   ins_encode %{
 6271     __ movl($mem$$Address, jint_cast($src$$constant));
 6272   %}
 6273   ins_pipe(ialu_mem_imm);
 6274 %}
 6275 
 6276 // Store Double
 6277 instruct storeD(memory mem, regD src)
 6278 %{
 6279   match(Set mem (StoreD mem src));
 6280 
 6281   ins_cost(95); // XXX
 6282   format %{ "movsd   $mem, $src\t# double" %}
 6283   ins_encode %{
 6284     __ movdbl($mem$$Address, $src$$XMMRegister);
 6285   %}
 6286   ins_pipe(pipe_slow); // XXX
 6287 %}
 6288 
 6289 // Store immediate double 0.0 (it is faster than store from XMM register)
 6290 instruct storeD0_imm(memory mem, immD0 src)
 6291 %{
 6292   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6293   match(Set mem (StoreD mem src));
 6294 
 6295   ins_cost(50);
 6296   format %{ "movq    $mem, $src\t# double 0." %}
 6297   ins_encode %{
 6298     __ movq($mem$$Address, $src$$constant);
 6299   %}
 6300   ins_pipe(ialu_mem_imm);
 6301 %}
 6302 
 6303 instruct storeD0(memory mem, immD0 zero)
 6304 %{
 6305   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6306   match(Set mem (StoreD mem zero));
 6307 
 6308   ins_cost(25); // XXX
 6309   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6310   ins_encode %{
 6311     __ movq($mem$$Address, r12);
 6312   %}
 6313   ins_pipe(ialu_mem_reg);
 6314 %}
 6315 
 6316 instruct storeSSI(stackSlotI dst, rRegI src)
 6317 %{
 6318   match(Set dst src);
 6319 
 6320   ins_cost(100);
 6321   format %{ "movl    $dst, $src\t# int stk" %}
 6322   opcode(0x89);
 6323   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6324   ins_pipe( ialu_mem_reg );
 6325 %}
 6326 
 6327 instruct storeSSL(stackSlotL dst, rRegL src)
 6328 %{
 6329   match(Set dst src);
 6330 
 6331   ins_cost(100);
 6332   format %{ "movq    $dst, $src\t# long stk" %}
 6333   opcode(0x89);
 6334   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6335   ins_pipe(ialu_mem_reg);
 6336 %}
 6337 
 6338 instruct storeSSP(stackSlotP dst, rRegP src)
 6339 %{
 6340   match(Set dst src);
 6341 
 6342   ins_cost(100);
 6343   format %{ "movq    $dst, $src\t# ptr stk" %}
 6344   opcode(0x89);
 6345   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6346   ins_pipe(ialu_mem_reg);
 6347 %}
 6348 
 6349 instruct storeSSF(stackSlotF dst, regF src)
 6350 %{
 6351   match(Set dst src);
 6352 
 6353   ins_cost(95); // XXX
 6354   format %{ "movss   $dst, $src\t# float stk" %}
 6355   ins_encode %{
 6356     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6357   %}
 6358   ins_pipe(pipe_slow); // XXX
 6359 %}
 6360 
 6361 instruct storeSSD(stackSlotD dst, regD src)
 6362 %{
 6363   match(Set dst src);
 6364 
 6365   ins_cost(95); // XXX
 6366   format %{ "movsd   $dst, $src\t# double stk" %}
 6367   ins_encode %{
 6368     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6369   %}
 6370   ins_pipe(pipe_slow); // XXX
 6371 %}
 6372 
 6373 instruct cacheWB(indirect addr)
 6374 %{
 6375   predicate(VM_Version::supports_data_cache_line_flush());
 6376   match(CacheWB addr);
 6377 
 6378   ins_cost(100);
 6379   format %{"cache wb $addr" %}
 6380   ins_encode %{
 6381     assert($addr->index_position() < 0, "should be");
 6382     assert($addr$$disp == 0, "should be");
 6383     __ cache_wb(Address($addr$$base$$Register, 0));
 6384   %}
 6385   ins_pipe(pipe_slow); // XXX
 6386 %}
 6387 
 6388 instruct cacheWBPreSync()
 6389 %{
 6390   predicate(VM_Version::supports_data_cache_line_flush());
 6391   match(CacheWBPreSync);
 6392 
 6393   ins_cost(100);
 6394   format %{"cache wb presync" %}
 6395   ins_encode %{
 6396     __ cache_wbsync(true);
 6397   %}
 6398   ins_pipe(pipe_slow); // XXX
 6399 %}
 6400 
 6401 instruct cacheWBPostSync()
 6402 %{
 6403   predicate(VM_Version::supports_data_cache_line_flush());
 6404   match(CacheWBPostSync);
 6405 
 6406   ins_cost(100);
 6407   format %{"cache wb postsync" %}
 6408   ins_encode %{
 6409     __ cache_wbsync(false);
 6410   %}
 6411   ins_pipe(pipe_slow); // XXX
 6412 %}
 6413 
 6414 //----------BSWAP Instructions-------------------------------------------------
 6415 instruct bytes_reverse_int(rRegI dst) %{
 6416   match(Set dst (ReverseBytesI dst));
 6417 
 6418   format %{ "bswapl  $dst" %}
 6419   ins_encode %{
 6420     __ bswapl($dst$$Register);
 6421   %}
 6422   ins_pipe( ialu_reg );
 6423 %}
 6424 
 6425 instruct bytes_reverse_long(rRegL dst) %{
 6426   match(Set dst (ReverseBytesL dst));
 6427 
 6428   format %{ "bswapq  $dst" %}
 6429   ins_encode %{
 6430     __ bswapq($dst$$Register);
 6431   %}
 6432   ins_pipe( ialu_reg);
 6433 %}
 6434 
 6435 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6436   match(Set dst (ReverseBytesUS dst));
 6437   effect(KILL cr);
 6438 
 6439   format %{ "bswapl  $dst\n\t"
 6440             "shrl    $dst,16\n\t" %}
 6441   ins_encode %{
 6442     __ bswapl($dst$$Register);
 6443     __ shrl($dst$$Register, 16);
 6444   %}
 6445   ins_pipe( ialu_reg );
 6446 %}
 6447 
 6448 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6449   match(Set dst (ReverseBytesS dst));
 6450   effect(KILL cr);
 6451 
 6452   format %{ "bswapl  $dst\n\t"
 6453             "sar     $dst,16\n\t" %}
 6454   ins_encode %{
 6455     __ bswapl($dst$$Register);
 6456     __ sarl($dst$$Register, 16);
 6457   %}
 6458   ins_pipe( ialu_reg );
 6459 %}
 6460 
 6461 //---------- Zeros Count Instructions ------------------------------------------
 6462 
 6463 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6464   predicate(UseCountLeadingZerosInstruction);
 6465   match(Set dst (CountLeadingZerosI src));
 6466   effect(KILL cr);
 6467 
 6468   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6469   ins_encode %{
 6470     __ lzcntl($dst$$Register, $src$$Register);
 6471   %}
 6472   ins_pipe(ialu_reg);
 6473 %}
 6474 
 6475 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6476   predicate(!UseCountLeadingZerosInstruction);
 6477   match(Set dst (CountLeadingZerosI src));
 6478   effect(KILL cr);
 6479 
 6480   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6481             "jnz     skip\n\t"
 6482             "movl    $dst, -1\n"
 6483       "skip:\n\t"
 6484             "negl    $dst\n\t"
 6485             "addl    $dst, 31" %}
 6486   ins_encode %{
 6487     Register Rdst = $dst$$Register;
 6488     Register Rsrc = $src$$Register;
 6489     Label skip;
 6490     __ bsrl(Rdst, Rsrc);
 6491     __ jccb(Assembler::notZero, skip);
 6492     __ movl(Rdst, -1);
 6493     __ bind(skip);
 6494     __ negl(Rdst);
 6495     __ addl(Rdst, BitsPerInt - 1);
 6496   %}
 6497   ins_pipe(ialu_reg);
 6498 %}
 6499 
 6500 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6501   predicate(UseCountLeadingZerosInstruction);
 6502   match(Set dst (CountLeadingZerosL src));
 6503   effect(KILL cr);
 6504 
 6505   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6506   ins_encode %{
 6507     __ lzcntq($dst$$Register, $src$$Register);
 6508   %}
 6509   ins_pipe(ialu_reg);
 6510 %}
 6511 
 6512 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6513   predicate(!UseCountLeadingZerosInstruction);
 6514   match(Set dst (CountLeadingZerosL src));
 6515   effect(KILL cr);
 6516 
 6517   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6518             "jnz     skip\n\t"
 6519             "movl    $dst, -1\n"
 6520       "skip:\n\t"
 6521             "negl    $dst\n\t"
 6522             "addl    $dst, 63" %}
 6523   ins_encode %{
 6524     Register Rdst = $dst$$Register;
 6525     Register Rsrc = $src$$Register;
 6526     Label skip;
 6527     __ bsrq(Rdst, Rsrc);
 6528     __ jccb(Assembler::notZero, skip);
 6529     __ movl(Rdst, -1);
 6530     __ bind(skip);
 6531     __ negl(Rdst);
 6532     __ addl(Rdst, BitsPerLong - 1);
 6533   %}
 6534   ins_pipe(ialu_reg);
 6535 %}
 6536 
 6537 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6538   predicate(UseCountTrailingZerosInstruction);
 6539   match(Set dst (CountTrailingZerosI src));
 6540   effect(KILL cr);
 6541 
 6542   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6543   ins_encode %{
 6544     __ tzcntl($dst$$Register, $src$$Register);
 6545   %}
 6546   ins_pipe(ialu_reg);
 6547 %}
 6548 
 6549 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6550   predicate(!UseCountTrailingZerosInstruction);
 6551   match(Set dst (CountTrailingZerosI src));
 6552   effect(KILL cr);
 6553 
 6554   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6555             "jnz     done\n\t"
 6556             "movl    $dst, 32\n"
 6557       "done:" %}
 6558   ins_encode %{
 6559     Register Rdst = $dst$$Register;
 6560     Label done;
 6561     __ bsfl(Rdst, $src$$Register);
 6562     __ jccb(Assembler::notZero, done);
 6563     __ movl(Rdst, BitsPerInt);
 6564     __ bind(done);
 6565   %}
 6566   ins_pipe(ialu_reg);
 6567 %}
 6568 
 6569 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6570   predicate(UseCountTrailingZerosInstruction);
 6571   match(Set dst (CountTrailingZerosL src));
 6572   effect(KILL cr);
 6573 
 6574   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6575   ins_encode %{
 6576     __ tzcntq($dst$$Register, $src$$Register);
 6577   %}
 6578   ins_pipe(ialu_reg);
 6579 %}
 6580 
 6581 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6582   predicate(!UseCountTrailingZerosInstruction);
 6583   match(Set dst (CountTrailingZerosL src));
 6584   effect(KILL cr);
 6585 
 6586   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6587             "jnz     done\n\t"
 6588             "movl    $dst, 64\n"
 6589       "done:" %}
 6590   ins_encode %{
 6591     Register Rdst = $dst$$Register;
 6592     Label done;
 6593     __ bsfq(Rdst, $src$$Register);
 6594     __ jccb(Assembler::notZero, done);
 6595     __ movl(Rdst, BitsPerLong);
 6596     __ bind(done);
 6597   %}
 6598   ins_pipe(ialu_reg);
 6599 %}
 6600 
 6601 
 6602 //---------- Population Count Instructions -------------------------------------
 6603 
 6604 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6605   predicate(UsePopCountInstruction);
 6606   match(Set dst (PopCountI src));
 6607   effect(KILL cr);
 6608 
 6609   format %{ "popcnt  $dst, $src" %}
 6610   ins_encode %{
 6611     __ popcntl($dst$$Register, $src$$Register);
 6612   %}
 6613   ins_pipe(ialu_reg);
 6614 %}
 6615 
 6616 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6617   predicate(UsePopCountInstruction);
 6618   match(Set dst (PopCountI (LoadI mem)));
 6619   effect(KILL cr);
 6620 
 6621   format %{ "popcnt  $dst, $mem" %}
 6622   ins_encode %{
 6623     __ popcntl($dst$$Register, $mem$$Address);
 6624   %}
 6625   ins_pipe(ialu_reg);
 6626 %}
 6627 
 6628 // Note: Long.bitCount(long) returns an int.
 6629 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6630   predicate(UsePopCountInstruction);
 6631   match(Set dst (PopCountL src));
 6632   effect(KILL cr);
 6633 
 6634   format %{ "popcnt  $dst, $src" %}
 6635   ins_encode %{
 6636     __ popcntq($dst$$Register, $src$$Register);
 6637   %}
 6638   ins_pipe(ialu_reg);
 6639 %}
 6640 
 6641 // Note: Long.bitCount(long) returns an int.
 6642 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6643   predicate(UsePopCountInstruction);
 6644   match(Set dst (PopCountL (LoadL mem)));
 6645   effect(KILL cr);
 6646 
 6647   format %{ "popcnt  $dst, $mem" %}
 6648   ins_encode %{
 6649     __ popcntq($dst$$Register, $mem$$Address);
 6650   %}
 6651   ins_pipe(ialu_reg);
 6652 %}
 6653 
 6654 
 6655 //----------MemBar Instructions-----------------------------------------------
 6656 // Memory barrier flavors
 6657 
 6658 instruct membar_acquire()
 6659 %{
 6660   match(MemBarAcquire);
 6661   match(LoadFence);
 6662   ins_cost(0);
 6663 
 6664   size(0);
 6665   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6666   ins_encode();
 6667   ins_pipe(empty);
 6668 %}
 6669 
 6670 instruct membar_acquire_lock()
 6671 %{
 6672   match(MemBarAcquireLock);
 6673   ins_cost(0);
 6674 
 6675   size(0);
 6676   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6677   ins_encode();
 6678   ins_pipe(empty);
 6679 %}
 6680 
 6681 instruct membar_release()
 6682 %{
 6683   match(MemBarRelease);
 6684   match(StoreFence);
 6685   ins_cost(0);
 6686 
 6687   size(0);
 6688   format %{ "MEMBAR-release ! (empty encoding)" %}
 6689   ins_encode();
 6690   ins_pipe(empty);
 6691 %}
 6692 
 6693 instruct membar_release_lock()
 6694 %{
 6695   match(MemBarReleaseLock);
 6696   ins_cost(0);
 6697 
 6698   size(0);
 6699   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6700   ins_encode();
 6701   ins_pipe(empty);
 6702 %}
 6703 
 6704 instruct membar_volatile(rFlagsReg cr) %{
 6705   match(MemBarVolatile);
 6706   effect(KILL cr);
 6707   ins_cost(400);
 6708 
 6709   format %{
 6710     $$template
 6711     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6712   %}
 6713   ins_encode %{
 6714     __ membar(Assembler::StoreLoad);
 6715   %}
 6716   ins_pipe(pipe_slow);
 6717 %}
 6718 
 6719 instruct unnecessary_membar_volatile()
 6720 %{
 6721   match(MemBarVolatile);
 6722   predicate(Matcher::post_store_load_barrier(n));
 6723   ins_cost(0);
 6724 
 6725   size(0);
 6726   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6727   ins_encode();
 6728   ins_pipe(empty);
 6729 %}
 6730 
 6731 instruct membar_storestore() %{
 6732   match(MemBarStoreStore);
 6733   match(StoreStoreFence);
 6734   ins_cost(0);
 6735 
 6736   size(0);
 6737   format %{ "MEMBAR-storestore (empty encoding)" %}
 6738   ins_encode( );
 6739   ins_pipe(empty);
 6740 %}
 6741 
 6742 //----------Move Instructions--------------------------------------------------
 6743 
 6744 instruct castX2P(rRegP dst, rRegL src)
 6745 %{
 6746   match(Set dst (CastX2P src));
 6747 
 6748   format %{ "movq    $dst, $src\t# long->ptr" %}
 6749   ins_encode %{
 6750     if ($dst$$reg != $src$$reg) {
 6751       __ movptr($dst$$Register, $src$$Register);
 6752     }
 6753   %}
 6754   ins_pipe(ialu_reg_reg); // XXX
 6755 %}
 6756 
 6757 instruct castP2X(rRegL dst, rRegP src)
 6758 %{
 6759   match(Set dst (CastP2X src));
 6760 
 6761   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6762   ins_encode %{
 6763     if ($dst$$reg != $src$$reg) {
 6764       __ movptr($dst$$Register, $src$$Register);
 6765     }
 6766   %}
 6767   ins_pipe(ialu_reg_reg); // XXX
 6768 %}
 6769 
 6770 // Convert oop into int for vectors alignment masking
 6771 instruct convP2I(rRegI dst, rRegP src)
 6772 %{
 6773   match(Set dst (ConvL2I (CastP2X src)));
 6774 
 6775   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6776   ins_encode %{
 6777     __ movl($dst$$Register, $src$$Register);
 6778   %}
 6779   ins_pipe(ialu_reg_reg); // XXX
 6780 %}
 6781 
 6782 // Convert compressed oop into int for vectors alignment masking
 6783 // in case of 32bit oops (heap < 4Gb).
 6784 instruct convN2I(rRegI dst, rRegN src)
 6785 %{
 6786   predicate(CompressedOops::shift() == 0);
 6787   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6788 
 6789   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6790   ins_encode %{
 6791     __ movl($dst$$Register, $src$$Register);
 6792   %}
 6793   ins_pipe(ialu_reg_reg); // XXX
 6794 %}
 6795 
 6796 // Convert oop pointer into compressed form
 6797 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6798   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6799   match(Set dst (EncodeP src));
 6800   effect(KILL cr);
 6801   format %{ "encode_heap_oop $dst,$src" %}
 6802   ins_encode %{
 6803     Register s = $src$$Register;
 6804     Register d = $dst$$Register;
 6805     if (s != d) {
 6806       __ movq(d, s);
 6807     }
 6808     __ encode_heap_oop(d);
 6809   %}
 6810   ins_pipe(ialu_reg_long);
 6811 %}
 6812 
 6813 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6814   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6815   match(Set dst (EncodeP src));
 6816   effect(KILL cr);
 6817   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6818   ins_encode %{
 6819     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6820   %}
 6821   ins_pipe(ialu_reg_long);
 6822 %}
 6823 
 6824 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6825   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6826             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6827   match(Set dst (DecodeN src));
 6828   effect(KILL cr);
 6829   format %{ "decode_heap_oop $dst,$src" %}
 6830   ins_encode %{
 6831     Register s = $src$$Register;
 6832     Register d = $dst$$Register;
 6833     if (s != d) {
 6834       __ movq(d, s);
 6835     }
 6836     __ decode_heap_oop(d);
 6837   %}
 6838   ins_pipe(ialu_reg_long);
 6839 %}
 6840 
 6841 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6842   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6843             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6844   match(Set dst (DecodeN src));
 6845   effect(KILL cr);
 6846   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6847   ins_encode %{
 6848     Register s = $src$$Register;
 6849     Register d = $dst$$Register;
 6850     if (s != d) {
 6851       __ decode_heap_oop_not_null(d, s);
 6852     } else {
 6853       __ decode_heap_oop_not_null(d);
 6854     }
 6855   %}
 6856   ins_pipe(ialu_reg_long);
 6857 %}
 6858 
 6859 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6860   match(Set dst (EncodePKlass src));
 6861   effect(TEMP dst, KILL cr);
 6862   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6863   ins_encode %{
 6864     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6865   %}
 6866   ins_pipe(ialu_reg_long);
 6867 %}
 6868 
 6869 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6870   match(Set dst (DecodeNKlass src));
 6871   effect(TEMP dst, KILL cr);
 6872   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6873   ins_encode %{
 6874     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6875   %}
 6876   ins_pipe(ialu_reg_long);
 6877 %}
 6878 
 6879 //----------Conditional Move---------------------------------------------------
 6880 // Jump
 6881 // dummy instruction for generating temp registers
 6882 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6883   match(Jump (LShiftL switch_val shift));
 6884   ins_cost(350);
 6885   predicate(false);
 6886   effect(TEMP dest);
 6887 
 6888   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6889             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6890   ins_encode %{
 6891     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6892     // to do that and the compiler is using that register as one it can allocate.
 6893     // So we build it all by hand.
 6894     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6895     // ArrayAddress dispatch(table, index);
 6896     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6897     __ lea($dest$$Register, $constantaddress);
 6898     __ jmp(dispatch);
 6899   %}
 6900   ins_pipe(pipe_jmp);
 6901 %}
 6902 
 6903 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6904   match(Jump (AddL (LShiftL switch_val shift) offset));
 6905   ins_cost(350);
 6906   effect(TEMP dest);
 6907 
 6908   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6909             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6910   ins_encode %{
 6911     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6912     // to do that and the compiler is using that register as one it can allocate.
 6913     // So we build it all by hand.
 6914     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6915     // ArrayAddress dispatch(table, index);
 6916     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6917     __ lea($dest$$Register, $constantaddress);
 6918     __ jmp(dispatch);
 6919   %}
 6920   ins_pipe(pipe_jmp);
 6921 %}
 6922 
 6923 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6924   match(Jump switch_val);
 6925   ins_cost(350);
 6926   effect(TEMP dest);
 6927 
 6928   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6929             "jmp     [$dest + $switch_val]\n\t" %}
 6930   ins_encode %{
 6931     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6932     // to do that and the compiler is using that register as one it can allocate.
 6933     // So we build it all by hand.
 6934     // Address index(noreg, switch_reg, Address::times_1);
 6935     // ArrayAddress dispatch(table, index);
 6936     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6937     __ lea($dest$$Register, $constantaddress);
 6938     __ jmp(dispatch);
 6939   %}
 6940   ins_pipe(pipe_jmp);
 6941 %}
 6942 
 6943 // Conditional move
 6944 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6945 %{
 6946   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6947 
 6948   ins_cost(200); // XXX
 6949   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6950   ins_encode %{
 6951     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6952   %}
 6953   ins_pipe(pipe_cmov_reg);
 6954 %}
 6955 
 6956 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6957   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6958 
 6959   ins_cost(200); // XXX
 6960   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6961   ins_encode %{
 6962     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6963   %}
 6964   ins_pipe(pipe_cmov_reg);
 6965 %}
 6966 
 6967 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6968   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6969   ins_cost(200);
 6970   expand %{
 6971     cmovI_regU(cop, cr, dst, src);
 6972   %}
 6973 %}
 6974 
 6975 // Conditional move
 6976 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6977   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6978 
 6979   ins_cost(250); // XXX
 6980   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6981   ins_encode %{
 6982     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6983   %}
 6984   ins_pipe(pipe_cmov_mem);
 6985 %}
 6986 
 6987 // Conditional move
 6988 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6989 %{
 6990   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6991 
 6992   ins_cost(250); // XXX
 6993   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6994   ins_encode %{
 6995     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6996   %}
 6997   ins_pipe(pipe_cmov_mem);
 6998 %}
 6999 
 7000 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7001   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7002   ins_cost(250);
 7003   expand %{
 7004     cmovI_memU(cop, cr, dst, src);
 7005   %}
 7006 %}
 7007 
 7008 // Conditional move
 7009 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7010 %{
 7011   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7012 
 7013   ins_cost(200); // XXX
 7014   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7015   ins_encode %{
 7016     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7017   %}
 7018   ins_pipe(pipe_cmov_reg);
 7019 %}
 7020 
 7021 // Conditional move
 7022 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7023 %{
 7024   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7025 
 7026   ins_cost(200); // XXX
 7027   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7028   ins_encode %{
 7029     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7030   %}
 7031   ins_pipe(pipe_cmov_reg);
 7032 %}
 7033 
 7034 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7035   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7036   ins_cost(200);
 7037   expand %{
 7038     cmovN_regU(cop, cr, dst, src);
 7039   %}
 7040 %}
 7041 
 7042 // Conditional move
 7043 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7044 %{
 7045   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7046 
 7047   ins_cost(200); // XXX
 7048   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7049   ins_encode %{
 7050     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7051   %}
 7052   ins_pipe(pipe_cmov_reg);  // XXX
 7053 %}
 7054 
 7055 // Conditional move
 7056 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7057 %{
 7058   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7059 
 7060   ins_cost(200); // XXX
 7061   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7062   ins_encode %{
 7063     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7064   %}
 7065   ins_pipe(pipe_cmov_reg); // XXX
 7066 %}
 7067 
 7068 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7069   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7070   ins_cost(200);
 7071   expand %{
 7072     cmovP_regU(cop, cr, dst, src);
 7073   %}
 7074 %}
 7075 
 7076 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7077 // correctly meets the two pointer arguments; one is an incoming
 7078 // register but the other is a memory operand.  ALSO appears to
 7079 // be buggy with implicit null checks.
 7080 //
 7081 //// Conditional move
 7082 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7083 //%{
 7084 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7085 //  ins_cost(250);
 7086 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7087 //  opcode(0x0F,0x40);
 7088 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7089 //  ins_pipe( pipe_cmov_mem );
 7090 //%}
 7091 //
 7092 //// Conditional move
 7093 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7094 //%{
 7095 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7096 //  ins_cost(250);
 7097 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7098 //  opcode(0x0F,0x40);
 7099 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7100 //  ins_pipe( pipe_cmov_mem );
 7101 //%}
 7102 
 7103 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7104 %{
 7105   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7106 
 7107   ins_cost(200); // XXX
 7108   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7109   ins_encode %{
 7110     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7111   %}
 7112   ins_pipe(pipe_cmov_reg);  // XXX
 7113 %}
 7114 
 7115 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7116 %{
 7117   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7118 
 7119   ins_cost(200); // XXX
 7120   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7121   ins_encode %{
 7122     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7123   %}
 7124   ins_pipe(pipe_cmov_mem);  // XXX
 7125 %}
 7126 
 7127 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7128 %{
 7129   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7130 
 7131   ins_cost(200); // XXX
 7132   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7133   ins_encode %{
 7134     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7135   %}
 7136   ins_pipe(pipe_cmov_reg); // XXX
 7137 %}
 7138 
 7139 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7140   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7141   ins_cost(200);
 7142   expand %{
 7143     cmovL_regU(cop, cr, dst, src);
 7144   %}
 7145 %}
 7146 
 7147 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7148 %{
 7149   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7150 
 7151   ins_cost(200); // XXX
 7152   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7153   ins_encode %{
 7154     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7155   %}
 7156   ins_pipe(pipe_cmov_mem); // XXX
 7157 %}
 7158 
 7159 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7160   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7161   ins_cost(200);
 7162   expand %{
 7163     cmovL_memU(cop, cr, dst, src);
 7164   %}
 7165 %}
 7166 
 7167 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7168 %{
 7169   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7170 
 7171   ins_cost(200); // XXX
 7172   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7173             "movss     $dst, $src\n"
 7174     "skip:" %}
 7175   ins_encode %{
 7176     Label Lskip;
 7177     // Invert sense of branch from sense of CMOV
 7178     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7179     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7180     __ bind(Lskip);
 7181   %}
 7182   ins_pipe(pipe_slow);
 7183 %}
 7184 
 7185 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7186 // %{
 7187 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7188 
 7189 //   ins_cost(200); // XXX
 7190 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7191 //             "movss     $dst, $src\n"
 7192 //     "skip:" %}
 7193 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7194 //   ins_pipe(pipe_slow);
 7195 // %}
 7196 
 7197 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7198 %{
 7199   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7200 
 7201   ins_cost(200); // XXX
 7202   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7203             "movss     $dst, $src\n"
 7204     "skip:" %}
 7205   ins_encode %{
 7206     Label Lskip;
 7207     // Invert sense of branch from sense of CMOV
 7208     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7209     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7210     __ bind(Lskip);
 7211   %}
 7212   ins_pipe(pipe_slow);
 7213 %}
 7214 
 7215 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7216   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7217   ins_cost(200);
 7218   expand %{
 7219     cmovF_regU(cop, cr, dst, src);
 7220   %}
 7221 %}
 7222 
 7223 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7224 %{
 7225   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7226 
 7227   ins_cost(200); // XXX
 7228   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7229             "movsd     $dst, $src\n"
 7230     "skip:" %}
 7231   ins_encode %{
 7232     Label Lskip;
 7233     // Invert sense of branch from sense of CMOV
 7234     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7235     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7236     __ bind(Lskip);
 7237   %}
 7238   ins_pipe(pipe_slow);
 7239 %}
 7240 
 7241 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7242 %{
 7243   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7244 
 7245   ins_cost(200); // XXX
 7246   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7247             "movsd     $dst, $src\n"
 7248     "skip:" %}
 7249   ins_encode %{
 7250     Label Lskip;
 7251     // Invert sense of branch from sense of CMOV
 7252     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7253     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7254     __ bind(Lskip);
 7255   %}
 7256   ins_pipe(pipe_slow);
 7257 %}
 7258 
 7259 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7260   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7261   ins_cost(200);
 7262   expand %{
 7263     cmovD_regU(cop, cr, dst, src);
 7264   %}
 7265 %}
 7266 
 7267 //----------Arithmetic Instructions--------------------------------------------
 7268 //----------Addition Instructions----------------------------------------------
 7269 
 7270 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7271 %{
 7272   match(Set dst (AddI dst src));
 7273   effect(KILL cr);
 7274 
 7275   format %{ "addl    $dst, $src\t# int" %}
 7276   ins_encode %{
 7277     __ addl($dst$$Register, $src$$Register);
 7278   %}
 7279   ins_pipe(ialu_reg_reg);
 7280 %}
 7281 
 7282 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7283 %{
 7284   match(Set dst (AddI dst src));
 7285   effect(KILL cr);
 7286 
 7287   format %{ "addl    $dst, $src\t# int" %}
 7288   ins_encode %{
 7289     __ addl($dst$$Register, $src$$constant);
 7290   %}
 7291   ins_pipe( ialu_reg );
 7292 %}
 7293 
 7294 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7295 %{
 7296   match(Set dst (AddI dst (LoadI src)));
 7297   effect(KILL cr);
 7298 
 7299   ins_cost(125); // XXX
 7300   format %{ "addl    $dst, $src\t# int" %}
 7301   ins_encode %{
 7302     __ addl($dst$$Register, $src$$Address);
 7303   %}
 7304   ins_pipe(ialu_reg_mem);
 7305 %}
 7306 
 7307 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7308 %{
 7309   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7310   effect(KILL cr);
 7311 
 7312   ins_cost(150); // XXX
 7313   format %{ "addl    $dst, $src\t# int" %}
 7314   ins_encode %{
 7315     __ addl($dst$$Address, $src$$Register);
 7316   %}
 7317   ins_pipe(ialu_mem_reg);
 7318 %}
 7319 
 7320 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7321 %{
 7322   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7323   effect(KILL cr);
 7324 
 7325   ins_cost(125); // XXX
 7326   format %{ "addl    $dst, $src\t# int" %}
 7327   ins_encode %{
 7328     __ addl($dst$$Address, $src$$constant);
 7329   %}
 7330   ins_pipe(ialu_mem_imm);
 7331 %}
 7332 
 7333 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7334 %{
 7335   predicate(UseIncDec);
 7336   match(Set dst (AddI dst src));
 7337   effect(KILL cr);
 7338 
 7339   format %{ "incl    $dst\t# int" %}
 7340   ins_encode %{
 7341     __ incrementl($dst$$Register);
 7342   %}
 7343   ins_pipe(ialu_reg);
 7344 %}
 7345 
 7346 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7347 %{
 7348   predicate(UseIncDec);
 7349   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7350   effect(KILL cr);
 7351 
 7352   ins_cost(125); // XXX
 7353   format %{ "incl    $dst\t# int" %}
 7354   ins_encode %{
 7355     __ incrementl($dst$$Address);
 7356   %}
 7357   ins_pipe(ialu_mem_imm);
 7358 %}
 7359 
 7360 // XXX why does that use AddI
 7361 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7362 %{
 7363   predicate(UseIncDec);
 7364   match(Set dst (AddI dst src));
 7365   effect(KILL cr);
 7366 
 7367   format %{ "decl    $dst\t# int" %}
 7368   ins_encode %{
 7369     __ decrementl($dst$$Register);
 7370   %}
 7371   ins_pipe(ialu_reg);
 7372 %}
 7373 
 7374 // XXX why does that use AddI
 7375 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7376 %{
 7377   predicate(UseIncDec);
 7378   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7379   effect(KILL cr);
 7380 
 7381   ins_cost(125); // XXX
 7382   format %{ "decl    $dst\t# int" %}
 7383   ins_encode %{
 7384     __ decrementl($dst$$Address);
 7385   %}
 7386   ins_pipe(ialu_mem_imm);
 7387 %}
 7388 
 7389 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7390 %{
 7391   predicate(VM_Version::supports_fast_2op_lea());
 7392   match(Set dst (AddI (LShiftI index scale) disp));
 7393 
 7394   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7395   ins_encode %{
 7396     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7397     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7398   %}
 7399   ins_pipe(ialu_reg_reg);
 7400 %}
 7401 
 7402 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7403 %{
 7404   predicate(VM_Version::supports_fast_3op_lea());
 7405   match(Set dst (AddI (AddI base index) disp));
 7406 
 7407   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7408   ins_encode %{
 7409     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7410   %}
 7411   ins_pipe(ialu_reg_reg);
 7412 %}
 7413 
 7414 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7415 %{
 7416   predicate(VM_Version::supports_fast_2op_lea());
 7417   match(Set dst (AddI base (LShiftI index scale)));
 7418 
 7419   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7420   ins_encode %{
 7421     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7422     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7423   %}
 7424   ins_pipe(ialu_reg_reg);
 7425 %}
 7426 
 7427 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7428 %{
 7429   predicate(VM_Version::supports_fast_3op_lea());
 7430   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7431 
 7432   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7433   ins_encode %{
 7434     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7435     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7436   %}
 7437   ins_pipe(ialu_reg_reg);
 7438 %}
 7439 
 7440 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7441 %{
 7442   match(Set dst (AddL dst src));
 7443   effect(KILL cr);
 7444 
 7445   format %{ "addq    $dst, $src\t# long" %}
 7446   ins_encode %{
 7447     __ addq($dst$$Register, $src$$Register);
 7448   %}
 7449   ins_pipe(ialu_reg_reg);
 7450 %}
 7451 
 7452 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7453 %{
 7454   match(Set dst (AddL dst src));
 7455   effect(KILL cr);
 7456 
 7457   format %{ "addq    $dst, $src\t# long" %}
 7458   ins_encode %{
 7459     __ addq($dst$$Register, $src$$constant);
 7460   %}
 7461   ins_pipe( ialu_reg );
 7462 %}
 7463 
 7464 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7465 %{
 7466   match(Set dst (AddL dst (LoadL src)));
 7467   effect(KILL cr);
 7468 
 7469   ins_cost(125); // XXX
 7470   format %{ "addq    $dst, $src\t# long" %}
 7471   ins_encode %{
 7472     __ addq($dst$$Register, $src$$Address);
 7473   %}
 7474   ins_pipe(ialu_reg_mem);
 7475 %}
 7476 
 7477 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7478 %{
 7479   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7480   effect(KILL cr);
 7481 
 7482   ins_cost(150); // XXX
 7483   format %{ "addq    $dst, $src\t# long" %}
 7484   ins_encode %{
 7485     __ addq($dst$$Address, $src$$Register);
 7486   %}
 7487   ins_pipe(ialu_mem_reg);
 7488 %}
 7489 
 7490 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7491 %{
 7492   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7493   effect(KILL cr);
 7494 
 7495   ins_cost(125); // XXX
 7496   format %{ "addq    $dst, $src\t# long" %}
 7497   ins_encode %{
 7498     __ addq($dst$$Address, $src$$constant);
 7499   %}
 7500   ins_pipe(ialu_mem_imm);
 7501 %}
 7502 
 7503 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7504 %{
 7505   predicate(UseIncDec);
 7506   match(Set dst (AddL dst src));
 7507   effect(KILL cr);
 7508 
 7509   format %{ "incq    $dst\t# long" %}
 7510   ins_encode %{
 7511     __ incrementq($dst$$Register);
 7512   %}
 7513   ins_pipe(ialu_reg);
 7514 %}
 7515 
 7516 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7517 %{
 7518   predicate(UseIncDec);
 7519   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7520   effect(KILL cr);
 7521 
 7522   ins_cost(125); // XXX
 7523   format %{ "incq    $dst\t# long" %}
 7524   ins_encode %{
 7525     __ incrementq($dst$$Address);
 7526   %}
 7527   ins_pipe(ialu_mem_imm);
 7528 %}
 7529 
 7530 // XXX why does that use AddL
 7531 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7532 %{
 7533   predicate(UseIncDec);
 7534   match(Set dst (AddL dst src));
 7535   effect(KILL cr);
 7536 
 7537   format %{ "decq    $dst\t# long" %}
 7538   ins_encode %{
 7539     __ decrementq($dst$$Register);
 7540   %}
 7541   ins_pipe(ialu_reg);
 7542 %}
 7543 
 7544 // XXX why does that use AddL
 7545 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7546 %{
 7547   predicate(UseIncDec);
 7548   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7549   effect(KILL cr);
 7550 
 7551   ins_cost(125); // XXX
 7552   format %{ "decq    $dst\t# long" %}
 7553   ins_encode %{
 7554     __ decrementq($dst$$Address);
 7555   %}
 7556   ins_pipe(ialu_mem_imm);
 7557 %}
 7558 
 7559 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7560 %{
 7561   predicate(VM_Version::supports_fast_2op_lea());
 7562   match(Set dst (AddL (LShiftL index scale) disp));
 7563 
 7564   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7565   ins_encode %{
 7566     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7567     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7568   %}
 7569   ins_pipe(ialu_reg_reg);
 7570 %}
 7571 
 7572 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7573 %{
 7574   predicate(VM_Version::supports_fast_3op_lea());
 7575   match(Set dst (AddL (AddL base index) disp));
 7576 
 7577   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7578   ins_encode %{
 7579     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7580   %}
 7581   ins_pipe(ialu_reg_reg);
 7582 %}
 7583 
 7584 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7585 %{
 7586   predicate(VM_Version::supports_fast_2op_lea());
 7587   match(Set dst (AddL base (LShiftL index scale)));
 7588 
 7589   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7590   ins_encode %{
 7591     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7592     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7593   %}
 7594   ins_pipe(ialu_reg_reg);
 7595 %}
 7596 
 7597 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7598 %{
 7599   predicate(VM_Version::supports_fast_3op_lea());
 7600   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7601 
 7602   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7603   ins_encode %{
 7604     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7605     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7606   %}
 7607   ins_pipe(ialu_reg_reg);
 7608 %}
 7609 
 7610 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7611 %{
 7612   match(Set dst (AddP dst src));
 7613   effect(KILL cr);
 7614 
 7615   format %{ "addq    $dst, $src\t# ptr" %}
 7616   ins_encode %{
 7617     __ addq($dst$$Register, $src$$Register);
 7618   %}
 7619   ins_pipe(ialu_reg_reg);
 7620 %}
 7621 
 7622 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7623 %{
 7624   match(Set dst (AddP dst src));
 7625   effect(KILL cr);
 7626 
 7627   format %{ "addq    $dst, $src\t# ptr" %}
 7628   ins_encode %{
 7629     __ addq($dst$$Register, $src$$constant);
 7630   %}
 7631   ins_pipe( ialu_reg );
 7632 %}
 7633 
 7634 // XXX addP mem ops ????
 7635 
 7636 instruct checkCastPP(rRegP dst)
 7637 %{
 7638   match(Set dst (CheckCastPP dst));
 7639 
 7640   size(0);
 7641   format %{ "# checkcastPP of $dst" %}
 7642   ins_encode(/* empty encoding */);
 7643   ins_pipe(empty);
 7644 %}
 7645 
 7646 instruct castPP(rRegP dst)
 7647 %{
 7648   match(Set dst (CastPP dst));
 7649 
 7650   size(0);
 7651   format %{ "# castPP of $dst" %}
 7652   ins_encode(/* empty encoding */);
 7653   ins_pipe(empty);
 7654 %}
 7655 
 7656 instruct castII(rRegI dst)
 7657 %{
 7658   match(Set dst (CastII dst));
 7659 
 7660   size(0);
 7661   format %{ "# castII of $dst" %}
 7662   ins_encode(/* empty encoding */);
 7663   ins_cost(0);
 7664   ins_pipe(empty);
 7665 %}
 7666 
 7667 instruct castLL(rRegL dst)
 7668 %{
 7669   match(Set dst (CastLL dst));
 7670 
 7671   size(0);
 7672   format %{ "# castLL of $dst" %}
 7673   ins_encode(/* empty encoding */);
 7674   ins_cost(0);
 7675   ins_pipe(empty);
 7676 %}
 7677 
 7678 instruct castFF(regF dst)
 7679 %{
 7680   match(Set dst (CastFF dst));
 7681 
 7682   size(0);
 7683   format %{ "# castFF of $dst" %}
 7684   ins_encode(/* empty encoding */);
 7685   ins_cost(0);
 7686   ins_pipe(empty);
 7687 %}
 7688 
 7689 instruct castDD(regD dst)
 7690 %{
 7691   match(Set dst (CastDD dst));
 7692 
 7693   size(0);
 7694   format %{ "# castDD of $dst" %}
 7695   ins_encode(/* empty encoding */);
 7696   ins_cost(0);
 7697   ins_pipe(empty);
 7698 %}
 7699 
 7700 // LoadP-locked same as a regular LoadP when used with compare-swap
 7701 instruct loadPLocked(rRegP dst, memory mem)
 7702 %{
 7703   match(Set dst (LoadPLocked mem));
 7704 
 7705   ins_cost(125); // XXX
 7706   format %{ "movq    $dst, $mem\t# ptr locked" %}
 7707   ins_encode %{
 7708     __ movq($dst$$Register, $mem$$Address);
 7709   %}
 7710   ins_pipe(ialu_reg_mem); // XXX
 7711 %}
 7712 
 7713 // Conditional-store of the updated heap-top.
 7714 // Used during allocation of the shared heap.
 7715 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
 7716 
 7717 instruct storePConditional(memory heap_top_ptr,
 7718                            rax_RegP oldval, rRegP newval,
 7719                            rFlagsReg cr)
 7720 %{
 7721   predicate(n->as_LoadStore()->barrier_data() == 0);
 7722   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 7723 
 7724   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
 7725             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
 7726   ins_encode %{
 7727     __ lock();
 7728     __ cmpxchgq($newval$$Register, $heap_top_ptr$$Address);
 7729   %}
 7730   ins_pipe(pipe_cmpxchg);
 7731 %}
 7732 
 7733 // Conditional-store of an int value.
 7734 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7735 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
 7736 %{
 7737   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 7738   effect(KILL oldval);
 7739 
 7740   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7741   opcode(0x0F, 0xB1);
 7742   ins_encode(lock_prefix,
 7743              REX_reg_mem(newval, mem),
 7744              OpcP, OpcS,
 7745              reg_mem(newval, mem));
 7746   ins_pipe(pipe_cmpxchg);
 7747 %}
 7748 
 7749 // Conditional-store of a long value.
 7750 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7751 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
 7752 %{
 7753   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 7754   effect(KILL oldval);
 7755 
 7756   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7757   ins_encode %{
 7758     __ lock();
 7759     __ cmpxchgq($newval$$Register, $mem$$Address);
 7760   %}
 7761   ins_pipe(pipe_cmpxchg);
 7762 %}
 7763 
 7764 
 7765 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7766 instruct compareAndSwapP(rRegI res,
 7767                          memory mem_ptr,
 7768                          rax_RegP oldval, rRegP newval,
 7769                          rFlagsReg cr)
 7770 %{
 7771   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7772   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7773   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7774   effect(KILL cr, KILL oldval);
 7775 
 7776   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7777             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7778             "sete    $res\n\t"
 7779             "movzbl  $res, $res" %}
 7780   ins_encode %{
 7781     __ lock();
 7782     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7783     __ sete($res$$Register);
 7784     __ movzbl($res$$Register, $res$$Register);
 7785   %}
 7786   ins_pipe( pipe_cmpxchg );
 7787 %}
 7788 
 7789 instruct compareAndSwapL(rRegI res,
 7790                          memory mem_ptr,
 7791                          rax_RegL oldval, rRegL newval,
 7792                          rFlagsReg cr)
 7793 %{
 7794   predicate(VM_Version::supports_cx8());
 7795   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7796   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7797   effect(KILL cr, KILL oldval);
 7798 
 7799   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7800             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7801             "sete    $res\n\t"
 7802             "movzbl  $res, $res" %}
 7803   ins_encode %{
 7804     __ lock();
 7805     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7806     __ sete($res$$Register);
 7807     __ movzbl($res$$Register, $res$$Register);
 7808   %}
 7809   ins_pipe( pipe_cmpxchg );
 7810 %}
 7811 
 7812 instruct compareAndSwapI(rRegI res,
 7813                          memory mem_ptr,
 7814                          rax_RegI oldval, rRegI newval,
 7815                          rFlagsReg cr)
 7816 %{
 7817   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7818   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7819   effect(KILL cr, KILL oldval);
 7820 
 7821   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7822             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7823             "sete    $res\n\t"
 7824             "movzbl  $res, $res" %}
 7825   ins_encode %{
 7826     __ lock();
 7827     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7828     __ sete($res$$Register);
 7829     __ movzbl($res$$Register, $res$$Register);
 7830   %}
 7831   ins_pipe( pipe_cmpxchg );
 7832 %}
 7833 
 7834 instruct compareAndSwapB(rRegI res,
 7835                          memory mem_ptr,
 7836                          rax_RegI oldval, rRegI newval,
 7837                          rFlagsReg cr)
 7838 %{
 7839   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7840   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7841   effect(KILL cr, KILL oldval);
 7842 
 7843   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7844             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7845             "sete    $res\n\t"
 7846             "movzbl  $res, $res" %}
 7847   ins_encode %{
 7848     __ lock();
 7849     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7850     __ sete($res$$Register);
 7851     __ movzbl($res$$Register, $res$$Register);
 7852   %}
 7853   ins_pipe( pipe_cmpxchg );
 7854 %}
 7855 
 7856 instruct compareAndSwapS(rRegI res,
 7857                          memory mem_ptr,
 7858                          rax_RegI oldval, rRegI newval,
 7859                          rFlagsReg cr)
 7860 %{
 7861   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7862   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7863   effect(KILL cr, KILL oldval);
 7864 
 7865   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7866             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7867             "sete    $res\n\t"
 7868             "movzbl  $res, $res" %}
 7869   ins_encode %{
 7870     __ lock();
 7871     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7872     __ sete($res$$Register);
 7873     __ movzbl($res$$Register, $res$$Register);
 7874   %}
 7875   ins_pipe( pipe_cmpxchg );
 7876 %}
 7877 
 7878 instruct compareAndSwapN(rRegI res,
 7879                           memory mem_ptr,
 7880                           rax_RegN oldval, rRegN newval,
 7881                           rFlagsReg cr) %{
 7882   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7883   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7884   effect(KILL cr, KILL oldval);
 7885 
 7886   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7887             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7888             "sete    $res\n\t"
 7889             "movzbl  $res, $res" %}
 7890   ins_encode %{
 7891     __ lock();
 7892     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7893     __ sete($res$$Register);
 7894     __ movzbl($res$$Register, $res$$Register);
 7895   %}
 7896   ins_pipe( pipe_cmpxchg );
 7897 %}
 7898 
 7899 instruct compareAndExchangeB(
 7900                          memory mem_ptr,
 7901                          rax_RegI oldval, rRegI newval,
 7902                          rFlagsReg cr)
 7903 %{
 7904   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7905   effect(KILL cr);
 7906 
 7907   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7908             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7909   ins_encode %{
 7910     __ lock();
 7911     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7912   %}
 7913   ins_pipe( pipe_cmpxchg );
 7914 %}
 7915 
 7916 instruct compareAndExchangeS(
 7917                          memory mem_ptr,
 7918                          rax_RegI oldval, rRegI newval,
 7919                          rFlagsReg cr)
 7920 %{
 7921   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7922   effect(KILL cr);
 7923 
 7924   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7925             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7926   ins_encode %{
 7927     __ lock();
 7928     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7929   %}
 7930   ins_pipe( pipe_cmpxchg );
 7931 %}
 7932 
 7933 instruct compareAndExchangeI(
 7934                          memory mem_ptr,
 7935                          rax_RegI oldval, rRegI newval,
 7936                          rFlagsReg cr)
 7937 %{
 7938   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7939   effect(KILL cr);
 7940 
 7941   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7942             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7943   ins_encode %{
 7944     __ lock();
 7945     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7946   %}
 7947   ins_pipe( pipe_cmpxchg );
 7948 %}
 7949 
 7950 instruct compareAndExchangeL(
 7951                          memory mem_ptr,
 7952                          rax_RegL oldval, rRegL newval,
 7953                          rFlagsReg cr)
 7954 %{
 7955   predicate(VM_Version::supports_cx8());
 7956   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7957   effect(KILL cr);
 7958 
 7959   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7960             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7961   ins_encode %{
 7962     __ lock();
 7963     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7964   %}
 7965   ins_pipe( pipe_cmpxchg );
 7966 %}
 7967 
 7968 instruct compareAndExchangeN(
 7969                           memory mem_ptr,
 7970                           rax_RegN oldval, rRegN newval,
 7971                           rFlagsReg cr) %{
 7972   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7973   effect(KILL cr);
 7974 
 7975   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7976             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7977   ins_encode %{
 7978     __ lock();
 7979     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7980   %}
 7981   ins_pipe( pipe_cmpxchg );
 7982 %}
 7983 
 7984 instruct compareAndExchangeP(
 7985                          memory mem_ptr,
 7986                          rax_RegP oldval, rRegP newval,
 7987                          rFlagsReg cr)
 7988 %{
 7989   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7990   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7991   effect(KILL cr);
 7992 
 7993   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7994             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7995   ins_encode %{
 7996     __ lock();
 7997     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7998   %}
 7999   ins_pipe( pipe_cmpxchg );
 8000 %}
 8001 
 8002 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8003   predicate(n->as_LoadStore()->result_not_used());
 8004   match(Set dummy (GetAndAddB mem add));
 8005   effect(KILL cr);
 8006   format %{ "ADDB  [$mem],$add" %}
 8007   ins_encode %{
 8008     __ lock();
 8009     __ addb($mem$$Address, $add$$constant);
 8010   %}
 8011   ins_pipe( pipe_cmpxchg );
 8012 %}
 8013 
 8014 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8015   match(Set newval (GetAndAddB mem newval));
 8016   effect(KILL cr);
 8017   format %{ "XADDB  [$mem],$newval" %}
 8018   ins_encode %{
 8019     __ lock();
 8020     __ xaddb($mem$$Address, $newval$$Register);
 8021   %}
 8022   ins_pipe( pipe_cmpxchg );
 8023 %}
 8024 
 8025 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8026   predicate(n->as_LoadStore()->result_not_used());
 8027   match(Set dummy (GetAndAddS mem add));
 8028   effect(KILL cr);
 8029   format %{ "ADDW  [$mem],$add" %}
 8030   ins_encode %{
 8031     __ lock();
 8032     __ addw($mem$$Address, $add$$constant);
 8033   %}
 8034   ins_pipe( pipe_cmpxchg );
 8035 %}
 8036 
 8037 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8038   match(Set newval (GetAndAddS mem newval));
 8039   effect(KILL cr);
 8040   format %{ "XADDW  [$mem],$newval" %}
 8041   ins_encode %{
 8042     __ lock();
 8043     __ xaddw($mem$$Address, $newval$$Register);
 8044   %}
 8045   ins_pipe( pipe_cmpxchg );
 8046 %}
 8047 
 8048 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8049   predicate(n->as_LoadStore()->result_not_used());
 8050   match(Set dummy (GetAndAddI mem add));
 8051   effect(KILL cr);
 8052   format %{ "ADDL  [$mem],$add" %}
 8053   ins_encode %{
 8054     __ lock();
 8055     __ addl($mem$$Address, $add$$constant);
 8056   %}
 8057   ins_pipe( pipe_cmpxchg );
 8058 %}
 8059 
 8060 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8061   match(Set newval (GetAndAddI mem newval));
 8062   effect(KILL cr);
 8063   format %{ "XADDL  [$mem],$newval" %}
 8064   ins_encode %{
 8065     __ lock();
 8066     __ xaddl($mem$$Address, $newval$$Register);
 8067   %}
 8068   ins_pipe( pipe_cmpxchg );
 8069 %}
 8070 
 8071 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8072   predicate(n->as_LoadStore()->result_not_used());
 8073   match(Set dummy (GetAndAddL mem add));
 8074   effect(KILL cr);
 8075   format %{ "ADDQ  [$mem],$add" %}
 8076   ins_encode %{
 8077     __ lock();
 8078     __ addq($mem$$Address, $add$$constant);
 8079   %}
 8080   ins_pipe( pipe_cmpxchg );
 8081 %}
 8082 
 8083 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8084   match(Set newval (GetAndAddL mem newval));
 8085   effect(KILL cr);
 8086   format %{ "XADDQ  [$mem],$newval" %}
 8087   ins_encode %{
 8088     __ lock();
 8089     __ xaddq($mem$$Address, $newval$$Register);
 8090   %}
 8091   ins_pipe( pipe_cmpxchg );
 8092 %}
 8093 
 8094 instruct xchgB( memory mem, rRegI newval) %{
 8095   match(Set newval (GetAndSetB mem newval));
 8096   format %{ "XCHGB  $newval,[$mem]" %}
 8097   ins_encode %{
 8098     __ xchgb($newval$$Register, $mem$$Address);
 8099   %}
 8100   ins_pipe( pipe_cmpxchg );
 8101 %}
 8102 
 8103 instruct xchgS( memory mem, rRegI newval) %{
 8104   match(Set newval (GetAndSetS mem newval));
 8105   format %{ "XCHGW  $newval,[$mem]" %}
 8106   ins_encode %{
 8107     __ xchgw($newval$$Register, $mem$$Address);
 8108   %}
 8109   ins_pipe( pipe_cmpxchg );
 8110 %}
 8111 
 8112 instruct xchgI( memory mem, rRegI newval) %{
 8113   match(Set newval (GetAndSetI mem newval));
 8114   format %{ "XCHGL  $newval,[$mem]" %}
 8115   ins_encode %{
 8116     __ xchgl($newval$$Register, $mem$$Address);
 8117   %}
 8118   ins_pipe( pipe_cmpxchg );
 8119 %}
 8120 
 8121 instruct xchgL( memory mem, rRegL newval) %{
 8122   match(Set newval (GetAndSetL mem newval));
 8123   format %{ "XCHGL  $newval,[$mem]" %}
 8124   ins_encode %{
 8125     __ xchgq($newval$$Register, $mem$$Address);
 8126   %}
 8127   ins_pipe( pipe_cmpxchg );
 8128 %}
 8129 
 8130 instruct xchgP( memory mem, rRegP newval) %{
 8131   match(Set newval (GetAndSetP mem newval));
 8132   predicate(n->as_LoadStore()->barrier_data() == 0);
 8133   format %{ "XCHGQ  $newval,[$mem]" %}
 8134   ins_encode %{
 8135     __ xchgq($newval$$Register, $mem$$Address);
 8136   %}
 8137   ins_pipe( pipe_cmpxchg );
 8138 %}
 8139 
 8140 instruct xchgN( memory mem, rRegN newval) %{
 8141   match(Set newval (GetAndSetN mem newval));
 8142   format %{ "XCHGL  $newval,$mem]" %}
 8143   ins_encode %{
 8144     __ xchgl($newval$$Register, $mem$$Address);
 8145   %}
 8146   ins_pipe( pipe_cmpxchg );
 8147 %}
 8148 
 8149 //----------Abs Instructions-------------------------------------------
 8150 
 8151 // Integer Absolute Instructions
 8152 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8153 %{
 8154   match(Set dst (AbsI src));
 8155   effect(TEMP dst, TEMP tmp, KILL cr);
 8156   format %{ "movl $tmp, $src\n\t"
 8157             "sarl $tmp, 31\n\t"
 8158             "movl $dst, $src\n\t"
 8159             "xorl $dst, $tmp\n\t"
 8160             "subl $dst, $tmp\n"
 8161           %}
 8162   ins_encode %{
 8163     __ movl($tmp$$Register, $src$$Register);
 8164     __ sarl($tmp$$Register, 31);
 8165     __ movl($dst$$Register, $src$$Register);
 8166     __ xorl($dst$$Register, $tmp$$Register);
 8167     __ subl($dst$$Register, $tmp$$Register);
 8168   %}
 8169 
 8170   ins_pipe(ialu_reg_reg);
 8171 %}
 8172 
 8173 // Long Absolute Instructions
 8174 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8175 %{
 8176   match(Set dst (AbsL src));
 8177   effect(TEMP dst, TEMP tmp, KILL cr);
 8178   format %{ "movq $tmp, $src\n\t"
 8179             "sarq $tmp, 63\n\t"
 8180             "movq $dst, $src\n\t"
 8181             "xorq $dst, $tmp\n\t"
 8182             "subq $dst, $tmp\n"
 8183           %}
 8184   ins_encode %{
 8185     __ movq($tmp$$Register, $src$$Register);
 8186     __ sarq($tmp$$Register, 63);
 8187     __ movq($dst$$Register, $src$$Register);
 8188     __ xorq($dst$$Register, $tmp$$Register);
 8189     __ subq($dst$$Register, $tmp$$Register);
 8190   %}
 8191 
 8192   ins_pipe(ialu_reg_reg);
 8193 %}
 8194 
 8195 //----------Subtraction Instructions-------------------------------------------
 8196 
 8197 // Integer Subtraction Instructions
 8198 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8199 %{
 8200   match(Set dst (SubI dst src));
 8201   effect(KILL cr);
 8202 
 8203   format %{ "subl    $dst, $src\t# int" %}
 8204   ins_encode %{
 8205     __ subl($dst$$Register, $src$$Register);
 8206   %}
 8207   ins_pipe(ialu_reg_reg);
 8208 %}
 8209 
 8210 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8211 %{
 8212   match(Set dst (SubI dst src));
 8213   effect(KILL cr);
 8214 
 8215   format %{ "subl    $dst, $src\t# int" %}
 8216   ins_encode %{
 8217     __ subl($dst$$Register, $src$$constant);
 8218   %}
 8219   ins_pipe(ialu_reg);
 8220 %}
 8221 
 8222 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8223 %{
 8224   match(Set dst (SubI dst (LoadI src)));
 8225   effect(KILL cr);
 8226 
 8227   ins_cost(125);
 8228   format %{ "subl    $dst, $src\t# int" %}
 8229   ins_encode %{
 8230     __ subl($dst$$Register, $src$$Address);
 8231   %}
 8232   ins_pipe(ialu_reg_mem);
 8233 %}
 8234 
 8235 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8236 %{
 8237   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8238   effect(KILL cr);
 8239 
 8240   ins_cost(150);
 8241   format %{ "subl    $dst, $src\t# int" %}
 8242   ins_encode %{
 8243     __ subl($dst$$Address, $src$$Register);
 8244   %}
 8245   ins_pipe(ialu_mem_reg);
 8246 %}
 8247 
 8248 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8249 %{
 8250   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8251   effect(KILL cr);
 8252 
 8253   ins_cost(125); // XXX
 8254   format %{ "subl    $dst, $src\t# int" %}
 8255   ins_encode %{
 8256     __ subl($dst$$Address, $src$$constant);
 8257   %}
 8258   ins_pipe(ialu_mem_imm);
 8259 %}
 8260 
 8261 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8262 %{
 8263   match(Set dst (SubL dst src));
 8264   effect(KILL cr);
 8265 
 8266   format %{ "subq    $dst, $src\t# long" %}
 8267   ins_encode %{
 8268     __ subq($dst$$Register, $src$$Register);
 8269   %}
 8270   ins_pipe(ialu_reg_reg);
 8271 %}
 8272 
 8273 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8274 %{
 8275   match(Set dst (SubL dst src));
 8276   effect(KILL cr);
 8277 
 8278   format %{ "subq    $dst, $src\t# long" %}
 8279   ins_encode %{
 8280     __ subq($dst$$Register, $src$$constant);
 8281   %}
 8282   ins_pipe(ialu_reg);
 8283 %}
 8284 
 8285 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8286 %{
 8287   match(Set dst (SubL dst (LoadL src)));
 8288   effect(KILL cr);
 8289 
 8290   ins_cost(125);
 8291   format %{ "subq    $dst, $src\t# long" %}
 8292   ins_encode %{
 8293     __ subq($dst$$Register, $src$$Address);
 8294   %}
 8295   ins_pipe(ialu_reg_mem);
 8296 %}
 8297 
 8298 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8299 %{
 8300   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8301   effect(KILL cr);
 8302 
 8303   ins_cost(150);
 8304   format %{ "subq    $dst, $src\t# long" %}
 8305   ins_encode %{
 8306     __ subq($dst$$Address, $src$$Register);
 8307   %}
 8308   ins_pipe(ialu_mem_reg);
 8309 %}
 8310 
 8311 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8312 %{
 8313   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8314   effect(KILL cr);
 8315 
 8316   ins_cost(125); // XXX
 8317   format %{ "subq    $dst, $src\t# long" %}
 8318   ins_encode %{
 8319     __ subq($dst$$Address, $src$$constant);
 8320   %}
 8321   ins_pipe(ialu_mem_imm);
 8322 %}
 8323 
 8324 // Subtract from a pointer
 8325 // XXX hmpf???
 8326 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8327 %{
 8328   match(Set dst (AddP dst (SubI zero src)));
 8329   effect(KILL cr);
 8330 
 8331   format %{ "subq    $dst, $src\t# ptr - int" %}
 8332   opcode(0x2B);
 8333   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8334   ins_pipe(ialu_reg_reg);
 8335 %}
 8336 
 8337 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8338 %{
 8339   match(Set dst (SubI zero dst));
 8340   effect(KILL cr);
 8341 
 8342   format %{ "negl    $dst\t# int" %}
 8343   ins_encode %{
 8344     __ negl($dst$$Register);
 8345   %}
 8346   ins_pipe(ialu_reg);
 8347 %}
 8348 
 8349 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8350 %{
 8351   match(Set dst (NegI dst));
 8352   effect(KILL cr);
 8353 
 8354   format %{ "negl    $dst\t# int" %}
 8355   ins_encode %{
 8356     __ negl($dst$$Register);
 8357   %}
 8358   ins_pipe(ialu_reg);
 8359 %}
 8360 
 8361 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8362 %{
 8363   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8364   effect(KILL cr);
 8365 
 8366   format %{ "negl    $dst\t# int" %}
 8367   ins_encode %{
 8368     __ negl($dst$$Address);
 8369   %}
 8370   ins_pipe(ialu_reg);
 8371 %}
 8372 
 8373 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8374 %{
 8375   match(Set dst (SubL zero dst));
 8376   effect(KILL cr);
 8377 
 8378   format %{ "negq    $dst\t# long" %}
 8379   ins_encode %{
 8380     __ negq($dst$$Register);
 8381   %}
 8382   ins_pipe(ialu_reg);
 8383 %}
 8384 
 8385 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8386 %{
 8387   match(Set dst (NegL dst));
 8388   effect(KILL cr);
 8389 
 8390   format %{ "negq    $dst\t# int" %}
 8391   ins_encode %{
 8392     __ negq($dst$$Register);
 8393   %}
 8394   ins_pipe(ialu_reg);
 8395 %}
 8396 
 8397 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8398 %{
 8399   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8400   effect(KILL cr);
 8401 
 8402   format %{ "negq    $dst\t# long" %}
 8403   ins_encode %{
 8404     __ negq($dst$$Address);
 8405   %}
 8406   ins_pipe(ialu_reg);
 8407 %}
 8408 
 8409 //----------Multiplication/Division Instructions-------------------------------
 8410 // Integer Multiplication Instructions
 8411 // Multiply Register
 8412 
 8413 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8414 %{
 8415   match(Set dst (MulI dst src));
 8416   effect(KILL cr);
 8417 
 8418   ins_cost(300);
 8419   format %{ "imull   $dst, $src\t# int" %}
 8420   ins_encode %{
 8421     __ imull($dst$$Register, $src$$Register);
 8422   %}
 8423   ins_pipe(ialu_reg_reg_alu0);
 8424 %}
 8425 
 8426 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8427 %{
 8428   match(Set dst (MulI src imm));
 8429   effect(KILL cr);
 8430 
 8431   ins_cost(300);
 8432   format %{ "imull   $dst, $src, $imm\t# int" %}
 8433   ins_encode %{
 8434     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8435   %}
 8436   ins_pipe(ialu_reg_reg_alu0);
 8437 %}
 8438 
 8439 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8440 %{
 8441   match(Set dst (MulI dst (LoadI src)));
 8442   effect(KILL cr);
 8443 
 8444   ins_cost(350);
 8445   format %{ "imull   $dst, $src\t# int" %}
 8446   ins_encode %{
 8447     __ imull($dst$$Register, $src$$Address);
 8448   %}
 8449   ins_pipe(ialu_reg_mem_alu0);
 8450 %}
 8451 
 8452 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8453 %{
 8454   match(Set dst (MulI (LoadI src) imm));
 8455   effect(KILL cr);
 8456 
 8457   ins_cost(300);
 8458   format %{ "imull   $dst, $src, $imm\t# int" %}
 8459   ins_encode %{
 8460     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8461   %}
 8462   ins_pipe(ialu_reg_mem_alu0);
 8463 %}
 8464 
 8465 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8466 %{
 8467   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8468   effect(KILL cr, KILL src2);
 8469 
 8470   expand %{ mulI_rReg(dst, src1, cr);
 8471            mulI_rReg(src2, src3, cr);
 8472            addI_rReg(dst, src2, cr); %}
 8473 %}
 8474 
 8475 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8476 %{
 8477   match(Set dst (MulL dst src));
 8478   effect(KILL cr);
 8479 
 8480   ins_cost(300);
 8481   format %{ "imulq   $dst, $src\t# long" %}
 8482   ins_encode %{
 8483     __ imulq($dst$$Register, $src$$Register);
 8484   %}
 8485   ins_pipe(ialu_reg_reg_alu0);
 8486 %}
 8487 
 8488 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8489 %{
 8490   match(Set dst (MulL src imm));
 8491   effect(KILL cr);
 8492 
 8493   ins_cost(300);
 8494   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8495   ins_encode %{
 8496     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8497   %}
 8498   ins_pipe(ialu_reg_reg_alu0);
 8499 %}
 8500 
 8501 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8502 %{
 8503   match(Set dst (MulL dst (LoadL src)));
 8504   effect(KILL cr);
 8505 
 8506   ins_cost(350);
 8507   format %{ "imulq   $dst, $src\t# long" %}
 8508   ins_encode %{
 8509     __ imulq($dst$$Register, $src$$Address);
 8510   %}
 8511   ins_pipe(ialu_reg_mem_alu0);
 8512 %}
 8513 
 8514 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8515 %{
 8516   match(Set dst (MulL (LoadL src) imm));
 8517   effect(KILL cr);
 8518 
 8519   ins_cost(300);
 8520   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8521   ins_encode %{
 8522     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8523   %}
 8524   ins_pipe(ialu_reg_mem_alu0);
 8525 %}
 8526 
 8527 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8528 %{
 8529   match(Set dst (MulHiL src rax));
 8530   effect(USE_KILL rax, KILL cr);
 8531 
 8532   ins_cost(300);
 8533   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8534   ins_encode %{
 8535     __ imulq($src$$Register);
 8536   %}
 8537   ins_pipe(ialu_reg_reg_alu0);
 8538 %}
 8539 
 8540 instruct umulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8541 %{
 8542   match(Set dst (UMulHiL src rax));
 8543   effect(USE_KILL rax, KILL cr);
 8544 
 8545   ins_cost(300);
 8546   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8547   ins_encode %{
 8548     __ mulq($src$$Register);
 8549   %}
 8550   ins_pipe(ialu_reg_reg_alu0);
 8551 %}
 8552 
 8553 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8554                    rFlagsReg cr)
 8555 %{
 8556   match(Set rax (NoOvfDivI rax div));
 8557   effect(KILL rdx, KILL cr);
 8558 
 8559   ins_cost(500);
 8560   format %{ "cdql\n\t"
 8561             "idivl   $div" %}
 8562   ins_encode %{
 8563     __ cdql();
 8564     __ idivl($div$$Register);
 8565   %}
 8566   ins_pipe(pipe_slow);
 8567 %}
 8568 
 8569 instruct divI_mem(rax_RegI rax, rdx_RegI rdx, memory div,
 8570                    rFlagsReg cr)
 8571 %{
 8572   match(Set rax (NoOvfDivI rax (LoadI div)));
 8573   effect(KILL rdx, KILL cr);
 8574 
 8575   ins_cost(575);
 8576   format %{ "cdql\n\t"
 8577             "idivl   $div" %}
 8578   ins_encode %{
 8579     __ cdql();
 8580     __ idivl($div$$Address);
 8581   %}
 8582   ins_pipe(pipe_slow);
 8583 %}
 8584 
 8585 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8586                    rFlagsReg cr)
 8587 %{
 8588   match(Set rax (NoOvfDivL rax div));
 8589   effect(KILL rdx, KILL cr);
 8590 
 8591   ins_cost(500);
 8592   format %{ "cdqq\n\t"
 8593             "idivq   $div" %}
 8594   ins_encode %{
 8595     __ cdqq();
 8596     __ idivq($div$$Register);
 8597   %}
 8598   ins_pipe(pipe_slow);
 8599 %}
 8600 
 8601 instruct divL_mem(rax_RegL rax, rdx_RegL rdx, memory div,
 8602                    rFlagsReg cr)
 8603 %{
 8604   match(Set rax (NoOvfDivL rax (LoadL div)));
 8605   effect(KILL rdx, KILL cr);
 8606 
 8607   ins_cost(575);
 8608   format %{ "cdqq\n\t"
 8609             "idivq   $div" %}
 8610   ins_encode %{
 8611     __ cdqq();
 8612     __ idivq($div$$Address);
 8613   %}
 8614   ins_pipe(pipe_slow);
 8615 %}
 8616 
 8617 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8618 %{
 8619   match(Set rax (UDivI rax div));
 8620   effect(KILL rdx, KILL cr);
 8621 
 8622   ins_cost(300);
 8623   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8624   ins_encode %{
 8625     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8626   %}
 8627   ins_pipe(ialu_reg_reg_alu0);
 8628 %}
 8629 
 8630 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8631 %{
 8632   match(Set rax (UDivL rax div));
 8633   effect(KILL rdx, KILL cr);
 8634 
 8635   ins_cost(300);
 8636   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8637   ins_encode %{
 8638      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8639   %}
 8640   ins_pipe(ialu_reg_reg_alu0);
 8641 %}
 8642 
 8643 // Integer DIVMOD with Register, both quotient and mod results
 8644 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8645                              rFlagsReg cr)
 8646 %{
 8647   match(NoOvfDivModI rax div);
 8648   effect(KILL cr);
 8649 
 8650   ins_cost(500);
 8651   format %{ "cdql\n\t"
 8652             "idivl   $div" %}
 8653   ins_encode %{
 8654     __ cdql();
 8655     __ idivl($div$$Register);
 8656   %}
 8657   ins_pipe(pipe_slow);
 8658 %}
 8659 
 8660 // Long DIVMOD with Register, both quotient and mod results
 8661 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8662                              rFlagsReg cr)
 8663 %{
 8664   match(NoOvfDivModL rax div);
 8665   effect(KILL cr);
 8666 
 8667   ins_cost(500);
 8668   format %{ "cdqq\n\t"
 8669             "idivq   $div" %}
 8670   ins_encode %{
 8671     __ cdqq();
 8672     __ idivq($div$$Register);
 8673   %}
 8674   ins_pipe(pipe_slow);
 8675 %}
 8676 
 8677 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8678 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8679                               no_rax_rdx_RegI div, rFlagsReg cr)
 8680 %{
 8681   match(UDivModI rax div);
 8682   effect(TEMP tmp, KILL cr);
 8683 
 8684   ins_cost(300);
 8685   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8686             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8687           %}
 8688   ins_encode %{
 8689     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8690   %}
 8691   ins_pipe(pipe_slow);
 8692 %}
 8693 
 8694 // Unsigned long DIVMOD with Register, both quotient and mod results
 8695 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8696                               no_rax_rdx_RegL div, rFlagsReg cr)
 8697 %{
 8698   match(UDivModL rax div);
 8699   effect(TEMP tmp, KILL cr);
 8700 
 8701   ins_cost(300);
 8702   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8703             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8704           %}
 8705   ins_encode %{
 8706     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8707   %}
 8708   ins_pipe(pipe_slow);
 8709 %}
 8710 
 8711 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8712                    rFlagsReg cr)
 8713 %{
 8714   match(Set rdx (NoOvfModI rax div));
 8715   effect(KILL rax, KILL cr);
 8716 
 8717   ins_cost(500);
 8718   format %{ "cdql\n\t"
 8719             "idivl   $div" %}
 8720   ins_encode %{
 8721     __ cdql();
 8722     __ idivl($div$$Register);
 8723   %}
 8724   ins_pipe(pipe_slow);
 8725 %}
 8726 
 8727 instruct modI_mem(rdx_RegI rdx, rax_RegI rax, memory div,
 8728                    rFlagsReg cr)
 8729 %{
 8730   match(Set rdx (NoOvfModI rax (LoadI div)));
 8731   effect(KILL rax, KILL cr);
 8732 
 8733   ins_cost(575);
 8734   format %{ "cdql\n\t"
 8735             "idivl   $div" %}
 8736   ins_encode %{
 8737     __ cdql();
 8738     __ idivl($div$$Address);
 8739   %}
 8740   ins_pipe(pipe_slow);
 8741 %}
 8742 
 8743 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8744                    rFlagsReg cr)
 8745 %{
 8746   match(Set rdx (NoOvfModL rax div));
 8747   effect(KILL rax, KILL cr);
 8748 
 8749   ins_cost(500);
 8750   format %{ "cdqq\n\t"
 8751             "idivq   $div" %}
 8752   ins_encode %{
 8753     __ cdqq();
 8754     __ idivq($div$$Register);
 8755   %}
 8756   ins_pipe(pipe_slow);
 8757 %}
 8758 
 8759 instruct modL_mem(rdx_RegL rdx, rax_RegL rax, memory div,
 8760                    rFlagsReg cr)
 8761 %{
 8762   match(Set rdx (NoOvfModL rax (LoadL div)));
 8763   effect(KILL rax, KILL cr);
 8764 
 8765   ins_cost(575);
 8766   format %{ "cdqq\n\t"
 8767             "idivq   $div" %}
 8768   ins_encode %{
 8769     __ cdqq();
 8770     __ idivq($div$$Address);
 8771   %}
 8772   ins_pipe(pipe_slow);
 8773 %}
 8774 
 8775 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8776 %{
 8777   match(Set rdx (UModI rax div));
 8778   effect(KILL rax, KILL cr);
 8779 
 8780   ins_cost(300);
 8781   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8782   ins_encode %{
 8783     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8784   %}
 8785   ins_pipe(ialu_reg_reg_alu0);
 8786 %}
 8787 
 8788 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8789 %{
 8790   match(Set rdx (UModL rax div));
 8791   effect(KILL rax, KILL cr);
 8792 
 8793   ins_cost(300);
 8794   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8795   ins_encode %{
 8796     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8797   %}
 8798   ins_pipe(ialu_reg_reg_alu0);
 8799 %}
 8800 
 8801 // Integer Shift Instructions
 8802 // Shift Left by one
 8803 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8804 %{
 8805   match(Set dst (LShiftI dst shift));
 8806   effect(KILL cr);
 8807 
 8808   format %{ "sall    $dst, $shift" %}
 8809   ins_encode %{
 8810     __ sall($dst$$Register, $shift$$constant);
 8811   %}
 8812   ins_pipe(ialu_reg);
 8813 %}
 8814 
 8815 // Shift Left by one
 8816 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8817 %{
 8818   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8819   effect(KILL cr);
 8820 
 8821   format %{ "sall    $dst, $shift\t" %}
 8822   ins_encode %{
 8823     __ sall($dst$$Address, $shift$$constant);
 8824   %}
 8825   ins_pipe(ialu_mem_imm);
 8826 %}
 8827 
 8828 // Shift Left by 8-bit immediate
 8829 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8830 %{
 8831   match(Set dst (LShiftI dst shift));
 8832   effect(KILL cr);
 8833 
 8834   format %{ "sall    $dst, $shift" %}
 8835   ins_encode %{
 8836     __ sall($dst$$Register, $shift$$constant);
 8837   %}
 8838   ins_pipe(ialu_reg);
 8839 %}
 8840 
 8841 // Shift Left by 8-bit immediate
 8842 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8843 %{
 8844   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8845   effect(KILL cr);
 8846 
 8847   format %{ "sall    $dst, $shift" %}
 8848   ins_encode %{
 8849     __ sall($dst$$Address, $shift$$constant);
 8850   %}
 8851   ins_pipe(ialu_mem_imm);
 8852 %}
 8853 
 8854 // Shift Left by variable
 8855 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8856 %{
 8857   match(Set dst (LShiftI dst shift));
 8858   effect(KILL cr);
 8859 
 8860   format %{ "sall    $dst, $shift" %}
 8861   ins_encode %{
 8862     __ sall($dst$$Register);
 8863   %}
 8864   ins_pipe(ialu_reg_reg);
 8865 %}
 8866 
 8867 // Shift Left by variable
 8868 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8869 %{
 8870   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8871   effect(KILL cr);
 8872 
 8873   format %{ "sall    $dst, $shift" %}
 8874   ins_encode %{
 8875     __ sall($dst$$Address);
 8876   %}
 8877   ins_pipe(ialu_mem_reg);
 8878 %}
 8879 
 8880 // Arithmetic shift right by one
 8881 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8882 %{
 8883   match(Set dst (RShiftI dst shift));
 8884   effect(KILL cr);
 8885 
 8886   format %{ "sarl    $dst, $shift" %}
 8887   ins_encode %{
 8888     __ sarl($dst$$Register, $shift$$constant);
 8889   %}
 8890   ins_pipe(ialu_reg);
 8891 %}
 8892 
 8893 // Arithmetic shift right by one
 8894 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8895 %{
 8896   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8897   effect(KILL cr);
 8898 
 8899   format %{ "sarl    $dst, $shift" %}
 8900   ins_encode %{
 8901     __ sarl($dst$$Address, $shift$$constant);
 8902   %}
 8903   ins_pipe(ialu_mem_imm);
 8904 %}
 8905 
 8906 // Arithmetic Shift Right by 8-bit immediate
 8907 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8908 %{
 8909   match(Set dst (RShiftI dst shift));
 8910   effect(KILL cr);
 8911 
 8912   format %{ "sarl    $dst, $shift" %}
 8913   ins_encode %{
 8914     __ sarl($dst$$Register, $shift$$constant);
 8915   %}
 8916   ins_pipe(ialu_mem_imm);
 8917 %}
 8918 
 8919 // Arithmetic Shift Right by 8-bit immediate
 8920 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8921 %{
 8922   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8923   effect(KILL cr);
 8924 
 8925   format %{ "sarl    $dst, $shift" %}
 8926   ins_encode %{
 8927     __ sarl($dst$$Address, $shift$$constant);
 8928   %}
 8929   ins_pipe(ialu_mem_imm);
 8930 %}
 8931 
 8932 // Arithmetic Shift Right by variable
 8933 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8934 %{
 8935   match(Set dst (RShiftI dst shift));
 8936   effect(KILL cr);
 8937   format %{ "sarl    $dst, $shift" %}
 8938   ins_encode %{
 8939     __ sarl($dst$$Register);
 8940   %}
 8941   ins_pipe(ialu_reg_reg);
 8942 %}
 8943 
 8944 // Arithmetic Shift Right by variable
 8945 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8946 %{
 8947   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8948   effect(KILL cr);
 8949 
 8950   format %{ "sarl    $dst, $shift" %}
 8951   ins_encode %{
 8952     __ sarl($dst$$Address);
 8953   %}
 8954   ins_pipe(ialu_mem_reg);
 8955 %}
 8956 
 8957 // Logical shift right by one
 8958 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8959 %{
 8960   match(Set dst (URShiftI dst shift));
 8961   effect(KILL cr);
 8962 
 8963   format %{ "shrl    $dst, $shift" %}
 8964   ins_encode %{
 8965     __ shrl($dst$$Register, $shift$$constant);
 8966   %}
 8967   ins_pipe(ialu_reg);
 8968 %}
 8969 
 8970 // Logical shift right by one
 8971 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8972 %{
 8973   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8974   effect(KILL cr);
 8975 
 8976   format %{ "shrl    $dst, $shift" %}
 8977   ins_encode %{
 8978     __ shrl($dst$$Address, $shift$$constant);
 8979   %}
 8980   ins_pipe(ialu_mem_imm);
 8981 %}
 8982 
 8983 // Logical Shift Right by 8-bit immediate
 8984 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8985 %{
 8986   match(Set dst (URShiftI dst shift));
 8987   effect(KILL cr);
 8988 
 8989   format %{ "shrl    $dst, $shift" %}
 8990   ins_encode %{
 8991     __ shrl($dst$$Register, $shift$$constant);
 8992   %}
 8993   ins_pipe(ialu_reg);
 8994 %}
 8995 
 8996 // Logical Shift Right by 8-bit immediate
 8997 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8998 %{
 8999   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9000   effect(KILL cr);
 9001 
 9002   format %{ "shrl    $dst, $shift" %}
 9003   ins_encode %{
 9004     __ shrl($dst$$Address, $shift$$constant);
 9005   %}
 9006   ins_pipe(ialu_mem_imm);
 9007 %}
 9008 
 9009 // Logical Shift Right by variable
 9010 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9011 %{
 9012   match(Set dst (URShiftI dst shift));
 9013   effect(KILL cr);
 9014 
 9015   format %{ "shrl    $dst, $shift" %}
 9016   ins_encode %{
 9017     __ shrl($dst$$Register);
 9018   %}
 9019   ins_pipe(ialu_reg_reg);
 9020 %}
 9021 
 9022 // Logical Shift Right by variable
 9023 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9024 %{
 9025   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9026   effect(KILL cr);
 9027 
 9028   format %{ "shrl    $dst, $shift" %}
 9029   ins_encode %{
 9030     __ shrl($dst$$Address);
 9031   %}
 9032   ins_pipe(ialu_mem_reg);
 9033 %}
 9034 
 9035 // Long Shift Instructions
 9036 // Shift Left by one
 9037 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9038 %{
 9039   match(Set dst (LShiftL dst shift));
 9040   effect(KILL cr);
 9041 
 9042   format %{ "salq    $dst, $shift" %}
 9043   ins_encode %{
 9044     __ salq($dst$$Register, $shift$$constant);
 9045   %}
 9046   ins_pipe(ialu_reg);
 9047 %}
 9048 
 9049 // Shift Left by one
 9050 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9051 %{
 9052   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9053   effect(KILL cr);
 9054 
 9055   format %{ "salq    $dst, $shift" %}
 9056   ins_encode %{
 9057     __ salq($dst$$Address, $shift$$constant);
 9058   %}
 9059   ins_pipe(ialu_mem_imm);
 9060 %}
 9061 
 9062 // Shift Left by 8-bit immediate
 9063 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9064 %{
 9065   match(Set dst (LShiftL dst shift));
 9066   effect(KILL cr);
 9067 
 9068   format %{ "salq    $dst, $shift" %}
 9069   ins_encode %{
 9070     __ salq($dst$$Register, $shift$$constant);
 9071   %}
 9072   ins_pipe(ialu_reg);
 9073 %}
 9074 
 9075 // Shift Left by 8-bit immediate
 9076 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9077 %{
 9078   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9079   effect(KILL cr);
 9080 
 9081   format %{ "salq    $dst, $shift" %}
 9082   ins_encode %{
 9083     __ salq($dst$$Address, $shift$$constant);
 9084   %}
 9085   ins_pipe(ialu_mem_imm);
 9086 %}
 9087 
 9088 // Shift Left by variable
 9089 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9090 %{
 9091   match(Set dst (LShiftL dst shift));
 9092   effect(KILL cr);
 9093 
 9094   format %{ "salq    $dst, $shift" %}
 9095   ins_encode %{
 9096     __ salq($dst$$Register);
 9097   %}
 9098   ins_pipe(ialu_reg_reg);
 9099 %}
 9100 
 9101 // Shift Left by variable
 9102 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9103 %{
 9104   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9105   effect(KILL cr);
 9106 
 9107   format %{ "salq    $dst, $shift" %}
 9108   ins_encode %{
 9109     __ salq($dst$$Address);
 9110   %}
 9111   ins_pipe(ialu_mem_reg);
 9112 %}
 9113 
 9114 // Arithmetic shift right by one
 9115 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9116 %{
 9117   match(Set dst (RShiftL dst shift));
 9118   effect(KILL cr);
 9119 
 9120   format %{ "sarq    $dst, $shift" %}
 9121   ins_encode %{
 9122     __ sarq($dst$$Register, $shift$$constant);
 9123   %}
 9124   ins_pipe(ialu_reg);
 9125 %}
 9126 
 9127 // Arithmetic shift right by one
 9128 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9129 %{
 9130   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9131   effect(KILL cr);
 9132 
 9133   format %{ "sarq    $dst, $shift" %}
 9134   ins_encode %{
 9135     __ sarq($dst$$Address, $shift$$constant);
 9136   %}
 9137   ins_pipe(ialu_mem_imm);
 9138 %}
 9139 
 9140 // Arithmetic Shift Right by 8-bit immediate
 9141 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9142 %{
 9143   match(Set dst (RShiftL dst shift));
 9144   effect(KILL cr);
 9145 
 9146   format %{ "sarq    $dst, $shift" %}
 9147   ins_encode %{
 9148     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9149   %}
 9150   ins_pipe(ialu_mem_imm);
 9151 %}
 9152 
 9153 // Arithmetic Shift Right by 8-bit immediate
 9154 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9155 %{
 9156   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9157   effect(KILL cr);
 9158 
 9159   format %{ "sarq    $dst, $shift" %}
 9160   ins_encode %{
 9161     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9162   %}
 9163   ins_pipe(ialu_mem_imm);
 9164 %}
 9165 
 9166 // Arithmetic Shift Right by variable
 9167 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9168 %{
 9169   match(Set dst (RShiftL dst shift));
 9170   effect(KILL cr);
 9171 
 9172   format %{ "sarq    $dst, $shift" %}
 9173   ins_encode %{
 9174     __ sarq($dst$$Register);
 9175   %}
 9176   ins_pipe(ialu_reg_reg);
 9177 %}
 9178 
 9179 // Arithmetic Shift Right by variable
 9180 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9181 %{
 9182   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9183   effect(KILL cr);
 9184 
 9185   format %{ "sarq    $dst, $shift" %}
 9186   ins_encode %{
 9187     __ sarq($dst$$Address);
 9188   %}
 9189   ins_pipe(ialu_mem_reg);
 9190 %}
 9191 
 9192 // Logical shift right by one
 9193 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9194 %{
 9195   match(Set dst (URShiftL dst shift));
 9196   effect(KILL cr);
 9197 
 9198   format %{ "shrq    $dst, $shift" %}
 9199   ins_encode %{
 9200     __ shrq($dst$$Register, $shift$$constant);
 9201   %}
 9202   ins_pipe(ialu_reg);
 9203 %}
 9204 
 9205 // Logical shift right by one
 9206 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9207 %{
 9208   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9209   effect(KILL cr);
 9210 
 9211   format %{ "shrq    $dst, $shift" %}
 9212   ins_encode %{
 9213     __ shrq($dst$$Address, $shift$$constant);
 9214   %}
 9215   ins_pipe(ialu_mem_imm);
 9216 %}
 9217 
 9218 // Logical Shift Right by 8-bit immediate
 9219 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9220 %{
 9221   match(Set dst (URShiftL dst shift));
 9222   effect(KILL cr);
 9223 
 9224   format %{ "shrq    $dst, $shift" %}
 9225   ins_encode %{
 9226     __ shrq($dst$$Register, $shift$$constant);
 9227   %}
 9228   ins_pipe(ialu_reg);
 9229 %}
 9230 
 9231 // Logical Shift Right by 8-bit immediate
 9232 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9233 %{
 9234   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9235   effect(KILL cr);
 9236 
 9237   format %{ "shrq    $dst, $shift" %}
 9238   ins_encode %{
 9239     __ shrq($dst$$Address, $shift$$constant);
 9240   %}
 9241   ins_pipe(ialu_mem_imm);
 9242 %}
 9243 
 9244 // Logical Shift Right by variable
 9245 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9246 %{
 9247   match(Set dst (URShiftL dst shift));
 9248   effect(KILL cr);
 9249 
 9250   format %{ "shrq    $dst, $shift" %}
 9251   ins_encode %{
 9252     __ shrq($dst$$Register);
 9253   %}
 9254   ins_pipe(ialu_reg_reg);
 9255 %}
 9256 
 9257 // Logical Shift Right by variable
 9258 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9259 %{
 9260   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9261   effect(KILL cr);
 9262 
 9263   format %{ "shrq    $dst, $shift" %}
 9264   ins_encode %{
 9265     __ shrq($dst$$Address);
 9266   %}
 9267   ins_pipe(ialu_mem_reg);
 9268 %}
 9269 
 9270 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9271 // This idiom is used by the compiler for the i2b bytecode.
 9272 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9273 %{
 9274   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9275 
 9276   format %{ "movsbl  $dst, $src\t# i2b" %}
 9277   ins_encode %{
 9278     __ movsbl($dst$$Register, $src$$Register);
 9279   %}
 9280   ins_pipe(ialu_reg_reg);
 9281 %}
 9282 
 9283 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9284 // This idiom is used by the compiler the i2s bytecode.
 9285 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9286 %{
 9287   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9288 
 9289   format %{ "movswl  $dst, $src\t# i2s" %}
 9290   ins_encode %{
 9291     __ movswl($dst$$Register, $src$$Register);
 9292   %}
 9293   ins_pipe(ialu_reg_reg);
 9294 %}
 9295 
 9296 // ROL/ROR instructions
 9297 
 9298 // Rotate left by constant.
 9299 instruct rolI_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9300 %{
 9301   predicate(n->bottom_type()->basic_type() == T_INT);
 9302   match(Set dst (RotateLeft dst shift));
 9303   effect(KILL cr);
 9304   format %{ "roll    $dst, $shift" %}
 9305   ins_encode %{
 9306     __ roll($dst$$Register, $shift$$constant);
 9307   %}
 9308   ins_pipe(ialu_reg);
 9309 %}
 9310 
 9311 // Rotate Left by variable
 9312 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9313 %{
 9314   predicate(n->bottom_type()->basic_type() == T_INT);
 9315   match(Set dst (RotateLeft dst shift));
 9316   effect(KILL cr);
 9317   format %{ "roll    $dst, $shift" %}
 9318   ins_encode %{
 9319     __ roll($dst$$Register);
 9320   %}
 9321   ins_pipe(ialu_reg_reg);
 9322 %}
 9323 
 9324 // Rotate Right by constant.
 9325 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9326 %{
 9327   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9328   match(Set dst (RotateRight dst shift));
 9329   effect(KILL cr);
 9330   format %{ "rorl    $dst, $shift" %}
 9331   ins_encode %{
 9332     __ rorl($dst$$Register, $shift$$constant);
 9333   %}
 9334   ins_pipe(ialu_reg);
 9335 %}
 9336 
 9337 // Rotate Right by constant.
 9338 instruct rorI_immI8(rRegI dst, immI8 shift)
 9339 %{
 9340   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9341   match(Set dst (RotateRight dst shift));
 9342   format %{ "rorxd     $dst, $shift" %}
 9343   ins_encode %{
 9344     __ rorxd($dst$$Register, $dst$$Register, $shift$$constant);
 9345   %}
 9346   ins_pipe(ialu_reg_reg);
 9347 %}
 9348 
 9349 // Rotate Right by variable
 9350 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9351 %{
 9352   predicate(n->bottom_type()->basic_type() == T_INT);
 9353   match(Set dst (RotateRight dst shift));
 9354   effect(KILL cr);
 9355   format %{ "rorl    $dst, $shift" %}
 9356   ins_encode %{
 9357     __ rorl($dst$$Register);
 9358   %}
 9359   ins_pipe(ialu_reg_reg);
 9360 %}
 9361 
 9362 
 9363 // Rotate Left by constant.
 9364 instruct rolL_immI8(rRegL dst, immI8 shift, rFlagsReg cr)
 9365 %{
 9366   predicate(n->bottom_type()->basic_type() == T_LONG);
 9367   match(Set dst (RotateLeft dst shift));
 9368   effect(KILL cr);
 9369   format %{ "rolq    $dst, $shift" %}
 9370   ins_encode %{
 9371     __ rolq($dst$$Register, $shift$$constant);
 9372   %}
 9373   ins_pipe(ialu_reg);
 9374 %}
 9375 
 9376 // Rotate Left by variable
 9377 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9378 %{
 9379   predicate(n->bottom_type()->basic_type() == T_LONG);
 9380   match(Set dst (RotateLeft dst shift));
 9381   effect(KILL cr);
 9382   format %{ "rolq    $dst, $shift" %}
 9383   ins_encode %{
 9384     __ rolq($dst$$Register);
 9385   %}
 9386   ins_pipe(ialu_reg_reg);
 9387 %}
 9388 
 9389 
 9390 // Rotate Right by constant.
 9391 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9392 %{
 9393   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9394   match(Set dst (RotateRight dst shift));
 9395   effect(KILL cr);
 9396   format %{ "rorq    $dst, $shift" %}
 9397   ins_encode %{
 9398     __ rorq($dst$$Register, $shift$$constant);
 9399   %}
 9400   ins_pipe(ialu_reg);
 9401 %}
 9402 
 9403 
 9404 // Rotate Right by constant
 9405 instruct rorL_immI8(rRegL dst, immI8 shift)
 9406 %{
 9407   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9408   match(Set dst (RotateRight dst shift));
 9409   format %{ "rorxq    $dst, $shift" %}
 9410   ins_encode %{
 9411     __ rorxq($dst$$Register, $dst$$Register, $shift$$constant);
 9412   %}
 9413   ins_pipe(ialu_reg_reg);
 9414 %}
 9415 
 9416 // Rotate Right by variable
 9417 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9418 %{
 9419   predicate(n->bottom_type()->basic_type() == T_LONG);
 9420   match(Set dst (RotateRight dst shift));
 9421   effect(KILL cr);
 9422   format %{ "rorq    $dst, $shift" %}
 9423   ins_encode %{
 9424     __ rorq($dst$$Register);
 9425   %}
 9426   ins_pipe(ialu_reg_reg);
 9427 %}
 9428 
 9429 
 9430 // Logical Instructions
 9431 
 9432 // Integer Logical Instructions
 9433 
 9434 // And Instructions
 9435 // And Register with Register
 9436 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9437 %{
 9438   match(Set dst (AndI dst src));
 9439   effect(KILL cr);
 9440 
 9441   format %{ "andl    $dst, $src\t# int" %}
 9442   ins_encode %{
 9443     __ andl($dst$$Register, $src$$Register);
 9444   %}
 9445   ins_pipe(ialu_reg_reg);
 9446 %}
 9447 
 9448 // And Register with Immediate 255
 9449 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
 9450 %{
 9451   match(Set dst (AndI dst src));
 9452 
 9453   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
 9454   ins_encode %{
 9455     __ movzbl($dst$$Register, $dst$$Register);
 9456   %}
 9457   ins_pipe(ialu_reg);
 9458 %}
 9459 
 9460 // And Register with Immediate 255 and promote to long
 9461 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9462 %{
 9463   match(Set dst (ConvI2L (AndI src mask)));
 9464 
 9465   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9466   ins_encode %{
 9467     __ movzbl($dst$$Register, $src$$Register);
 9468   %}
 9469   ins_pipe(ialu_reg);
 9470 %}
 9471 
 9472 // And Register with Immediate 65535
 9473 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
 9474 %{
 9475   match(Set dst (AndI dst src));
 9476 
 9477   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
 9478   ins_encode %{
 9479     __ movzwl($dst$$Register, $dst$$Register);
 9480   %}
 9481   ins_pipe(ialu_reg);
 9482 %}
 9483 
 9484 // And Register with Immediate 65535 and promote to long
 9485 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9486 %{
 9487   match(Set dst (ConvI2L (AndI src mask)));
 9488 
 9489   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9490   ins_encode %{
 9491     __ movzwl($dst$$Register, $src$$Register);
 9492   %}
 9493   ins_pipe(ialu_reg);
 9494 %}
 9495 
 9496 // Can skip int2long conversions after AND with small bitmask
 9497 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9498 %{
 9499   predicate(VM_Version::supports_bmi2());
 9500   ins_cost(125);
 9501   effect(TEMP tmp, KILL cr);
 9502   match(Set dst (ConvI2L (AndI src mask)));
 9503   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9504   ins_encode %{
 9505     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9506     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9507   %}
 9508   ins_pipe(ialu_reg_reg);
 9509 %}
 9510 
 9511 // And Register with Immediate
 9512 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9513 %{
 9514   match(Set dst (AndI dst src));
 9515   effect(KILL cr);
 9516 
 9517   format %{ "andl    $dst, $src\t# int" %}
 9518   ins_encode %{
 9519     __ andl($dst$$Register, $src$$constant);
 9520   %}
 9521   ins_pipe(ialu_reg);
 9522 %}
 9523 
 9524 // And Register with Memory
 9525 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9526 %{
 9527   match(Set dst (AndI dst (LoadI src)));
 9528   effect(KILL cr);
 9529 
 9530   ins_cost(125);
 9531   format %{ "andl    $dst, $src\t# int" %}
 9532   ins_encode %{
 9533     __ andl($dst$$Register, $src$$Address);
 9534   %}
 9535   ins_pipe(ialu_reg_mem);
 9536 %}
 9537 
 9538 // And Memory with Register
 9539 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9540 %{
 9541   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9542   effect(KILL cr);
 9543 
 9544   ins_cost(150);
 9545   format %{ "andb    $dst, $src\t# byte" %}
 9546   ins_encode %{
 9547     __ andb($dst$$Address, $src$$Register);
 9548   %}
 9549   ins_pipe(ialu_mem_reg);
 9550 %}
 9551 
 9552 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9553 %{
 9554   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9555   effect(KILL cr);
 9556 
 9557   ins_cost(150);
 9558   format %{ "andl    $dst, $src\t# int" %}
 9559   ins_encode %{
 9560     __ andl($dst$$Address, $src$$Register);
 9561   %}
 9562   ins_pipe(ialu_mem_reg);
 9563 %}
 9564 
 9565 // And Memory with Immediate
 9566 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9567 %{
 9568   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9569   effect(KILL cr);
 9570 
 9571   ins_cost(125);
 9572   format %{ "andl    $dst, $src\t# int" %}
 9573   ins_encode %{
 9574     __ andl($dst$$Address, $src$$constant);
 9575   %}
 9576   ins_pipe(ialu_mem_imm);
 9577 %}
 9578 
 9579 // BMI1 instructions
 9580 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9581   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9582   predicate(UseBMI1Instructions);
 9583   effect(KILL cr);
 9584 
 9585   ins_cost(125);
 9586   format %{ "andnl  $dst, $src1, $src2" %}
 9587 
 9588   ins_encode %{
 9589     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9590   %}
 9591   ins_pipe(ialu_reg_mem);
 9592 %}
 9593 
 9594 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9595   match(Set dst (AndI (XorI src1 minus_1) src2));
 9596   predicate(UseBMI1Instructions);
 9597   effect(KILL cr);
 9598 
 9599   format %{ "andnl  $dst, $src1, $src2" %}
 9600 
 9601   ins_encode %{
 9602     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9603   %}
 9604   ins_pipe(ialu_reg);
 9605 %}
 9606 
 9607 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9608   match(Set dst (AndI (SubI imm_zero src) src));
 9609   predicate(UseBMI1Instructions);
 9610   effect(KILL cr);
 9611 
 9612   format %{ "blsil  $dst, $src" %}
 9613 
 9614   ins_encode %{
 9615     __ blsil($dst$$Register, $src$$Register);
 9616   %}
 9617   ins_pipe(ialu_reg);
 9618 %}
 9619 
 9620 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9621   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9622   predicate(UseBMI1Instructions);
 9623   effect(KILL cr);
 9624 
 9625   ins_cost(125);
 9626   format %{ "blsil  $dst, $src" %}
 9627 
 9628   ins_encode %{
 9629     __ blsil($dst$$Register, $src$$Address);
 9630   %}
 9631   ins_pipe(ialu_reg_mem);
 9632 %}
 9633 
 9634 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9635 %{
 9636   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9637   predicate(UseBMI1Instructions);
 9638   effect(KILL cr);
 9639 
 9640   ins_cost(125);
 9641   format %{ "blsmskl $dst, $src" %}
 9642 
 9643   ins_encode %{
 9644     __ blsmskl($dst$$Register, $src$$Address);
 9645   %}
 9646   ins_pipe(ialu_reg_mem);
 9647 %}
 9648 
 9649 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9650 %{
 9651   match(Set dst (XorI (AddI src minus_1) src));
 9652   predicate(UseBMI1Instructions);
 9653   effect(KILL cr);
 9654 
 9655   format %{ "blsmskl $dst, $src" %}
 9656 
 9657   ins_encode %{
 9658     __ blsmskl($dst$$Register, $src$$Register);
 9659   %}
 9660 
 9661   ins_pipe(ialu_reg);
 9662 %}
 9663 
 9664 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9665 %{
 9666   match(Set dst (AndI (AddI src minus_1) src) );
 9667   predicate(UseBMI1Instructions);
 9668   effect(KILL cr);
 9669 
 9670   format %{ "blsrl  $dst, $src" %}
 9671 
 9672   ins_encode %{
 9673     __ blsrl($dst$$Register, $src$$Register);
 9674   %}
 9675 
 9676   ins_pipe(ialu_reg_mem);
 9677 %}
 9678 
 9679 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9680 %{
 9681   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9682   predicate(UseBMI1Instructions);
 9683   effect(KILL cr);
 9684 
 9685   ins_cost(125);
 9686   format %{ "blsrl  $dst, $src" %}
 9687 
 9688   ins_encode %{
 9689     __ blsrl($dst$$Register, $src$$Address);
 9690   %}
 9691 
 9692   ins_pipe(ialu_reg);
 9693 %}
 9694 
 9695 // Or Instructions
 9696 // Or Register with Register
 9697 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9698 %{
 9699   match(Set dst (OrI dst src));
 9700   effect(KILL cr);
 9701 
 9702   format %{ "orl     $dst, $src\t# int" %}
 9703   ins_encode %{
 9704     __ orl($dst$$Register, $src$$Register);
 9705   %}
 9706   ins_pipe(ialu_reg_reg);
 9707 %}
 9708 
 9709 // Or Register with Immediate
 9710 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9711 %{
 9712   match(Set dst (OrI dst src));
 9713   effect(KILL cr);
 9714 
 9715   format %{ "orl     $dst, $src\t# int" %}
 9716   ins_encode %{
 9717     __ orl($dst$$Register, $src$$constant);
 9718   %}
 9719   ins_pipe(ialu_reg);
 9720 %}
 9721 
 9722 // Or Register with Memory
 9723 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9724 %{
 9725   match(Set dst (OrI dst (LoadI src)));
 9726   effect(KILL cr);
 9727 
 9728   ins_cost(125);
 9729   format %{ "orl     $dst, $src\t# int" %}
 9730   ins_encode %{
 9731     __ orl($dst$$Register, $src$$Address);
 9732   %}
 9733   ins_pipe(ialu_reg_mem);
 9734 %}
 9735 
 9736 // Or Memory with Register
 9737 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9738 %{
 9739   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9740   effect(KILL cr);
 9741 
 9742   ins_cost(150);
 9743   format %{ "orb    $dst, $src\t# byte" %}
 9744   ins_encode %{
 9745     __ orb($dst$$Address, $src$$Register);
 9746   %}
 9747   ins_pipe(ialu_mem_reg);
 9748 %}
 9749 
 9750 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9751 %{
 9752   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9753   effect(KILL cr);
 9754 
 9755   ins_cost(150);
 9756   format %{ "orl     $dst, $src\t# int" %}
 9757   ins_encode %{
 9758     __ orl($dst$$Address, $src$$Register);
 9759   %}
 9760   ins_pipe(ialu_mem_reg);
 9761 %}
 9762 
 9763 // Or Memory with Immediate
 9764 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9765 %{
 9766   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9767   effect(KILL cr);
 9768 
 9769   ins_cost(125);
 9770   format %{ "orl     $dst, $src\t# int" %}
 9771   ins_encode %{
 9772     __ orl($dst$$Address, $src$$constant);
 9773   %}
 9774   ins_pipe(ialu_mem_imm);
 9775 %}
 9776 
 9777 // Xor Instructions
 9778 // Xor Register with Register
 9779 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9780 %{
 9781   match(Set dst (XorI dst src));
 9782   effect(KILL cr);
 9783 
 9784   format %{ "xorl    $dst, $src\t# int" %}
 9785   ins_encode %{
 9786     __ xorl($dst$$Register, $src$$Register);
 9787   %}
 9788   ins_pipe(ialu_reg_reg);
 9789 %}
 9790 
 9791 // Xor Register with Immediate -1
 9792 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9793   match(Set dst (XorI dst imm));
 9794 
 9795   format %{ "not    $dst" %}
 9796   ins_encode %{
 9797      __ notl($dst$$Register);
 9798   %}
 9799   ins_pipe(ialu_reg);
 9800 %}
 9801 
 9802 // Xor Register with Immediate
 9803 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9804 %{
 9805   match(Set dst (XorI dst src));
 9806   effect(KILL cr);
 9807 
 9808   format %{ "xorl    $dst, $src\t# int" %}
 9809   ins_encode %{
 9810     __ xorl($dst$$Register, $src$$constant);
 9811   %}
 9812   ins_pipe(ialu_reg);
 9813 %}
 9814 
 9815 // Xor Register with Memory
 9816 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9817 %{
 9818   match(Set dst (XorI dst (LoadI src)));
 9819   effect(KILL cr);
 9820 
 9821   ins_cost(125);
 9822   format %{ "xorl    $dst, $src\t# int" %}
 9823   ins_encode %{
 9824     __ xorl($dst$$Register, $src$$Address);
 9825   %}
 9826   ins_pipe(ialu_reg_mem);
 9827 %}
 9828 
 9829 // Xor Memory with Register
 9830 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9831 %{
 9832   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9833   effect(KILL cr);
 9834 
 9835   ins_cost(150);
 9836   format %{ "xorb    $dst, $src\t# byte" %}
 9837   ins_encode %{
 9838     __ xorb($dst$$Address, $src$$Register);
 9839   %}
 9840   ins_pipe(ialu_mem_reg);
 9841 %}
 9842 
 9843 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9844 %{
 9845   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9846   effect(KILL cr);
 9847 
 9848   ins_cost(150);
 9849   format %{ "xorl    $dst, $src\t# int" %}
 9850   ins_encode %{
 9851     __ xorl($dst$$Address, $src$$Register);
 9852   %}
 9853   ins_pipe(ialu_mem_reg);
 9854 %}
 9855 
 9856 // Xor Memory with Immediate
 9857 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9858 %{
 9859   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9860   effect(KILL cr);
 9861 
 9862   ins_cost(125);
 9863   format %{ "xorl    $dst, $src\t# int" %}
 9864   ins_encode %{
 9865     __ xorl($dst$$Address, $src$$constant);
 9866   %}
 9867   ins_pipe(ialu_mem_imm);
 9868 %}
 9869 
 9870 
 9871 // Long Logical Instructions
 9872 
 9873 // And Instructions
 9874 // And Register with Register
 9875 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9876 %{
 9877   match(Set dst (AndL dst src));
 9878   effect(KILL cr);
 9879 
 9880   format %{ "andq    $dst, $src\t# long" %}
 9881   ins_encode %{
 9882     __ andq($dst$$Register, $src$$Register);
 9883   %}
 9884   ins_pipe(ialu_reg_reg);
 9885 %}
 9886 
 9887 // And Register with Immediate 255
 9888 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
 9889 %{
 9890   match(Set dst (AndL dst src));
 9891 
 9892   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
 9893   ins_encode %{
 9894     __ movzbq($dst$$Register, $dst$$Register);
 9895   %}
 9896   ins_pipe(ialu_reg);
 9897 %}
 9898 
 9899 // And Register with Immediate 65535
 9900 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
 9901 %{
 9902   match(Set dst (AndL dst src));
 9903 
 9904   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
 9905   ins_encode %{
 9906     __ movzwq($dst$$Register, $dst$$Register);
 9907   %}
 9908   ins_pipe(ialu_reg);
 9909 %}
 9910 
 9911 // And Register with Immediate
 9912 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9913 %{
 9914   match(Set dst (AndL dst src));
 9915   effect(KILL cr);
 9916 
 9917   format %{ "andq    $dst, $src\t# long" %}
 9918   ins_encode %{
 9919     __ andq($dst$$Register, $src$$constant);
 9920   %}
 9921   ins_pipe(ialu_reg);
 9922 %}
 9923 
 9924 // And Register with Memory
 9925 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9926 %{
 9927   match(Set dst (AndL dst (LoadL src)));
 9928   effect(KILL cr);
 9929 
 9930   ins_cost(125);
 9931   format %{ "andq    $dst, $src\t# long" %}
 9932   ins_encode %{
 9933     __ andq($dst$$Register, $src$$Address);
 9934   %}
 9935   ins_pipe(ialu_reg_mem);
 9936 %}
 9937 
 9938 // And Memory with Register
 9939 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9940 %{
 9941   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9942   effect(KILL cr);
 9943 
 9944   ins_cost(150);
 9945   format %{ "andq    $dst, $src\t# long" %}
 9946   ins_encode %{
 9947     __ andq($dst$$Address, $src$$Register);
 9948   %}
 9949   ins_pipe(ialu_mem_reg);
 9950 %}
 9951 
 9952 // And Memory with Immediate
 9953 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9954 %{
 9955   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9956   effect(KILL cr);
 9957 
 9958   ins_cost(125);
 9959   format %{ "andq    $dst, $src\t# long" %}
 9960   ins_encode %{
 9961     __ andq($dst$$Address, $src$$constant);
 9962   %}
 9963   ins_pipe(ialu_mem_imm);
 9964 %}
 9965 
 9966 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9967 %{
 9968   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9969   // because AND/OR works well enough for 8/32-bit values.
 9970   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9971 
 9972   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9973   effect(KILL cr);
 9974 
 9975   ins_cost(125);
 9976   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9977   ins_encode %{
 9978     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9979   %}
 9980   ins_pipe(ialu_mem_imm);
 9981 %}
 9982 
 9983 // BMI1 instructions
 9984 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9985   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9986   predicate(UseBMI1Instructions);
 9987   effect(KILL cr);
 9988 
 9989   ins_cost(125);
 9990   format %{ "andnq  $dst, $src1, $src2" %}
 9991 
 9992   ins_encode %{
 9993     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9994   %}
 9995   ins_pipe(ialu_reg_mem);
 9996 %}
 9997 
 9998 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9999   match(Set dst (AndL (XorL src1 minus_1) src2));
10000   predicate(UseBMI1Instructions);
10001   effect(KILL cr);
10002 
10003   format %{ "andnq  $dst, $src1, $src2" %}
10004 
10005   ins_encode %{
10006   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10007   %}
10008   ins_pipe(ialu_reg_mem);
10009 %}
10010 
10011 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10012   match(Set dst (AndL (SubL imm_zero src) src));
10013   predicate(UseBMI1Instructions);
10014   effect(KILL cr);
10015 
10016   format %{ "blsiq  $dst, $src" %}
10017 
10018   ins_encode %{
10019     __ blsiq($dst$$Register, $src$$Register);
10020   %}
10021   ins_pipe(ialu_reg);
10022 %}
10023 
10024 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10025   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10026   predicate(UseBMI1Instructions);
10027   effect(KILL cr);
10028 
10029   ins_cost(125);
10030   format %{ "blsiq  $dst, $src" %}
10031 
10032   ins_encode %{
10033     __ blsiq($dst$$Register, $src$$Address);
10034   %}
10035   ins_pipe(ialu_reg_mem);
10036 %}
10037 
10038 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10039 %{
10040   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10041   predicate(UseBMI1Instructions);
10042   effect(KILL cr);
10043 
10044   ins_cost(125);
10045   format %{ "blsmskq $dst, $src" %}
10046 
10047   ins_encode %{
10048     __ blsmskq($dst$$Register, $src$$Address);
10049   %}
10050   ins_pipe(ialu_reg_mem);
10051 %}
10052 
10053 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10054 %{
10055   match(Set dst (XorL (AddL src minus_1) src));
10056   predicate(UseBMI1Instructions);
10057   effect(KILL cr);
10058 
10059   format %{ "blsmskq $dst, $src" %}
10060 
10061   ins_encode %{
10062     __ blsmskq($dst$$Register, $src$$Register);
10063   %}
10064 
10065   ins_pipe(ialu_reg);
10066 %}
10067 
10068 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10069 %{
10070   match(Set dst (AndL (AddL src minus_1) src) );
10071   predicate(UseBMI1Instructions);
10072   effect(KILL cr);
10073 
10074   format %{ "blsrq  $dst, $src" %}
10075 
10076   ins_encode %{
10077     __ blsrq($dst$$Register, $src$$Register);
10078   %}
10079 
10080   ins_pipe(ialu_reg);
10081 %}
10082 
10083 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10084 %{
10085   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10086   predicate(UseBMI1Instructions);
10087   effect(KILL cr);
10088 
10089   ins_cost(125);
10090   format %{ "blsrq  $dst, $src" %}
10091 
10092   ins_encode %{
10093     __ blsrq($dst$$Register, $src$$Address);
10094   %}
10095 
10096   ins_pipe(ialu_reg);
10097 %}
10098 
10099 // Or Instructions
10100 // Or Register with Register
10101 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10102 %{
10103   match(Set dst (OrL dst src));
10104   effect(KILL cr);
10105 
10106   format %{ "orq     $dst, $src\t# long" %}
10107   ins_encode %{
10108     __ orq($dst$$Register, $src$$Register);
10109   %}
10110   ins_pipe(ialu_reg_reg);
10111 %}
10112 
10113 // Use any_RegP to match R15 (TLS register) without spilling.
10114 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10115   match(Set dst (OrL dst (CastP2X src)));
10116   effect(KILL cr);
10117 
10118   format %{ "orq     $dst, $src\t# long" %}
10119   ins_encode %{
10120     __ orq($dst$$Register, $src$$Register);
10121   %}
10122   ins_pipe(ialu_reg_reg);
10123 %}
10124 
10125 
10126 // Or Register with Immediate
10127 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10128 %{
10129   match(Set dst (OrL dst src));
10130   effect(KILL cr);
10131 
10132   format %{ "orq     $dst, $src\t# long" %}
10133   ins_encode %{
10134     __ orq($dst$$Register, $src$$constant);
10135   %}
10136   ins_pipe(ialu_reg);
10137 %}
10138 
10139 // Or Register with Memory
10140 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10141 %{
10142   match(Set dst (OrL dst (LoadL src)));
10143   effect(KILL cr);
10144 
10145   ins_cost(125);
10146   format %{ "orq     $dst, $src\t# long" %}
10147   ins_encode %{
10148     __ orq($dst$$Register, $src$$Address);
10149   %}
10150   ins_pipe(ialu_reg_mem);
10151 %}
10152 
10153 // Or Memory with Register
10154 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10155 %{
10156   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10157   effect(KILL cr);
10158 
10159   ins_cost(150);
10160   format %{ "orq     $dst, $src\t# long" %}
10161   ins_encode %{
10162     __ orq($dst$$Address, $src$$Register);
10163   %}
10164   ins_pipe(ialu_mem_reg);
10165 %}
10166 
10167 // Or Memory with Immediate
10168 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10169 %{
10170   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10171   effect(KILL cr);
10172 
10173   ins_cost(125);
10174   format %{ "orq     $dst, $src\t# long" %}
10175   ins_encode %{
10176     __ orq($dst$$Address, $src$$constant);
10177   %}
10178   ins_pipe(ialu_mem_imm);
10179 %}
10180 
10181 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10182 %{
10183   // con should be a pure 64-bit power of 2 immediate
10184   // because AND/OR works well enough for 8/32-bit values.
10185   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10186 
10187   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10188   effect(KILL cr);
10189 
10190   ins_cost(125);
10191   format %{ "btsq    $dst, log2($con)\t# long" %}
10192   ins_encode %{
10193     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10194   %}
10195   ins_pipe(ialu_mem_imm);
10196 %}
10197 
10198 // Xor Instructions
10199 // Xor Register with Register
10200 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10201 %{
10202   match(Set dst (XorL dst src));
10203   effect(KILL cr);
10204 
10205   format %{ "xorq    $dst, $src\t# long" %}
10206   ins_encode %{
10207     __ xorq($dst$$Register, $src$$Register);
10208   %}
10209   ins_pipe(ialu_reg_reg);
10210 %}
10211 
10212 // Xor Register with Immediate -1
10213 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10214   match(Set dst (XorL dst imm));
10215 
10216   format %{ "notq   $dst" %}
10217   ins_encode %{
10218      __ notq($dst$$Register);
10219   %}
10220   ins_pipe(ialu_reg);
10221 %}
10222 
10223 // Xor Register with Immediate
10224 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10225 %{
10226   match(Set dst (XorL dst src));
10227   effect(KILL cr);
10228 
10229   format %{ "xorq    $dst, $src\t# long" %}
10230   ins_encode %{
10231     __ xorq($dst$$Register, $src$$constant);
10232   %}
10233   ins_pipe(ialu_reg);
10234 %}
10235 
10236 // Xor Register with Memory
10237 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10238 %{
10239   match(Set dst (XorL dst (LoadL src)));
10240   effect(KILL cr);
10241 
10242   ins_cost(125);
10243   format %{ "xorq    $dst, $src\t# long" %}
10244   ins_encode %{
10245     __ xorq($dst$$Register, $src$$Address);
10246   %}
10247   ins_pipe(ialu_reg_mem);
10248 %}
10249 
10250 // Xor Memory with Register
10251 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10252 %{
10253   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10254   effect(KILL cr);
10255 
10256   ins_cost(150);
10257   format %{ "xorq    $dst, $src\t# long" %}
10258   ins_encode %{
10259     __ xorq($dst$$Address, $src$$Register);
10260   %}
10261   ins_pipe(ialu_mem_reg);
10262 %}
10263 
10264 // Xor Memory with Immediate
10265 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10266 %{
10267   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10268   effect(KILL cr);
10269 
10270   ins_cost(125);
10271   format %{ "xorq    $dst, $src\t# long" %}
10272   ins_encode %{
10273     __ xorq($dst$$Address, $src$$constant);
10274   %}
10275   ins_pipe(ialu_mem_imm);
10276 %}
10277 
10278 // Convert Int to Boolean
10279 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10280 %{
10281   match(Set dst (Conv2B src));
10282   effect(KILL cr);
10283 
10284   format %{ "testl   $src, $src\t# ci2b\n\t"
10285             "setnz   $dst\n\t"
10286             "movzbl  $dst, $dst" %}
10287   ins_encode %{
10288     __ testl($src$$Register, $src$$Register);
10289     __ set_byte_if_not_zero($dst$$Register);
10290     __ movzbl($dst$$Register, $dst$$Register);
10291   %}
10292   ins_pipe(pipe_slow); // XXX
10293 %}
10294 
10295 // Convert Pointer to Boolean
10296 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10297 %{
10298   match(Set dst (Conv2B src));
10299   effect(KILL cr);
10300 
10301   format %{ "testq   $src, $src\t# cp2b\n\t"
10302             "setnz   $dst\n\t"
10303             "movzbl  $dst, $dst" %}
10304   ins_encode %{
10305     __ testq($src$$Register, $src$$Register);
10306     __ set_byte_if_not_zero($dst$$Register);
10307     __ movzbl($dst$$Register, $dst$$Register);
10308   %}
10309   ins_pipe(pipe_slow); // XXX
10310 %}
10311 
10312 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10313 %{
10314   match(Set dst (CmpLTMask p q));
10315   effect(KILL cr);
10316 
10317   ins_cost(400);
10318   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10319             "setlt   $dst\n\t"
10320             "movzbl  $dst, $dst\n\t"
10321             "negl    $dst" %}
10322   ins_encode %{
10323     __ cmpl($p$$Register, $q$$Register);
10324     __ setl($dst$$Register);
10325     __ movzbl($dst$$Register, $dst$$Register);
10326     __ negl($dst$$Register);
10327   %}
10328   ins_pipe(pipe_slow);
10329 %}
10330 
10331 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10332 %{
10333   match(Set dst (CmpLTMask dst zero));
10334   effect(KILL cr);
10335 
10336   ins_cost(100);
10337   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10338   ins_encode %{
10339     __ sarl($dst$$Register, 31);
10340   %}
10341   ins_pipe(ialu_reg);
10342 %}
10343 
10344 /* Better to save a register than avoid a branch */
10345 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10346 %{
10347   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10348   effect(KILL cr);
10349   ins_cost(300);
10350   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10351             "jge     done\n\t"
10352             "addl    $p,$y\n"
10353             "done:   " %}
10354   ins_encode %{
10355     Register Rp = $p$$Register;
10356     Register Rq = $q$$Register;
10357     Register Ry = $y$$Register;
10358     Label done;
10359     __ subl(Rp, Rq);
10360     __ jccb(Assembler::greaterEqual, done);
10361     __ addl(Rp, Ry);
10362     __ bind(done);
10363   %}
10364   ins_pipe(pipe_cmplt);
10365 %}
10366 
10367 /* Better to save a register than avoid a branch */
10368 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10369 %{
10370   match(Set y (AndI (CmpLTMask p q) y));
10371   effect(KILL cr);
10372 
10373   ins_cost(300);
10374 
10375   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10376             "jlt     done\n\t"
10377             "xorl    $y, $y\n"
10378             "done:   " %}
10379   ins_encode %{
10380     Register Rp = $p$$Register;
10381     Register Rq = $q$$Register;
10382     Register Ry = $y$$Register;
10383     Label done;
10384     __ cmpl(Rp, Rq);
10385     __ jccb(Assembler::less, done);
10386     __ xorl(Ry, Ry);
10387     __ bind(done);
10388   %}
10389   ins_pipe(pipe_cmplt);
10390 %}
10391 
10392 
10393 //---------- FP Instructions------------------------------------------------
10394 
10395 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10396 %{
10397   match(Set cr (CmpF src1 src2));
10398 
10399   ins_cost(145);
10400   format %{ "ucomiss $src1, $src2\n\t"
10401             "jnp,s   exit\n\t"
10402             "pushfq\t# saw NaN, set CF\n\t"
10403             "andq    [rsp], #0xffffff2b\n\t"
10404             "popfq\n"
10405     "exit:" %}
10406   ins_encode %{
10407     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10408     emit_cmpfp_fixup(_masm);
10409   %}
10410   ins_pipe(pipe_slow);
10411 %}
10412 
10413 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10414   match(Set cr (CmpF src1 src2));
10415 
10416   ins_cost(100);
10417   format %{ "ucomiss $src1, $src2" %}
10418   ins_encode %{
10419     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10420   %}
10421   ins_pipe(pipe_slow);
10422 %}
10423 
10424 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10425 %{
10426   match(Set cr (CmpF src1 (LoadF src2)));
10427 
10428   ins_cost(145);
10429   format %{ "ucomiss $src1, $src2\n\t"
10430             "jnp,s   exit\n\t"
10431             "pushfq\t# saw NaN, set CF\n\t"
10432             "andq    [rsp], #0xffffff2b\n\t"
10433             "popfq\n"
10434     "exit:" %}
10435   ins_encode %{
10436     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10437     emit_cmpfp_fixup(_masm);
10438   %}
10439   ins_pipe(pipe_slow);
10440 %}
10441 
10442 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10443   match(Set cr (CmpF src1 (LoadF src2)));
10444 
10445   ins_cost(100);
10446   format %{ "ucomiss $src1, $src2" %}
10447   ins_encode %{
10448     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10449   %}
10450   ins_pipe(pipe_slow);
10451 %}
10452 
10453 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10454   match(Set cr (CmpF src con));
10455 
10456   ins_cost(145);
10457   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10458             "jnp,s   exit\n\t"
10459             "pushfq\t# saw NaN, set CF\n\t"
10460             "andq    [rsp], #0xffffff2b\n\t"
10461             "popfq\n"
10462     "exit:" %}
10463   ins_encode %{
10464     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10465     emit_cmpfp_fixup(_masm);
10466   %}
10467   ins_pipe(pipe_slow);
10468 %}
10469 
10470 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10471   match(Set cr (CmpF src con));
10472   ins_cost(100);
10473   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10474   ins_encode %{
10475     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10476   %}
10477   ins_pipe(pipe_slow);
10478 %}
10479 
10480 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10481 %{
10482   match(Set cr (CmpD src1 src2));
10483 
10484   ins_cost(145);
10485   format %{ "ucomisd $src1, $src2\n\t"
10486             "jnp,s   exit\n\t"
10487             "pushfq\t# saw NaN, set CF\n\t"
10488             "andq    [rsp], #0xffffff2b\n\t"
10489             "popfq\n"
10490     "exit:" %}
10491   ins_encode %{
10492     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10493     emit_cmpfp_fixup(_masm);
10494   %}
10495   ins_pipe(pipe_slow);
10496 %}
10497 
10498 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10499   match(Set cr (CmpD src1 src2));
10500 
10501   ins_cost(100);
10502   format %{ "ucomisd $src1, $src2 test" %}
10503   ins_encode %{
10504     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10505   %}
10506   ins_pipe(pipe_slow);
10507 %}
10508 
10509 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10510 %{
10511   match(Set cr (CmpD src1 (LoadD src2)));
10512 
10513   ins_cost(145);
10514   format %{ "ucomisd $src1, $src2\n\t"
10515             "jnp,s   exit\n\t"
10516             "pushfq\t# saw NaN, set CF\n\t"
10517             "andq    [rsp], #0xffffff2b\n\t"
10518             "popfq\n"
10519     "exit:" %}
10520   ins_encode %{
10521     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10522     emit_cmpfp_fixup(_masm);
10523   %}
10524   ins_pipe(pipe_slow);
10525 %}
10526 
10527 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10528   match(Set cr (CmpD src1 (LoadD src2)));
10529 
10530   ins_cost(100);
10531   format %{ "ucomisd $src1, $src2" %}
10532   ins_encode %{
10533     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10534   %}
10535   ins_pipe(pipe_slow);
10536 %}
10537 
10538 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10539   match(Set cr (CmpD src con));
10540 
10541   ins_cost(145);
10542   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10543             "jnp,s   exit\n\t"
10544             "pushfq\t# saw NaN, set CF\n\t"
10545             "andq    [rsp], #0xffffff2b\n\t"
10546             "popfq\n"
10547     "exit:" %}
10548   ins_encode %{
10549     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10550     emit_cmpfp_fixup(_masm);
10551   %}
10552   ins_pipe(pipe_slow);
10553 %}
10554 
10555 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10556   match(Set cr (CmpD src con));
10557   ins_cost(100);
10558   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10559   ins_encode %{
10560     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10561   %}
10562   ins_pipe(pipe_slow);
10563 %}
10564 
10565 // Compare into -1,0,1
10566 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10567 %{
10568   match(Set dst (CmpF3 src1 src2));
10569   effect(KILL cr);
10570 
10571   ins_cost(275);
10572   format %{ "ucomiss $src1, $src2\n\t"
10573             "movl    $dst, #-1\n\t"
10574             "jp,s    done\n\t"
10575             "jb,s    done\n\t"
10576             "setne   $dst\n\t"
10577             "movzbl  $dst, $dst\n"
10578     "done:" %}
10579   ins_encode %{
10580     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10581     emit_cmpfp3(_masm, $dst$$Register);
10582   %}
10583   ins_pipe(pipe_slow);
10584 %}
10585 
10586 // Compare into -1,0,1
10587 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10588 %{
10589   match(Set dst (CmpF3 src1 (LoadF src2)));
10590   effect(KILL cr);
10591 
10592   ins_cost(275);
10593   format %{ "ucomiss $src1, $src2\n\t"
10594             "movl    $dst, #-1\n\t"
10595             "jp,s    done\n\t"
10596             "jb,s    done\n\t"
10597             "setne   $dst\n\t"
10598             "movzbl  $dst, $dst\n"
10599     "done:" %}
10600   ins_encode %{
10601     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10602     emit_cmpfp3(_masm, $dst$$Register);
10603   %}
10604   ins_pipe(pipe_slow);
10605 %}
10606 
10607 // Compare into -1,0,1
10608 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10609   match(Set dst (CmpF3 src con));
10610   effect(KILL cr);
10611 
10612   ins_cost(275);
10613   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10614             "movl    $dst, #-1\n\t"
10615             "jp,s    done\n\t"
10616             "jb,s    done\n\t"
10617             "setne   $dst\n\t"
10618             "movzbl  $dst, $dst\n"
10619     "done:" %}
10620   ins_encode %{
10621     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10622     emit_cmpfp3(_masm, $dst$$Register);
10623   %}
10624   ins_pipe(pipe_slow);
10625 %}
10626 
10627 // Compare into -1,0,1
10628 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10629 %{
10630   match(Set dst (CmpD3 src1 src2));
10631   effect(KILL cr);
10632 
10633   ins_cost(275);
10634   format %{ "ucomisd $src1, $src2\n\t"
10635             "movl    $dst, #-1\n\t"
10636             "jp,s    done\n\t"
10637             "jb,s    done\n\t"
10638             "setne   $dst\n\t"
10639             "movzbl  $dst, $dst\n"
10640     "done:" %}
10641   ins_encode %{
10642     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10643     emit_cmpfp3(_masm, $dst$$Register);
10644   %}
10645   ins_pipe(pipe_slow);
10646 %}
10647 
10648 // Compare into -1,0,1
10649 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10650 %{
10651   match(Set dst (CmpD3 src1 (LoadD src2)));
10652   effect(KILL cr);
10653 
10654   ins_cost(275);
10655   format %{ "ucomisd $src1, $src2\n\t"
10656             "movl    $dst, #-1\n\t"
10657             "jp,s    done\n\t"
10658             "jb,s    done\n\t"
10659             "setne   $dst\n\t"
10660             "movzbl  $dst, $dst\n"
10661     "done:" %}
10662   ins_encode %{
10663     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10664     emit_cmpfp3(_masm, $dst$$Register);
10665   %}
10666   ins_pipe(pipe_slow);
10667 %}
10668 
10669 // Compare into -1,0,1
10670 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10671   match(Set dst (CmpD3 src con));
10672   effect(KILL cr);
10673 
10674   ins_cost(275);
10675   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10676             "movl    $dst, #-1\n\t"
10677             "jp,s    done\n\t"
10678             "jb,s    done\n\t"
10679             "setne   $dst\n\t"
10680             "movzbl  $dst, $dst\n"
10681     "done:" %}
10682   ins_encode %{
10683     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10684     emit_cmpfp3(_masm, $dst$$Register);
10685   %}
10686   ins_pipe(pipe_slow);
10687 %}
10688 
10689 //----------Arithmetic Conversion Instructions---------------------------------
10690 
10691 instruct convF2D_reg_reg(regD dst, regF src)
10692 %{
10693   match(Set dst (ConvF2D src));
10694 
10695   format %{ "cvtss2sd $dst, $src" %}
10696   ins_encode %{
10697     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10698   %}
10699   ins_pipe(pipe_slow); // XXX
10700 %}
10701 
10702 instruct convF2D_reg_mem(regD dst, memory src)
10703 %{
10704   match(Set dst (ConvF2D (LoadF src)));
10705 
10706   format %{ "cvtss2sd $dst, $src" %}
10707   ins_encode %{
10708     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10709   %}
10710   ins_pipe(pipe_slow); // XXX
10711 %}
10712 
10713 instruct convD2F_reg_reg(regF dst, regD src)
10714 %{
10715   match(Set dst (ConvD2F src));
10716 
10717   format %{ "cvtsd2ss $dst, $src" %}
10718   ins_encode %{
10719     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10720   %}
10721   ins_pipe(pipe_slow); // XXX
10722 %}
10723 
10724 instruct convD2F_reg_mem(regF dst, memory src)
10725 %{
10726   match(Set dst (ConvD2F (LoadD src)));
10727 
10728   format %{ "cvtsd2ss $dst, $src" %}
10729   ins_encode %{
10730     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10731   %}
10732   ins_pipe(pipe_slow); // XXX
10733 %}
10734 
10735 // XXX do mem variants
10736 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10737 %{
10738   match(Set dst (ConvF2I src));
10739   effect(KILL cr);
10740   format %{ "convert_f2i $dst,$src" %}
10741   ins_encode %{
10742     __ convert_f2i($dst$$Register, $src$$XMMRegister);
10743   %}
10744   ins_pipe(pipe_slow);
10745 %}
10746 
10747 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10748 %{
10749   match(Set dst (ConvF2L src));
10750   effect(KILL cr);
10751   format %{ "convert_f2l $dst,$src"%}
10752   ins_encode %{
10753     __ convert_f2l($dst$$Register, $src$$XMMRegister);
10754   %}
10755   ins_pipe(pipe_slow);
10756 %}
10757 
10758 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10759 %{
10760   match(Set dst (ConvD2I src));
10761   effect(KILL cr);
10762   format %{ "convert_d2i $dst,$src"%}
10763   ins_encode %{
10764     __ convert_d2i($dst$$Register, $src$$XMMRegister);
10765   %}
10766   ins_pipe(pipe_slow);
10767 %}
10768 
10769 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10770 %{
10771   match(Set dst (ConvD2L src));
10772   effect(KILL cr);
10773   format %{ "convert_d2l $dst,$src"%}
10774   ins_encode %{
10775     __ convert_d2l($dst$$Register, $src$$XMMRegister);
10776   %}
10777   ins_pipe(pipe_slow);
10778 %}
10779 
10780 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10781 %{
10782   match(Set dst (RoundD src));
10783   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10784   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10785   ins_encode %{
10786     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10787   %}
10788   ins_pipe(pipe_slow);
10789 %}
10790 
10791 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10792 %{
10793   match(Set dst (RoundF src));
10794   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10795   format %{ "round_float $dst,$src" %}
10796   ins_encode %{
10797     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10798   %}
10799   ins_pipe(pipe_slow);
10800 %}
10801 
10802 instruct convI2F_reg_reg(regF dst, rRegI src)
10803 %{
10804   predicate(!UseXmmI2F);
10805   match(Set dst (ConvI2F src));
10806 
10807   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10808   ins_encode %{
10809     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10810   %}
10811   ins_pipe(pipe_slow); // XXX
10812 %}
10813 
10814 instruct convI2F_reg_mem(regF dst, memory src)
10815 %{
10816   match(Set dst (ConvI2F (LoadI src)));
10817 
10818   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10819   ins_encode %{
10820     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10821   %}
10822   ins_pipe(pipe_slow); // XXX
10823 %}
10824 
10825 instruct convI2D_reg_reg(regD dst, rRegI src)
10826 %{
10827   predicate(!UseXmmI2D);
10828   match(Set dst (ConvI2D src));
10829 
10830   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10831   ins_encode %{
10832     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10833   %}
10834   ins_pipe(pipe_slow); // XXX
10835 %}
10836 
10837 instruct convI2D_reg_mem(regD dst, memory src)
10838 %{
10839   match(Set dst (ConvI2D (LoadI src)));
10840 
10841   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10842   ins_encode %{
10843     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10844   %}
10845   ins_pipe(pipe_slow); // XXX
10846 %}
10847 
10848 instruct convXI2F_reg(regF dst, rRegI src)
10849 %{
10850   predicate(UseXmmI2F);
10851   match(Set dst (ConvI2F src));
10852 
10853   format %{ "movdl $dst, $src\n\t"
10854             "cvtdq2psl $dst, $dst\t# i2f" %}
10855   ins_encode %{
10856     __ movdl($dst$$XMMRegister, $src$$Register);
10857     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10858   %}
10859   ins_pipe(pipe_slow); // XXX
10860 %}
10861 
10862 instruct convXI2D_reg(regD dst, rRegI src)
10863 %{
10864   predicate(UseXmmI2D);
10865   match(Set dst (ConvI2D src));
10866 
10867   format %{ "movdl $dst, $src\n\t"
10868             "cvtdq2pdl $dst, $dst\t# i2d" %}
10869   ins_encode %{
10870     __ movdl($dst$$XMMRegister, $src$$Register);
10871     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10872   %}
10873   ins_pipe(pipe_slow); // XXX
10874 %}
10875 
10876 instruct convL2F_reg_reg(regF dst, rRegL src)
10877 %{
10878   match(Set dst (ConvL2F src));
10879 
10880   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10881   ins_encode %{
10882     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10883   %}
10884   ins_pipe(pipe_slow); // XXX
10885 %}
10886 
10887 instruct convL2F_reg_mem(regF dst, memory src)
10888 %{
10889   match(Set dst (ConvL2F (LoadL src)));
10890 
10891   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10892   ins_encode %{
10893     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10894   %}
10895   ins_pipe(pipe_slow); // XXX
10896 %}
10897 
10898 instruct convL2D_reg_reg(regD dst, rRegL src)
10899 %{
10900   match(Set dst (ConvL2D src));
10901 
10902   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10903   ins_encode %{
10904     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10905   %}
10906   ins_pipe(pipe_slow); // XXX
10907 %}
10908 
10909 instruct convL2D_reg_mem(regD dst, memory src)
10910 %{
10911   match(Set dst (ConvL2D (LoadL src)));
10912 
10913   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10914   ins_encode %{
10915     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10916   %}
10917   ins_pipe(pipe_slow); // XXX
10918 %}
10919 
10920 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10921 %{
10922   match(Set dst (ConvI2L src));
10923 
10924   ins_cost(125);
10925   format %{ "movslq  $dst, $src\t# i2l" %}
10926   ins_encode %{
10927     __ movslq($dst$$Register, $src$$Register);
10928   %}
10929   ins_pipe(ialu_reg_reg);
10930 %}
10931 
10932 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10933 // %{
10934 //   match(Set dst (ConvI2L src));
10935 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10936 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10937 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10938 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10939 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10940 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10941 
10942 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10943 //   ins_encode(enc_copy(dst, src));
10944 // //   opcode(0x63); // needs REX.W
10945 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10946 //   ins_pipe(ialu_reg_reg);
10947 // %}
10948 
10949 // Zero-extend convert int to long
10950 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10951 %{
10952   match(Set dst (AndL (ConvI2L src) mask));
10953 
10954   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10955   ins_encode %{
10956     if ($dst$$reg != $src$$reg) {
10957       __ movl($dst$$Register, $src$$Register);
10958     }
10959   %}
10960   ins_pipe(ialu_reg_reg);
10961 %}
10962 
10963 // Zero-extend convert int to long
10964 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10965 %{
10966   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10967 
10968   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10969   ins_encode %{
10970     __ movl($dst$$Register, $src$$Address);
10971   %}
10972   ins_pipe(ialu_reg_mem);
10973 %}
10974 
10975 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10976 %{
10977   match(Set dst (AndL src mask));
10978 
10979   format %{ "movl    $dst, $src\t# zero-extend long" %}
10980   ins_encode %{
10981     __ movl($dst$$Register, $src$$Register);
10982   %}
10983   ins_pipe(ialu_reg_reg);
10984 %}
10985 
10986 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10987 %{
10988   match(Set dst (ConvL2I src));
10989 
10990   format %{ "movl    $dst, $src\t# l2i" %}
10991   ins_encode %{
10992     __ movl($dst$$Register, $src$$Register);
10993   %}
10994   ins_pipe(ialu_reg_reg);
10995 %}
10996 
10997 
10998 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10999   match(Set dst (MoveF2I src));
11000   effect(DEF dst, USE src);
11001 
11002   ins_cost(125);
11003   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11004   ins_encode %{
11005     __ movl($dst$$Register, Address(rsp, $src$$disp));
11006   %}
11007   ins_pipe(ialu_reg_mem);
11008 %}
11009 
11010 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11011   match(Set dst (MoveI2F src));
11012   effect(DEF dst, USE src);
11013 
11014   ins_cost(125);
11015   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11016   ins_encode %{
11017     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11018   %}
11019   ins_pipe(pipe_slow);
11020 %}
11021 
11022 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11023   match(Set dst (MoveD2L src));
11024   effect(DEF dst, USE src);
11025 
11026   ins_cost(125);
11027   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11028   ins_encode %{
11029     __ movq($dst$$Register, Address(rsp, $src$$disp));
11030   %}
11031   ins_pipe(ialu_reg_mem);
11032 %}
11033 
11034 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11035   predicate(!UseXmmLoadAndClearUpper);
11036   match(Set dst (MoveL2D src));
11037   effect(DEF dst, USE src);
11038 
11039   ins_cost(125);
11040   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11041   ins_encode %{
11042     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11043   %}
11044   ins_pipe(pipe_slow);
11045 %}
11046 
11047 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11048   predicate(UseXmmLoadAndClearUpper);
11049   match(Set dst (MoveL2D src));
11050   effect(DEF dst, USE src);
11051 
11052   ins_cost(125);
11053   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11054   ins_encode %{
11055     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11056   %}
11057   ins_pipe(pipe_slow);
11058 %}
11059 
11060 
11061 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11062   match(Set dst (MoveF2I src));
11063   effect(DEF dst, USE src);
11064 
11065   ins_cost(95); // XXX
11066   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11067   ins_encode %{
11068     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11069   %}
11070   ins_pipe(pipe_slow);
11071 %}
11072 
11073 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11074   match(Set dst (MoveI2F src));
11075   effect(DEF dst, USE src);
11076 
11077   ins_cost(100);
11078   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11079   ins_encode %{
11080     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11081   %}
11082   ins_pipe( ialu_mem_reg );
11083 %}
11084 
11085 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11086   match(Set dst (MoveD2L src));
11087   effect(DEF dst, USE src);
11088 
11089   ins_cost(95); // XXX
11090   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11091   ins_encode %{
11092     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11093   %}
11094   ins_pipe(pipe_slow);
11095 %}
11096 
11097 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11098   match(Set dst (MoveL2D src));
11099   effect(DEF dst, USE src);
11100 
11101   ins_cost(100);
11102   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11103   ins_encode %{
11104     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11105   %}
11106   ins_pipe(ialu_mem_reg);
11107 %}
11108 
11109 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11110   match(Set dst (MoveF2I src));
11111   effect(DEF dst, USE src);
11112   ins_cost(85);
11113   format %{ "movd    $dst,$src\t# MoveF2I" %}
11114   ins_encode %{
11115     __ movdl($dst$$Register, $src$$XMMRegister);
11116   %}
11117   ins_pipe( pipe_slow );
11118 %}
11119 
11120 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11121   match(Set dst (MoveD2L src));
11122   effect(DEF dst, USE src);
11123   ins_cost(85);
11124   format %{ "movd    $dst,$src\t# MoveD2L" %}
11125   ins_encode %{
11126     __ movdq($dst$$Register, $src$$XMMRegister);
11127   %}
11128   ins_pipe( pipe_slow );
11129 %}
11130 
11131 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11132   match(Set dst (MoveI2F src));
11133   effect(DEF dst, USE src);
11134   ins_cost(100);
11135   format %{ "movd    $dst,$src\t# MoveI2F" %}
11136   ins_encode %{
11137     __ movdl($dst$$XMMRegister, $src$$Register);
11138   %}
11139   ins_pipe( pipe_slow );
11140 %}
11141 
11142 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11143   match(Set dst (MoveL2D src));
11144   effect(DEF dst, USE src);
11145   ins_cost(100);
11146   format %{ "movd    $dst,$src\t# MoveL2D" %}
11147   ins_encode %{
11148      __ movdq($dst$$XMMRegister, $src$$Register);
11149   %}
11150   ins_pipe( pipe_slow );
11151 %}
11152 
11153 // Fast clearing of an array
11154 // Small ClearArray non-AVX512.
11155 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11156                   Universe dummy, rFlagsReg cr)
11157 %{
11158   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11159   match(Set dummy (ClearArray cnt base));
11160   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11161 
11162   format %{ $$template
11163     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11164     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11165     $$emit$$"jg      LARGE\n\t"
11166     $$emit$$"dec     rcx\n\t"
11167     $$emit$$"js      DONE\t# Zero length\n\t"
11168     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11169     $$emit$$"dec     rcx\n\t"
11170     $$emit$$"jge     LOOP\n\t"
11171     $$emit$$"jmp     DONE\n\t"
11172     $$emit$$"# LARGE:\n\t"
11173     if (UseFastStosb) {
11174        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11175        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11176     } else if (UseXMMForObjInit) {
11177        $$emit$$"mov     rdi,rax\n\t"
11178        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11179        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11180        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11181        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11182        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11183        $$emit$$"add     0x40,rax\n\t"
11184        $$emit$$"# L_zero_64_bytes:\n\t"
11185        $$emit$$"sub     0x8,rcx\n\t"
11186        $$emit$$"jge     L_loop\n\t"
11187        $$emit$$"add     0x4,rcx\n\t"
11188        $$emit$$"jl      L_tail\n\t"
11189        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11190        $$emit$$"add     0x20,rax\n\t"
11191        $$emit$$"sub     0x4,rcx\n\t"
11192        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11193        $$emit$$"add     0x4,rcx\n\t"
11194        $$emit$$"jle     L_end\n\t"
11195        $$emit$$"dec     rcx\n\t"
11196        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11197        $$emit$$"vmovq   xmm0,(rax)\n\t"
11198        $$emit$$"add     0x8,rax\n\t"
11199        $$emit$$"dec     rcx\n\t"
11200        $$emit$$"jge     L_sloop\n\t"
11201        $$emit$$"# L_end:\n\t"
11202     } else {
11203        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11204     }
11205     $$emit$$"# DONE"
11206   %}
11207   ins_encode %{
11208     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11209                  $tmp$$XMMRegister, false, knoreg);
11210   %}
11211   ins_pipe(pipe_slow);
11212 %}
11213 
11214 // Small ClearArray AVX512 non-constant length.
11215 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11216                        Universe dummy, rFlagsReg cr)
11217 %{
11218   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11219   match(Set dummy (ClearArray cnt base));
11220   ins_cost(125);
11221   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11222 
11223   format %{ $$template
11224     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11225     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11226     $$emit$$"jg      LARGE\n\t"
11227     $$emit$$"dec     rcx\n\t"
11228     $$emit$$"js      DONE\t# Zero length\n\t"
11229     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11230     $$emit$$"dec     rcx\n\t"
11231     $$emit$$"jge     LOOP\n\t"
11232     $$emit$$"jmp     DONE\n\t"
11233     $$emit$$"# LARGE:\n\t"
11234     if (UseFastStosb) {
11235        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11236        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11237     } else if (UseXMMForObjInit) {
11238        $$emit$$"mov     rdi,rax\n\t"
11239        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11240        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11241        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11242        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11243        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11244        $$emit$$"add     0x40,rax\n\t"
11245        $$emit$$"# L_zero_64_bytes:\n\t"
11246        $$emit$$"sub     0x8,rcx\n\t"
11247        $$emit$$"jge     L_loop\n\t"
11248        $$emit$$"add     0x4,rcx\n\t"
11249        $$emit$$"jl      L_tail\n\t"
11250        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11251        $$emit$$"add     0x20,rax\n\t"
11252        $$emit$$"sub     0x4,rcx\n\t"
11253        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11254        $$emit$$"add     0x4,rcx\n\t"
11255        $$emit$$"jle     L_end\n\t"
11256        $$emit$$"dec     rcx\n\t"
11257        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11258        $$emit$$"vmovq   xmm0,(rax)\n\t"
11259        $$emit$$"add     0x8,rax\n\t"
11260        $$emit$$"dec     rcx\n\t"
11261        $$emit$$"jge     L_sloop\n\t"
11262        $$emit$$"# L_end:\n\t"
11263     } else {
11264        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11265     }
11266     $$emit$$"# DONE"
11267   %}
11268   ins_encode %{
11269     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11270                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11271   %}
11272   ins_pipe(pipe_slow);
11273 %}
11274 
11275 // Large ClearArray non-AVX512.
11276 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11277                         Universe dummy, rFlagsReg cr)
11278 %{
11279   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11280   match(Set dummy (ClearArray cnt base));
11281   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11282 
11283   format %{ $$template
11284     if (UseFastStosb) {
11285        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11286        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11287        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11288     } else if (UseXMMForObjInit) {
11289        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11290        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11291        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11292        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11293        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11294        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11295        $$emit$$"add     0x40,rax\n\t"
11296        $$emit$$"# L_zero_64_bytes:\n\t"
11297        $$emit$$"sub     0x8,rcx\n\t"
11298        $$emit$$"jge     L_loop\n\t"
11299        $$emit$$"add     0x4,rcx\n\t"
11300        $$emit$$"jl      L_tail\n\t"
11301        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11302        $$emit$$"add     0x20,rax\n\t"
11303        $$emit$$"sub     0x4,rcx\n\t"
11304        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11305        $$emit$$"add     0x4,rcx\n\t"
11306        $$emit$$"jle     L_end\n\t"
11307        $$emit$$"dec     rcx\n\t"
11308        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11309        $$emit$$"vmovq   xmm0,(rax)\n\t"
11310        $$emit$$"add     0x8,rax\n\t"
11311        $$emit$$"dec     rcx\n\t"
11312        $$emit$$"jge     L_sloop\n\t"
11313        $$emit$$"# L_end:\n\t"
11314     } else {
11315        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11316        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11317     }
11318   %}
11319   ins_encode %{
11320     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11321                  $tmp$$XMMRegister, true, knoreg);
11322   %}
11323   ins_pipe(pipe_slow);
11324 %}
11325 
11326 // Large ClearArray AVX512.
11327 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11328                              Universe dummy, rFlagsReg cr)
11329 %{
11330   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11331   match(Set dummy (ClearArray cnt base));
11332   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11333 
11334   format %{ $$template
11335     if (UseFastStosb) {
11336        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11337        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11338        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11339     } else if (UseXMMForObjInit) {
11340        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11341        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11342        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11343        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11344        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11345        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11346        $$emit$$"add     0x40,rax\n\t"
11347        $$emit$$"# L_zero_64_bytes:\n\t"
11348        $$emit$$"sub     0x8,rcx\n\t"
11349        $$emit$$"jge     L_loop\n\t"
11350        $$emit$$"add     0x4,rcx\n\t"
11351        $$emit$$"jl      L_tail\n\t"
11352        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11353        $$emit$$"add     0x20,rax\n\t"
11354        $$emit$$"sub     0x4,rcx\n\t"
11355        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11356        $$emit$$"add     0x4,rcx\n\t"
11357        $$emit$$"jle     L_end\n\t"
11358        $$emit$$"dec     rcx\n\t"
11359        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11360        $$emit$$"vmovq   xmm0,(rax)\n\t"
11361        $$emit$$"add     0x8,rax\n\t"
11362        $$emit$$"dec     rcx\n\t"
11363        $$emit$$"jge     L_sloop\n\t"
11364        $$emit$$"# L_end:\n\t"
11365     } else {
11366        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11367        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11368     }
11369   %}
11370   ins_encode %{
11371     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11372                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11373   %}
11374   ins_pipe(pipe_slow);
11375 %}
11376 
11377 // Small ClearArray AVX512 constant length.
11378 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11379 %{
11380   predicate(!((ClearArrayNode*)n)->is_large() &&
11381               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11382   match(Set dummy (ClearArray cnt base));
11383   ins_cost(100);
11384   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11385   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11386   ins_encode %{
11387    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11388   %}
11389   ins_pipe(pipe_slow);
11390 %}
11391 
11392 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11393                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11394 %{
11395   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11396   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11397   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11398 
11399   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11400   ins_encode %{
11401     __ string_compare($str1$$Register, $str2$$Register,
11402                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11403                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11404   %}
11405   ins_pipe( pipe_slow );
11406 %}
11407 
11408 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11409                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11410 %{
11411   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11412   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11413   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11414 
11415   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11416   ins_encode %{
11417     __ string_compare($str1$$Register, $str2$$Register,
11418                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11419                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11420   %}
11421   ins_pipe( pipe_slow );
11422 %}
11423 
11424 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11425                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11426 %{
11427   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11428   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11429   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11430 
11431   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11432   ins_encode %{
11433     __ string_compare($str1$$Register, $str2$$Register,
11434                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11435                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11436   %}
11437   ins_pipe( pipe_slow );
11438 %}
11439 
11440 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11441                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11442 %{
11443   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11444   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11445   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11446 
11447   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11448   ins_encode %{
11449     __ string_compare($str1$$Register, $str2$$Register,
11450                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11451                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11452   %}
11453   ins_pipe( pipe_slow );
11454 %}
11455 
11456 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11457                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11458 %{
11459   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11460   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11461   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11462 
11463   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11464   ins_encode %{
11465     __ string_compare($str1$$Register, $str2$$Register,
11466                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11467                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11468   %}
11469   ins_pipe( pipe_slow );
11470 %}
11471 
11472 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11473                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11474 %{
11475   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11476   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11477   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11478 
11479   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11480   ins_encode %{
11481     __ string_compare($str1$$Register, $str2$$Register,
11482                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11483                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11484   %}
11485   ins_pipe( pipe_slow );
11486 %}
11487 
11488 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11489                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11490 %{
11491   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11492   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11493   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11494 
11495   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11496   ins_encode %{
11497     __ string_compare($str2$$Register, $str1$$Register,
11498                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11499                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11500   %}
11501   ins_pipe( pipe_slow );
11502 %}
11503 
11504 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11505                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11506 %{
11507   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11508   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11509   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11510 
11511   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11512   ins_encode %{
11513     __ string_compare($str2$$Register, $str1$$Register,
11514                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11515                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11516   %}
11517   ins_pipe( pipe_slow );
11518 %}
11519 
11520 // fast search of substring with known size.
11521 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11522                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11523 %{
11524   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11525   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11526   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11527 
11528   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11529   ins_encode %{
11530     int icnt2 = (int)$int_cnt2$$constant;
11531     if (icnt2 >= 16) {
11532       // IndexOf for constant substrings with size >= 16 elements
11533       // which don't need to be loaded through stack.
11534       __ string_indexofC8($str1$$Register, $str2$$Register,
11535                           $cnt1$$Register, $cnt2$$Register,
11536                           icnt2, $result$$Register,
11537                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11538     } else {
11539       // Small strings are loaded through stack if they cross page boundary.
11540       __ string_indexof($str1$$Register, $str2$$Register,
11541                         $cnt1$$Register, $cnt2$$Register,
11542                         icnt2, $result$$Register,
11543                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11544     }
11545   %}
11546   ins_pipe( pipe_slow );
11547 %}
11548 
11549 // fast search of substring with known size.
11550 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11551                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11552 %{
11553   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11554   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11555   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11556 
11557   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11558   ins_encode %{
11559     int icnt2 = (int)$int_cnt2$$constant;
11560     if (icnt2 >= 8) {
11561       // IndexOf for constant substrings with size >= 8 elements
11562       // which don't need to be loaded through stack.
11563       __ string_indexofC8($str1$$Register, $str2$$Register,
11564                           $cnt1$$Register, $cnt2$$Register,
11565                           icnt2, $result$$Register,
11566                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11567     } else {
11568       // Small strings are loaded through stack if they cross page boundary.
11569       __ string_indexof($str1$$Register, $str2$$Register,
11570                         $cnt1$$Register, $cnt2$$Register,
11571                         icnt2, $result$$Register,
11572                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11573     }
11574   %}
11575   ins_pipe( pipe_slow );
11576 %}
11577 
11578 // fast search of substring with known size.
11579 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11580                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11581 %{
11582   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11583   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11584   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11585 
11586   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11587   ins_encode %{
11588     int icnt2 = (int)$int_cnt2$$constant;
11589     if (icnt2 >= 8) {
11590       // IndexOf for constant substrings with size >= 8 elements
11591       // which don't need to be loaded through stack.
11592       __ string_indexofC8($str1$$Register, $str2$$Register,
11593                           $cnt1$$Register, $cnt2$$Register,
11594                           icnt2, $result$$Register,
11595                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11596     } else {
11597       // Small strings are loaded through stack if they cross page boundary.
11598       __ string_indexof($str1$$Register, $str2$$Register,
11599                         $cnt1$$Register, $cnt2$$Register,
11600                         icnt2, $result$$Register,
11601                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11602     }
11603   %}
11604   ins_pipe( pipe_slow );
11605 %}
11606 
11607 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11608                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11609 %{
11610   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11611   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11612   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11613 
11614   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11615   ins_encode %{
11616     __ string_indexof($str1$$Register, $str2$$Register,
11617                       $cnt1$$Register, $cnt2$$Register,
11618                       (-1), $result$$Register,
11619                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11620   %}
11621   ins_pipe( pipe_slow );
11622 %}
11623 
11624 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11625                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11626 %{
11627   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11628   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11629   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11630 
11631   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11632   ins_encode %{
11633     __ string_indexof($str1$$Register, $str2$$Register,
11634                       $cnt1$$Register, $cnt2$$Register,
11635                       (-1), $result$$Register,
11636                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11637   %}
11638   ins_pipe( pipe_slow );
11639 %}
11640 
11641 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11642                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11643 %{
11644   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11645   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11646   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11647 
11648   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11649   ins_encode %{
11650     __ string_indexof($str1$$Register, $str2$$Register,
11651                       $cnt1$$Register, $cnt2$$Register,
11652                       (-1), $result$$Register,
11653                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11654   %}
11655   ins_pipe( pipe_slow );
11656 %}
11657 
11658 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11659                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11660 %{
11661   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11662   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11663   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11664   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11665   ins_encode %{
11666     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11667                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11668   %}
11669   ins_pipe( pipe_slow );
11670 %}
11671 
11672 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11673                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11674 %{
11675   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11676   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11677   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11678   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11679   ins_encode %{
11680     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11681                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11682   %}
11683   ins_pipe( pipe_slow );
11684 %}
11685 
11686 // fast string equals
11687 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11688                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11689 %{
11690   predicate(!VM_Version::supports_avx512vlbw());
11691   match(Set result (StrEquals (Binary str1 str2) cnt));
11692   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11693 
11694   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11695   ins_encode %{
11696     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11697                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11698                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11699   %}
11700   ins_pipe( pipe_slow );
11701 %}
11702 
11703 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11704                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11705 %{
11706   predicate(VM_Version::supports_avx512vlbw());
11707   match(Set result (StrEquals (Binary str1 str2) cnt));
11708   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11709 
11710   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11711   ins_encode %{
11712     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11713                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11714                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11715   %}
11716   ins_pipe( pipe_slow );
11717 %}
11718 
11719 // fast array equals
11720 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11721                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11722 %{
11723   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11724   match(Set result (AryEq ary1 ary2));
11725   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11726 
11727   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11728   ins_encode %{
11729     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11730                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11731                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11732   %}
11733   ins_pipe( pipe_slow );
11734 %}
11735 
11736 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11737                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11738 %{
11739   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11740   match(Set result (AryEq ary1 ary2));
11741   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11742 
11743   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11744   ins_encode %{
11745     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11746                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11747                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11748   %}
11749   ins_pipe( pipe_slow );
11750 %}
11751 
11752 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11753                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11754 %{
11755   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11756   match(Set result (AryEq ary1 ary2));
11757   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11758 
11759   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11760   ins_encode %{
11761     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11762                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11763                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11764   %}
11765   ins_pipe( pipe_slow );
11766 %}
11767 
11768 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11769                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11770 %{
11771   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11772   match(Set result (AryEq ary1 ary2));
11773   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11774 
11775   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11776   ins_encode %{
11777     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11778                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11779                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11780   %}
11781   ins_pipe( pipe_slow );
11782 %}
11783 
11784 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11785                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11786 %{
11787   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11788   match(Set result (CountPositives ary1 len));
11789   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11790 
11791   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11792   ins_encode %{
11793     __ count_positives($ary1$$Register, $len$$Register,
11794                        $result$$Register, $tmp3$$Register,
11795                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11796   %}
11797   ins_pipe( pipe_slow );
11798 %}
11799 
11800 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11801                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11802 %{
11803   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11804   match(Set result (CountPositives ary1 len));
11805   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11806 
11807   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11808   ins_encode %{
11809     __ count_positives($ary1$$Register, $len$$Register,
11810                        $result$$Register, $tmp3$$Register,
11811                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11812   %}
11813   ins_pipe( pipe_slow );
11814 %}
11815 
11816 // fast char[] to byte[] compression
11817 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11818                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11819   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11820   match(Set result (StrCompressedCopy src (Binary dst len)));
11821   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11822          USE_KILL len, KILL tmp5, KILL cr);
11823 
11824   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11825   ins_encode %{
11826     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11827                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11828                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11829                            knoreg, knoreg);
11830   %}
11831   ins_pipe( pipe_slow );
11832 %}
11833 
11834 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11835                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11836   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11837   match(Set result (StrCompressedCopy src (Binary dst len)));
11838   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11839          USE_KILL len, KILL tmp5, KILL cr);
11840 
11841   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11842   ins_encode %{
11843     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11844                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11845                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11846                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11847   %}
11848   ins_pipe( pipe_slow );
11849 %}
11850 // fast byte[] to char[] inflation
11851 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11852                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11853   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11854   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11855   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11856 
11857   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11858   ins_encode %{
11859     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11860                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11861   %}
11862   ins_pipe( pipe_slow );
11863 %}
11864 
11865 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11866                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11867   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11868   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11869   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11870 
11871   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11872   ins_encode %{
11873     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11874                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11875   %}
11876   ins_pipe( pipe_slow );
11877 %}
11878 
11879 // encode char[] to byte[] in ISO_8859_1
11880 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11881                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11882                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11883   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11884   match(Set result (EncodeISOArray src (Binary dst len)));
11885   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11886 
11887   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11888   ins_encode %{
11889     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11890                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11891                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11892   %}
11893   ins_pipe( pipe_slow );
11894 %}
11895 
11896 // encode char[] to byte[] in ASCII
11897 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11898                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11899                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11900   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11901   match(Set result (EncodeISOArray src (Binary dst len)));
11902   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11903 
11904   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11905   ins_encode %{
11906     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11907                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11908                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11909   %}
11910   ins_pipe( pipe_slow );
11911 %}
11912 
11913 //----------Overflow Math Instructions-----------------------------------------
11914 
11915 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11916 %{
11917   match(Set cr (OverflowAddI op1 op2));
11918   effect(DEF cr, USE_KILL op1, USE op2);
11919 
11920   format %{ "addl    $op1, $op2\t# overflow check int" %}
11921 
11922   ins_encode %{
11923     __ addl($op1$$Register, $op2$$Register);
11924   %}
11925   ins_pipe(ialu_reg_reg);
11926 %}
11927 
11928 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11929 %{
11930   match(Set cr (OverflowAddI op1 op2));
11931   effect(DEF cr, USE_KILL op1, USE op2);
11932 
11933   format %{ "addl    $op1, $op2\t# overflow check int" %}
11934 
11935   ins_encode %{
11936     __ addl($op1$$Register, $op2$$constant);
11937   %}
11938   ins_pipe(ialu_reg_reg);
11939 %}
11940 
11941 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11942 %{
11943   match(Set cr (OverflowAddL op1 op2));
11944   effect(DEF cr, USE_KILL op1, USE op2);
11945 
11946   format %{ "addq    $op1, $op2\t# overflow check long" %}
11947   ins_encode %{
11948     __ addq($op1$$Register, $op2$$Register);
11949   %}
11950   ins_pipe(ialu_reg_reg);
11951 %}
11952 
11953 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11954 %{
11955   match(Set cr (OverflowAddL op1 op2));
11956   effect(DEF cr, USE_KILL op1, USE op2);
11957 
11958   format %{ "addq    $op1, $op2\t# overflow check long" %}
11959   ins_encode %{
11960     __ addq($op1$$Register, $op2$$constant);
11961   %}
11962   ins_pipe(ialu_reg_reg);
11963 %}
11964 
11965 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11966 %{
11967   match(Set cr (OverflowSubI op1 op2));
11968 
11969   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11970   ins_encode %{
11971     __ cmpl($op1$$Register, $op2$$Register);
11972   %}
11973   ins_pipe(ialu_reg_reg);
11974 %}
11975 
11976 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11977 %{
11978   match(Set cr (OverflowSubI op1 op2));
11979 
11980   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11981   ins_encode %{
11982     __ cmpl($op1$$Register, $op2$$constant);
11983   %}
11984   ins_pipe(ialu_reg_reg);
11985 %}
11986 
11987 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11988 %{
11989   match(Set cr (OverflowSubL op1 op2));
11990 
11991   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11992   ins_encode %{
11993     __ cmpq($op1$$Register, $op2$$Register);
11994   %}
11995   ins_pipe(ialu_reg_reg);
11996 %}
11997 
11998 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11999 %{
12000   match(Set cr (OverflowSubL op1 op2));
12001 
12002   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12003   ins_encode %{
12004     __ cmpq($op1$$Register, $op2$$constant);
12005   %}
12006   ins_pipe(ialu_reg_reg);
12007 %}
12008 
12009 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12010 %{
12011   match(Set cr (OverflowSubI zero op2));
12012   effect(DEF cr, USE_KILL op2);
12013 
12014   format %{ "negl    $op2\t# overflow check int" %}
12015   ins_encode %{
12016     __ negl($op2$$Register);
12017   %}
12018   ins_pipe(ialu_reg_reg);
12019 %}
12020 
12021 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12022 %{
12023   match(Set cr (OverflowSubL zero op2));
12024   effect(DEF cr, USE_KILL op2);
12025 
12026   format %{ "negq    $op2\t# overflow check long" %}
12027   ins_encode %{
12028     __ negq($op2$$Register);
12029   %}
12030   ins_pipe(ialu_reg_reg);
12031 %}
12032 
12033 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12034 %{
12035   match(Set cr (OverflowMulI op1 op2));
12036   effect(DEF cr, USE_KILL op1, USE op2);
12037 
12038   format %{ "imull    $op1, $op2\t# overflow check int" %}
12039   ins_encode %{
12040     __ imull($op1$$Register, $op2$$Register);
12041   %}
12042   ins_pipe(ialu_reg_reg_alu0);
12043 %}
12044 
12045 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12046 %{
12047   match(Set cr (OverflowMulI op1 op2));
12048   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12049 
12050   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12051   ins_encode %{
12052     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12053   %}
12054   ins_pipe(ialu_reg_reg_alu0);
12055 %}
12056 
12057 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12058 %{
12059   match(Set cr (OverflowMulL op1 op2));
12060   effect(DEF cr, USE_KILL op1, USE op2);
12061 
12062   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12063   ins_encode %{
12064     __ imulq($op1$$Register, $op2$$Register);
12065   %}
12066   ins_pipe(ialu_reg_reg_alu0);
12067 %}
12068 
12069 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12070 %{
12071   match(Set cr (OverflowMulL op1 op2));
12072   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12073 
12074   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12075   ins_encode %{
12076     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12077   %}
12078   ins_pipe(ialu_reg_reg_alu0);
12079 %}
12080 
12081 
12082 //----------Control Flow Instructions------------------------------------------
12083 // Signed compare Instructions
12084 
12085 // XXX more variants!!
12086 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12087 %{
12088   match(Set cr (CmpI op1 op2));
12089   effect(DEF cr, USE op1, USE op2);
12090 
12091   format %{ "cmpl    $op1, $op2" %}
12092   ins_encode %{
12093     __ cmpl($op1$$Register, $op2$$Register);
12094   %}
12095   ins_pipe(ialu_cr_reg_reg);
12096 %}
12097 
12098 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12099 %{
12100   match(Set cr (CmpI op1 op2));
12101 
12102   format %{ "cmpl    $op1, $op2" %}
12103   ins_encode %{
12104     __ cmpl($op1$$Register, $op2$$constant);
12105   %}
12106   ins_pipe(ialu_cr_reg_imm);
12107 %}
12108 
12109 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12110 %{
12111   match(Set cr (CmpI op1 (LoadI op2)));
12112 
12113   ins_cost(500); // XXX
12114   format %{ "cmpl    $op1, $op2" %}
12115   ins_encode %{
12116     __ cmpl($op1$$Register, $op2$$Address);
12117   %}
12118   ins_pipe(ialu_cr_reg_mem);
12119 %}
12120 
12121 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12122 %{
12123   match(Set cr (CmpI src zero));
12124 
12125   format %{ "testl   $src, $src" %}
12126   ins_encode %{
12127     __ testl($src$$Register, $src$$Register);
12128   %}
12129   ins_pipe(ialu_cr_reg_imm);
12130 %}
12131 
12132 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12133 %{
12134   match(Set cr (CmpI (AndI src con) zero));
12135 
12136   format %{ "testl   $src, $con" %}
12137   ins_encode %{
12138     __ testl($src$$Register, $con$$constant);
12139   %}
12140   ins_pipe(ialu_cr_reg_imm);
12141 %}
12142 
12143 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12144 %{
12145   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12146 
12147   format %{ "testl   $src, $mem" %}
12148   ins_encode %{
12149     __ testl($src$$Register, $mem$$Address);
12150   %}
12151   ins_pipe(ialu_cr_reg_mem);
12152 %}
12153 
12154 // Unsigned compare Instructions; really, same as signed except they
12155 // produce an rFlagsRegU instead of rFlagsReg.
12156 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12157 %{
12158   match(Set cr (CmpU op1 op2));
12159 
12160   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12161   ins_encode %{
12162     __ cmpl($op1$$Register, $op2$$Register);
12163   %}
12164   ins_pipe(ialu_cr_reg_reg);
12165 %}
12166 
12167 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12168 %{
12169   match(Set cr (CmpU op1 op2));
12170 
12171   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12172   ins_encode %{
12173     __ cmpl($op1$$Register, $op2$$constant);
12174   %}
12175   ins_pipe(ialu_cr_reg_imm);
12176 %}
12177 
12178 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12179 %{
12180   match(Set cr (CmpU op1 (LoadI op2)));
12181 
12182   ins_cost(500); // XXX
12183   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12184   ins_encode %{
12185     __ cmpl($op1$$Register, $op2$$Address);
12186   %}
12187   ins_pipe(ialu_cr_reg_mem);
12188 %}
12189 
12190 // // // Cisc-spilled version of cmpU_rReg
12191 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12192 // //%{
12193 // //  match(Set cr (CmpU (LoadI op1) op2));
12194 // //
12195 // //  format %{ "CMPu   $op1,$op2" %}
12196 // //  ins_cost(500);
12197 // //  opcode(0x39);  /* Opcode 39 /r */
12198 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12199 // //%}
12200 
12201 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12202 %{
12203   match(Set cr (CmpU src zero));
12204 
12205   format %{ "testl   $src, $src\t# unsigned" %}
12206   ins_encode %{
12207     __ testl($src$$Register, $src$$Register);
12208   %}
12209   ins_pipe(ialu_cr_reg_imm);
12210 %}
12211 
12212 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12213 %{
12214   match(Set cr (CmpP op1 op2));
12215 
12216   format %{ "cmpq    $op1, $op2\t# ptr" %}
12217   ins_encode %{
12218     __ cmpq($op1$$Register, $op2$$Register);
12219   %}
12220   ins_pipe(ialu_cr_reg_reg);
12221 %}
12222 
12223 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12224 %{
12225   match(Set cr (CmpP op1 (LoadP op2)));
12226   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12227 
12228   ins_cost(500); // XXX
12229   format %{ "cmpq    $op1, $op2\t# ptr" %}
12230   ins_encode %{
12231     __ cmpq($op1$$Register, $op2$$Address);
12232   %}
12233   ins_pipe(ialu_cr_reg_mem);
12234 %}
12235 
12236 // // // Cisc-spilled version of cmpP_rReg
12237 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12238 // //%{
12239 // //  match(Set cr (CmpP (LoadP op1) op2));
12240 // //
12241 // //  format %{ "CMPu   $op1,$op2" %}
12242 // //  ins_cost(500);
12243 // //  opcode(0x39);  /* Opcode 39 /r */
12244 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12245 // //%}
12246 
12247 // XXX this is generalized by compP_rReg_mem???
12248 // Compare raw pointer (used in out-of-heap check).
12249 // Only works because non-oop pointers must be raw pointers
12250 // and raw pointers have no anti-dependencies.
12251 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12252 %{
12253   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12254             n->in(2)->as_Load()->barrier_data() == 0);
12255   match(Set cr (CmpP op1 (LoadP op2)));
12256 
12257   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12258   ins_encode %{
12259     __ cmpq($op1$$Register, $op2$$Address);
12260   %}
12261   ins_pipe(ialu_cr_reg_mem);
12262 %}
12263 
12264 // This will generate a signed flags result. This should be OK since
12265 // any compare to a zero should be eq/neq.
12266 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12267 %{
12268   match(Set cr (CmpP src zero));
12269 
12270   format %{ "testq   $src, $src\t# ptr" %}
12271   ins_encode %{
12272     __ testq($src$$Register, $src$$Register);
12273   %}
12274   ins_pipe(ialu_cr_reg_imm);
12275 %}
12276 
12277 // This will generate a signed flags result. This should be OK since
12278 // any compare to a zero should be eq/neq.
12279 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12280 %{
12281   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12282             n->in(1)->as_Load()->barrier_data() == 0);
12283   match(Set cr (CmpP (LoadP op) zero));
12284 
12285   ins_cost(500); // XXX
12286   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12287   ins_encode %{
12288     __ testq($op$$Address, 0xFFFFFFFF);
12289   %}
12290   ins_pipe(ialu_cr_reg_imm);
12291 %}
12292 
12293 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12294 %{
12295   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12296             n->in(1)->as_Load()->barrier_data() == 0);
12297   match(Set cr (CmpP (LoadP mem) zero));
12298 
12299   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12300   ins_encode %{
12301     __ cmpq(r12, $mem$$Address);
12302   %}
12303   ins_pipe(ialu_cr_reg_mem);
12304 %}
12305 
12306 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12307 %{
12308   match(Set cr (CmpN op1 op2));
12309 
12310   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12311   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12312   ins_pipe(ialu_cr_reg_reg);
12313 %}
12314 
12315 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12316 %{
12317   match(Set cr (CmpN src (LoadN mem)));
12318 
12319   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12320   ins_encode %{
12321     __ cmpl($src$$Register, $mem$$Address);
12322   %}
12323   ins_pipe(ialu_cr_reg_mem);
12324 %}
12325 
12326 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12327   match(Set cr (CmpN op1 op2));
12328 
12329   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12330   ins_encode %{
12331     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12332   %}
12333   ins_pipe(ialu_cr_reg_imm);
12334 %}
12335 
12336 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12337 %{
12338   match(Set cr (CmpN src (LoadN mem)));
12339 
12340   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12341   ins_encode %{
12342     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12343   %}
12344   ins_pipe(ialu_cr_reg_mem);
12345 %}
12346 
12347 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12348   match(Set cr (CmpN op1 op2));
12349 
12350   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12351   ins_encode %{
12352     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12353   %}
12354   ins_pipe(ialu_cr_reg_imm);
12355 %}
12356 
12357 // Disabled because the compressed Klass* in header cannot be safely
12358 // accessed. TODO: Re-enable it as soon as synchronization does not
12359 // overload the upper header bits anymore.
12360 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12361 %{
12362   predicate(false);
12363   match(Set cr (CmpN src (LoadNKlass mem)));
12364 
12365   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12366   ins_encode %{
12367     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12368   %}
12369   ins_pipe(ialu_cr_reg_mem);
12370 %}
12371 
12372 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12373   match(Set cr (CmpN src zero));
12374 
12375   format %{ "testl   $src, $src\t# compressed ptr" %}
12376   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12377   ins_pipe(ialu_cr_reg_imm);
12378 %}
12379 
12380 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12381 %{
12382   predicate(CompressedOops::base() != NULL);
12383   match(Set cr (CmpN (LoadN mem) zero));
12384 
12385   ins_cost(500); // XXX
12386   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12387   ins_encode %{
12388     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12389   %}
12390   ins_pipe(ialu_cr_reg_mem);
12391 %}
12392 
12393 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12394 %{
12395   predicate(CompressedOops::base() == NULL);
12396   match(Set cr (CmpN (LoadN mem) zero));
12397 
12398   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12399   ins_encode %{
12400     __ cmpl(r12, $mem$$Address);
12401   %}
12402   ins_pipe(ialu_cr_reg_mem);
12403 %}
12404 
12405 // Yanked all unsigned pointer compare operations.
12406 // Pointer compares are done with CmpP which is already unsigned.
12407 
12408 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12409 %{
12410   match(Set cr (CmpL op1 op2));
12411 
12412   format %{ "cmpq    $op1, $op2" %}
12413   ins_encode %{
12414     __ cmpq($op1$$Register, $op2$$Register);
12415   %}
12416   ins_pipe(ialu_cr_reg_reg);
12417 %}
12418 
12419 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12420 %{
12421   match(Set cr (CmpL op1 op2));
12422 
12423   format %{ "cmpq    $op1, $op2" %}
12424   ins_encode %{
12425     __ cmpq($op1$$Register, $op2$$constant);
12426   %}
12427   ins_pipe(ialu_cr_reg_imm);
12428 %}
12429 
12430 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12431 %{
12432   match(Set cr (CmpL op1 (LoadL op2)));
12433 
12434   format %{ "cmpq    $op1, $op2" %}
12435   ins_encode %{
12436     __ cmpq($op1$$Register, $op2$$Address);
12437   %}
12438   ins_pipe(ialu_cr_reg_mem);
12439 %}
12440 
12441 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12442 %{
12443   match(Set cr (CmpL src zero));
12444 
12445   format %{ "testq   $src, $src" %}
12446   ins_encode %{
12447     __ testq($src$$Register, $src$$Register);
12448   %}
12449   ins_pipe(ialu_cr_reg_imm);
12450 %}
12451 
12452 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12453 %{
12454   match(Set cr (CmpL (AndL src con) zero));
12455 
12456   format %{ "testq   $src, $con\t# long" %}
12457   ins_encode %{
12458     __ testq($src$$Register, $con$$constant);
12459   %}
12460   ins_pipe(ialu_cr_reg_imm);
12461 %}
12462 
12463 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12464 %{
12465   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12466 
12467   format %{ "testq   $src, $mem" %}
12468   ins_encode %{
12469     __ testq($src$$Register, $mem$$Address);
12470   %}
12471   ins_pipe(ialu_cr_reg_mem);
12472 %}
12473 
12474 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12475 %{
12476   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12477 
12478   format %{ "testq   $src, $mem" %}
12479   ins_encode %{
12480     __ testq($src$$Register, $mem$$Address);
12481   %}
12482   ins_pipe(ialu_cr_reg_mem);
12483 %}
12484 
12485 // Manifest a CmpL result in an integer register.  Very painful.
12486 // This is the test to avoid.
12487 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12488 %{
12489   match(Set dst (CmpL3 src1 src2));
12490   effect(KILL flags);
12491 
12492   ins_cost(275); // XXX
12493   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12494             "movl    $dst, -1\n\t"
12495             "jl,s    done\n\t"
12496             "setne   $dst\n\t"
12497             "movzbl  $dst, $dst\n\t"
12498     "done:" %}
12499   ins_encode %{
12500     Label done;
12501     __ cmpq($src1$$Register, $src2$$Register);
12502     __ movl($dst$$Register, -1);
12503     __ jccb(Assembler::less, done);
12504     __ setne($dst$$Register);
12505     __ movzbl($dst$$Register, $dst$$Register);
12506     __ bind(done);
12507   %}
12508   ins_pipe(pipe_slow);
12509 %}
12510 
12511 // Unsigned long compare Instructions; really, same as signed long except they
12512 // produce an rFlagsRegU instead of rFlagsReg.
12513 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12514 %{
12515   match(Set cr (CmpUL op1 op2));
12516 
12517   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12518   ins_encode %{
12519     __ cmpq($op1$$Register, $op2$$Register);
12520   %}
12521   ins_pipe(ialu_cr_reg_reg);
12522 %}
12523 
12524 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12525 %{
12526   match(Set cr (CmpUL op1 op2));
12527 
12528   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12529   ins_encode %{
12530     __ cmpq($op1$$Register, $op2$$constant);
12531   %}
12532   ins_pipe(ialu_cr_reg_imm);
12533 %}
12534 
12535 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12536 %{
12537   match(Set cr (CmpUL op1 (LoadL op2)));
12538 
12539   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12540   ins_encode %{
12541     __ cmpq($op1$$Register, $op2$$Address);
12542   %}
12543   ins_pipe(ialu_cr_reg_mem);
12544 %}
12545 
12546 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12547 %{
12548   match(Set cr (CmpUL src zero));
12549 
12550   format %{ "testq   $src, $src\t# unsigned" %}
12551   ins_encode %{
12552     __ testq($src$$Register, $src$$Register);
12553   %}
12554   ins_pipe(ialu_cr_reg_imm);
12555 %}
12556 
12557 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12558 %{
12559   match(Set cr (CmpI (LoadB mem) imm));
12560 
12561   ins_cost(125);
12562   format %{ "cmpb    $mem, $imm" %}
12563   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12564   ins_pipe(ialu_cr_reg_mem);
12565 %}
12566 
12567 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12568 %{
12569   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12570 
12571   ins_cost(125);
12572   format %{ "testb   $mem, $imm\t# ubyte" %}
12573   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12574   ins_pipe(ialu_cr_reg_mem);
12575 %}
12576 
12577 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12578 %{
12579   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12580 
12581   ins_cost(125);
12582   format %{ "testb   $mem, $imm\t# byte" %}
12583   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12584   ins_pipe(ialu_cr_reg_mem);
12585 %}
12586 
12587 //----------Max and Min--------------------------------------------------------
12588 // Min Instructions
12589 
12590 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12591 %{
12592   effect(USE_DEF dst, USE src, USE cr);
12593 
12594   format %{ "cmovlgt $dst, $src\t# min" %}
12595   ins_encode %{
12596     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12597   %}
12598   ins_pipe(pipe_cmov_reg);
12599 %}
12600 
12601 
12602 instruct minI_rReg(rRegI dst, rRegI src)
12603 %{
12604   match(Set dst (MinI dst src));
12605 
12606   ins_cost(200);
12607   expand %{
12608     rFlagsReg cr;
12609     compI_rReg(cr, dst, src);
12610     cmovI_reg_g(dst, src, cr);
12611   %}
12612 %}
12613 
12614 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12615 %{
12616   effect(USE_DEF dst, USE src, USE cr);
12617 
12618   format %{ "cmovllt $dst, $src\t# max" %}
12619   ins_encode %{
12620     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12621   %}
12622   ins_pipe(pipe_cmov_reg);
12623 %}
12624 
12625 
12626 instruct maxI_rReg(rRegI dst, rRegI src)
12627 %{
12628   match(Set dst (MaxI dst src));
12629 
12630   ins_cost(200);
12631   expand %{
12632     rFlagsReg cr;
12633     compI_rReg(cr, dst, src);
12634     cmovI_reg_l(dst, src, cr);
12635   %}
12636 %}
12637 
12638 // ============================================================================
12639 // Branch Instructions
12640 
12641 // Jump Direct - Label defines a relative address from JMP+1
12642 instruct jmpDir(label labl)
12643 %{
12644   match(Goto);
12645   effect(USE labl);
12646 
12647   ins_cost(300);
12648   format %{ "jmp     $labl" %}
12649   size(5);
12650   ins_encode %{
12651     Label* L = $labl$$label;
12652     __ jmp(*L, false); // Always long jump
12653   %}
12654   ins_pipe(pipe_jmp);
12655 %}
12656 
12657 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12658 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12659 %{
12660   match(If cop cr);
12661   effect(USE labl);
12662 
12663   ins_cost(300);
12664   format %{ "j$cop     $labl" %}
12665   size(6);
12666   ins_encode %{
12667     Label* L = $labl$$label;
12668     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12669   %}
12670   ins_pipe(pipe_jcc);
12671 %}
12672 
12673 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12674 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12675 %{
12676   match(CountedLoopEnd cop cr);
12677   effect(USE labl);
12678 
12679   ins_cost(300);
12680   format %{ "j$cop     $labl\t# loop end" %}
12681   size(6);
12682   ins_encode %{
12683     Label* L = $labl$$label;
12684     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12685   %}
12686   ins_pipe(pipe_jcc);
12687 %}
12688 
12689 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12690 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12691   match(CountedLoopEnd cop cmp);
12692   effect(USE labl);
12693 
12694   ins_cost(300);
12695   format %{ "j$cop,u   $labl\t# loop end" %}
12696   size(6);
12697   ins_encode %{
12698     Label* L = $labl$$label;
12699     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12700   %}
12701   ins_pipe(pipe_jcc);
12702 %}
12703 
12704 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12705   match(CountedLoopEnd cop cmp);
12706   effect(USE labl);
12707 
12708   ins_cost(200);
12709   format %{ "j$cop,u   $labl\t# loop end" %}
12710   size(6);
12711   ins_encode %{
12712     Label* L = $labl$$label;
12713     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12714   %}
12715   ins_pipe(pipe_jcc);
12716 %}
12717 
12718 // Jump Direct Conditional - using unsigned comparison
12719 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12720   match(If cop cmp);
12721   effect(USE labl);
12722 
12723   ins_cost(300);
12724   format %{ "j$cop,u   $labl" %}
12725   size(6);
12726   ins_encode %{
12727     Label* L = $labl$$label;
12728     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12729   %}
12730   ins_pipe(pipe_jcc);
12731 %}
12732 
12733 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12734   match(If cop cmp);
12735   effect(USE labl);
12736 
12737   ins_cost(200);
12738   format %{ "j$cop,u   $labl" %}
12739   size(6);
12740   ins_encode %{
12741     Label* L = $labl$$label;
12742     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12743   %}
12744   ins_pipe(pipe_jcc);
12745 %}
12746 
12747 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12748   match(If cop cmp);
12749   effect(USE labl);
12750 
12751   ins_cost(200);
12752   format %{ $$template
12753     if ($cop$$cmpcode == Assembler::notEqual) {
12754       $$emit$$"jp,u    $labl\n\t"
12755       $$emit$$"j$cop,u   $labl"
12756     } else {
12757       $$emit$$"jp,u    done\n\t"
12758       $$emit$$"j$cop,u   $labl\n\t"
12759       $$emit$$"done:"
12760     }
12761   %}
12762   ins_encode %{
12763     Label* l = $labl$$label;
12764     if ($cop$$cmpcode == Assembler::notEqual) {
12765       __ jcc(Assembler::parity, *l, false);
12766       __ jcc(Assembler::notEqual, *l, false);
12767     } else if ($cop$$cmpcode == Assembler::equal) {
12768       Label done;
12769       __ jccb(Assembler::parity, done);
12770       __ jcc(Assembler::equal, *l, false);
12771       __ bind(done);
12772     } else {
12773        ShouldNotReachHere();
12774     }
12775   %}
12776   ins_pipe(pipe_jcc);
12777 %}
12778 
12779 // ============================================================================
12780 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12781 // superklass array for an instance of the superklass.  Set a hidden
12782 // internal cache on a hit (cache is checked with exposed code in
12783 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12784 // encoding ALSO sets flags.
12785 
12786 instruct partialSubtypeCheck(rdi_RegP result,
12787                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12788                              rFlagsReg cr)
12789 %{
12790   match(Set result (PartialSubtypeCheck sub super));
12791   effect(KILL rcx, KILL cr);
12792 
12793   ins_cost(1100);  // slightly larger than the next version
12794   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12795             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12796             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12797             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12798             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12799             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12800             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12801     "miss:\t" %}
12802 
12803   opcode(0x1); // Force a XOR of RDI
12804   ins_encode(enc_PartialSubtypeCheck());
12805   ins_pipe(pipe_slow);
12806 %}
12807 
12808 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12809                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12810                                      immP0 zero,
12811                                      rdi_RegP result)
12812 %{
12813   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12814   effect(KILL rcx, KILL result);
12815 
12816   ins_cost(1000);
12817   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12818             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12819             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12820             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12821             "jne,s   miss\t\t# Missed: flags nz\n\t"
12822             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12823     "miss:\t" %}
12824 
12825   opcode(0x0); // No need to XOR RDI
12826   ins_encode(enc_PartialSubtypeCheck());
12827   ins_pipe(pipe_slow);
12828 %}
12829 
12830 // ============================================================================
12831 // Branch Instructions -- short offset versions
12832 //
12833 // These instructions are used to replace jumps of a long offset (the default
12834 // match) with jumps of a shorter offset.  These instructions are all tagged
12835 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12836 // match rules in general matching.  Instead, the ADLC generates a conversion
12837 // method in the MachNode which can be used to do in-place replacement of the
12838 // long variant with the shorter variant.  The compiler will determine if a
12839 // branch can be taken by the is_short_branch_offset() predicate in the machine
12840 // specific code section of the file.
12841 
12842 // Jump Direct - Label defines a relative address from JMP+1
12843 instruct jmpDir_short(label labl) %{
12844   match(Goto);
12845   effect(USE labl);
12846 
12847   ins_cost(300);
12848   format %{ "jmp,s   $labl" %}
12849   size(2);
12850   ins_encode %{
12851     Label* L = $labl$$label;
12852     __ jmpb(*L);
12853   %}
12854   ins_pipe(pipe_jmp);
12855   ins_short_branch(1);
12856 %}
12857 
12858 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12859 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12860   match(If cop cr);
12861   effect(USE labl);
12862 
12863   ins_cost(300);
12864   format %{ "j$cop,s   $labl" %}
12865   size(2);
12866   ins_encode %{
12867     Label* L = $labl$$label;
12868     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12869   %}
12870   ins_pipe(pipe_jcc);
12871   ins_short_branch(1);
12872 %}
12873 
12874 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12875 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12876   match(CountedLoopEnd cop cr);
12877   effect(USE labl);
12878 
12879   ins_cost(300);
12880   format %{ "j$cop,s   $labl\t# loop end" %}
12881   size(2);
12882   ins_encode %{
12883     Label* L = $labl$$label;
12884     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12885   %}
12886   ins_pipe(pipe_jcc);
12887   ins_short_branch(1);
12888 %}
12889 
12890 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12891 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12892   match(CountedLoopEnd cop cmp);
12893   effect(USE labl);
12894 
12895   ins_cost(300);
12896   format %{ "j$cop,us  $labl\t# loop end" %}
12897   size(2);
12898   ins_encode %{
12899     Label* L = $labl$$label;
12900     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12901   %}
12902   ins_pipe(pipe_jcc);
12903   ins_short_branch(1);
12904 %}
12905 
12906 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12907   match(CountedLoopEnd cop cmp);
12908   effect(USE labl);
12909 
12910   ins_cost(300);
12911   format %{ "j$cop,us  $labl\t# loop end" %}
12912   size(2);
12913   ins_encode %{
12914     Label* L = $labl$$label;
12915     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12916   %}
12917   ins_pipe(pipe_jcc);
12918   ins_short_branch(1);
12919 %}
12920 
12921 // Jump Direct Conditional - using unsigned comparison
12922 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12923   match(If cop cmp);
12924   effect(USE labl);
12925 
12926   ins_cost(300);
12927   format %{ "j$cop,us  $labl" %}
12928   size(2);
12929   ins_encode %{
12930     Label* L = $labl$$label;
12931     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12932   %}
12933   ins_pipe(pipe_jcc);
12934   ins_short_branch(1);
12935 %}
12936 
12937 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12938   match(If cop cmp);
12939   effect(USE labl);
12940 
12941   ins_cost(300);
12942   format %{ "j$cop,us  $labl" %}
12943   size(2);
12944   ins_encode %{
12945     Label* L = $labl$$label;
12946     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12947   %}
12948   ins_pipe(pipe_jcc);
12949   ins_short_branch(1);
12950 %}
12951 
12952 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12953   match(If cop cmp);
12954   effect(USE labl);
12955 
12956   ins_cost(300);
12957   format %{ $$template
12958     if ($cop$$cmpcode == Assembler::notEqual) {
12959       $$emit$$"jp,u,s  $labl\n\t"
12960       $$emit$$"j$cop,u,s  $labl"
12961     } else {
12962       $$emit$$"jp,u,s  done\n\t"
12963       $$emit$$"j$cop,u,s  $labl\n\t"
12964       $$emit$$"done:"
12965     }
12966   %}
12967   size(4);
12968   ins_encode %{
12969     Label* l = $labl$$label;
12970     if ($cop$$cmpcode == Assembler::notEqual) {
12971       __ jccb(Assembler::parity, *l);
12972       __ jccb(Assembler::notEqual, *l);
12973     } else if ($cop$$cmpcode == Assembler::equal) {
12974       Label done;
12975       __ jccb(Assembler::parity, done);
12976       __ jccb(Assembler::equal, *l);
12977       __ bind(done);
12978     } else {
12979        ShouldNotReachHere();
12980     }
12981   %}
12982   ins_pipe(pipe_jcc);
12983   ins_short_branch(1);
12984 %}
12985 
12986 // ============================================================================
12987 // inlined locking and unlocking
12988 
12989 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12990   predicate(Compile::current()->use_rtm());
12991   match(Set cr (FastLock object box));
12992   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12993   ins_cost(300);
12994   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12995   ins_encode %{
12996     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12997                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12998                  _rtm_counters, _stack_rtm_counters,
12999                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13000                  true, ra_->C->profile_rtm());
13001   %}
13002   ins_pipe(pipe_slow);
13003 %}
13004 
13005 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13006   predicate(!Compile::current()->use_rtm());
13007   match(Set cr (FastLock object box));
13008   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
13009   ins_cost(300);
13010   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13011   ins_encode %{
13012     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13013                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
13014   %}
13015   ins_pipe(pipe_slow);
13016 %}
13017 
13018 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13019   match(Set cr (FastUnlock object box));
13020   effect(TEMP tmp, USE_KILL box);
13021   ins_cost(300);
13022   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13023   ins_encode %{
13024     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13025   %}
13026   ins_pipe(pipe_slow);
13027 %}
13028 
13029 
13030 // ============================================================================
13031 // Safepoint Instructions
13032 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13033 %{
13034   match(SafePoint poll);
13035   effect(KILL cr, USE poll);
13036 
13037   format %{ "testl   rax, [$poll]\t"
13038             "# Safepoint: poll for GC" %}
13039   ins_cost(125);
13040   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13041   ins_encode %{
13042     __ relocate(relocInfo::poll_type);
13043     address pre_pc = __ pc();
13044     __ testl(rax, Address($poll$$Register, 0));
13045     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13046   %}
13047   ins_pipe(ialu_reg_mem);
13048 %}
13049 
13050 instruct mask_all_evexL(kReg dst, rRegL src) %{
13051   match(Set dst (MaskAll src));
13052   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13053   ins_encode %{
13054     int mask_len = Matcher::vector_length(this);
13055     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13056   %}
13057   ins_pipe( pipe_slow );
13058 %}
13059 
13060 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13061   predicate(Matcher::vector_length(n) > 32);
13062   match(Set dst (MaskAll src));
13063   effect(TEMP tmp);
13064   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13065   ins_encode %{
13066     int mask_len = Matcher::vector_length(this);
13067     __ movslq($tmp$$Register, $src$$Register);
13068     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13069   %}
13070   ins_pipe( pipe_slow );
13071 %}
13072 
13073 // ============================================================================
13074 // Procedure Call/Return Instructions
13075 // Call Java Static Instruction
13076 // Note: If this code changes, the corresponding ret_addr_offset() and
13077 //       compute_padding() functions will have to be adjusted.
13078 instruct CallStaticJavaDirect(method meth) %{
13079   match(CallStaticJava);
13080   effect(USE meth);
13081 
13082   ins_cost(300);
13083   format %{ "call,static " %}
13084   opcode(0xE8); /* E8 cd */
13085   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13086   ins_pipe(pipe_slow);
13087   ins_alignment(4);
13088 %}
13089 
13090 // Call Java Dynamic Instruction
13091 // Note: If this code changes, the corresponding ret_addr_offset() and
13092 //       compute_padding() functions will have to be adjusted.
13093 instruct CallDynamicJavaDirect(method meth)
13094 %{
13095   match(CallDynamicJava);
13096   effect(USE meth);
13097 
13098   ins_cost(300);
13099   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13100             "call,dynamic " %}
13101   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13102   ins_pipe(pipe_slow);
13103   ins_alignment(4);
13104 %}
13105 
13106 // Call Runtime Instruction
13107 instruct CallRuntimeDirect(method meth)
13108 %{
13109   match(CallRuntime);
13110   effect(USE meth);
13111 
13112   ins_cost(300);
13113   format %{ "call,runtime " %}
13114   ins_encode(clear_avx, Java_To_Runtime(meth));
13115   ins_pipe(pipe_slow);
13116 %}
13117 
13118 // Call runtime without safepoint
13119 instruct CallLeafDirect(method meth)
13120 %{
13121   match(CallLeaf);
13122   effect(USE meth);
13123 
13124   ins_cost(300);
13125   format %{ "call_leaf,runtime " %}
13126   ins_encode(clear_avx, Java_To_Runtime(meth));
13127   ins_pipe(pipe_slow);
13128 %}
13129 
13130 // Call runtime without safepoint and with vector arguments
13131 instruct CallLeafDirectVector(method meth)
13132 %{
13133   match(CallLeafVector);
13134   effect(USE meth);
13135 
13136   ins_cost(300);
13137   format %{ "call_leaf,vector " %}
13138   ins_encode(Java_To_Runtime(meth));
13139   ins_pipe(pipe_slow);
13140 %}
13141 
13142 //
13143 instruct CallNativeDirect(method meth)
13144 %{
13145   match(CallNative);
13146   effect(USE meth);
13147 
13148   ins_cost(300);
13149   format %{ "call_native " %}
13150   ins_encode(clear_avx, Java_To_Runtime(meth));
13151   ins_pipe(pipe_slow);
13152 %}
13153 
13154 // Call runtime without safepoint
13155 instruct CallLeafNoFPDirect(method meth)
13156 %{
13157   match(CallLeafNoFP);
13158   effect(USE meth);
13159 
13160   ins_cost(300);
13161   format %{ "call_leaf_nofp,runtime " %}
13162   ins_encode(clear_avx, Java_To_Runtime(meth));
13163   ins_pipe(pipe_slow);
13164 %}
13165 
13166 // Return Instruction
13167 // Remove the return address & jump to it.
13168 // Notice: We always emit a nop after a ret to make sure there is room
13169 // for safepoint patching
13170 instruct Ret()
13171 %{
13172   match(Return);
13173 
13174   format %{ "ret" %}
13175   ins_encode %{
13176     __ ret(0);
13177   %}
13178   ins_pipe(pipe_jmp);
13179 %}
13180 
13181 // Tail Call; Jump from runtime stub to Java code.
13182 // Also known as an 'interprocedural jump'.
13183 // Target of jump will eventually return to caller.
13184 // TailJump below removes the return address.
13185 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13186 %{
13187   match(TailCall jump_target method_ptr);
13188 
13189   ins_cost(300);
13190   format %{ "jmp     $jump_target\t# rbx holds method" %}
13191   ins_encode %{
13192     __ jmp($jump_target$$Register);
13193   %}
13194   ins_pipe(pipe_jmp);
13195 %}
13196 
13197 // Tail Jump; remove the return address; jump to target.
13198 // TailCall above leaves the return address around.
13199 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13200 %{
13201   match(TailJump jump_target ex_oop);
13202 
13203   ins_cost(300);
13204   format %{ "popq    rdx\t# pop return address\n\t"
13205             "jmp     $jump_target" %}
13206   ins_encode %{
13207     __ popq(as_Register(RDX_enc));
13208     __ jmp($jump_target$$Register);
13209   %}
13210   ins_pipe(pipe_jmp);
13211 %}
13212 
13213 // Create exception oop: created by stack-crawling runtime code.
13214 // Created exception is now available to this handler, and is setup
13215 // just prior to jumping to this handler.  No code emitted.
13216 instruct CreateException(rax_RegP ex_oop)
13217 %{
13218   match(Set ex_oop (CreateEx));
13219 
13220   size(0);
13221   // use the following format syntax
13222   format %{ "# exception oop is in rax; no code emitted" %}
13223   ins_encode();
13224   ins_pipe(empty);
13225 %}
13226 
13227 // Rethrow exception:
13228 // The exception oop will come in the first argument position.
13229 // Then JUMP (not call) to the rethrow stub code.
13230 instruct RethrowException()
13231 %{
13232   match(Rethrow);
13233 
13234   // use the following format syntax
13235   format %{ "jmp     rethrow_stub" %}
13236   ins_encode(enc_rethrow);
13237   ins_pipe(pipe_jmp);
13238 %}
13239 
13240 // ============================================================================
13241 // This name is KNOWN by the ADLC and cannot be changed.
13242 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13243 // for this guy.
13244 instruct tlsLoadP(r15_RegP dst) %{
13245   match(Set dst (ThreadLocal));
13246   effect(DEF dst);
13247 
13248   size(0);
13249   format %{ "# TLS is in R15" %}
13250   ins_encode( /*empty encoding*/ );
13251   ins_pipe(ialu_reg_reg);
13252 %}
13253 
13254 
13255 //----------PEEPHOLE RULES-----------------------------------------------------
13256 // These must follow all instruction definitions as they use the names
13257 // defined in the instructions definitions.
13258 //
13259 // peepmatch ( root_instr_name [preceding_instruction]* );
13260 //
13261 // peepconstraint %{
13262 // (instruction_number.operand_name relational_op instruction_number.operand_name
13263 //  [, ...] );
13264 // // instruction numbers are zero-based using left to right order in peepmatch
13265 //
13266 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13267 // // provide an instruction_number.operand_name for each operand that appears
13268 // // in the replacement instruction's match rule
13269 //
13270 // ---------VM FLAGS---------------------------------------------------------
13271 //
13272 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13273 //
13274 // Each peephole rule is given an identifying number starting with zero and
13275 // increasing by one in the order seen by the parser.  An individual peephole
13276 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13277 // on the command-line.
13278 //
13279 // ---------CURRENT LIMITATIONS----------------------------------------------
13280 //
13281 // Only match adjacent instructions in same basic block
13282 // Only equality constraints
13283 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13284 // Only one replacement instruction
13285 //
13286 // ---------EXAMPLE----------------------------------------------------------
13287 //
13288 // // pertinent parts of existing instructions in architecture description
13289 // instruct movI(rRegI dst, rRegI src)
13290 // %{
13291 //   match(Set dst (CopyI src));
13292 // %}
13293 //
13294 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13295 // %{
13296 //   match(Set dst (AddI dst src));
13297 //   effect(KILL cr);
13298 // %}
13299 //
13300 // // Change (inc mov) to lea
13301 // peephole %{
13302 //   // increment preceded by register-register move
13303 //   peepmatch ( incI_rReg movI );
13304 //   // require that the destination register of the increment
13305 //   // match the destination register of the move
13306 //   peepconstraint ( 0.dst == 1.dst );
13307 //   // construct a replacement instruction that sets
13308 //   // the destination to ( move's source register + one )
13309 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13310 // %}
13311 //
13312 
13313 // Implementation no longer uses movX instructions since
13314 // machine-independent system no longer uses CopyX nodes.
13315 //
13316 // peephole
13317 // %{
13318 //   peepmatch (incI_rReg movI);
13319 //   peepconstraint (0.dst == 1.dst);
13320 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13321 // %}
13322 
13323 // peephole
13324 // %{
13325 //   peepmatch (decI_rReg movI);
13326 //   peepconstraint (0.dst == 1.dst);
13327 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13328 // %}
13329 
13330 // peephole
13331 // %{
13332 //   peepmatch (addI_rReg_imm movI);
13333 //   peepconstraint (0.dst == 1.dst);
13334 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13335 // %}
13336 
13337 // peephole
13338 // %{
13339 //   peepmatch (incL_rReg movL);
13340 //   peepconstraint (0.dst == 1.dst);
13341 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13342 // %}
13343 
13344 // peephole
13345 // %{
13346 //   peepmatch (decL_rReg movL);
13347 //   peepconstraint (0.dst == 1.dst);
13348 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13349 // %}
13350 
13351 // peephole
13352 // %{
13353 //   peepmatch (addL_rReg_imm movL);
13354 //   peepconstraint (0.dst == 1.dst);
13355 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13356 // %}
13357 
13358 // peephole
13359 // %{
13360 //   peepmatch (addP_rReg_imm movP);
13361 //   peepconstraint (0.dst == 1.dst);
13362 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13363 // %}
13364 
13365 // // Change load of spilled value to only a spill
13366 // instruct storeI(memory mem, rRegI src)
13367 // %{
13368 //   match(Set mem (StoreI mem src));
13369 // %}
13370 //
13371 // instruct loadI(rRegI dst, memory mem)
13372 // %{
13373 //   match(Set dst (LoadI mem));
13374 // %}
13375 //
13376 
13377 peephole
13378 %{
13379   peepmatch (loadI storeI);
13380   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13381   peepreplace (storeI(1.mem 1.mem 1.src));
13382 %}
13383 
13384 peephole
13385 %{
13386   peepmatch (loadL storeL);
13387   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13388   peepreplace (storeL(1.mem 1.mem 1.src));
13389 %}
13390 
13391 //----------SMARTSPILL RULES---------------------------------------------------
13392 // These must follow all instruction definitions as they use the names
13393 // defined in the instructions definitions.