1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 
  323 source_hpp %{
  324 
  325 #include "peephole_x86_64.hpp"
  326 
  327 %}
  328 
  329 // Register masks
  330 source_hpp %{
  331 
  332 extern RegMask _ANY_REG_mask;
  333 extern RegMask _PTR_REG_mask;
  334 extern RegMask _PTR_REG_NO_RBP_mask;
  335 extern RegMask _PTR_NO_RAX_REG_mask;
  336 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  337 extern RegMask _LONG_REG_mask;
  338 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  339 extern RegMask _LONG_NO_RCX_REG_mask;
  340 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  341 extern RegMask _INT_REG_mask;
  342 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  343 extern RegMask _INT_NO_RCX_REG_mask;
  344 extern RegMask _INT_NO_RBP_R13_REG_mask;
  345 extern RegMask _FLOAT_REG_mask;
  346 
  347 extern RegMask _STACK_OR_PTR_REG_mask;
  348 extern RegMask _STACK_OR_LONG_REG_mask;
  349 extern RegMask _STACK_OR_INT_REG_mask;
  350 
  351 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  352 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  353 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_INT_REG_mask;
  380 
  381 static bool need_r12_heapbase() {
  382   return UseCompressedOops;
  383 }
  384 
  385 void reg_mask_init() {
  386   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  387   // We derive a number of subsets from it.
  388   _ANY_REG_mask = _ALL_REG_mask;
  389 
  390   if (PreserveFramePointer) {
  391     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  392     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  393   }
  394   if (need_r12_heapbase()) {
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  396     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  397   }
  398 
  399   _PTR_REG_mask = _ANY_REG_mask;
  400   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  401   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  404 
  405   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  406   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  407 
  408   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  409   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  410   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  411 
  412   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  413   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  414   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  415 
  416   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  417   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  418   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  419 
  420   _LONG_REG_mask = _PTR_REG_mask;
  421   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  422   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  423 
  424   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  425   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  426   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  429 
  430   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  431   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  432   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  433 
  434   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  435   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  436   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  439 
  440   _INT_REG_mask = _ALL_INT_REG_mask;
  441   if (PreserveFramePointer) {
  442     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  443   }
  444   if (need_r12_heapbase()) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  446   }
  447 
  448   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  449   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  450 
  451   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  452   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  453   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  454 
  455   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  456   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  457 
  458   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  459   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  460   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  461 
  462   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  463   // from the float_reg_legacy/float_reg_evex register class.
  464   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  465 }
  466 
  467 static bool generate_vzeroupper(Compile* C) {
  468   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  469 }
  470 
  471 static int clear_avx_size() {
  472   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  473 }
  474 
  475 // !!!!! Special hack to get all types of calls to specify the byte offset
  476 //       from the start of the call to the point where the return address
  477 //       will point.
  478 int MachCallStaticJavaNode::ret_addr_offset()
  479 {
  480   int offset = 5; // 5 bytes from start of call to where return address points
  481   offset += clear_avx_size();
  482   return offset;
  483 }
  484 
  485 int MachCallDynamicJavaNode::ret_addr_offset()
  486 {
  487   int offset = 15; // 15 bytes from start of call to where return address points
  488   offset += clear_avx_size();
  489   return offset;
  490 }
  491 
  492 int MachCallRuntimeNode::ret_addr_offset() {
  493   int offset = 13; // movq r10,#addr; callq (r10)
  494   if (this->ideal_Opcode() != Op_CallLeafVector) {
  495     offset += clear_avx_size();
  496   }
  497   return offset;
  498 }
  499 //
  500 // Compute padding required for nodes which need alignment
  501 //
  502 
  503 // The address of the call instruction needs to be 4-byte aligned to
  504 // ensure that it does not span a cache line so that it can be patched.
  505 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  506 {
  507   current_offset += clear_avx_size(); // skip vzeroupper
  508   current_offset += 1; // skip call opcode byte
  509   return align_up(current_offset, alignment_required()) - current_offset;
  510 }
  511 
  512 // The address of the call instruction needs to be 4-byte aligned to
  513 // ensure that it does not span a cache line so that it can be patched.
  514 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  515 {
  516   current_offset += clear_avx_size(); // skip vzeroupper
  517   current_offset += 11; // skip movq instruction + call opcode byte
  518   return align_up(current_offset, alignment_required()) - current_offset;
  519 }
  520 
  521 // EMIT_RM()
  522 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  523   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  524   cbuf.insts()->emit_int8(c);
  525 }
  526 
  527 // EMIT_CC()
  528 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  529   unsigned char c = (unsigned char) (f1 | f2);
  530   cbuf.insts()->emit_int8(c);
  531 }
  532 
  533 // EMIT_OPCODE()
  534 void emit_opcode(CodeBuffer &cbuf, int code) {
  535   cbuf.insts()->emit_int8((unsigned char) code);
  536 }
  537 
  538 // EMIT_OPCODE() w/ relocation information
  539 void emit_opcode(CodeBuffer &cbuf,
  540                  int code, relocInfo::relocType reloc, int offset, int format)
  541 {
  542   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  543   emit_opcode(cbuf, code);
  544 }
  545 
  546 // EMIT_D8()
  547 void emit_d8(CodeBuffer &cbuf, int d8) {
  548   cbuf.insts()->emit_int8((unsigned char) d8);
  549 }
  550 
  551 // EMIT_D16()
  552 void emit_d16(CodeBuffer &cbuf, int d16) {
  553   cbuf.insts()->emit_int16(d16);
  554 }
  555 
  556 // EMIT_D32()
  557 void emit_d32(CodeBuffer &cbuf, int d32) {
  558   cbuf.insts()->emit_int32(d32);
  559 }
  560 
  561 // EMIT_D64()
  562 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  563   cbuf.insts()->emit_int64(d64);
  564 }
  565 
  566 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  567 void emit_d32_reloc(CodeBuffer& cbuf,
  568                     int d32,
  569                     relocInfo::relocType reloc,
  570                     int format)
  571 {
  572   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  573   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  574   cbuf.insts()->emit_int32(d32);
  575 }
  576 
  577 // emit 32 bit value and construct relocation entry from RelocationHolder
  578 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  579 #ifdef ASSERT
  580   if (rspec.reloc()->type() == relocInfo::oop_type &&
  581       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  582     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  583     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  584   }
  585 #endif
  586   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  587   cbuf.insts()->emit_int32(d32);
  588 }
  589 
  590 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  591   address next_ip = cbuf.insts_end() + 4;
  592   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  593                  external_word_Relocation::spec(addr),
  594                  RELOC_DISP32);
  595 }
  596 
  597 
  598 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  599 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  600   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  601   cbuf.insts()->emit_int64(d64);
  602 }
  603 
  604 // emit 64 bit value and construct relocation entry from RelocationHolder
  605 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  606 #ifdef ASSERT
  607   if (rspec.reloc()->type() == relocInfo::oop_type &&
  608       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  609     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  610     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  611   }
  612 #endif
  613   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  614   cbuf.insts()->emit_int64(d64);
  615 }
  616 
  617 // Access stack slot for load or store
  618 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  619 {
  620   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  621   if (-0x80 <= disp && disp < 0x80) {
  622     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  623     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  624     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  625   } else {
  626     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  627     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  628     emit_d32(cbuf, disp);     // Displacement // R/M byte
  629   }
  630 }
  631 
  632    // rRegI ereg, memory mem) %{    // emit_reg_mem
  633 void encode_RegMem(CodeBuffer &cbuf,
  634                    int reg,
  635                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  636 {
  637   assert(disp_reloc == relocInfo::none, "cannot have disp");
  638   int regenc = reg & 7;
  639   int baseenc = base & 7;
  640   int indexenc = index & 7;
  641 
  642   // There is no index & no scale, use form without SIB byte
  643   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  644     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  645     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  646       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  647     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  648       // If 8-bit displacement, mode 0x1
  649       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  650       emit_d8(cbuf, disp);
  651     } else {
  652       // If 32-bit displacement
  653       if (base == -1) { // Special flag for absolute address
  654         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  655         if (disp_reloc != relocInfo::none) {
  656           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  657         } else {
  658           emit_d32(cbuf, disp);
  659         }
  660       } else {
  661         // Normal base + offset
  662         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  663         if (disp_reloc != relocInfo::none) {
  664           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  665         } else {
  666           emit_d32(cbuf, disp);
  667         }
  668       }
  669     }
  670   } else {
  671     // Else, encode with the SIB byte
  672     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  673     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  674       // If no displacement
  675       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  676       emit_rm(cbuf, scale, indexenc, baseenc);
  677     } else {
  678       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  679         // If 8-bit displacement, mode 0x1
  680         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  681         emit_rm(cbuf, scale, indexenc, baseenc);
  682         emit_d8(cbuf, disp);
  683       } else {
  684         // If 32-bit displacement
  685         if (base == 0x04 ) {
  686           emit_rm(cbuf, 0x2, regenc, 0x4);
  687           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  688         } else {
  689           emit_rm(cbuf, 0x2, regenc, 0x4);
  690           emit_rm(cbuf, scale, indexenc, baseenc); // *
  691         }
  692         if (disp_reloc != relocInfo::none) {
  693           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  694         } else {
  695           emit_d32(cbuf, disp);
  696         }
  697       }
  698     }
  699   }
  700 }
  701 
  702 // This could be in MacroAssembler but it's fairly C2 specific
  703 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  704   Label exit;
  705   __ jccb(Assembler::noParity, exit);
  706   __ pushf();
  707   //
  708   // comiss/ucomiss instructions set ZF,PF,CF flags and
  709   // zero OF,AF,SF for NaN values.
  710   // Fixup flags by zeroing ZF,PF so that compare of NaN
  711   // values returns 'less than' result (CF is set).
  712   // Leave the rest of flags unchanged.
  713   //
  714   //    7 6 5 4 3 2 1 0
  715   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  716   //    0 0 1 0 1 0 1 1   (0x2B)
  717   //
  718   __ andq(Address(rsp, 0), 0xffffff2b);
  719   __ popf();
  720   __ bind(exit);
  721 }
  722 
  723 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  724   Label done;
  725   __ movl(dst, -1);
  726   __ jcc(Assembler::parity, done);
  727   __ jcc(Assembler::below, done);
  728   __ setb(Assembler::notEqual, dst);
  729   __ movzbl(dst, dst);
  730   __ bind(done);
  731 }
  732 
  733 // Math.min()    # Math.max()
  734 // --------------------------
  735 // ucomis[s/d]   #
  736 // ja   -> b     # a
  737 // jp   -> NaN   # NaN
  738 // jb   -> a     # b
  739 // je            #
  740 // |-jz -> a | b # a & b
  741 // |    -> a     #
  742 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  743                      XMMRegister a, XMMRegister b,
  744                      XMMRegister xmmt, Register rt,
  745                      bool min, bool single) {
  746 
  747   Label nan, zero, below, above, done;
  748 
  749   if (single)
  750     __ ucomiss(a, b);
  751   else
  752     __ ucomisd(a, b);
  753 
  754   if (dst->encoding() != (min ? b : a)->encoding())
  755     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  756   else
  757     __ jccb(Assembler::above, done);
  758 
  759   __ jccb(Assembler::parity, nan);  // PF=1
  760   __ jccb(Assembler::below, below); // CF=1
  761 
  762   // equal
  763   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  764   if (single) {
  765     __ ucomiss(a, xmmt);
  766     __ jccb(Assembler::equal, zero);
  767 
  768     __ movflt(dst, a);
  769     __ jmp(done);
  770   }
  771   else {
  772     __ ucomisd(a, xmmt);
  773     __ jccb(Assembler::equal, zero);
  774 
  775     __ movdbl(dst, a);
  776     __ jmp(done);
  777   }
  778 
  779   __ bind(zero);
  780   if (min)
  781     __ vpor(dst, a, b, Assembler::AVX_128bit);
  782   else
  783     __ vpand(dst, a, b, Assembler::AVX_128bit);
  784 
  785   __ jmp(done);
  786 
  787   __ bind(above);
  788   if (single)
  789     __ movflt(dst, min ? b : a);
  790   else
  791     __ movdbl(dst, min ? b : a);
  792 
  793   __ jmp(done);
  794 
  795   __ bind(nan);
  796   if (single) {
  797     __ movl(rt, 0x7fc00000); // Float.NaN
  798     __ movdl(dst, rt);
  799   }
  800   else {
  801     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  802     __ movdq(dst, rt);
  803   }
  804   __ jmp(done);
  805 
  806   __ bind(below);
  807   if (single)
  808     __ movflt(dst, min ? a : b);
  809   else
  810     __ movdbl(dst, min ? a : b);
  811 
  812   __ bind(done);
  813 }
  814 
  815 //=============================================================================
  816 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  817 
  818 int ConstantTable::calculate_table_base_offset() const {
  819   return 0;  // absolute addressing, no offset
  820 }
  821 
  822 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  823 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  824   ShouldNotReachHere();
  825 }
  826 
  827 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  828   // Empty encoding
  829 }
  830 
  831 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  832   return 0;
  833 }
  834 
  835 #ifndef PRODUCT
  836 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  837   st->print("# MachConstantBaseNode (empty encoding)");
  838 }
  839 #endif
  840 
  841 
  842 //=============================================================================
  843 #ifndef PRODUCT
  844 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  845   Compile* C = ra_->C;
  846 
  847   int framesize = C->output()->frame_size_in_bytes();
  848   int bangsize = C->output()->bang_size_in_bytes();
  849   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  850   // Remove wordSize for return addr which is already pushed.
  851   framesize -= wordSize;
  852 
  853   if (C->output()->need_stack_bang(bangsize)) {
  854     framesize -= wordSize;
  855     st->print("# stack bang (%d bytes)", bangsize);
  856     st->print("\n\t");
  857     st->print("pushq   rbp\t# Save rbp");
  858     if (PreserveFramePointer) {
  859         st->print("\n\t");
  860         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  861     }
  862     if (framesize) {
  863       st->print("\n\t");
  864       st->print("subq    rsp, #%d\t# Create frame",framesize);
  865     }
  866   } else {
  867     st->print("subq    rsp, #%d\t# Create frame",framesize);
  868     st->print("\n\t");
  869     framesize -= wordSize;
  870     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  871     if (PreserveFramePointer) {
  872       st->print("\n\t");
  873       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  874       if (framesize > 0) {
  875         st->print("\n\t");
  876         st->print("addq    rbp, #%d", framesize);
  877       }
  878     }
  879   }
  880 
  881   if (VerifyStackAtCalls) {
  882     st->print("\n\t");
  883     framesize -= wordSize;
  884     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  885 #ifdef ASSERT
  886     st->print("\n\t");
  887     st->print("# stack alignment check");
  888 #endif
  889   }
  890   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  891     st->print("\n\t");
  892     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  893     st->print("\n\t");
  894     st->print("je      fast_entry\t");
  895     st->print("\n\t");
  896     st->print("call    #nmethod_entry_barrier_stub\t");
  897     st->print("\n\tfast_entry:");
  898   }
  899   st->cr();
  900 }
  901 #endif
  902 
  903 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  904   Compile* C = ra_->C;
  905   C2_MacroAssembler _masm(&cbuf);
  906 
  907   int framesize = C->output()->frame_size_in_bytes();
  908   int bangsize = C->output()->bang_size_in_bytes();
  909 
  910   if (C->clinit_barrier_on_entry()) {
  911     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  912     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  913 
  914     Label L_skip_barrier;
  915     Register klass = rscratch1;
  916 
  917     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  918     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  919 
  920     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  921 
  922     __ bind(L_skip_barrier);
  923   }
  924 
  925   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  926 
  927   C->output()->set_frame_complete(cbuf.insts_size());
  928 
  929   if (C->has_mach_constant_base_node()) {
  930     // NOTE: We set the table base offset here because users might be
  931     // emitted before MachConstantBaseNode.
  932     ConstantTable& constant_table = C->output()->constant_table();
  933     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  934   }
  935 }
  936 
  937 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  938 {
  939   return MachNode::size(ra_); // too many variables; just compute it
  940                               // the hard way
  941 }
  942 
  943 int MachPrologNode::reloc() const
  944 {
  945   return 0; // a large enough number
  946 }
  947 
  948 //=============================================================================
  949 #ifndef PRODUCT
  950 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  951 {
  952   Compile* C = ra_->C;
  953   if (generate_vzeroupper(C)) {
  954     st->print("vzeroupper");
  955     st->cr(); st->print("\t");
  956   }
  957 
  958   int framesize = C->output()->frame_size_in_bytes();
  959   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  960   // Remove word for return adr already pushed
  961   // and RBP
  962   framesize -= 2*wordSize;
  963 
  964   if (framesize) {
  965     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  966     st->print("\t");
  967   }
  968 
  969   st->print_cr("popq    rbp");
  970   if (do_polling() && C->is_method_compilation()) {
  971     st->print("\t");
  972     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  973                  "ja      #safepoint_stub\t"
  974                  "# Safepoint: poll for GC");
  975   }
  976 }
  977 #endif
  978 
  979 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  980 {
  981   Compile* C = ra_->C;
  982   MacroAssembler _masm(&cbuf);
  983 
  984   if (generate_vzeroupper(C)) {
  985     // Clear upper bits of YMM registers when current compiled code uses
  986     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  987     __ vzeroupper();
  988   }
  989 
  990   int framesize = C->output()->frame_size_in_bytes();
  991   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  992   // Remove word for return adr already pushed
  993   // and RBP
  994   framesize -= 2*wordSize;
  995 
  996   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  997 
  998   if (framesize) {
  999     emit_opcode(cbuf, Assembler::REX_W);
 1000     if (framesize < 0x80) {
 1001       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 1002       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 1003       emit_d8(cbuf, framesize);
 1004     } else {
 1005       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 1006       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 1007       emit_d32(cbuf, framesize);
 1008     }
 1009   }
 1010 
 1011   // popq rbp
 1012   emit_opcode(cbuf, 0x58 | RBP_enc);
 1013 
 1014   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1015     __ reserved_stack_check();
 1016   }
 1017 
 1018   if (do_polling() && C->is_method_compilation()) {
 1019     MacroAssembler _masm(&cbuf);
 1020     Label dummy_label;
 1021     Label* code_stub = &dummy_label;
 1022     if (!C->output()->in_scratch_emit_size()) {
 1023       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1024     }
 1025     __ relocate(relocInfo::poll_return_type);
 1026     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1027   }
 1028 }
 1029 
 1030 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1031 {
 1032   return MachNode::size(ra_); // too many variables; just compute it
 1033                               // the hard way
 1034 }
 1035 
 1036 int MachEpilogNode::reloc() const
 1037 {
 1038   return 2; // a large enough number
 1039 }
 1040 
 1041 const Pipeline* MachEpilogNode::pipeline() const
 1042 {
 1043   return MachNode::pipeline_class();
 1044 }
 1045 
 1046 //=============================================================================
 1047 
 1048 enum RC {
 1049   rc_bad,
 1050   rc_int,
 1051   rc_kreg,
 1052   rc_float,
 1053   rc_stack
 1054 };
 1055 
 1056 static enum RC rc_class(OptoReg::Name reg)
 1057 {
 1058   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1059 
 1060   if (OptoReg::is_stack(reg)) return rc_stack;
 1061 
 1062   VMReg r = OptoReg::as_VMReg(reg);
 1063 
 1064   if (r->is_Register()) return rc_int;
 1065 
 1066   if (r->is_KRegister()) return rc_kreg;
 1067 
 1068   assert(r->is_XMMRegister(), "must be");
 1069   return rc_float;
 1070 }
 1071 
 1072 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1073 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1074                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1075 
 1076 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1077                      int stack_offset, int reg, uint ireg, outputStream* st);
 1078 
 1079 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1080                                       int dst_offset, uint ireg, outputStream* st) {
 1081   if (cbuf) {
 1082     MacroAssembler _masm(cbuf);
 1083     switch (ireg) {
 1084     case Op_VecS:
 1085       __ movq(Address(rsp, -8), rax);
 1086       __ movl(rax, Address(rsp, src_offset));
 1087       __ movl(Address(rsp, dst_offset), rax);
 1088       __ movq(rax, Address(rsp, -8));
 1089       break;
 1090     case Op_VecD:
 1091       __ pushq(Address(rsp, src_offset));
 1092       __ popq (Address(rsp, dst_offset));
 1093       break;
 1094     case Op_VecX:
 1095       __ pushq(Address(rsp, src_offset));
 1096       __ popq (Address(rsp, dst_offset));
 1097       __ pushq(Address(rsp, src_offset+8));
 1098       __ popq (Address(rsp, dst_offset+8));
 1099       break;
 1100     case Op_VecY:
 1101       __ vmovdqu(Address(rsp, -32), xmm0);
 1102       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1103       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1104       __ vmovdqu(xmm0, Address(rsp, -32));
 1105       break;
 1106     case Op_VecZ:
 1107       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1108       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1109       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1110       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1111       break;
 1112     default:
 1113       ShouldNotReachHere();
 1114     }
 1115 #ifndef PRODUCT
 1116   } else {
 1117     switch (ireg) {
 1118     case Op_VecS:
 1119       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1120                 "movl    rax, [rsp + #%d]\n\t"
 1121                 "movl    [rsp + #%d], rax\n\t"
 1122                 "movq    rax, [rsp - #8]",
 1123                 src_offset, dst_offset);
 1124       break;
 1125     case Op_VecD:
 1126       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1127                 "popq    [rsp + #%d]",
 1128                 src_offset, dst_offset);
 1129       break;
 1130      case Op_VecX:
 1131       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1132                 "popq    [rsp + #%d]\n\t"
 1133                 "pushq   [rsp + #%d]\n\t"
 1134                 "popq    [rsp + #%d]",
 1135                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1136       break;
 1137     case Op_VecY:
 1138       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1139                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1140                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1141                 "vmovdqu xmm0, [rsp - #32]",
 1142                 src_offset, dst_offset);
 1143       break;
 1144     case Op_VecZ:
 1145       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1146                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1147                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1148                 "vmovdqu xmm0, [rsp - #64]",
 1149                 src_offset, dst_offset);
 1150       break;
 1151     default:
 1152       ShouldNotReachHere();
 1153     }
 1154 #endif
 1155   }
 1156 }
 1157 
 1158 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1159                                        PhaseRegAlloc* ra_,
 1160                                        bool do_size,
 1161                                        outputStream* st) const {
 1162   assert(cbuf != NULL || st  != NULL, "sanity");
 1163   // Get registers to move
 1164   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1165   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1166   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1167   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1168 
 1169   enum RC src_second_rc = rc_class(src_second);
 1170   enum RC src_first_rc = rc_class(src_first);
 1171   enum RC dst_second_rc = rc_class(dst_second);
 1172   enum RC dst_first_rc = rc_class(dst_first);
 1173 
 1174   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1175          "must move at least 1 register" );
 1176 
 1177   if (src_first == dst_first && src_second == dst_second) {
 1178     // Self copy, no move
 1179     return 0;
 1180   }
 1181   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1182     uint ireg = ideal_reg();
 1183     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1184     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1185     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1186       // mem -> mem
 1187       int src_offset = ra_->reg2offset(src_first);
 1188       int dst_offset = ra_->reg2offset(dst_first);
 1189       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1190     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1191       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1192     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1193       int stack_offset = ra_->reg2offset(dst_first);
 1194       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1195     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1196       int stack_offset = ra_->reg2offset(src_first);
 1197       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1198     } else {
 1199       ShouldNotReachHere();
 1200     }
 1201     return 0;
 1202   }
 1203   if (src_first_rc == rc_stack) {
 1204     // mem ->
 1205     if (dst_first_rc == rc_stack) {
 1206       // mem -> mem
 1207       assert(src_second != dst_first, "overlap");
 1208       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1209           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1210         // 64-bit
 1211         int src_offset = ra_->reg2offset(src_first);
 1212         int dst_offset = ra_->reg2offset(dst_first);
 1213         if (cbuf) {
 1214           MacroAssembler _masm(cbuf);
 1215           __ pushq(Address(rsp, src_offset));
 1216           __ popq (Address(rsp, dst_offset));
 1217 #ifndef PRODUCT
 1218         } else {
 1219           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1220                     "popq    [rsp + #%d]",
 1221                      src_offset, dst_offset);
 1222 #endif
 1223         }
 1224       } else {
 1225         // 32-bit
 1226         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1227         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1228         // No pushl/popl, so:
 1229         int src_offset = ra_->reg2offset(src_first);
 1230         int dst_offset = ra_->reg2offset(dst_first);
 1231         if (cbuf) {
 1232           MacroAssembler _masm(cbuf);
 1233           __ movq(Address(rsp, -8), rax);
 1234           __ movl(rax, Address(rsp, src_offset));
 1235           __ movl(Address(rsp, dst_offset), rax);
 1236           __ movq(rax, Address(rsp, -8));
 1237 #ifndef PRODUCT
 1238         } else {
 1239           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1240                     "movl    rax, [rsp + #%d]\n\t"
 1241                     "movl    [rsp + #%d], rax\n\t"
 1242                     "movq    rax, [rsp - #8]",
 1243                      src_offset, dst_offset);
 1244 #endif
 1245         }
 1246       }
 1247       return 0;
 1248     } else if (dst_first_rc == rc_int) {
 1249       // mem -> gpr
 1250       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1251           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1252         // 64-bit
 1253         int offset = ra_->reg2offset(src_first);
 1254         if (cbuf) {
 1255           MacroAssembler _masm(cbuf);
 1256           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1257 #ifndef PRODUCT
 1258         } else {
 1259           st->print("movq    %s, [rsp + #%d]\t# spill",
 1260                      Matcher::regName[dst_first],
 1261                      offset);
 1262 #endif
 1263         }
 1264       } else {
 1265         // 32-bit
 1266         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1267         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1268         int offset = ra_->reg2offset(src_first);
 1269         if (cbuf) {
 1270           MacroAssembler _masm(cbuf);
 1271           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1272 #ifndef PRODUCT
 1273         } else {
 1274           st->print("movl    %s, [rsp + #%d]\t# spill",
 1275                      Matcher::regName[dst_first],
 1276                      offset);
 1277 #endif
 1278         }
 1279       }
 1280       return 0;
 1281     } else if (dst_first_rc == rc_float) {
 1282       // mem-> xmm
 1283       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1284           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1285         // 64-bit
 1286         int offset = ra_->reg2offset(src_first);
 1287         if (cbuf) {
 1288           MacroAssembler _masm(cbuf);
 1289           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1290 #ifndef PRODUCT
 1291         } else {
 1292           st->print("%s  %s, [rsp + #%d]\t# spill",
 1293                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1294                      Matcher::regName[dst_first],
 1295                      offset);
 1296 #endif
 1297         }
 1298       } else {
 1299         // 32-bit
 1300         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1301         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1302         int offset = ra_->reg2offset(src_first);
 1303         if (cbuf) {
 1304           MacroAssembler _masm(cbuf);
 1305           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1306 #ifndef PRODUCT
 1307         } else {
 1308           st->print("movss   %s, [rsp + #%d]\t# spill",
 1309                      Matcher::regName[dst_first],
 1310                      offset);
 1311 #endif
 1312         }
 1313       }
 1314       return 0;
 1315     } else if (dst_first_rc == rc_kreg) {
 1316       // mem -> kreg
 1317       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1318           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1319         // 64-bit
 1320         int offset = ra_->reg2offset(src_first);
 1321         if (cbuf) {
 1322           MacroAssembler _masm(cbuf);
 1323           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1324 #ifndef PRODUCT
 1325         } else {
 1326           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1327                      Matcher::regName[dst_first],
 1328                      offset);
 1329 #endif
 1330         }
 1331       }
 1332       return 0;
 1333     }
 1334   } else if (src_first_rc == rc_int) {
 1335     // gpr ->
 1336     if (dst_first_rc == rc_stack) {
 1337       // gpr -> mem
 1338       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1339           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1340         // 64-bit
 1341         int offset = ra_->reg2offset(dst_first);
 1342         if (cbuf) {
 1343           MacroAssembler _masm(cbuf);
 1344           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1345 #ifndef PRODUCT
 1346         } else {
 1347           st->print("movq    [rsp + #%d], %s\t# spill",
 1348                      offset,
 1349                      Matcher::regName[src_first]);
 1350 #endif
 1351         }
 1352       } else {
 1353         // 32-bit
 1354         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1355         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1356         int offset = ra_->reg2offset(dst_first);
 1357         if (cbuf) {
 1358           MacroAssembler _masm(cbuf);
 1359           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1360 #ifndef PRODUCT
 1361         } else {
 1362           st->print("movl    [rsp + #%d], %s\t# spill",
 1363                      offset,
 1364                      Matcher::regName[src_first]);
 1365 #endif
 1366         }
 1367       }
 1368       return 0;
 1369     } else if (dst_first_rc == rc_int) {
 1370       // gpr -> gpr
 1371       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1372           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1373         // 64-bit
 1374         if (cbuf) {
 1375           MacroAssembler _masm(cbuf);
 1376           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1377                   as_Register(Matcher::_regEncode[src_first]));
 1378 #ifndef PRODUCT
 1379         } else {
 1380           st->print("movq    %s, %s\t# spill",
 1381                      Matcher::regName[dst_first],
 1382                      Matcher::regName[src_first]);
 1383 #endif
 1384         }
 1385         return 0;
 1386       } else {
 1387         // 32-bit
 1388         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1389         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1390         if (cbuf) {
 1391           MacroAssembler _masm(cbuf);
 1392           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1393                   as_Register(Matcher::_regEncode[src_first]));
 1394 #ifndef PRODUCT
 1395         } else {
 1396           st->print("movl    %s, %s\t# spill",
 1397                      Matcher::regName[dst_first],
 1398                      Matcher::regName[src_first]);
 1399 #endif
 1400         }
 1401         return 0;
 1402       }
 1403     } else if (dst_first_rc == rc_float) {
 1404       // gpr -> xmm
 1405       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1406           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1407         // 64-bit
 1408         if (cbuf) {
 1409           MacroAssembler _masm(cbuf);
 1410           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1411 #ifndef PRODUCT
 1412         } else {
 1413           st->print("movdq   %s, %s\t# spill",
 1414                      Matcher::regName[dst_first],
 1415                      Matcher::regName[src_first]);
 1416 #endif
 1417         }
 1418       } else {
 1419         // 32-bit
 1420         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1421         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1422         if (cbuf) {
 1423           MacroAssembler _masm(cbuf);
 1424           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1425 #ifndef PRODUCT
 1426         } else {
 1427           st->print("movdl   %s, %s\t# spill",
 1428                      Matcher::regName[dst_first],
 1429                      Matcher::regName[src_first]);
 1430 #endif
 1431         }
 1432       }
 1433       return 0;
 1434     } else if (dst_first_rc == rc_kreg) {
 1435       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1436           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1437         // 64-bit
 1438         if (cbuf) {
 1439           MacroAssembler _masm(cbuf);
 1440           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1441   #ifndef PRODUCT
 1442         } else {
 1443            st->print("kmovq   %s, %s\t# spill",
 1444                        Matcher::regName[dst_first],
 1445                        Matcher::regName[src_first]);
 1446   #endif
 1447         }
 1448       }
 1449       Unimplemented();
 1450       return 0;
 1451     }
 1452   } else if (src_first_rc == rc_float) {
 1453     // xmm ->
 1454     if (dst_first_rc == rc_stack) {
 1455       // xmm -> mem
 1456       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1457           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1458         // 64-bit
 1459         int offset = ra_->reg2offset(dst_first);
 1460         if (cbuf) {
 1461           MacroAssembler _masm(cbuf);
 1462           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1463 #ifndef PRODUCT
 1464         } else {
 1465           st->print("movsd   [rsp + #%d], %s\t# spill",
 1466                      offset,
 1467                      Matcher::regName[src_first]);
 1468 #endif
 1469         }
 1470       } else {
 1471         // 32-bit
 1472         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1473         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1474         int offset = ra_->reg2offset(dst_first);
 1475         if (cbuf) {
 1476           MacroAssembler _masm(cbuf);
 1477           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1478 #ifndef PRODUCT
 1479         } else {
 1480           st->print("movss   [rsp + #%d], %s\t# spill",
 1481                      offset,
 1482                      Matcher::regName[src_first]);
 1483 #endif
 1484         }
 1485       }
 1486       return 0;
 1487     } else if (dst_first_rc == rc_int) {
 1488       // xmm -> gpr
 1489       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1490           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1491         // 64-bit
 1492         if (cbuf) {
 1493           MacroAssembler _masm(cbuf);
 1494           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1495 #ifndef PRODUCT
 1496         } else {
 1497           st->print("movdq   %s, %s\t# spill",
 1498                      Matcher::regName[dst_first],
 1499                      Matcher::regName[src_first]);
 1500 #endif
 1501         }
 1502       } else {
 1503         // 32-bit
 1504         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1505         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1506         if (cbuf) {
 1507           MacroAssembler _masm(cbuf);
 1508           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1509 #ifndef PRODUCT
 1510         } else {
 1511           st->print("movdl   %s, %s\t# spill",
 1512                      Matcher::regName[dst_first],
 1513                      Matcher::regName[src_first]);
 1514 #endif
 1515         }
 1516       }
 1517       return 0;
 1518     } else if (dst_first_rc == rc_float) {
 1519       // xmm -> xmm
 1520       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1521           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1522         // 64-bit
 1523         if (cbuf) {
 1524           MacroAssembler _masm(cbuf);
 1525           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1526 #ifndef PRODUCT
 1527         } else {
 1528           st->print("%s  %s, %s\t# spill",
 1529                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1530                      Matcher::regName[dst_first],
 1531                      Matcher::regName[src_first]);
 1532 #endif
 1533         }
 1534       } else {
 1535         // 32-bit
 1536         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1537         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1538         if (cbuf) {
 1539           MacroAssembler _masm(cbuf);
 1540           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1541 #ifndef PRODUCT
 1542         } else {
 1543           st->print("%s  %s, %s\t# spill",
 1544                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1545                      Matcher::regName[dst_first],
 1546                      Matcher::regName[src_first]);
 1547 #endif
 1548         }
 1549       }
 1550       return 0;
 1551     } else if (dst_first_rc == rc_kreg) {
 1552       assert(false, "Illegal spilling");
 1553       return 0;
 1554     }
 1555   } else if (src_first_rc == rc_kreg) {
 1556     if (dst_first_rc == rc_stack) {
 1557       // mem -> kreg
 1558       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1559           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1560         // 64-bit
 1561         int offset = ra_->reg2offset(dst_first);
 1562         if (cbuf) {
 1563           MacroAssembler _masm(cbuf);
 1564           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1565 #ifndef PRODUCT
 1566         } else {
 1567           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1568                      offset,
 1569                      Matcher::regName[src_first]);
 1570 #endif
 1571         }
 1572       }
 1573       return 0;
 1574     } else if (dst_first_rc == rc_int) {
 1575       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1576           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1577         // 64-bit
 1578         if (cbuf) {
 1579           MacroAssembler _masm(cbuf);
 1580           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1581 #ifndef PRODUCT
 1582         } else {
 1583          st->print("kmovq   %s, %s\t# spill",
 1584                      Matcher::regName[dst_first],
 1585                      Matcher::regName[src_first]);
 1586 #endif
 1587         }
 1588       }
 1589       Unimplemented();
 1590       return 0;
 1591     } else if (dst_first_rc == rc_kreg) {
 1592       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1593           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1594         // 64-bit
 1595         if (cbuf) {
 1596           MacroAssembler _masm(cbuf);
 1597           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1598 #ifndef PRODUCT
 1599         } else {
 1600          st->print("kmovq   %s, %s\t# spill",
 1601                      Matcher::regName[dst_first],
 1602                      Matcher::regName[src_first]);
 1603 #endif
 1604         }
 1605       }
 1606       return 0;
 1607     } else if (dst_first_rc == rc_float) {
 1608       assert(false, "Illegal spill");
 1609       return 0;
 1610     }
 1611   }
 1612 
 1613   assert(0," foo ");
 1614   Unimplemented();
 1615   return 0;
 1616 }
 1617 
 1618 #ifndef PRODUCT
 1619 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1620   implementation(NULL, ra_, false, st);
 1621 }
 1622 #endif
 1623 
 1624 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1625   implementation(&cbuf, ra_, false, NULL);
 1626 }
 1627 
 1628 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1629   return MachNode::size(ra_);
 1630 }
 1631 
 1632 //=============================================================================
 1633 #ifndef PRODUCT
 1634 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1635 {
 1636   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1637   int reg = ra_->get_reg_first(this);
 1638   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1639             Matcher::regName[reg], offset);
 1640 }
 1641 #endif
 1642 
 1643 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1644 {
 1645   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1646   int reg = ra_->get_encode(this);
 1647   if (offset >= 0x80) {
 1648     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1649     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1650     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1651     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1652     emit_d32(cbuf, offset);
 1653   } else {
 1654     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1655     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1656     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1657     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1658     emit_d8(cbuf, offset);
 1659   }
 1660 }
 1661 
 1662 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1663 {
 1664   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1665   return (offset < 0x80) ? 5 : 8; // REX
 1666 }
 1667 
 1668 //=============================================================================
 1669 #ifndef PRODUCT
 1670 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1671 {
 1672   if (UseCompressedClassPointers) {
 1673     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1674     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1675     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1676   } else {
 1677     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1678                  "# Inline cache check");
 1679   }
 1680   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1681   st->print_cr("\tnop\t# nops to align entry point");
 1682 }
 1683 #endif
 1684 
 1685 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1686 {
 1687   MacroAssembler masm(&cbuf);
 1688   uint insts_size = cbuf.insts_size();
 1689   if (UseCompressedClassPointers) {
 1690     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1691     masm.cmpptr(rax, rscratch1);
 1692   } else {
 1693     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1694   }
 1695 
 1696   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1697 
 1698   /* WARNING these NOPs are critical so that verified entry point is properly
 1699      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1700   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1701   if (OptoBreakpoint) {
 1702     // Leave space for int3
 1703     nops_cnt -= 1;
 1704   }
 1705   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1706   if (nops_cnt > 0)
 1707     masm.nop(nops_cnt);
 1708 }
 1709 
 1710 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1711 {
 1712   return MachNode::size(ra_); // too many variables; just compute it
 1713                               // the hard way
 1714 }
 1715 
 1716 
 1717 //=============================================================================
 1718 
 1719 const bool Matcher::supports_vector_calling_convention(void) {
 1720   if (EnableVectorSupport && UseVectorStubs) {
 1721     return true;
 1722   }
 1723   return false;
 1724 }
 1725 
 1726 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1727   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1728   int lo = XMM0_num;
 1729   int hi = XMM0b_num;
 1730   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1731   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1732   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1733   return OptoRegPair(hi, lo);
 1734 }
 1735 
 1736 // Is this branch offset short enough that a short branch can be used?
 1737 //
 1738 // NOTE: If the platform does not provide any short branch variants, then
 1739 //       this method should return false for offset 0.
 1740 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1741   // The passed offset is relative to address of the branch.
 1742   // On 86 a branch displacement is calculated relative to address
 1743   // of a next instruction.
 1744   offset -= br_size;
 1745 
 1746   // the short version of jmpConUCF2 contains multiple branches,
 1747   // making the reach slightly less
 1748   if (rule == jmpConUCF2_rule)
 1749     return (-126 <= offset && offset <= 125);
 1750   return (-128 <= offset && offset <= 127);
 1751 }
 1752 
 1753 // Return whether or not this register is ever used as an argument.
 1754 // This function is used on startup to build the trampoline stubs in
 1755 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1756 // call in the trampoline, and arguments in those registers not be
 1757 // available to the callee.
 1758 bool Matcher::can_be_java_arg(int reg)
 1759 {
 1760   return
 1761     reg ==  RDI_num || reg == RDI_H_num ||
 1762     reg ==  RSI_num || reg == RSI_H_num ||
 1763     reg ==  RDX_num || reg == RDX_H_num ||
 1764     reg ==  RCX_num || reg == RCX_H_num ||
 1765     reg ==   R8_num || reg ==  R8_H_num ||
 1766     reg ==   R9_num || reg ==  R9_H_num ||
 1767     reg ==  R12_num || reg == R12_H_num ||
 1768     reg == XMM0_num || reg == XMM0b_num ||
 1769     reg == XMM1_num || reg == XMM1b_num ||
 1770     reg == XMM2_num || reg == XMM2b_num ||
 1771     reg == XMM3_num || reg == XMM3b_num ||
 1772     reg == XMM4_num || reg == XMM4b_num ||
 1773     reg == XMM5_num || reg == XMM5b_num ||
 1774     reg == XMM6_num || reg == XMM6b_num ||
 1775     reg == XMM7_num || reg == XMM7b_num;
 1776 }
 1777 
 1778 bool Matcher::is_spillable_arg(int reg)
 1779 {
 1780   return can_be_java_arg(reg);
 1781 }
 1782 
 1783 uint Matcher::int_pressure_limit()
 1784 {
 1785   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1786 }
 1787 
 1788 uint Matcher::float_pressure_limit()
 1789 {
 1790   // After experiment around with different values, the following default threshold
 1791   // works best for LCM's register pressure scheduling on x64.
 1792   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1793   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1794   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1795 }
 1796 
 1797 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1798   // In 64 bit mode a code which use multiply when
 1799   // devisor is constant is faster than hardware
 1800   // DIV instruction (it uses MulHiL).
 1801   return false;
 1802 }
 1803 
 1804 // Register for DIVI projection of divmodI
 1805 RegMask Matcher::divI_proj_mask() {
 1806   return INT_RAX_REG_mask();
 1807 }
 1808 
 1809 // Register for MODI projection of divmodI
 1810 RegMask Matcher::modI_proj_mask() {
 1811   return INT_RDX_REG_mask();
 1812 }
 1813 
 1814 // Register for DIVL projection of divmodL
 1815 RegMask Matcher::divL_proj_mask() {
 1816   return LONG_RAX_REG_mask();
 1817 }
 1818 
 1819 // Register for MODL projection of divmodL
 1820 RegMask Matcher::modL_proj_mask() {
 1821   return LONG_RDX_REG_mask();
 1822 }
 1823 
 1824 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1825 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1826     return NO_REG_mask();
 1827 }
 1828 
 1829 %}
 1830 
 1831 //----------ENCODING BLOCK-----------------------------------------------------
 1832 // This block specifies the encoding classes used by the compiler to
 1833 // output byte streams.  Encoding classes are parameterized macros
 1834 // used by Machine Instruction Nodes in order to generate the bit
 1835 // encoding of the instruction.  Operands specify their base encoding
 1836 // interface with the interface keyword.  There are currently
 1837 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1838 // COND_INTER.  REG_INTER causes an operand to generate a function
 1839 // which returns its register number when queried.  CONST_INTER causes
 1840 // an operand to generate a function which returns the value of the
 1841 // constant when queried.  MEMORY_INTER causes an operand to generate
 1842 // four functions which return the Base Register, the Index Register,
 1843 // the Scale Value, and the Offset Value of the operand when queried.
 1844 // COND_INTER causes an operand to generate six functions which return
 1845 // the encoding code (ie - encoding bits for the instruction)
 1846 // associated with each basic boolean condition for a conditional
 1847 // instruction.
 1848 //
 1849 // Instructions specify two basic values for encoding.  Again, a
 1850 // function is available to check if the constant displacement is an
 1851 // oop. They use the ins_encode keyword to specify their encoding
 1852 // classes (which must be a sequence of enc_class names, and their
 1853 // parameters, specified in the encoding block), and they use the
 1854 // opcode keyword to specify, in order, their primary, secondary, and
 1855 // tertiary opcode.  Only the opcode sections which a particular
 1856 // instruction needs for encoding need to be specified.
 1857 encode %{
 1858   // Build emit functions for each basic byte or larger field in the
 1859   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1860   // from C++ code in the enc_class source block.  Emit functions will
 1861   // live in the main source block for now.  In future, we can
 1862   // generalize this by adding a syntax that specifies the sizes of
 1863   // fields in an order, so that the adlc can build the emit functions
 1864   // automagically
 1865 
 1866   // Emit primary opcode
 1867   enc_class OpcP
 1868   %{
 1869     emit_opcode(cbuf, $primary);
 1870   %}
 1871 
 1872   // Emit secondary opcode
 1873   enc_class OpcS
 1874   %{
 1875     emit_opcode(cbuf, $secondary);
 1876   %}
 1877 
 1878   // Emit tertiary opcode
 1879   enc_class OpcT
 1880   %{
 1881     emit_opcode(cbuf, $tertiary);
 1882   %}
 1883 
 1884   // Emit opcode directly
 1885   enc_class Opcode(immI d8)
 1886   %{
 1887     emit_opcode(cbuf, $d8$$constant);
 1888   %}
 1889 
 1890   // Emit size prefix
 1891   enc_class SizePrefix
 1892   %{
 1893     emit_opcode(cbuf, 0x66);
 1894   %}
 1895 
 1896   enc_class reg(rRegI reg)
 1897   %{
 1898     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1899   %}
 1900 
 1901   enc_class reg_reg(rRegI dst, rRegI src)
 1902   %{
 1903     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1904   %}
 1905 
 1906   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1907   %{
 1908     emit_opcode(cbuf, $opcode$$constant);
 1909     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1910   %}
 1911 
 1912   enc_class cdql_enc(no_rax_rdx_RegI div)
 1913   %{
 1914     // Full implementation of Java idiv and irem; checks for
 1915     // special case as described in JVM spec., p.243 & p.271.
 1916     //
 1917     //         normal case                           special case
 1918     //
 1919     // input : rax: dividend                         min_int
 1920     //         reg: divisor                          -1
 1921     //
 1922     // output: rax: quotient  (= rax idiv reg)       min_int
 1923     //         rdx: remainder (= rax irem reg)       0
 1924     //
 1925     //  Code sequnce:
 1926     //
 1927     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1928     //    5:   75 07/08                jne    e <normal>
 1929     //    7:   33 d2                   xor    %edx,%edx
 1930     //  [div >= 8 -> offset + 1]
 1931     //  [REX_B]
 1932     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1933     //    c:   74 03/04                je     11 <done>
 1934     // 000000000000000e <normal>:
 1935     //    e:   99                      cltd
 1936     //  [div >= 8 -> offset + 1]
 1937     //  [REX_B]
 1938     //    f:   f7 f9                   idiv   $div
 1939     // 0000000000000011 <done>:
 1940     MacroAssembler _masm(&cbuf);
 1941     Label normal;
 1942     Label done;
 1943 
 1944     // cmp    $0x80000000,%eax
 1945     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1946 
 1947     // jne    e <normal>
 1948     __ jccb(Assembler::notEqual, normal);
 1949 
 1950     // xor    %edx,%edx
 1951     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1952 
 1953     // cmp    $0xffffffffffffffff,%ecx
 1954     __ cmpl($div$$Register, -1);
 1955 
 1956     // je     11 <done>
 1957     __ jccb(Assembler::equal, done);
 1958 
 1959     // <normal>
 1960     // cltd
 1961     __ bind(normal);
 1962     __ cdql();
 1963 
 1964     // idivl
 1965     // <done>
 1966     __ idivl($div$$Register);
 1967     __ bind(done);
 1968   %}
 1969 
 1970   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1971   %{
 1972     // Full implementation of Java ldiv and lrem; checks for
 1973     // special case as described in JVM spec., p.243 & p.271.
 1974     //
 1975     //         normal case                           special case
 1976     //
 1977     // input : rax: dividend                         min_long
 1978     //         reg: divisor                          -1
 1979     //
 1980     // output: rax: quotient  (= rax idiv reg)       min_long
 1981     //         rdx: remainder (= rax irem reg)       0
 1982     //
 1983     //  Code sequnce:
 1984     //
 1985     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1986     //    7:   00 00 80
 1987     //    a:   48 39 d0                cmp    %rdx,%rax
 1988     //    d:   75 08                   jne    17 <normal>
 1989     //    f:   33 d2                   xor    %edx,%edx
 1990     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1991     //   15:   74 05                   je     1c <done>
 1992     // 0000000000000017 <normal>:
 1993     //   17:   48 99                   cqto
 1994     //   19:   48 f7 f9                idiv   $div
 1995     // 000000000000001c <done>:
 1996     MacroAssembler _masm(&cbuf);
 1997     Label normal;
 1998     Label done;
 1999 
 2000     // mov    $0x8000000000000000,%rdx
 2001     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 2002 
 2003     // cmp    %rdx,%rax
 2004     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 2005 
 2006     // jne    17 <normal>
 2007     __ jccb(Assembler::notEqual, normal);
 2008 
 2009     // xor    %edx,%edx
 2010     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 2011 
 2012     // cmp    $0xffffffffffffffff,$div
 2013     __ cmpq($div$$Register, -1);
 2014 
 2015     // je     1e <done>
 2016     __ jccb(Assembler::equal, done);
 2017 
 2018     // <normal>
 2019     // cqto
 2020     __ bind(normal);
 2021     __ cdqq();
 2022 
 2023     // idivq (note: must be emitted by the user of this rule)
 2024     // <done>
 2025     __ idivq($div$$Register);
 2026     __ bind(done);
 2027   %}
 2028 
 2029   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2030   enc_class OpcSE(immI imm)
 2031   %{
 2032     // Emit primary opcode and set sign-extend bit
 2033     // Check for 8-bit immediate, and set sign extend bit in opcode
 2034     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2035       emit_opcode(cbuf, $primary | 0x02);
 2036     } else {
 2037       // 32-bit immediate
 2038       emit_opcode(cbuf, $primary);
 2039     }
 2040   %}
 2041 
 2042   enc_class OpcSErm(rRegI dst, immI imm)
 2043   %{
 2044     // OpcSEr/m
 2045     int dstenc = $dst$$reg;
 2046     if (dstenc >= 8) {
 2047       emit_opcode(cbuf, Assembler::REX_B);
 2048       dstenc -= 8;
 2049     }
 2050     // Emit primary opcode and set sign-extend bit
 2051     // Check for 8-bit immediate, and set sign extend bit in opcode
 2052     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2053       emit_opcode(cbuf, $primary | 0x02);
 2054     } else {
 2055       // 32-bit immediate
 2056       emit_opcode(cbuf, $primary);
 2057     }
 2058     // Emit r/m byte with secondary opcode, after primary opcode.
 2059     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2060   %}
 2061 
 2062   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2063   %{
 2064     // OpcSEr/m
 2065     int dstenc = $dst$$reg;
 2066     if (dstenc < 8) {
 2067       emit_opcode(cbuf, Assembler::REX_W);
 2068     } else {
 2069       emit_opcode(cbuf, Assembler::REX_WB);
 2070       dstenc -= 8;
 2071     }
 2072     // Emit primary opcode and set sign-extend bit
 2073     // Check for 8-bit immediate, and set sign extend bit in opcode
 2074     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2075       emit_opcode(cbuf, $primary | 0x02);
 2076     } else {
 2077       // 32-bit immediate
 2078       emit_opcode(cbuf, $primary);
 2079     }
 2080     // Emit r/m byte with secondary opcode, after primary opcode.
 2081     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2082   %}
 2083 
 2084   enc_class Con8or32(immI imm)
 2085   %{
 2086     // Check for 8-bit immediate, and set sign extend bit in opcode
 2087     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2088       $$$emit8$imm$$constant;
 2089     } else {
 2090       // 32-bit immediate
 2091       $$$emit32$imm$$constant;
 2092     }
 2093   %}
 2094 
 2095   enc_class opc2_reg(rRegI dst)
 2096   %{
 2097     // BSWAP
 2098     emit_cc(cbuf, $secondary, $dst$$reg);
 2099   %}
 2100 
 2101   enc_class opc3_reg(rRegI dst)
 2102   %{
 2103     // BSWAP
 2104     emit_cc(cbuf, $tertiary, $dst$$reg);
 2105   %}
 2106 
 2107   enc_class reg_opc(rRegI div)
 2108   %{
 2109     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2110     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2111   %}
 2112 
 2113   enc_class enc_cmov(cmpOp cop)
 2114   %{
 2115     // CMOV
 2116     $$$emit8$primary;
 2117     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2118   %}
 2119 
 2120   enc_class enc_PartialSubtypeCheck()
 2121   %{
 2122     Register Rrdi = as_Register(RDI_enc); // result register
 2123     Register Rrax = as_Register(RAX_enc); // super class
 2124     Register Rrcx = as_Register(RCX_enc); // killed
 2125     Register Rrsi = as_Register(RSI_enc); // sub class
 2126     Label miss;
 2127     const bool set_cond_codes = true;
 2128 
 2129     MacroAssembler _masm(&cbuf);
 2130     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2131                                      NULL, &miss,
 2132                                      /*set_cond_codes:*/ true);
 2133     if ($primary) {
 2134       __ xorptr(Rrdi, Rrdi);
 2135     }
 2136     __ bind(miss);
 2137   %}
 2138 
 2139   enc_class clear_avx %{
 2140     debug_only(int off0 = cbuf.insts_size());
 2141     if (generate_vzeroupper(Compile::current())) {
 2142       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2143       // Clear upper bits of YMM registers when current compiled code uses
 2144       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2145       MacroAssembler _masm(&cbuf);
 2146       __ vzeroupper();
 2147     }
 2148     debug_only(int off1 = cbuf.insts_size());
 2149     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2150   %}
 2151 
 2152   enc_class Java_To_Runtime(method meth) %{
 2153     // No relocation needed
 2154     MacroAssembler _masm(&cbuf);
 2155     __ mov64(r10, (int64_t) $meth$$method);
 2156     __ call(r10);
 2157     __ post_call_nop();
 2158   %}
 2159 
 2160   enc_class Java_Static_Call(method meth)
 2161   %{
 2162     // JAVA STATIC CALL
 2163     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2164     // determine who we intended to call.
 2165     MacroAssembler _masm(&cbuf);
 2166     cbuf.set_insts_mark();
 2167     $$$emit8$primary;
 2168 
 2169     if (!_method) {
 2170       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2171                      runtime_call_Relocation::spec(),
 2172                      RELOC_DISP32);
 2173     } else {
 2174       int method_index = resolved_method_index(cbuf);
 2175       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2176                                                   : static_call_Relocation::spec(method_index);
 2177       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2178                      rspec, RELOC_DISP32);
 2179       address mark = cbuf.insts_mark();
 2180       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 2181         // Calls of the same statically bound method can share
 2182         // a stub to the interpreter.
 2183         cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
 2184       } else {
 2185         // Emit stubs for static call.
 2186         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2187         if (stub == NULL) {
 2188           ciEnv::current()->record_failure("CodeCache is full");
 2189           return;
 2190         }
 2191       }
 2192     }
 2193     _masm.clear_inst_mark();
 2194     __ post_call_nop();
 2195   %}
 2196 
 2197   enc_class Java_Dynamic_Call(method meth) %{
 2198     MacroAssembler _masm(&cbuf);
 2199     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2200     __ post_call_nop();
 2201   %}
 2202 
 2203   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2204   %{
 2205     // SAL, SAR, SHR
 2206     int dstenc = $dst$$reg;
 2207     if (dstenc >= 8) {
 2208       emit_opcode(cbuf, Assembler::REX_B);
 2209       dstenc -= 8;
 2210     }
 2211     $$$emit8$primary;
 2212     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2213     $$$emit8$shift$$constant;
 2214   %}
 2215 
 2216   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2217   %{
 2218     // SAL, SAR, SHR
 2219     int dstenc = $dst$$reg;
 2220     if (dstenc < 8) {
 2221       emit_opcode(cbuf, Assembler::REX_W);
 2222     } else {
 2223       emit_opcode(cbuf, Assembler::REX_WB);
 2224       dstenc -= 8;
 2225     }
 2226     $$$emit8$primary;
 2227     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2228     $$$emit8$shift$$constant;
 2229   %}
 2230 
 2231   enc_class load_immI(rRegI dst, immI src)
 2232   %{
 2233     int dstenc = $dst$$reg;
 2234     if (dstenc >= 8) {
 2235       emit_opcode(cbuf, Assembler::REX_B);
 2236       dstenc -= 8;
 2237     }
 2238     emit_opcode(cbuf, 0xB8 | dstenc);
 2239     $$$emit32$src$$constant;
 2240   %}
 2241 
 2242   enc_class load_immL(rRegL dst, immL src)
 2243   %{
 2244     int dstenc = $dst$$reg;
 2245     if (dstenc < 8) {
 2246       emit_opcode(cbuf, Assembler::REX_W);
 2247     } else {
 2248       emit_opcode(cbuf, Assembler::REX_WB);
 2249       dstenc -= 8;
 2250     }
 2251     emit_opcode(cbuf, 0xB8 | dstenc);
 2252     emit_d64(cbuf, $src$$constant);
 2253   %}
 2254 
 2255   enc_class load_immUL32(rRegL dst, immUL32 src)
 2256   %{
 2257     // same as load_immI, but this time we care about zeroes in the high word
 2258     int dstenc = $dst$$reg;
 2259     if (dstenc >= 8) {
 2260       emit_opcode(cbuf, Assembler::REX_B);
 2261       dstenc -= 8;
 2262     }
 2263     emit_opcode(cbuf, 0xB8 | dstenc);
 2264     $$$emit32$src$$constant;
 2265   %}
 2266 
 2267   enc_class load_immL32(rRegL dst, immL32 src)
 2268   %{
 2269     int dstenc = $dst$$reg;
 2270     if (dstenc < 8) {
 2271       emit_opcode(cbuf, Assembler::REX_W);
 2272     } else {
 2273       emit_opcode(cbuf, Assembler::REX_WB);
 2274       dstenc -= 8;
 2275     }
 2276     emit_opcode(cbuf, 0xC7);
 2277     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2278     $$$emit32$src$$constant;
 2279   %}
 2280 
 2281   enc_class load_immP31(rRegP dst, immP32 src)
 2282   %{
 2283     // same as load_immI, but this time we care about zeroes in the high word
 2284     int dstenc = $dst$$reg;
 2285     if (dstenc >= 8) {
 2286       emit_opcode(cbuf, Assembler::REX_B);
 2287       dstenc -= 8;
 2288     }
 2289     emit_opcode(cbuf, 0xB8 | dstenc);
 2290     $$$emit32$src$$constant;
 2291   %}
 2292 
 2293   enc_class load_immP(rRegP dst, immP src)
 2294   %{
 2295     int dstenc = $dst$$reg;
 2296     if (dstenc < 8) {
 2297       emit_opcode(cbuf, Assembler::REX_W);
 2298     } else {
 2299       emit_opcode(cbuf, Assembler::REX_WB);
 2300       dstenc -= 8;
 2301     }
 2302     emit_opcode(cbuf, 0xB8 | dstenc);
 2303     // This next line should be generated from ADLC
 2304     if ($src->constant_reloc() != relocInfo::none) {
 2305       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2306     } else {
 2307       emit_d64(cbuf, $src$$constant);
 2308     }
 2309   %}
 2310 
 2311   enc_class Con32(immI src)
 2312   %{
 2313     // Output immediate
 2314     $$$emit32$src$$constant;
 2315   %}
 2316 
 2317   enc_class Con32F_as_bits(immF src)
 2318   %{
 2319     // Output Float immediate bits
 2320     jfloat jf = $src$$constant;
 2321     jint jf_as_bits = jint_cast(jf);
 2322     emit_d32(cbuf, jf_as_bits);
 2323   %}
 2324 
 2325   enc_class Con16(immI src)
 2326   %{
 2327     // Output immediate
 2328     $$$emit16$src$$constant;
 2329   %}
 2330 
 2331   // How is this different from Con32??? XXX
 2332   enc_class Con_d32(immI src)
 2333   %{
 2334     emit_d32(cbuf,$src$$constant);
 2335   %}
 2336 
 2337   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2338     // Output immediate memory reference
 2339     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2340     emit_d32(cbuf, 0x00);
 2341   %}
 2342 
 2343   enc_class lock_prefix()
 2344   %{
 2345     emit_opcode(cbuf, 0xF0); // lock
 2346   %}
 2347 
 2348   enc_class REX_mem(memory mem)
 2349   %{
 2350     if ($mem$$base >= 8) {
 2351       if ($mem$$index < 8) {
 2352         emit_opcode(cbuf, Assembler::REX_B);
 2353       } else {
 2354         emit_opcode(cbuf, Assembler::REX_XB);
 2355       }
 2356     } else {
 2357       if ($mem$$index >= 8) {
 2358         emit_opcode(cbuf, Assembler::REX_X);
 2359       }
 2360     }
 2361   %}
 2362 
 2363   enc_class REX_mem_wide(memory mem)
 2364   %{
 2365     if ($mem$$base >= 8) {
 2366       if ($mem$$index < 8) {
 2367         emit_opcode(cbuf, Assembler::REX_WB);
 2368       } else {
 2369         emit_opcode(cbuf, Assembler::REX_WXB);
 2370       }
 2371     } else {
 2372       if ($mem$$index < 8) {
 2373         emit_opcode(cbuf, Assembler::REX_W);
 2374       } else {
 2375         emit_opcode(cbuf, Assembler::REX_WX);
 2376       }
 2377     }
 2378   %}
 2379 
 2380   // for byte regs
 2381   enc_class REX_breg(rRegI reg)
 2382   %{
 2383     if ($reg$$reg >= 4) {
 2384       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2385     }
 2386   %}
 2387 
 2388   // for byte regs
 2389   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2390   %{
 2391     if ($dst$$reg < 8) {
 2392       if ($src$$reg >= 4) {
 2393         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2394       }
 2395     } else {
 2396       if ($src$$reg < 8) {
 2397         emit_opcode(cbuf, Assembler::REX_R);
 2398       } else {
 2399         emit_opcode(cbuf, Assembler::REX_RB);
 2400       }
 2401     }
 2402   %}
 2403 
 2404   // for byte regs
 2405   enc_class REX_breg_mem(rRegI reg, memory mem)
 2406   %{
 2407     if ($reg$$reg < 8) {
 2408       if ($mem$$base < 8) {
 2409         if ($mem$$index >= 8) {
 2410           emit_opcode(cbuf, Assembler::REX_X);
 2411         } else if ($reg$$reg >= 4) {
 2412           emit_opcode(cbuf, Assembler::REX);
 2413         }
 2414       } else {
 2415         if ($mem$$index < 8) {
 2416           emit_opcode(cbuf, Assembler::REX_B);
 2417         } else {
 2418           emit_opcode(cbuf, Assembler::REX_XB);
 2419         }
 2420       }
 2421     } else {
 2422       if ($mem$$base < 8) {
 2423         if ($mem$$index < 8) {
 2424           emit_opcode(cbuf, Assembler::REX_R);
 2425         } else {
 2426           emit_opcode(cbuf, Assembler::REX_RX);
 2427         }
 2428       } else {
 2429         if ($mem$$index < 8) {
 2430           emit_opcode(cbuf, Assembler::REX_RB);
 2431         } else {
 2432           emit_opcode(cbuf, Assembler::REX_RXB);
 2433         }
 2434       }
 2435     }
 2436   %}
 2437 
 2438   enc_class REX_reg(rRegI reg)
 2439   %{
 2440     if ($reg$$reg >= 8) {
 2441       emit_opcode(cbuf, Assembler::REX_B);
 2442     }
 2443   %}
 2444 
 2445   enc_class REX_reg_wide(rRegI reg)
 2446   %{
 2447     if ($reg$$reg < 8) {
 2448       emit_opcode(cbuf, Assembler::REX_W);
 2449     } else {
 2450       emit_opcode(cbuf, Assembler::REX_WB);
 2451     }
 2452   %}
 2453 
 2454   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2455   %{
 2456     if ($dst$$reg < 8) {
 2457       if ($src$$reg >= 8) {
 2458         emit_opcode(cbuf, Assembler::REX_B);
 2459       }
 2460     } else {
 2461       if ($src$$reg < 8) {
 2462         emit_opcode(cbuf, Assembler::REX_R);
 2463       } else {
 2464         emit_opcode(cbuf, Assembler::REX_RB);
 2465       }
 2466     }
 2467   %}
 2468 
 2469   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2470   %{
 2471     if ($dst$$reg < 8) {
 2472       if ($src$$reg < 8) {
 2473         emit_opcode(cbuf, Assembler::REX_W);
 2474       } else {
 2475         emit_opcode(cbuf, Assembler::REX_WB);
 2476       }
 2477     } else {
 2478       if ($src$$reg < 8) {
 2479         emit_opcode(cbuf, Assembler::REX_WR);
 2480       } else {
 2481         emit_opcode(cbuf, Assembler::REX_WRB);
 2482       }
 2483     }
 2484   %}
 2485 
 2486   enc_class REX_reg_mem(rRegI reg, memory mem)
 2487   %{
 2488     if ($reg$$reg < 8) {
 2489       if ($mem$$base < 8) {
 2490         if ($mem$$index >= 8) {
 2491           emit_opcode(cbuf, Assembler::REX_X);
 2492         }
 2493       } else {
 2494         if ($mem$$index < 8) {
 2495           emit_opcode(cbuf, Assembler::REX_B);
 2496         } else {
 2497           emit_opcode(cbuf, Assembler::REX_XB);
 2498         }
 2499       }
 2500     } else {
 2501       if ($mem$$base < 8) {
 2502         if ($mem$$index < 8) {
 2503           emit_opcode(cbuf, Assembler::REX_R);
 2504         } else {
 2505           emit_opcode(cbuf, Assembler::REX_RX);
 2506         }
 2507       } else {
 2508         if ($mem$$index < 8) {
 2509           emit_opcode(cbuf, Assembler::REX_RB);
 2510         } else {
 2511           emit_opcode(cbuf, Assembler::REX_RXB);
 2512         }
 2513       }
 2514     }
 2515   %}
 2516 
 2517   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2518   %{
 2519     if ($reg$$reg < 8) {
 2520       if ($mem$$base < 8) {
 2521         if ($mem$$index < 8) {
 2522           emit_opcode(cbuf, Assembler::REX_W);
 2523         } else {
 2524           emit_opcode(cbuf, Assembler::REX_WX);
 2525         }
 2526       } else {
 2527         if ($mem$$index < 8) {
 2528           emit_opcode(cbuf, Assembler::REX_WB);
 2529         } else {
 2530           emit_opcode(cbuf, Assembler::REX_WXB);
 2531         }
 2532       }
 2533     } else {
 2534       if ($mem$$base < 8) {
 2535         if ($mem$$index < 8) {
 2536           emit_opcode(cbuf, Assembler::REX_WR);
 2537         } else {
 2538           emit_opcode(cbuf, Assembler::REX_WRX);
 2539         }
 2540       } else {
 2541         if ($mem$$index < 8) {
 2542           emit_opcode(cbuf, Assembler::REX_WRB);
 2543         } else {
 2544           emit_opcode(cbuf, Assembler::REX_WRXB);
 2545         }
 2546       }
 2547     }
 2548   %}
 2549 
 2550   enc_class reg_mem(rRegI ereg, memory mem)
 2551   %{
 2552     // High registers handle in encode_RegMem
 2553     int reg = $ereg$$reg;
 2554     int base = $mem$$base;
 2555     int index = $mem$$index;
 2556     int scale = $mem$$scale;
 2557     int disp = $mem$$disp;
 2558     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2559 
 2560     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2561   %}
 2562 
 2563   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2564   %{
 2565     int rm_byte_opcode = $rm_opcode$$constant;
 2566 
 2567     // High registers handle in encode_RegMem
 2568     int base = $mem$$base;
 2569     int index = $mem$$index;
 2570     int scale = $mem$$scale;
 2571     int displace = $mem$$disp;
 2572 
 2573     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2574                                             // working with static
 2575                                             // globals
 2576     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2577                   disp_reloc);
 2578   %}
 2579 
 2580   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2581   %{
 2582     int reg_encoding = $dst$$reg;
 2583     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2584     int index        = 0x04;            // 0x04 indicates no index
 2585     int scale        = 0x00;            // 0x00 indicates no scale
 2586     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2587     relocInfo::relocType disp_reloc = relocInfo::none;
 2588     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2589                   disp_reloc);
 2590   %}
 2591 
 2592   enc_class neg_reg(rRegI dst)
 2593   %{
 2594     int dstenc = $dst$$reg;
 2595     if (dstenc >= 8) {
 2596       emit_opcode(cbuf, Assembler::REX_B);
 2597       dstenc -= 8;
 2598     }
 2599     // NEG $dst
 2600     emit_opcode(cbuf, 0xF7);
 2601     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2602   %}
 2603 
 2604   enc_class neg_reg_wide(rRegI dst)
 2605   %{
 2606     int dstenc = $dst$$reg;
 2607     if (dstenc < 8) {
 2608       emit_opcode(cbuf, Assembler::REX_W);
 2609     } else {
 2610       emit_opcode(cbuf, Assembler::REX_WB);
 2611       dstenc -= 8;
 2612     }
 2613     // NEG $dst
 2614     emit_opcode(cbuf, 0xF7);
 2615     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2616   %}
 2617 
 2618   enc_class setLT_reg(rRegI dst)
 2619   %{
 2620     int dstenc = $dst$$reg;
 2621     if (dstenc >= 8) {
 2622       emit_opcode(cbuf, Assembler::REX_B);
 2623       dstenc -= 8;
 2624     } else if (dstenc >= 4) {
 2625       emit_opcode(cbuf, Assembler::REX);
 2626     }
 2627     // SETLT $dst
 2628     emit_opcode(cbuf, 0x0F);
 2629     emit_opcode(cbuf, 0x9C);
 2630     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2631   %}
 2632 
 2633   enc_class setNZ_reg(rRegI dst)
 2634   %{
 2635     int dstenc = $dst$$reg;
 2636     if (dstenc >= 8) {
 2637       emit_opcode(cbuf, Assembler::REX_B);
 2638       dstenc -= 8;
 2639     } else if (dstenc >= 4) {
 2640       emit_opcode(cbuf, Assembler::REX);
 2641     }
 2642     // SETNZ $dst
 2643     emit_opcode(cbuf, 0x0F);
 2644     emit_opcode(cbuf, 0x95);
 2645     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2646   %}
 2647 
 2648 
 2649   // Compare the lonogs and set -1, 0, or 1 into dst
 2650   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2651   %{
 2652     int src1enc = $src1$$reg;
 2653     int src2enc = $src2$$reg;
 2654     int dstenc = $dst$$reg;
 2655 
 2656     // cmpq $src1, $src2
 2657     if (src1enc < 8) {
 2658       if (src2enc < 8) {
 2659         emit_opcode(cbuf, Assembler::REX_W);
 2660       } else {
 2661         emit_opcode(cbuf, Assembler::REX_WB);
 2662       }
 2663     } else {
 2664       if (src2enc < 8) {
 2665         emit_opcode(cbuf, Assembler::REX_WR);
 2666       } else {
 2667         emit_opcode(cbuf, Assembler::REX_WRB);
 2668       }
 2669     }
 2670     emit_opcode(cbuf, 0x3B);
 2671     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2672 
 2673     // movl $dst, -1
 2674     if (dstenc >= 8) {
 2675       emit_opcode(cbuf, Assembler::REX_B);
 2676     }
 2677     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2678     emit_d32(cbuf, -1);
 2679 
 2680     // jl,s done
 2681     emit_opcode(cbuf, 0x7C);
 2682     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2683 
 2684     // setne $dst
 2685     if (dstenc >= 4) {
 2686       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2687     }
 2688     emit_opcode(cbuf, 0x0F);
 2689     emit_opcode(cbuf, 0x95);
 2690     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2691 
 2692     // movzbl $dst, $dst
 2693     if (dstenc >= 4) {
 2694       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2695     }
 2696     emit_opcode(cbuf, 0x0F);
 2697     emit_opcode(cbuf, 0xB6);
 2698     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2699   %}
 2700 
 2701   enc_class Push_ResultXD(regD dst) %{
 2702     MacroAssembler _masm(&cbuf);
 2703     __ fstp_d(Address(rsp, 0));
 2704     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2705     __ addptr(rsp, 8);
 2706   %}
 2707 
 2708   enc_class Push_SrcXD(regD src) %{
 2709     MacroAssembler _masm(&cbuf);
 2710     __ subptr(rsp, 8);
 2711     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2712     __ fld_d(Address(rsp, 0));
 2713   %}
 2714 
 2715 
 2716   enc_class enc_rethrow()
 2717   %{
 2718     cbuf.set_insts_mark();
 2719     emit_opcode(cbuf, 0xE9); // jmp entry
 2720     emit_d32_reloc(cbuf,
 2721                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2722                    runtime_call_Relocation::spec(),
 2723                    RELOC_DISP32);
 2724   %}
 2725 
 2726 %}
 2727 
 2728 
 2729 
 2730 //----------FRAME--------------------------------------------------------------
 2731 // Definition of frame structure and management information.
 2732 //
 2733 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2734 //                             |   (to get allocators register number
 2735 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2736 //  r   CALLER     |        |
 2737 //  o     |        +--------+      pad to even-align allocators stack-slot
 2738 //  w     V        |  pad0  |        numbers; owned by CALLER
 2739 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2740 //  h     ^        |   in   |  5
 2741 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2742 //  |     |        |        |  3
 2743 //  |     |        +--------+
 2744 //  V     |        | old out|      Empty on Intel, window on Sparc
 2745 //        |    old |preserve|      Must be even aligned.
 2746 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2747 //        |        |   in   |  3   area for Intel ret address
 2748 //     Owned by    |preserve|      Empty on Sparc.
 2749 //       SELF      +--------+
 2750 //        |        |  pad2  |  2   pad to align old SP
 2751 //        |        +--------+  1
 2752 //        |        | locks  |  0
 2753 //        |        +--------+----> OptoReg::stack0(), even aligned
 2754 //        |        |  pad1  | 11   pad to align new SP
 2755 //        |        +--------+
 2756 //        |        |        | 10
 2757 //        |        | spills |  9   spills
 2758 //        V        |        |  8   (pad0 slot for callee)
 2759 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2760 //        ^        |  out   |  7
 2761 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2762 //     Owned by    +--------+
 2763 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2764 //        |    new |preserve|      Must be even-aligned.
 2765 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2766 //        |        |        |
 2767 //
 2768 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2769 //         known from SELF's arguments and the Java calling convention.
 2770 //         Region 6-7 is determined per call site.
 2771 // Note 2: If the calling convention leaves holes in the incoming argument
 2772 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2773 //         are owned by the CALLEE.  Holes should not be necessary in the
 2774 //         incoming area, as the Java calling convention is completely under
 2775 //         the control of the AD file.  Doubles can be sorted and packed to
 2776 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2777 //         varargs C calling conventions.
 2778 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2779 //         even aligned with pad0 as needed.
 2780 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2781 //         region 6-11 is even aligned; it may be padded out more so that
 2782 //         the region from SP to FP meets the minimum stack alignment.
 2783 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2784 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2785 //         SP meets the minimum alignment.
 2786 
 2787 frame
 2788 %{
 2789   // These three registers define part of the calling convention
 2790   // between compiled code and the interpreter.
 2791   inline_cache_reg(RAX);                // Inline Cache Register
 2792 
 2793   // Optional: name the operand used by cisc-spilling to access
 2794   // [stack_pointer + offset]
 2795   cisc_spilling_operand_name(indOffset32);
 2796 
 2797   // Number of stack slots consumed by locking an object
 2798   sync_stack_slots(2);
 2799 
 2800   // Compiled code's Frame Pointer
 2801   frame_pointer(RSP);
 2802 
 2803   // Interpreter stores its frame pointer in a register which is
 2804   // stored to the stack by I2CAdaptors.
 2805   // I2CAdaptors convert from interpreted java to compiled java.
 2806   interpreter_frame_pointer(RBP);
 2807 
 2808   // Stack alignment requirement
 2809   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2810 
 2811   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2812   // for calls to C.  Supports the var-args backing area for register parms.
 2813   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2814 
 2815   // The after-PROLOG location of the return address.  Location of
 2816   // return address specifies a type (REG or STACK) and a number
 2817   // representing the register number (i.e. - use a register name) or
 2818   // stack slot.
 2819   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2820   // Otherwise, it is above the locks and verification slot and alignment word
 2821   return_addr(STACK - 2 +
 2822               align_up((Compile::current()->in_preserve_stack_slots() +
 2823                         Compile::current()->fixed_slots()),
 2824                        stack_alignment_in_slots()));
 2825 
 2826   // Location of compiled Java return values.  Same as C for now.
 2827   return_value
 2828   %{
 2829     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2830            "only return normal values");
 2831 
 2832     static const int lo[Op_RegL + 1] = {
 2833       0,
 2834       0,
 2835       RAX_num,  // Op_RegN
 2836       RAX_num,  // Op_RegI
 2837       RAX_num,  // Op_RegP
 2838       XMM0_num, // Op_RegF
 2839       XMM0_num, // Op_RegD
 2840       RAX_num   // Op_RegL
 2841     };
 2842     static const int hi[Op_RegL + 1] = {
 2843       0,
 2844       0,
 2845       OptoReg::Bad, // Op_RegN
 2846       OptoReg::Bad, // Op_RegI
 2847       RAX_H_num,    // Op_RegP
 2848       OptoReg::Bad, // Op_RegF
 2849       XMM0b_num,    // Op_RegD
 2850       RAX_H_num     // Op_RegL
 2851     };
 2852     // Excluded flags and vector registers.
 2853     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2854     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2855   %}
 2856 %}
 2857 
 2858 //----------ATTRIBUTES---------------------------------------------------------
 2859 //----------Operand Attributes-------------------------------------------------
 2860 op_attrib op_cost(0);        // Required cost attribute
 2861 
 2862 //----------Instruction Attributes---------------------------------------------
 2863 ins_attrib ins_cost(100);       // Required cost attribute
 2864 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2865 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2866                                 // a non-matching short branch variant
 2867                                 // of some long branch?
 2868 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2869                                 // be a power of 2) specifies the
 2870                                 // alignment that some part of the
 2871                                 // instruction (not necessarily the
 2872                                 // start) requires.  If > 1, a
 2873                                 // compute_padding() function must be
 2874                                 // provided for the instruction
 2875 
 2876 //----------OPERANDS-----------------------------------------------------------
 2877 // Operand definitions must precede instruction definitions for correct parsing
 2878 // in the ADLC because operands constitute user defined types which are used in
 2879 // instruction definitions.
 2880 
 2881 //----------Simple Operands----------------------------------------------------
 2882 // Immediate Operands
 2883 // Integer Immediate
 2884 operand immI()
 2885 %{
 2886   match(ConI);
 2887 
 2888   op_cost(10);
 2889   format %{ %}
 2890   interface(CONST_INTER);
 2891 %}
 2892 
 2893 // Constant for test vs zero
 2894 operand immI_0()
 2895 %{
 2896   predicate(n->get_int() == 0);
 2897   match(ConI);
 2898 
 2899   op_cost(0);
 2900   format %{ %}
 2901   interface(CONST_INTER);
 2902 %}
 2903 
 2904 // Constant for increment
 2905 operand immI_1()
 2906 %{
 2907   predicate(n->get_int() == 1);
 2908   match(ConI);
 2909 
 2910   op_cost(0);
 2911   format %{ %}
 2912   interface(CONST_INTER);
 2913 %}
 2914 
 2915 // Constant for decrement
 2916 operand immI_M1()
 2917 %{
 2918   predicate(n->get_int() == -1);
 2919   match(ConI);
 2920 
 2921   op_cost(0);
 2922   format %{ %}
 2923   interface(CONST_INTER);
 2924 %}
 2925 
 2926 operand immI_2()
 2927 %{
 2928   predicate(n->get_int() == 2);
 2929   match(ConI);
 2930 
 2931   op_cost(0);
 2932   format %{ %}
 2933   interface(CONST_INTER);
 2934 %}
 2935 
 2936 operand immI_4()
 2937 %{
 2938   predicate(n->get_int() == 4);
 2939   match(ConI);
 2940 
 2941   op_cost(0);
 2942   format %{ %}
 2943   interface(CONST_INTER);
 2944 %}
 2945 
 2946 operand immI_8()
 2947 %{
 2948   predicate(n->get_int() == 8);
 2949   match(ConI);
 2950 
 2951   op_cost(0);
 2952   format %{ %}
 2953   interface(CONST_INTER);
 2954 %}
 2955 
 2956 // Valid scale values for addressing modes
 2957 operand immI2()
 2958 %{
 2959   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2960   match(ConI);
 2961 
 2962   format %{ %}
 2963   interface(CONST_INTER);
 2964 %}
 2965 
 2966 operand immU7()
 2967 %{
 2968   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2969   match(ConI);
 2970 
 2971   op_cost(5);
 2972   format %{ %}
 2973   interface(CONST_INTER);
 2974 %}
 2975 
 2976 operand immI8()
 2977 %{
 2978   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2979   match(ConI);
 2980 
 2981   op_cost(5);
 2982   format %{ %}
 2983   interface(CONST_INTER);
 2984 %}
 2985 
 2986 operand immU8()
 2987 %{
 2988   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2989   match(ConI);
 2990 
 2991   op_cost(5);
 2992   format %{ %}
 2993   interface(CONST_INTER);
 2994 %}
 2995 
 2996 operand immI16()
 2997 %{
 2998   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2999   match(ConI);
 3000 
 3001   op_cost(10);
 3002   format %{ %}
 3003   interface(CONST_INTER);
 3004 %}
 3005 
 3006 // Int Immediate non-negative
 3007 operand immU31()
 3008 %{
 3009   predicate(n->get_int() >= 0);
 3010   match(ConI);
 3011 
 3012   op_cost(0);
 3013   format %{ %}
 3014   interface(CONST_INTER);
 3015 %}
 3016 
 3017 // Constant for long shifts
 3018 operand immI_32()
 3019 %{
 3020   predicate( n->get_int() == 32 );
 3021   match(ConI);
 3022 
 3023   op_cost(0);
 3024   format %{ %}
 3025   interface(CONST_INTER);
 3026 %}
 3027 
 3028 // Constant for long shifts
 3029 operand immI_64()
 3030 %{
 3031   predicate( n->get_int() == 64 );
 3032   match(ConI);
 3033 
 3034   op_cost(0);
 3035   format %{ %}
 3036   interface(CONST_INTER);
 3037 %}
 3038 
 3039 // Pointer Immediate
 3040 operand immP()
 3041 %{
 3042   match(ConP);
 3043 
 3044   op_cost(10);
 3045   format %{ %}
 3046   interface(CONST_INTER);
 3047 %}
 3048 
 3049 // NULL Pointer Immediate
 3050 operand immP0()
 3051 %{
 3052   predicate(n->get_ptr() == 0);
 3053   match(ConP);
 3054 
 3055   op_cost(5);
 3056   format %{ %}
 3057   interface(CONST_INTER);
 3058 %}
 3059 
 3060 // Pointer Immediate
 3061 operand immN() %{
 3062   match(ConN);
 3063 
 3064   op_cost(10);
 3065   format %{ %}
 3066   interface(CONST_INTER);
 3067 %}
 3068 
 3069 operand immNKlass() %{
 3070   match(ConNKlass);
 3071 
 3072   op_cost(10);
 3073   format %{ %}
 3074   interface(CONST_INTER);
 3075 %}
 3076 
 3077 // NULL Pointer Immediate
 3078 operand immN0() %{
 3079   predicate(n->get_narrowcon() == 0);
 3080   match(ConN);
 3081 
 3082   op_cost(5);
 3083   format %{ %}
 3084   interface(CONST_INTER);
 3085 %}
 3086 
 3087 operand immP31()
 3088 %{
 3089   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3090             && (n->get_ptr() >> 31) == 0);
 3091   match(ConP);
 3092 
 3093   op_cost(5);
 3094   format %{ %}
 3095   interface(CONST_INTER);
 3096 %}
 3097 
 3098 
 3099 // Long Immediate
 3100 operand immL()
 3101 %{
 3102   match(ConL);
 3103 
 3104   op_cost(20);
 3105   format %{ %}
 3106   interface(CONST_INTER);
 3107 %}
 3108 
 3109 // Long Immediate 8-bit
 3110 operand immL8()
 3111 %{
 3112   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3113   match(ConL);
 3114 
 3115   op_cost(5);
 3116   format %{ %}
 3117   interface(CONST_INTER);
 3118 %}
 3119 
 3120 // Long Immediate 32-bit unsigned
 3121 operand immUL32()
 3122 %{
 3123   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3124   match(ConL);
 3125 
 3126   op_cost(10);
 3127   format %{ %}
 3128   interface(CONST_INTER);
 3129 %}
 3130 
 3131 // Long Immediate 32-bit signed
 3132 operand immL32()
 3133 %{
 3134   predicate(n->get_long() == (int) (n->get_long()));
 3135   match(ConL);
 3136 
 3137   op_cost(15);
 3138   format %{ %}
 3139   interface(CONST_INTER);
 3140 %}
 3141 
 3142 operand immL_Pow2()
 3143 %{
 3144   predicate(is_power_of_2((julong)n->get_long()));
 3145   match(ConL);
 3146 
 3147   op_cost(15);
 3148   format %{ %}
 3149   interface(CONST_INTER);
 3150 %}
 3151 
 3152 operand immL_NotPow2()
 3153 %{
 3154   predicate(is_power_of_2((julong)~n->get_long()));
 3155   match(ConL);
 3156 
 3157   op_cost(15);
 3158   format %{ %}
 3159   interface(CONST_INTER);
 3160 %}
 3161 
 3162 // Long Immediate zero
 3163 operand immL0()
 3164 %{
 3165   predicate(n->get_long() == 0L);
 3166   match(ConL);
 3167 
 3168   op_cost(10);
 3169   format %{ %}
 3170   interface(CONST_INTER);
 3171 %}
 3172 
 3173 // Constant for increment
 3174 operand immL1()
 3175 %{
 3176   predicate(n->get_long() == 1);
 3177   match(ConL);
 3178 
 3179   format %{ %}
 3180   interface(CONST_INTER);
 3181 %}
 3182 
 3183 // Constant for decrement
 3184 operand immL_M1()
 3185 %{
 3186   predicate(n->get_long() == -1);
 3187   match(ConL);
 3188 
 3189   format %{ %}
 3190   interface(CONST_INTER);
 3191 %}
 3192 
 3193 // Long Immediate: the value 10
 3194 operand immL10()
 3195 %{
 3196   predicate(n->get_long() == 10);
 3197   match(ConL);
 3198 
 3199   format %{ %}
 3200   interface(CONST_INTER);
 3201 %}
 3202 
 3203 // Long immediate from 0 to 127.
 3204 // Used for a shorter form of long mul by 10.
 3205 operand immL_127()
 3206 %{
 3207   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3208   match(ConL);
 3209 
 3210   op_cost(10);
 3211   format %{ %}
 3212   interface(CONST_INTER);
 3213 %}
 3214 
 3215 // Long Immediate: low 32-bit mask
 3216 operand immL_32bits()
 3217 %{
 3218   predicate(n->get_long() == 0xFFFFFFFFL);
 3219   match(ConL);
 3220   op_cost(20);
 3221 
 3222   format %{ %}
 3223   interface(CONST_INTER);
 3224 %}
 3225 
 3226 // Int Immediate: 2^n-1, positive
 3227 operand immI_Pow2M1()
 3228 %{
 3229   predicate((n->get_int() > 0)
 3230             && is_power_of_2(n->get_int() + 1));
 3231   match(ConI);
 3232 
 3233   op_cost(20);
 3234   format %{ %}
 3235   interface(CONST_INTER);
 3236 %}
 3237 
 3238 // Float Immediate zero
 3239 operand immF0()
 3240 %{
 3241   predicate(jint_cast(n->getf()) == 0);
 3242   match(ConF);
 3243 
 3244   op_cost(5);
 3245   format %{ %}
 3246   interface(CONST_INTER);
 3247 %}
 3248 
 3249 // Float Immediate
 3250 operand immF()
 3251 %{
 3252   match(ConF);
 3253 
 3254   op_cost(15);
 3255   format %{ %}
 3256   interface(CONST_INTER);
 3257 %}
 3258 
 3259 // Double Immediate zero
 3260 operand immD0()
 3261 %{
 3262   predicate(jlong_cast(n->getd()) == 0);
 3263   match(ConD);
 3264 
 3265   op_cost(5);
 3266   format %{ %}
 3267   interface(CONST_INTER);
 3268 %}
 3269 
 3270 // Double Immediate
 3271 operand immD()
 3272 %{
 3273   match(ConD);
 3274 
 3275   op_cost(15);
 3276   format %{ %}
 3277   interface(CONST_INTER);
 3278 %}
 3279 
 3280 // Immediates for special shifts (sign extend)
 3281 
 3282 // Constants for increment
 3283 operand immI_16()
 3284 %{
 3285   predicate(n->get_int() == 16);
 3286   match(ConI);
 3287 
 3288   format %{ %}
 3289   interface(CONST_INTER);
 3290 %}
 3291 
 3292 operand immI_24()
 3293 %{
 3294   predicate(n->get_int() == 24);
 3295   match(ConI);
 3296 
 3297   format %{ %}
 3298   interface(CONST_INTER);
 3299 %}
 3300 
 3301 // Constant for byte-wide masking
 3302 operand immI_255()
 3303 %{
 3304   predicate(n->get_int() == 255);
 3305   match(ConI);
 3306 
 3307   format %{ %}
 3308   interface(CONST_INTER);
 3309 %}
 3310 
 3311 // Constant for short-wide masking
 3312 operand immI_65535()
 3313 %{
 3314   predicate(n->get_int() == 65535);
 3315   match(ConI);
 3316 
 3317   format %{ %}
 3318   interface(CONST_INTER);
 3319 %}
 3320 
 3321 // Constant for byte-wide masking
 3322 operand immL_255()
 3323 %{
 3324   predicate(n->get_long() == 255);
 3325   match(ConL);
 3326 
 3327   format %{ %}
 3328   interface(CONST_INTER);
 3329 %}
 3330 
 3331 // Constant for short-wide masking
 3332 operand immL_65535()
 3333 %{
 3334   predicate(n->get_long() == 65535);
 3335   match(ConL);
 3336 
 3337   format %{ %}
 3338   interface(CONST_INTER);
 3339 %}
 3340 
 3341 operand kReg()
 3342 %{
 3343   constraint(ALLOC_IN_RC(vectmask_reg));
 3344   match(RegVectMask);
 3345   format %{%}
 3346   interface(REG_INTER);
 3347 %}
 3348 
 3349 operand kReg_K1()
 3350 %{
 3351   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3352   match(RegVectMask);
 3353   format %{%}
 3354   interface(REG_INTER);
 3355 %}
 3356 
 3357 operand kReg_K2()
 3358 %{
 3359   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3360   match(RegVectMask);
 3361   format %{%}
 3362   interface(REG_INTER);
 3363 %}
 3364 
 3365 // Special Registers
 3366 operand kReg_K3()
 3367 %{
 3368   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3369   match(RegVectMask);
 3370   format %{%}
 3371   interface(REG_INTER);
 3372 %}
 3373 
 3374 operand kReg_K4()
 3375 %{
 3376   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3377   match(RegVectMask);
 3378   format %{%}
 3379   interface(REG_INTER);
 3380 %}
 3381 
 3382 operand kReg_K5()
 3383 %{
 3384   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3385   match(RegVectMask);
 3386   format %{%}
 3387   interface(REG_INTER);
 3388 %}
 3389 
 3390 operand kReg_K6()
 3391 %{
 3392   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3393   match(RegVectMask);
 3394   format %{%}
 3395   interface(REG_INTER);
 3396 %}
 3397 
 3398 // Special Registers
 3399 operand kReg_K7()
 3400 %{
 3401   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3402   match(RegVectMask);
 3403   format %{%}
 3404   interface(REG_INTER);
 3405 %}
 3406 
 3407 // Register Operands
 3408 // Integer Register
 3409 operand rRegI()
 3410 %{
 3411   constraint(ALLOC_IN_RC(int_reg));
 3412   match(RegI);
 3413 
 3414   match(rax_RegI);
 3415   match(rbx_RegI);
 3416   match(rcx_RegI);
 3417   match(rdx_RegI);
 3418   match(rdi_RegI);
 3419 
 3420   format %{ %}
 3421   interface(REG_INTER);
 3422 %}
 3423 
 3424 // Special Registers
 3425 operand rax_RegI()
 3426 %{
 3427   constraint(ALLOC_IN_RC(int_rax_reg));
 3428   match(RegI);
 3429   match(rRegI);
 3430 
 3431   format %{ "RAX" %}
 3432   interface(REG_INTER);
 3433 %}
 3434 
 3435 // Special Registers
 3436 operand rbx_RegI()
 3437 %{
 3438   constraint(ALLOC_IN_RC(int_rbx_reg));
 3439   match(RegI);
 3440   match(rRegI);
 3441 
 3442   format %{ "RBX" %}
 3443   interface(REG_INTER);
 3444 %}
 3445 
 3446 operand rcx_RegI()
 3447 %{
 3448   constraint(ALLOC_IN_RC(int_rcx_reg));
 3449   match(RegI);
 3450   match(rRegI);
 3451 
 3452   format %{ "RCX" %}
 3453   interface(REG_INTER);
 3454 %}
 3455 
 3456 operand rdx_RegI()
 3457 %{
 3458   constraint(ALLOC_IN_RC(int_rdx_reg));
 3459   match(RegI);
 3460   match(rRegI);
 3461 
 3462   format %{ "RDX" %}
 3463   interface(REG_INTER);
 3464 %}
 3465 
 3466 operand rdi_RegI()
 3467 %{
 3468   constraint(ALLOC_IN_RC(int_rdi_reg));
 3469   match(RegI);
 3470   match(rRegI);
 3471 
 3472   format %{ "RDI" %}
 3473   interface(REG_INTER);
 3474 %}
 3475 
 3476 operand no_rax_rdx_RegI()
 3477 %{
 3478   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3479   match(RegI);
 3480   match(rbx_RegI);
 3481   match(rcx_RegI);
 3482   match(rdi_RegI);
 3483 
 3484   format %{ %}
 3485   interface(REG_INTER);
 3486 %}
 3487 
 3488 operand no_rbp_r13_RegI()
 3489 %{
 3490   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3491   match(RegI);
 3492   match(rRegI);
 3493   match(rax_RegI);
 3494   match(rbx_RegI);
 3495   match(rcx_RegI);
 3496   match(rdx_RegI);
 3497   match(rdi_RegI);
 3498 
 3499   format %{ %}
 3500   interface(REG_INTER);
 3501 %}
 3502 
 3503 // Pointer Register
 3504 operand any_RegP()
 3505 %{
 3506   constraint(ALLOC_IN_RC(any_reg));
 3507   match(RegP);
 3508   match(rax_RegP);
 3509   match(rbx_RegP);
 3510   match(rdi_RegP);
 3511   match(rsi_RegP);
 3512   match(rbp_RegP);
 3513   match(r15_RegP);
 3514   match(rRegP);
 3515 
 3516   format %{ %}
 3517   interface(REG_INTER);
 3518 %}
 3519 
 3520 operand rRegP()
 3521 %{
 3522   constraint(ALLOC_IN_RC(ptr_reg));
 3523   match(RegP);
 3524   match(rax_RegP);
 3525   match(rbx_RegP);
 3526   match(rdi_RegP);
 3527   match(rsi_RegP);
 3528   match(rbp_RegP);  // See Q&A below about
 3529   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3530 
 3531   format %{ %}
 3532   interface(REG_INTER);
 3533 %}
 3534 
 3535 operand rRegN() %{
 3536   constraint(ALLOC_IN_RC(int_reg));
 3537   match(RegN);
 3538 
 3539   format %{ %}
 3540   interface(REG_INTER);
 3541 %}
 3542 
 3543 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3544 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3545 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3546 // The output of an instruction is controlled by the allocator, which respects
 3547 // register class masks, not match rules.  Unless an instruction mentions
 3548 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3549 // by the allocator as an input.
 3550 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3551 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3552 // result, RBP is not included in the output of the instruction either.
 3553 
 3554 operand no_rax_RegP()
 3555 %{
 3556   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3557   match(RegP);
 3558   match(rbx_RegP);
 3559   match(rsi_RegP);
 3560   match(rdi_RegP);
 3561 
 3562   format %{ %}
 3563   interface(REG_INTER);
 3564 %}
 3565 
 3566 // This operand is not allowed to use RBP even if
 3567 // RBP is not used to hold the frame pointer.
 3568 operand no_rbp_RegP()
 3569 %{
 3570   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3571   match(RegP);
 3572   match(rbx_RegP);
 3573   match(rsi_RegP);
 3574   match(rdi_RegP);
 3575 
 3576   format %{ %}
 3577   interface(REG_INTER);
 3578 %}
 3579 
 3580 operand no_rax_rbx_RegP()
 3581 %{
 3582   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3583   match(RegP);
 3584   match(rsi_RegP);
 3585   match(rdi_RegP);
 3586 
 3587   format %{ %}
 3588   interface(REG_INTER);
 3589 %}
 3590 
 3591 // Special Registers
 3592 // Return a pointer value
 3593 operand rax_RegP()
 3594 %{
 3595   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3596   match(RegP);
 3597   match(rRegP);
 3598 
 3599   format %{ %}
 3600   interface(REG_INTER);
 3601 %}
 3602 
 3603 // Special Registers
 3604 // Return a compressed pointer value
 3605 operand rax_RegN()
 3606 %{
 3607   constraint(ALLOC_IN_RC(int_rax_reg));
 3608   match(RegN);
 3609   match(rRegN);
 3610 
 3611   format %{ %}
 3612   interface(REG_INTER);
 3613 %}
 3614 
 3615 // Used in AtomicAdd
 3616 operand rbx_RegP()
 3617 %{
 3618   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3619   match(RegP);
 3620   match(rRegP);
 3621 
 3622   format %{ %}
 3623   interface(REG_INTER);
 3624 %}
 3625 
 3626 operand rsi_RegP()
 3627 %{
 3628   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3629   match(RegP);
 3630   match(rRegP);
 3631 
 3632   format %{ %}
 3633   interface(REG_INTER);
 3634 %}
 3635 
 3636 operand rbp_RegP()
 3637 %{
 3638   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3639   match(RegP);
 3640   match(rRegP);
 3641 
 3642   format %{ %}
 3643   interface(REG_INTER);
 3644 %}
 3645 
 3646 // Used in rep stosq
 3647 operand rdi_RegP()
 3648 %{
 3649   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3650   match(RegP);
 3651   match(rRegP);
 3652 
 3653   format %{ %}
 3654   interface(REG_INTER);
 3655 %}
 3656 
 3657 operand r15_RegP()
 3658 %{
 3659   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3660   match(RegP);
 3661   match(rRegP);
 3662 
 3663   format %{ %}
 3664   interface(REG_INTER);
 3665 %}
 3666 
 3667 operand rRegL()
 3668 %{
 3669   constraint(ALLOC_IN_RC(long_reg));
 3670   match(RegL);
 3671   match(rax_RegL);
 3672   match(rdx_RegL);
 3673 
 3674   format %{ %}
 3675   interface(REG_INTER);
 3676 %}
 3677 
 3678 // Special Registers
 3679 operand no_rax_rdx_RegL()
 3680 %{
 3681   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3682   match(RegL);
 3683   match(rRegL);
 3684 
 3685   format %{ %}
 3686   interface(REG_INTER);
 3687 %}
 3688 
 3689 operand rax_RegL()
 3690 %{
 3691   constraint(ALLOC_IN_RC(long_rax_reg));
 3692   match(RegL);
 3693   match(rRegL);
 3694 
 3695   format %{ "RAX" %}
 3696   interface(REG_INTER);
 3697 %}
 3698 
 3699 operand rcx_RegL()
 3700 %{
 3701   constraint(ALLOC_IN_RC(long_rcx_reg));
 3702   match(RegL);
 3703   match(rRegL);
 3704 
 3705   format %{ %}
 3706   interface(REG_INTER);
 3707 %}
 3708 
 3709 operand rdx_RegL()
 3710 %{
 3711   constraint(ALLOC_IN_RC(long_rdx_reg));
 3712   match(RegL);
 3713   match(rRegL);
 3714 
 3715   format %{ %}
 3716   interface(REG_INTER);
 3717 %}
 3718 
 3719 operand no_rbp_r13_RegL()
 3720 %{
 3721   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3722   match(RegL);
 3723   match(rRegL);
 3724   match(rax_RegL);
 3725   match(rcx_RegL);
 3726   match(rdx_RegL);
 3727 
 3728   format %{ %}
 3729   interface(REG_INTER);
 3730 %}
 3731 
 3732 // Flags register, used as output of compare instructions
 3733 operand rFlagsReg()
 3734 %{
 3735   constraint(ALLOC_IN_RC(int_flags));
 3736   match(RegFlags);
 3737 
 3738   format %{ "RFLAGS" %}
 3739   interface(REG_INTER);
 3740 %}
 3741 
 3742 // Flags register, used as output of FLOATING POINT compare instructions
 3743 operand rFlagsRegU()
 3744 %{
 3745   constraint(ALLOC_IN_RC(int_flags));
 3746   match(RegFlags);
 3747 
 3748   format %{ "RFLAGS_U" %}
 3749   interface(REG_INTER);
 3750 %}
 3751 
 3752 operand rFlagsRegUCF() %{
 3753   constraint(ALLOC_IN_RC(int_flags));
 3754   match(RegFlags);
 3755   predicate(false);
 3756 
 3757   format %{ "RFLAGS_U_CF" %}
 3758   interface(REG_INTER);
 3759 %}
 3760 
 3761 // Float register operands
 3762 operand regF() %{
 3763    constraint(ALLOC_IN_RC(float_reg));
 3764    match(RegF);
 3765 
 3766    format %{ %}
 3767    interface(REG_INTER);
 3768 %}
 3769 
 3770 // Float register operands
 3771 operand legRegF() %{
 3772    constraint(ALLOC_IN_RC(float_reg_legacy));
 3773    match(RegF);
 3774 
 3775    format %{ %}
 3776    interface(REG_INTER);
 3777 %}
 3778 
 3779 // Float register operands
 3780 operand vlRegF() %{
 3781    constraint(ALLOC_IN_RC(float_reg_vl));
 3782    match(RegF);
 3783 
 3784    format %{ %}
 3785    interface(REG_INTER);
 3786 %}
 3787 
 3788 // Double register operands
 3789 operand regD() %{
 3790    constraint(ALLOC_IN_RC(double_reg));
 3791    match(RegD);
 3792 
 3793    format %{ %}
 3794    interface(REG_INTER);
 3795 %}
 3796 
 3797 // Double register operands
 3798 operand legRegD() %{
 3799    constraint(ALLOC_IN_RC(double_reg_legacy));
 3800    match(RegD);
 3801 
 3802    format %{ %}
 3803    interface(REG_INTER);
 3804 %}
 3805 
 3806 // Double register operands
 3807 operand vlRegD() %{
 3808    constraint(ALLOC_IN_RC(double_reg_vl));
 3809    match(RegD);
 3810 
 3811    format %{ %}
 3812    interface(REG_INTER);
 3813 %}
 3814 
 3815 //----------Memory Operands----------------------------------------------------
 3816 // Direct Memory Operand
 3817 // operand direct(immP addr)
 3818 // %{
 3819 //   match(addr);
 3820 
 3821 //   format %{ "[$addr]" %}
 3822 //   interface(MEMORY_INTER) %{
 3823 //     base(0xFFFFFFFF);
 3824 //     index(0x4);
 3825 //     scale(0x0);
 3826 //     disp($addr);
 3827 //   %}
 3828 // %}
 3829 
 3830 // Indirect Memory Operand
 3831 operand indirect(any_RegP reg)
 3832 %{
 3833   constraint(ALLOC_IN_RC(ptr_reg));
 3834   match(reg);
 3835 
 3836   format %{ "[$reg]" %}
 3837   interface(MEMORY_INTER) %{
 3838     base($reg);
 3839     index(0x4);
 3840     scale(0x0);
 3841     disp(0x0);
 3842   %}
 3843 %}
 3844 
 3845 // Indirect Memory Plus Short Offset Operand
 3846 operand indOffset8(any_RegP reg, immL8 off)
 3847 %{
 3848   constraint(ALLOC_IN_RC(ptr_reg));
 3849   match(AddP reg off);
 3850 
 3851   format %{ "[$reg + $off (8-bit)]" %}
 3852   interface(MEMORY_INTER) %{
 3853     base($reg);
 3854     index(0x4);
 3855     scale(0x0);
 3856     disp($off);
 3857   %}
 3858 %}
 3859 
 3860 // Indirect Memory Plus Long Offset Operand
 3861 operand indOffset32(any_RegP reg, immL32 off)
 3862 %{
 3863   constraint(ALLOC_IN_RC(ptr_reg));
 3864   match(AddP reg off);
 3865 
 3866   format %{ "[$reg + $off (32-bit)]" %}
 3867   interface(MEMORY_INTER) %{
 3868     base($reg);
 3869     index(0x4);
 3870     scale(0x0);
 3871     disp($off);
 3872   %}
 3873 %}
 3874 
 3875 // Indirect Memory Plus Index Register Plus Offset Operand
 3876 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3877 %{
 3878   constraint(ALLOC_IN_RC(ptr_reg));
 3879   match(AddP (AddP reg lreg) off);
 3880 
 3881   op_cost(10);
 3882   format %{"[$reg + $off + $lreg]" %}
 3883   interface(MEMORY_INTER) %{
 3884     base($reg);
 3885     index($lreg);
 3886     scale(0x0);
 3887     disp($off);
 3888   %}
 3889 %}
 3890 
 3891 // Indirect Memory Plus Index Register Plus Offset Operand
 3892 operand indIndex(any_RegP reg, rRegL lreg)
 3893 %{
 3894   constraint(ALLOC_IN_RC(ptr_reg));
 3895   match(AddP reg lreg);
 3896 
 3897   op_cost(10);
 3898   format %{"[$reg + $lreg]" %}
 3899   interface(MEMORY_INTER) %{
 3900     base($reg);
 3901     index($lreg);
 3902     scale(0x0);
 3903     disp(0x0);
 3904   %}
 3905 %}
 3906 
 3907 // Indirect Memory Times Scale Plus Index Register
 3908 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3909 %{
 3910   constraint(ALLOC_IN_RC(ptr_reg));
 3911   match(AddP reg (LShiftL lreg scale));
 3912 
 3913   op_cost(10);
 3914   format %{"[$reg + $lreg << $scale]" %}
 3915   interface(MEMORY_INTER) %{
 3916     base($reg);
 3917     index($lreg);
 3918     scale($scale);
 3919     disp(0x0);
 3920   %}
 3921 %}
 3922 
 3923 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3924 %{
 3925   constraint(ALLOC_IN_RC(ptr_reg));
 3926   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3927   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3928 
 3929   op_cost(10);
 3930   format %{"[$reg + pos $idx << $scale]" %}
 3931   interface(MEMORY_INTER) %{
 3932     base($reg);
 3933     index($idx);
 3934     scale($scale);
 3935     disp(0x0);
 3936   %}
 3937 %}
 3938 
 3939 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3940 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3941 %{
 3942   constraint(ALLOC_IN_RC(ptr_reg));
 3943   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3944 
 3945   op_cost(10);
 3946   format %{"[$reg + $off + $lreg << $scale]" %}
 3947   interface(MEMORY_INTER) %{
 3948     base($reg);
 3949     index($lreg);
 3950     scale($scale);
 3951     disp($off);
 3952   %}
 3953 %}
 3954 
 3955 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3956 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3957 %{
 3958   constraint(ALLOC_IN_RC(ptr_reg));
 3959   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3960   match(AddP (AddP reg (ConvI2L idx)) off);
 3961 
 3962   op_cost(10);
 3963   format %{"[$reg + $off + $idx]" %}
 3964   interface(MEMORY_INTER) %{
 3965     base($reg);
 3966     index($idx);
 3967     scale(0x0);
 3968     disp($off);
 3969   %}
 3970 %}
 3971 
 3972 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3973 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3974 %{
 3975   constraint(ALLOC_IN_RC(ptr_reg));
 3976   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3977   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3978 
 3979   op_cost(10);
 3980   format %{"[$reg + $off + $idx << $scale]" %}
 3981   interface(MEMORY_INTER) %{
 3982     base($reg);
 3983     index($idx);
 3984     scale($scale);
 3985     disp($off);
 3986   %}
 3987 %}
 3988 
 3989 // Indirect Narrow Oop Plus Offset Operand
 3990 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3991 // we can't free r12 even with CompressedOops::base() == NULL.
 3992 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3993   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3994   constraint(ALLOC_IN_RC(ptr_reg));
 3995   match(AddP (DecodeN reg) off);
 3996 
 3997   op_cost(10);
 3998   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3999   interface(MEMORY_INTER) %{
 4000     base(0xc); // R12
 4001     index($reg);
 4002     scale(0x3);
 4003     disp($off);
 4004   %}
 4005 %}
 4006 
 4007 // Indirect Memory Operand
 4008 operand indirectNarrow(rRegN reg)
 4009 %{
 4010   predicate(CompressedOops::shift() == 0);
 4011   constraint(ALLOC_IN_RC(ptr_reg));
 4012   match(DecodeN reg);
 4013 
 4014   format %{ "[$reg]" %}
 4015   interface(MEMORY_INTER) %{
 4016     base($reg);
 4017     index(0x4);
 4018     scale(0x0);
 4019     disp(0x0);
 4020   %}
 4021 %}
 4022 
 4023 // Indirect Memory Plus Short Offset Operand
 4024 operand indOffset8Narrow(rRegN reg, immL8 off)
 4025 %{
 4026   predicate(CompressedOops::shift() == 0);
 4027   constraint(ALLOC_IN_RC(ptr_reg));
 4028   match(AddP (DecodeN reg) off);
 4029 
 4030   format %{ "[$reg + $off (8-bit)]" %}
 4031   interface(MEMORY_INTER) %{
 4032     base($reg);
 4033     index(0x4);
 4034     scale(0x0);
 4035     disp($off);
 4036   %}
 4037 %}
 4038 
 4039 // Indirect Memory Plus Long Offset Operand
 4040 operand indOffset32Narrow(rRegN reg, immL32 off)
 4041 %{
 4042   predicate(CompressedOops::shift() == 0);
 4043   constraint(ALLOC_IN_RC(ptr_reg));
 4044   match(AddP (DecodeN reg) off);
 4045 
 4046   format %{ "[$reg + $off (32-bit)]" %}
 4047   interface(MEMORY_INTER) %{
 4048     base($reg);
 4049     index(0x4);
 4050     scale(0x0);
 4051     disp($off);
 4052   %}
 4053 %}
 4054 
 4055 // Indirect Memory Plus Index Register Plus Offset Operand
 4056 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4057 %{
 4058   predicate(CompressedOops::shift() == 0);
 4059   constraint(ALLOC_IN_RC(ptr_reg));
 4060   match(AddP (AddP (DecodeN reg) lreg) off);
 4061 
 4062   op_cost(10);
 4063   format %{"[$reg + $off + $lreg]" %}
 4064   interface(MEMORY_INTER) %{
 4065     base($reg);
 4066     index($lreg);
 4067     scale(0x0);
 4068     disp($off);
 4069   %}
 4070 %}
 4071 
 4072 // Indirect Memory Plus Index Register Plus Offset Operand
 4073 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4074 %{
 4075   predicate(CompressedOops::shift() == 0);
 4076   constraint(ALLOC_IN_RC(ptr_reg));
 4077   match(AddP (DecodeN reg) lreg);
 4078 
 4079   op_cost(10);
 4080   format %{"[$reg + $lreg]" %}
 4081   interface(MEMORY_INTER) %{
 4082     base($reg);
 4083     index($lreg);
 4084     scale(0x0);
 4085     disp(0x0);
 4086   %}
 4087 %}
 4088 
 4089 // Indirect Memory Times Scale Plus Index Register
 4090 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4091 %{
 4092   predicate(CompressedOops::shift() == 0);
 4093   constraint(ALLOC_IN_RC(ptr_reg));
 4094   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4095 
 4096   op_cost(10);
 4097   format %{"[$reg + $lreg << $scale]" %}
 4098   interface(MEMORY_INTER) %{
 4099     base($reg);
 4100     index($lreg);
 4101     scale($scale);
 4102     disp(0x0);
 4103   %}
 4104 %}
 4105 
 4106 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4107 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4108 %{
 4109   predicate(CompressedOops::shift() == 0);
 4110   constraint(ALLOC_IN_RC(ptr_reg));
 4111   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4112 
 4113   op_cost(10);
 4114   format %{"[$reg + $off + $lreg << $scale]" %}
 4115   interface(MEMORY_INTER) %{
 4116     base($reg);
 4117     index($lreg);
 4118     scale($scale);
 4119     disp($off);
 4120   %}
 4121 %}
 4122 
 4123 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4124 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4125 %{
 4126   constraint(ALLOC_IN_RC(ptr_reg));
 4127   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4128   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4129 
 4130   op_cost(10);
 4131   format %{"[$reg + $off + $idx]" %}
 4132   interface(MEMORY_INTER) %{
 4133     base($reg);
 4134     index($idx);
 4135     scale(0x0);
 4136     disp($off);
 4137   %}
 4138 %}
 4139 
 4140 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4141 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4142 %{
 4143   constraint(ALLOC_IN_RC(ptr_reg));
 4144   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4145   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4146 
 4147   op_cost(10);
 4148   format %{"[$reg + $off + $idx << $scale]" %}
 4149   interface(MEMORY_INTER) %{
 4150     base($reg);
 4151     index($idx);
 4152     scale($scale);
 4153     disp($off);
 4154   %}
 4155 %}
 4156 
 4157 //----------Special Memory Operands--------------------------------------------
 4158 // Stack Slot Operand - This operand is used for loading and storing temporary
 4159 //                      values on the stack where a match requires a value to
 4160 //                      flow through memory.
 4161 operand stackSlotP(sRegP reg)
 4162 %{
 4163   constraint(ALLOC_IN_RC(stack_slots));
 4164   // No match rule because this operand is only generated in matching
 4165 
 4166   format %{ "[$reg]" %}
 4167   interface(MEMORY_INTER) %{
 4168     base(0x4);   // RSP
 4169     index(0x4);  // No Index
 4170     scale(0x0);  // No Scale
 4171     disp($reg);  // Stack Offset
 4172   %}
 4173 %}
 4174 
 4175 operand stackSlotI(sRegI reg)
 4176 %{
 4177   constraint(ALLOC_IN_RC(stack_slots));
 4178   // No match rule because this operand is only generated in matching
 4179 
 4180   format %{ "[$reg]" %}
 4181   interface(MEMORY_INTER) %{
 4182     base(0x4);   // RSP
 4183     index(0x4);  // No Index
 4184     scale(0x0);  // No Scale
 4185     disp($reg);  // Stack Offset
 4186   %}
 4187 %}
 4188 
 4189 operand stackSlotF(sRegF reg)
 4190 %{
 4191   constraint(ALLOC_IN_RC(stack_slots));
 4192   // No match rule because this operand is only generated in matching
 4193 
 4194   format %{ "[$reg]" %}
 4195   interface(MEMORY_INTER) %{
 4196     base(0x4);   // RSP
 4197     index(0x4);  // No Index
 4198     scale(0x0);  // No Scale
 4199     disp($reg);  // Stack Offset
 4200   %}
 4201 %}
 4202 
 4203 operand stackSlotD(sRegD reg)
 4204 %{
 4205   constraint(ALLOC_IN_RC(stack_slots));
 4206   // No match rule because this operand is only generated in matching
 4207 
 4208   format %{ "[$reg]" %}
 4209   interface(MEMORY_INTER) %{
 4210     base(0x4);   // RSP
 4211     index(0x4);  // No Index
 4212     scale(0x0);  // No Scale
 4213     disp($reg);  // Stack Offset
 4214   %}
 4215 %}
 4216 operand stackSlotL(sRegL reg)
 4217 %{
 4218   constraint(ALLOC_IN_RC(stack_slots));
 4219   // No match rule because this operand is only generated in matching
 4220 
 4221   format %{ "[$reg]" %}
 4222   interface(MEMORY_INTER) %{
 4223     base(0x4);   // RSP
 4224     index(0x4);  // No Index
 4225     scale(0x0);  // No Scale
 4226     disp($reg);  // Stack Offset
 4227   %}
 4228 %}
 4229 
 4230 //----------Conditional Branch Operands----------------------------------------
 4231 // Comparison Op  - This is the operation of the comparison, and is limited to
 4232 //                  the following set of codes:
 4233 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4234 //
 4235 // Other attributes of the comparison, such as unsignedness, are specified
 4236 // by the comparison instruction that sets a condition code flags register.
 4237 // That result is represented by a flags operand whose subtype is appropriate
 4238 // to the unsignedness (etc.) of the comparison.
 4239 //
 4240 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4241 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4242 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4243 
 4244 // Comparison Code
 4245 operand cmpOp()
 4246 %{
 4247   match(Bool);
 4248 
 4249   format %{ "" %}
 4250   interface(COND_INTER) %{
 4251     equal(0x4, "e");
 4252     not_equal(0x5, "ne");
 4253     less(0xC, "l");
 4254     greater_equal(0xD, "ge");
 4255     less_equal(0xE, "le");
 4256     greater(0xF, "g");
 4257     overflow(0x0, "o");
 4258     no_overflow(0x1, "no");
 4259   %}
 4260 %}
 4261 
 4262 // Comparison Code, unsigned compare.  Used by FP also, with
 4263 // C2 (unordered) turned into GT or LT already.  The other bits
 4264 // C0 and C3 are turned into Carry & Zero flags.
 4265 operand cmpOpU()
 4266 %{
 4267   match(Bool);
 4268 
 4269   format %{ "" %}
 4270   interface(COND_INTER) %{
 4271     equal(0x4, "e");
 4272     not_equal(0x5, "ne");
 4273     less(0x2, "b");
 4274     greater_equal(0x3, "ae");
 4275     less_equal(0x6, "be");
 4276     greater(0x7, "a");
 4277     overflow(0x0, "o");
 4278     no_overflow(0x1, "no");
 4279   %}
 4280 %}
 4281 
 4282 
 4283 // Floating comparisons that don't require any fixup for the unordered case,
 4284 // If both inputs of the comparison are the same, ZF is always set so we
 4285 // don't need to use cmpOpUCF2 for eq/ne
 4286 operand cmpOpUCF() %{
 4287   match(Bool);
 4288   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4289             n->as_Bool()->_test._test == BoolTest::ge ||
 4290             n->as_Bool()->_test._test == BoolTest::le ||
 4291             n->as_Bool()->_test._test == BoolTest::gt ||
 4292             n->in(1)->in(1) == n->in(1)->in(2));
 4293   format %{ "" %}
 4294   interface(COND_INTER) %{
 4295     equal(0xb, "np");
 4296     not_equal(0xa, "p");
 4297     less(0x2, "b");
 4298     greater_equal(0x3, "ae");
 4299     less_equal(0x6, "be");
 4300     greater(0x7, "a");
 4301     overflow(0x0, "o");
 4302     no_overflow(0x1, "no");
 4303   %}
 4304 %}
 4305 
 4306 
 4307 // Floating comparisons that can be fixed up with extra conditional jumps
 4308 operand cmpOpUCF2() %{
 4309   match(Bool);
 4310   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 4311              n->as_Bool()->_test._test == BoolTest::eq) &&
 4312             n->in(1)->in(1) != n->in(1)->in(2));
 4313   format %{ "" %}
 4314   interface(COND_INTER) %{
 4315     equal(0x4, "e");
 4316     not_equal(0x5, "ne");
 4317     less(0x2, "b");
 4318     greater_equal(0x3, "ae");
 4319     less_equal(0x6, "be");
 4320     greater(0x7, "a");
 4321     overflow(0x0, "o");
 4322     no_overflow(0x1, "no");
 4323   %}
 4324 %}
 4325 
 4326 //----------OPERAND CLASSES----------------------------------------------------
 4327 // Operand Classes are groups of operands that are used as to simplify
 4328 // instruction definitions by not requiring the AD writer to specify separate
 4329 // instructions for every form of operand when the instruction accepts
 4330 // multiple operand types with the same basic encoding and format.  The classic
 4331 // case of this is memory operands.
 4332 
 4333 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4334                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4335                indCompressedOopOffset,
 4336                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4337                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4338                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4339 
 4340 //----------PIPELINE-----------------------------------------------------------
 4341 // Rules which define the behavior of the target architectures pipeline.
 4342 pipeline %{
 4343 
 4344 //----------ATTRIBUTES---------------------------------------------------------
 4345 attributes %{
 4346   variable_size_instructions;        // Fixed size instructions
 4347   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4348   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4349   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4350   instruction_fetch_units = 1;       // of 16 bytes
 4351 
 4352   // List of nop instructions
 4353   nops( MachNop );
 4354 %}
 4355 
 4356 //----------RESOURCES----------------------------------------------------------
 4357 // Resources are the functional units available to the machine
 4358 
 4359 // Generic P2/P3 pipeline
 4360 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4361 // 3 instructions decoded per cycle.
 4362 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4363 // 3 ALU op, only ALU0 handles mul instructions.
 4364 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4365            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4366            BR, FPU,
 4367            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4368 
 4369 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4370 // Pipeline Description specifies the stages in the machine's pipeline
 4371 
 4372 // Generic P2/P3 pipeline
 4373 pipe_desc(S0, S1, S2, S3, S4, S5);
 4374 
 4375 //----------PIPELINE CLASSES---------------------------------------------------
 4376 // Pipeline Classes describe the stages in which input and output are
 4377 // referenced by the hardware pipeline.
 4378 
 4379 // Naming convention: ialu or fpu
 4380 // Then: _reg
 4381 // Then: _reg if there is a 2nd register
 4382 // Then: _long if it's a pair of instructions implementing a long
 4383 // Then: _fat if it requires the big decoder
 4384 //   Or: _mem if it requires the big decoder and a memory unit.
 4385 
 4386 // Integer ALU reg operation
 4387 pipe_class ialu_reg(rRegI dst)
 4388 %{
 4389     single_instruction;
 4390     dst    : S4(write);
 4391     dst    : S3(read);
 4392     DECODE : S0;        // any decoder
 4393     ALU    : S3;        // any alu
 4394 %}
 4395 
 4396 // Long ALU reg operation
 4397 pipe_class ialu_reg_long(rRegL dst)
 4398 %{
 4399     instruction_count(2);
 4400     dst    : S4(write);
 4401     dst    : S3(read);
 4402     DECODE : S0(2);     // any 2 decoders
 4403     ALU    : S3(2);     // both alus
 4404 %}
 4405 
 4406 // Integer ALU reg operation using big decoder
 4407 pipe_class ialu_reg_fat(rRegI dst)
 4408 %{
 4409     single_instruction;
 4410     dst    : S4(write);
 4411     dst    : S3(read);
 4412     D0     : S0;        // big decoder only
 4413     ALU    : S3;        // any alu
 4414 %}
 4415 
 4416 // Integer ALU reg-reg operation
 4417 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4418 %{
 4419     single_instruction;
 4420     dst    : S4(write);
 4421     src    : S3(read);
 4422     DECODE : S0;        // any decoder
 4423     ALU    : S3;        // any alu
 4424 %}
 4425 
 4426 // Integer ALU reg-reg operation
 4427 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4428 %{
 4429     single_instruction;
 4430     dst    : S4(write);
 4431     src    : S3(read);
 4432     D0     : S0;        // big decoder only
 4433     ALU    : S3;        // any alu
 4434 %}
 4435 
 4436 // Integer ALU reg-mem operation
 4437 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4438 %{
 4439     single_instruction;
 4440     dst    : S5(write);
 4441     mem    : S3(read);
 4442     D0     : S0;        // big decoder only
 4443     ALU    : S4;        // any alu
 4444     MEM    : S3;        // any mem
 4445 %}
 4446 
 4447 // Integer mem operation (prefetch)
 4448 pipe_class ialu_mem(memory mem)
 4449 %{
 4450     single_instruction;
 4451     mem    : S3(read);
 4452     D0     : S0;        // big decoder only
 4453     MEM    : S3;        // any mem
 4454 %}
 4455 
 4456 // Integer Store to Memory
 4457 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4458 %{
 4459     single_instruction;
 4460     mem    : S3(read);
 4461     src    : S5(read);
 4462     D0     : S0;        // big decoder only
 4463     ALU    : S4;        // any alu
 4464     MEM    : S3;
 4465 %}
 4466 
 4467 // // Long Store to Memory
 4468 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4469 // %{
 4470 //     instruction_count(2);
 4471 //     mem    : S3(read);
 4472 //     src    : S5(read);
 4473 //     D0     : S0(2);          // big decoder only; twice
 4474 //     ALU    : S4(2);     // any 2 alus
 4475 //     MEM    : S3(2);  // Both mems
 4476 // %}
 4477 
 4478 // Integer Store to Memory
 4479 pipe_class ialu_mem_imm(memory mem)
 4480 %{
 4481     single_instruction;
 4482     mem    : S3(read);
 4483     D0     : S0;        // big decoder only
 4484     ALU    : S4;        // any alu
 4485     MEM    : S3;
 4486 %}
 4487 
 4488 // Integer ALU0 reg-reg operation
 4489 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4490 %{
 4491     single_instruction;
 4492     dst    : S4(write);
 4493     src    : S3(read);
 4494     D0     : S0;        // Big decoder only
 4495     ALU0   : S3;        // only alu0
 4496 %}
 4497 
 4498 // Integer ALU0 reg-mem operation
 4499 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4500 %{
 4501     single_instruction;
 4502     dst    : S5(write);
 4503     mem    : S3(read);
 4504     D0     : S0;        // big decoder only
 4505     ALU0   : S4;        // ALU0 only
 4506     MEM    : S3;        // any mem
 4507 %}
 4508 
 4509 // Integer ALU reg-reg operation
 4510 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4511 %{
 4512     single_instruction;
 4513     cr     : S4(write);
 4514     src1   : S3(read);
 4515     src2   : S3(read);
 4516     DECODE : S0;        // any decoder
 4517     ALU    : S3;        // any alu
 4518 %}
 4519 
 4520 // Integer ALU reg-imm operation
 4521 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4522 %{
 4523     single_instruction;
 4524     cr     : S4(write);
 4525     src1   : S3(read);
 4526     DECODE : S0;        // any decoder
 4527     ALU    : S3;        // any alu
 4528 %}
 4529 
 4530 // Integer ALU reg-mem operation
 4531 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4532 %{
 4533     single_instruction;
 4534     cr     : S4(write);
 4535     src1   : S3(read);
 4536     src2   : S3(read);
 4537     D0     : S0;        // big decoder only
 4538     ALU    : S4;        // any alu
 4539     MEM    : S3;
 4540 %}
 4541 
 4542 // Conditional move reg-reg
 4543 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4544 %{
 4545     instruction_count(4);
 4546     y      : S4(read);
 4547     q      : S3(read);
 4548     p      : S3(read);
 4549     DECODE : S0(4);     // any decoder
 4550 %}
 4551 
 4552 // Conditional move reg-reg
 4553 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4554 %{
 4555     single_instruction;
 4556     dst    : S4(write);
 4557     src    : S3(read);
 4558     cr     : S3(read);
 4559     DECODE : S0;        // any decoder
 4560 %}
 4561 
 4562 // Conditional move reg-mem
 4563 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4564 %{
 4565     single_instruction;
 4566     dst    : S4(write);
 4567     src    : S3(read);
 4568     cr     : S3(read);
 4569     DECODE : S0;        // any decoder
 4570     MEM    : S3;
 4571 %}
 4572 
 4573 // Conditional move reg-reg long
 4574 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4575 %{
 4576     single_instruction;
 4577     dst    : S4(write);
 4578     src    : S3(read);
 4579     cr     : S3(read);
 4580     DECODE : S0(2);     // any 2 decoders
 4581 %}
 4582 
 4583 // XXX
 4584 // // Conditional move double reg-reg
 4585 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4586 // %{
 4587 //     single_instruction;
 4588 //     dst    : S4(write);
 4589 //     src    : S3(read);
 4590 //     cr     : S3(read);
 4591 //     DECODE : S0;     // any decoder
 4592 // %}
 4593 
 4594 // Float reg-reg operation
 4595 pipe_class fpu_reg(regD dst)
 4596 %{
 4597     instruction_count(2);
 4598     dst    : S3(read);
 4599     DECODE : S0(2);     // any 2 decoders
 4600     FPU    : S3;
 4601 %}
 4602 
 4603 // Float reg-reg operation
 4604 pipe_class fpu_reg_reg(regD dst, regD src)
 4605 %{
 4606     instruction_count(2);
 4607     dst    : S4(write);
 4608     src    : S3(read);
 4609     DECODE : S0(2);     // any 2 decoders
 4610     FPU    : S3;
 4611 %}
 4612 
 4613 // Float reg-reg operation
 4614 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4615 %{
 4616     instruction_count(3);
 4617     dst    : S4(write);
 4618     src1   : S3(read);
 4619     src2   : S3(read);
 4620     DECODE : S0(3);     // any 3 decoders
 4621     FPU    : S3(2);
 4622 %}
 4623 
 4624 // Float reg-reg operation
 4625 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4626 %{
 4627     instruction_count(4);
 4628     dst    : S4(write);
 4629     src1   : S3(read);
 4630     src2   : S3(read);
 4631     src3   : S3(read);
 4632     DECODE : S0(4);     // any 3 decoders
 4633     FPU    : S3(2);
 4634 %}
 4635 
 4636 // Float reg-reg operation
 4637 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4638 %{
 4639     instruction_count(4);
 4640     dst    : S4(write);
 4641     src1   : S3(read);
 4642     src2   : S3(read);
 4643     src3   : S3(read);
 4644     DECODE : S1(3);     // any 3 decoders
 4645     D0     : S0;        // Big decoder only
 4646     FPU    : S3(2);
 4647     MEM    : S3;
 4648 %}
 4649 
 4650 // Float reg-mem operation
 4651 pipe_class fpu_reg_mem(regD dst, memory mem)
 4652 %{
 4653     instruction_count(2);
 4654     dst    : S5(write);
 4655     mem    : S3(read);
 4656     D0     : S0;        // big decoder only
 4657     DECODE : S1;        // any decoder for FPU POP
 4658     FPU    : S4;
 4659     MEM    : S3;        // any mem
 4660 %}
 4661 
 4662 // Float reg-mem operation
 4663 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4664 %{
 4665     instruction_count(3);
 4666     dst    : S5(write);
 4667     src1   : S3(read);
 4668     mem    : S3(read);
 4669     D0     : S0;        // big decoder only
 4670     DECODE : S1(2);     // any decoder for FPU POP
 4671     FPU    : S4;
 4672     MEM    : S3;        // any mem
 4673 %}
 4674 
 4675 // Float mem-reg operation
 4676 pipe_class fpu_mem_reg(memory mem, regD src)
 4677 %{
 4678     instruction_count(2);
 4679     src    : S5(read);
 4680     mem    : S3(read);
 4681     DECODE : S0;        // any decoder for FPU PUSH
 4682     D0     : S1;        // big decoder only
 4683     FPU    : S4;
 4684     MEM    : S3;        // any mem
 4685 %}
 4686 
 4687 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4688 %{
 4689     instruction_count(3);
 4690     src1   : S3(read);
 4691     src2   : S3(read);
 4692     mem    : S3(read);
 4693     DECODE : S0(2);     // any decoder for FPU PUSH
 4694     D0     : S1;        // big decoder only
 4695     FPU    : S4;
 4696     MEM    : S3;        // any mem
 4697 %}
 4698 
 4699 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4700 %{
 4701     instruction_count(3);
 4702     src1   : S3(read);
 4703     src2   : S3(read);
 4704     mem    : S4(read);
 4705     DECODE : S0;        // any decoder for FPU PUSH
 4706     D0     : S0(2);     // big decoder only
 4707     FPU    : S4;
 4708     MEM    : S3(2);     // any mem
 4709 %}
 4710 
 4711 pipe_class fpu_mem_mem(memory dst, memory src1)
 4712 %{
 4713     instruction_count(2);
 4714     src1   : S3(read);
 4715     dst    : S4(read);
 4716     D0     : S0(2);     // big decoder only
 4717     MEM    : S3(2);     // any mem
 4718 %}
 4719 
 4720 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4721 %{
 4722     instruction_count(3);
 4723     src1   : S3(read);
 4724     src2   : S3(read);
 4725     dst    : S4(read);
 4726     D0     : S0(3);     // big decoder only
 4727     FPU    : S4;
 4728     MEM    : S3(3);     // any mem
 4729 %}
 4730 
 4731 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4732 %{
 4733     instruction_count(3);
 4734     src1   : S4(read);
 4735     mem    : S4(read);
 4736     DECODE : S0;        // any decoder for FPU PUSH
 4737     D0     : S0(2);     // big decoder only
 4738     FPU    : S4;
 4739     MEM    : S3(2);     // any mem
 4740 %}
 4741 
 4742 // Float load constant
 4743 pipe_class fpu_reg_con(regD dst)
 4744 %{
 4745     instruction_count(2);
 4746     dst    : S5(write);
 4747     D0     : S0;        // big decoder only for the load
 4748     DECODE : S1;        // any decoder for FPU POP
 4749     FPU    : S4;
 4750     MEM    : S3;        // any mem
 4751 %}
 4752 
 4753 // Float load constant
 4754 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4755 %{
 4756     instruction_count(3);
 4757     dst    : S5(write);
 4758     src    : S3(read);
 4759     D0     : S0;        // big decoder only for the load
 4760     DECODE : S1(2);     // any decoder for FPU POP
 4761     FPU    : S4;
 4762     MEM    : S3;        // any mem
 4763 %}
 4764 
 4765 // UnConditional branch
 4766 pipe_class pipe_jmp(label labl)
 4767 %{
 4768     single_instruction;
 4769     BR   : S3;
 4770 %}
 4771 
 4772 // Conditional branch
 4773 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4774 %{
 4775     single_instruction;
 4776     cr    : S1(read);
 4777     BR    : S3;
 4778 %}
 4779 
 4780 // Allocation idiom
 4781 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4782 %{
 4783     instruction_count(1); force_serialization;
 4784     fixed_latency(6);
 4785     heap_ptr : S3(read);
 4786     DECODE   : S0(3);
 4787     D0       : S2;
 4788     MEM      : S3;
 4789     ALU      : S3(2);
 4790     dst      : S5(write);
 4791     BR       : S5;
 4792 %}
 4793 
 4794 // Generic big/slow expanded idiom
 4795 pipe_class pipe_slow()
 4796 %{
 4797     instruction_count(10); multiple_bundles; force_serialization;
 4798     fixed_latency(100);
 4799     D0  : S0(2);
 4800     MEM : S3(2);
 4801 %}
 4802 
 4803 // The real do-nothing guy
 4804 pipe_class empty()
 4805 %{
 4806     instruction_count(0);
 4807 %}
 4808 
 4809 // Define the class for the Nop node
 4810 define
 4811 %{
 4812    MachNop = empty;
 4813 %}
 4814 
 4815 %}
 4816 
 4817 //----------INSTRUCTIONS-------------------------------------------------------
 4818 //
 4819 // match      -- States which machine-independent subtree may be replaced
 4820 //               by this instruction.
 4821 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4822 //               selection to identify a minimum cost tree of machine
 4823 //               instructions that matches a tree of machine-independent
 4824 //               instructions.
 4825 // format     -- A string providing the disassembly for this instruction.
 4826 //               The value of an instruction's operand may be inserted
 4827 //               by referring to it with a '$' prefix.
 4828 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4829 //               to within an encode class as $primary, $secondary, and $tertiary
 4830 //               rrspectively.  The primary opcode is commonly used to
 4831 //               indicate the type of machine instruction, while secondary
 4832 //               and tertiary are often used for prefix options or addressing
 4833 //               modes.
 4834 // ins_encode -- A list of encode classes with parameters. The encode class
 4835 //               name must have been defined in an 'enc_class' specification
 4836 //               in the encode section of the architecture description.
 4837 
 4838 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4839 // Load Float
 4840 instruct MoveF2VL(vlRegF dst, regF src) %{
 4841   match(Set dst src);
 4842   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4843   ins_encode %{
 4844     ShouldNotReachHere();
 4845   %}
 4846   ins_pipe( fpu_reg_reg );
 4847 %}
 4848 
 4849 // Load Float
 4850 instruct MoveF2LEG(legRegF dst, regF src) %{
 4851   match(Set dst src);
 4852   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4853   ins_encode %{
 4854     ShouldNotReachHere();
 4855   %}
 4856   ins_pipe( fpu_reg_reg );
 4857 %}
 4858 
 4859 // Load Float
 4860 instruct MoveVL2F(regF dst, vlRegF src) %{
 4861   match(Set dst src);
 4862   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4863   ins_encode %{
 4864     ShouldNotReachHere();
 4865   %}
 4866   ins_pipe( fpu_reg_reg );
 4867 %}
 4868 
 4869 // Load Float
 4870 instruct MoveLEG2F(regF dst, legRegF src) %{
 4871   match(Set dst src);
 4872   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4873   ins_encode %{
 4874     ShouldNotReachHere();
 4875   %}
 4876   ins_pipe( fpu_reg_reg );
 4877 %}
 4878 
 4879 // Load Double
 4880 instruct MoveD2VL(vlRegD dst, regD src) %{
 4881   match(Set dst src);
 4882   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4883   ins_encode %{
 4884     ShouldNotReachHere();
 4885   %}
 4886   ins_pipe( fpu_reg_reg );
 4887 %}
 4888 
 4889 // Load Double
 4890 instruct MoveD2LEG(legRegD dst, regD src) %{
 4891   match(Set dst src);
 4892   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4893   ins_encode %{
 4894     ShouldNotReachHere();
 4895   %}
 4896   ins_pipe( fpu_reg_reg );
 4897 %}
 4898 
 4899 // Load Double
 4900 instruct MoveVL2D(regD dst, vlRegD src) %{
 4901   match(Set dst src);
 4902   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4903   ins_encode %{
 4904     ShouldNotReachHere();
 4905   %}
 4906   ins_pipe( fpu_reg_reg );
 4907 %}
 4908 
 4909 // Load Double
 4910 instruct MoveLEG2D(regD dst, legRegD src) %{
 4911   match(Set dst src);
 4912   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4913   ins_encode %{
 4914     ShouldNotReachHere();
 4915   %}
 4916   ins_pipe( fpu_reg_reg );
 4917 %}
 4918 
 4919 //----------Load/Store/Move Instructions---------------------------------------
 4920 //----------Load Instructions--------------------------------------------------
 4921 
 4922 // Load Byte (8 bit signed)
 4923 instruct loadB(rRegI dst, memory mem)
 4924 %{
 4925   match(Set dst (LoadB mem));
 4926 
 4927   ins_cost(125);
 4928   format %{ "movsbl  $dst, $mem\t# byte" %}
 4929 
 4930   ins_encode %{
 4931     __ movsbl($dst$$Register, $mem$$Address);
 4932   %}
 4933 
 4934   ins_pipe(ialu_reg_mem);
 4935 %}
 4936 
 4937 // Load Byte (8 bit signed) into Long Register
 4938 instruct loadB2L(rRegL dst, memory mem)
 4939 %{
 4940   match(Set dst (ConvI2L (LoadB mem)));
 4941 
 4942   ins_cost(125);
 4943   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4944 
 4945   ins_encode %{
 4946     __ movsbq($dst$$Register, $mem$$Address);
 4947   %}
 4948 
 4949   ins_pipe(ialu_reg_mem);
 4950 %}
 4951 
 4952 // Load Unsigned Byte (8 bit UNsigned)
 4953 instruct loadUB(rRegI dst, memory mem)
 4954 %{
 4955   match(Set dst (LoadUB mem));
 4956 
 4957   ins_cost(125);
 4958   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4959 
 4960   ins_encode %{
 4961     __ movzbl($dst$$Register, $mem$$Address);
 4962   %}
 4963 
 4964   ins_pipe(ialu_reg_mem);
 4965 %}
 4966 
 4967 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4968 instruct loadUB2L(rRegL dst, memory mem)
 4969 %{
 4970   match(Set dst (ConvI2L (LoadUB mem)));
 4971 
 4972   ins_cost(125);
 4973   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4974 
 4975   ins_encode %{
 4976     __ movzbq($dst$$Register, $mem$$Address);
 4977   %}
 4978 
 4979   ins_pipe(ialu_reg_mem);
 4980 %}
 4981 
 4982 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4983 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4984   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4985   effect(KILL cr);
 4986 
 4987   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4988             "andl    $dst, right_n_bits($mask, 8)" %}
 4989   ins_encode %{
 4990     Register Rdst = $dst$$Register;
 4991     __ movzbq(Rdst, $mem$$Address);
 4992     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4993   %}
 4994   ins_pipe(ialu_reg_mem);
 4995 %}
 4996 
 4997 // Load Short (16 bit signed)
 4998 instruct loadS(rRegI dst, memory mem)
 4999 %{
 5000   match(Set dst (LoadS mem));
 5001 
 5002   ins_cost(125);
 5003   format %{ "movswl $dst, $mem\t# short" %}
 5004 
 5005   ins_encode %{
 5006     __ movswl($dst$$Register, $mem$$Address);
 5007   %}
 5008 
 5009   ins_pipe(ialu_reg_mem);
 5010 %}
 5011 
 5012 // Load Short (16 bit signed) to Byte (8 bit signed)
 5013 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5014   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5015 
 5016   ins_cost(125);
 5017   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5018   ins_encode %{
 5019     __ movsbl($dst$$Register, $mem$$Address);
 5020   %}
 5021   ins_pipe(ialu_reg_mem);
 5022 %}
 5023 
 5024 // Load Short (16 bit signed) into Long Register
 5025 instruct loadS2L(rRegL dst, memory mem)
 5026 %{
 5027   match(Set dst (ConvI2L (LoadS mem)));
 5028 
 5029   ins_cost(125);
 5030   format %{ "movswq $dst, $mem\t# short -> long" %}
 5031 
 5032   ins_encode %{
 5033     __ movswq($dst$$Register, $mem$$Address);
 5034   %}
 5035 
 5036   ins_pipe(ialu_reg_mem);
 5037 %}
 5038 
 5039 // Load Unsigned Short/Char (16 bit UNsigned)
 5040 instruct loadUS(rRegI dst, memory mem)
 5041 %{
 5042   match(Set dst (LoadUS mem));
 5043 
 5044   ins_cost(125);
 5045   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5046 
 5047   ins_encode %{
 5048     __ movzwl($dst$$Register, $mem$$Address);
 5049   %}
 5050 
 5051   ins_pipe(ialu_reg_mem);
 5052 %}
 5053 
 5054 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5055 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5056   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5057 
 5058   ins_cost(125);
 5059   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5060   ins_encode %{
 5061     __ movsbl($dst$$Register, $mem$$Address);
 5062   %}
 5063   ins_pipe(ialu_reg_mem);
 5064 %}
 5065 
 5066 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5067 instruct loadUS2L(rRegL dst, memory mem)
 5068 %{
 5069   match(Set dst (ConvI2L (LoadUS mem)));
 5070 
 5071   ins_cost(125);
 5072   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5073 
 5074   ins_encode %{
 5075     __ movzwq($dst$$Register, $mem$$Address);
 5076   %}
 5077 
 5078   ins_pipe(ialu_reg_mem);
 5079 %}
 5080 
 5081 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5082 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5083   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5084 
 5085   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5086   ins_encode %{
 5087     __ movzbq($dst$$Register, $mem$$Address);
 5088   %}
 5089   ins_pipe(ialu_reg_mem);
 5090 %}
 5091 
 5092 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5093 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5094   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5095   effect(KILL cr);
 5096 
 5097   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5098             "andl    $dst, right_n_bits($mask, 16)" %}
 5099   ins_encode %{
 5100     Register Rdst = $dst$$Register;
 5101     __ movzwq(Rdst, $mem$$Address);
 5102     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5103   %}
 5104   ins_pipe(ialu_reg_mem);
 5105 %}
 5106 
 5107 // Load Integer
 5108 instruct loadI(rRegI dst, memory mem)
 5109 %{
 5110   match(Set dst (LoadI mem));
 5111 
 5112   ins_cost(125);
 5113   format %{ "movl    $dst, $mem\t# int" %}
 5114 
 5115   ins_encode %{
 5116     __ movl($dst$$Register, $mem$$Address);
 5117   %}
 5118 
 5119   ins_pipe(ialu_reg_mem);
 5120 %}
 5121 
 5122 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5123 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5124   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5125 
 5126   ins_cost(125);
 5127   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5128   ins_encode %{
 5129     __ movsbl($dst$$Register, $mem$$Address);
 5130   %}
 5131   ins_pipe(ialu_reg_mem);
 5132 %}
 5133 
 5134 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5135 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5136   match(Set dst (AndI (LoadI mem) mask));
 5137 
 5138   ins_cost(125);
 5139   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5140   ins_encode %{
 5141     __ movzbl($dst$$Register, $mem$$Address);
 5142   %}
 5143   ins_pipe(ialu_reg_mem);
 5144 %}
 5145 
 5146 // Load Integer (32 bit signed) to Short (16 bit signed)
 5147 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5148   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5149 
 5150   ins_cost(125);
 5151   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5152   ins_encode %{
 5153     __ movswl($dst$$Register, $mem$$Address);
 5154   %}
 5155   ins_pipe(ialu_reg_mem);
 5156 %}
 5157 
 5158 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5159 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5160   match(Set dst (AndI (LoadI mem) mask));
 5161 
 5162   ins_cost(125);
 5163   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5164   ins_encode %{
 5165     __ movzwl($dst$$Register, $mem$$Address);
 5166   %}
 5167   ins_pipe(ialu_reg_mem);
 5168 %}
 5169 
 5170 // Load Integer into Long Register
 5171 instruct loadI2L(rRegL dst, memory mem)
 5172 %{
 5173   match(Set dst (ConvI2L (LoadI mem)));
 5174 
 5175   ins_cost(125);
 5176   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5177 
 5178   ins_encode %{
 5179     __ movslq($dst$$Register, $mem$$Address);
 5180   %}
 5181 
 5182   ins_pipe(ialu_reg_mem);
 5183 %}
 5184 
 5185 // Load Integer with mask 0xFF into Long Register
 5186 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5187   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5188 
 5189   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5190   ins_encode %{
 5191     __ movzbq($dst$$Register, $mem$$Address);
 5192   %}
 5193   ins_pipe(ialu_reg_mem);
 5194 %}
 5195 
 5196 // Load Integer with mask 0xFFFF into Long Register
 5197 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5198   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5199 
 5200   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5201   ins_encode %{
 5202     __ movzwq($dst$$Register, $mem$$Address);
 5203   %}
 5204   ins_pipe(ialu_reg_mem);
 5205 %}
 5206 
 5207 // Load Integer with a 31-bit mask into Long Register
 5208 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5209   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5210   effect(KILL cr);
 5211 
 5212   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5213             "andl    $dst, $mask" %}
 5214   ins_encode %{
 5215     Register Rdst = $dst$$Register;
 5216     __ movl(Rdst, $mem$$Address);
 5217     __ andl(Rdst, $mask$$constant);
 5218   %}
 5219   ins_pipe(ialu_reg_mem);
 5220 %}
 5221 
 5222 // Load Unsigned Integer into Long Register
 5223 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5224 %{
 5225   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5226 
 5227   ins_cost(125);
 5228   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5229 
 5230   ins_encode %{
 5231     __ movl($dst$$Register, $mem$$Address);
 5232   %}
 5233 
 5234   ins_pipe(ialu_reg_mem);
 5235 %}
 5236 
 5237 // Load Long
 5238 instruct loadL(rRegL dst, memory mem)
 5239 %{
 5240   match(Set dst (LoadL mem));
 5241 
 5242   ins_cost(125);
 5243   format %{ "movq    $dst, $mem\t# long" %}
 5244 
 5245   ins_encode %{
 5246     __ movq($dst$$Register, $mem$$Address);
 5247   %}
 5248 
 5249   ins_pipe(ialu_reg_mem); // XXX
 5250 %}
 5251 
 5252 // Load Range
 5253 instruct loadRange(rRegI dst, memory mem)
 5254 %{
 5255   match(Set dst (LoadRange mem));
 5256 
 5257   ins_cost(125); // XXX
 5258   format %{ "movl    $dst, $mem\t# range" %}
 5259   ins_encode %{
 5260     __ movl($dst$$Register, $mem$$Address);
 5261   %}
 5262   ins_pipe(ialu_reg_mem);
 5263 %}
 5264 
 5265 // Load Pointer
 5266 instruct loadP(rRegP dst, memory mem)
 5267 %{
 5268   match(Set dst (LoadP mem));
 5269   predicate(n->as_Load()->barrier_data() == 0);
 5270 
 5271   ins_cost(125); // XXX
 5272   format %{ "movq    $dst, $mem\t# ptr" %}
 5273   ins_encode %{
 5274     __ movq($dst$$Register, $mem$$Address);
 5275   %}
 5276   ins_pipe(ialu_reg_mem); // XXX
 5277 %}
 5278 
 5279 // Load Compressed Pointer
 5280 instruct loadN(rRegN dst, memory mem)
 5281 %{
 5282    match(Set dst (LoadN mem));
 5283 
 5284    ins_cost(125); // XXX
 5285    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5286    ins_encode %{
 5287      __ movl($dst$$Register, $mem$$Address);
 5288    %}
 5289    ins_pipe(ialu_reg_mem); // XXX
 5290 %}
 5291 
 5292 
 5293 // Load Klass Pointer
 5294 instruct loadKlass(rRegP dst, memory mem)
 5295 %{
 5296   match(Set dst (LoadKlass mem));
 5297 
 5298   ins_cost(125); // XXX
 5299   format %{ "movq    $dst, $mem\t# class" %}
 5300   ins_encode %{
 5301     __ movq($dst$$Register, $mem$$Address);
 5302   %}
 5303   ins_pipe(ialu_reg_mem); // XXX
 5304 %}
 5305 
 5306 // Load narrow Klass Pointer
 5307 instruct loadNKlass(rRegN dst, memory mem)
 5308 %{
 5309   match(Set dst (LoadNKlass mem));
 5310 
 5311   ins_cost(125); // XXX
 5312   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5313   ins_encode %{
 5314     __ movl($dst$$Register, $mem$$Address);
 5315   %}
 5316   ins_pipe(ialu_reg_mem); // XXX
 5317 %}
 5318 
 5319 // Load Float
 5320 instruct loadF(regF dst, memory mem)
 5321 %{
 5322   match(Set dst (LoadF mem));
 5323 
 5324   ins_cost(145); // XXX
 5325   format %{ "movss   $dst, $mem\t# float" %}
 5326   ins_encode %{
 5327     __ movflt($dst$$XMMRegister, $mem$$Address);
 5328   %}
 5329   ins_pipe(pipe_slow); // XXX
 5330 %}
 5331 
 5332 // Load Double
 5333 instruct loadD_partial(regD dst, memory mem)
 5334 %{
 5335   predicate(!UseXmmLoadAndClearUpper);
 5336   match(Set dst (LoadD mem));
 5337 
 5338   ins_cost(145); // XXX
 5339   format %{ "movlpd  $dst, $mem\t# double" %}
 5340   ins_encode %{
 5341     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5342   %}
 5343   ins_pipe(pipe_slow); // XXX
 5344 %}
 5345 
 5346 instruct loadD(regD dst, memory mem)
 5347 %{
 5348   predicate(UseXmmLoadAndClearUpper);
 5349   match(Set dst (LoadD mem));
 5350 
 5351   ins_cost(145); // XXX
 5352   format %{ "movsd   $dst, $mem\t# double" %}
 5353   ins_encode %{
 5354     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5355   %}
 5356   ins_pipe(pipe_slow); // XXX
 5357 %}
 5358 
 5359 
 5360 // Following pseudo code describes the algorithm for max[FD]:
 5361 // Min algorithm is on similar lines
 5362 //  btmp = (b < +0.0) ? a : b
 5363 //  atmp = (b < +0.0) ? b : a
 5364 //  Tmp  = Max_Float(atmp , btmp)
 5365 //  Res  = (atmp == NaN) ? atmp : Tmp
 5366 
 5367 // max = java.lang.Math.max(float a, float b)
 5368 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5369   predicate(UseAVX > 0 && !n->is_reduction());
 5370   match(Set dst (MaxF a b));
 5371   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5372   format %{
 5373      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5374      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5375      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5376      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5377      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5378   %}
 5379   ins_encode %{
 5380     int vector_len = Assembler::AVX_128bit;
 5381     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5382     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5383     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5384     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5385     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5386  %}
 5387   ins_pipe( pipe_slow );
 5388 %}
 5389 
 5390 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5391   predicate(UseAVX > 0 && n->is_reduction());
 5392   match(Set dst (MaxF a b));
 5393   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5394 
 5395   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5396   ins_encode %{
 5397     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5398                     false /*min*/, true /*single*/);
 5399   %}
 5400   ins_pipe( pipe_slow );
 5401 %}
 5402 
 5403 // max = java.lang.Math.max(double a, double b)
 5404 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5405   predicate(UseAVX > 0 && !n->is_reduction());
 5406   match(Set dst (MaxD a b));
 5407   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5408   format %{
 5409      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5410      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5411      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5412      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5413      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5414   %}
 5415   ins_encode %{
 5416     int vector_len = Assembler::AVX_128bit;
 5417     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5418     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5419     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5420     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5421     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5422   %}
 5423   ins_pipe( pipe_slow );
 5424 %}
 5425 
 5426 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5427   predicate(UseAVX > 0 && n->is_reduction());
 5428   match(Set dst (MaxD a b));
 5429   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5430 
 5431   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5432   ins_encode %{
 5433     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5434                     false /*min*/, false /*single*/);
 5435   %}
 5436   ins_pipe( pipe_slow );
 5437 %}
 5438 
 5439 // min = java.lang.Math.min(float a, float b)
 5440 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5441   predicate(UseAVX > 0 && !n->is_reduction());
 5442   match(Set dst (MinF a b));
 5443   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5444   format %{
 5445      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5446      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5447      "vminss           $tmp,$atmp,$btmp           \n\t"
 5448      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5449      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5450   %}
 5451   ins_encode %{
 5452     int vector_len = Assembler::AVX_128bit;
 5453     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5454     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5455     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5456     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5457     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5458   %}
 5459   ins_pipe( pipe_slow );
 5460 %}
 5461 
 5462 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5463   predicate(UseAVX > 0 && n->is_reduction());
 5464   match(Set dst (MinF a b));
 5465   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5466 
 5467   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5468   ins_encode %{
 5469     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5470                     true /*min*/, true /*single*/);
 5471   %}
 5472   ins_pipe( pipe_slow );
 5473 %}
 5474 
 5475 // min = java.lang.Math.min(double a, double b)
 5476 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5477   predicate(UseAVX > 0 && !n->is_reduction());
 5478   match(Set dst (MinD a b));
 5479   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5480   format %{
 5481      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5482      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5483      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5484      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5485      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5486   %}
 5487   ins_encode %{
 5488     int vector_len = Assembler::AVX_128bit;
 5489     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5490     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5491     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5492     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5493     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5494   %}
 5495   ins_pipe( pipe_slow );
 5496 %}
 5497 
 5498 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5499   predicate(UseAVX > 0 && n->is_reduction());
 5500   match(Set dst (MinD a b));
 5501   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5502 
 5503   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5504   ins_encode %{
 5505     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5506                     true /*min*/, false /*single*/);
 5507   %}
 5508   ins_pipe( pipe_slow );
 5509 %}
 5510 
 5511 // Load Effective Address
 5512 instruct leaP8(rRegP dst, indOffset8 mem)
 5513 %{
 5514   match(Set dst mem);
 5515 
 5516   ins_cost(110); // XXX
 5517   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5518   ins_encode %{
 5519     __ leaq($dst$$Register, $mem$$Address);
 5520   %}
 5521   ins_pipe(ialu_reg_reg_fat);
 5522 %}
 5523 
 5524 instruct leaP32(rRegP dst, indOffset32 mem)
 5525 %{
 5526   match(Set dst mem);
 5527 
 5528   ins_cost(110);
 5529   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5530   ins_encode %{
 5531     __ leaq($dst$$Register, $mem$$Address);
 5532   %}
 5533   ins_pipe(ialu_reg_reg_fat);
 5534 %}
 5535 
 5536 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5537 %{
 5538   match(Set dst mem);
 5539 
 5540   ins_cost(110);
 5541   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5542   ins_encode %{
 5543     __ leaq($dst$$Register, $mem$$Address);
 5544   %}
 5545   ins_pipe(ialu_reg_reg_fat);
 5546 %}
 5547 
 5548 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5549 %{
 5550   match(Set dst mem);
 5551 
 5552   ins_cost(110);
 5553   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5554   ins_encode %{
 5555     __ leaq($dst$$Register, $mem$$Address);
 5556   %}
 5557   ins_pipe(ialu_reg_reg_fat);
 5558 %}
 5559 
 5560 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5561 %{
 5562   match(Set dst mem);
 5563 
 5564   ins_cost(110);
 5565   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5566   ins_encode %{
 5567     __ leaq($dst$$Register, $mem$$Address);
 5568   %}
 5569   ins_pipe(ialu_reg_reg_fat);
 5570 %}
 5571 
 5572 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5573 %{
 5574   match(Set dst mem);
 5575 
 5576   ins_cost(110);
 5577   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5578   ins_encode %{
 5579     __ leaq($dst$$Register, $mem$$Address);
 5580   %}
 5581   ins_pipe(ialu_reg_reg_fat);
 5582 %}
 5583 
 5584 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5585 %{
 5586   match(Set dst mem);
 5587 
 5588   ins_cost(110);
 5589   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5590   ins_encode %{
 5591     __ leaq($dst$$Register, $mem$$Address);
 5592   %}
 5593   ins_pipe(ialu_reg_reg_fat);
 5594 %}
 5595 
 5596 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5597 %{
 5598   match(Set dst mem);
 5599 
 5600   ins_cost(110);
 5601   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5602   ins_encode %{
 5603     __ leaq($dst$$Register, $mem$$Address);
 5604   %}
 5605   ins_pipe(ialu_reg_reg_fat);
 5606 %}
 5607 
 5608 // Load Effective Address which uses Narrow (32-bits) oop
 5609 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5610 %{
 5611   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5612   match(Set dst mem);
 5613 
 5614   ins_cost(110);
 5615   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5616   ins_encode %{
 5617     __ leaq($dst$$Register, $mem$$Address);
 5618   %}
 5619   ins_pipe(ialu_reg_reg_fat);
 5620 %}
 5621 
 5622 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5623 %{
 5624   predicate(CompressedOops::shift() == 0);
 5625   match(Set dst mem);
 5626 
 5627   ins_cost(110); // XXX
 5628   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5629   ins_encode %{
 5630     __ leaq($dst$$Register, $mem$$Address);
 5631   %}
 5632   ins_pipe(ialu_reg_reg_fat);
 5633 %}
 5634 
 5635 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5636 %{
 5637   predicate(CompressedOops::shift() == 0);
 5638   match(Set dst mem);
 5639 
 5640   ins_cost(110);
 5641   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5642   ins_encode %{
 5643     __ leaq($dst$$Register, $mem$$Address);
 5644   %}
 5645   ins_pipe(ialu_reg_reg_fat);
 5646 %}
 5647 
 5648 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5649 %{
 5650   predicate(CompressedOops::shift() == 0);
 5651   match(Set dst mem);
 5652 
 5653   ins_cost(110);
 5654   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5655   ins_encode %{
 5656     __ leaq($dst$$Register, $mem$$Address);
 5657   %}
 5658   ins_pipe(ialu_reg_reg_fat);
 5659 %}
 5660 
 5661 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5662 %{
 5663   predicate(CompressedOops::shift() == 0);
 5664   match(Set dst mem);
 5665 
 5666   ins_cost(110);
 5667   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5668   ins_encode %{
 5669     __ leaq($dst$$Register, $mem$$Address);
 5670   %}
 5671   ins_pipe(ialu_reg_reg_fat);
 5672 %}
 5673 
 5674 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5675 %{
 5676   predicate(CompressedOops::shift() == 0);
 5677   match(Set dst mem);
 5678 
 5679   ins_cost(110);
 5680   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5681   ins_encode %{
 5682     __ leaq($dst$$Register, $mem$$Address);
 5683   %}
 5684   ins_pipe(ialu_reg_reg_fat);
 5685 %}
 5686 
 5687 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5688 %{
 5689   predicate(CompressedOops::shift() == 0);
 5690   match(Set dst mem);
 5691 
 5692   ins_cost(110);
 5693   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5694   ins_encode %{
 5695     __ leaq($dst$$Register, $mem$$Address);
 5696   %}
 5697   ins_pipe(ialu_reg_reg_fat);
 5698 %}
 5699 
 5700 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5701 %{
 5702   predicate(CompressedOops::shift() == 0);
 5703   match(Set dst mem);
 5704 
 5705   ins_cost(110);
 5706   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5707   ins_encode %{
 5708     __ leaq($dst$$Register, $mem$$Address);
 5709   %}
 5710   ins_pipe(ialu_reg_reg_fat);
 5711 %}
 5712 
 5713 instruct loadConI(rRegI dst, immI src)
 5714 %{
 5715   match(Set dst src);
 5716 
 5717   format %{ "movl    $dst, $src\t# int" %}
 5718   ins_encode %{
 5719     __ movl($dst$$Register, $src$$constant);
 5720   %}
 5721   ins_pipe(ialu_reg_fat); // XXX
 5722 %}
 5723 
 5724 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5725 %{
 5726   match(Set dst src);
 5727   effect(KILL cr);
 5728 
 5729   ins_cost(50);
 5730   format %{ "xorl    $dst, $dst\t# int" %}
 5731   ins_encode %{
 5732     __ xorl($dst$$Register, $dst$$Register);
 5733   %}
 5734   ins_pipe(ialu_reg);
 5735 %}
 5736 
 5737 instruct loadConL(rRegL dst, immL src)
 5738 %{
 5739   match(Set dst src);
 5740 
 5741   ins_cost(150);
 5742   format %{ "movq    $dst, $src\t# long" %}
 5743   ins_encode %{
 5744     __ mov64($dst$$Register, $src$$constant);
 5745   %}
 5746   ins_pipe(ialu_reg);
 5747 %}
 5748 
 5749 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5750 %{
 5751   match(Set dst src);
 5752   effect(KILL cr);
 5753 
 5754   ins_cost(50);
 5755   format %{ "xorl    $dst, $dst\t# long" %}
 5756   ins_encode %{
 5757     __ xorl($dst$$Register, $dst$$Register);
 5758   %}
 5759   ins_pipe(ialu_reg); // XXX
 5760 %}
 5761 
 5762 instruct loadConUL32(rRegL dst, immUL32 src)
 5763 %{
 5764   match(Set dst src);
 5765 
 5766   ins_cost(60);
 5767   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5768   ins_encode %{
 5769     __ movl($dst$$Register, $src$$constant);
 5770   %}
 5771   ins_pipe(ialu_reg);
 5772 %}
 5773 
 5774 instruct loadConL32(rRegL dst, immL32 src)
 5775 %{
 5776   match(Set dst src);
 5777 
 5778   ins_cost(70);
 5779   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5780   ins_encode %{
 5781     __ movq($dst$$Register, $src$$constant);
 5782   %}
 5783   ins_pipe(ialu_reg);
 5784 %}
 5785 
 5786 instruct loadConP(rRegP dst, immP con) %{
 5787   match(Set dst con);
 5788 
 5789   format %{ "movq    $dst, $con\t# ptr" %}
 5790   ins_encode %{
 5791     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5792   %}
 5793   ins_pipe(ialu_reg_fat); // XXX
 5794 %}
 5795 
 5796 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5797 %{
 5798   match(Set dst src);
 5799   effect(KILL cr);
 5800 
 5801   ins_cost(50);
 5802   format %{ "xorl    $dst, $dst\t# ptr" %}
 5803   ins_encode %{
 5804     __ xorl($dst$$Register, $dst$$Register);
 5805   %}
 5806   ins_pipe(ialu_reg);
 5807 %}
 5808 
 5809 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5810 %{
 5811   match(Set dst src);
 5812   effect(KILL cr);
 5813 
 5814   ins_cost(60);
 5815   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5816   ins_encode %{
 5817     __ movl($dst$$Register, $src$$constant);
 5818   %}
 5819   ins_pipe(ialu_reg);
 5820 %}
 5821 
 5822 instruct loadConF(regF dst, immF con) %{
 5823   match(Set dst con);
 5824   ins_cost(125);
 5825   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5826   ins_encode %{
 5827     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5828   %}
 5829   ins_pipe(pipe_slow);
 5830 %}
 5831 
 5832 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5833   match(Set dst src);
 5834   effect(KILL cr);
 5835   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5836   ins_encode %{
 5837     __ xorq($dst$$Register, $dst$$Register);
 5838   %}
 5839   ins_pipe(ialu_reg);
 5840 %}
 5841 
 5842 instruct loadConN(rRegN dst, immN src) %{
 5843   match(Set dst src);
 5844 
 5845   ins_cost(125);
 5846   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5847   ins_encode %{
 5848     address con = (address)$src$$constant;
 5849     if (con == NULL) {
 5850       ShouldNotReachHere();
 5851     } else {
 5852       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5853     }
 5854   %}
 5855   ins_pipe(ialu_reg_fat); // XXX
 5856 %}
 5857 
 5858 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5859   match(Set dst src);
 5860 
 5861   ins_cost(125);
 5862   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5863   ins_encode %{
 5864     address con = (address)$src$$constant;
 5865     if (con == NULL) {
 5866       ShouldNotReachHere();
 5867     } else {
 5868       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5869     }
 5870   %}
 5871   ins_pipe(ialu_reg_fat); // XXX
 5872 %}
 5873 
 5874 instruct loadConF0(regF dst, immF0 src)
 5875 %{
 5876   match(Set dst src);
 5877   ins_cost(100);
 5878 
 5879   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5880   ins_encode %{
 5881     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5882   %}
 5883   ins_pipe(pipe_slow);
 5884 %}
 5885 
 5886 // Use the same format since predicate() can not be used here.
 5887 instruct loadConD(regD dst, immD con) %{
 5888   match(Set dst con);
 5889   ins_cost(125);
 5890   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5891   ins_encode %{
 5892     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5893   %}
 5894   ins_pipe(pipe_slow);
 5895 %}
 5896 
 5897 instruct loadConD0(regD dst, immD0 src)
 5898 %{
 5899   match(Set dst src);
 5900   ins_cost(100);
 5901 
 5902   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5903   ins_encode %{
 5904     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5905   %}
 5906   ins_pipe(pipe_slow);
 5907 %}
 5908 
 5909 instruct loadSSI(rRegI dst, stackSlotI src)
 5910 %{
 5911   match(Set dst src);
 5912 
 5913   ins_cost(125);
 5914   format %{ "movl    $dst, $src\t# int stk" %}
 5915   opcode(0x8B);
 5916   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5917   ins_pipe(ialu_reg_mem);
 5918 %}
 5919 
 5920 instruct loadSSL(rRegL dst, stackSlotL src)
 5921 %{
 5922   match(Set dst src);
 5923 
 5924   ins_cost(125);
 5925   format %{ "movq    $dst, $src\t# long stk" %}
 5926   opcode(0x8B);
 5927   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5928   ins_pipe(ialu_reg_mem);
 5929 %}
 5930 
 5931 instruct loadSSP(rRegP dst, stackSlotP src)
 5932 %{
 5933   match(Set dst src);
 5934 
 5935   ins_cost(125);
 5936   format %{ "movq    $dst, $src\t# ptr stk" %}
 5937   opcode(0x8B);
 5938   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5939   ins_pipe(ialu_reg_mem);
 5940 %}
 5941 
 5942 instruct loadSSF(regF dst, stackSlotF src)
 5943 %{
 5944   match(Set dst src);
 5945 
 5946   ins_cost(125);
 5947   format %{ "movss   $dst, $src\t# float stk" %}
 5948   ins_encode %{
 5949     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5950   %}
 5951   ins_pipe(pipe_slow); // XXX
 5952 %}
 5953 
 5954 // Use the same format since predicate() can not be used here.
 5955 instruct loadSSD(regD dst, stackSlotD src)
 5956 %{
 5957   match(Set dst src);
 5958 
 5959   ins_cost(125);
 5960   format %{ "movsd   $dst, $src\t# double stk" %}
 5961   ins_encode  %{
 5962     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5963   %}
 5964   ins_pipe(pipe_slow); // XXX
 5965 %}
 5966 
 5967 // Prefetch instructions for allocation.
 5968 // Must be safe to execute with invalid address (cannot fault).
 5969 
 5970 instruct prefetchAlloc( memory mem ) %{
 5971   predicate(AllocatePrefetchInstr==3);
 5972   match(PrefetchAllocation mem);
 5973   ins_cost(125);
 5974 
 5975   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5976   ins_encode %{
 5977     __ prefetchw($mem$$Address);
 5978   %}
 5979   ins_pipe(ialu_mem);
 5980 %}
 5981 
 5982 instruct prefetchAllocNTA( memory mem ) %{
 5983   predicate(AllocatePrefetchInstr==0);
 5984   match(PrefetchAllocation mem);
 5985   ins_cost(125);
 5986 
 5987   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5988   ins_encode %{
 5989     __ prefetchnta($mem$$Address);
 5990   %}
 5991   ins_pipe(ialu_mem);
 5992 %}
 5993 
 5994 instruct prefetchAllocT0( memory mem ) %{
 5995   predicate(AllocatePrefetchInstr==1);
 5996   match(PrefetchAllocation mem);
 5997   ins_cost(125);
 5998 
 5999   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 6000   ins_encode %{
 6001     __ prefetcht0($mem$$Address);
 6002   %}
 6003   ins_pipe(ialu_mem);
 6004 %}
 6005 
 6006 instruct prefetchAllocT2( memory mem ) %{
 6007   predicate(AllocatePrefetchInstr==2);
 6008   match(PrefetchAllocation mem);
 6009   ins_cost(125);
 6010 
 6011   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6012   ins_encode %{
 6013     __ prefetcht2($mem$$Address);
 6014   %}
 6015   ins_pipe(ialu_mem);
 6016 %}
 6017 
 6018 //----------Store Instructions-------------------------------------------------
 6019 
 6020 // Store Byte
 6021 instruct storeB(memory mem, rRegI src)
 6022 %{
 6023   match(Set mem (StoreB mem src));
 6024 
 6025   ins_cost(125); // XXX
 6026   format %{ "movb    $mem, $src\t# byte" %}
 6027   ins_encode %{
 6028     __ movb($mem$$Address, $src$$Register);
 6029   %}
 6030   ins_pipe(ialu_mem_reg);
 6031 %}
 6032 
 6033 // Store Char/Short
 6034 instruct storeC(memory mem, rRegI src)
 6035 %{
 6036   match(Set mem (StoreC mem src));
 6037 
 6038   ins_cost(125); // XXX
 6039   format %{ "movw    $mem, $src\t# char/short" %}
 6040   ins_encode %{
 6041     __ movw($mem$$Address, $src$$Register);
 6042   %}
 6043   ins_pipe(ialu_mem_reg);
 6044 %}
 6045 
 6046 // Store Integer
 6047 instruct storeI(memory mem, rRegI src)
 6048 %{
 6049   match(Set mem (StoreI mem src));
 6050 
 6051   ins_cost(125); // XXX
 6052   format %{ "movl    $mem, $src\t# int" %}
 6053   ins_encode %{
 6054     __ movl($mem$$Address, $src$$Register);
 6055   %}
 6056   ins_pipe(ialu_mem_reg);
 6057 %}
 6058 
 6059 // Store Long
 6060 instruct storeL(memory mem, rRegL src)
 6061 %{
 6062   match(Set mem (StoreL mem src));
 6063 
 6064   ins_cost(125); // XXX
 6065   format %{ "movq    $mem, $src\t# long" %}
 6066   ins_encode %{
 6067     __ movq($mem$$Address, $src$$Register);
 6068   %}
 6069   ins_pipe(ialu_mem_reg); // XXX
 6070 %}
 6071 
 6072 // Store Pointer
 6073 instruct storeP(memory mem, any_RegP src)
 6074 %{
 6075   match(Set mem (StoreP mem src));
 6076 
 6077   ins_cost(125); // XXX
 6078   format %{ "movq    $mem, $src\t# ptr" %}
 6079   ins_encode %{
 6080     __ movq($mem$$Address, $src$$Register);
 6081   %}
 6082   ins_pipe(ialu_mem_reg);
 6083 %}
 6084 
 6085 instruct storeImmP0(memory mem, immP0 zero)
 6086 %{
 6087   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6088   match(Set mem (StoreP mem zero));
 6089 
 6090   ins_cost(125); // XXX
 6091   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6092   ins_encode %{
 6093     __ movq($mem$$Address, r12);
 6094   %}
 6095   ins_pipe(ialu_mem_reg);
 6096 %}
 6097 
 6098 // Store NULL Pointer, mark word, or other simple pointer constant.
 6099 instruct storeImmP(memory mem, immP31 src)
 6100 %{
 6101   match(Set mem (StoreP mem src));
 6102 
 6103   ins_cost(150); // XXX
 6104   format %{ "movq    $mem, $src\t# ptr" %}
 6105   ins_encode %{
 6106     __ movq($mem$$Address, $src$$constant);
 6107   %}
 6108   ins_pipe(ialu_mem_imm);
 6109 %}
 6110 
 6111 // Store Compressed Pointer
 6112 instruct storeN(memory mem, rRegN src)
 6113 %{
 6114   match(Set mem (StoreN mem src));
 6115 
 6116   ins_cost(125); // XXX
 6117   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6118   ins_encode %{
 6119     __ movl($mem$$Address, $src$$Register);
 6120   %}
 6121   ins_pipe(ialu_mem_reg);
 6122 %}
 6123 
 6124 instruct storeNKlass(memory mem, rRegN src)
 6125 %{
 6126   match(Set mem (StoreNKlass mem src));
 6127 
 6128   ins_cost(125); // XXX
 6129   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6130   ins_encode %{
 6131     __ movl($mem$$Address, $src$$Register);
 6132   %}
 6133   ins_pipe(ialu_mem_reg);
 6134 %}
 6135 
 6136 instruct storeImmN0(memory mem, immN0 zero)
 6137 %{
 6138   predicate(CompressedOops::base() == NULL);
 6139   match(Set mem (StoreN mem zero));
 6140 
 6141   ins_cost(125); // XXX
 6142   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6143   ins_encode %{
 6144     __ movl($mem$$Address, r12);
 6145   %}
 6146   ins_pipe(ialu_mem_reg);
 6147 %}
 6148 
 6149 instruct storeImmN(memory mem, immN src)
 6150 %{
 6151   match(Set mem (StoreN mem src));
 6152 
 6153   ins_cost(150); // XXX
 6154   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6155   ins_encode %{
 6156     address con = (address)$src$$constant;
 6157     if (con == NULL) {
 6158       __ movl($mem$$Address, 0);
 6159     } else {
 6160       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6161     }
 6162   %}
 6163   ins_pipe(ialu_mem_imm);
 6164 %}
 6165 
 6166 instruct storeImmNKlass(memory mem, immNKlass src)
 6167 %{
 6168   match(Set mem (StoreNKlass mem src));
 6169 
 6170   ins_cost(150); // XXX
 6171   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6172   ins_encode %{
 6173     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6174   %}
 6175   ins_pipe(ialu_mem_imm);
 6176 %}
 6177 
 6178 // Store Integer Immediate
 6179 instruct storeImmI0(memory mem, immI_0 zero)
 6180 %{
 6181   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6182   match(Set mem (StoreI mem zero));
 6183 
 6184   ins_cost(125); // XXX
 6185   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6186   ins_encode %{
 6187     __ movl($mem$$Address, r12);
 6188   %}
 6189   ins_pipe(ialu_mem_reg);
 6190 %}
 6191 
 6192 instruct storeImmI(memory mem, immI src)
 6193 %{
 6194   match(Set mem (StoreI mem src));
 6195 
 6196   ins_cost(150);
 6197   format %{ "movl    $mem, $src\t# int" %}
 6198   ins_encode %{
 6199     __ movl($mem$$Address, $src$$constant);
 6200   %}
 6201   ins_pipe(ialu_mem_imm);
 6202 %}
 6203 
 6204 // Store Long Immediate
 6205 instruct storeImmL0(memory mem, immL0 zero)
 6206 %{
 6207   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6208   match(Set mem (StoreL mem zero));
 6209 
 6210   ins_cost(125); // XXX
 6211   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6212   ins_encode %{
 6213     __ movq($mem$$Address, r12);
 6214   %}
 6215   ins_pipe(ialu_mem_reg);
 6216 %}
 6217 
 6218 instruct storeImmL(memory mem, immL32 src)
 6219 %{
 6220   match(Set mem (StoreL mem src));
 6221 
 6222   ins_cost(150);
 6223   format %{ "movq    $mem, $src\t# long" %}
 6224   ins_encode %{
 6225     __ movq($mem$$Address, $src$$constant);
 6226   %}
 6227   ins_pipe(ialu_mem_imm);
 6228 %}
 6229 
 6230 // Store Short/Char Immediate
 6231 instruct storeImmC0(memory mem, immI_0 zero)
 6232 %{
 6233   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6234   match(Set mem (StoreC mem zero));
 6235 
 6236   ins_cost(125); // XXX
 6237   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6238   ins_encode %{
 6239     __ movw($mem$$Address, r12);
 6240   %}
 6241   ins_pipe(ialu_mem_reg);
 6242 %}
 6243 
 6244 instruct storeImmI16(memory mem, immI16 src)
 6245 %{
 6246   predicate(UseStoreImmI16);
 6247   match(Set mem (StoreC mem src));
 6248 
 6249   ins_cost(150);
 6250   format %{ "movw    $mem, $src\t# short/char" %}
 6251   ins_encode %{
 6252     __ movw($mem$$Address, $src$$constant);
 6253   %}
 6254   ins_pipe(ialu_mem_imm);
 6255 %}
 6256 
 6257 // Store Byte Immediate
 6258 instruct storeImmB0(memory mem, immI_0 zero)
 6259 %{
 6260   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6261   match(Set mem (StoreB mem zero));
 6262 
 6263   ins_cost(125); // XXX
 6264   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6265   ins_encode %{
 6266     __ movb($mem$$Address, r12);
 6267   %}
 6268   ins_pipe(ialu_mem_reg);
 6269 %}
 6270 
 6271 instruct storeImmB(memory mem, immI8 src)
 6272 %{
 6273   match(Set mem (StoreB mem src));
 6274 
 6275   ins_cost(150); // XXX
 6276   format %{ "movb    $mem, $src\t# byte" %}
 6277   ins_encode %{
 6278     __ movb($mem$$Address, $src$$constant);
 6279   %}
 6280   ins_pipe(ialu_mem_imm);
 6281 %}
 6282 
 6283 // Store CMS card-mark Immediate
 6284 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6285 %{
 6286   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6287   match(Set mem (StoreCM mem zero));
 6288 
 6289   ins_cost(125); // XXX
 6290   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6291   ins_encode %{
 6292     __ movb($mem$$Address, r12);
 6293   %}
 6294   ins_pipe(ialu_mem_reg);
 6295 %}
 6296 
 6297 instruct storeImmCM0(memory mem, immI_0 src)
 6298 %{
 6299   match(Set mem (StoreCM mem src));
 6300 
 6301   ins_cost(150); // XXX
 6302   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6303   ins_encode %{
 6304     __ movb($mem$$Address, $src$$constant);
 6305   %}
 6306   ins_pipe(ialu_mem_imm);
 6307 %}
 6308 
 6309 // Store Float
 6310 instruct storeF(memory mem, regF src)
 6311 %{
 6312   match(Set mem (StoreF mem src));
 6313 
 6314   ins_cost(95); // XXX
 6315   format %{ "movss   $mem, $src\t# float" %}
 6316   ins_encode %{
 6317     __ movflt($mem$$Address, $src$$XMMRegister);
 6318   %}
 6319   ins_pipe(pipe_slow); // XXX
 6320 %}
 6321 
 6322 // Store immediate Float value (it is faster than store from XMM register)
 6323 instruct storeF0(memory mem, immF0 zero)
 6324 %{
 6325   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6326   match(Set mem (StoreF mem zero));
 6327 
 6328   ins_cost(25); // XXX
 6329   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6330   ins_encode %{
 6331     __ movl($mem$$Address, r12);
 6332   %}
 6333   ins_pipe(ialu_mem_reg);
 6334 %}
 6335 
 6336 instruct storeF_imm(memory mem, immF src)
 6337 %{
 6338   match(Set mem (StoreF mem src));
 6339 
 6340   ins_cost(50);
 6341   format %{ "movl    $mem, $src\t# float" %}
 6342   ins_encode %{
 6343     __ movl($mem$$Address, jint_cast($src$$constant));
 6344   %}
 6345   ins_pipe(ialu_mem_imm);
 6346 %}
 6347 
 6348 // Store Double
 6349 instruct storeD(memory mem, regD src)
 6350 %{
 6351   match(Set mem (StoreD mem src));
 6352 
 6353   ins_cost(95); // XXX
 6354   format %{ "movsd   $mem, $src\t# double" %}
 6355   ins_encode %{
 6356     __ movdbl($mem$$Address, $src$$XMMRegister);
 6357   %}
 6358   ins_pipe(pipe_slow); // XXX
 6359 %}
 6360 
 6361 // Store immediate double 0.0 (it is faster than store from XMM register)
 6362 instruct storeD0_imm(memory mem, immD0 src)
 6363 %{
 6364   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6365   match(Set mem (StoreD mem src));
 6366 
 6367   ins_cost(50);
 6368   format %{ "movq    $mem, $src\t# double 0." %}
 6369   ins_encode %{
 6370     __ movq($mem$$Address, $src$$constant);
 6371   %}
 6372   ins_pipe(ialu_mem_imm);
 6373 %}
 6374 
 6375 instruct storeD0(memory mem, immD0 zero)
 6376 %{
 6377   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6378   match(Set mem (StoreD mem zero));
 6379 
 6380   ins_cost(25); // XXX
 6381   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6382   ins_encode %{
 6383     __ movq($mem$$Address, r12);
 6384   %}
 6385   ins_pipe(ialu_mem_reg);
 6386 %}
 6387 
 6388 instruct storeSSI(stackSlotI dst, rRegI src)
 6389 %{
 6390   match(Set dst src);
 6391 
 6392   ins_cost(100);
 6393   format %{ "movl    $dst, $src\t# int stk" %}
 6394   opcode(0x89);
 6395   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6396   ins_pipe( ialu_mem_reg );
 6397 %}
 6398 
 6399 instruct storeSSL(stackSlotL dst, rRegL src)
 6400 %{
 6401   match(Set dst src);
 6402 
 6403   ins_cost(100);
 6404   format %{ "movq    $dst, $src\t# long stk" %}
 6405   opcode(0x89);
 6406   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6407   ins_pipe(ialu_mem_reg);
 6408 %}
 6409 
 6410 instruct storeSSP(stackSlotP dst, rRegP src)
 6411 %{
 6412   match(Set dst src);
 6413 
 6414   ins_cost(100);
 6415   format %{ "movq    $dst, $src\t# ptr stk" %}
 6416   opcode(0x89);
 6417   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6418   ins_pipe(ialu_mem_reg);
 6419 %}
 6420 
 6421 instruct storeSSF(stackSlotF dst, regF src)
 6422 %{
 6423   match(Set dst src);
 6424 
 6425   ins_cost(95); // XXX
 6426   format %{ "movss   $dst, $src\t# float stk" %}
 6427   ins_encode %{
 6428     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6429   %}
 6430   ins_pipe(pipe_slow); // XXX
 6431 %}
 6432 
 6433 instruct storeSSD(stackSlotD dst, regD src)
 6434 %{
 6435   match(Set dst src);
 6436 
 6437   ins_cost(95); // XXX
 6438   format %{ "movsd   $dst, $src\t# double stk" %}
 6439   ins_encode %{
 6440     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6441   %}
 6442   ins_pipe(pipe_slow); // XXX
 6443 %}
 6444 
 6445 instruct cacheWB(indirect addr)
 6446 %{
 6447   predicate(VM_Version::supports_data_cache_line_flush());
 6448   match(CacheWB addr);
 6449 
 6450   ins_cost(100);
 6451   format %{"cache wb $addr" %}
 6452   ins_encode %{
 6453     assert($addr->index_position() < 0, "should be");
 6454     assert($addr$$disp == 0, "should be");
 6455     __ cache_wb(Address($addr$$base$$Register, 0));
 6456   %}
 6457   ins_pipe(pipe_slow); // XXX
 6458 %}
 6459 
 6460 instruct cacheWBPreSync()
 6461 %{
 6462   predicate(VM_Version::supports_data_cache_line_flush());
 6463   match(CacheWBPreSync);
 6464 
 6465   ins_cost(100);
 6466   format %{"cache wb presync" %}
 6467   ins_encode %{
 6468     __ cache_wbsync(true);
 6469   %}
 6470   ins_pipe(pipe_slow); // XXX
 6471 %}
 6472 
 6473 instruct cacheWBPostSync()
 6474 %{
 6475   predicate(VM_Version::supports_data_cache_line_flush());
 6476   match(CacheWBPostSync);
 6477 
 6478   ins_cost(100);
 6479   format %{"cache wb postsync" %}
 6480   ins_encode %{
 6481     __ cache_wbsync(false);
 6482   %}
 6483   ins_pipe(pipe_slow); // XXX
 6484 %}
 6485 
 6486 //----------BSWAP Instructions-------------------------------------------------
 6487 instruct bytes_reverse_int(rRegI dst) %{
 6488   match(Set dst (ReverseBytesI dst));
 6489 
 6490   format %{ "bswapl  $dst" %}
 6491   ins_encode %{
 6492     __ bswapl($dst$$Register);
 6493   %}
 6494   ins_pipe( ialu_reg );
 6495 %}
 6496 
 6497 instruct bytes_reverse_long(rRegL dst) %{
 6498   match(Set dst (ReverseBytesL dst));
 6499 
 6500   format %{ "bswapq  $dst" %}
 6501   ins_encode %{
 6502     __ bswapq($dst$$Register);
 6503   %}
 6504   ins_pipe( ialu_reg);
 6505 %}
 6506 
 6507 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6508   match(Set dst (ReverseBytesUS dst));
 6509   effect(KILL cr);
 6510 
 6511   format %{ "bswapl  $dst\n\t"
 6512             "shrl    $dst,16\n\t" %}
 6513   ins_encode %{
 6514     __ bswapl($dst$$Register);
 6515     __ shrl($dst$$Register, 16);
 6516   %}
 6517   ins_pipe( ialu_reg );
 6518 %}
 6519 
 6520 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6521   match(Set dst (ReverseBytesS dst));
 6522   effect(KILL cr);
 6523 
 6524   format %{ "bswapl  $dst\n\t"
 6525             "sar     $dst,16\n\t" %}
 6526   ins_encode %{
 6527     __ bswapl($dst$$Register);
 6528     __ sarl($dst$$Register, 16);
 6529   %}
 6530   ins_pipe( ialu_reg );
 6531 %}
 6532 
 6533 //---------- Zeros Count Instructions ------------------------------------------
 6534 
 6535 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6536   predicate(UseCountLeadingZerosInstruction);
 6537   match(Set dst (CountLeadingZerosI src));
 6538   effect(KILL cr);
 6539 
 6540   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6541   ins_encode %{
 6542     __ lzcntl($dst$$Register, $src$$Register);
 6543   %}
 6544   ins_pipe(ialu_reg);
 6545 %}
 6546 
 6547 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6548   predicate(UseCountLeadingZerosInstruction);
 6549   match(Set dst (CountLeadingZerosI (LoadI src)));
 6550   effect(KILL cr);
 6551   ins_cost(175);
 6552   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6553   ins_encode %{
 6554     __ lzcntl($dst$$Register, $src$$Address);
 6555   %}
 6556   ins_pipe(ialu_reg_mem);
 6557 %}
 6558 
 6559 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6560   predicate(!UseCountLeadingZerosInstruction);
 6561   match(Set dst (CountLeadingZerosI src));
 6562   effect(KILL cr);
 6563 
 6564   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6565             "jnz     skip\n\t"
 6566             "movl    $dst, -1\n"
 6567       "skip:\n\t"
 6568             "negl    $dst\n\t"
 6569             "addl    $dst, 31" %}
 6570   ins_encode %{
 6571     Register Rdst = $dst$$Register;
 6572     Register Rsrc = $src$$Register;
 6573     Label skip;
 6574     __ bsrl(Rdst, Rsrc);
 6575     __ jccb(Assembler::notZero, skip);
 6576     __ movl(Rdst, -1);
 6577     __ bind(skip);
 6578     __ negl(Rdst);
 6579     __ addl(Rdst, BitsPerInt - 1);
 6580   %}
 6581   ins_pipe(ialu_reg);
 6582 %}
 6583 
 6584 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6585   predicate(UseCountLeadingZerosInstruction);
 6586   match(Set dst (CountLeadingZerosL src));
 6587   effect(KILL cr);
 6588 
 6589   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6590   ins_encode %{
 6591     __ lzcntq($dst$$Register, $src$$Register);
 6592   %}
 6593   ins_pipe(ialu_reg);
 6594 %}
 6595 
 6596 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6597   predicate(UseCountLeadingZerosInstruction);
 6598   match(Set dst (CountLeadingZerosL (LoadL src)));
 6599   effect(KILL cr);
 6600   ins_cost(175);
 6601   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6602   ins_encode %{
 6603     __ lzcntq($dst$$Register, $src$$Address);
 6604   %}
 6605   ins_pipe(ialu_reg_mem);
 6606 %}
 6607 
 6608 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6609   predicate(!UseCountLeadingZerosInstruction);
 6610   match(Set dst (CountLeadingZerosL src));
 6611   effect(KILL cr);
 6612 
 6613   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6614             "jnz     skip\n\t"
 6615             "movl    $dst, -1\n"
 6616       "skip:\n\t"
 6617             "negl    $dst\n\t"
 6618             "addl    $dst, 63" %}
 6619   ins_encode %{
 6620     Register Rdst = $dst$$Register;
 6621     Register Rsrc = $src$$Register;
 6622     Label skip;
 6623     __ bsrq(Rdst, Rsrc);
 6624     __ jccb(Assembler::notZero, skip);
 6625     __ movl(Rdst, -1);
 6626     __ bind(skip);
 6627     __ negl(Rdst);
 6628     __ addl(Rdst, BitsPerLong - 1);
 6629   %}
 6630   ins_pipe(ialu_reg);
 6631 %}
 6632 
 6633 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6634   predicate(UseCountTrailingZerosInstruction);
 6635   match(Set dst (CountTrailingZerosI src));
 6636   effect(KILL cr);
 6637 
 6638   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6639   ins_encode %{
 6640     __ tzcntl($dst$$Register, $src$$Register);
 6641   %}
 6642   ins_pipe(ialu_reg);
 6643 %}
 6644 
 6645 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6646   predicate(UseCountTrailingZerosInstruction);
 6647   match(Set dst (CountTrailingZerosI (LoadI src)));
 6648   effect(KILL cr);
 6649   ins_cost(175);
 6650   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6651   ins_encode %{
 6652     __ tzcntl($dst$$Register, $src$$Address);
 6653   %}
 6654   ins_pipe(ialu_reg_mem);
 6655 %}
 6656 
 6657 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6658   predicate(!UseCountTrailingZerosInstruction);
 6659   match(Set dst (CountTrailingZerosI src));
 6660   effect(KILL cr);
 6661 
 6662   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6663             "jnz     done\n\t"
 6664             "movl    $dst, 32\n"
 6665       "done:" %}
 6666   ins_encode %{
 6667     Register Rdst = $dst$$Register;
 6668     Label done;
 6669     __ bsfl(Rdst, $src$$Register);
 6670     __ jccb(Assembler::notZero, done);
 6671     __ movl(Rdst, BitsPerInt);
 6672     __ bind(done);
 6673   %}
 6674   ins_pipe(ialu_reg);
 6675 %}
 6676 
 6677 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6678   predicate(UseCountTrailingZerosInstruction);
 6679   match(Set dst (CountTrailingZerosL src));
 6680   effect(KILL cr);
 6681 
 6682   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6683   ins_encode %{
 6684     __ tzcntq($dst$$Register, $src$$Register);
 6685   %}
 6686   ins_pipe(ialu_reg);
 6687 %}
 6688 
 6689 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6690   predicate(UseCountTrailingZerosInstruction);
 6691   match(Set dst (CountTrailingZerosL (LoadL src)));
 6692   effect(KILL cr);
 6693   ins_cost(175);
 6694   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6695   ins_encode %{
 6696     __ tzcntq($dst$$Register, $src$$Address);
 6697   %}
 6698   ins_pipe(ialu_reg_mem);
 6699 %}
 6700 
 6701 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6702   predicate(!UseCountTrailingZerosInstruction);
 6703   match(Set dst (CountTrailingZerosL src));
 6704   effect(KILL cr);
 6705 
 6706   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6707             "jnz     done\n\t"
 6708             "movl    $dst, 64\n"
 6709       "done:" %}
 6710   ins_encode %{
 6711     Register Rdst = $dst$$Register;
 6712     Label done;
 6713     __ bsfq(Rdst, $src$$Register);
 6714     __ jccb(Assembler::notZero, done);
 6715     __ movl(Rdst, BitsPerLong);
 6716     __ bind(done);
 6717   %}
 6718   ins_pipe(ialu_reg);
 6719 %}
 6720 
 6721 //--------------- Reverse Operation Instructions ----------------
 6722 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 6723   predicate(!VM_Version::supports_gfni());
 6724   match(Set dst (ReverseI src));
 6725   effect(TEMP dst, TEMP rtmp, KILL cr);
 6726   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 6727   ins_encode %{
 6728     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 6729   %}
 6730   ins_pipe( ialu_reg );
 6731 %}
 6732 
 6733 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6734   predicate(VM_Version::supports_gfni());
 6735   match(Set dst (ReverseI src));
 6736   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6737   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6738   ins_encode %{
 6739     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 6740   %}
 6741   ins_pipe( ialu_reg );
 6742 %}
 6743 
 6744 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 6745   predicate(!VM_Version::supports_gfni());
 6746   match(Set dst (ReverseL src));
 6747   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 6748   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 6749   ins_encode %{
 6750     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 6751   %}
 6752   ins_pipe( ialu_reg );
 6753 %}
 6754 
 6755 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6756   predicate(VM_Version::supports_gfni());
 6757   match(Set dst (ReverseL src));
 6758   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6759   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6760   ins_encode %{
 6761     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 6762   %}
 6763   ins_pipe( ialu_reg );
 6764 %}
 6765 
 6766 //---------- Population Count Instructions -------------------------------------
 6767 
 6768 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6769   predicate(UsePopCountInstruction);
 6770   match(Set dst (PopCountI src));
 6771   effect(KILL cr);
 6772 
 6773   format %{ "popcnt  $dst, $src" %}
 6774   ins_encode %{
 6775     __ popcntl($dst$$Register, $src$$Register);
 6776   %}
 6777   ins_pipe(ialu_reg);
 6778 %}
 6779 
 6780 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6781   predicate(UsePopCountInstruction);
 6782   match(Set dst (PopCountI (LoadI mem)));
 6783   effect(KILL cr);
 6784 
 6785   format %{ "popcnt  $dst, $mem" %}
 6786   ins_encode %{
 6787     __ popcntl($dst$$Register, $mem$$Address);
 6788   %}
 6789   ins_pipe(ialu_reg);
 6790 %}
 6791 
 6792 // Note: Long.bitCount(long) returns an int.
 6793 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6794   predicate(UsePopCountInstruction);
 6795   match(Set dst (PopCountL src));
 6796   effect(KILL cr);
 6797 
 6798   format %{ "popcnt  $dst, $src" %}
 6799   ins_encode %{
 6800     __ popcntq($dst$$Register, $src$$Register);
 6801   %}
 6802   ins_pipe(ialu_reg);
 6803 %}
 6804 
 6805 // Note: Long.bitCount(long) returns an int.
 6806 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6807   predicate(UsePopCountInstruction);
 6808   match(Set dst (PopCountL (LoadL mem)));
 6809   effect(KILL cr);
 6810 
 6811   format %{ "popcnt  $dst, $mem" %}
 6812   ins_encode %{
 6813     __ popcntq($dst$$Register, $mem$$Address);
 6814   %}
 6815   ins_pipe(ialu_reg);
 6816 %}
 6817 
 6818 
 6819 //----------MemBar Instructions-----------------------------------------------
 6820 // Memory barrier flavors
 6821 
 6822 instruct membar_acquire()
 6823 %{
 6824   match(MemBarAcquire);
 6825   match(LoadFence);
 6826   ins_cost(0);
 6827 
 6828   size(0);
 6829   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6830   ins_encode();
 6831   ins_pipe(empty);
 6832 %}
 6833 
 6834 instruct membar_acquire_lock()
 6835 %{
 6836   match(MemBarAcquireLock);
 6837   ins_cost(0);
 6838 
 6839   size(0);
 6840   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6841   ins_encode();
 6842   ins_pipe(empty);
 6843 %}
 6844 
 6845 instruct membar_release()
 6846 %{
 6847   match(MemBarRelease);
 6848   match(StoreFence);
 6849   ins_cost(0);
 6850 
 6851   size(0);
 6852   format %{ "MEMBAR-release ! (empty encoding)" %}
 6853   ins_encode();
 6854   ins_pipe(empty);
 6855 %}
 6856 
 6857 instruct membar_release_lock()
 6858 %{
 6859   match(MemBarReleaseLock);
 6860   ins_cost(0);
 6861 
 6862   size(0);
 6863   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6864   ins_encode();
 6865   ins_pipe(empty);
 6866 %}
 6867 
 6868 instruct membar_volatile(rFlagsReg cr) %{
 6869   match(MemBarVolatile);
 6870   effect(KILL cr);
 6871   ins_cost(400);
 6872 
 6873   format %{
 6874     $$template
 6875     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6876   %}
 6877   ins_encode %{
 6878     __ membar(Assembler::StoreLoad);
 6879   %}
 6880   ins_pipe(pipe_slow);
 6881 %}
 6882 
 6883 instruct unnecessary_membar_volatile()
 6884 %{
 6885   match(MemBarVolatile);
 6886   predicate(Matcher::post_store_load_barrier(n));
 6887   ins_cost(0);
 6888 
 6889   size(0);
 6890   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6891   ins_encode();
 6892   ins_pipe(empty);
 6893 %}
 6894 
 6895 instruct membar_storestore() %{
 6896   match(MemBarStoreStore);
 6897   match(StoreStoreFence);
 6898   ins_cost(0);
 6899 
 6900   size(0);
 6901   format %{ "MEMBAR-storestore (empty encoding)" %}
 6902   ins_encode( );
 6903   ins_pipe(empty);
 6904 %}
 6905 
 6906 //----------Move Instructions--------------------------------------------------
 6907 
 6908 instruct castX2P(rRegP dst, rRegL src)
 6909 %{
 6910   match(Set dst (CastX2P src));
 6911 
 6912   format %{ "movq    $dst, $src\t# long->ptr" %}
 6913   ins_encode %{
 6914     if ($dst$$reg != $src$$reg) {
 6915       __ movptr($dst$$Register, $src$$Register);
 6916     }
 6917   %}
 6918   ins_pipe(ialu_reg_reg); // XXX
 6919 %}
 6920 
 6921 instruct castP2X(rRegL dst, rRegP src)
 6922 %{
 6923   match(Set dst (CastP2X src));
 6924 
 6925   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6926   ins_encode %{
 6927     if ($dst$$reg != $src$$reg) {
 6928       __ movptr($dst$$Register, $src$$Register);
 6929     }
 6930   %}
 6931   ins_pipe(ialu_reg_reg); // XXX
 6932 %}
 6933 
 6934 // Convert oop into int for vectors alignment masking
 6935 instruct convP2I(rRegI dst, rRegP src)
 6936 %{
 6937   match(Set dst (ConvL2I (CastP2X src)));
 6938 
 6939   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6940   ins_encode %{
 6941     __ movl($dst$$Register, $src$$Register);
 6942   %}
 6943   ins_pipe(ialu_reg_reg); // XXX
 6944 %}
 6945 
 6946 // Convert compressed oop into int for vectors alignment masking
 6947 // in case of 32bit oops (heap < 4Gb).
 6948 instruct convN2I(rRegI dst, rRegN src)
 6949 %{
 6950   predicate(CompressedOops::shift() == 0);
 6951   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6952 
 6953   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6954   ins_encode %{
 6955     __ movl($dst$$Register, $src$$Register);
 6956   %}
 6957   ins_pipe(ialu_reg_reg); // XXX
 6958 %}
 6959 
 6960 // Convert oop pointer into compressed form
 6961 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6962   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6963   match(Set dst (EncodeP src));
 6964   effect(KILL cr);
 6965   format %{ "encode_heap_oop $dst,$src" %}
 6966   ins_encode %{
 6967     Register s = $src$$Register;
 6968     Register d = $dst$$Register;
 6969     if (s != d) {
 6970       __ movq(d, s);
 6971     }
 6972     __ encode_heap_oop(d);
 6973   %}
 6974   ins_pipe(ialu_reg_long);
 6975 %}
 6976 
 6977 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6978   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6979   match(Set dst (EncodeP src));
 6980   effect(KILL cr);
 6981   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6982   ins_encode %{
 6983     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6984   %}
 6985   ins_pipe(ialu_reg_long);
 6986 %}
 6987 
 6988 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6989   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6990             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6991   match(Set dst (DecodeN src));
 6992   effect(KILL cr);
 6993   format %{ "decode_heap_oop $dst,$src" %}
 6994   ins_encode %{
 6995     Register s = $src$$Register;
 6996     Register d = $dst$$Register;
 6997     if (s != d) {
 6998       __ movq(d, s);
 6999     }
 7000     __ decode_heap_oop(d);
 7001   %}
 7002   ins_pipe(ialu_reg_long);
 7003 %}
 7004 
 7005 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7006   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 7007             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 7008   match(Set dst (DecodeN src));
 7009   effect(KILL cr);
 7010   format %{ "decode_heap_oop_not_null $dst,$src" %}
 7011   ins_encode %{
 7012     Register s = $src$$Register;
 7013     Register d = $dst$$Register;
 7014     if (s != d) {
 7015       __ decode_heap_oop_not_null(d, s);
 7016     } else {
 7017       __ decode_heap_oop_not_null(d);
 7018     }
 7019   %}
 7020   ins_pipe(ialu_reg_long);
 7021 %}
 7022 
 7023 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 7024   match(Set dst (EncodePKlass src));
 7025   effect(TEMP dst, KILL cr);
 7026   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 7027   ins_encode %{
 7028     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7029   %}
 7030   ins_pipe(ialu_reg_long);
 7031 %}
 7032 
 7033 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7034   match(Set dst (DecodeNKlass src));
 7035   effect(TEMP dst, KILL cr);
 7036   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 7037   ins_encode %{
 7038     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7039   %}
 7040   ins_pipe(ialu_reg_long);
 7041 %}
 7042 
 7043 //----------Conditional Move---------------------------------------------------
 7044 // Jump
 7045 // dummy instruction for generating temp registers
 7046 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 7047   match(Jump (LShiftL switch_val shift));
 7048   ins_cost(350);
 7049   predicate(false);
 7050   effect(TEMP dest);
 7051 
 7052   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7053             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 7054   ins_encode %{
 7055     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7056     // to do that and the compiler is using that register as one it can allocate.
 7057     // So we build it all by hand.
 7058     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 7059     // ArrayAddress dispatch(table, index);
 7060     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 7061     __ lea($dest$$Register, $constantaddress);
 7062     __ jmp(dispatch);
 7063   %}
 7064   ins_pipe(pipe_jmp);
 7065 %}
 7066 
 7067 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 7068   match(Jump (AddL (LShiftL switch_val shift) offset));
 7069   ins_cost(350);
 7070   effect(TEMP dest);
 7071 
 7072   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7073             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 7074   ins_encode %{
 7075     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7076     // to do that and the compiler is using that register as one it can allocate.
 7077     // So we build it all by hand.
 7078     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7079     // ArrayAddress dispatch(table, index);
 7080     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7081     __ lea($dest$$Register, $constantaddress);
 7082     __ jmp(dispatch);
 7083   %}
 7084   ins_pipe(pipe_jmp);
 7085 %}
 7086 
 7087 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 7088   match(Jump switch_val);
 7089   ins_cost(350);
 7090   effect(TEMP dest);
 7091 
 7092   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7093             "jmp     [$dest + $switch_val]\n\t" %}
 7094   ins_encode %{
 7095     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7096     // to do that and the compiler is using that register as one it can allocate.
 7097     // So we build it all by hand.
 7098     // Address index(noreg, switch_reg, Address::times_1);
 7099     // ArrayAddress dispatch(table, index);
 7100     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7101     __ lea($dest$$Register, $constantaddress);
 7102     __ jmp(dispatch);
 7103   %}
 7104   ins_pipe(pipe_jmp);
 7105 %}
 7106 
 7107 // Conditional move
 7108 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7109 %{
 7110   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7111   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7112 
 7113   ins_cost(100); // XXX
 7114   format %{ "setbn$cop $dst\t# signed, int" %}
 7115   ins_encode %{
 7116     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7117     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7118   %}
 7119   ins_pipe(ialu_reg);
 7120 %}
 7121 
 7122 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7123 %{
 7124   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7125 
 7126   ins_cost(200); // XXX
 7127   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7128   ins_encode %{
 7129     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7130   %}
 7131   ins_pipe(pipe_cmov_reg);
 7132 %}
 7133 
 7134 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7135 %{
 7136   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7137   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7138 
 7139   ins_cost(100); // XXX
 7140   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7141   ins_encode %{
 7142     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7143     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7144   %}
 7145   ins_pipe(ialu_reg);
 7146 %}
 7147 
 7148 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7149   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7150 
 7151   ins_cost(200); // XXX
 7152   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7153   ins_encode %{
 7154     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7155   %}
 7156   ins_pipe(pipe_cmov_reg);
 7157 %}
 7158 
 7159 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7160 %{
 7161   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7162   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7163 
 7164   ins_cost(100); // XXX
 7165   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7166   ins_encode %{
 7167     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7168     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7169   %}
 7170   ins_pipe(ialu_reg);
 7171 %}
 7172 
 7173 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7174   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7175   ins_cost(200);
 7176   expand %{
 7177     cmovI_regU(cop, cr, dst, src);
 7178   %}
 7179 %}
 7180 
 7181 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7182   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7183   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7184 
 7185   ins_cost(200); // XXX
 7186   format %{ "cmovpl  $dst, $src\n\t"
 7187             "cmovnel $dst, $src" %}
 7188   ins_encode %{
 7189     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7190     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7191   %}
 7192   ins_pipe(pipe_cmov_reg);
 7193 %}
 7194 
 7195 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7196 // inputs of the CMove
 7197 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7198   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7199   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7200 
 7201   ins_cost(200); // XXX
 7202   format %{ "cmovpl  $dst, $src\n\t"
 7203             "cmovnel $dst, $src" %}
 7204   ins_encode %{
 7205     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7206     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7207   %}
 7208   ins_pipe(pipe_cmov_reg);
 7209 %}
 7210 
 7211 // Conditional move
 7212 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7213   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7214 
 7215   ins_cost(250); // XXX
 7216   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7217   ins_encode %{
 7218     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7219   %}
 7220   ins_pipe(pipe_cmov_mem);
 7221 %}
 7222 
 7223 // Conditional move
 7224 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7225 %{
 7226   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7227 
 7228   ins_cost(250); // XXX
 7229   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7230   ins_encode %{
 7231     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7232   %}
 7233   ins_pipe(pipe_cmov_mem);
 7234 %}
 7235 
 7236 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7237   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7238   ins_cost(250);
 7239   expand %{
 7240     cmovI_memU(cop, cr, dst, src);
 7241   %}
 7242 %}
 7243 
 7244 // Conditional move
 7245 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7246 %{
 7247   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7248 
 7249   ins_cost(200); // XXX
 7250   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7251   ins_encode %{
 7252     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7253   %}
 7254   ins_pipe(pipe_cmov_reg);
 7255 %}
 7256 
 7257 // Conditional move
 7258 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7259 %{
 7260   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7261 
 7262   ins_cost(200); // XXX
 7263   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7264   ins_encode %{
 7265     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7266   %}
 7267   ins_pipe(pipe_cmov_reg);
 7268 %}
 7269 
 7270 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7271   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7272   ins_cost(200);
 7273   expand %{
 7274     cmovN_regU(cop, cr, dst, src);
 7275   %}
 7276 %}
 7277 
 7278 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7279   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7280   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7281 
 7282   ins_cost(200); // XXX
 7283   format %{ "cmovpl  $dst, $src\n\t"
 7284             "cmovnel $dst, $src" %}
 7285   ins_encode %{
 7286     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7287     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7288   %}
 7289   ins_pipe(pipe_cmov_reg);
 7290 %}
 7291 
 7292 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7293 // inputs of the CMove
 7294 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7295   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7296   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 7297 
 7298   ins_cost(200); // XXX
 7299   format %{ "cmovpl  $dst, $src\n\t"
 7300             "cmovnel $dst, $src" %}
 7301   ins_encode %{
 7302     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7303     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7304   %}
 7305   ins_pipe(pipe_cmov_reg);
 7306 %}
 7307 
 7308 // Conditional move
 7309 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7310 %{
 7311   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7312 
 7313   ins_cost(200); // XXX
 7314   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7315   ins_encode %{
 7316     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7317   %}
 7318   ins_pipe(pipe_cmov_reg);  // XXX
 7319 %}
 7320 
 7321 // Conditional move
 7322 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7323 %{
 7324   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7325 
 7326   ins_cost(200); // XXX
 7327   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7328   ins_encode %{
 7329     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7330   %}
 7331   ins_pipe(pipe_cmov_reg); // XXX
 7332 %}
 7333 
 7334 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7335   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7336   ins_cost(200);
 7337   expand %{
 7338     cmovP_regU(cop, cr, dst, src);
 7339   %}
 7340 %}
 7341 
 7342 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7343   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7344   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7345 
 7346   ins_cost(200); // XXX
 7347   format %{ "cmovpq  $dst, $src\n\t"
 7348             "cmovneq $dst, $src" %}
 7349   ins_encode %{
 7350     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7351     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7352   %}
 7353   ins_pipe(pipe_cmov_reg);
 7354 %}
 7355 
 7356 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7357 // inputs of the CMove
 7358 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7359   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7360   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 7361 
 7362   ins_cost(200); // XXX
 7363   format %{ "cmovpq  $dst, $src\n\t"
 7364             "cmovneq $dst, $src" %}
 7365   ins_encode %{
 7366     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7367     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7368   %}
 7369   ins_pipe(pipe_cmov_reg);
 7370 %}
 7371 
 7372 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7373 // correctly meets the two pointer arguments; one is an incoming
 7374 // register but the other is a memory operand.  ALSO appears to
 7375 // be buggy with implicit null checks.
 7376 //
 7377 //// Conditional move
 7378 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7379 //%{
 7380 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7381 //  ins_cost(250);
 7382 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7383 //  opcode(0x0F,0x40);
 7384 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7385 //  ins_pipe( pipe_cmov_mem );
 7386 //%}
 7387 //
 7388 //// Conditional move
 7389 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7390 //%{
 7391 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7392 //  ins_cost(250);
 7393 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7394 //  opcode(0x0F,0x40);
 7395 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7396 //  ins_pipe( pipe_cmov_mem );
 7397 //%}
 7398 
 7399 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7400 %{
 7401   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7402   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7403 
 7404   ins_cost(100); // XXX
 7405   format %{ "setbn$cop $dst\t# signed, long" %}
 7406   ins_encode %{
 7407     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7408     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7409   %}
 7410   ins_pipe(ialu_reg);
 7411 %}
 7412 
 7413 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7414 %{
 7415   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7416 
 7417   ins_cost(200); // XXX
 7418   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7419   ins_encode %{
 7420     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7421   %}
 7422   ins_pipe(pipe_cmov_reg);  // XXX
 7423 %}
 7424 
 7425 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7426 %{
 7427   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7428 
 7429   ins_cost(200); // XXX
 7430   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7431   ins_encode %{
 7432     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7433   %}
 7434   ins_pipe(pipe_cmov_mem);  // XXX
 7435 %}
 7436 
 7437 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7438 %{
 7439   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7440   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7441 
 7442   ins_cost(100); // XXX
 7443   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7444   ins_encode %{
 7445     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7446     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7447   %}
 7448   ins_pipe(ialu_reg);
 7449 %}
 7450 
 7451 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7452 %{
 7453   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7454 
 7455   ins_cost(200); // XXX
 7456   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7457   ins_encode %{
 7458     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7459   %}
 7460   ins_pipe(pipe_cmov_reg); // XXX
 7461 %}
 7462 
 7463 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7464 %{
 7465   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7466   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7467 
 7468   ins_cost(100); // XXX
 7469   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7470   ins_encode %{
 7471     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7472     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7473   %}
 7474   ins_pipe(ialu_reg);
 7475 %}
 7476 
 7477 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7478   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7479   ins_cost(200);
 7480   expand %{
 7481     cmovL_regU(cop, cr, dst, src);
 7482   %}
 7483 %}
 7484 
 7485 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7486   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7487   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7488 
 7489   ins_cost(200); // XXX
 7490   format %{ "cmovpq  $dst, $src\n\t"
 7491             "cmovneq $dst, $src" %}
 7492   ins_encode %{
 7493     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7494     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7495   %}
 7496   ins_pipe(pipe_cmov_reg);
 7497 %}
 7498 
 7499 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7500 // inputs of the CMove
 7501 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7502   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7503   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7504 
 7505   ins_cost(200); // XXX
 7506   format %{ "cmovpq  $dst, $src\n\t"
 7507             "cmovneq $dst, $src" %}
 7508   ins_encode %{
 7509     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7510     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7511   %}
 7512   ins_pipe(pipe_cmov_reg);
 7513 %}
 7514 
 7515 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7516 %{
 7517   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7518 
 7519   ins_cost(200); // XXX
 7520   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7521   ins_encode %{
 7522     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7523   %}
 7524   ins_pipe(pipe_cmov_mem); // XXX
 7525 %}
 7526 
 7527 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7528   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7529   ins_cost(200);
 7530   expand %{
 7531     cmovL_memU(cop, cr, dst, src);
 7532   %}
 7533 %}
 7534 
 7535 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7536 %{
 7537   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7538 
 7539   ins_cost(200); // XXX
 7540   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7541             "movss     $dst, $src\n"
 7542     "skip:" %}
 7543   ins_encode %{
 7544     Label Lskip;
 7545     // Invert sense of branch from sense of CMOV
 7546     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7547     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7548     __ bind(Lskip);
 7549   %}
 7550   ins_pipe(pipe_slow);
 7551 %}
 7552 
 7553 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7554 // %{
 7555 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7556 
 7557 //   ins_cost(200); // XXX
 7558 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7559 //             "movss     $dst, $src\n"
 7560 //     "skip:" %}
 7561 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7562 //   ins_pipe(pipe_slow);
 7563 // %}
 7564 
 7565 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7566 %{
 7567   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7568 
 7569   ins_cost(200); // XXX
 7570   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7571             "movss     $dst, $src\n"
 7572     "skip:" %}
 7573   ins_encode %{
 7574     Label Lskip;
 7575     // Invert sense of branch from sense of CMOV
 7576     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7577     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7578     __ bind(Lskip);
 7579   %}
 7580   ins_pipe(pipe_slow);
 7581 %}
 7582 
 7583 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7584   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7585   ins_cost(200);
 7586   expand %{
 7587     cmovF_regU(cop, cr, dst, src);
 7588   %}
 7589 %}
 7590 
 7591 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7592 %{
 7593   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7594 
 7595   ins_cost(200); // XXX
 7596   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7597             "movsd     $dst, $src\n"
 7598     "skip:" %}
 7599   ins_encode %{
 7600     Label Lskip;
 7601     // Invert sense of branch from sense of CMOV
 7602     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7603     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7604     __ bind(Lskip);
 7605   %}
 7606   ins_pipe(pipe_slow);
 7607 %}
 7608 
 7609 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7610 %{
 7611   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7612 
 7613   ins_cost(200); // XXX
 7614   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7615             "movsd     $dst, $src\n"
 7616     "skip:" %}
 7617   ins_encode %{
 7618     Label Lskip;
 7619     // Invert sense of branch from sense of CMOV
 7620     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7621     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7622     __ bind(Lskip);
 7623   %}
 7624   ins_pipe(pipe_slow);
 7625 %}
 7626 
 7627 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7628   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7629   ins_cost(200);
 7630   expand %{
 7631     cmovD_regU(cop, cr, dst, src);
 7632   %}
 7633 %}
 7634 
 7635 //----------Arithmetic Instructions--------------------------------------------
 7636 //----------Addition Instructions----------------------------------------------
 7637 
 7638 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7639 %{
 7640   match(Set dst (AddI dst src));
 7641   effect(KILL cr);
 7642 
 7643   format %{ "addl    $dst, $src\t# int" %}
 7644   ins_encode %{
 7645     __ addl($dst$$Register, $src$$Register);
 7646   %}
 7647   ins_pipe(ialu_reg_reg);
 7648 %}
 7649 
 7650 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7651 %{
 7652   match(Set dst (AddI dst src));
 7653   effect(KILL cr);
 7654 
 7655   format %{ "addl    $dst, $src\t# int" %}
 7656   ins_encode %{
 7657     __ addl($dst$$Register, $src$$constant);
 7658   %}
 7659   ins_pipe( ialu_reg );
 7660 %}
 7661 
 7662 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7663 %{
 7664   match(Set dst (AddI dst (LoadI src)));
 7665   effect(KILL cr);
 7666 
 7667   ins_cost(150); // XXX
 7668   format %{ "addl    $dst, $src\t# int" %}
 7669   ins_encode %{
 7670     __ addl($dst$$Register, $src$$Address);
 7671   %}
 7672   ins_pipe(ialu_reg_mem);
 7673 %}
 7674 
 7675 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7676 %{
 7677   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7678   effect(KILL cr);
 7679 
 7680   ins_cost(150); // XXX
 7681   format %{ "addl    $dst, $src\t# int" %}
 7682   ins_encode %{
 7683     __ addl($dst$$Address, $src$$Register);
 7684   %}
 7685   ins_pipe(ialu_mem_reg);
 7686 %}
 7687 
 7688 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7689 %{
 7690   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7691   effect(KILL cr);
 7692 
 7693   ins_cost(125); // XXX
 7694   format %{ "addl    $dst, $src\t# int" %}
 7695   ins_encode %{
 7696     __ addl($dst$$Address, $src$$constant);
 7697   %}
 7698   ins_pipe(ialu_mem_imm);
 7699 %}
 7700 
 7701 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7702 %{
 7703   predicate(UseIncDec);
 7704   match(Set dst (AddI dst src));
 7705   effect(KILL cr);
 7706 
 7707   format %{ "incl    $dst\t# int" %}
 7708   ins_encode %{
 7709     __ incrementl($dst$$Register);
 7710   %}
 7711   ins_pipe(ialu_reg);
 7712 %}
 7713 
 7714 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7715 %{
 7716   predicate(UseIncDec);
 7717   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7718   effect(KILL cr);
 7719 
 7720   ins_cost(125); // XXX
 7721   format %{ "incl    $dst\t# int" %}
 7722   ins_encode %{
 7723     __ incrementl($dst$$Address);
 7724   %}
 7725   ins_pipe(ialu_mem_imm);
 7726 %}
 7727 
 7728 // XXX why does that use AddI
 7729 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7730 %{
 7731   predicate(UseIncDec);
 7732   match(Set dst (AddI dst src));
 7733   effect(KILL cr);
 7734 
 7735   format %{ "decl    $dst\t# int" %}
 7736   ins_encode %{
 7737     __ decrementl($dst$$Register);
 7738   %}
 7739   ins_pipe(ialu_reg);
 7740 %}
 7741 
 7742 // XXX why does that use AddI
 7743 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7744 %{
 7745   predicate(UseIncDec);
 7746   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7747   effect(KILL cr);
 7748 
 7749   ins_cost(125); // XXX
 7750   format %{ "decl    $dst\t# int" %}
 7751   ins_encode %{
 7752     __ decrementl($dst$$Address);
 7753   %}
 7754   ins_pipe(ialu_mem_imm);
 7755 %}
 7756 
 7757 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7758 %{
 7759   predicate(VM_Version::supports_fast_2op_lea());
 7760   match(Set dst (AddI (LShiftI index scale) disp));
 7761 
 7762   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7763   ins_encode %{
 7764     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7765     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7766   %}
 7767   ins_pipe(ialu_reg_reg);
 7768 %}
 7769 
 7770 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7771 %{
 7772   predicate(VM_Version::supports_fast_3op_lea());
 7773   match(Set dst (AddI (AddI base index) disp));
 7774 
 7775   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7776   ins_encode %{
 7777     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7778   %}
 7779   ins_pipe(ialu_reg_reg);
 7780 %}
 7781 
 7782 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7783 %{
 7784   predicate(VM_Version::supports_fast_2op_lea());
 7785   match(Set dst (AddI base (LShiftI index scale)));
 7786 
 7787   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7788   ins_encode %{
 7789     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7790     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7791   %}
 7792   ins_pipe(ialu_reg_reg);
 7793 %}
 7794 
 7795 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7796 %{
 7797   predicate(VM_Version::supports_fast_3op_lea());
 7798   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7799 
 7800   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7801   ins_encode %{
 7802     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7803     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7804   %}
 7805   ins_pipe(ialu_reg_reg);
 7806 %}
 7807 
 7808 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7809 %{
 7810   match(Set dst (AddL dst src));
 7811   effect(KILL cr);
 7812 
 7813   format %{ "addq    $dst, $src\t# long" %}
 7814   ins_encode %{
 7815     __ addq($dst$$Register, $src$$Register);
 7816   %}
 7817   ins_pipe(ialu_reg_reg);
 7818 %}
 7819 
 7820 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7821 %{
 7822   match(Set dst (AddL dst src));
 7823   effect(KILL cr);
 7824 
 7825   format %{ "addq    $dst, $src\t# long" %}
 7826   ins_encode %{
 7827     __ addq($dst$$Register, $src$$constant);
 7828   %}
 7829   ins_pipe( ialu_reg );
 7830 %}
 7831 
 7832 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7833 %{
 7834   match(Set dst (AddL dst (LoadL src)));
 7835   effect(KILL cr);
 7836 
 7837   ins_cost(150); // XXX
 7838   format %{ "addq    $dst, $src\t# long" %}
 7839   ins_encode %{
 7840     __ addq($dst$$Register, $src$$Address);
 7841   %}
 7842   ins_pipe(ialu_reg_mem);
 7843 %}
 7844 
 7845 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7846 %{
 7847   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7848   effect(KILL cr);
 7849 
 7850   ins_cost(150); // XXX
 7851   format %{ "addq    $dst, $src\t# long" %}
 7852   ins_encode %{
 7853     __ addq($dst$$Address, $src$$Register);
 7854   %}
 7855   ins_pipe(ialu_mem_reg);
 7856 %}
 7857 
 7858 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7859 %{
 7860   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7861   effect(KILL cr);
 7862 
 7863   ins_cost(125); // XXX
 7864   format %{ "addq    $dst, $src\t# long" %}
 7865   ins_encode %{
 7866     __ addq($dst$$Address, $src$$constant);
 7867   %}
 7868   ins_pipe(ialu_mem_imm);
 7869 %}
 7870 
 7871 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7872 %{
 7873   predicate(UseIncDec);
 7874   match(Set dst (AddL dst src));
 7875   effect(KILL cr);
 7876 
 7877   format %{ "incq    $dst\t# long" %}
 7878   ins_encode %{
 7879     __ incrementq($dst$$Register);
 7880   %}
 7881   ins_pipe(ialu_reg);
 7882 %}
 7883 
 7884 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7885 %{
 7886   predicate(UseIncDec);
 7887   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7888   effect(KILL cr);
 7889 
 7890   ins_cost(125); // XXX
 7891   format %{ "incq    $dst\t# long" %}
 7892   ins_encode %{
 7893     __ incrementq($dst$$Address);
 7894   %}
 7895   ins_pipe(ialu_mem_imm);
 7896 %}
 7897 
 7898 // XXX why does that use AddL
 7899 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7900 %{
 7901   predicate(UseIncDec);
 7902   match(Set dst (AddL dst src));
 7903   effect(KILL cr);
 7904 
 7905   format %{ "decq    $dst\t# long" %}
 7906   ins_encode %{
 7907     __ decrementq($dst$$Register);
 7908   %}
 7909   ins_pipe(ialu_reg);
 7910 %}
 7911 
 7912 // XXX why does that use AddL
 7913 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7914 %{
 7915   predicate(UseIncDec);
 7916   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7917   effect(KILL cr);
 7918 
 7919   ins_cost(125); // XXX
 7920   format %{ "decq    $dst\t# long" %}
 7921   ins_encode %{
 7922     __ decrementq($dst$$Address);
 7923   %}
 7924   ins_pipe(ialu_mem_imm);
 7925 %}
 7926 
 7927 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7928 %{
 7929   predicate(VM_Version::supports_fast_2op_lea());
 7930   match(Set dst (AddL (LShiftL index scale) disp));
 7931 
 7932   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7933   ins_encode %{
 7934     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7935     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7936   %}
 7937   ins_pipe(ialu_reg_reg);
 7938 %}
 7939 
 7940 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7941 %{
 7942   predicate(VM_Version::supports_fast_3op_lea());
 7943   match(Set dst (AddL (AddL base index) disp));
 7944 
 7945   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7946   ins_encode %{
 7947     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7948   %}
 7949   ins_pipe(ialu_reg_reg);
 7950 %}
 7951 
 7952 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7953 %{
 7954   predicate(VM_Version::supports_fast_2op_lea());
 7955   match(Set dst (AddL base (LShiftL index scale)));
 7956 
 7957   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7958   ins_encode %{
 7959     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7960     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7961   %}
 7962   ins_pipe(ialu_reg_reg);
 7963 %}
 7964 
 7965 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7966 %{
 7967   predicate(VM_Version::supports_fast_3op_lea());
 7968   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7969 
 7970   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7971   ins_encode %{
 7972     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7973     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7974   %}
 7975   ins_pipe(ialu_reg_reg);
 7976 %}
 7977 
 7978 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7979 %{
 7980   match(Set dst (AddP dst src));
 7981   effect(KILL cr);
 7982 
 7983   format %{ "addq    $dst, $src\t# ptr" %}
 7984   ins_encode %{
 7985     __ addq($dst$$Register, $src$$Register);
 7986   %}
 7987   ins_pipe(ialu_reg_reg);
 7988 %}
 7989 
 7990 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7991 %{
 7992   match(Set dst (AddP dst src));
 7993   effect(KILL cr);
 7994 
 7995   format %{ "addq    $dst, $src\t# ptr" %}
 7996   ins_encode %{
 7997     __ addq($dst$$Register, $src$$constant);
 7998   %}
 7999   ins_pipe( ialu_reg );
 8000 %}
 8001 
 8002 // XXX addP mem ops ????
 8003 
 8004 instruct checkCastPP(rRegP dst)
 8005 %{
 8006   match(Set dst (CheckCastPP dst));
 8007 
 8008   size(0);
 8009   format %{ "# checkcastPP of $dst" %}
 8010   ins_encode(/* empty encoding */);
 8011   ins_pipe(empty);
 8012 %}
 8013 
 8014 instruct castPP(rRegP dst)
 8015 %{
 8016   match(Set dst (CastPP dst));
 8017 
 8018   size(0);
 8019   format %{ "# castPP of $dst" %}
 8020   ins_encode(/* empty encoding */);
 8021   ins_pipe(empty);
 8022 %}
 8023 
 8024 instruct castII(rRegI dst)
 8025 %{
 8026   match(Set dst (CastII dst));
 8027 
 8028   size(0);
 8029   format %{ "# castII of $dst" %}
 8030   ins_encode(/* empty encoding */);
 8031   ins_cost(0);
 8032   ins_pipe(empty);
 8033 %}
 8034 
 8035 instruct castLL(rRegL dst)
 8036 %{
 8037   match(Set dst (CastLL dst));
 8038 
 8039   size(0);
 8040   format %{ "# castLL of $dst" %}
 8041   ins_encode(/* empty encoding */);
 8042   ins_cost(0);
 8043   ins_pipe(empty);
 8044 %}
 8045 
 8046 instruct castFF(regF dst)
 8047 %{
 8048   match(Set dst (CastFF dst));
 8049 
 8050   size(0);
 8051   format %{ "# castFF of $dst" %}
 8052   ins_encode(/* empty encoding */);
 8053   ins_cost(0);
 8054   ins_pipe(empty);
 8055 %}
 8056 
 8057 instruct castDD(regD dst)
 8058 %{
 8059   match(Set dst (CastDD dst));
 8060 
 8061   size(0);
 8062   format %{ "# castDD of $dst" %}
 8063   ins_encode(/* empty encoding */);
 8064   ins_cost(0);
 8065   ins_pipe(empty);
 8066 %}
 8067 
 8068 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 8069 instruct compareAndSwapP(rRegI res,
 8070                          memory mem_ptr,
 8071                          rax_RegP oldval, rRegP newval,
 8072                          rFlagsReg cr)
 8073 %{
 8074   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8075   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 8076   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 8077   effect(KILL cr, KILL oldval);
 8078 
 8079   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8080             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8081             "sete    $res\n\t"
 8082             "movzbl  $res, $res" %}
 8083   ins_encode %{
 8084     __ lock();
 8085     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8086     __ sete($res$$Register);
 8087     __ movzbl($res$$Register, $res$$Register);
 8088   %}
 8089   ins_pipe( pipe_cmpxchg );
 8090 %}
 8091 
 8092 instruct compareAndSwapL(rRegI res,
 8093                          memory mem_ptr,
 8094                          rax_RegL oldval, rRegL newval,
 8095                          rFlagsReg cr)
 8096 %{
 8097   predicate(VM_Version::supports_cx8());
 8098   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 8099   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 8100   effect(KILL cr, KILL oldval);
 8101 
 8102   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8103             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8104             "sete    $res\n\t"
 8105             "movzbl  $res, $res" %}
 8106   ins_encode %{
 8107     __ lock();
 8108     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8109     __ sete($res$$Register);
 8110     __ movzbl($res$$Register, $res$$Register);
 8111   %}
 8112   ins_pipe( pipe_cmpxchg );
 8113 %}
 8114 
 8115 instruct compareAndSwapI(rRegI res,
 8116                          memory mem_ptr,
 8117                          rax_RegI oldval, rRegI newval,
 8118                          rFlagsReg cr)
 8119 %{
 8120   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 8121   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 8122   effect(KILL cr, KILL oldval);
 8123 
 8124   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8125             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8126             "sete    $res\n\t"
 8127             "movzbl  $res, $res" %}
 8128   ins_encode %{
 8129     __ lock();
 8130     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8131     __ sete($res$$Register);
 8132     __ movzbl($res$$Register, $res$$Register);
 8133   %}
 8134   ins_pipe( pipe_cmpxchg );
 8135 %}
 8136 
 8137 instruct compareAndSwapB(rRegI res,
 8138                          memory mem_ptr,
 8139                          rax_RegI oldval, rRegI newval,
 8140                          rFlagsReg cr)
 8141 %{
 8142   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 8143   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 8144   effect(KILL cr, KILL oldval);
 8145 
 8146   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8147             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8148             "sete    $res\n\t"
 8149             "movzbl  $res, $res" %}
 8150   ins_encode %{
 8151     __ lock();
 8152     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8153     __ sete($res$$Register);
 8154     __ movzbl($res$$Register, $res$$Register);
 8155   %}
 8156   ins_pipe( pipe_cmpxchg );
 8157 %}
 8158 
 8159 instruct compareAndSwapS(rRegI res,
 8160                          memory mem_ptr,
 8161                          rax_RegI oldval, rRegI newval,
 8162                          rFlagsReg cr)
 8163 %{
 8164   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 8165   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 8166   effect(KILL cr, KILL oldval);
 8167 
 8168   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8169             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8170             "sete    $res\n\t"
 8171             "movzbl  $res, $res" %}
 8172   ins_encode %{
 8173     __ lock();
 8174     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8175     __ sete($res$$Register);
 8176     __ movzbl($res$$Register, $res$$Register);
 8177   %}
 8178   ins_pipe( pipe_cmpxchg );
 8179 %}
 8180 
 8181 instruct compareAndSwapN(rRegI res,
 8182                           memory mem_ptr,
 8183                           rax_RegN oldval, rRegN newval,
 8184                           rFlagsReg cr) %{
 8185   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 8186   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 8187   effect(KILL cr, KILL oldval);
 8188 
 8189   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8190             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8191             "sete    $res\n\t"
 8192             "movzbl  $res, $res" %}
 8193   ins_encode %{
 8194     __ lock();
 8195     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8196     __ sete($res$$Register);
 8197     __ movzbl($res$$Register, $res$$Register);
 8198   %}
 8199   ins_pipe( pipe_cmpxchg );
 8200 %}
 8201 
 8202 instruct compareAndExchangeB(
 8203                          memory mem_ptr,
 8204                          rax_RegI oldval, rRegI newval,
 8205                          rFlagsReg cr)
 8206 %{
 8207   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 8208   effect(KILL cr);
 8209 
 8210   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8211             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8212   ins_encode %{
 8213     __ lock();
 8214     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8215   %}
 8216   ins_pipe( pipe_cmpxchg );
 8217 %}
 8218 
 8219 instruct compareAndExchangeS(
 8220                          memory mem_ptr,
 8221                          rax_RegI oldval, rRegI newval,
 8222                          rFlagsReg cr)
 8223 %{
 8224   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 8225   effect(KILL cr);
 8226 
 8227   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8228             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8229   ins_encode %{
 8230     __ lock();
 8231     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8232   %}
 8233   ins_pipe( pipe_cmpxchg );
 8234 %}
 8235 
 8236 instruct compareAndExchangeI(
 8237                          memory mem_ptr,
 8238                          rax_RegI oldval, rRegI newval,
 8239                          rFlagsReg cr)
 8240 %{
 8241   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 8242   effect(KILL cr);
 8243 
 8244   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8245             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8246   ins_encode %{
 8247     __ lock();
 8248     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8249   %}
 8250   ins_pipe( pipe_cmpxchg );
 8251 %}
 8252 
 8253 instruct compareAndExchangeL(
 8254                          memory mem_ptr,
 8255                          rax_RegL oldval, rRegL newval,
 8256                          rFlagsReg cr)
 8257 %{
 8258   predicate(VM_Version::supports_cx8());
 8259   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 8260   effect(KILL cr);
 8261 
 8262   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8263             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8264   ins_encode %{
 8265     __ lock();
 8266     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8267   %}
 8268   ins_pipe( pipe_cmpxchg );
 8269 %}
 8270 
 8271 instruct compareAndExchangeN(
 8272                           memory mem_ptr,
 8273                           rax_RegN oldval, rRegN newval,
 8274                           rFlagsReg cr) %{
 8275   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 8276   effect(KILL cr);
 8277 
 8278   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8279             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8280   ins_encode %{
 8281     __ lock();
 8282     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8283   %}
 8284   ins_pipe( pipe_cmpxchg );
 8285 %}
 8286 
 8287 instruct compareAndExchangeP(
 8288                          memory mem_ptr,
 8289                          rax_RegP oldval, rRegP newval,
 8290                          rFlagsReg cr)
 8291 %{
 8292   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8293   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 8294   effect(KILL cr);
 8295 
 8296   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8297             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8298   ins_encode %{
 8299     __ lock();
 8300     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8301   %}
 8302   ins_pipe( pipe_cmpxchg );
 8303 %}
 8304 
 8305 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8306   predicate(n->as_LoadStore()->result_not_used());
 8307   match(Set dummy (GetAndAddB mem add));
 8308   effect(KILL cr);
 8309   format %{ "ADDB  [$mem],$add" %}
 8310   ins_encode %{
 8311     __ lock();
 8312     __ addb($mem$$Address, $add$$constant);
 8313   %}
 8314   ins_pipe( pipe_cmpxchg );
 8315 %}
 8316 
 8317 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8318   match(Set newval (GetAndAddB mem newval));
 8319   effect(KILL cr);
 8320   format %{ "XADDB  [$mem],$newval" %}
 8321   ins_encode %{
 8322     __ lock();
 8323     __ xaddb($mem$$Address, $newval$$Register);
 8324   %}
 8325   ins_pipe( pipe_cmpxchg );
 8326 %}
 8327 
 8328 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8329   predicate(n->as_LoadStore()->result_not_used());
 8330   match(Set dummy (GetAndAddS mem add));
 8331   effect(KILL cr);
 8332   format %{ "ADDW  [$mem],$add" %}
 8333   ins_encode %{
 8334     __ lock();
 8335     __ addw($mem$$Address, $add$$constant);
 8336   %}
 8337   ins_pipe( pipe_cmpxchg );
 8338 %}
 8339 
 8340 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8341   match(Set newval (GetAndAddS mem newval));
 8342   effect(KILL cr);
 8343   format %{ "XADDW  [$mem],$newval" %}
 8344   ins_encode %{
 8345     __ lock();
 8346     __ xaddw($mem$$Address, $newval$$Register);
 8347   %}
 8348   ins_pipe( pipe_cmpxchg );
 8349 %}
 8350 
 8351 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8352   predicate(n->as_LoadStore()->result_not_used());
 8353   match(Set dummy (GetAndAddI mem add));
 8354   effect(KILL cr);
 8355   format %{ "ADDL  [$mem],$add" %}
 8356   ins_encode %{
 8357     __ lock();
 8358     __ addl($mem$$Address, $add$$constant);
 8359   %}
 8360   ins_pipe( pipe_cmpxchg );
 8361 %}
 8362 
 8363 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8364   match(Set newval (GetAndAddI mem newval));
 8365   effect(KILL cr);
 8366   format %{ "XADDL  [$mem],$newval" %}
 8367   ins_encode %{
 8368     __ lock();
 8369     __ xaddl($mem$$Address, $newval$$Register);
 8370   %}
 8371   ins_pipe( pipe_cmpxchg );
 8372 %}
 8373 
 8374 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8375   predicate(n->as_LoadStore()->result_not_used());
 8376   match(Set dummy (GetAndAddL mem add));
 8377   effect(KILL cr);
 8378   format %{ "ADDQ  [$mem],$add" %}
 8379   ins_encode %{
 8380     __ lock();
 8381     __ addq($mem$$Address, $add$$constant);
 8382   %}
 8383   ins_pipe( pipe_cmpxchg );
 8384 %}
 8385 
 8386 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8387   match(Set newval (GetAndAddL mem newval));
 8388   effect(KILL cr);
 8389   format %{ "XADDQ  [$mem],$newval" %}
 8390   ins_encode %{
 8391     __ lock();
 8392     __ xaddq($mem$$Address, $newval$$Register);
 8393   %}
 8394   ins_pipe( pipe_cmpxchg );
 8395 %}
 8396 
 8397 instruct xchgB( memory mem, rRegI newval) %{
 8398   match(Set newval (GetAndSetB mem newval));
 8399   format %{ "XCHGB  $newval,[$mem]" %}
 8400   ins_encode %{
 8401     __ xchgb($newval$$Register, $mem$$Address);
 8402   %}
 8403   ins_pipe( pipe_cmpxchg );
 8404 %}
 8405 
 8406 instruct xchgS( memory mem, rRegI newval) %{
 8407   match(Set newval (GetAndSetS mem newval));
 8408   format %{ "XCHGW  $newval,[$mem]" %}
 8409   ins_encode %{
 8410     __ xchgw($newval$$Register, $mem$$Address);
 8411   %}
 8412   ins_pipe( pipe_cmpxchg );
 8413 %}
 8414 
 8415 instruct xchgI( memory mem, rRegI newval) %{
 8416   match(Set newval (GetAndSetI mem newval));
 8417   format %{ "XCHGL  $newval,[$mem]" %}
 8418   ins_encode %{
 8419     __ xchgl($newval$$Register, $mem$$Address);
 8420   %}
 8421   ins_pipe( pipe_cmpxchg );
 8422 %}
 8423 
 8424 instruct xchgL( memory mem, rRegL newval) %{
 8425   match(Set newval (GetAndSetL mem newval));
 8426   format %{ "XCHGL  $newval,[$mem]" %}
 8427   ins_encode %{
 8428     __ xchgq($newval$$Register, $mem$$Address);
 8429   %}
 8430   ins_pipe( pipe_cmpxchg );
 8431 %}
 8432 
 8433 instruct xchgP( memory mem, rRegP newval) %{
 8434   match(Set newval (GetAndSetP mem newval));
 8435   predicate(n->as_LoadStore()->barrier_data() == 0);
 8436   format %{ "XCHGQ  $newval,[$mem]" %}
 8437   ins_encode %{
 8438     __ xchgq($newval$$Register, $mem$$Address);
 8439   %}
 8440   ins_pipe( pipe_cmpxchg );
 8441 %}
 8442 
 8443 instruct xchgN( memory mem, rRegN newval) %{
 8444   match(Set newval (GetAndSetN mem newval));
 8445   format %{ "XCHGL  $newval,$mem]" %}
 8446   ins_encode %{
 8447     __ xchgl($newval$$Register, $mem$$Address);
 8448   %}
 8449   ins_pipe( pipe_cmpxchg );
 8450 %}
 8451 
 8452 //----------Abs Instructions-------------------------------------------
 8453 
 8454 // Integer Absolute Instructions
 8455 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8456 %{
 8457   match(Set dst (AbsI src));
 8458   effect(TEMP dst, TEMP tmp, KILL cr);
 8459   format %{ "movl $tmp, $src\n\t"
 8460             "sarl $tmp, 31\n\t"
 8461             "movl $dst, $src\n\t"
 8462             "xorl $dst, $tmp\n\t"
 8463             "subl $dst, $tmp\n"
 8464           %}
 8465   ins_encode %{
 8466     __ movl($tmp$$Register, $src$$Register);
 8467     __ sarl($tmp$$Register, 31);
 8468     __ movl($dst$$Register, $src$$Register);
 8469     __ xorl($dst$$Register, $tmp$$Register);
 8470     __ subl($dst$$Register, $tmp$$Register);
 8471   %}
 8472 
 8473   ins_pipe(ialu_reg_reg);
 8474 %}
 8475 
 8476 // Long Absolute Instructions
 8477 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8478 %{
 8479   match(Set dst (AbsL src));
 8480   effect(TEMP dst, TEMP tmp, KILL cr);
 8481   format %{ "movq $tmp, $src\n\t"
 8482             "sarq $tmp, 63\n\t"
 8483             "movq $dst, $src\n\t"
 8484             "xorq $dst, $tmp\n\t"
 8485             "subq $dst, $tmp\n"
 8486           %}
 8487   ins_encode %{
 8488     __ movq($tmp$$Register, $src$$Register);
 8489     __ sarq($tmp$$Register, 63);
 8490     __ movq($dst$$Register, $src$$Register);
 8491     __ xorq($dst$$Register, $tmp$$Register);
 8492     __ subq($dst$$Register, $tmp$$Register);
 8493   %}
 8494 
 8495   ins_pipe(ialu_reg_reg);
 8496 %}
 8497 
 8498 //----------Subtraction Instructions-------------------------------------------
 8499 
 8500 // Integer Subtraction Instructions
 8501 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8502 %{
 8503   match(Set dst (SubI dst src));
 8504   effect(KILL cr);
 8505 
 8506   format %{ "subl    $dst, $src\t# int" %}
 8507   ins_encode %{
 8508     __ subl($dst$$Register, $src$$Register);
 8509   %}
 8510   ins_pipe(ialu_reg_reg);
 8511 %}
 8512 
 8513 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8514 %{
 8515   match(Set dst (SubI dst (LoadI src)));
 8516   effect(KILL cr);
 8517 
 8518   ins_cost(150);
 8519   format %{ "subl    $dst, $src\t# int" %}
 8520   ins_encode %{
 8521     __ subl($dst$$Register, $src$$Address);
 8522   %}
 8523   ins_pipe(ialu_reg_mem);
 8524 %}
 8525 
 8526 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8527 %{
 8528   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8529   effect(KILL cr);
 8530 
 8531   ins_cost(150);
 8532   format %{ "subl    $dst, $src\t# int" %}
 8533   ins_encode %{
 8534     __ subl($dst$$Address, $src$$Register);
 8535   %}
 8536   ins_pipe(ialu_mem_reg);
 8537 %}
 8538 
 8539 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8540 %{
 8541   match(Set dst (SubL dst src));
 8542   effect(KILL cr);
 8543 
 8544   format %{ "subq    $dst, $src\t# long" %}
 8545   ins_encode %{
 8546     __ subq($dst$$Register, $src$$Register);
 8547   %}
 8548   ins_pipe(ialu_reg_reg);
 8549 %}
 8550 
 8551 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8552 %{
 8553   match(Set dst (SubL dst (LoadL src)));
 8554   effect(KILL cr);
 8555 
 8556   ins_cost(150);
 8557   format %{ "subq    $dst, $src\t# long" %}
 8558   ins_encode %{
 8559     __ subq($dst$$Register, $src$$Address);
 8560   %}
 8561   ins_pipe(ialu_reg_mem);
 8562 %}
 8563 
 8564 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8565 %{
 8566   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8567   effect(KILL cr);
 8568 
 8569   ins_cost(150);
 8570   format %{ "subq    $dst, $src\t# long" %}
 8571   ins_encode %{
 8572     __ subq($dst$$Address, $src$$Register);
 8573   %}
 8574   ins_pipe(ialu_mem_reg);
 8575 %}
 8576 
 8577 // Subtract from a pointer
 8578 // XXX hmpf???
 8579 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8580 %{
 8581   match(Set dst (AddP dst (SubI zero src)));
 8582   effect(KILL cr);
 8583 
 8584   format %{ "subq    $dst, $src\t# ptr - int" %}
 8585   opcode(0x2B);
 8586   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8587   ins_pipe(ialu_reg_reg);
 8588 %}
 8589 
 8590 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8591 %{
 8592   match(Set dst (SubI zero dst));
 8593   effect(KILL cr);
 8594 
 8595   format %{ "negl    $dst\t# int" %}
 8596   ins_encode %{
 8597     __ negl($dst$$Register);
 8598   %}
 8599   ins_pipe(ialu_reg);
 8600 %}
 8601 
 8602 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8603 %{
 8604   match(Set dst (NegI dst));
 8605   effect(KILL cr);
 8606 
 8607   format %{ "negl    $dst\t# int" %}
 8608   ins_encode %{
 8609     __ negl($dst$$Register);
 8610   %}
 8611   ins_pipe(ialu_reg);
 8612 %}
 8613 
 8614 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8615 %{
 8616   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8617   effect(KILL cr);
 8618 
 8619   format %{ "negl    $dst\t# int" %}
 8620   ins_encode %{
 8621     __ negl($dst$$Address);
 8622   %}
 8623   ins_pipe(ialu_reg);
 8624 %}
 8625 
 8626 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8627 %{
 8628   match(Set dst (SubL zero dst));
 8629   effect(KILL cr);
 8630 
 8631   format %{ "negq    $dst\t# long" %}
 8632   ins_encode %{
 8633     __ negq($dst$$Register);
 8634   %}
 8635   ins_pipe(ialu_reg);
 8636 %}
 8637 
 8638 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8639 %{
 8640   match(Set dst (NegL dst));
 8641   effect(KILL cr);
 8642 
 8643   format %{ "negq    $dst\t# int" %}
 8644   ins_encode %{
 8645     __ negq($dst$$Register);
 8646   %}
 8647   ins_pipe(ialu_reg);
 8648 %}
 8649 
 8650 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8651 %{
 8652   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8653   effect(KILL cr);
 8654 
 8655   format %{ "negq    $dst\t# long" %}
 8656   ins_encode %{
 8657     __ negq($dst$$Address);
 8658   %}
 8659   ins_pipe(ialu_reg);
 8660 %}
 8661 
 8662 //----------Multiplication/Division Instructions-------------------------------
 8663 // Integer Multiplication Instructions
 8664 // Multiply Register
 8665 
 8666 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8667 %{
 8668   match(Set dst (MulI dst src));
 8669   effect(KILL cr);
 8670 
 8671   ins_cost(300);
 8672   format %{ "imull   $dst, $src\t# int" %}
 8673   ins_encode %{
 8674     __ imull($dst$$Register, $src$$Register);
 8675   %}
 8676   ins_pipe(ialu_reg_reg_alu0);
 8677 %}
 8678 
 8679 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8680 %{
 8681   match(Set dst (MulI src imm));
 8682   effect(KILL cr);
 8683 
 8684   ins_cost(300);
 8685   format %{ "imull   $dst, $src, $imm\t# int" %}
 8686   ins_encode %{
 8687     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8688   %}
 8689   ins_pipe(ialu_reg_reg_alu0);
 8690 %}
 8691 
 8692 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8693 %{
 8694   match(Set dst (MulI dst (LoadI src)));
 8695   effect(KILL cr);
 8696 
 8697   ins_cost(350);
 8698   format %{ "imull   $dst, $src\t# int" %}
 8699   ins_encode %{
 8700     __ imull($dst$$Register, $src$$Address);
 8701   %}
 8702   ins_pipe(ialu_reg_mem_alu0);
 8703 %}
 8704 
 8705 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8706 %{
 8707   match(Set dst (MulI (LoadI src) imm));
 8708   effect(KILL cr);
 8709 
 8710   ins_cost(300);
 8711   format %{ "imull   $dst, $src, $imm\t# int" %}
 8712   ins_encode %{
 8713     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8714   %}
 8715   ins_pipe(ialu_reg_mem_alu0);
 8716 %}
 8717 
 8718 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8719 %{
 8720   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8721   effect(KILL cr, KILL src2);
 8722 
 8723   expand %{ mulI_rReg(dst, src1, cr);
 8724            mulI_rReg(src2, src3, cr);
 8725            addI_rReg(dst, src2, cr); %}
 8726 %}
 8727 
 8728 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8729 %{
 8730   match(Set dst (MulL dst src));
 8731   effect(KILL cr);
 8732 
 8733   ins_cost(300);
 8734   format %{ "imulq   $dst, $src\t# long" %}
 8735   ins_encode %{
 8736     __ imulq($dst$$Register, $src$$Register);
 8737   %}
 8738   ins_pipe(ialu_reg_reg_alu0);
 8739 %}
 8740 
 8741 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8742 %{
 8743   match(Set dst (MulL src imm));
 8744   effect(KILL cr);
 8745 
 8746   ins_cost(300);
 8747   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8748   ins_encode %{
 8749     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8750   %}
 8751   ins_pipe(ialu_reg_reg_alu0);
 8752 %}
 8753 
 8754 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8755 %{
 8756   match(Set dst (MulL dst (LoadL src)));
 8757   effect(KILL cr);
 8758 
 8759   ins_cost(350);
 8760   format %{ "imulq   $dst, $src\t# long" %}
 8761   ins_encode %{
 8762     __ imulq($dst$$Register, $src$$Address);
 8763   %}
 8764   ins_pipe(ialu_reg_mem_alu0);
 8765 %}
 8766 
 8767 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8768 %{
 8769   match(Set dst (MulL (LoadL src) imm));
 8770   effect(KILL cr);
 8771 
 8772   ins_cost(300);
 8773   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8774   ins_encode %{
 8775     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8776   %}
 8777   ins_pipe(ialu_reg_mem_alu0);
 8778 %}
 8779 
 8780 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8781 %{
 8782   match(Set dst (MulHiL src rax));
 8783   effect(USE_KILL rax, KILL cr);
 8784 
 8785   ins_cost(300);
 8786   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8787   ins_encode %{
 8788     __ imulq($src$$Register);
 8789   %}
 8790   ins_pipe(ialu_reg_reg_alu0);
 8791 %}
 8792 
 8793 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8794 %{
 8795   match(Set dst (UMulHiL src rax));
 8796   effect(USE_KILL rax, KILL cr);
 8797 
 8798   ins_cost(300);
 8799   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8800   ins_encode %{
 8801     __ mulq($src$$Register);
 8802   %}
 8803   ins_pipe(ialu_reg_reg_alu0);
 8804 %}
 8805 
 8806 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8807                    rFlagsReg cr)
 8808 %{
 8809   match(Set rax (DivI rax div));
 8810   effect(KILL rdx, KILL cr);
 8811 
 8812   ins_cost(30*100+10*100); // XXX
 8813   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8814             "jne,s   normal\n\t"
 8815             "xorl    rdx, rdx\n\t"
 8816             "cmpl    $div, -1\n\t"
 8817             "je,s    done\n"
 8818     "normal: cdql\n\t"
 8819             "idivl   $div\n"
 8820     "done:"        %}
 8821   ins_encode(cdql_enc(div));
 8822   ins_pipe(ialu_reg_reg_alu0);
 8823 %}
 8824 
 8825 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8826                    rFlagsReg cr)
 8827 %{
 8828   match(Set rax (DivL rax div));
 8829   effect(KILL rdx, KILL cr);
 8830 
 8831   ins_cost(30*100+10*100); // XXX
 8832   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8833             "cmpq    rax, rdx\n\t"
 8834             "jne,s   normal\n\t"
 8835             "xorl    rdx, rdx\n\t"
 8836             "cmpq    $div, -1\n\t"
 8837             "je,s    done\n"
 8838     "normal: cdqq\n\t"
 8839             "idivq   $div\n"
 8840     "done:"        %}
 8841   ins_encode(cdqq_enc(div));
 8842   ins_pipe(ialu_reg_reg_alu0);
 8843 %}
 8844 
 8845 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8846 %{
 8847   match(Set rax (UDivI rax div));
 8848   effect(KILL rdx, KILL cr);
 8849 
 8850   ins_cost(300);
 8851   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8852   ins_encode %{
 8853     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8854   %}
 8855   ins_pipe(ialu_reg_reg_alu0);
 8856 %}
 8857 
 8858 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8859 %{
 8860   match(Set rax (UDivL rax div));
 8861   effect(KILL rdx, KILL cr);
 8862 
 8863   ins_cost(300);
 8864   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8865   ins_encode %{
 8866      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8867   %}
 8868   ins_pipe(ialu_reg_reg_alu0);
 8869 %}
 8870 
 8871 // Integer DIVMOD with Register, both quotient and mod results
 8872 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8873                              rFlagsReg cr)
 8874 %{
 8875   match(DivModI rax div);
 8876   effect(KILL cr);
 8877 
 8878   ins_cost(30*100+10*100); // XXX
 8879   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8880             "jne,s   normal\n\t"
 8881             "xorl    rdx, rdx\n\t"
 8882             "cmpl    $div, -1\n\t"
 8883             "je,s    done\n"
 8884     "normal: cdql\n\t"
 8885             "idivl   $div\n"
 8886     "done:"        %}
 8887   ins_encode(cdql_enc(div));
 8888   ins_pipe(pipe_slow);
 8889 %}
 8890 
 8891 // Long DIVMOD with Register, both quotient and mod results
 8892 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8893                              rFlagsReg cr)
 8894 %{
 8895   match(DivModL rax div);
 8896   effect(KILL cr);
 8897 
 8898   ins_cost(30*100+10*100); // XXX
 8899   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8900             "cmpq    rax, rdx\n\t"
 8901             "jne,s   normal\n\t"
 8902             "xorl    rdx, rdx\n\t"
 8903             "cmpq    $div, -1\n\t"
 8904             "je,s    done\n"
 8905     "normal: cdqq\n\t"
 8906             "idivq   $div\n"
 8907     "done:"        %}
 8908   ins_encode(cdqq_enc(div));
 8909   ins_pipe(pipe_slow);
 8910 %}
 8911 
 8912 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8913 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8914                               no_rax_rdx_RegI div, rFlagsReg cr)
 8915 %{
 8916   match(UDivModI rax div);
 8917   effect(TEMP tmp, KILL cr);
 8918 
 8919   ins_cost(300);
 8920   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8921             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8922           %}
 8923   ins_encode %{
 8924     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8925   %}
 8926   ins_pipe(pipe_slow);
 8927 %}
 8928 
 8929 // Unsigned long DIVMOD with Register, both quotient and mod results
 8930 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8931                               no_rax_rdx_RegL div, rFlagsReg cr)
 8932 %{
 8933   match(UDivModL rax div);
 8934   effect(TEMP tmp, KILL cr);
 8935 
 8936   ins_cost(300);
 8937   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8938             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8939           %}
 8940   ins_encode %{
 8941     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8942   %}
 8943   ins_pipe(pipe_slow);
 8944 %}
 8945 
 8946 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8947                    rFlagsReg cr)
 8948 %{
 8949   match(Set rdx (ModI rax div));
 8950   effect(KILL rax, KILL cr);
 8951 
 8952   ins_cost(300); // XXX
 8953   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8954             "jne,s   normal\n\t"
 8955             "xorl    rdx, rdx\n\t"
 8956             "cmpl    $div, -1\n\t"
 8957             "je,s    done\n"
 8958     "normal: cdql\n\t"
 8959             "idivl   $div\n"
 8960     "done:"        %}
 8961   ins_encode(cdql_enc(div));
 8962   ins_pipe(ialu_reg_reg_alu0);
 8963 %}
 8964 
 8965 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8966                    rFlagsReg cr)
 8967 %{
 8968   match(Set rdx (ModL rax div));
 8969   effect(KILL rax, KILL cr);
 8970 
 8971   ins_cost(300); // XXX
 8972   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8973             "cmpq    rax, rdx\n\t"
 8974             "jne,s   normal\n\t"
 8975             "xorl    rdx, rdx\n\t"
 8976             "cmpq    $div, -1\n\t"
 8977             "je,s    done\n"
 8978     "normal: cdqq\n\t"
 8979             "idivq   $div\n"
 8980     "done:"        %}
 8981   ins_encode(cdqq_enc(div));
 8982   ins_pipe(ialu_reg_reg_alu0);
 8983 %}
 8984 
 8985 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8986 %{
 8987   match(Set rdx (UModI rax div));
 8988   effect(KILL rax, KILL cr);
 8989 
 8990   ins_cost(300);
 8991   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8992   ins_encode %{
 8993     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8994   %}
 8995   ins_pipe(ialu_reg_reg_alu0);
 8996 %}
 8997 
 8998 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8999 %{
 9000   match(Set rdx (UModL rax div));
 9001   effect(KILL rax, KILL cr);
 9002 
 9003   ins_cost(300);
 9004   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9005   ins_encode %{
 9006     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9007   %}
 9008   ins_pipe(ialu_reg_reg_alu0);
 9009 %}
 9010 
 9011 // Integer Shift Instructions
 9012 // Shift Left by one, two, three
 9013 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 9014 %{
 9015   match(Set dst (LShiftI dst shift));
 9016   effect(KILL cr);
 9017 
 9018   format %{ "sall    $dst, $shift" %}
 9019   ins_encode %{
 9020     __ sall($dst$$Register, $shift$$constant);
 9021   %}
 9022   ins_pipe(ialu_reg);
 9023 %}
 9024 
 9025 // Shift Left by 8-bit immediate
 9026 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9027 %{
 9028   match(Set dst (LShiftI dst shift));
 9029   effect(KILL cr);
 9030 
 9031   format %{ "sall    $dst, $shift" %}
 9032   ins_encode %{
 9033     __ sall($dst$$Register, $shift$$constant);
 9034   %}
 9035   ins_pipe(ialu_reg);
 9036 %}
 9037 
 9038 // Shift Left by 8-bit immediate
 9039 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9040 %{
 9041   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9042   effect(KILL cr);
 9043 
 9044   format %{ "sall    $dst, $shift" %}
 9045   ins_encode %{
 9046     __ sall($dst$$Address, $shift$$constant);
 9047   %}
 9048   ins_pipe(ialu_mem_imm);
 9049 %}
 9050 
 9051 // Shift Left by variable
 9052 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9053 %{
 9054   predicate(!VM_Version::supports_bmi2());
 9055   match(Set dst (LShiftI dst shift));
 9056   effect(KILL cr);
 9057 
 9058   format %{ "sall    $dst, $shift" %}
 9059   ins_encode %{
 9060     __ sall($dst$$Register);
 9061   %}
 9062   ins_pipe(ialu_reg_reg);
 9063 %}
 9064 
 9065 // Shift Left by variable
 9066 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9067 %{
 9068   predicate(!VM_Version::supports_bmi2());
 9069   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9070   effect(KILL cr);
 9071 
 9072   format %{ "sall    $dst, $shift" %}
 9073   ins_encode %{
 9074     __ sall($dst$$Address);
 9075   %}
 9076   ins_pipe(ialu_mem_reg);
 9077 %}
 9078 
 9079 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9080 %{
 9081   predicate(VM_Version::supports_bmi2());
 9082   match(Set dst (LShiftI src shift));
 9083 
 9084   format %{ "shlxl   $dst, $src, $shift" %}
 9085   ins_encode %{
 9086     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9087   %}
 9088   ins_pipe(ialu_reg_reg);
 9089 %}
 9090 
 9091 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9092 %{
 9093   predicate(VM_Version::supports_bmi2());
 9094   match(Set dst (LShiftI (LoadI src) shift));
 9095   ins_cost(175);
 9096   format %{ "shlxl   $dst, $src, $shift" %}
 9097   ins_encode %{
 9098     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9099   %}
 9100   ins_pipe(ialu_reg_mem);
 9101 %}
 9102 
 9103 // Arithmetic Shift Right by 8-bit immediate
 9104 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9105 %{
 9106   match(Set dst (RShiftI dst shift));
 9107   effect(KILL cr);
 9108 
 9109   format %{ "sarl    $dst, $shift" %}
 9110   ins_encode %{
 9111     __ sarl($dst$$Register, $shift$$constant);
 9112   %}
 9113   ins_pipe(ialu_mem_imm);
 9114 %}
 9115 
 9116 // Arithmetic Shift Right by 8-bit immediate
 9117 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9118 %{
 9119   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9120   effect(KILL cr);
 9121 
 9122   format %{ "sarl    $dst, $shift" %}
 9123   ins_encode %{
 9124     __ sarl($dst$$Address, $shift$$constant);
 9125   %}
 9126   ins_pipe(ialu_mem_imm);
 9127 %}
 9128 
 9129 // Arithmetic Shift Right by variable
 9130 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9131 %{
 9132   predicate(!VM_Version::supports_bmi2());
 9133   match(Set dst (RShiftI dst shift));
 9134   effect(KILL cr);
 9135   format %{ "sarl    $dst, $shift" %}
 9136   ins_encode %{
 9137     __ sarl($dst$$Register);
 9138   %}
 9139   ins_pipe(ialu_reg_reg);
 9140 %}
 9141 
 9142 // Arithmetic Shift Right by variable
 9143 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9144 %{
 9145   predicate(!VM_Version::supports_bmi2());
 9146   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9147   effect(KILL cr);
 9148 
 9149   format %{ "sarl    $dst, $shift" %}
 9150   ins_encode %{
 9151     __ sarl($dst$$Address);
 9152   %}
 9153   ins_pipe(ialu_mem_reg);
 9154 %}
 9155 
 9156 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9157 %{
 9158   predicate(VM_Version::supports_bmi2());
 9159   match(Set dst (RShiftI src shift));
 9160 
 9161   format %{ "sarxl   $dst, $src, $shift" %}
 9162   ins_encode %{
 9163     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9164   %}
 9165   ins_pipe(ialu_reg_reg);
 9166 %}
 9167 
 9168 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9169 %{
 9170   predicate(VM_Version::supports_bmi2());
 9171   match(Set dst (RShiftI (LoadI src) shift));
 9172   ins_cost(175);
 9173   format %{ "sarxl   $dst, $src, $shift" %}
 9174   ins_encode %{
 9175     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9176   %}
 9177   ins_pipe(ialu_reg_mem);
 9178 %}
 9179 
 9180 // Logical Shift Right by 8-bit immediate
 9181 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9182 %{
 9183   match(Set dst (URShiftI dst shift));
 9184   effect(KILL cr);
 9185 
 9186   format %{ "shrl    $dst, $shift" %}
 9187   ins_encode %{
 9188     __ shrl($dst$$Register, $shift$$constant);
 9189   %}
 9190   ins_pipe(ialu_reg);
 9191 %}
 9192 
 9193 // Logical Shift Right by 8-bit immediate
 9194 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9195 %{
 9196   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9197   effect(KILL cr);
 9198 
 9199   format %{ "shrl    $dst, $shift" %}
 9200   ins_encode %{
 9201     __ shrl($dst$$Address, $shift$$constant);
 9202   %}
 9203   ins_pipe(ialu_mem_imm);
 9204 %}
 9205 
 9206 // Logical Shift Right by variable
 9207 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9208 %{
 9209   predicate(!VM_Version::supports_bmi2());
 9210   match(Set dst (URShiftI dst shift));
 9211   effect(KILL cr);
 9212 
 9213   format %{ "shrl    $dst, $shift" %}
 9214   ins_encode %{
 9215     __ shrl($dst$$Register);
 9216   %}
 9217   ins_pipe(ialu_reg_reg);
 9218 %}
 9219 
 9220 // Logical Shift Right by variable
 9221 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9222 %{
 9223   predicate(!VM_Version::supports_bmi2());
 9224   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9225   effect(KILL cr);
 9226 
 9227   format %{ "shrl    $dst, $shift" %}
 9228   ins_encode %{
 9229     __ shrl($dst$$Address);
 9230   %}
 9231   ins_pipe(ialu_mem_reg);
 9232 %}
 9233 
 9234 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9235 %{
 9236   predicate(VM_Version::supports_bmi2());
 9237   match(Set dst (URShiftI src shift));
 9238 
 9239   format %{ "shrxl   $dst, $src, $shift" %}
 9240   ins_encode %{
 9241     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9242   %}
 9243   ins_pipe(ialu_reg_reg);
 9244 %}
 9245 
 9246 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9247 %{
 9248   predicate(VM_Version::supports_bmi2());
 9249   match(Set dst (URShiftI (LoadI src) shift));
 9250   ins_cost(175);
 9251   format %{ "shrxl   $dst, $src, $shift" %}
 9252   ins_encode %{
 9253     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9254   %}
 9255   ins_pipe(ialu_reg_mem);
 9256 %}
 9257 
 9258 // Long Shift Instructions
 9259 // Shift Left by one, two, three
 9260 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9261 %{
 9262   match(Set dst (LShiftL dst shift));
 9263   effect(KILL cr);
 9264 
 9265   format %{ "salq    $dst, $shift" %}
 9266   ins_encode %{
 9267     __ salq($dst$$Register, $shift$$constant);
 9268   %}
 9269   ins_pipe(ialu_reg);
 9270 %}
 9271 
 9272 // Shift Left by 8-bit immediate
 9273 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9274 %{
 9275   match(Set dst (LShiftL dst shift));
 9276   effect(KILL cr);
 9277 
 9278   format %{ "salq    $dst, $shift" %}
 9279   ins_encode %{
 9280     __ salq($dst$$Register, $shift$$constant);
 9281   %}
 9282   ins_pipe(ialu_reg);
 9283 %}
 9284 
 9285 // Shift Left by 8-bit immediate
 9286 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9287 %{
 9288   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9289   effect(KILL cr);
 9290 
 9291   format %{ "salq    $dst, $shift" %}
 9292   ins_encode %{
 9293     __ salq($dst$$Address, $shift$$constant);
 9294   %}
 9295   ins_pipe(ialu_mem_imm);
 9296 %}
 9297 
 9298 // Shift Left by variable
 9299 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9300 %{
 9301   predicate(!VM_Version::supports_bmi2());
 9302   match(Set dst (LShiftL dst shift));
 9303   effect(KILL cr);
 9304 
 9305   format %{ "salq    $dst, $shift" %}
 9306   ins_encode %{
 9307     __ salq($dst$$Register);
 9308   %}
 9309   ins_pipe(ialu_reg_reg);
 9310 %}
 9311 
 9312 // Shift Left by variable
 9313 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9314 %{
 9315   predicate(!VM_Version::supports_bmi2());
 9316   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9317   effect(KILL cr);
 9318 
 9319   format %{ "salq    $dst, $shift" %}
 9320   ins_encode %{
 9321     __ salq($dst$$Address);
 9322   %}
 9323   ins_pipe(ialu_mem_reg);
 9324 %}
 9325 
 9326 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9327 %{
 9328   predicate(VM_Version::supports_bmi2());
 9329   match(Set dst (LShiftL src shift));
 9330 
 9331   format %{ "shlxq   $dst, $src, $shift" %}
 9332   ins_encode %{
 9333     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9334   %}
 9335   ins_pipe(ialu_reg_reg);
 9336 %}
 9337 
 9338 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9339 %{
 9340   predicate(VM_Version::supports_bmi2());
 9341   match(Set dst (LShiftL (LoadL src) shift));
 9342   ins_cost(175);
 9343   format %{ "shlxq   $dst, $src, $shift" %}
 9344   ins_encode %{
 9345     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9346   %}
 9347   ins_pipe(ialu_reg_mem);
 9348 %}
 9349 
 9350 // Arithmetic Shift Right by 8-bit immediate
 9351 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9352 %{
 9353   match(Set dst (RShiftL dst shift));
 9354   effect(KILL cr);
 9355 
 9356   format %{ "sarq    $dst, $shift" %}
 9357   ins_encode %{
 9358     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9359   %}
 9360   ins_pipe(ialu_mem_imm);
 9361 %}
 9362 
 9363 // Arithmetic Shift Right by 8-bit immediate
 9364 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9365 %{
 9366   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9367   effect(KILL cr);
 9368 
 9369   format %{ "sarq    $dst, $shift" %}
 9370   ins_encode %{
 9371     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9372   %}
 9373   ins_pipe(ialu_mem_imm);
 9374 %}
 9375 
 9376 // Arithmetic Shift Right by variable
 9377 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9378 %{
 9379   predicate(!VM_Version::supports_bmi2());
 9380   match(Set dst (RShiftL dst shift));
 9381   effect(KILL cr);
 9382 
 9383   format %{ "sarq    $dst, $shift" %}
 9384   ins_encode %{
 9385     __ sarq($dst$$Register);
 9386   %}
 9387   ins_pipe(ialu_reg_reg);
 9388 %}
 9389 
 9390 // Arithmetic Shift Right by variable
 9391 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9392 %{
 9393   predicate(!VM_Version::supports_bmi2());
 9394   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9395   effect(KILL cr);
 9396 
 9397   format %{ "sarq    $dst, $shift" %}
 9398   ins_encode %{
 9399     __ sarq($dst$$Address);
 9400   %}
 9401   ins_pipe(ialu_mem_reg);
 9402 %}
 9403 
 9404 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9405 %{
 9406   predicate(VM_Version::supports_bmi2());
 9407   match(Set dst (RShiftL src shift));
 9408 
 9409   format %{ "sarxq   $dst, $src, $shift" %}
 9410   ins_encode %{
 9411     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9412   %}
 9413   ins_pipe(ialu_reg_reg);
 9414 %}
 9415 
 9416 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9417 %{
 9418   predicate(VM_Version::supports_bmi2());
 9419   match(Set dst (RShiftL (LoadL src) shift));
 9420   ins_cost(175);
 9421   format %{ "sarxq   $dst, $src, $shift" %}
 9422   ins_encode %{
 9423     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9424   %}
 9425   ins_pipe(ialu_reg_mem);
 9426 %}
 9427 
 9428 // Logical Shift Right by 8-bit immediate
 9429 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9430 %{
 9431   match(Set dst (URShiftL dst shift));
 9432   effect(KILL cr);
 9433 
 9434   format %{ "shrq    $dst, $shift" %}
 9435   ins_encode %{
 9436     __ shrq($dst$$Register, $shift$$constant);
 9437   %}
 9438   ins_pipe(ialu_reg);
 9439 %}
 9440 
 9441 // Logical Shift Right by 8-bit immediate
 9442 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9443 %{
 9444   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9445   effect(KILL cr);
 9446 
 9447   format %{ "shrq    $dst, $shift" %}
 9448   ins_encode %{
 9449     __ shrq($dst$$Address, $shift$$constant);
 9450   %}
 9451   ins_pipe(ialu_mem_imm);
 9452 %}
 9453 
 9454 // Logical Shift Right by variable
 9455 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9456 %{
 9457   predicate(!VM_Version::supports_bmi2());
 9458   match(Set dst (URShiftL dst shift));
 9459   effect(KILL cr);
 9460 
 9461   format %{ "shrq    $dst, $shift" %}
 9462   ins_encode %{
 9463     __ shrq($dst$$Register);
 9464   %}
 9465   ins_pipe(ialu_reg_reg);
 9466 %}
 9467 
 9468 // Logical Shift Right by variable
 9469 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9470 %{
 9471   predicate(!VM_Version::supports_bmi2());
 9472   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9473   effect(KILL cr);
 9474 
 9475   format %{ "shrq    $dst, $shift" %}
 9476   ins_encode %{
 9477     __ shrq($dst$$Address);
 9478   %}
 9479   ins_pipe(ialu_mem_reg);
 9480 %}
 9481 
 9482 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9483 %{
 9484   predicate(VM_Version::supports_bmi2());
 9485   match(Set dst (URShiftL src shift));
 9486 
 9487   format %{ "shrxq   $dst, $src, $shift" %}
 9488   ins_encode %{
 9489     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9490   %}
 9491   ins_pipe(ialu_reg_reg);
 9492 %}
 9493 
 9494 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9495 %{
 9496   predicate(VM_Version::supports_bmi2());
 9497   match(Set dst (URShiftL (LoadL src) shift));
 9498   ins_cost(175);
 9499   format %{ "shrxq   $dst, $src, $shift" %}
 9500   ins_encode %{
 9501     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9502   %}
 9503   ins_pipe(ialu_reg_mem);
 9504 %}
 9505 
 9506 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9507 // This idiom is used by the compiler for the i2b bytecode.
 9508 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9509 %{
 9510   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9511 
 9512   format %{ "movsbl  $dst, $src\t# i2b" %}
 9513   ins_encode %{
 9514     __ movsbl($dst$$Register, $src$$Register);
 9515   %}
 9516   ins_pipe(ialu_reg_reg);
 9517 %}
 9518 
 9519 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9520 // This idiom is used by the compiler the i2s bytecode.
 9521 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9522 %{
 9523   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9524 
 9525   format %{ "movswl  $dst, $src\t# i2s" %}
 9526   ins_encode %{
 9527     __ movswl($dst$$Register, $src$$Register);
 9528   %}
 9529   ins_pipe(ialu_reg_reg);
 9530 %}
 9531 
 9532 // ROL/ROR instructions
 9533 
 9534 // Rotate left by constant.
 9535 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9536 %{
 9537   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9538   match(Set dst (RotateLeft dst shift));
 9539   effect(KILL cr);
 9540   format %{ "roll    $dst, $shift" %}
 9541   ins_encode %{
 9542     __ roll($dst$$Register, $shift$$constant);
 9543   %}
 9544   ins_pipe(ialu_reg);
 9545 %}
 9546 
 9547 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9548 %{
 9549   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9550   match(Set dst (RotateLeft src shift));
 9551   format %{ "rolxl   $dst, $src, $shift" %}
 9552   ins_encode %{
 9553     int shift = 32 - ($shift$$constant & 31);
 9554     __ rorxl($dst$$Register, $src$$Register, shift);
 9555   %}
 9556   ins_pipe(ialu_reg_reg);
 9557 %}
 9558 
 9559 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9560 %{
 9561   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9562   match(Set dst (RotateLeft (LoadI src) shift));
 9563   ins_cost(175);
 9564   format %{ "rolxl   $dst, $src, $shift" %}
 9565   ins_encode %{
 9566     int shift = 32 - ($shift$$constant & 31);
 9567     __ rorxl($dst$$Register, $src$$Address, shift);
 9568   %}
 9569   ins_pipe(ialu_reg_mem);
 9570 %}
 9571 
 9572 // Rotate Left by variable
 9573 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9574 %{
 9575   predicate(n->bottom_type()->basic_type() == T_INT);
 9576   match(Set dst (RotateLeft dst shift));
 9577   effect(KILL cr);
 9578   format %{ "roll    $dst, $shift" %}
 9579   ins_encode %{
 9580     __ roll($dst$$Register);
 9581   %}
 9582   ins_pipe(ialu_reg_reg);
 9583 %}
 9584 
 9585 // Rotate Right by constant.
 9586 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9587 %{
 9588   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9589   match(Set dst (RotateRight dst shift));
 9590   effect(KILL cr);
 9591   format %{ "rorl    $dst, $shift" %}
 9592   ins_encode %{
 9593     __ rorl($dst$$Register, $shift$$constant);
 9594   %}
 9595   ins_pipe(ialu_reg);
 9596 %}
 9597 
 9598 // Rotate Right by constant.
 9599 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9600 %{
 9601   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9602   match(Set dst (RotateRight src shift));
 9603   format %{ "rorxl   $dst, $src, $shift" %}
 9604   ins_encode %{
 9605     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9606   %}
 9607   ins_pipe(ialu_reg_reg);
 9608 %}
 9609 
 9610 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9611 %{
 9612   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9613   match(Set dst (RotateRight (LoadI src) shift));
 9614   ins_cost(175);
 9615   format %{ "rorxl   $dst, $src, $shift" %}
 9616   ins_encode %{
 9617     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9618   %}
 9619   ins_pipe(ialu_reg_mem);
 9620 %}
 9621 
 9622 // Rotate Right by variable
 9623 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9624 %{
 9625   predicate(n->bottom_type()->basic_type() == T_INT);
 9626   match(Set dst (RotateRight dst shift));
 9627   effect(KILL cr);
 9628   format %{ "rorl    $dst, $shift" %}
 9629   ins_encode %{
 9630     __ rorl($dst$$Register);
 9631   %}
 9632   ins_pipe(ialu_reg_reg);
 9633 %}
 9634 
 9635 // Rotate Left by constant.
 9636 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9637 %{
 9638   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9639   match(Set dst (RotateLeft dst shift));
 9640   effect(KILL cr);
 9641   format %{ "rolq    $dst, $shift" %}
 9642   ins_encode %{
 9643     __ rolq($dst$$Register, $shift$$constant);
 9644   %}
 9645   ins_pipe(ialu_reg);
 9646 %}
 9647 
 9648 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9649 %{
 9650   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9651   match(Set dst (RotateLeft src shift));
 9652   format %{ "rolxq   $dst, $src, $shift" %}
 9653   ins_encode %{
 9654     int shift = 64 - ($shift$$constant & 63);
 9655     __ rorxq($dst$$Register, $src$$Register, shift);
 9656   %}
 9657   ins_pipe(ialu_reg_reg);
 9658 %}
 9659 
 9660 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9661 %{
 9662   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9663   match(Set dst (RotateLeft (LoadL src) shift));
 9664   ins_cost(175);
 9665   format %{ "rolxq   $dst, $src, $shift" %}
 9666   ins_encode %{
 9667     int shift = 64 - ($shift$$constant & 63);
 9668     __ rorxq($dst$$Register, $src$$Address, shift);
 9669   %}
 9670   ins_pipe(ialu_reg_mem);
 9671 %}
 9672 
 9673 // Rotate Left by variable
 9674 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9675 %{
 9676   predicate(n->bottom_type()->basic_type() == T_LONG);
 9677   match(Set dst (RotateLeft dst shift));
 9678   effect(KILL cr);
 9679   format %{ "rolq    $dst, $shift" %}
 9680   ins_encode %{
 9681     __ rolq($dst$$Register);
 9682   %}
 9683   ins_pipe(ialu_reg_reg);
 9684 %}
 9685 
 9686 // Rotate Right by constant.
 9687 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9688 %{
 9689   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9690   match(Set dst (RotateRight dst shift));
 9691   effect(KILL cr);
 9692   format %{ "rorq    $dst, $shift" %}
 9693   ins_encode %{
 9694     __ rorq($dst$$Register, $shift$$constant);
 9695   %}
 9696   ins_pipe(ialu_reg);
 9697 %}
 9698 
 9699 // Rotate Right by constant
 9700 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9701 %{
 9702   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9703   match(Set dst (RotateRight src shift));
 9704   format %{ "rorxq   $dst, $src, $shift" %}
 9705   ins_encode %{
 9706     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9707   %}
 9708   ins_pipe(ialu_reg_reg);
 9709 %}
 9710 
 9711 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9712 %{
 9713   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9714   match(Set dst (RotateRight (LoadL src) shift));
 9715   ins_cost(175);
 9716   format %{ "rorxq   $dst, $src, $shift" %}
 9717   ins_encode %{
 9718     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9719   %}
 9720   ins_pipe(ialu_reg_mem);
 9721 %}
 9722 
 9723 // Rotate Right by variable
 9724 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9725 %{
 9726   predicate(n->bottom_type()->basic_type() == T_LONG);
 9727   match(Set dst (RotateRight dst shift));
 9728   effect(KILL cr);
 9729   format %{ "rorq    $dst, $shift" %}
 9730   ins_encode %{
 9731     __ rorq($dst$$Register);
 9732   %}
 9733   ins_pipe(ialu_reg_reg);
 9734 %}
 9735 
 9736 //----------------------------- CompressBits/ExpandBits ------------------------
 9737 
 9738 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9739   predicate(n->bottom_type()->isa_long());
 9740   match(Set dst (CompressBits src mask));
 9741   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9742   ins_encode %{
 9743     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9744   %}
 9745   ins_pipe( pipe_slow );
 9746 %}
 9747 
 9748 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9749   predicate(n->bottom_type()->isa_long());
 9750   match(Set dst (ExpandBits src mask));
 9751   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9752   ins_encode %{
 9753     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 9754   %}
 9755   ins_pipe( pipe_slow );
 9756 %}
 9757 
 9758 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9759   predicate(n->bottom_type()->isa_long());
 9760   match(Set dst (CompressBits src (LoadL mask)));
 9761   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9762   ins_encode %{
 9763     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 9764   %}
 9765   ins_pipe( pipe_slow );
 9766 %}
 9767 
 9768 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9769   predicate(n->bottom_type()->isa_long());
 9770   match(Set dst (ExpandBits src (LoadL mask)));
 9771   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9772   ins_encode %{
 9773     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 9774   %}
 9775   ins_pipe( pipe_slow );
 9776 %}
 9777 
 9778 
 9779 // Logical Instructions
 9780 
 9781 // Integer Logical Instructions
 9782 
 9783 // And Instructions
 9784 // And Register with Register
 9785 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9786 %{
 9787   match(Set dst (AndI dst src));
 9788   effect(KILL cr);
 9789 
 9790   format %{ "andl    $dst, $src\t# int" %}
 9791   ins_encode %{
 9792     __ andl($dst$$Register, $src$$Register);
 9793   %}
 9794   ins_pipe(ialu_reg_reg);
 9795 %}
 9796 
 9797 // And Register with Immediate 255
 9798 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 9799 %{
 9800   match(Set dst (AndI src mask));
 9801 
 9802   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 9803   ins_encode %{
 9804     __ movzbl($dst$$Register, $src$$Register);
 9805   %}
 9806   ins_pipe(ialu_reg);
 9807 %}
 9808 
 9809 // And Register with Immediate 255 and promote to long
 9810 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9811 %{
 9812   match(Set dst (ConvI2L (AndI src mask)));
 9813 
 9814   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9815   ins_encode %{
 9816     __ movzbl($dst$$Register, $src$$Register);
 9817   %}
 9818   ins_pipe(ialu_reg);
 9819 %}
 9820 
 9821 // And Register with Immediate 65535
 9822 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 9823 %{
 9824   match(Set dst (AndI src mask));
 9825 
 9826   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 9827   ins_encode %{
 9828     __ movzwl($dst$$Register, $src$$Register);
 9829   %}
 9830   ins_pipe(ialu_reg);
 9831 %}
 9832 
 9833 // And Register with Immediate 65535 and promote to long
 9834 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9835 %{
 9836   match(Set dst (ConvI2L (AndI src mask)));
 9837 
 9838   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9839   ins_encode %{
 9840     __ movzwl($dst$$Register, $src$$Register);
 9841   %}
 9842   ins_pipe(ialu_reg);
 9843 %}
 9844 
 9845 // Can skip int2long conversions after AND with small bitmask
 9846 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9847 %{
 9848   predicate(VM_Version::supports_bmi2());
 9849   ins_cost(125);
 9850   effect(TEMP tmp, KILL cr);
 9851   match(Set dst (ConvI2L (AndI src mask)));
 9852   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9853   ins_encode %{
 9854     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9855     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9856   %}
 9857   ins_pipe(ialu_reg_reg);
 9858 %}
 9859 
 9860 // And Register with Immediate
 9861 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9862 %{
 9863   match(Set dst (AndI dst src));
 9864   effect(KILL cr);
 9865 
 9866   format %{ "andl    $dst, $src\t# int" %}
 9867   ins_encode %{
 9868     __ andl($dst$$Register, $src$$constant);
 9869   %}
 9870   ins_pipe(ialu_reg);
 9871 %}
 9872 
 9873 // And Register with Memory
 9874 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9875 %{
 9876   match(Set dst (AndI dst (LoadI src)));
 9877   effect(KILL cr);
 9878 
 9879   ins_cost(150);
 9880   format %{ "andl    $dst, $src\t# int" %}
 9881   ins_encode %{
 9882     __ andl($dst$$Register, $src$$Address);
 9883   %}
 9884   ins_pipe(ialu_reg_mem);
 9885 %}
 9886 
 9887 // And Memory with Register
 9888 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9889 %{
 9890   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9891   effect(KILL cr);
 9892 
 9893   ins_cost(150);
 9894   format %{ "andb    $dst, $src\t# byte" %}
 9895   ins_encode %{
 9896     __ andb($dst$$Address, $src$$Register);
 9897   %}
 9898   ins_pipe(ialu_mem_reg);
 9899 %}
 9900 
 9901 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9902 %{
 9903   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9904   effect(KILL cr);
 9905 
 9906   ins_cost(150);
 9907   format %{ "andl    $dst, $src\t# int" %}
 9908   ins_encode %{
 9909     __ andl($dst$$Address, $src$$Register);
 9910   %}
 9911   ins_pipe(ialu_mem_reg);
 9912 %}
 9913 
 9914 // And Memory with Immediate
 9915 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9916 %{
 9917   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9918   effect(KILL cr);
 9919 
 9920   ins_cost(125);
 9921   format %{ "andl    $dst, $src\t# int" %}
 9922   ins_encode %{
 9923     __ andl($dst$$Address, $src$$constant);
 9924   %}
 9925   ins_pipe(ialu_mem_imm);
 9926 %}
 9927 
 9928 // BMI1 instructions
 9929 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9930   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9931   predicate(UseBMI1Instructions);
 9932   effect(KILL cr);
 9933 
 9934   ins_cost(125);
 9935   format %{ "andnl  $dst, $src1, $src2" %}
 9936 
 9937   ins_encode %{
 9938     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9939   %}
 9940   ins_pipe(ialu_reg_mem);
 9941 %}
 9942 
 9943 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9944   match(Set dst (AndI (XorI src1 minus_1) src2));
 9945   predicate(UseBMI1Instructions);
 9946   effect(KILL cr);
 9947 
 9948   format %{ "andnl  $dst, $src1, $src2" %}
 9949 
 9950   ins_encode %{
 9951     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9952   %}
 9953   ins_pipe(ialu_reg);
 9954 %}
 9955 
 9956 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9957   match(Set dst (AndI (SubI imm_zero src) src));
 9958   predicate(UseBMI1Instructions);
 9959   effect(KILL cr);
 9960 
 9961   format %{ "blsil  $dst, $src" %}
 9962 
 9963   ins_encode %{
 9964     __ blsil($dst$$Register, $src$$Register);
 9965   %}
 9966   ins_pipe(ialu_reg);
 9967 %}
 9968 
 9969 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9970   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9971   predicate(UseBMI1Instructions);
 9972   effect(KILL cr);
 9973 
 9974   ins_cost(125);
 9975   format %{ "blsil  $dst, $src" %}
 9976 
 9977   ins_encode %{
 9978     __ blsil($dst$$Register, $src$$Address);
 9979   %}
 9980   ins_pipe(ialu_reg_mem);
 9981 %}
 9982 
 9983 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9984 %{
 9985   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9986   predicate(UseBMI1Instructions);
 9987   effect(KILL cr);
 9988 
 9989   ins_cost(125);
 9990   format %{ "blsmskl $dst, $src" %}
 9991 
 9992   ins_encode %{
 9993     __ blsmskl($dst$$Register, $src$$Address);
 9994   %}
 9995   ins_pipe(ialu_reg_mem);
 9996 %}
 9997 
 9998 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9999 %{
10000   match(Set dst (XorI (AddI src minus_1) src));
10001   predicate(UseBMI1Instructions);
10002   effect(KILL cr);
10003 
10004   format %{ "blsmskl $dst, $src" %}
10005 
10006   ins_encode %{
10007     __ blsmskl($dst$$Register, $src$$Register);
10008   %}
10009 
10010   ins_pipe(ialu_reg);
10011 %}
10012 
10013 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10014 %{
10015   match(Set dst (AndI (AddI src minus_1) src) );
10016   predicate(UseBMI1Instructions);
10017   effect(KILL cr);
10018 
10019   format %{ "blsrl  $dst, $src" %}
10020 
10021   ins_encode %{
10022     __ blsrl($dst$$Register, $src$$Register);
10023   %}
10024 
10025   ins_pipe(ialu_reg_mem);
10026 %}
10027 
10028 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10029 %{
10030   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10031   predicate(UseBMI1Instructions);
10032   effect(KILL cr);
10033 
10034   ins_cost(125);
10035   format %{ "blsrl  $dst, $src" %}
10036 
10037   ins_encode %{
10038     __ blsrl($dst$$Register, $src$$Address);
10039   %}
10040 
10041   ins_pipe(ialu_reg);
10042 %}
10043 
10044 // Or Instructions
10045 // Or Register with Register
10046 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10047 %{
10048   match(Set dst (OrI dst src));
10049   effect(KILL cr);
10050 
10051   format %{ "orl     $dst, $src\t# int" %}
10052   ins_encode %{
10053     __ orl($dst$$Register, $src$$Register);
10054   %}
10055   ins_pipe(ialu_reg_reg);
10056 %}
10057 
10058 // Or Register with Immediate
10059 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10060 %{
10061   match(Set dst (OrI dst src));
10062   effect(KILL cr);
10063 
10064   format %{ "orl     $dst, $src\t# int" %}
10065   ins_encode %{
10066     __ orl($dst$$Register, $src$$constant);
10067   %}
10068   ins_pipe(ialu_reg);
10069 %}
10070 
10071 // Or Register with Memory
10072 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10073 %{
10074   match(Set dst (OrI dst (LoadI src)));
10075   effect(KILL cr);
10076 
10077   ins_cost(150);
10078   format %{ "orl     $dst, $src\t# int" %}
10079   ins_encode %{
10080     __ orl($dst$$Register, $src$$Address);
10081   %}
10082   ins_pipe(ialu_reg_mem);
10083 %}
10084 
10085 // Or Memory with Register
10086 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10087 %{
10088   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10089   effect(KILL cr);
10090 
10091   ins_cost(150);
10092   format %{ "orb    $dst, $src\t# byte" %}
10093   ins_encode %{
10094     __ orb($dst$$Address, $src$$Register);
10095   %}
10096   ins_pipe(ialu_mem_reg);
10097 %}
10098 
10099 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10100 %{
10101   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10102   effect(KILL cr);
10103 
10104   ins_cost(150);
10105   format %{ "orl     $dst, $src\t# int" %}
10106   ins_encode %{
10107     __ orl($dst$$Address, $src$$Register);
10108   %}
10109   ins_pipe(ialu_mem_reg);
10110 %}
10111 
10112 // Or Memory with Immediate
10113 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10114 %{
10115   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10116   effect(KILL cr);
10117 
10118   ins_cost(125);
10119   format %{ "orl     $dst, $src\t# int" %}
10120   ins_encode %{
10121     __ orl($dst$$Address, $src$$constant);
10122   %}
10123   ins_pipe(ialu_mem_imm);
10124 %}
10125 
10126 // Xor Instructions
10127 // Xor Register with Register
10128 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10129 %{
10130   match(Set dst (XorI dst src));
10131   effect(KILL cr);
10132 
10133   format %{ "xorl    $dst, $src\t# int" %}
10134   ins_encode %{
10135     __ xorl($dst$$Register, $src$$Register);
10136   %}
10137   ins_pipe(ialu_reg_reg);
10138 %}
10139 
10140 // Xor Register with Immediate -1
10141 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
10142   match(Set dst (XorI dst imm));
10143 
10144   format %{ "not    $dst" %}
10145   ins_encode %{
10146      __ notl($dst$$Register);
10147   %}
10148   ins_pipe(ialu_reg);
10149 %}
10150 
10151 // Xor Register with Immediate
10152 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10153 %{
10154   match(Set dst (XorI dst src));
10155   effect(KILL cr);
10156 
10157   format %{ "xorl    $dst, $src\t# int" %}
10158   ins_encode %{
10159     __ xorl($dst$$Register, $src$$constant);
10160   %}
10161   ins_pipe(ialu_reg);
10162 %}
10163 
10164 // Xor Register with Memory
10165 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10166 %{
10167   match(Set dst (XorI dst (LoadI src)));
10168   effect(KILL cr);
10169 
10170   ins_cost(150);
10171   format %{ "xorl    $dst, $src\t# int" %}
10172   ins_encode %{
10173     __ xorl($dst$$Register, $src$$Address);
10174   %}
10175   ins_pipe(ialu_reg_mem);
10176 %}
10177 
10178 // Xor Memory with Register
10179 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10180 %{
10181   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10182   effect(KILL cr);
10183 
10184   ins_cost(150);
10185   format %{ "xorb    $dst, $src\t# byte" %}
10186   ins_encode %{
10187     __ xorb($dst$$Address, $src$$Register);
10188   %}
10189   ins_pipe(ialu_mem_reg);
10190 %}
10191 
10192 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10193 %{
10194   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10195   effect(KILL cr);
10196 
10197   ins_cost(150);
10198   format %{ "xorl    $dst, $src\t# int" %}
10199   ins_encode %{
10200     __ xorl($dst$$Address, $src$$Register);
10201   %}
10202   ins_pipe(ialu_mem_reg);
10203 %}
10204 
10205 // Xor Memory with Immediate
10206 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10207 %{
10208   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10209   effect(KILL cr);
10210 
10211   ins_cost(125);
10212   format %{ "xorl    $dst, $src\t# int" %}
10213   ins_encode %{
10214     __ xorl($dst$$Address, $src$$constant);
10215   %}
10216   ins_pipe(ialu_mem_imm);
10217 %}
10218 
10219 
10220 // Long Logical Instructions
10221 
10222 // And Instructions
10223 // And Register with Register
10224 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10225 %{
10226   match(Set dst (AndL dst src));
10227   effect(KILL cr);
10228 
10229   format %{ "andq    $dst, $src\t# long" %}
10230   ins_encode %{
10231     __ andq($dst$$Register, $src$$Register);
10232   %}
10233   ins_pipe(ialu_reg_reg);
10234 %}
10235 
10236 // And Register with Immediate 255
10237 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10238 %{
10239   match(Set dst (AndL src mask));
10240 
10241   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10242   ins_encode %{
10243     // movzbl zeroes out the upper 32-bit and does not need REX.W
10244     __ movzbl($dst$$Register, $src$$Register);
10245   %}
10246   ins_pipe(ialu_reg);
10247 %}
10248 
10249 // And Register with Immediate 65535
10250 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10251 %{
10252   match(Set dst (AndL src mask));
10253 
10254   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10255   ins_encode %{
10256     // movzwl zeroes out the upper 32-bit and does not need REX.W
10257     __ movzwl($dst$$Register, $src$$Register);
10258   %}
10259   ins_pipe(ialu_reg);
10260 %}
10261 
10262 // And Register with Immediate
10263 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10264 %{
10265   match(Set dst (AndL dst src));
10266   effect(KILL cr);
10267 
10268   format %{ "andq    $dst, $src\t# long" %}
10269   ins_encode %{
10270     __ andq($dst$$Register, $src$$constant);
10271   %}
10272   ins_pipe(ialu_reg);
10273 %}
10274 
10275 // And Register with Memory
10276 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10277 %{
10278   match(Set dst (AndL dst (LoadL src)));
10279   effect(KILL cr);
10280 
10281   ins_cost(150);
10282   format %{ "andq    $dst, $src\t# long" %}
10283   ins_encode %{
10284     __ andq($dst$$Register, $src$$Address);
10285   %}
10286   ins_pipe(ialu_reg_mem);
10287 %}
10288 
10289 // And Memory with Register
10290 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10291 %{
10292   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10293   effect(KILL cr);
10294 
10295   ins_cost(150);
10296   format %{ "andq    $dst, $src\t# long" %}
10297   ins_encode %{
10298     __ andq($dst$$Address, $src$$Register);
10299   %}
10300   ins_pipe(ialu_mem_reg);
10301 %}
10302 
10303 // And Memory with Immediate
10304 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10305 %{
10306   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10307   effect(KILL cr);
10308 
10309   ins_cost(125);
10310   format %{ "andq    $dst, $src\t# long" %}
10311   ins_encode %{
10312     __ andq($dst$$Address, $src$$constant);
10313   %}
10314   ins_pipe(ialu_mem_imm);
10315 %}
10316 
10317 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10318 %{
10319   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10320   // because AND/OR works well enough for 8/32-bit values.
10321   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10322 
10323   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10324   effect(KILL cr);
10325 
10326   ins_cost(125);
10327   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10328   ins_encode %{
10329     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10330   %}
10331   ins_pipe(ialu_mem_imm);
10332 %}
10333 
10334 // BMI1 instructions
10335 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10336   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10337   predicate(UseBMI1Instructions);
10338   effect(KILL cr);
10339 
10340   ins_cost(125);
10341   format %{ "andnq  $dst, $src1, $src2" %}
10342 
10343   ins_encode %{
10344     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10345   %}
10346   ins_pipe(ialu_reg_mem);
10347 %}
10348 
10349 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10350   match(Set dst (AndL (XorL src1 minus_1) src2));
10351   predicate(UseBMI1Instructions);
10352   effect(KILL cr);
10353 
10354   format %{ "andnq  $dst, $src1, $src2" %}
10355 
10356   ins_encode %{
10357   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10358   %}
10359   ins_pipe(ialu_reg_mem);
10360 %}
10361 
10362 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10363   match(Set dst (AndL (SubL imm_zero src) src));
10364   predicate(UseBMI1Instructions);
10365   effect(KILL cr);
10366 
10367   format %{ "blsiq  $dst, $src" %}
10368 
10369   ins_encode %{
10370     __ blsiq($dst$$Register, $src$$Register);
10371   %}
10372   ins_pipe(ialu_reg);
10373 %}
10374 
10375 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10376   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10377   predicate(UseBMI1Instructions);
10378   effect(KILL cr);
10379 
10380   ins_cost(125);
10381   format %{ "blsiq  $dst, $src" %}
10382 
10383   ins_encode %{
10384     __ blsiq($dst$$Register, $src$$Address);
10385   %}
10386   ins_pipe(ialu_reg_mem);
10387 %}
10388 
10389 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10390 %{
10391   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10392   predicate(UseBMI1Instructions);
10393   effect(KILL cr);
10394 
10395   ins_cost(125);
10396   format %{ "blsmskq $dst, $src" %}
10397 
10398   ins_encode %{
10399     __ blsmskq($dst$$Register, $src$$Address);
10400   %}
10401   ins_pipe(ialu_reg_mem);
10402 %}
10403 
10404 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10405 %{
10406   match(Set dst (XorL (AddL src minus_1) src));
10407   predicate(UseBMI1Instructions);
10408   effect(KILL cr);
10409 
10410   format %{ "blsmskq $dst, $src" %}
10411 
10412   ins_encode %{
10413     __ blsmskq($dst$$Register, $src$$Register);
10414   %}
10415 
10416   ins_pipe(ialu_reg);
10417 %}
10418 
10419 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10420 %{
10421   match(Set dst (AndL (AddL src minus_1) src) );
10422   predicate(UseBMI1Instructions);
10423   effect(KILL cr);
10424 
10425   format %{ "blsrq  $dst, $src" %}
10426 
10427   ins_encode %{
10428     __ blsrq($dst$$Register, $src$$Register);
10429   %}
10430 
10431   ins_pipe(ialu_reg);
10432 %}
10433 
10434 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10435 %{
10436   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10437   predicate(UseBMI1Instructions);
10438   effect(KILL cr);
10439 
10440   ins_cost(125);
10441   format %{ "blsrq  $dst, $src" %}
10442 
10443   ins_encode %{
10444     __ blsrq($dst$$Register, $src$$Address);
10445   %}
10446 
10447   ins_pipe(ialu_reg);
10448 %}
10449 
10450 // Or Instructions
10451 // Or Register with Register
10452 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10453 %{
10454   match(Set dst (OrL dst src));
10455   effect(KILL cr);
10456 
10457   format %{ "orq     $dst, $src\t# long" %}
10458   ins_encode %{
10459     __ orq($dst$$Register, $src$$Register);
10460   %}
10461   ins_pipe(ialu_reg_reg);
10462 %}
10463 
10464 // Use any_RegP to match R15 (TLS register) without spilling.
10465 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10466   match(Set dst (OrL dst (CastP2X src)));
10467   effect(KILL cr);
10468 
10469   format %{ "orq     $dst, $src\t# long" %}
10470   ins_encode %{
10471     __ orq($dst$$Register, $src$$Register);
10472   %}
10473   ins_pipe(ialu_reg_reg);
10474 %}
10475 
10476 
10477 // Or Register with Immediate
10478 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10479 %{
10480   match(Set dst (OrL dst src));
10481   effect(KILL cr);
10482 
10483   format %{ "orq     $dst, $src\t# long" %}
10484   ins_encode %{
10485     __ orq($dst$$Register, $src$$constant);
10486   %}
10487   ins_pipe(ialu_reg);
10488 %}
10489 
10490 // Or Register with Memory
10491 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10492 %{
10493   match(Set dst (OrL dst (LoadL src)));
10494   effect(KILL cr);
10495 
10496   ins_cost(150);
10497   format %{ "orq     $dst, $src\t# long" %}
10498   ins_encode %{
10499     __ orq($dst$$Register, $src$$Address);
10500   %}
10501   ins_pipe(ialu_reg_mem);
10502 %}
10503 
10504 // Or Memory with Register
10505 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10506 %{
10507   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10508   effect(KILL cr);
10509 
10510   ins_cost(150);
10511   format %{ "orq     $dst, $src\t# long" %}
10512   ins_encode %{
10513     __ orq($dst$$Address, $src$$Register);
10514   %}
10515   ins_pipe(ialu_mem_reg);
10516 %}
10517 
10518 // Or Memory with Immediate
10519 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10520 %{
10521   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10522   effect(KILL cr);
10523 
10524   ins_cost(125);
10525   format %{ "orq     $dst, $src\t# long" %}
10526   ins_encode %{
10527     __ orq($dst$$Address, $src$$constant);
10528   %}
10529   ins_pipe(ialu_mem_imm);
10530 %}
10531 
10532 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10533 %{
10534   // con should be a pure 64-bit power of 2 immediate
10535   // because AND/OR works well enough for 8/32-bit values.
10536   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10537 
10538   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10539   effect(KILL cr);
10540 
10541   ins_cost(125);
10542   format %{ "btsq    $dst, log2($con)\t# long" %}
10543   ins_encode %{
10544     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10545   %}
10546   ins_pipe(ialu_mem_imm);
10547 %}
10548 
10549 // Xor Instructions
10550 // Xor Register with Register
10551 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10552 %{
10553   match(Set dst (XorL dst src));
10554   effect(KILL cr);
10555 
10556   format %{ "xorq    $dst, $src\t# long" %}
10557   ins_encode %{
10558     __ xorq($dst$$Register, $src$$Register);
10559   %}
10560   ins_pipe(ialu_reg_reg);
10561 %}
10562 
10563 // Xor Register with Immediate -1
10564 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10565   match(Set dst (XorL dst imm));
10566 
10567   format %{ "notq   $dst" %}
10568   ins_encode %{
10569      __ notq($dst$$Register);
10570   %}
10571   ins_pipe(ialu_reg);
10572 %}
10573 
10574 // Xor Register with Immediate
10575 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10576 %{
10577   match(Set dst (XorL dst src));
10578   effect(KILL cr);
10579 
10580   format %{ "xorq    $dst, $src\t# long" %}
10581   ins_encode %{
10582     __ xorq($dst$$Register, $src$$constant);
10583   %}
10584   ins_pipe(ialu_reg);
10585 %}
10586 
10587 // Xor Register with Memory
10588 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10589 %{
10590   match(Set dst (XorL dst (LoadL src)));
10591   effect(KILL cr);
10592 
10593   ins_cost(150);
10594   format %{ "xorq    $dst, $src\t# long" %}
10595   ins_encode %{
10596     __ xorq($dst$$Register, $src$$Address);
10597   %}
10598   ins_pipe(ialu_reg_mem);
10599 %}
10600 
10601 // Xor Memory with Register
10602 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10603 %{
10604   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10605   effect(KILL cr);
10606 
10607   ins_cost(150);
10608   format %{ "xorq    $dst, $src\t# long" %}
10609   ins_encode %{
10610     __ xorq($dst$$Address, $src$$Register);
10611   %}
10612   ins_pipe(ialu_mem_reg);
10613 %}
10614 
10615 // Xor Memory with Immediate
10616 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10617 %{
10618   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10619   effect(KILL cr);
10620 
10621   ins_cost(125);
10622   format %{ "xorq    $dst, $src\t# long" %}
10623   ins_encode %{
10624     __ xorq($dst$$Address, $src$$constant);
10625   %}
10626   ins_pipe(ialu_mem_imm);
10627 %}
10628 
10629 // Convert Int to Boolean
10630 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10631 %{
10632   match(Set dst (Conv2B src));
10633   effect(KILL cr);
10634 
10635   format %{ "testl   $src, $src\t# ci2b\n\t"
10636             "setnz   $dst\n\t"
10637             "movzbl  $dst, $dst" %}
10638   ins_encode %{
10639     __ testl($src$$Register, $src$$Register);
10640     __ set_byte_if_not_zero($dst$$Register);
10641     __ movzbl($dst$$Register, $dst$$Register);
10642   %}
10643   ins_pipe(pipe_slow); // XXX
10644 %}
10645 
10646 // Convert Pointer to Boolean
10647 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10648 %{
10649   match(Set dst (Conv2B src));
10650   effect(KILL cr);
10651 
10652   format %{ "testq   $src, $src\t# cp2b\n\t"
10653             "setnz   $dst\n\t"
10654             "movzbl  $dst, $dst" %}
10655   ins_encode %{
10656     __ testq($src$$Register, $src$$Register);
10657     __ set_byte_if_not_zero($dst$$Register);
10658     __ movzbl($dst$$Register, $dst$$Register);
10659   %}
10660   ins_pipe(pipe_slow); // XXX
10661 %}
10662 
10663 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10664 %{
10665   match(Set dst (CmpLTMask p q));
10666   effect(KILL cr);
10667 
10668   ins_cost(400);
10669   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10670             "setlt   $dst\n\t"
10671             "movzbl  $dst, $dst\n\t"
10672             "negl    $dst" %}
10673   ins_encode %{
10674     __ cmpl($p$$Register, $q$$Register);
10675     __ setl($dst$$Register);
10676     __ movzbl($dst$$Register, $dst$$Register);
10677     __ negl($dst$$Register);
10678   %}
10679   ins_pipe(pipe_slow);
10680 %}
10681 
10682 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10683 %{
10684   match(Set dst (CmpLTMask dst zero));
10685   effect(KILL cr);
10686 
10687   ins_cost(100);
10688   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10689   ins_encode %{
10690     __ sarl($dst$$Register, 31);
10691   %}
10692   ins_pipe(ialu_reg);
10693 %}
10694 
10695 /* Better to save a register than avoid a branch */
10696 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10697 %{
10698   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10699   effect(KILL cr);
10700   ins_cost(300);
10701   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10702             "jge     done\n\t"
10703             "addl    $p,$y\n"
10704             "done:   " %}
10705   ins_encode %{
10706     Register Rp = $p$$Register;
10707     Register Rq = $q$$Register;
10708     Register Ry = $y$$Register;
10709     Label done;
10710     __ subl(Rp, Rq);
10711     __ jccb(Assembler::greaterEqual, done);
10712     __ addl(Rp, Ry);
10713     __ bind(done);
10714   %}
10715   ins_pipe(pipe_cmplt);
10716 %}
10717 
10718 /* Better to save a register than avoid a branch */
10719 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10720 %{
10721   match(Set y (AndI (CmpLTMask p q) y));
10722   effect(KILL cr);
10723 
10724   ins_cost(300);
10725 
10726   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10727             "jlt     done\n\t"
10728             "xorl    $y, $y\n"
10729             "done:   " %}
10730   ins_encode %{
10731     Register Rp = $p$$Register;
10732     Register Rq = $q$$Register;
10733     Register Ry = $y$$Register;
10734     Label done;
10735     __ cmpl(Rp, Rq);
10736     __ jccb(Assembler::less, done);
10737     __ xorl(Ry, Ry);
10738     __ bind(done);
10739   %}
10740   ins_pipe(pipe_cmplt);
10741 %}
10742 
10743 
10744 //---------- FP Instructions------------------------------------------------
10745 
10746 // Really expensive, avoid
10747 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10748 %{
10749   match(Set cr (CmpF src1 src2));
10750 
10751   ins_cost(500);
10752   format %{ "ucomiss $src1, $src2\n\t"
10753             "jnp,s   exit\n\t"
10754             "pushfq\t# saw NaN, set CF\n\t"
10755             "andq    [rsp], #0xffffff2b\n\t"
10756             "popfq\n"
10757     "exit:" %}
10758   ins_encode %{
10759     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10760     emit_cmpfp_fixup(_masm);
10761   %}
10762   ins_pipe(pipe_slow);
10763 %}
10764 
10765 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10766   match(Set cr (CmpF src1 src2));
10767 
10768   ins_cost(100);
10769   format %{ "ucomiss $src1, $src2" %}
10770   ins_encode %{
10771     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10772   %}
10773   ins_pipe(pipe_slow);
10774 %}
10775 
10776 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10777   match(Set cr (CmpF src1 (LoadF src2)));
10778 
10779   ins_cost(100);
10780   format %{ "ucomiss $src1, $src2" %}
10781   ins_encode %{
10782     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10783   %}
10784   ins_pipe(pipe_slow);
10785 %}
10786 
10787 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10788   match(Set cr (CmpF src con));
10789   ins_cost(100);
10790   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10791   ins_encode %{
10792     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10793   %}
10794   ins_pipe(pipe_slow);
10795 %}
10796 
10797 // Really expensive, avoid
10798 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10799 %{
10800   match(Set cr (CmpD src1 src2));
10801 
10802   ins_cost(500);
10803   format %{ "ucomisd $src1, $src2\n\t"
10804             "jnp,s   exit\n\t"
10805             "pushfq\t# saw NaN, set CF\n\t"
10806             "andq    [rsp], #0xffffff2b\n\t"
10807             "popfq\n"
10808     "exit:" %}
10809   ins_encode %{
10810     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10811     emit_cmpfp_fixup(_masm);
10812   %}
10813   ins_pipe(pipe_slow);
10814 %}
10815 
10816 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10817   match(Set cr (CmpD src1 src2));
10818 
10819   ins_cost(100);
10820   format %{ "ucomisd $src1, $src2 test" %}
10821   ins_encode %{
10822     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10823   %}
10824   ins_pipe(pipe_slow);
10825 %}
10826 
10827 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10828   match(Set cr (CmpD src1 (LoadD src2)));
10829 
10830   ins_cost(100);
10831   format %{ "ucomisd $src1, $src2" %}
10832   ins_encode %{
10833     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10834   %}
10835   ins_pipe(pipe_slow);
10836 %}
10837 
10838 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10839   match(Set cr (CmpD src con));
10840   ins_cost(100);
10841   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10842   ins_encode %{
10843     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10844   %}
10845   ins_pipe(pipe_slow);
10846 %}
10847 
10848 // Compare into -1,0,1
10849 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10850 %{
10851   match(Set dst (CmpF3 src1 src2));
10852   effect(KILL cr);
10853 
10854   ins_cost(275);
10855   format %{ "ucomiss $src1, $src2\n\t"
10856             "movl    $dst, #-1\n\t"
10857             "jp,s    done\n\t"
10858             "jb,s    done\n\t"
10859             "setne   $dst\n\t"
10860             "movzbl  $dst, $dst\n"
10861     "done:" %}
10862   ins_encode %{
10863     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10864     emit_cmpfp3(_masm, $dst$$Register);
10865   %}
10866   ins_pipe(pipe_slow);
10867 %}
10868 
10869 // Compare into -1,0,1
10870 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10871 %{
10872   match(Set dst (CmpF3 src1 (LoadF src2)));
10873   effect(KILL cr);
10874 
10875   ins_cost(275);
10876   format %{ "ucomiss $src1, $src2\n\t"
10877             "movl    $dst, #-1\n\t"
10878             "jp,s    done\n\t"
10879             "jb,s    done\n\t"
10880             "setne   $dst\n\t"
10881             "movzbl  $dst, $dst\n"
10882     "done:" %}
10883   ins_encode %{
10884     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10885     emit_cmpfp3(_masm, $dst$$Register);
10886   %}
10887   ins_pipe(pipe_slow);
10888 %}
10889 
10890 // Compare into -1,0,1
10891 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10892   match(Set dst (CmpF3 src con));
10893   effect(KILL cr);
10894 
10895   ins_cost(275);
10896   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10897             "movl    $dst, #-1\n\t"
10898             "jp,s    done\n\t"
10899             "jb,s    done\n\t"
10900             "setne   $dst\n\t"
10901             "movzbl  $dst, $dst\n"
10902     "done:" %}
10903   ins_encode %{
10904     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10905     emit_cmpfp3(_masm, $dst$$Register);
10906   %}
10907   ins_pipe(pipe_slow);
10908 %}
10909 
10910 // Compare into -1,0,1
10911 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10912 %{
10913   match(Set dst (CmpD3 src1 src2));
10914   effect(KILL cr);
10915 
10916   ins_cost(275);
10917   format %{ "ucomisd $src1, $src2\n\t"
10918             "movl    $dst, #-1\n\t"
10919             "jp,s    done\n\t"
10920             "jb,s    done\n\t"
10921             "setne   $dst\n\t"
10922             "movzbl  $dst, $dst\n"
10923     "done:" %}
10924   ins_encode %{
10925     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10926     emit_cmpfp3(_masm, $dst$$Register);
10927   %}
10928   ins_pipe(pipe_slow);
10929 %}
10930 
10931 // Compare into -1,0,1
10932 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10933 %{
10934   match(Set dst (CmpD3 src1 (LoadD src2)));
10935   effect(KILL cr);
10936 
10937   ins_cost(275);
10938   format %{ "ucomisd $src1, $src2\n\t"
10939             "movl    $dst, #-1\n\t"
10940             "jp,s    done\n\t"
10941             "jb,s    done\n\t"
10942             "setne   $dst\n\t"
10943             "movzbl  $dst, $dst\n"
10944     "done:" %}
10945   ins_encode %{
10946     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10947     emit_cmpfp3(_masm, $dst$$Register);
10948   %}
10949   ins_pipe(pipe_slow);
10950 %}
10951 
10952 // Compare into -1,0,1
10953 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10954   match(Set dst (CmpD3 src con));
10955   effect(KILL cr);
10956 
10957   ins_cost(275);
10958   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10959             "movl    $dst, #-1\n\t"
10960             "jp,s    done\n\t"
10961             "jb,s    done\n\t"
10962             "setne   $dst\n\t"
10963             "movzbl  $dst, $dst\n"
10964     "done:" %}
10965   ins_encode %{
10966     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10967     emit_cmpfp3(_masm, $dst$$Register);
10968   %}
10969   ins_pipe(pipe_slow);
10970 %}
10971 
10972 //----------Arithmetic Conversion Instructions---------------------------------
10973 
10974 instruct convF2D_reg_reg(regD dst, regF src)
10975 %{
10976   match(Set dst (ConvF2D src));
10977 
10978   format %{ "cvtss2sd $dst, $src" %}
10979   ins_encode %{
10980     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10981   %}
10982   ins_pipe(pipe_slow); // XXX
10983 %}
10984 
10985 instruct convF2D_reg_mem(regD dst, memory src)
10986 %{
10987   match(Set dst (ConvF2D (LoadF src)));
10988 
10989   format %{ "cvtss2sd $dst, $src" %}
10990   ins_encode %{
10991     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10992   %}
10993   ins_pipe(pipe_slow); // XXX
10994 %}
10995 
10996 instruct convD2F_reg_reg(regF dst, regD src)
10997 %{
10998   match(Set dst (ConvD2F src));
10999 
11000   format %{ "cvtsd2ss $dst, $src" %}
11001   ins_encode %{
11002     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11003   %}
11004   ins_pipe(pipe_slow); // XXX
11005 %}
11006 
11007 instruct convD2F_reg_mem(regF dst, memory src)
11008 %{
11009   match(Set dst (ConvD2F (LoadD src)));
11010 
11011   format %{ "cvtsd2ss $dst, $src" %}
11012   ins_encode %{
11013     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11014   %}
11015   ins_pipe(pipe_slow); // XXX
11016 %}
11017 
11018 // XXX do mem variants
11019 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11020 %{
11021   match(Set dst (ConvF2I src));
11022   effect(KILL cr);
11023   format %{ "convert_f2i $dst,$src" %}
11024   ins_encode %{
11025     __ convert_f2i($dst$$Register, $src$$XMMRegister);
11026   %}
11027   ins_pipe(pipe_slow);
11028 %}
11029 
11030 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11031 %{
11032   match(Set dst (ConvF2L src));
11033   effect(KILL cr);
11034   format %{ "convert_f2l $dst,$src"%}
11035   ins_encode %{
11036     __ convert_f2l($dst$$Register, $src$$XMMRegister);
11037   %}
11038   ins_pipe(pipe_slow);
11039 %}
11040 
11041 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11042 %{
11043   match(Set dst (ConvD2I src));
11044   effect(KILL cr);
11045   format %{ "convert_d2i $dst,$src"%}
11046   ins_encode %{
11047     __ convert_d2i($dst$$Register, $src$$XMMRegister);
11048   %}
11049   ins_pipe(pipe_slow);
11050 %}
11051 
11052 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11053 %{
11054   match(Set dst (ConvD2L src));
11055   effect(KILL cr);
11056   format %{ "convert_d2l $dst,$src"%}
11057   ins_encode %{
11058     __ convert_d2l($dst$$Register, $src$$XMMRegister);
11059   %}
11060   ins_pipe(pipe_slow);
11061 %}
11062 
11063 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11064 %{
11065   match(Set dst (RoundD src));
11066   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11067   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11068   ins_encode %{
11069     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11070   %}
11071   ins_pipe(pipe_slow);
11072 %}
11073 
11074 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11075 %{
11076   match(Set dst (RoundF src));
11077   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11078   format %{ "round_float $dst,$src" %}
11079   ins_encode %{
11080     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11081   %}
11082   ins_pipe(pipe_slow);
11083 %}
11084 
11085 instruct convI2F_reg_reg(regF dst, rRegI src)
11086 %{
11087   predicate(!UseXmmI2F);
11088   match(Set dst (ConvI2F src));
11089 
11090   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11091   ins_encode %{
11092     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11093   %}
11094   ins_pipe(pipe_slow); // XXX
11095 %}
11096 
11097 instruct convI2F_reg_mem(regF dst, memory src)
11098 %{
11099   match(Set dst (ConvI2F (LoadI src)));
11100 
11101   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11102   ins_encode %{
11103     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11104   %}
11105   ins_pipe(pipe_slow); // XXX
11106 %}
11107 
11108 instruct convI2D_reg_reg(regD dst, rRegI src)
11109 %{
11110   predicate(!UseXmmI2D);
11111   match(Set dst (ConvI2D src));
11112 
11113   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11114   ins_encode %{
11115     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11116   %}
11117   ins_pipe(pipe_slow); // XXX
11118 %}
11119 
11120 instruct convI2D_reg_mem(regD dst, memory src)
11121 %{
11122   match(Set dst (ConvI2D (LoadI src)));
11123 
11124   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11125   ins_encode %{
11126     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11127   %}
11128   ins_pipe(pipe_slow); // XXX
11129 %}
11130 
11131 instruct convXI2F_reg(regF dst, rRegI src)
11132 %{
11133   predicate(UseXmmI2F);
11134   match(Set dst (ConvI2F src));
11135 
11136   format %{ "movdl $dst, $src\n\t"
11137             "cvtdq2psl $dst, $dst\t# i2f" %}
11138   ins_encode %{
11139     __ movdl($dst$$XMMRegister, $src$$Register);
11140     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11141   %}
11142   ins_pipe(pipe_slow); // XXX
11143 %}
11144 
11145 instruct convXI2D_reg(regD dst, rRegI src)
11146 %{
11147   predicate(UseXmmI2D);
11148   match(Set dst (ConvI2D src));
11149 
11150   format %{ "movdl $dst, $src\n\t"
11151             "cvtdq2pdl $dst, $dst\t# i2d" %}
11152   ins_encode %{
11153     __ movdl($dst$$XMMRegister, $src$$Register);
11154     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11155   %}
11156   ins_pipe(pipe_slow); // XXX
11157 %}
11158 
11159 instruct convL2F_reg_reg(regF dst, rRegL src)
11160 %{
11161   match(Set dst (ConvL2F src));
11162 
11163   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11164   ins_encode %{
11165     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11166   %}
11167   ins_pipe(pipe_slow); // XXX
11168 %}
11169 
11170 instruct convL2F_reg_mem(regF dst, memory src)
11171 %{
11172   match(Set dst (ConvL2F (LoadL src)));
11173 
11174   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11175   ins_encode %{
11176     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11177   %}
11178   ins_pipe(pipe_slow); // XXX
11179 %}
11180 
11181 instruct convL2D_reg_reg(regD dst, rRegL src)
11182 %{
11183   match(Set dst (ConvL2D src));
11184 
11185   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11186   ins_encode %{
11187     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11188   %}
11189   ins_pipe(pipe_slow); // XXX
11190 %}
11191 
11192 instruct convL2D_reg_mem(regD dst, memory src)
11193 %{
11194   match(Set dst (ConvL2D (LoadL src)));
11195 
11196   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11197   ins_encode %{
11198     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11199   %}
11200   ins_pipe(pipe_slow); // XXX
11201 %}
11202 
11203 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11204 %{
11205   match(Set dst (ConvI2L src));
11206 
11207   ins_cost(125);
11208   format %{ "movslq  $dst, $src\t# i2l" %}
11209   ins_encode %{
11210     __ movslq($dst$$Register, $src$$Register);
11211   %}
11212   ins_pipe(ialu_reg_reg);
11213 %}
11214 
11215 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11216 // %{
11217 //   match(Set dst (ConvI2L src));
11218 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11219 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11220 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11221 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11222 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11223 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11224 
11225 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11226 //   ins_encode(enc_copy(dst, src));
11227 // //   opcode(0x63); // needs REX.W
11228 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11229 //   ins_pipe(ialu_reg_reg);
11230 // %}
11231 
11232 // Zero-extend convert int to long
11233 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11234 %{
11235   match(Set dst (AndL (ConvI2L src) mask));
11236 
11237   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11238   ins_encode %{
11239     if ($dst$$reg != $src$$reg) {
11240       __ movl($dst$$Register, $src$$Register);
11241     }
11242   %}
11243   ins_pipe(ialu_reg_reg);
11244 %}
11245 
11246 // Zero-extend convert int to long
11247 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11248 %{
11249   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11250 
11251   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11252   ins_encode %{
11253     __ movl($dst$$Register, $src$$Address);
11254   %}
11255   ins_pipe(ialu_reg_mem);
11256 %}
11257 
11258 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11259 %{
11260   match(Set dst (AndL src mask));
11261 
11262   format %{ "movl    $dst, $src\t# zero-extend long" %}
11263   ins_encode %{
11264     __ movl($dst$$Register, $src$$Register);
11265   %}
11266   ins_pipe(ialu_reg_reg);
11267 %}
11268 
11269 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11270 %{
11271   match(Set dst (ConvL2I src));
11272 
11273   format %{ "movl    $dst, $src\t# l2i" %}
11274   ins_encode %{
11275     __ movl($dst$$Register, $src$$Register);
11276   %}
11277   ins_pipe(ialu_reg_reg);
11278 %}
11279 
11280 
11281 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11282   match(Set dst (MoveF2I src));
11283   effect(DEF dst, USE src);
11284 
11285   ins_cost(125);
11286   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11287   ins_encode %{
11288     __ movl($dst$$Register, Address(rsp, $src$$disp));
11289   %}
11290   ins_pipe(ialu_reg_mem);
11291 %}
11292 
11293 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11294   match(Set dst (MoveI2F src));
11295   effect(DEF dst, USE src);
11296 
11297   ins_cost(125);
11298   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11299   ins_encode %{
11300     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11301   %}
11302   ins_pipe(pipe_slow);
11303 %}
11304 
11305 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11306   match(Set dst (MoveD2L src));
11307   effect(DEF dst, USE src);
11308 
11309   ins_cost(125);
11310   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11311   ins_encode %{
11312     __ movq($dst$$Register, Address(rsp, $src$$disp));
11313   %}
11314   ins_pipe(ialu_reg_mem);
11315 %}
11316 
11317 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11318   predicate(!UseXmmLoadAndClearUpper);
11319   match(Set dst (MoveL2D src));
11320   effect(DEF dst, USE src);
11321 
11322   ins_cost(125);
11323   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11324   ins_encode %{
11325     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11326   %}
11327   ins_pipe(pipe_slow);
11328 %}
11329 
11330 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11331   predicate(UseXmmLoadAndClearUpper);
11332   match(Set dst (MoveL2D src));
11333   effect(DEF dst, USE src);
11334 
11335   ins_cost(125);
11336   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11337   ins_encode %{
11338     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11339   %}
11340   ins_pipe(pipe_slow);
11341 %}
11342 
11343 
11344 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11345   match(Set dst (MoveF2I src));
11346   effect(DEF dst, USE src);
11347 
11348   ins_cost(95); // XXX
11349   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11350   ins_encode %{
11351     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11352   %}
11353   ins_pipe(pipe_slow);
11354 %}
11355 
11356 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11357   match(Set dst (MoveI2F src));
11358   effect(DEF dst, USE src);
11359 
11360   ins_cost(100);
11361   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11362   ins_encode %{
11363     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11364   %}
11365   ins_pipe( ialu_mem_reg );
11366 %}
11367 
11368 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11369   match(Set dst (MoveD2L src));
11370   effect(DEF dst, USE src);
11371 
11372   ins_cost(95); // XXX
11373   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11374   ins_encode %{
11375     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11376   %}
11377   ins_pipe(pipe_slow);
11378 %}
11379 
11380 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11381   match(Set dst (MoveL2D src));
11382   effect(DEF dst, USE src);
11383 
11384   ins_cost(100);
11385   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11386   ins_encode %{
11387     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11388   %}
11389   ins_pipe(ialu_mem_reg);
11390 %}
11391 
11392 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11393   match(Set dst (MoveF2I src));
11394   effect(DEF dst, USE src);
11395   ins_cost(85);
11396   format %{ "movd    $dst,$src\t# MoveF2I" %}
11397   ins_encode %{
11398     __ movdl($dst$$Register, $src$$XMMRegister);
11399   %}
11400   ins_pipe( pipe_slow );
11401 %}
11402 
11403 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11404   match(Set dst (MoveD2L src));
11405   effect(DEF dst, USE src);
11406   ins_cost(85);
11407   format %{ "movd    $dst,$src\t# MoveD2L" %}
11408   ins_encode %{
11409     __ movdq($dst$$Register, $src$$XMMRegister);
11410   %}
11411   ins_pipe( pipe_slow );
11412 %}
11413 
11414 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11415   match(Set dst (MoveI2F src));
11416   effect(DEF dst, USE src);
11417   ins_cost(100);
11418   format %{ "movd    $dst,$src\t# MoveI2F" %}
11419   ins_encode %{
11420     __ movdl($dst$$XMMRegister, $src$$Register);
11421   %}
11422   ins_pipe( pipe_slow );
11423 %}
11424 
11425 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11426   match(Set dst (MoveL2D src));
11427   effect(DEF dst, USE src);
11428   ins_cost(100);
11429   format %{ "movd    $dst,$src\t# MoveL2D" %}
11430   ins_encode %{
11431      __ movdq($dst$$XMMRegister, $src$$Register);
11432   %}
11433   ins_pipe( pipe_slow );
11434 %}
11435 
11436 // Fast clearing of an array
11437 // Small ClearArray non-AVX512.
11438 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11439                   Universe dummy, rFlagsReg cr)
11440 %{
11441   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11442   match(Set dummy (ClearArray cnt base));
11443   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11444 
11445   format %{ $$template
11446     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11447     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11448     $$emit$$"jg      LARGE\n\t"
11449     $$emit$$"dec     rcx\n\t"
11450     $$emit$$"js      DONE\t# Zero length\n\t"
11451     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11452     $$emit$$"dec     rcx\n\t"
11453     $$emit$$"jge     LOOP\n\t"
11454     $$emit$$"jmp     DONE\n\t"
11455     $$emit$$"# LARGE:\n\t"
11456     if (UseFastStosb) {
11457        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11458        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11459     } else if (UseXMMForObjInit) {
11460        $$emit$$"mov     rdi,rax\n\t"
11461        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11462        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11463        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11464        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11465        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11466        $$emit$$"add     0x40,rax\n\t"
11467        $$emit$$"# L_zero_64_bytes:\n\t"
11468        $$emit$$"sub     0x8,rcx\n\t"
11469        $$emit$$"jge     L_loop\n\t"
11470        $$emit$$"add     0x4,rcx\n\t"
11471        $$emit$$"jl      L_tail\n\t"
11472        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11473        $$emit$$"add     0x20,rax\n\t"
11474        $$emit$$"sub     0x4,rcx\n\t"
11475        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11476        $$emit$$"add     0x4,rcx\n\t"
11477        $$emit$$"jle     L_end\n\t"
11478        $$emit$$"dec     rcx\n\t"
11479        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11480        $$emit$$"vmovq   xmm0,(rax)\n\t"
11481        $$emit$$"add     0x8,rax\n\t"
11482        $$emit$$"dec     rcx\n\t"
11483        $$emit$$"jge     L_sloop\n\t"
11484        $$emit$$"# L_end:\n\t"
11485     } else {
11486        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11487     }
11488     $$emit$$"# DONE"
11489   %}
11490   ins_encode %{
11491     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11492                  $tmp$$XMMRegister, false, knoreg);
11493   %}
11494   ins_pipe(pipe_slow);
11495 %}
11496 
11497 // Small ClearArray AVX512 non-constant length.
11498 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11499                        Universe dummy, rFlagsReg cr)
11500 %{
11501   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11502   match(Set dummy (ClearArray cnt base));
11503   ins_cost(125);
11504   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11505 
11506   format %{ $$template
11507     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11508     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11509     $$emit$$"jg      LARGE\n\t"
11510     $$emit$$"dec     rcx\n\t"
11511     $$emit$$"js      DONE\t# Zero length\n\t"
11512     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11513     $$emit$$"dec     rcx\n\t"
11514     $$emit$$"jge     LOOP\n\t"
11515     $$emit$$"jmp     DONE\n\t"
11516     $$emit$$"# LARGE:\n\t"
11517     if (UseFastStosb) {
11518        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11519        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11520     } else if (UseXMMForObjInit) {
11521        $$emit$$"mov     rdi,rax\n\t"
11522        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11523        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11524        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11525        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11526        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11527        $$emit$$"add     0x40,rax\n\t"
11528        $$emit$$"# L_zero_64_bytes:\n\t"
11529        $$emit$$"sub     0x8,rcx\n\t"
11530        $$emit$$"jge     L_loop\n\t"
11531        $$emit$$"add     0x4,rcx\n\t"
11532        $$emit$$"jl      L_tail\n\t"
11533        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11534        $$emit$$"add     0x20,rax\n\t"
11535        $$emit$$"sub     0x4,rcx\n\t"
11536        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11537        $$emit$$"add     0x4,rcx\n\t"
11538        $$emit$$"jle     L_end\n\t"
11539        $$emit$$"dec     rcx\n\t"
11540        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11541        $$emit$$"vmovq   xmm0,(rax)\n\t"
11542        $$emit$$"add     0x8,rax\n\t"
11543        $$emit$$"dec     rcx\n\t"
11544        $$emit$$"jge     L_sloop\n\t"
11545        $$emit$$"# L_end:\n\t"
11546     } else {
11547        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11548     }
11549     $$emit$$"# DONE"
11550   %}
11551   ins_encode %{
11552     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11553                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11554   %}
11555   ins_pipe(pipe_slow);
11556 %}
11557 
11558 // Large ClearArray non-AVX512.
11559 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11560                         Universe dummy, rFlagsReg cr)
11561 %{
11562   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11563   match(Set dummy (ClearArray cnt base));
11564   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11565 
11566   format %{ $$template
11567     if (UseFastStosb) {
11568        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11569        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11570        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11571     } else if (UseXMMForObjInit) {
11572        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11573        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11574        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11575        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11576        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11577        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11578        $$emit$$"add     0x40,rax\n\t"
11579        $$emit$$"# L_zero_64_bytes:\n\t"
11580        $$emit$$"sub     0x8,rcx\n\t"
11581        $$emit$$"jge     L_loop\n\t"
11582        $$emit$$"add     0x4,rcx\n\t"
11583        $$emit$$"jl      L_tail\n\t"
11584        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11585        $$emit$$"add     0x20,rax\n\t"
11586        $$emit$$"sub     0x4,rcx\n\t"
11587        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11588        $$emit$$"add     0x4,rcx\n\t"
11589        $$emit$$"jle     L_end\n\t"
11590        $$emit$$"dec     rcx\n\t"
11591        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11592        $$emit$$"vmovq   xmm0,(rax)\n\t"
11593        $$emit$$"add     0x8,rax\n\t"
11594        $$emit$$"dec     rcx\n\t"
11595        $$emit$$"jge     L_sloop\n\t"
11596        $$emit$$"# L_end:\n\t"
11597     } else {
11598        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11599        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11600     }
11601   %}
11602   ins_encode %{
11603     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11604                  $tmp$$XMMRegister, true, knoreg);
11605   %}
11606   ins_pipe(pipe_slow);
11607 %}
11608 
11609 // Large ClearArray AVX512.
11610 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11611                              Universe dummy, rFlagsReg cr)
11612 %{
11613   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11614   match(Set dummy (ClearArray cnt base));
11615   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11616 
11617   format %{ $$template
11618     if (UseFastStosb) {
11619        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11620        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11621        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11622     } else if (UseXMMForObjInit) {
11623        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11624        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11625        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11626        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11627        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11628        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11629        $$emit$$"add     0x40,rax\n\t"
11630        $$emit$$"# L_zero_64_bytes:\n\t"
11631        $$emit$$"sub     0x8,rcx\n\t"
11632        $$emit$$"jge     L_loop\n\t"
11633        $$emit$$"add     0x4,rcx\n\t"
11634        $$emit$$"jl      L_tail\n\t"
11635        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11636        $$emit$$"add     0x20,rax\n\t"
11637        $$emit$$"sub     0x4,rcx\n\t"
11638        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11639        $$emit$$"add     0x4,rcx\n\t"
11640        $$emit$$"jle     L_end\n\t"
11641        $$emit$$"dec     rcx\n\t"
11642        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11643        $$emit$$"vmovq   xmm0,(rax)\n\t"
11644        $$emit$$"add     0x8,rax\n\t"
11645        $$emit$$"dec     rcx\n\t"
11646        $$emit$$"jge     L_sloop\n\t"
11647        $$emit$$"# L_end:\n\t"
11648     } else {
11649        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11650        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11651     }
11652   %}
11653   ins_encode %{
11654     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11655                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11656   %}
11657   ins_pipe(pipe_slow);
11658 %}
11659 
11660 // Small ClearArray AVX512 constant length.
11661 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11662 %{
11663   predicate(!((ClearArrayNode*)n)->is_large() &&
11664               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11665   match(Set dummy (ClearArray cnt base));
11666   ins_cost(100);
11667   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11668   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11669   ins_encode %{
11670    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11671   %}
11672   ins_pipe(pipe_slow);
11673 %}
11674 
11675 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11676                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11677 %{
11678   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11679   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11680   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11681 
11682   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11683   ins_encode %{
11684     __ string_compare($str1$$Register, $str2$$Register,
11685                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11686                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11687   %}
11688   ins_pipe( pipe_slow );
11689 %}
11690 
11691 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11692                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11693 %{
11694   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11695   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11696   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11697 
11698   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11699   ins_encode %{
11700     __ string_compare($str1$$Register, $str2$$Register,
11701                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11702                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11703   %}
11704   ins_pipe( pipe_slow );
11705 %}
11706 
11707 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11708                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11709 %{
11710   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11711   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11712   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11713 
11714   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11715   ins_encode %{
11716     __ string_compare($str1$$Register, $str2$$Register,
11717                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11718                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11719   %}
11720   ins_pipe( pipe_slow );
11721 %}
11722 
11723 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11724                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11725 %{
11726   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11727   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11728   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11729 
11730   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11731   ins_encode %{
11732     __ string_compare($str1$$Register, $str2$$Register,
11733                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11734                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11735   %}
11736   ins_pipe( pipe_slow );
11737 %}
11738 
11739 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11740                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11741 %{
11742   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11743   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11744   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11745 
11746   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11747   ins_encode %{
11748     __ string_compare($str1$$Register, $str2$$Register,
11749                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11750                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11751   %}
11752   ins_pipe( pipe_slow );
11753 %}
11754 
11755 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11756                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11757 %{
11758   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11759   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11760   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11761 
11762   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11763   ins_encode %{
11764     __ string_compare($str1$$Register, $str2$$Register,
11765                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11766                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11767   %}
11768   ins_pipe( pipe_slow );
11769 %}
11770 
11771 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11772                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11773 %{
11774   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11775   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11776   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11777 
11778   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11779   ins_encode %{
11780     __ string_compare($str2$$Register, $str1$$Register,
11781                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11782                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11783   %}
11784   ins_pipe( pipe_slow );
11785 %}
11786 
11787 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11788                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11789 %{
11790   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11791   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11792   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11793 
11794   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11795   ins_encode %{
11796     __ string_compare($str2$$Register, $str1$$Register,
11797                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11798                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11799   %}
11800   ins_pipe( pipe_slow );
11801 %}
11802 
11803 // fast search of substring with known size.
11804 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11805                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11806 %{
11807   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11808   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11809   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11810 
11811   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11812   ins_encode %{
11813     int icnt2 = (int)$int_cnt2$$constant;
11814     if (icnt2 >= 16) {
11815       // IndexOf for constant substrings with size >= 16 elements
11816       // which don't need to be loaded through stack.
11817       __ string_indexofC8($str1$$Register, $str2$$Register,
11818                           $cnt1$$Register, $cnt2$$Register,
11819                           icnt2, $result$$Register,
11820                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11821     } else {
11822       // Small strings are loaded through stack if they cross page boundary.
11823       __ string_indexof($str1$$Register, $str2$$Register,
11824                         $cnt1$$Register, $cnt2$$Register,
11825                         icnt2, $result$$Register,
11826                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11827     }
11828   %}
11829   ins_pipe( pipe_slow );
11830 %}
11831 
11832 // fast search of substring with known size.
11833 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11834                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11835 %{
11836   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11837   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11838   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11839 
11840   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11841   ins_encode %{
11842     int icnt2 = (int)$int_cnt2$$constant;
11843     if (icnt2 >= 8) {
11844       // IndexOf for constant substrings with size >= 8 elements
11845       // which don't need to be loaded through stack.
11846       __ string_indexofC8($str1$$Register, $str2$$Register,
11847                           $cnt1$$Register, $cnt2$$Register,
11848                           icnt2, $result$$Register,
11849                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11850     } else {
11851       // Small strings are loaded through stack if they cross page boundary.
11852       __ string_indexof($str1$$Register, $str2$$Register,
11853                         $cnt1$$Register, $cnt2$$Register,
11854                         icnt2, $result$$Register,
11855                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11856     }
11857   %}
11858   ins_pipe( pipe_slow );
11859 %}
11860 
11861 // fast search of substring with known size.
11862 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11863                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11864 %{
11865   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11866   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11867   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11868 
11869   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11870   ins_encode %{
11871     int icnt2 = (int)$int_cnt2$$constant;
11872     if (icnt2 >= 8) {
11873       // IndexOf for constant substrings with size >= 8 elements
11874       // which don't need to be loaded through stack.
11875       __ string_indexofC8($str1$$Register, $str2$$Register,
11876                           $cnt1$$Register, $cnt2$$Register,
11877                           icnt2, $result$$Register,
11878                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11879     } else {
11880       // Small strings are loaded through stack if they cross page boundary.
11881       __ string_indexof($str1$$Register, $str2$$Register,
11882                         $cnt1$$Register, $cnt2$$Register,
11883                         icnt2, $result$$Register,
11884                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11885     }
11886   %}
11887   ins_pipe( pipe_slow );
11888 %}
11889 
11890 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11891                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11892 %{
11893   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11894   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11895   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11896 
11897   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11898   ins_encode %{
11899     __ string_indexof($str1$$Register, $str2$$Register,
11900                       $cnt1$$Register, $cnt2$$Register,
11901                       (-1), $result$$Register,
11902                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11903   %}
11904   ins_pipe( pipe_slow );
11905 %}
11906 
11907 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11908                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11909 %{
11910   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11911   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11912   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11913 
11914   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11915   ins_encode %{
11916     __ string_indexof($str1$$Register, $str2$$Register,
11917                       $cnt1$$Register, $cnt2$$Register,
11918                       (-1), $result$$Register,
11919                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11920   %}
11921   ins_pipe( pipe_slow );
11922 %}
11923 
11924 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11925                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11926 %{
11927   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11928   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11929   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11930 
11931   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11932   ins_encode %{
11933     __ string_indexof($str1$$Register, $str2$$Register,
11934                       $cnt1$$Register, $cnt2$$Register,
11935                       (-1), $result$$Register,
11936                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11937   %}
11938   ins_pipe( pipe_slow );
11939 %}
11940 
11941 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11942                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11943 %{
11944   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11945   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11946   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11947   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11948   ins_encode %{
11949     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11950                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11951   %}
11952   ins_pipe( pipe_slow );
11953 %}
11954 
11955 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11956                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11957 %{
11958   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11959   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11960   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11961   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11962   ins_encode %{
11963     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11964                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11965   %}
11966   ins_pipe( pipe_slow );
11967 %}
11968 
11969 // fast string equals
11970 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11971                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11972 %{
11973   predicate(!VM_Version::supports_avx512vlbw());
11974   match(Set result (StrEquals (Binary str1 str2) cnt));
11975   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11976 
11977   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11978   ins_encode %{
11979     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11980                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11981                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11982   %}
11983   ins_pipe( pipe_slow );
11984 %}
11985 
11986 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11987                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11988 %{
11989   predicate(VM_Version::supports_avx512vlbw());
11990   match(Set result (StrEquals (Binary str1 str2) cnt));
11991   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11992 
11993   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11994   ins_encode %{
11995     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11996                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11997                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11998   %}
11999   ins_pipe( pipe_slow );
12000 %}
12001 
12002 // fast array equals
12003 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12004                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12005 %{
12006   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12007   match(Set result (AryEq ary1 ary2));
12008   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12009 
12010   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12011   ins_encode %{
12012     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12013                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12014                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12015   %}
12016   ins_pipe( pipe_slow );
12017 %}
12018 
12019 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12020                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12021 %{
12022   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12023   match(Set result (AryEq ary1 ary2));
12024   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12025 
12026   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12027   ins_encode %{
12028     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12029                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12030                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12031   %}
12032   ins_pipe( pipe_slow );
12033 %}
12034 
12035 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12036                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12037 %{
12038   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12039   match(Set result (AryEq ary1 ary2));
12040   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12041 
12042   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12043   ins_encode %{
12044     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12045                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12046                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12047   %}
12048   ins_pipe( pipe_slow );
12049 %}
12050 
12051 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12052                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12053 %{
12054   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12055   match(Set result (AryEq ary1 ary2));
12056   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12057 
12058   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12059   ins_encode %{
12060     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12061                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12062                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12063   %}
12064   ins_pipe( pipe_slow );
12065 %}
12066 
12067 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12068                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12069 %{
12070   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12071   match(Set result (CountPositives ary1 len));
12072   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12073 
12074   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12075   ins_encode %{
12076     __ count_positives($ary1$$Register, $len$$Register,
12077                        $result$$Register, $tmp3$$Register,
12078                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12079   %}
12080   ins_pipe( pipe_slow );
12081 %}
12082 
12083 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12084                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12085 %{
12086   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12087   match(Set result (CountPositives ary1 len));
12088   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12089 
12090   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12091   ins_encode %{
12092     __ count_positives($ary1$$Register, $len$$Register,
12093                        $result$$Register, $tmp3$$Register,
12094                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12095   %}
12096   ins_pipe( pipe_slow );
12097 %}
12098 
12099 // fast char[] to byte[] compression
12100 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12101                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12102   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12103   match(Set result (StrCompressedCopy src (Binary dst len)));
12104   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12105          USE_KILL len, KILL tmp5, KILL cr);
12106 
12107   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12108   ins_encode %{
12109     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12110                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12111                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12112                            knoreg, knoreg);
12113   %}
12114   ins_pipe( pipe_slow );
12115 %}
12116 
12117 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12118                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12119   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12120   match(Set result (StrCompressedCopy src (Binary dst len)));
12121   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12122          USE_KILL len, KILL tmp5, KILL cr);
12123 
12124   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12125   ins_encode %{
12126     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12127                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12128                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12129                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12130   %}
12131   ins_pipe( pipe_slow );
12132 %}
12133 // fast byte[] to char[] inflation
12134 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12135                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12136   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12137   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12138   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12139 
12140   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12141   ins_encode %{
12142     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12143                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12144   %}
12145   ins_pipe( pipe_slow );
12146 %}
12147 
12148 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12149                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12150   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12151   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12152   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12153 
12154   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12155   ins_encode %{
12156     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12157                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12158   %}
12159   ins_pipe( pipe_slow );
12160 %}
12161 
12162 // encode char[] to byte[] in ISO_8859_1
12163 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12164                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12165                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12166   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12167   match(Set result (EncodeISOArray src (Binary dst len)));
12168   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12169 
12170   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12171   ins_encode %{
12172     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12173                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12174                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12175   %}
12176   ins_pipe( pipe_slow );
12177 %}
12178 
12179 // encode char[] to byte[] in ASCII
12180 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12181                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12182                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12183   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12184   match(Set result (EncodeISOArray src (Binary dst len)));
12185   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12186 
12187   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12188   ins_encode %{
12189     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12190                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12191                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12192   %}
12193   ins_pipe( pipe_slow );
12194 %}
12195 
12196 //----------Overflow Math Instructions-----------------------------------------
12197 
12198 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12199 %{
12200   match(Set cr (OverflowAddI op1 op2));
12201   effect(DEF cr, USE_KILL op1, USE op2);
12202 
12203   format %{ "addl    $op1, $op2\t# overflow check int" %}
12204 
12205   ins_encode %{
12206     __ addl($op1$$Register, $op2$$Register);
12207   %}
12208   ins_pipe(ialu_reg_reg);
12209 %}
12210 
12211 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12212 %{
12213   match(Set cr (OverflowAddI op1 op2));
12214   effect(DEF cr, USE_KILL op1, USE op2);
12215 
12216   format %{ "addl    $op1, $op2\t# overflow check int" %}
12217 
12218   ins_encode %{
12219     __ addl($op1$$Register, $op2$$constant);
12220   %}
12221   ins_pipe(ialu_reg_reg);
12222 %}
12223 
12224 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12225 %{
12226   match(Set cr (OverflowAddL op1 op2));
12227   effect(DEF cr, USE_KILL op1, USE op2);
12228 
12229   format %{ "addq    $op1, $op2\t# overflow check long" %}
12230   ins_encode %{
12231     __ addq($op1$$Register, $op2$$Register);
12232   %}
12233   ins_pipe(ialu_reg_reg);
12234 %}
12235 
12236 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12237 %{
12238   match(Set cr (OverflowAddL op1 op2));
12239   effect(DEF cr, USE_KILL op1, USE op2);
12240 
12241   format %{ "addq    $op1, $op2\t# overflow check long" %}
12242   ins_encode %{
12243     __ addq($op1$$Register, $op2$$constant);
12244   %}
12245   ins_pipe(ialu_reg_reg);
12246 %}
12247 
12248 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12249 %{
12250   match(Set cr (OverflowSubI op1 op2));
12251 
12252   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12253   ins_encode %{
12254     __ cmpl($op1$$Register, $op2$$Register);
12255   %}
12256   ins_pipe(ialu_reg_reg);
12257 %}
12258 
12259 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12260 %{
12261   match(Set cr (OverflowSubI op1 op2));
12262 
12263   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12264   ins_encode %{
12265     __ cmpl($op1$$Register, $op2$$constant);
12266   %}
12267   ins_pipe(ialu_reg_reg);
12268 %}
12269 
12270 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12271 %{
12272   match(Set cr (OverflowSubL op1 op2));
12273 
12274   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12275   ins_encode %{
12276     __ cmpq($op1$$Register, $op2$$Register);
12277   %}
12278   ins_pipe(ialu_reg_reg);
12279 %}
12280 
12281 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12282 %{
12283   match(Set cr (OverflowSubL op1 op2));
12284 
12285   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12286   ins_encode %{
12287     __ cmpq($op1$$Register, $op2$$constant);
12288   %}
12289   ins_pipe(ialu_reg_reg);
12290 %}
12291 
12292 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12293 %{
12294   match(Set cr (OverflowSubI zero op2));
12295   effect(DEF cr, USE_KILL op2);
12296 
12297   format %{ "negl    $op2\t# overflow check int" %}
12298   ins_encode %{
12299     __ negl($op2$$Register);
12300   %}
12301   ins_pipe(ialu_reg_reg);
12302 %}
12303 
12304 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12305 %{
12306   match(Set cr (OverflowSubL zero op2));
12307   effect(DEF cr, USE_KILL op2);
12308 
12309   format %{ "negq    $op2\t# overflow check long" %}
12310   ins_encode %{
12311     __ negq($op2$$Register);
12312   %}
12313   ins_pipe(ialu_reg_reg);
12314 %}
12315 
12316 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12317 %{
12318   match(Set cr (OverflowMulI op1 op2));
12319   effect(DEF cr, USE_KILL op1, USE op2);
12320 
12321   format %{ "imull    $op1, $op2\t# overflow check int" %}
12322   ins_encode %{
12323     __ imull($op1$$Register, $op2$$Register);
12324   %}
12325   ins_pipe(ialu_reg_reg_alu0);
12326 %}
12327 
12328 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12329 %{
12330   match(Set cr (OverflowMulI op1 op2));
12331   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12332 
12333   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12334   ins_encode %{
12335     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12336   %}
12337   ins_pipe(ialu_reg_reg_alu0);
12338 %}
12339 
12340 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12341 %{
12342   match(Set cr (OverflowMulL op1 op2));
12343   effect(DEF cr, USE_KILL op1, USE op2);
12344 
12345   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12346   ins_encode %{
12347     __ imulq($op1$$Register, $op2$$Register);
12348   %}
12349   ins_pipe(ialu_reg_reg_alu0);
12350 %}
12351 
12352 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12353 %{
12354   match(Set cr (OverflowMulL op1 op2));
12355   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12356 
12357   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12358   ins_encode %{
12359     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12360   %}
12361   ins_pipe(ialu_reg_reg_alu0);
12362 %}
12363 
12364 
12365 //----------Control Flow Instructions------------------------------------------
12366 // Signed compare Instructions
12367 
12368 // XXX more variants!!
12369 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12370 %{
12371   match(Set cr (CmpI op1 op2));
12372   effect(DEF cr, USE op1, USE op2);
12373 
12374   format %{ "cmpl    $op1, $op2" %}
12375   ins_encode %{
12376     __ cmpl($op1$$Register, $op2$$Register);
12377   %}
12378   ins_pipe(ialu_cr_reg_reg);
12379 %}
12380 
12381 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12382 %{
12383   match(Set cr (CmpI op1 op2));
12384 
12385   format %{ "cmpl    $op1, $op2" %}
12386   ins_encode %{
12387     __ cmpl($op1$$Register, $op2$$constant);
12388   %}
12389   ins_pipe(ialu_cr_reg_imm);
12390 %}
12391 
12392 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12393 %{
12394   match(Set cr (CmpI op1 (LoadI op2)));
12395 
12396   ins_cost(500); // XXX
12397   format %{ "cmpl    $op1, $op2" %}
12398   ins_encode %{
12399     __ cmpl($op1$$Register, $op2$$Address);
12400   %}
12401   ins_pipe(ialu_cr_reg_mem);
12402 %}
12403 
12404 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12405 %{
12406   match(Set cr (CmpI src zero));
12407 
12408   format %{ "testl   $src, $src" %}
12409   ins_encode %{
12410     __ testl($src$$Register, $src$$Register);
12411   %}
12412   ins_pipe(ialu_cr_reg_imm);
12413 %}
12414 
12415 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12416 %{
12417   match(Set cr (CmpI (AndI src con) zero));
12418 
12419   format %{ "testl   $src, $con" %}
12420   ins_encode %{
12421     __ testl($src$$Register, $con$$constant);
12422   %}
12423   ins_pipe(ialu_cr_reg_imm);
12424 %}
12425 
12426 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12427 %{
12428   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12429 
12430   format %{ "testl   $src, $mem" %}
12431   ins_encode %{
12432     __ testl($src$$Register, $mem$$Address);
12433   %}
12434   ins_pipe(ialu_cr_reg_mem);
12435 %}
12436 
12437 // Unsigned compare Instructions; really, same as signed except they
12438 // produce an rFlagsRegU instead of rFlagsReg.
12439 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12440 %{
12441   match(Set cr (CmpU op1 op2));
12442 
12443   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12444   ins_encode %{
12445     __ cmpl($op1$$Register, $op2$$Register);
12446   %}
12447   ins_pipe(ialu_cr_reg_reg);
12448 %}
12449 
12450 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12451 %{
12452   match(Set cr (CmpU op1 op2));
12453 
12454   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12455   ins_encode %{
12456     __ cmpl($op1$$Register, $op2$$constant);
12457   %}
12458   ins_pipe(ialu_cr_reg_imm);
12459 %}
12460 
12461 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12462 %{
12463   match(Set cr (CmpU op1 (LoadI op2)));
12464 
12465   ins_cost(500); // XXX
12466   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12467   ins_encode %{
12468     __ cmpl($op1$$Register, $op2$$Address);
12469   %}
12470   ins_pipe(ialu_cr_reg_mem);
12471 %}
12472 
12473 // // // Cisc-spilled version of cmpU_rReg
12474 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12475 // //%{
12476 // //  match(Set cr (CmpU (LoadI op1) op2));
12477 // //
12478 // //  format %{ "CMPu   $op1,$op2" %}
12479 // //  ins_cost(500);
12480 // //  opcode(0x39);  /* Opcode 39 /r */
12481 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12482 // //%}
12483 
12484 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12485 %{
12486   match(Set cr (CmpU src zero));
12487 
12488   format %{ "testl   $src, $src\t# unsigned" %}
12489   ins_encode %{
12490     __ testl($src$$Register, $src$$Register);
12491   %}
12492   ins_pipe(ialu_cr_reg_imm);
12493 %}
12494 
12495 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12496 %{
12497   match(Set cr (CmpP op1 op2));
12498 
12499   format %{ "cmpq    $op1, $op2\t# ptr" %}
12500   ins_encode %{
12501     __ cmpq($op1$$Register, $op2$$Register);
12502   %}
12503   ins_pipe(ialu_cr_reg_reg);
12504 %}
12505 
12506 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12507 %{
12508   match(Set cr (CmpP op1 (LoadP op2)));
12509   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12510 
12511   ins_cost(500); // XXX
12512   format %{ "cmpq    $op1, $op2\t# ptr" %}
12513   ins_encode %{
12514     __ cmpq($op1$$Register, $op2$$Address);
12515   %}
12516   ins_pipe(ialu_cr_reg_mem);
12517 %}
12518 
12519 // // // Cisc-spilled version of cmpP_rReg
12520 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12521 // //%{
12522 // //  match(Set cr (CmpP (LoadP op1) op2));
12523 // //
12524 // //  format %{ "CMPu   $op1,$op2" %}
12525 // //  ins_cost(500);
12526 // //  opcode(0x39);  /* Opcode 39 /r */
12527 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12528 // //%}
12529 
12530 // XXX this is generalized by compP_rReg_mem???
12531 // Compare raw pointer (used in out-of-heap check).
12532 // Only works because non-oop pointers must be raw pointers
12533 // and raw pointers have no anti-dependencies.
12534 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12535 %{
12536   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12537             n->in(2)->as_Load()->barrier_data() == 0);
12538   match(Set cr (CmpP op1 (LoadP op2)));
12539 
12540   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12541   ins_encode %{
12542     __ cmpq($op1$$Register, $op2$$Address);
12543   %}
12544   ins_pipe(ialu_cr_reg_mem);
12545 %}
12546 
12547 // This will generate a signed flags result. This should be OK since
12548 // any compare to a zero should be eq/neq.
12549 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12550 %{
12551   match(Set cr (CmpP src zero));
12552 
12553   format %{ "testq   $src, $src\t# ptr" %}
12554   ins_encode %{
12555     __ testq($src$$Register, $src$$Register);
12556   %}
12557   ins_pipe(ialu_cr_reg_imm);
12558 %}
12559 
12560 // This will generate a signed flags result. This should be OK since
12561 // any compare to a zero should be eq/neq.
12562 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12563 %{
12564   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12565             n->in(1)->as_Load()->barrier_data() == 0);
12566   match(Set cr (CmpP (LoadP op) zero));
12567 
12568   ins_cost(500); // XXX
12569   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12570   ins_encode %{
12571     __ testq($op$$Address, 0xFFFFFFFF);
12572   %}
12573   ins_pipe(ialu_cr_reg_imm);
12574 %}
12575 
12576 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12577 %{
12578   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12579             n->in(1)->as_Load()->barrier_data() == 0);
12580   match(Set cr (CmpP (LoadP mem) zero));
12581 
12582   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12583   ins_encode %{
12584     __ cmpq(r12, $mem$$Address);
12585   %}
12586   ins_pipe(ialu_cr_reg_mem);
12587 %}
12588 
12589 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12590 %{
12591   match(Set cr (CmpN op1 op2));
12592 
12593   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12594   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12595   ins_pipe(ialu_cr_reg_reg);
12596 %}
12597 
12598 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12599 %{
12600   match(Set cr (CmpN src (LoadN mem)));
12601 
12602   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12603   ins_encode %{
12604     __ cmpl($src$$Register, $mem$$Address);
12605   %}
12606   ins_pipe(ialu_cr_reg_mem);
12607 %}
12608 
12609 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12610   match(Set cr (CmpN op1 op2));
12611 
12612   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12613   ins_encode %{
12614     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12615   %}
12616   ins_pipe(ialu_cr_reg_imm);
12617 %}
12618 
12619 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12620 %{
12621   match(Set cr (CmpN src (LoadN mem)));
12622 
12623   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12624   ins_encode %{
12625     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12626   %}
12627   ins_pipe(ialu_cr_reg_mem);
12628 %}
12629 
12630 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12631   match(Set cr (CmpN op1 op2));
12632 
12633   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12634   ins_encode %{
12635     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12636   %}
12637   ins_pipe(ialu_cr_reg_imm);
12638 %}
12639 
12640 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12641 %{
12642   match(Set cr (CmpN src (LoadNKlass mem)));
12643 
12644   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12645   ins_encode %{
12646     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12647   %}
12648   ins_pipe(ialu_cr_reg_mem);
12649 %}
12650 
12651 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12652   match(Set cr (CmpN src zero));
12653 
12654   format %{ "testl   $src, $src\t# compressed ptr" %}
12655   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12656   ins_pipe(ialu_cr_reg_imm);
12657 %}
12658 
12659 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12660 %{
12661   predicate(CompressedOops::base() != NULL);
12662   match(Set cr (CmpN (LoadN mem) zero));
12663 
12664   ins_cost(500); // XXX
12665   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12666   ins_encode %{
12667     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12668   %}
12669   ins_pipe(ialu_cr_reg_mem);
12670 %}
12671 
12672 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12673 %{
12674   predicate(CompressedOops::base() == NULL);
12675   match(Set cr (CmpN (LoadN mem) zero));
12676 
12677   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12678   ins_encode %{
12679     __ cmpl(r12, $mem$$Address);
12680   %}
12681   ins_pipe(ialu_cr_reg_mem);
12682 %}
12683 
12684 // Yanked all unsigned pointer compare operations.
12685 // Pointer compares are done with CmpP which is already unsigned.
12686 
12687 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12688 %{
12689   match(Set cr (CmpL op1 op2));
12690 
12691   format %{ "cmpq    $op1, $op2" %}
12692   ins_encode %{
12693     __ cmpq($op1$$Register, $op2$$Register);
12694   %}
12695   ins_pipe(ialu_cr_reg_reg);
12696 %}
12697 
12698 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12699 %{
12700   match(Set cr (CmpL op1 op2));
12701 
12702   format %{ "cmpq    $op1, $op2" %}
12703   ins_encode %{
12704     __ cmpq($op1$$Register, $op2$$constant);
12705   %}
12706   ins_pipe(ialu_cr_reg_imm);
12707 %}
12708 
12709 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12710 %{
12711   match(Set cr (CmpL op1 (LoadL op2)));
12712 
12713   format %{ "cmpq    $op1, $op2" %}
12714   ins_encode %{
12715     __ cmpq($op1$$Register, $op2$$Address);
12716   %}
12717   ins_pipe(ialu_cr_reg_mem);
12718 %}
12719 
12720 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12721 %{
12722   match(Set cr (CmpL src zero));
12723 
12724   format %{ "testq   $src, $src" %}
12725   ins_encode %{
12726     __ testq($src$$Register, $src$$Register);
12727   %}
12728   ins_pipe(ialu_cr_reg_imm);
12729 %}
12730 
12731 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12732 %{
12733   match(Set cr (CmpL (AndL src con) zero));
12734 
12735   format %{ "testq   $src, $con\t# long" %}
12736   ins_encode %{
12737     __ testq($src$$Register, $con$$constant);
12738   %}
12739   ins_pipe(ialu_cr_reg_imm);
12740 %}
12741 
12742 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12743 %{
12744   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12745 
12746   format %{ "testq   $src, $mem" %}
12747   ins_encode %{
12748     __ testq($src$$Register, $mem$$Address);
12749   %}
12750   ins_pipe(ialu_cr_reg_mem);
12751 %}
12752 
12753 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12754 %{
12755   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12756 
12757   format %{ "testq   $src, $mem" %}
12758   ins_encode %{
12759     __ testq($src$$Register, $mem$$Address);
12760   %}
12761   ins_pipe(ialu_cr_reg_mem);
12762 %}
12763 
12764 // Manifest a CmpU result in an integer register.  Very painful.
12765 // This is the test to avoid.
12766 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
12767 %{
12768   match(Set dst (CmpU3 src1 src2));
12769   effect(KILL flags);
12770 
12771   ins_cost(275); // XXX
12772   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
12773             "movl    $dst, -1\n\t"
12774             "jb,u    done\n\t"
12775             "setne   $dst\n\t"
12776             "movzbl  $dst, $dst\n\t"
12777     "done:" %}
12778   ins_encode %{
12779     Label done;
12780     __ cmpl($src1$$Register, $src2$$Register);
12781     __ movl($dst$$Register, -1);
12782     __ jccb(Assembler::below, done);
12783     __ setne($dst$$Register);
12784     __ movzbl($dst$$Register, $dst$$Register);
12785     __ bind(done);
12786   %}
12787   ins_pipe(pipe_slow);
12788 %}
12789 
12790 // Manifest a CmpL result in an integer register.  Very painful.
12791 // This is the test to avoid.
12792 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12793 %{
12794   match(Set dst (CmpL3 src1 src2));
12795   effect(KILL flags);
12796 
12797   ins_cost(275); // XXX
12798   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12799             "movl    $dst, -1\n\t"
12800             "jl,s    done\n\t"
12801             "setne   $dst\n\t"
12802             "movzbl  $dst, $dst\n\t"
12803     "done:" %}
12804   ins_encode %{
12805     Label done;
12806     __ cmpq($src1$$Register, $src2$$Register);
12807     __ movl($dst$$Register, -1);
12808     __ jccb(Assembler::less, done);
12809     __ setne($dst$$Register);
12810     __ movzbl($dst$$Register, $dst$$Register);
12811     __ bind(done);
12812   %}
12813   ins_pipe(pipe_slow);
12814 %}
12815 
12816 // Manifest a CmpUL result in an integer register.  Very painful.
12817 // This is the test to avoid.
12818 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12819 %{
12820   match(Set dst (CmpUL3 src1 src2));
12821   effect(KILL flags);
12822 
12823   ins_cost(275); // XXX
12824   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12825             "movl    $dst, -1\n\t"
12826             "jb,u    done\n\t"
12827             "setne   $dst\n\t"
12828             "movzbl  $dst, $dst\n\t"
12829     "done:" %}
12830   ins_encode %{
12831     Label done;
12832     __ cmpq($src1$$Register, $src2$$Register);
12833     __ movl($dst$$Register, -1);
12834     __ jccb(Assembler::below, done);
12835     __ setne($dst$$Register);
12836     __ movzbl($dst$$Register, $dst$$Register);
12837     __ bind(done);
12838   %}
12839   ins_pipe(pipe_slow);
12840 %}
12841 
12842 // Unsigned long compare Instructions; really, same as signed long except they
12843 // produce an rFlagsRegU instead of rFlagsReg.
12844 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12845 %{
12846   match(Set cr (CmpUL op1 op2));
12847 
12848   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12849   ins_encode %{
12850     __ cmpq($op1$$Register, $op2$$Register);
12851   %}
12852   ins_pipe(ialu_cr_reg_reg);
12853 %}
12854 
12855 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12856 %{
12857   match(Set cr (CmpUL op1 op2));
12858 
12859   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12860   ins_encode %{
12861     __ cmpq($op1$$Register, $op2$$constant);
12862   %}
12863   ins_pipe(ialu_cr_reg_imm);
12864 %}
12865 
12866 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12867 %{
12868   match(Set cr (CmpUL op1 (LoadL op2)));
12869 
12870   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12871   ins_encode %{
12872     __ cmpq($op1$$Register, $op2$$Address);
12873   %}
12874   ins_pipe(ialu_cr_reg_mem);
12875 %}
12876 
12877 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12878 %{
12879   match(Set cr (CmpUL src zero));
12880 
12881   format %{ "testq   $src, $src\t# unsigned" %}
12882   ins_encode %{
12883     __ testq($src$$Register, $src$$Register);
12884   %}
12885   ins_pipe(ialu_cr_reg_imm);
12886 %}
12887 
12888 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12889 %{
12890   match(Set cr (CmpI (LoadB mem) imm));
12891 
12892   ins_cost(125);
12893   format %{ "cmpb    $mem, $imm" %}
12894   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12895   ins_pipe(ialu_cr_reg_mem);
12896 %}
12897 
12898 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12899 %{
12900   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12901 
12902   ins_cost(125);
12903   format %{ "testb   $mem, $imm\t# ubyte" %}
12904   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12905   ins_pipe(ialu_cr_reg_mem);
12906 %}
12907 
12908 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12909 %{
12910   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12911 
12912   ins_cost(125);
12913   format %{ "testb   $mem, $imm\t# byte" %}
12914   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12915   ins_pipe(ialu_cr_reg_mem);
12916 %}
12917 
12918 //----------Max and Min--------------------------------------------------------
12919 // Min Instructions
12920 
12921 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12922 %{
12923   effect(USE_DEF dst, USE src, USE cr);
12924 
12925   format %{ "cmovlgt $dst, $src\t# min" %}
12926   ins_encode %{
12927     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12928   %}
12929   ins_pipe(pipe_cmov_reg);
12930 %}
12931 
12932 
12933 instruct minI_rReg(rRegI dst, rRegI src)
12934 %{
12935   match(Set dst (MinI dst src));
12936 
12937   ins_cost(200);
12938   expand %{
12939     rFlagsReg cr;
12940     compI_rReg(cr, dst, src);
12941     cmovI_reg_g(dst, src, cr);
12942   %}
12943 %}
12944 
12945 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12946 %{
12947   effect(USE_DEF dst, USE src, USE cr);
12948 
12949   format %{ "cmovllt $dst, $src\t# max" %}
12950   ins_encode %{
12951     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12952   %}
12953   ins_pipe(pipe_cmov_reg);
12954 %}
12955 
12956 
12957 instruct maxI_rReg(rRegI dst, rRegI src)
12958 %{
12959   match(Set dst (MaxI dst src));
12960 
12961   ins_cost(200);
12962   expand %{
12963     rFlagsReg cr;
12964     compI_rReg(cr, dst, src);
12965     cmovI_reg_l(dst, src, cr);
12966   %}
12967 %}
12968 
12969 // ============================================================================
12970 // Branch Instructions
12971 
12972 // Jump Direct - Label defines a relative address from JMP+1
12973 instruct jmpDir(label labl)
12974 %{
12975   match(Goto);
12976   effect(USE labl);
12977 
12978   ins_cost(300);
12979   format %{ "jmp     $labl" %}
12980   size(5);
12981   ins_encode %{
12982     Label* L = $labl$$label;
12983     __ jmp(*L, false); // Always long jump
12984   %}
12985   ins_pipe(pipe_jmp);
12986 %}
12987 
12988 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12989 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12990 %{
12991   match(If cop cr);
12992   effect(USE labl);
12993 
12994   ins_cost(300);
12995   format %{ "j$cop     $labl" %}
12996   size(6);
12997   ins_encode %{
12998     Label* L = $labl$$label;
12999     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13000   %}
13001   ins_pipe(pipe_jcc);
13002 %}
13003 
13004 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13005 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13006 %{
13007   match(CountedLoopEnd cop cr);
13008   effect(USE labl);
13009 
13010   ins_cost(300);
13011   format %{ "j$cop     $labl\t# loop end" %}
13012   size(6);
13013   ins_encode %{
13014     Label* L = $labl$$label;
13015     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13016   %}
13017   ins_pipe(pipe_jcc);
13018 %}
13019 
13020 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13021 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13022   match(CountedLoopEnd cop cmp);
13023   effect(USE labl);
13024 
13025   ins_cost(300);
13026   format %{ "j$cop,u   $labl\t# loop end" %}
13027   size(6);
13028   ins_encode %{
13029     Label* L = $labl$$label;
13030     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13031   %}
13032   ins_pipe(pipe_jcc);
13033 %}
13034 
13035 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13036   match(CountedLoopEnd cop cmp);
13037   effect(USE labl);
13038 
13039   ins_cost(200);
13040   format %{ "j$cop,u   $labl\t# loop end" %}
13041   size(6);
13042   ins_encode %{
13043     Label* L = $labl$$label;
13044     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13045   %}
13046   ins_pipe(pipe_jcc);
13047 %}
13048 
13049 // Jump Direct Conditional - using unsigned comparison
13050 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13051   match(If cop cmp);
13052   effect(USE labl);
13053 
13054   ins_cost(300);
13055   format %{ "j$cop,u   $labl" %}
13056   size(6);
13057   ins_encode %{
13058     Label* L = $labl$$label;
13059     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13060   %}
13061   ins_pipe(pipe_jcc);
13062 %}
13063 
13064 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13065   match(If cop cmp);
13066   effect(USE labl);
13067 
13068   ins_cost(200);
13069   format %{ "j$cop,u   $labl" %}
13070   size(6);
13071   ins_encode %{
13072     Label* L = $labl$$label;
13073     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13074   %}
13075   ins_pipe(pipe_jcc);
13076 %}
13077 
13078 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13079   match(If cop cmp);
13080   effect(USE labl);
13081 
13082   ins_cost(200);
13083   format %{ $$template
13084     if ($cop$$cmpcode == Assembler::notEqual) {
13085       $$emit$$"jp,u    $labl\n\t"
13086       $$emit$$"j$cop,u   $labl"
13087     } else {
13088       $$emit$$"jp,u    done\n\t"
13089       $$emit$$"j$cop,u   $labl\n\t"
13090       $$emit$$"done:"
13091     }
13092   %}
13093   ins_encode %{
13094     Label* l = $labl$$label;
13095     if ($cop$$cmpcode == Assembler::notEqual) {
13096       __ jcc(Assembler::parity, *l, false);
13097       __ jcc(Assembler::notEqual, *l, false);
13098     } else if ($cop$$cmpcode == Assembler::equal) {
13099       Label done;
13100       __ jccb(Assembler::parity, done);
13101       __ jcc(Assembler::equal, *l, false);
13102       __ bind(done);
13103     } else {
13104        ShouldNotReachHere();
13105     }
13106   %}
13107   ins_pipe(pipe_jcc);
13108 %}
13109 
13110 // ============================================================================
13111 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13112 // superklass array for an instance of the superklass.  Set a hidden
13113 // internal cache on a hit (cache is checked with exposed code in
13114 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13115 // encoding ALSO sets flags.
13116 
13117 instruct partialSubtypeCheck(rdi_RegP result,
13118                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13119                              rFlagsReg cr)
13120 %{
13121   match(Set result (PartialSubtypeCheck sub super));
13122   effect(KILL rcx, KILL cr);
13123 
13124   ins_cost(1100);  // slightly larger than the next version
13125   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13126             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13127             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13128             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13129             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13130             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13131             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13132     "miss:\t" %}
13133 
13134   opcode(0x1); // Force a XOR of RDI
13135   ins_encode(enc_PartialSubtypeCheck());
13136   ins_pipe(pipe_slow);
13137 %}
13138 
13139 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
13140                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13141                                      immP0 zero,
13142                                      rdi_RegP result)
13143 %{
13144   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13145   effect(KILL rcx, KILL result);
13146 
13147   ins_cost(1000);
13148   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13149             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13150             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13151             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
13152             "jne,s   miss\t\t# Missed: flags nz\n\t"
13153             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13154     "miss:\t" %}
13155 
13156   opcode(0x0); // No need to XOR RDI
13157   ins_encode(enc_PartialSubtypeCheck());
13158   ins_pipe(pipe_slow);
13159 %}
13160 
13161 // ============================================================================
13162 // Branch Instructions -- short offset versions
13163 //
13164 // These instructions are used to replace jumps of a long offset (the default
13165 // match) with jumps of a shorter offset.  These instructions are all tagged
13166 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13167 // match rules in general matching.  Instead, the ADLC generates a conversion
13168 // method in the MachNode which can be used to do in-place replacement of the
13169 // long variant with the shorter variant.  The compiler will determine if a
13170 // branch can be taken by the is_short_branch_offset() predicate in the machine
13171 // specific code section of the file.
13172 
13173 // Jump Direct - Label defines a relative address from JMP+1
13174 instruct jmpDir_short(label labl) %{
13175   match(Goto);
13176   effect(USE labl);
13177 
13178   ins_cost(300);
13179   format %{ "jmp,s   $labl" %}
13180   size(2);
13181   ins_encode %{
13182     Label* L = $labl$$label;
13183     __ jmpb(*L);
13184   %}
13185   ins_pipe(pipe_jmp);
13186   ins_short_branch(1);
13187 %}
13188 
13189 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13190 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13191   match(If cop cr);
13192   effect(USE labl);
13193 
13194   ins_cost(300);
13195   format %{ "j$cop,s   $labl" %}
13196   size(2);
13197   ins_encode %{
13198     Label* L = $labl$$label;
13199     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13200   %}
13201   ins_pipe(pipe_jcc);
13202   ins_short_branch(1);
13203 %}
13204 
13205 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13206 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13207   match(CountedLoopEnd cop cr);
13208   effect(USE labl);
13209 
13210   ins_cost(300);
13211   format %{ "j$cop,s   $labl\t# loop end" %}
13212   size(2);
13213   ins_encode %{
13214     Label* L = $labl$$label;
13215     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13216   %}
13217   ins_pipe(pipe_jcc);
13218   ins_short_branch(1);
13219 %}
13220 
13221 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13222 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13223   match(CountedLoopEnd cop cmp);
13224   effect(USE labl);
13225 
13226   ins_cost(300);
13227   format %{ "j$cop,us  $labl\t# loop end" %}
13228   size(2);
13229   ins_encode %{
13230     Label* L = $labl$$label;
13231     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13232   %}
13233   ins_pipe(pipe_jcc);
13234   ins_short_branch(1);
13235 %}
13236 
13237 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13238   match(CountedLoopEnd cop cmp);
13239   effect(USE labl);
13240 
13241   ins_cost(300);
13242   format %{ "j$cop,us  $labl\t# loop end" %}
13243   size(2);
13244   ins_encode %{
13245     Label* L = $labl$$label;
13246     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13247   %}
13248   ins_pipe(pipe_jcc);
13249   ins_short_branch(1);
13250 %}
13251 
13252 // Jump Direct Conditional - using unsigned comparison
13253 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13254   match(If cop cmp);
13255   effect(USE labl);
13256 
13257   ins_cost(300);
13258   format %{ "j$cop,us  $labl" %}
13259   size(2);
13260   ins_encode %{
13261     Label* L = $labl$$label;
13262     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13263   %}
13264   ins_pipe(pipe_jcc);
13265   ins_short_branch(1);
13266 %}
13267 
13268 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13269   match(If cop cmp);
13270   effect(USE labl);
13271 
13272   ins_cost(300);
13273   format %{ "j$cop,us  $labl" %}
13274   size(2);
13275   ins_encode %{
13276     Label* L = $labl$$label;
13277     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13278   %}
13279   ins_pipe(pipe_jcc);
13280   ins_short_branch(1);
13281 %}
13282 
13283 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13284   match(If cop cmp);
13285   effect(USE labl);
13286 
13287   ins_cost(300);
13288   format %{ $$template
13289     if ($cop$$cmpcode == Assembler::notEqual) {
13290       $$emit$$"jp,u,s  $labl\n\t"
13291       $$emit$$"j$cop,u,s  $labl"
13292     } else {
13293       $$emit$$"jp,u,s  done\n\t"
13294       $$emit$$"j$cop,u,s  $labl\n\t"
13295       $$emit$$"done:"
13296     }
13297   %}
13298   size(4);
13299   ins_encode %{
13300     Label* l = $labl$$label;
13301     if ($cop$$cmpcode == Assembler::notEqual) {
13302       __ jccb(Assembler::parity, *l);
13303       __ jccb(Assembler::notEqual, *l);
13304     } else if ($cop$$cmpcode == Assembler::equal) {
13305       Label done;
13306       __ jccb(Assembler::parity, done);
13307       __ jccb(Assembler::equal, *l);
13308       __ bind(done);
13309     } else {
13310        ShouldNotReachHere();
13311     }
13312   %}
13313   ins_pipe(pipe_jcc);
13314   ins_short_branch(1);
13315 %}
13316 
13317 // ============================================================================
13318 // inlined locking and unlocking
13319 
13320 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
13321   predicate(Compile::current()->use_rtm());
13322   match(Set cr (FastLock object box));
13323   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
13324   ins_cost(300);
13325   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13326   ins_encode %{
13327     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13328                  $scr$$Register, $cx1$$Register, $cx2$$Register,
13329                  _rtm_counters, _stack_rtm_counters,
13330                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13331                  true, ra_->C->profile_rtm());
13332   %}
13333   ins_pipe(pipe_slow);
13334 %}
13335 
13336 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13337   predicate(!Compile::current()->use_rtm());
13338   match(Set cr (FastLock object box));
13339   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
13340   ins_cost(300);
13341   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13342   ins_encode %{
13343     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13344                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
13345   %}
13346   ins_pipe(pipe_slow);
13347 %}
13348 
13349 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13350   match(Set cr (FastUnlock object box));
13351   effect(TEMP tmp, USE_KILL box);
13352   ins_cost(300);
13353   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13354   ins_encode %{
13355     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13356   %}
13357   ins_pipe(pipe_slow);
13358 %}
13359 
13360 
13361 // ============================================================================
13362 // Safepoint Instructions
13363 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13364 %{
13365   match(SafePoint poll);
13366   effect(KILL cr, USE poll);
13367 
13368   format %{ "testl   rax, [$poll]\t"
13369             "# Safepoint: poll for GC" %}
13370   ins_cost(125);
13371   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13372   ins_encode %{
13373     __ relocate(relocInfo::poll_type);
13374     address pre_pc = __ pc();
13375     __ testl(rax, Address($poll$$Register, 0));
13376     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13377   %}
13378   ins_pipe(ialu_reg_mem);
13379 %}
13380 
13381 instruct mask_all_evexL(kReg dst, rRegL src) %{
13382   match(Set dst (MaskAll src));
13383   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13384   ins_encode %{
13385     int mask_len = Matcher::vector_length(this);
13386     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13387   %}
13388   ins_pipe( pipe_slow );
13389 %}
13390 
13391 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13392   predicate(Matcher::vector_length(n) > 32);
13393   match(Set dst (MaskAll src));
13394   effect(TEMP tmp);
13395   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13396   ins_encode %{
13397     int mask_len = Matcher::vector_length(this);
13398     __ movslq($tmp$$Register, $src$$Register);
13399     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13400   %}
13401   ins_pipe( pipe_slow );
13402 %}
13403 
13404 // ============================================================================
13405 // Procedure Call/Return Instructions
13406 // Call Java Static Instruction
13407 // Note: If this code changes, the corresponding ret_addr_offset() and
13408 //       compute_padding() functions will have to be adjusted.
13409 instruct CallStaticJavaDirect(method meth) %{
13410   match(CallStaticJava);
13411   effect(USE meth);
13412 
13413   ins_cost(300);
13414   format %{ "call,static " %}
13415   opcode(0xE8); /* E8 cd */
13416   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13417   ins_pipe(pipe_slow);
13418   ins_alignment(4);
13419 %}
13420 
13421 // Call Java Dynamic Instruction
13422 // Note: If this code changes, the corresponding ret_addr_offset() and
13423 //       compute_padding() functions will have to be adjusted.
13424 instruct CallDynamicJavaDirect(method meth)
13425 %{
13426   match(CallDynamicJava);
13427   effect(USE meth);
13428 
13429   ins_cost(300);
13430   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13431             "call,dynamic " %}
13432   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13433   ins_pipe(pipe_slow);
13434   ins_alignment(4);
13435 %}
13436 
13437 // Call Runtime Instruction
13438 instruct CallRuntimeDirect(method meth)
13439 %{
13440   match(CallRuntime);
13441   effect(USE meth);
13442 
13443   ins_cost(300);
13444   format %{ "call,runtime " %}
13445   ins_encode(clear_avx, Java_To_Runtime(meth));
13446   ins_pipe(pipe_slow);
13447 %}
13448 
13449 // Call runtime without safepoint
13450 instruct CallLeafDirect(method meth)
13451 %{
13452   match(CallLeaf);
13453   effect(USE meth);
13454 
13455   ins_cost(300);
13456   format %{ "call_leaf,runtime " %}
13457   ins_encode(clear_avx, Java_To_Runtime(meth));
13458   ins_pipe(pipe_slow);
13459 %}
13460 
13461 // Call runtime without safepoint and with vector arguments
13462 instruct CallLeafDirectVector(method meth)
13463 %{
13464   match(CallLeafVector);
13465   effect(USE meth);
13466 
13467   ins_cost(300);
13468   format %{ "call_leaf,vector " %}
13469   ins_encode(Java_To_Runtime(meth));
13470   ins_pipe(pipe_slow);
13471 %}
13472 
13473 // Call runtime without safepoint
13474 instruct CallLeafNoFPDirect(method meth)
13475 %{
13476   match(CallLeafNoFP);
13477   effect(USE meth);
13478 
13479   ins_cost(300);
13480   format %{ "call_leaf_nofp,runtime " %}
13481   ins_encode(clear_avx, Java_To_Runtime(meth));
13482   ins_pipe(pipe_slow);
13483 %}
13484 
13485 // Return Instruction
13486 // Remove the return address & jump to it.
13487 // Notice: We always emit a nop after a ret to make sure there is room
13488 // for safepoint patching
13489 instruct Ret()
13490 %{
13491   match(Return);
13492 
13493   format %{ "ret" %}
13494   ins_encode %{
13495     __ ret(0);
13496   %}
13497   ins_pipe(pipe_jmp);
13498 %}
13499 
13500 // Tail Call; Jump from runtime stub to Java code.
13501 // Also known as an 'interprocedural jump'.
13502 // Target of jump will eventually return to caller.
13503 // TailJump below removes the return address.
13504 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13505 %{
13506   match(TailCall jump_target method_ptr);
13507 
13508   ins_cost(300);
13509   format %{ "jmp     $jump_target\t# rbx holds method" %}
13510   ins_encode %{
13511     __ jmp($jump_target$$Register);
13512   %}
13513   ins_pipe(pipe_jmp);
13514 %}
13515 
13516 // Tail Jump; remove the return address; jump to target.
13517 // TailCall above leaves the return address around.
13518 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13519 %{
13520   match(TailJump jump_target ex_oop);
13521 
13522   ins_cost(300);
13523   format %{ "popq    rdx\t# pop return address\n\t"
13524             "jmp     $jump_target" %}
13525   ins_encode %{
13526     __ popq(as_Register(RDX_enc));
13527     __ jmp($jump_target$$Register);
13528   %}
13529   ins_pipe(pipe_jmp);
13530 %}
13531 
13532 // Create exception oop: created by stack-crawling runtime code.
13533 // Created exception is now available to this handler, and is setup
13534 // just prior to jumping to this handler.  No code emitted.
13535 instruct CreateException(rax_RegP ex_oop)
13536 %{
13537   match(Set ex_oop (CreateEx));
13538 
13539   size(0);
13540   // use the following format syntax
13541   format %{ "# exception oop is in rax; no code emitted" %}
13542   ins_encode();
13543   ins_pipe(empty);
13544 %}
13545 
13546 // Rethrow exception:
13547 // The exception oop will come in the first argument position.
13548 // Then JUMP (not call) to the rethrow stub code.
13549 instruct RethrowException()
13550 %{
13551   match(Rethrow);
13552 
13553   // use the following format syntax
13554   format %{ "jmp     rethrow_stub" %}
13555   ins_encode(enc_rethrow);
13556   ins_pipe(pipe_jmp);
13557 %}
13558 
13559 // ============================================================================
13560 // This name is KNOWN by the ADLC and cannot be changed.
13561 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13562 // for this guy.
13563 instruct tlsLoadP(r15_RegP dst) %{
13564   match(Set dst (ThreadLocal));
13565   effect(DEF dst);
13566 
13567   size(0);
13568   format %{ "# TLS is in R15" %}
13569   ins_encode( /*empty encoding*/ );
13570   ins_pipe(ialu_reg_reg);
13571 %}
13572 
13573 instruct rdtsc(rax_RegL dst, rdx_RegL tmp) %{
13574   match(Set dst (Timestamp));
13575   effect(DEF dst, KILL tmp);
13576 
13577   format %{ "RDTSC" %}
13578   ins_encode %{
13579     __ rdtsc();
13580     __ shlptr(rdx, 32);
13581     __ orptr(rax, rdx);
13582   %}
13583   ins_pipe(ialu_reg_reg);
13584 %}
13585 
13586 instruct rdtscp(rax_RegL dst, rdx_RegL tmp, rcx_RegL dead) %{
13587   match(Set dst (TimestampSerial));
13588   effect(DEF dst, KILL tmp, KILL dead);
13589 
13590   format %{ "RDTSCP" %}
13591   ins_encode %{
13592     __ rdtscp();
13593     __ shlptr(rdx, 32);
13594     __ orptr(rax, rdx);
13595   %}
13596   ins_pipe(ialu_reg_reg);
13597 %}
13598 
13599 
13600 //----------PEEPHOLE RULES-----------------------------------------------------
13601 // These must follow all instruction definitions as they use the names
13602 // defined in the instructions definitions.
13603 //
13604 // peeppredicate ( rule_predicate );
13605 // // the predicate unless which the peephole rule will be ignored
13606 //
13607 // peepmatch ( root_instr_name [preceding_instruction]* );
13608 //
13609 // peepprocedure ( procedure_name );
13610 // // provide a procedure name to perform the optimization, the procedure should
13611 // // reside in the architecture dependent peephole file, the method has the
13612 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
13613 // // with the arguments being the basic block, the current node index inside the
13614 // // block, the register allocator, the functions upon invoked return a new node
13615 // // defined in peepreplace, and the rules of the nodes appearing in the
13616 // // corresponding peepmatch, the function return true if successful, else
13617 // // return false
13618 //
13619 // peepconstraint %{
13620 // (instruction_number.operand_name relational_op instruction_number.operand_name
13621 //  [, ...] );
13622 // // instruction numbers are zero-based using left to right order in peepmatch
13623 //
13624 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13625 // // provide an instruction_number.operand_name for each operand that appears
13626 // // in the replacement instruction's match rule
13627 //
13628 // ---------VM FLAGS---------------------------------------------------------
13629 //
13630 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13631 //
13632 // Each peephole rule is given an identifying number starting with zero and
13633 // increasing by one in the order seen by the parser.  An individual peephole
13634 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13635 // on the command-line.
13636 //
13637 // ---------CURRENT LIMITATIONS----------------------------------------------
13638 //
13639 // Only transformations inside a basic block (do we need more for peephole)
13640 //
13641 // ---------EXAMPLE----------------------------------------------------------
13642 //
13643 // // pertinent parts of existing instructions in architecture description
13644 // instruct movI(rRegI dst, rRegI src)
13645 // %{
13646 //   match(Set dst (CopyI src));
13647 // %}
13648 //
13649 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13650 // %{
13651 //   match(Set dst (AddI dst src));
13652 //   effect(KILL cr);
13653 // %}
13654 //
13655 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
13656 // %{
13657 //   match(Set dst (AddI dst src));
13658 // %}
13659 //
13660 // 1. Simple replacement
13661 // - Only match adjacent instructions in same basic block
13662 // - Only equality constraints
13663 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
13664 // - Only one replacement instruction
13665 //
13666 // // Change (inc mov) to lea
13667 // peephole %{
13668 //   // lea should only be emitted when beneficial
13669 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
13670 //   // increment preceded by register-register move
13671 //   peepmatch ( incI_rReg movI );
13672 //   // require that the destination register of the increment
13673 //   // match the destination register of the move
13674 //   peepconstraint ( 0.dst == 1.dst );
13675 //   // construct a replacement instruction that sets
13676 //   // the destination to ( move's source register + one )
13677 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13678 // %}
13679 //
13680 // 2. Procedural replacement
13681 // - More flexible finding relevent nodes
13682 // - More flexible constraints
13683 // - More flexible transformations
13684 // - May utilise architecture-dependent API more effectively
13685 // - Currently only one replacement instruction due to adlc parsing capabilities
13686 //
13687 // // Change (inc mov) to lea
13688 // peephole %{
13689 //   // lea should only be emitted when beneficial
13690 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
13691 //   // the rule numbers of these nodes inside are passed into the function below
13692 //   peepmatch ( incI_rReg movI );
13693 //   // the method that takes the responsibility of transformation
13694 //   peepprocedure ( inc_mov_to_lea );
13695 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
13696 //   // node is passed into the function above
13697 //   peepreplace ( leaI_rReg_immI() );
13698 // %}
13699 
13700 // These instructions is not matched by the matcher but used by the peephole
13701 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
13702 %{
13703   predicate(false);
13704   match(Set dst (AddI src1 src2));
13705   format %{ "leal    $dst, [$src1 + $src2]" %}
13706   ins_encode %{
13707     Register dst = $dst$$Register;
13708     Register src1 = $src1$$Register;
13709     Register src2 = $src2$$Register;
13710     if (src1 != rbp && src1 != r13) {
13711       __ leal(dst, Address(src1, src2, Address::times_1));
13712     } else {
13713       assert(src2 != rbp && src2 != r13, "");
13714       __ leal(dst, Address(src2, src1, Address::times_1));
13715     }
13716   %}
13717   ins_pipe(ialu_reg_reg);
13718 %}
13719 
13720 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
13721 %{
13722   predicate(false);
13723   match(Set dst (AddI src1 src2));
13724   format %{ "leal    $dst, [$src1 + $src2]" %}
13725   ins_encode %{
13726     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
13727   %}
13728   ins_pipe(ialu_reg_reg);
13729 %}
13730 
13731 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
13732 %{
13733   predicate(false);
13734   match(Set dst (LShiftI src shift));
13735   format %{ "leal    $dst, [$src << $shift]" %}
13736   ins_encode %{
13737     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
13738     Register src = $src$$Register;
13739     if (scale == Address::times_2 && src != rbp && src != r13) {
13740       __ leal($dst$$Register, Address(src, src, Address::times_1));
13741     } else {
13742       __ leal($dst$$Register, Address(noreg, src, scale));
13743     }
13744   %}
13745   ins_pipe(ialu_reg_reg);
13746 %}
13747 
13748 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
13749 %{
13750   predicate(false);
13751   match(Set dst (AddL src1 src2));
13752   format %{ "leaq    $dst, [$src1 + $src2]" %}
13753   ins_encode %{
13754     Register dst = $dst$$Register;
13755     Register src1 = $src1$$Register;
13756     Register src2 = $src2$$Register;
13757     if (src1 != rbp && src1 != r13) {
13758       __ leaq(dst, Address(src1, src2, Address::times_1));
13759     } else {
13760       assert(src2 != rbp && src2 != r13, "");
13761       __ leaq(dst, Address(src2, src1, Address::times_1));
13762     }
13763   %}
13764   ins_pipe(ialu_reg_reg);
13765 %}
13766 
13767 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
13768 %{
13769   predicate(false);
13770   match(Set dst (AddL src1 src2));
13771   format %{ "leaq    $dst, [$src1 + $src2]" %}
13772   ins_encode %{
13773     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
13774   %}
13775   ins_pipe(ialu_reg_reg);
13776 %}
13777 
13778 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
13779 %{
13780   predicate(false);
13781   match(Set dst (LShiftL src shift));
13782   format %{ "leaq    $dst, [$src << $shift]" %}
13783   ins_encode %{
13784     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
13785     Register src = $src$$Register;
13786     if (scale == Address::times_2 && src != rbp && src != r13) {
13787       __ leaq($dst$$Register, Address(src, src, Address::times_1));
13788     } else {
13789       __ leaq($dst$$Register, Address(noreg, src, scale));
13790     }
13791   %}
13792   ins_pipe(ialu_reg_reg);
13793 %}
13794 
13795 peephole
13796 %{
13797   peeppredicate(VM_Version::supports_fast_2op_lea());
13798   peepmatch (addI_rReg);
13799   peepprocedure (lea_coalesce_reg);
13800   peepreplace (leaI_rReg_rReg_peep());
13801 %}
13802 
13803 peephole
13804 %{
13805   peeppredicate(VM_Version::supports_fast_2op_lea());
13806   peepmatch (addI_rReg_imm);
13807   peepprocedure (lea_coalesce_imm);
13808   peepreplace (leaI_rReg_immI_peep());
13809 %}
13810 
13811 peephole
13812 %{
13813   peeppredicate(VM_Version::supports_fast_2op_lea());
13814   peepmatch (incI_rReg);
13815   peepprocedure (lea_coalesce_imm);
13816   peepreplace (leaI_rReg_immI_peep());
13817 %}
13818 
13819 peephole
13820 %{
13821   peeppredicate(VM_Version::supports_fast_2op_lea());
13822   peepmatch (decI_rReg);
13823   peepprocedure (lea_coalesce_imm);
13824   peepreplace (leaI_rReg_immI_peep());
13825 %}
13826 
13827 peephole
13828 %{
13829   peeppredicate(VM_Version::supports_fast_2op_lea());
13830   peepmatch (salI_rReg_immI2);
13831   peepprocedure (lea_coalesce_imm);
13832   peepreplace (leaI_rReg_immI2_peep());
13833 %}
13834 
13835 peephole
13836 %{
13837   peeppredicate(VM_Version::supports_fast_2op_lea());
13838   peepmatch (addL_rReg);
13839   peepprocedure (lea_coalesce_reg);
13840   peepreplace (leaL_rReg_rReg_peep());
13841 %}
13842 
13843 peephole
13844 %{
13845   peeppredicate(VM_Version::supports_fast_2op_lea());
13846   peepmatch (addL_rReg_imm);
13847   peepprocedure (lea_coalesce_imm);
13848   peepreplace (leaL_rReg_immL32_peep());
13849 %}
13850 
13851 peephole
13852 %{
13853   peeppredicate(VM_Version::supports_fast_2op_lea());
13854   peepmatch (incL_rReg);
13855   peepprocedure (lea_coalesce_imm);
13856   peepreplace (leaL_rReg_immL32_peep());
13857 %}
13858 
13859 peephole
13860 %{
13861   peeppredicate(VM_Version::supports_fast_2op_lea());
13862   peepmatch (decL_rReg);
13863   peepprocedure (lea_coalesce_imm);
13864   peepreplace (leaL_rReg_immL32_peep());
13865 %}
13866 
13867 peephole
13868 %{
13869   peeppredicate(VM_Version::supports_fast_2op_lea());
13870   peepmatch (salL_rReg_immI2);
13871   peepprocedure (lea_coalesce_imm);
13872   peepreplace (leaL_rReg_immI2_peep());
13873 %}
13874 
13875 //----------SMARTSPILL RULES---------------------------------------------------
13876 // These must follow all instruction definitions as they use the names
13877 // defined in the instructions definitions.