1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RDX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RDI int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 
  323 source_hpp %{
  324 
  325 #include "peephole_x86_64.hpp"
  326 
  327 %}
  328 
  329 // Register masks
  330 source_hpp %{
  331 
  332 extern RegMask _ANY_REG_mask;
  333 extern RegMask _PTR_REG_mask;
  334 extern RegMask _PTR_REG_NO_RBP_mask;
  335 extern RegMask _PTR_NO_RAX_REG_mask;
  336 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  337 extern RegMask _LONG_REG_mask;
  338 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  339 extern RegMask _LONG_NO_RCX_REG_mask;
  340 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  341 extern RegMask _INT_REG_mask;
  342 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  343 extern RegMask _INT_NO_RCX_REG_mask;
  344 extern RegMask _INT_NO_RBP_R13_REG_mask;
  345 extern RegMask _FLOAT_REG_mask;
  346 
  347 extern RegMask _STACK_OR_PTR_REG_mask;
  348 extern RegMask _STACK_OR_LONG_REG_mask;
  349 extern RegMask _STACK_OR_INT_REG_mask;
  350 
  351 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  352 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  353 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_INT_REG_mask;
  380 
  381 static bool need_r12_heapbase() {
  382   return UseCompressedOops;
  383 }
  384 
  385 void reg_mask_init() {
  386   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  387   // We derive a number of subsets from it.
  388   _ANY_REG_mask = _ALL_REG_mask;
  389 
  390   if (PreserveFramePointer) {
  391     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  392     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  393   }
  394   if (need_r12_heapbase()) {
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  396     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  397   }
  398 
  399   _PTR_REG_mask = _ANY_REG_mask;
  400   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  401   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  404 
  405   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  406   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  407 
  408   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  409   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  410   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  411 
  412   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  413   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  414   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  415 
  416   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  417   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  418   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  419 
  420   _LONG_REG_mask = _PTR_REG_mask;
  421   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  422   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  423 
  424   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  425   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  426   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  429 
  430   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  431   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  432   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  433 
  434   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  435   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  436   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  439 
  440   _INT_REG_mask = _ALL_INT_REG_mask;
  441   if (PreserveFramePointer) {
  442     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  443   }
  444   if (need_r12_heapbase()) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  446   }
  447 
  448   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  449   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  450 
  451   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  452   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  453   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  454 
  455   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  456   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  457 
  458   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  459   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  460   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  461 
  462   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  463   // from the float_reg_legacy/float_reg_evex register class.
  464   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  465 }
  466 
  467 static bool generate_vzeroupper(Compile* C) {
  468   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  469 }
  470 
  471 static int clear_avx_size() {
  472   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  473 }
  474 
  475 // !!!!! Special hack to get all types of calls to specify the byte offset
  476 //       from the start of the call to the point where the return address
  477 //       will point.
  478 int MachCallStaticJavaNode::ret_addr_offset()
  479 {
  480   int offset = 5; // 5 bytes from start of call to where return address points
  481   offset += clear_avx_size();
  482   return offset;
  483 }
  484 
  485 int MachCallDynamicJavaNode::ret_addr_offset()
  486 {
  487   int offset = 15; // 15 bytes from start of call to where return address points
  488   offset += clear_avx_size();
  489   return offset;
  490 }
  491 
  492 int MachCallRuntimeNode::ret_addr_offset() {
  493   int offset = 13; // movq r10,#addr; callq (r10)
  494   if (this->ideal_Opcode() != Op_CallLeafVector) {
  495     offset += clear_avx_size();
  496   }
  497   return offset;
  498 }
  499 //
  500 // Compute padding required for nodes which need alignment
  501 //
  502 
  503 // The address of the call instruction needs to be 4-byte aligned to
  504 // ensure that it does not span a cache line so that it can be patched.
  505 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  506 {
  507   current_offset += clear_avx_size(); // skip vzeroupper
  508   current_offset += 1; // skip call opcode byte
  509   return align_up(current_offset, alignment_required()) - current_offset;
  510 }
  511 
  512 // The address of the call instruction needs to be 4-byte aligned to
  513 // ensure that it does not span a cache line so that it can be patched.
  514 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  515 {
  516   current_offset += clear_avx_size(); // skip vzeroupper
  517   current_offset += 11; // skip movq instruction + call opcode byte
  518   return align_up(current_offset, alignment_required()) - current_offset;
  519 }
  520 
  521 // This could be in MacroAssembler but it's fairly C2 specific
  522 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  523   Label exit;
  524   __ jccb(Assembler::noParity, exit);
  525   __ pushf();
  526   //
  527   // comiss/ucomiss instructions set ZF,PF,CF flags and
  528   // zero OF,AF,SF for NaN values.
  529   // Fixup flags by zeroing ZF,PF so that compare of NaN
  530   // values returns 'less than' result (CF is set).
  531   // Leave the rest of flags unchanged.
  532   //
  533   //    7 6 5 4 3 2 1 0
  534   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  535   //    0 0 1 0 1 0 1 1   (0x2B)
  536   //
  537   __ andq(Address(rsp, 0), 0xffffff2b);
  538   __ popf();
  539   __ bind(exit);
  540 }
  541 
  542 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  543   Label done;
  544   __ movl(dst, -1);
  545   __ jcc(Assembler::parity, done);
  546   __ jcc(Assembler::below, done);
  547   __ setb(Assembler::notEqual, dst);
  548   __ movzbl(dst, dst);
  549   __ bind(done);
  550 }
  551 
  552 // Math.min()    # Math.max()
  553 // --------------------------
  554 // ucomis[s/d]   #
  555 // ja   -> b     # a
  556 // jp   -> NaN   # NaN
  557 // jb   -> a     # b
  558 // je            #
  559 // |-jz -> a | b # a & b
  560 // |    -> a     #
  561 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  562                      XMMRegister a, XMMRegister b,
  563                      XMMRegister xmmt, Register rt,
  564                      bool min, bool single) {
  565 
  566   Label nan, zero, below, above, done;
  567 
  568   if (single)
  569     __ ucomiss(a, b);
  570   else
  571     __ ucomisd(a, b);
  572 
  573   if (dst->encoding() != (min ? b : a)->encoding())
  574     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  575   else
  576     __ jccb(Assembler::above, done);
  577 
  578   __ jccb(Assembler::parity, nan);  // PF=1
  579   __ jccb(Assembler::below, below); // CF=1
  580 
  581   // equal
  582   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  583   if (single) {
  584     __ ucomiss(a, xmmt);
  585     __ jccb(Assembler::equal, zero);
  586 
  587     __ movflt(dst, a);
  588     __ jmp(done);
  589   }
  590   else {
  591     __ ucomisd(a, xmmt);
  592     __ jccb(Assembler::equal, zero);
  593 
  594     __ movdbl(dst, a);
  595     __ jmp(done);
  596   }
  597 
  598   __ bind(zero);
  599   if (min)
  600     __ vpor(dst, a, b, Assembler::AVX_128bit);
  601   else
  602     __ vpand(dst, a, b, Assembler::AVX_128bit);
  603 
  604   __ jmp(done);
  605 
  606   __ bind(above);
  607   if (single)
  608     __ movflt(dst, min ? b : a);
  609   else
  610     __ movdbl(dst, min ? b : a);
  611 
  612   __ jmp(done);
  613 
  614   __ bind(nan);
  615   if (single) {
  616     __ movl(rt, 0x7fc00000); // Float.NaN
  617     __ movdl(dst, rt);
  618   }
  619   else {
  620     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  621     __ movdq(dst, rt);
  622   }
  623   __ jmp(done);
  624 
  625   __ bind(below);
  626   if (single)
  627     __ movflt(dst, min ? a : b);
  628   else
  629     __ movdbl(dst, min ? a : b);
  630 
  631   __ bind(done);
  632 }
  633 
  634 //=============================================================================
  635 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  636 
  637 int ConstantTable::calculate_table_base_offset() const {
  638   return 0;  // absolute addressing, no offset
  639 }
  640 
  641 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  642 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  643   ShouldNotReachHere();
  644 }
  645 
  646 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  647   // Empty encoding
  648 }
  649 
  650 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  651   return 0;
  652 }
  653 
  654 #ifndef PRODUCT
  655 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  656   st->print("# MachConstantBaseNode (empty encoding)");
  657 }
  658 #endif
  659 
  660 
  661 //=============================================================================
  662 #ifndef PRODUCT
  663 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  664   Compile* C = ra_->C;
  665 
  666   int framesize = C->output()->frame_size_in_bytes();
  667   int bangsize = C->output()->bang_size_in_bytes();
  668   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  669   // Remove wordSize for return addr which is already pushed.
  670   framesize -= wordSize;
  671 
  672   if (C->output()->need_stack_bang(bangsize)) {
  673     framesize -= wordSize;
  674     st->print("# stack bang (%d bytes)", bangsize);
  675     st->print("\n\t");
  676     st->print("pushq   rbp\t# Save rbp");
  677     if (PreserveFramePointer) {
  678         st->print("\n\t");
  679         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  680     }
  681     if (framesize) {
  682       st->print("\n\t");
  683       st->print("subq    rsp, #%d\t# Create frame",framesize);
  684     }
  685   } else {
  686     st->print("subq    rsp, #%d\t# Create frame",framesize);
  687     st->print("\n\t");
  688     framesize -= wordSize;
  689     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  690     if (PreserveFramePointer) {
  691       st->print("\n\t");
  692       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  693       if (framesize > 0) {
  694         st->print("\n\t");
  695         st->print("addq    rbp, #%d", framesize);
  696       }
  697     }
  698   }
  699 
  700   if (VerifyStackAtCalls) {
  701     st->print("\n\t");
  702     framesize -= wordSize;
  703     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  704 #ifdef ASSERT
  705     st->print("\n\t");
  706     st->print("# stack alignment check");
  707 #endif
  708   }
  709   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  710     st->print("\n\t");
  711     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  712     st->print("\n\t");
  713     st->print("je      fast_entry\t");
  714     st->print("\n\t");
  715     st->print("call    #nmethod_entry_barrier_stub\t");
  716     st->print("\n\tfast_entry:");
  717   }
  718   st->cr();
  719 }
  720 #endif
  721 
  722 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  723   Compile* C = ra_->C;
  724   C2_MacroAssembler _masm(&cbuf);
  725 
  726   int framesize = C->output()->frame_size_in_bytes();
  727   int bangsize = C->output()->bang_size_in_bytes();
  728 
  729   if (C->clinit_barrier_on_entry()) {
  730     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  731     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  732 
  733     Label L_skip_barrier;
  734     Register klass = rscratch1;
  735 
  736     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  737     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  738 
  739     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  740 
  741     __ bind(L_skip_barrier);
  742   }
  743 
  744   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  745 
  746   C->output()->set_frame_complete(cbuf.insts_size());
  747 
  748   if (C->has_mach_constant_base_node()) {
  749     // NOTE: We set the table base offset here because users might be
  750     // emitted before MachConstantBaseNode.
  751     ConstantTable& constant_table = C->output()->constant_table();
  752     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  753   }
  754 }
  755 
  756 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  757 {
  758   return MachNode::size(ra_); // too many variables; just compute it
  759                               // the hard way
  760 }
  761 
  762 int MachPrologNode::reloc() const
  763 {
  764   return 0; // a large enough number
  765 }
  766 
  767 //=============================================================================
  768 #ifndef PRODUCT
  769 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  770 {
  771   Compile* C = ra_->C;
  772   if (generate_vzeroupper(C)) {
  773     st->print("vzeroupper");
  774     st->cr(); st->print("\t");
  775   }
  776 
  777   int framesize = C->output()->frame_size_in_bytes();
  778   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  779   // Remove word for return adr already pushed
  780   // and RBP
  781   framesize -= 2*wordSize;
  782 
  783   if (framesize) {
  784     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  785     st->print("\t");
  786   }
  787 
  788   st->print_cr("popq    rbp");
  789   if (do_polling() && C->is_method_compilation()) {
  790     st->print("\t");
  791     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  792                  "ja      #safepoint_stub\t"
  793                  "# Safepoint: poll for GC");
  794   }
  795 }
  796 #endif
  797 
  798 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  799 {
  800   Compile* C = ra_->C;
  801   MacroAssembler _masm(&cbuf);
  802 
  803   if (generate_vzeroupper(C)) {
  804     // Clear upper bits of YMM registers when current compiled code uses
  805     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  806     __ vzeroupper();
  807   }
  808 
  809   int framesize = C->output()->frame_size_in_bytes();
  810   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  811   // Remove word for return adr already pushed
  812   // and RBP
  813   framesize -= 2*wordSize;
  814 
  815   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  816 
  817   if (framesize) {
  818     __ addq(rsp, framesize);
  819   }
  820 
  821   __ popq(rbp);
  822 
  823   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  824     __ reserved_stack_check();
  825   }
  826 
  827   if (do_polling() && C->is_method_compilation()) {
  828     MacroAssembler _masm(&cbuf);
  829     Label dummy_label;
  830     Label* code_stub = &dummy_label;
  831     if (!C->output()->in_scratch_emit_size()) {
  832       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  833       C->output()->add_stub(stub);
  834       code_stub = &stub->entry();
  835     }
  836     __ relocate(relocInfo::poll_return_type);
  837     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  838   }
  839 }
  840 
  841 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  842 {
  843   return MachNode::size(ra_); // too many variables; just compute it
  844                               // the hard way
  845 }
  846 
  847 int MachEpilogNode::reloc() const
  848 {
  849   return 2; // a large enough number
  850 }
  851 
  852 const Pipeline* MachEpilogNode::pipeline() const
  853 {
  854   return MachNode::pipeline_class();
  855 }
  856 
  857 //=============================================================================
  858 
  859 enum RC {
  860   rc_bad,
  861   rc_int,
  862   rc_kreg,
  863   rc_float,
  864   rc_stack
  865 };
  866 
  867 static enum RC rc_class(OptoReg::Name reg)
  868 {
  869   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  870 
  871   if (OptoReg::is_stack(reg)) return rc_stack;
  872 
  873   VMReg r = OptoReg::as_VMReg(reg);
  874 
  875   if (r->is_Register()) return rc_int;
  876 
  877   if (r->is_KRegister()) return rc_kreg;
  878 
  879   assert(r->is_XMMRegister(), "must be");
  880   return rc_float;
  881 }
  882 
  883 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  884 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
  885                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  886 
  887 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
  888                      int stack_offset, int reg, uint ireg, outputStream* st);
  889 
  890 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
  891                                       int dst_offset, uint ireg, outputStream* st) {
  892   if (cbuf) {
  893     MacroAssembler _masm(cbuf);
  894     switch (ireg) {
  895     case Op_VecS:
  896       __ movq(Address(rsp, -8), rax);
  897       __ movl(rax, Address(rsp, src_offset));
  898       __ movl(Address(rsp, dst_offset), rax);
  899       __ movq(rax, Address(rsp, -8));
  900       break;
  901     case Op_VecD:
  902       __ pushq(Address(rsp, src_offset));
  903       __ popq (Address(rsp, dst_offset));
  904       break;
  905     case Op_VecX:
  906       __ pushq(Address(rsp, src_offset));
  907       __ popq (Address(rsp, dst_offset));
  908       __ pushq(Address(rsp, src_offset+8));
  909       __ popq (Address(rsp, dst_offset+8));
  910       break;
  911     case Op_VecY:
  912       __ vmovdqu(Address(rsp, -32), xmm0);
  913       __ vmovdqu(xmm0, Address(rsp, src_offset));
  914       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  915       __ vmovdqu(xmm0, Address(rsp, -32));
  916       break;
  917     case Op_VecZ:
  918       __ evmovdquq(Address(rsp, -64), xmm0, 2);
  919       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
  920       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
  921       __ evmovdquq(xmm0, Address(rsp, -64), 2);
  922       break;
  923     default:
  924       ShouldNotReachHere();
  925     }
  926 #ifndef PRODUCT
  927   } else {
  928     switch (ireg) {
  929     case Op_VecS:
  930       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
  931                 "movl    rax, [rsp + #%d]\n\t"
  932                 "movl    [rsp + #%d], rax\n\t"
  933                 "movq    rax, [rsp - #8]",
  934                 src_offset, dst_offset);
  935       break;
  936     case Op_VecD:
  937       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
  938                 "popq    [rsp + #%d]",
  939                 src_offset, dst_offset);
  940       break;
  941      case Op_VecX:
  942       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
  943                 "popq    [rsp + #%d]\n\t"
  944                 "pushq   [rsp + #%d]\n\t"
  945                 "popq    [rsp + #%d]",
  946                 src_offset, dst_offset, src_offset+8, dst_offset+8);
  947       break;
  948     case Op_VecY:
  949       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
  950                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  951                 "vmovdqu [rsp + #%d], xmm0\n\t"
  952                 "vmovdqu xmm0, [rsp - #32]",
  953                 src_offset, dst_offset);
  954       break;
  955     case Op_VecZ:
  956       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
  957                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  958                 "vmovdqu [rsp + #%d], xmm0\n\t"
  959                 "vmovdqu xmm0, [rsp - #64]",
  960                 src_offset, dst_offset);
  961       break;
  962     default:
  963       ShouldNotReachHere();
  964     }
  965 #endif
  966   }
  967 }
  968 
  969 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
  970                                        PhaseRegAlloc* ra_,
  971                                        bool do_size,
  972                                        outputStream* st) const {
  973   assert(cbuf != NULL || st  != NULL, "sanity");
  974   // Get registers to move
  975   OptoReg::Name src_second = ra_->get_reg_second(in(1));
  976   OptoReg::Name src_first = ra_->get_reg_first(in(1));
  977   OptoReg::Name dst_second = ra_->get_reg_second(this);
  978   OptoReg::Name dst_first = ra_->get_reg_first(this);
  979 
  980   enum RC src_second_rc = rc_class(src_second);
  981   enum RC src_first_rc = rc_class(src_first);
  982   enum RC dst_second_rc = rc_class(dst_second);
  983   enum RC dst_first_rc = rc_class(dst_first);
  984 
  985   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
  986          "must move at least 1 register" );
  987 
  988   if (src_first == dst_first && src_second == dst_second) {
  989     // Self copy, no move
  990     return 0;
  991   }
  992   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
  993     uint ireg = ideal_reg();
  994     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
  995     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
  996     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
  997       // mem -> mem
  998       int src_offset = ra_->reg2offset(src_first);
  999       int dst_offset = ra_->reg2offset(dst_first);
 1000       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1001     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1002       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1003     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1004       int stack_offset = ra_->reg2offset(dst_first);
 1005       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1006     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1007       int stack_offset = ra_->reg2offset(src_first);
 1008       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1009     } else {
 1010       ShouldNotReachHere();
 1011     }
 1012     return 0;
 1013   }
 1014   if (src_first_rc == rc_stack) {
 1015     // mem ->
 1016     if (dst_first_rc == rc_stack) {
 1017       // mem -> mem
 1018       assert(src_second != dst_first, "overlap");
 1019       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1020           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1021         // 64-bit
 1022         int src_offset = ra_->reg2offset(src_first);
 1023         int dst_offset = ra_->reg2offset(dst_first);
 1024         if (cbuf) {
 1025           MacroAssembler _masm(cbuf);
 1026           __ pushq(Address(rsp, src_offset));
 1027           __ popq (Address(rsp, dst_offset));
 1028 #ifndef PRODUCT
 1029         } else {
 1030           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1031                     "popq    [rsp + #%d]",
 1032                      src_offset, dst_offset);
 1033 #endif
 1034         }
 1035       } else {
 1036         // 32-bit
 1037         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1038         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1039         // No pushl/popl, so:
 1040         int src_offset = ra_->reg2offset(src_first);
 1041         int dst_offset = ra_->reg2offset(dst_first);
 1042         if (cbuf) {
 1043           MacroAssembler _masm(cbuf);
 1044           __ movq(Address(rsp, -8), rax);
 1045           __ movl(rax, Address(rsp, src_offset));
 1046           __ movl(Address(rsp, dst_offset), rax);
 1047           __ movq(rax, Address(rsp, -8));
 1048 #ifndef PRODUCT
 1049         } else {
 1050           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                     "movl    rax, [rsp + #%d]\n\t"
 1052                     "movl    [rsp + #%d], rax\n\t"
 1053                     "movq    rax, [rsp - #8]",
 1054                      src_offset, dst_offset);
 1055 #endif
 1056         }
 1057       }
 1058       return 0;
 1059     } else if (dst_first_rc == rc_int) {
 1060       // mem -> gpr
 1061       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1062           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1063         // 64-bit
 1064         int offset = ra_->reg2offset(src_first);
 1065         if (cbuf) {
 1066           MacroAssembler _masm(cbuf);
 1067           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1068 #ifndef PRODUCT
 1069         } else {
 1070           st->print("movq    %s, [rsp + #%d]\t# spill",
 1071                      Matcher::regName[dst_first],
 1072                      offset);
 1073 #endif
 1074         }
 1075       } else {
 1076         // 32-bit
 1077         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1078         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1079         int offset = ra_->reg2offset(src_first);
 1080         if (cbuf) {
 1081           MacroAssembler _masm(cbuf);
 1082           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1083 #ifndef PRODUCT
 1084         } else {
 1085           st->print("movl    %s, [rsp + #%d]\t# spill",
 1086                      Matcher::regName[dst_first],
 1087                      offset);
 1088 #endif
 1089         }
 1090       }
 1091       return 0;
 1092     } else if (dst_first_rc == rc_float) {
 1093       // mem-> xmm
 1094       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1095           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1096         // 64-bit
 1097         int offset = ra_->reg2offset(src_first);
 1098         if (cbuf) {
 1099           MacroAssembler _masm(cbuf);
 1100           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1101 #ifndef PRODUCT
 1102         } else {
 1103           st->print("%s  %s, [rsp + #%d]\t# spill",
 1104                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1105                      Matcher::regName[dst_first],
 1106                      offset);
 1107 #endif
 1108         }
 1109       } else {
 1110         // 32-bit
 1111         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1112         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1113         int offset = ra_->reg2offset(src_first);
 1114         if (cbuf) {
 1115           MacroAssembler _masm(cbuf);
 1116           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1117 #ifndef PRODUCT
 1118         } else {
 1119           st->print("movss   %s, [rsp + #%d]\t# spill",
 1120                      Matcher::regName[dst_first],
 1121                      offset);
 1122 #endif
 1123         }
 1124       }
 1125       return 0;
 1126     } else if (dst_first_rc == rc_kreg) {
 1127       // mem -> kreg
 1128       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1129           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1130         // 64-bit
 1131         int offset = ra_->reg2offset(src_first);
 1132         if (cbuf) {
 1133           MacroAssembler _masm(cbuf);
 1134           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1135 #ifndef PRODUCT
 1136         } else {
 1137           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1138                      Matcher::regName[dst_first],
 1139                      offset);
 1140 #endif
 1141         }
 1142       }
 1143       return 0;
 1144     }
 1145   } else if (src_first_rc == rc_int) {
 1146     // gpr ->
 1147     if (dst_first_rc == rc_stack) {
 1148       // gpr -> mem
 1149       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1150           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1151         // 64-bit
 1152         int offset = ra_->reg2offset(dst_first);
 1153         if (cbuf) {
 1154           MacroAssembler _masm(cbuf);
 1155           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1156 #ifndef PRODUCT
 1157         } else {
 1158           st->print("movq    [rsp + #%d], %s\t# spill",
 1159                      offset,
 1160                      Matcher::regName[src_first]);
 1161 #endif
 1162         }
 1163       } else {
 1164         // 32-bit
 1165         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1166         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1167         int offset = ra_->reg2offset(dst_first);
 1168         if (cbuf) {
 1169           MacroAssembler _masm(cbuf);
 1170           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1171 #ifndef PRODUCT
 1172         } else {
 1173           st->print("movl    [rsp + #%d], %s\t# spill",
 1174                      offset,
 1175                      Matcher::regName[src_first]);
 1176 #endif
 1177         }
 1178       }
 1179       return 0;
 1180     } else if (dst_first_rc == rc_int) {
 1181       // gpr -> gpr
 1182       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1183           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1184         // 64-bit
 1185         if (cbuf) {
 1186           MacroAssembler _masm(cbuf);
 1187           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1188                   as_Register(Matcher::_regEncode[src_first]));
 1189 #ifndef PRODUCT
 1190         } else {
 1191           st->print("movq    %s, %s\t# spill",
 1192                      Matcher::regName[dst_first],
 1193                      Matcher::regName[src_first]);
 1194 #endif
 1195         }
 1196         return 0;
 1197       } else {
 1198         // 32-bit
 1199         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1200         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1201         if (cbuf) {
 1202           MacroAssembler _masm(cbuf);
 1203           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1204                   as_Register(Matcher::_regEncode[src_first]));
 1205 #ifndef PRODUCT
 1206         } else {
 1207           st->print("movl    %s, %s\t# spill",
 1208                      Matcher::regName[dst_first],
 1209                      Matcher::regName[src_first]);
 1210 #endif
 1211         }
 1212         return 0;
 1213       }
 1214     } else if (dst_first_rc == rc_float) {
 1215       // gpr -> xmm
 1216       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1217           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1218         // 64-bit
 1219         if (cbuf) {
 1220           MacroAssembler _masm(cbuf);
 1221           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1222 #ifndef PRODUCT
 1223         } else {
 1224           st->print("movdq   %s, %s\t# spill",
 1225                      Matcher::regName[dst_first],
 1226                      Matcher::regName[src_first]);
 1227 #endif
 1228         }
 1229       } else {
 1230         // 32-bit
 1231         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1232         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1233         if (cbuf) {
 1234           MacroAssembler _masm(cbuf);
 1235           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1236 #ifndef PRODUCT
 1237         } else {
 1238           st->print("movdl   %s, %s\t# spill",
 1239                      Matcher::regName[dst_first],
 1240                      Matcher::regName[src_first]);
 1241 #endif
 1242         }
 1243       }
 1244       return 0;
 1245     } else if (dst_first_rc == rc_kreg) {
 1246       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1247           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1248         // 64-bit
 1249         if (cbuf) {
 1250           MacroAssembler _masm(cbuf);
 1251           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1252   #ifndef PRODUCT
 1253         } else {
 1254            st->print("kmovq   %s, %s\t# spill",
 1255                        Matcher::regName[dst_first],
 1256                        Matcher::regName[src_first]);
 1257   #endif
 1258         }
 1259       }
 1260       Unimplemented();
 1261       return 0;
 1262     }
 1263   } else if (src_first_rc == rc_float) {
 1264     // xmm ->
 1265     if (dst_first_rc == rc_stack) {
 1266       // xmm -> mem
 1267       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1268           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1269         // 64-bit
 1270         int offset = ra_->reg2offset(dst_first);
 1271         if (cbuf) {
 1272           MacroAssembler _masm(cbuf);
 1273           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1274 #ifndef PRODUCT
 1275         } else {
 1276           st->print("movsd   [rsp + #%d], %s\t# spill",
 1277                      offset,
 1278                      Matcher::regName[src_first]);
 1279 #endif
 1280         }
 1281       } else {
 1282         // 32-bit
 1283         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1284         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1285         int offset = ra_->reg2offset(dst_first);
 1286         if (cbuf) {
 1287           MacroAssembler _masm(cbuf);
 1288           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1289 #ifndef PRODUCT
 1290         } else {
 1291           st->print("movss   [rsp + #%d], %s\t# spill",
 1292                      offset,
 1293                      Matcher::regName[src_first]);
 1294 #endif
 1295         }
 1296       }
 1297       return 0;
 1298     } else if (dst_first_rc == rc_int) {
 1299       // xmm -> gpr
 1300       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1301           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1302         // 64-bit
 1303         if (cbuf) {
 1304           MacroAssembler _masm(cbuf);
 1305           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1306 #ifndef PRODUCT
 1307         } else {
 1308           st->print("movdq   %s, %s\t# spill",
 1309                      Matcher::regName[dst_first],
 1310                      Matcher::regName[src_first]);
 1311 #endif
 1312         }
 1313       } else {
 1314         // 32-bit
 1315         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1316         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1317         if (cbuf) {
 1318           MacroAssembler _masm(cbuf);
 1319           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1320 #ifndef PRODUCT
 1321         } else {
 1322           st->print("movdl   %s, %s\t# spill",
 1323                      Matcher::regName[dst_first],
 1324                      Matcher::regName[src_first]);
 1325 #endif
 1326         }
 1327       }
 1328       return 0;
 1329     } else if (dst_first_rc == rc_float) {
 1330       // xmm -> xmm
 1331       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1332           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1333         // 64-bit
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("%s  %s, %s\t# spill",
 1340                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1341                      Matcher::regName[dst_first],
 1342                      Matcher::regName[src_first]);
 1343 #endif
 1344         }
 1345       } else {
 1346         // 32-bit
 1347         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1348         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("%s  %s, %s\t# spill",
 1355                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1356                      Matcher::regName[dst_first],
 1357                      Matcher::regName[src_first]);
 1358 #endif
 1359         }
 1360       }
 1361       return 0;
 1362     } else if (dst_first_rc == rc_kreg) {
 1363       assert(false, "Illegal spilling");
 1364       return 0;
 1365     }
 1366   } else if (src_first_rc == rc_kreg) {
 1367     if (dst_first_rc == rc_stack) {
 1368       // mem -> kreg
 1369       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1370           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1371         // 64-bit
 1372         int offset = ra_->reg2offset(dst_first);
 1373         if (cbuf) {
 1374           MacroAssembler _masm(cbuf);
 1375           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1376 #ifndef PRODUCT
 1377         } else {
 1378           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1379                      offset,
 1380                      Matcher::regName[src_first]);
 1381 #endif
 1382         }
 1383       }
 1384       return 0;
 1385     } else if (dst_first_rc == rc_int) {
 1386       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1387           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1388         // 64-bit
 1389         if (cbuf) {
 1390           MacroAssembler _masm(cbuf);
 1391           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1392 #ifndef PRODUCT
 1393         } else {
 1394          st->print("kmovq   %s, %s\t# spill",
 1395                      Matcher::regName[dst_first],
 1396                      Matcher::regName[src_first]);
 1397 #endif
 1398         }
 1399       }
 1400       Unimplemented();
 1401       return 0;
 1402     } else if (dst_first_rc == rc_kreg) {
 1403       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1404           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1405         // 64-bit
 1406         if (cbuf) {
 1407           MacroAssembler _masm(cbuf);
 1408           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411          st->print("kmovq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       }
 1417       return 0;
 1418     } else if (dst_first_rc == rc_float) {
 1419       assert(false, "Illegal spill");
 1420       return 0;
 1421     }
 1422   }
 1423 
 1424   assert(0," foo ");
 1425   Unimplemented();
 1426   return 0;
 1427 }
 1428 
 1429 #ifndef PRODUCT
 1430 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1431   implementation(NULL, ra_, false, st);
 1432 }
 1433 #endif
 1434 
 1435 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1436   implementation(&cbuf, ra_, false, NULL);
 1437 }
 1438 
 1439 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1440   return MachNode::size(ra_);
 1441 }
 1442 
 1443 //=============================================================================
 1444 #ifndef PRODUCT
 1445 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1446 {
 1447   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1448   int reg = ra_->get_reg_first(this);
 1449   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1450             Matcher::regName[reg], offset);
 1451 }
 1452 #endif
 1453 
 1454 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1455 {
 1456   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1457   int reg = ra_->get_encode(this);
 1458 
 1459   MacroAssembler masm(&cbuf);
 1460   masm.lea(as_Register(reg), Address(rsp, offset));
 1461 }
 1462 
 1463 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1464 {
 1465   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1466   return (offset < 0x80) ? 5 : 8; // REX
 1467 }
 1468 
 1469 //=============================================================================
 1470 #ifndef PRODUCT
 1471 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1472 {
 1473   if (UseCompressedClassPointers) {
 1474     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1475     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1476     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1477   } else {
 1478     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1479                  "# Inline cache check");
 1480   }
 1481   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1482   st->print_cr("\tnop\t# nops to align entry point");
 1483 }
 1484 #endif
 1485 
 1486 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1487 {
 1488   MacroAssembler masm(&cbuf);
 1489   uint insts_size = cbuf.insts_size();
 1490   if (UseCompressedClassPointers) {
 1491     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1492     masm.cmpptr(rax, rscratch1);
 1493   } else {
 1494     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1495   }
 1496 
 1497   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1498 
 1499   /* WARNING these NOPs are critical so that verified entry point is properly
 1500      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1501   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1502   if (OptoBreakpoint) {
 1503     // Leave space for int3
 1504     nops_cnt -= 1;
 1505   }
 1506   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1507   if (nops_cnt > 0)
 1508     masm.nop(nops_cnt);
 1509 }
 1510 
 1511 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1512 {
 1513   return MachNode::size(ra_); // too many variables; just compute it
 1514                               // the hard way
 1515 }
 1516 
 1517 
 1518 //=============================================================================
 1519 
 1520 bool Matcher::supports_vector_calling_convention(void) {
 1521   if (EnableVectorSupport && UseVectorStubs) {
 1522     return true;
 1523   }
 1524   return false;
 1525 }
 1526 
 1527 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1528   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1529   int lo = XMM0_num;
 1530   int hi = XMM0b_num;
 1531   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1532   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1533   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1534   return OptoRegPair(hi, lo);
 1535 }
 1536 
 1537 // Is this branch offset short enough that a short branch can be used?
 1538 //
 1539 // NOTE: If the platform does not provide any short branch variants, then
 1540 //       this method should return false for offset 0.
 1541 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1542   // The passed offset is relative to address of the branch.
 1543   // On 86 a branch displacement is calculated relative to address
 1544   // of a next instruction.
 1545   offset -= br_size;
 1546 
 1547   // the short version of jmpConUCF2 contains multiple branches,
 1548   // making the reach slightly less
 1549   if (rule == jmpConUCF2_rule)
 1550     return (-126 <= offset && offset <= 125);
 1551   return (-128 <= offset && offset <= 127);
 1552 }
 1553 
 1554 // Return whether or not this register is ever used as an argument.
 1555 // This function is used on startup to build the trampoline stubs in
 1556 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1557 // call in the trampoline, and arguments in those registers not be
 1558 // available to the callee.
 1559 bool Matcher::can_be_java_arg(int reg)
 1560 {
 1561   return
 1562     reg ==  RDI_num || reg == RDI_H_num ||
 1563     reg ==  RSI_num || reg == RSI_H_num ||
 1564     reg ==  RDX_num || reg == RDX_H_num ||
 1565     reg ==  RCX_num || reg == RCX_H_num ||
 1566     reg ==   R8_num || reg ==  R8_H_num ||
 1567     reg ==   R9_num || reg ==  R9_H_num ||
 1568     reg ==  R12_num || reg == R12_H_num ||
 1569     reg == XMM0_num || reg == XMM0b_num ||
 1570     reg == XMM1_num || reg == XMM1b_num ||
 1571     reg == XMM2_num || reg == XMM2b_num ||
 1572     reg == XMM3_num || reg == XMM3b_num ||
 1573     reg == XMM4_num || reg == XMM4b_num ||
 1574     reg == XMM5_num || reg == XMM5b_num ||
 1575     reg == XMM6_num || reg == XMM6b_num ||
 1576     reg == XMM7_num || reg == XMM7b_num;
 1577 }
 1578 
 1579 bool Matcher::is_spillable_arg(int reg)
 1580 {
 1581   return can_be_java_arg(reg);
 1582 }
 1583 
 1584 uint Matcher::int_pressure_limit()
 1585 {
 1586   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1587 }
 1588 
 1589 uint Matcher::float_pressure_limit()
 1590 {
 1591   // After experiment around with different values, the following default threshold
 1592   // works best for LCM's register pressure scheduling on x64.
 1593   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1594   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1595   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1596 }
 1597 
 1598 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1599   // In 64 bit mode a code which use multiply when
 1600   // devisor is constant is faster than hardware
 1601   // DIV instruction (it uses MulHiL).
 1602   return false;
 1603 }
 1604 
 1605 // Register for DIVI projection of divmodI
 1606 RegMask Matcher::divI_proj_mask() {
 1607   return INT_RAX_REG_mask();
 1608 }
 1609 
 1610 // Register for MODI projection of divmodI
 1611 RegMask Matcher::modI_proj_mask() {
 1612   return INT_RDX_REG_mask();
 1613 }
 1614 
 1615 // Register for DIVL projection of divmodL
 1616 RegMask Matcher::divL_proj_mask() {
 1617   return LONG_RAX_REG_mask();
 1618 }
 1619 
 1620 // Register for MODL projection of divmodL
 1621 RegMask Matcher::modL_proj_mask() {
 1622   return LONG_RDX_REG_mask();
 1623 }
 1624 
 1625 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1626 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1627     return NO_REG_mask();
 1628 }
 1629 
 1630 %}
 1631 
 1632 //----------ENCODING BLOCK-----------------------------------------------------
 1633 // This block specifies the encoding classes used by the compiler to
 1634 // output byte streams.  Encoding classes are parameterized macros
 1635 // used by Machine Instruction Nodes in order to generate the bit
 1636 // encoding of the instruction.  Operands specify their base encoding
 1637 // interface with the interface keyword.  There are currently
 1638 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1639 // COND_INTER.  REG_INTER causes an operand to generate a function
 1640 // which returns its register number when queried.  CONST_INTER causes
 1641 // an operand to generate a function which returns the value of the
 1642 // constant when queried.  MEMORY_INTER causes an operand to generate
 1643 // four functions which return the Base Register, the Index Register,
 1644 // the Scale Value, and the Offset Value of the operand when queried.
 1645 // COND_INTER causes an operand to generate six functions which return
 1646 // the encoding code (ie - encoding bits for the instruction)
 1647 // associated with each basic boolean condition for a conditional
 1648 // instruction.
 1649 //
 1650 // Instructions specify two basic values for encoding.  Again, a
 1651 // function is available to check if the constant displacement is an
 1652 // oop. They use the ins_encode keyword to specify their encoding
 1653 // classes (which must be a sequence of enc_class names, and their
 1654 // parameters, specified in the encoding block), and they use the
 1655 // opcode keyword to specify, in order, their primary, secondary, and
 1656 // tertiary opcode.  Only the opcode sections which a particular
 1657 // instruction needs for encoding need to be specified.
 1658 encode %{
 1659   enc_class cdql_enc(no_rax_rdx_RegI div)
 1660   %{
 1661     // Full implementation of Java idiv and irem; checks for
 1662     // special case as described in JVM spec., p.243 & p.271.
 1663     //
 1664     //         normal case                           special case
 1665     //
 1666     // input : rax: dividend                         min_int
 1667     //         reg: divisor                          -1
 1668     //
 1669     // output: rax: quotient  (= rax idiv reg)       min_int
 1670     //         rdx: remainder (= rax irem reg)       0
 1671     //
 1672     //  Code sequnce:
 1673     //
 1674     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1675     //    5:   75 07/08                jne    e <normal>
 1676     //    7:   33 d2                   xor    %edx,%edx
 1677     //  [div >= 8 -> offset + 1]
 1678     //  [REX_B]
 1679     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1680     //    c:   74 03/04                je     11 <done>
 1681     // 000000000000000e <normal>:
 1682     //    e:   99                      cltd
 1683     //  [div >= 8 -> offset + 1]
 1684     //  [REX_B]
 1685     //    f:   f7 f9                   idiv   $div
 1686     // 0000000000000011 <done>:
 1687     MacroAssembler _masm(&cbuf);
 1688     Label normal;
 1689     Label done;
 1690 
 1691     // cmp    $0x80000000,%eax
 1692     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1693 
 1694     // jne    e <normal>
 1695     __ jccb(Assembler::notEqual, normal);
 1696 
 1697     // xor    %edx,%edx
 1698     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1699 
 1700     // cmp    $0xffffffffffffffff,%ecx
 1701     __ cmpl($div$$Register, -1);
 1702 
 1703     // je     11 <done>
 1704     __ jccb(Assembler::equal, done);
 1705 
 1706     // <normal>
 1707     // cltd
 1708     __ bind(normal);
 1709     __ cdql();
 1710 
 1711     // idivl
 1712     // <done>
 1713     __ idivl($div$$Register);
 1714     __ bind(done);
 1715   %}
 1716 
 1717   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1718   %{
 1719     // Full implementation of Java ldiv and lrem; checks for
 1720     // special case as described in JVM spec., p.243 & p.271.
 1721     //
 1722     //         normal case                           special case
 1723     //
 1724     // input : rax: dividend                         min_long
 1725     //         reg: divisor                          -1
 1726     //
 1727     // output: rax: quotient  (= rax idiv reg)       min_long
 1728     //         rdx: remainder (= rax irem reg)       0
 1729     //
 1730     //  Code sequnce:
 1731     //
 1732     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1733     //    7:   00 00 80
 1734     //    a:   48 39 d0                cmp    %rdx,%rax
 1735     //    d:   75 08                   jne    17 <normal>
 1736     //    f:   33 d2                   xor    %edx,%edx
 1737     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1738     //   15:   74 05                   je     1c <done>
 1739     // 0000000000000017 <normal>:
 1740     //   17:   48 99                   cqto
 1741     //   19:   48 f7 f9                idiv   $div
 1742     // 000000000000001c <done>:
 1743     MacroAssembler _masm(&cbuf);
 1744     Label normal;
 1745     Label done;
 1746 
 1747     // mov    $0x8000000000000000,%rdx
 1748     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1749 
 1750     // cmp    %rdx,%rax
 1751     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1752 
 1753     // jne    17 <normal>
 1754     __ jccb(Assembler::notEqual, normal);
 1755 
 1756     // xor    %edx,%edx
 1757     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1758 
 1759     // cmp    $0xffffffffffffffff,$div
 1760     __ cmpq($div$$Register, -1);
 1761 
 1762     // je     1e <done>
 1763     __ jccb(Assembler::equal, done);
 1764 
 1765     // <normal>
 1766     // cqto
 1767     __ bind(normal);
 1768     __ cdqq();
 1769 
 1770     // idivq (note: must be emitted by the user of this rule)
 1771     // <done>
 1772     __ idivq($div$$Register);
 1773     __ bind(done);
 1774   %}
 1775 
 1776   enc_class enc_PartialSubtypeCheck()
 1777   %{
 1778     Register Rrdi = as_Register(RDI_enc); // result register
 1779     Register Rrax = as_Register(RAX_enc); // super class
 1780     Register Rrcx = as_Register(RCX_enc); // killed
 1781     Register Rrsi = as_Register(RSI_enc); // sub class
 1782     Label miss;
 1783     const bool set_cond_codes = true;
 1784 
 1785     MacroAssembler _masm(&cbuf);
 1786     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 1787                                      NULL, &miss,
 1788                                      /*set_cond_codes:*/ true);
 1789     if ($primary) {
 1790       __ xorptr(Rrdi, Rrdi);
 1791     }
 1792     __ bind(miss);
 1793   %}
 1794 
 1795   enc_class clear_avx %{
 1796     debug_only(int off0 = cbuf.insts_size());
 1797     if (generate_vzeroupper(Compile::current())) {
 1798       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1799       // Clear upper bits of YMM registers when current compiled code uses
 1800       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1801       MacroAssembler _masm(&cbuf);
 1802       __ vzeroupper();
 1803     }
 1804     debug_only(int off1 = cbuf.insts_size());
 1805     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1806   %}
 1807 
 1808   enc_class Java_To_Runtime(method meth) %{
 1809     // No relocation needed
 1810     MacroAssembler _masm(&cbuf);
 1811     __ mov64(r10, (int64_t) $meth$$method);
 1812     __ call(r10);
 1813     __ post_call_nop();
 1814   %}
 1815 
 1816   enc_class Java_Static_Call(method meth)
 1817   %{
 1818     // JAVA STATIC CALL
 1819     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1820     // determine who we intended to call.
 1821     MacroAssembler _masm(&cbuf);
 1822 
 1823     if (!_method) {
 1824       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1825     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1826       // The NOP here is purely to ensure that eliding a call to
 1827       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1828       __ addr_nop_5();
 1829       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1830     } else {
 1831       int method_index = resolved_method_index(cbuf);
 1832       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1833                                                   : static_call_Relocation::spec(method_index);
 1834       address mark = __ pc();
 1835       int call_offset = __ offset();
 1836       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1837       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1838         // Calls of the same statically bound method can share
 1839         // a stub to the interpreter.
 1840         cbuf.shared_stub_to_interp_for(_method, call_offset);
 1841       } else {
 1842         // Emit stubs for static call.
 1843         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 1844         if (stub == NULL) {
 1845           ciEnv::current()->record_failure("CodeCache is full");
 1846           return;
 1847         }
 1848       }
 1849     }
 1850     __ post_call_nop();
 1851   %}
 1852 
 1853   enc_class Java_Dynamic_Call(method meth) %{
 1854     MacroAssembler _masm(&cbuf);
 1855     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 1856     __ post_call_nop();
 1857   %}
 1858 
 1859 %}
 1860 
 1861 
 1862 
 1863 //----------FRAME--------------------------------------------------------------
 1864 // Definition of frame structure and management information.
 1865 //
 1866 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1867 //                             |   (to get allocators register number
 1868 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1869 //  r   CALLER     |        |
 1870 //  o     |        +--------+      pad to even-align allocators stack-slot
 1871 //  w     V        |  pad0  |        numbers; owned by CALLER
 1872 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1873 //  h     ^        |   in   |  5
 1874 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1875 //  |     |        |        |  3
 1876 //  |     |        +--------+
 1877 //  V     |        | old out|      Empty on Intel, window on Sparc
 1878 //        |    old |preserve|      Must be even aligned.
 1879 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1880 //        |        |   in   |  3   area for Intel ret address
 1881 //     Owned by    |preserve|      Empty on Sparc.
 1882 //       SELF      +--------+
 1883 //        |        |  pad2  |  2   pad to align old SP
 1884 //        |        +--------+  1
 1885 //        |        | locks  |  0
 1886 //        |        +--------+----> OptoReg::stack0(), even aligned
 1887 //        |        |  pad1  | 11   pad to align new SP
 1888 //        |        +--------+
 1889 //        |        |        | 10
 1890 //        |        | spills |  9   spills
 1891 //        V        |        |  8   (pad0 slot for callee)
 1892 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1893 //        ^        |  out   |  7
 1894 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1895 //     Owned by    +--------+
 1896 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1897 //        |    new |preserve|      Must be even-aligned.
 1898 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1899 //        |        |        |
 1900 //
 1901 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1902 //         known from SELF's arguments and the Java calling convention.
 1903 //         Region 6-7 is determined per call site.
 1904 // Note 2: If the calling convention leaves holes in the incoming argument
 1905 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1906 //         are owned by the CALLEE.  Holes should not be necessary in the
 1907 //         incoming area, as the Java calling convention is completely under
 1908 //         the control of the AD file.  Doubles can be sorted and packed to
 1909 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1910 //         varargs C calling conventions.
 1911 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1912 //         even aligned with pad0 as needed.
 1913 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1914 //         region 6-11 is even aligned; it may be padded out more so that
 1915 //         the region from SP to FP meets the minimum stack alignment.
 1916 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1917 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1918 //         SP meets the minimum alignment.
 1919 
 1920 frame
 1921 %{
 1922   // These three registers define part of the calling convention
 1923   // between compiled code and the interpreter.
 1924   inline_cache_reg(RAX);                // Inline Cache Register
 1925 
 1926   // Optional: name the operand used by cisc-spilling to access
 1927   // [stack_pointer + offset]
 1928   cisc_spilling_operand_name(indOffset32);
 1929 
 1930   // Number of stack slots consumed by locking an object
 1931   sync_stack_slots(2);
 1932 
 1933   // Compiled code's Frame Pointer
 1934   frame_pointer(RSP);
 1935 
 1936   // Interpreter stores its frame pointer in a register which is
 1937   // stored to the stack by I2CAdaptors.
 1938   // I2CAdaptors convert from interpreted java to compiled java.
 1939   interpreter_frame_pointer(RBP);
 1940 
 1941   // Stack alignment requirement
 1942   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1943 
 1944   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1945   // for calls to C.  Supports the var-args backing area for register parms.
 1946   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1947 
 1948   // The after-PROLOG location of the return address.  Location of
 1949   // return address specifies a type (REG or STACK) and a number
 1950   // representing the register number (i.e. - use a register name) or
 1951   // stack slot.
 1952   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1953   // Otherwise, it is above the locks and verification slot and alignment word
 1954   return_addr(STACK - 2 +
 1955               align_up((Compile::current()->in_preserve_stack_slots() +
 1956                         Compile::current()->fixed_slots()),
 1957                        stack_alignment_in_slots()));
 1958 
 1959   // Location of compiled Java return values.  Same as C for now.
 1960   return_value
 1961   %{
 1962     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 1963            "only return normal values");
 1964 
 1965     static const int lo[Op_RegL + 1] = {
 1966       0,
 1967       0,
 1968       RAX_num,  // Op_RegN
 1969       RAX_num,  // Op_RegI
 1970       RAX_num,  // Op_RegP
 1971       XMM0_num, // Op_RegF
 1972       XMM0_num, // Op_RegD
 1973       RAX_num   // Op_RegL
 1974     };
 1975     static const int hi[Op_RegL + 1] = {
 1976       0,
 1977       0,
 1978       OptoReg::Bad, // Op_RegN
 1979       OptoReg::Bad, // Op_RegI
 1980       RAX_H_num,    // Op_RegP
 1981       OptoReg::Bad, // Op_RegF
 1982       XMM0b_num,    // Op_RegD
 1983       RAX_H_num     // Op_RegL
 1984     };
 1985     // Excluded flags and vector registers.
 1986     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 1987     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 1988   %}
 1989 %}
 1990 
 1991 //----------ATTRIBUTES---------------------------------------------------------
 1992 //----------Operand Attributes-------------------------------------------------
 1993 op_attrib op_cost(0);        // Required cost attribute
 1994 
 1995 //----------Instruction Attributes---------------------------------------------
 1996 ins_attrib ins_cost(100);       // Required cost attribute
 1997 ins_attrib ins_size(8);         // Required size attribute (in bits)
 1998 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 1999                                 // a non-matching short branch variant
 2000                                 // of some long branch?
 2001 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2002                                 // be a power of 2) specifies the
 2003                                 // alignment that some part of the
 2004                                 // instruction (not necessarily the
 2005                                 // start) requires.  If > 1, a
 2006                                 // compute_padding() function must be
 2007                                 // provided for the instruction
 2008 
 2009 //----------OPERANDS-----------------------------------------------------------
 2010 // Operand definitions must precede instruction definitions for correct parsing
 2011 // in the ADLC because operands constitute user defined types which are used in
 2012 // instruction definitions.
 2013 
 2014 //----------Simple Operands----------------------------------------------------
 2015 // Immediate Operands
 2016 // Integer Immediate
 2017 operand immI()
 2018 %{
 2019   match(ConI);
 2020 
 2021   op_cost(10);
 2022   format %{ %}
 2023   interface(CONST_INTER);
 2024 %}
 2025 
 2026 // Constant for test vs zero
 2027 operand immI_0()
 2028 %{
 2029   predicate(n->get_int() == 0);
 2030   match(ConI);
 2031 
 2032   op_cost(0);
 2033   format %{ %}
 2034   interface(CONST_INTER);
 2035 %}
 2036 
 2037 // Constant for increment
 2038 operand immI_1()
 2039 %{
 2040   predicate(n->get_int() == 1);
 2041   match(ConI);
 2042 
 2043   op_cost(0);
 2044   format %{ %}
 2045   interface(CONST_INTER);
 2046 %}
 2047 
 2048 // Constant for decrement
 2049 operand immI_M1()
 2050 %{
 2051   predicate(n->get_int() == -1);
 2052   match(ConI);
 2053 
 2054   op_cost(0);
 2055   format %{ %}
 2056   interface(CONST_INTER);
 2057 %}
 2058 
 2059 operand immI_2()
 2060 %{
 2061   predicate(n->get_int() == 2);
 2062   match(ConI);
 2063 
 2064   op_cost(0);
 2065   format %{ %}
 2066   interface(CONST_INTER);
 2067 %}
 2068 
 2069 operand immI_4()
 2070 %{
 2071   predicate(n->get_int() == 4);
 2072   match(ConI);
 2073 
 2074   op_cost(0);
 2075   format %{ %}
 2076   interface(CONST_INTER);
 2077 %}
 2078 
 2079 operand immI_8()
 2080 %{
 2081   predicate(n->get_int() == 8);
 2082   match(ConI);
 2083 
 2084   op_cost(0);
 2085   format %{ %}
 2086   interface(CONST_INTER);
 2087 %}
 2088 
 2089 // Valid scale values for addressing modes
 2090 operand immI2()
 2091 %{
 2092   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2093   match(ConI);
 2094 
 2095   format %{ %}
 2096   interface(CONST_INTER);
 2097 %}
 2098 
 2099 operand immU7()
 2100 %{
 2101   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2102   match(ConI);
 2103 
 2104   op_cost(5);
 2105   format %{ %}
 2106   interface(CONST_INTER);
 2107 %}
 2108 
 2109 operand immI8()
 2110 %{
 2111   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2112   match(ConI);
 2113 
 2114   op_cost(5);
 2115   format %{ %}
 2116   interface(CONST_INTER);
 2117 %}
 2118 
 2119 operand immU8()
 2120 %{
 2121   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2122   match(ConI);
 2123 
 2124   op_cost(5);
 2125   format %{ %}
 2126   interface(CONST_INTER);
 2127 %}
 2128 
 2129 operand immI16()
 2130 %{
 2131   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2132   match(ConI);
 2133 
 2134   op_cost(10);
 2135   format %{ %}
 2136   interface(CONST_INTER);
 2137 %}
 2138 
 2139 // Int Immediate non-negative
 2140 operand immU31()
 2141 %{
 2142   predicate(n->get_int() >= 0);
 2143   match(ConI);
 2144 
 2145   op_cost(0);
 2146   format %{ %}
 2147   interface(CONST_INTER);
 2148 %}
 2149 
 2150 // Constant for long shifts
 2151 operand immI_32()
 2152 %{
 2153   predicate( n->get_int() == 32 );
 2154   match(ConI);
 2155 
 2156   op_cost(0);
 2157   format %{ %}
 2158   interface(CONST_INTER);
 2159 %}
 2160 
 2161 // Constant for long shifts
 2162 operand immI_64()
 2163 %{
 2164   predicate( n->get_int() == 64 );
 2165   match(ConI);
 2166 
 2167   op_cost(0);
 2168   format %{ %}
 2169   interface(CONST_INTER);
 2170 %}
 2171 
 2172 // Pointer Immediate
 2173 operand immP()
 2174 %{
 2175   match(ConP);
 2176 
 2177   op_cost(10);
 2178   format %{ %}
 2179   interface(CONST_INTER);
 2180 %}
 2181 
 2182 // NULL Pointer Immediate
 2183 operand immP0()
 2184 %{
 2185   predicate(n->get_ptr() == 0);
 2186   match(ConP);
 2187 
 2188   op_cost(5);
 2189   format %{ %}
 2190   interface(CONST_INTER);
 2191 %}
 2192 
 2193 // Pointer Immediate
 2194 operand immN() %{
 2195   match(ConN);
 2196 
 2197   op_cost(10);
 2198   format %{ %}
 2199   interface(CONST_INTER);
 2200 %}
 2201 
 2202 operand immNKlass() %{
 2203   match(ConNKlass);
 2204 
 2205   op_cost(10);
 2206   format %{ %}
 2207   interface(CONST_INTER);
 2208 %}
 2209 
 2210 // NULL Pointer Immediate
 2211 operand immN0() %{
 2212   predicate(n->get_narrowcon() == 0);
 2213   match(ConN);
 2214 
 2215   op_cost(5);
 2216   format %{ %}
 2217   interface(CONST_INTER);
 2218 %}
 2219 
 2220 operand immP31()
 2221 %{
 2222   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2223             && (n->get_ptr() >> 31) == 0);
 2224   match(ConP);
 2225 
 2226   op_cost(5);
 2227   format %{ %}
 2228   interface(CONST_INTER);
 2229 %}
 2230 
 2231 
 2232 // Long Immediate
 2233 operand immL()
 2234 %{
 2235   match(ConL);
 2236 
 2237   op_cost(20);
 2238   format %{ %}
 2239   interface(CONST_INTER);
 2240 %}
 2241 
 2242 // Long Immediate 8-bit
 2243 operand immL8()
 2244 %{
 2245   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2246   match(ConL);
 2247 
 2248   op_cost(5);
 2249   format %{ %}
 2250   interface(CONST_INTER);
 2251 %}
 2252 
 2253 // Long Immediate 32-bit unsigned
 2254 operand immUL32()
 2255 %{
 2256   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2257   match(ConL);
 2258 
 2259   op_cost(10);
 2260   format %{ %}
 2261   interface(CONST_INTER);
 2262 %}
 2263 
 2264 // Long Immediate 32-bit signed
 2265 operand immL32()
 2266 %{
 2267   predicate(n->get_long() == (int) (n->get_long()));
 2268   match(ConL);
 2269 
 2270   op_cost(15);
 2271   format %{ %}
 2272   interface(CONST_INTER);
 2273 %}
 2274 
 2275 operand immL_Pow2()
 2276 %{
 2277   predicate(is_power_of_2((julong)n->get_long()));
 2278   match(ConL);
 2279 
 2280   op_cost(15);
 2281   format %{ %}
 2282   interface(CONST_INTER);
 2283 %}
 2284 
 2285 operand immL_NotPow2()
 2286 %{
 2287   predicate(is_power_of_2((julong)~n->get_long()));
 2288   match(ConL);
 2289 
 2290   op_cost(15);
 2291   format %{ %}
 2292   interface(CONST_INTER);
 2293 %}
 2294 
 2295 // Long Immediate zero
 2296 operand immL0()
 2297 %{
 2298   predicate(n->get_long() == 0L);
 2299   match(ConL);
 2300 
 2301   op_cost(10);
 2302   format %{ %}
 2303   interface(CONST_INTER);
 2304 %}
 2305 
 2306 // Constant for increment
 2307 operand immL1()
 2308 %{
 2309   predicate(n->get_long() == 1);
 2310   match(ConL);
 2311 
 2312   format %{ %}
 2313   interface(CONST_INTER);
 2314 %}
 2315 
 2316 // Constant for decrement
 2317 operand immL_M1()
 2318 %{
 2319   predicate(n->get_long() == -1);
 2320   match(ConL);
 2321 
 2322   format %{ %}
 2323   interface(CONST_INTER);
 2324 %}
 2325 
 2326 // Long Immediate: the value 10
 2327 operand immL10()
 2328 %{
 2329   predicate(n->get_long() == 10);
 2330   match(ConL);
 2331 
 2332   format %{ %}
 2333   interface(CONST_INTER);
 2334 %}
 2335 
 2336 // Long immediate from 0 to 127.
 2337 // Used for a shorter form of long mul by 10.
 2338 operand immL_127()
 2339 %{
 2340   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 2341   match(ConL);
 2342 
 2343   op_cost(10);
 2344   format %{ %}
 2345   interface(CONST_INTER);
 2346 %}
 2347 
 2348 // Long Immediate: low 32-bit mask
 2349 operand immL_32bits()
 2350 %{
 2351   predicate(n->get_long() == 0xFFFFFFFFL);
 2352   match(ConL);
 2353   op_cost(20);
 2354 
 2355   format %{ %}
 2356   interface(CONST_INTER);
 2357 %}
 2358 
 2359 // Int Immediate: 2^n-1, positive
 2360 operand immI_Pow2M1()
 2361 %{
 2362   predicate((n->get_int() > 0)
 2363             && is_power_of_2((juint)n->get_int() + 1));
 2364   match(ConI);
 2365 
 2366   op_cost(20);
 2367   format %{ %}
 2368   interface(CONST_INTER);
 2369 %}
 2370 
 2371 // Float Immediate zero
 2372 operand immF0()
 2373 %{
 2374   predicate(jint_cast(n->getf()) == 0);
 2375   match(ConF);
 2376 
 2377   op_cost(5);
 2378   format %{ %}
 2379   interface(CONST_INTER);
 2380 %}
 2381 
 2382 // Float Immediate
 2383 operand immF()
 2384 %{
 2385   match(ConF);
 2386 
 2387   op_cost(15);
 2388   format %{ %}
 2389   interface(CONST_INTER);
 2390 %}
 2391 
 2392 // Double Immediate zero
 2393 operand immD0()
 2394 %{
 2395   predicate(jlong_cast(n->getd()) == 0);
 2396   match(ConD);
 2397 
 2398   op_cost(5);
 2399   format %{ %}
 2400   interface(CONST_INTER);
 2401 %}
 2402 
 2403 // Double Immediate
 2404 operand immD()
 2405 %{
 2406   match(ConD);
 2407 
 2408   op_cost(15);
 2409   format %{ %}
 2410   interface(CONST_INTER);
 2411 %}
 2412 
 2413 // Immediates for special shifts (sign extend)
 2414 
 2415 // Constants for increment
 2416 operand immI_16()
 2417 %{
 2418   predicate(n->get_int() == 16);
 2419   match(ConI);
 2420 
 2421   format %{ %}
 2422   interface(CONST_INTER);
 2423 %}
 2424 
 2425 operand immI_24()
 2426 %{
 2427   predicate(n->get_int() == 24);
 2428   match(ConI);
 2429 
 2430   format %{ %}
 2431   interface(CONST_INTER);
 2432 %}
 2433 
 2434 // Constant for byte-wide masking
 2435 operand immI_255()
 2436 %{
 2437   predicate(n->get_int() == 255);
 2438   match(ConI);
 2439 
 2440   format %{ %}
 2441   interface(CONST_INTER);
 2442 %}
 2443 
 2444 // Constant for short-wide masking
 2445 operand immI_65535()
 2446 %{
 2447   predicate(n->get_int() == 65535);
 2448   match(ConI);
 2449 
 2450   format %{ %}
 2451   interface(CONST_INTER);
 2452 %}
 2453 
 2454 // Constant for byte-wide masking
 2455 operand immL_255()
 2456 %{
 2457   predicate(n->get_long() == 255);
 2458   match(ConL);
 2459 
 2460   format %{ %}
 2461   interface(CONST_INTER);
 2462 %}
 2463 
 2464 // Constant for short-wide masking
 2465 operand immL_65535()
 2466 %{
 2467   predicate(n->get_long() == 65535);
 2468   match(ConL);
 2469 
 2470   format %{ %}
 2471   interface(CONST_INTER);
 2472 %}
 2473 
 2474 operand kReg()
 2475 %{
 2476   constraint(ALLOC_IN_RC(vectmask_reg));
 2477   match(RegVectMask);
 2478   format %{%}
 2479   interface(REG_INTER);
 2480 %}
 2481 
 2482 operand kReg_K1()
 2483 %{
 2484   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 2485   match(RegVectMask);
 2486   format %{%}
 2487   interface(REG_INTER);
 2488 %}
 2489 
 2490 operand kReg_K2()
 2491 %{
 2492   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 2493   match(RegVectMask);
 2494   format %{%}
 2495   interface(REG_INTER);
 2496 %}
 2497 
 2498 // Special Registers
 2499 operand kReg_K3()
 2500 %{
 2501   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 2502   match(RegVectMask);
 2503   format %{%}
 2504   interface(REG_INTER);
 2505 %}
 2506 
 2507 operand kReg_K4()
 2508 %{
 2509   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 2510   match(RegVectMask);
 2511   format %{%}
 2512   interface(REG_INTER);
 2513 %}
 2514 
 2515 operand kReg_K5()
 2516 %{
 2517   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 2518   match(RegVectMask);
 2519   format %{%}
 2520   interface(REG_INTER);
 2521 %}
 2522 
 2523 operand kReg_K6()
 2524 %{
 2525   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 2526   match(RegVectMask);
 2527   format %{%}
 2528   interface(REG_INTER);
 2529 %}
 2530 
 2531 // Special Registers
 2532 operand kReg_K7()
 2533 %{
 2534   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 2535   match(RegVectMask);
 2536   format %{%}
 2537   interface(REG_INTER);
 2538 %}
 2539 
 2540 // Register Operands
 2541 // Integer Register
 2542 operand rRegI()
 2543 %{
 2544   constraint(ALLOC_IN_RC(int_reg));
 2545   match(RegI);
 2546 
 2547   match(rax_RegI);
 2548   match(rbx_RegI);
 2549   match(rcx_RegI);
 2550   match(rdx_RegI);
 2551   match(rdi_RegI);
 2552 
 2553   format %{ %}
 2554   interface(REG_INTER);
 2555 %}
 2556 
 2557 // Special Registers
 2558 operand rax_RegI()
 2559 %{
 2560   constraint(ALLOC_IN_RC(int_rax_reg));
 2561   match(RegI);
 2562   match(rRegI);
 2563 
 2564   format %{ "RAX" %}
 2565   interface(REG_INTER);
 2566 %}
 2567 
 2568 // Special Registers
 2569 operand rbx_RegI()
 2570 %{
 2571   constraint(ALLOC_IN_RC(int_rbx_reg));
 2572   match(RegI);
 2573   match(rRegI);
 2574 
 2575   format %{ "RBX" %}
 2576   interface(REG_INTER);
 2577 %}
 2578 
 2579 operand rcx_RegI()
 2580 %{
 2581   constraint(ALLOC_IN_RC(int_rcx_reg));
 2582   match(RegI);
 2583   match(rRegI);
 2584 
 2585   format %{ "RCX" %}
 2586   interface(REG_INTER);
 2587 %}
 2588 
 2589 operand rdx_RegI()
 2590 %{
 2591   constraint(ALLOC_IN_RC(int_rdx_reg));
 2592   match(RegI);
 2593   match(rRegI);
 2594 
 2595   format %{ "RDX" %}
 2596   interface(REG_INTER);
 2597 %}
 2598 
 2599 operand rdi_RegI()
 2600 %{
 2601   constraint(ALLOC_IN_RC(int_rdi_reg));
 2602   match(RegI);
 2603   match(rRegI);
 2604 
 2605   format %{ "RDI" %}
 2606   interface(REG_INTER);
 2607 %}
 2608 
 2609 operand no_rax_rdx_RegI()
 2610 %{
 2611   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2612   match(RegI);
 2613   match(rbx_RegI);
 2614   match(rcx_RegI);
 2615   match(rdi_RegI);
 2616 
 2617   format %{ %}
 2618   interface(REG_INTER);
 2619 %}
 2620 
 2621 operand no_rbp_r13_RegI()
 2622 %{
 2623   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2624   match(RegI);
 2625   match(rRegI);
 2626   match(rax_RegI);
 2627   match(rbx_RegI);
 2628   match(rcx_RegI);
 2629   match(rdx_RegI);
 2630   match(rdi_RegI);
 2631 
 2632   format %{ %}
 2633   interface(REG_INTER);
 2634 %}
 2635 
 2636 // Pointer Register
 2637 operand any_RegP()
 2638 %{
 2639   constraint(ALLOC_IN_RC(any_reg));
 2640   match(RegP);
 2641   match(rax_RegP);
 2642   match(rbx_RegP);
 2643   match(rdi_RegP);
 2644   match(rsi_RegP);
 2645   match(rbp_RegP);
 2646   match(r15_RegP);
 2647   match(rRegP);
 2648 
 2649   format %{ %}
 2650   interface(REG_INTER);
 2651 %}
 2652 
 2653 operand rRegP()
 2654 %{
 2655   constraint(ALLOC_IN_RC(ptr_reg));
 2656   match(RegP);
 2657   match(rax_RegP);
 2658   match(rbx_RegP);
 2659   match(rdi_RegP);
 2660   match(rsi_RegP);
 2661   match(rbp_RegP);  // See Q&A below about
 2662   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2663 
 2664   format %{ %}
 2665   interface(REG_INTER);
 2666 %}
 2667 
 2668 operand rRegN() %{
 2669   constraint(ALLOC_IN_RC(int_reg));
 2670   match(RegN);
 2671 
 2672   format %{ %}
 2673   interface(REG_INTER);
 2674 %}
 2675 
 2676 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2677 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2678 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2679 // The output of an instruction is controlled by the allocator, which respects
 2680 // register class masks, not match rules.  Unless an instruction mentions
 2681 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2682 // by the allocator as an input.
 2683 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2684 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2685 // result, RBP is not included in the output of the instruction either.
 2686 
 2687 operand no_rax_RegP()
 2688 %{
 2689   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 2690   match(RegP);
 2691   match(rbx_RegP);
 2692   match(rsi_RegP);
 2693   match(rdi_RegP);
 2694 
 2695   format %{ %}
 2696   interface(REG_INTER);
 2697 %}
 2698 
 2699 // This operand is not allowed to use RBP even if
 2700 // RBP is not used to hold the frame pointer.
 2701 operand no_rbp_RegP()
 2702 %{
 2703   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2704   match(RegP);
 2705   match(rbx_RegP);
 2706   match(rsi_RegP);
 2707   match(rdi_RegP);
 2708 
 2709   format %{ %}
 2710   interface(REG_INTER);
 2711 %}
 2712 
 2713 operand no_rax_rbx_RegP()
 2714 %{
 2715   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 2716   match(RegP);
 2717   match(rsi_RegP);
 2718   match(rdi_RegP);
 2719 
 2720   format %{ %}
 2721   interface(REG_INTER);
 2722 %}
 2723 
 2724 // Special Registers
 2725 // Return a pointer value
 2726 operand rax_RegP()
 2727 %{
 2728   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2729   match(RegP);
 2730   match(rRegP);
 2731 
 2732   format %{ %}
 2733   interface(REG_INTER);
 2734 %}
 2735 
 2736 // Special Registers
 2737 // Return a compressed pointer value
 2738 operand rax_RegN()
 2739 %{
 2740   constraint(ALLOC_IN_RC(int_rax_reg));
 2741   match(RegN);
 2742   match(rRegN);
 2743 
 2744   format %{ %}
 2745   interface(REG_INTER);
 2746 %}
 2747 
 2748 // Used in AtomicAdd
 2749 operand rbx_RegP()
 2750 %{
 2751   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2752   match(RegP);
 2753   match(rRegP);
 2754 
 2755   format %{ %}
 2756   interface(REG_INTER);
 2757 %}
 2758 
 2759 operand rsi_RegP()
 2760 %{
 2761   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2762   match(RegP);
 2763   match(rRegP);
 2764 
 2765   format %{ %}
 2766   interface(REG_INTER);
 2767 %}
 2768 
 2769 operand rbp_RegP()
 2770 %{
 2771   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2772   match(RegP);
 2773   match(rRegP);
 2774 
 2775   format %{ %}
 2776   interface(REG_INTER);
 2777 %}
 2778 
 2779 // Used in rep stosq
 2780 operand rdi_RegP()
 2781 %{
 2782   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2783   match(RegP);
 2784   match(rRegP);
 2785 
 2786   format %{ %}
 2787   interface(REG_INTER);
 2788 %}
 2789 
 2790 operand r15_RegP()
 2791 %{
 2792   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2793   match(RegP);
 2794   match(rRegP);
 2795 
 2796   format %{ %}
 2797   interface(REG_INTER);
 2798 %}
 2799 
 2800 operand rRegL()
 2801 %{
 2802   constraint(ALLOC_IN_RC(long_reg));
 2803   match(RegL);
 2804   match(rax_RegL);
 2805   match(rdx_RegL);
 2806 
 2807   format %{ %}
 2808   interface(REG_INTER);
 2809 %}
 2810 
 2811 // Special Registers
 2812 operand no_rax_rdx_RegL()
 2813 %{
 2814   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2815   match(RegL);
 2816   match(rRegL);
 2817 
 2818   format %{ %}
 2819   interface(REG_INTER);
 2820 %}
 2821 
 2822 operand rax_RegL()
 2823 %{
 2824   constraint(ALLOC_IN_RC(long_rax_reg));
 2825   match(RegL);
 2826   match(rRegL);
 2827 
 2828   format %{ "RAX" %}
 2829   interface(REG_INTER);
 2830 %}
 2831 
 2832 operand rcx_RegL()
 2833 %{
 2834   constraint(ALLOC_IN_RC(long_rcx_reg));
 2835   match(RegL);
 2836   match(rRegL);
 2837 
 2838   format %{ %}
 2839   interface(REG_INTER);
 2840 %}
 2841 
 2842 operand rdx_RegL()
 2843 %{
 2844   constraint(ALLOC_IN_RC(long_rdx_reg));
 2845   match(RegL);
 2846   match(rRegL);
 2847 
 2848   format %{ %}
 2849   interface(REG_INTER);
 2850 %}
 2851 
 2852 operand no_rbp_r13_RegL()
 2853 %{
 2854   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2855   match(RegL);
 2856   match(rRegL);
 2857   match(rax_RegL);
 2858   match(rcx_RegL);
 2859   match(rdx_RegL);
 2860 
 2861   format %{ %}
 2862   interface(REG_INTER);
 2863 %}
 2864 
 2865 // Flags register, used as output of compare instructions
 2866 operand rFlagsReg()
 2867 %{
 2868   constraint(ALLOC_IN_RC(int_flags));
 2869   match(RegFlags);
 2870 
 2871   format %{ "RFLAGS" %}
 2872   interface(REG_INTER);
 2873 %}
 2874 
 2875 // Flags register, used as output of FLOATING POINT compare instructions
 2876 operand rFlagsRegU()
 2877 %{
 2878   constraint(ALLOC_IN_RC(int_flags));
 2879   match(RegFlags);
 2880 
 2881   format %{ "RFLAGS_U" %}
 2882   interface(REG_INTER);
 2883 %}
 2884 
 2885 operand rFlagsRegUCF() %{
 2886   constraint(ALLOC_IN_RC(int_flags));
 2887   match(RegFlags);
 2888   predicate(false);
 2889 
 2890   format %{ "RFLAGS_U_CF" %}
 2891   interface(REG_INTER);
 2892 %}
 2893 
 2894 // Float register operands
 2895 operand regF() %{
 2896    constraint(ALLOC_IN_RC(float_reg));
 2897    match(RegF);
 2898 
 2899    format %{ %}
 2900    interface(REG_INTER);
 2901 %}
 2902 
 2903 // Float register operands
 2904 operand legRegF() %{
 2905    constraint(ALLOC_IN_RC(float_reg_legacy));
 2906    match(RegF);
 2907 
 2908    format %{ %}
 2909    interface(REG_INTER);
 2910 %}
 2911 
 2912 // Float register operands
 2913 operand vlRegF() %{
 2914    constraint(ALLOC_IN_RC(float_reg_vl));
 2915    match(RegF);
 2916 
 2917    format %{ %}
 2918    interface(REG_INTER);
 2919 %}
 2920 
 2921 // Double register operands
 2922 operand regD() %{
 2923    constraint(ALLOC_IN_RC(double_reg));
 2924    match(RegD);
 2925 
 2926    format %{ %}
 2927    interface(REG_INTER);
 2928 %}
 2929 
 2930 // Double register operands
 2931 operand legRegD() %{
 2932    constraint(ALLOC_IN_RC(double_reg_legacy));
 2933    match(RegD);
 2934 
 2935    format %{ %}
 2936    interface(REG_INTER);
 2937 %}
 2938 
 2939 // Double register operands
 2940 operand vlRegD() %{
 2941    constraint(ALLOC_IN_RC(double_reg_vl));
 2942    match(RegD);
 2943 
 2944    format %{ %}
 2945    interface(REG_INTER);
 2946 %}
 2947 
 2948 //----------Memory Operands----------------------------------------------------
 2949 // Direct Memory Operand
 2950 // operand direct(immP addr)
 2951 // %{
 2952 //   match(addr);
 2953 
 2954 //   format %{ "[$addr]" %}
 2955 //   interface(MEMORY_INTER) %{
 2956 //     base(0xFFFFFFFF);
 2957 //     index(0x4);
 2958 //     scale(0x0);
 2959 //     disp($addr);
 2960 //   %}
 2961 // %}
 2962 
 2963 // Indirect Memory Operand
 2964 operand indirect(any_RegP reg)
 2965 %{
 2966   constraint(ALLOC_IN_RC(ptr_reg));
 2967   match(reg);
 2968 
 2969   format %{ "[$reg]" %}
 2970   interface(MEMORY_INTER) %{
 2971     base($reg);
 2972     index(0x4);
 2973     scale(0x0);
 2974     disp(0x0);
 2975   %}
 2976 %}
 2977 
 2978 // Indirect Memory Plus Short Offset Operand
 2979 operand indOffset8(any_RegP reg, immL8 off)
 2980 %{
 2981   constraint(ALLOC_IN_RC(ptr_reg));
 2982   match(AddP reg off);
 2983 
 2984   format %{ "[$reg + $off (8-bit)]" %}
 2985   interface(MEMORY_INTER) %{
 2986     base($reg);
 2987     index(0x4);
 2988     scale(0x0);
 2989     disp($off);
 2990   %}
 2991 %}
 2992 
 2993 // Indirect Memory Plus Long Offset Operand
 2994 operand indOffset32(any_RegP reg, immL32 off)
 2995 %{
 2996   constraint(ALLOC_IN_RC(ptr_reg));
 2997   match(AddP reg off);
 2998 
 2999   format %{ "[$reg + $off (32-bit)]" %}
 3000   interface(MEMORY_INTER) %{
 3001     base($reg);
 3002     index(0x4);
 3003     scale(0x0);
 3004     disp($off);
 3005   %}
 3006 %}
 3007 
 3008 // Indirect Memory Plus Index Register Plus Offset Operand
 3009 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3010 %{
 3011   constraint(ALLOC_IN_RC(ptr_reg));
 3012   match(AddP (AddP reg lreg) off);
 3013 
 3014   op_cost(10);
 3015   format %{"[$reg + $off + $lreg]" %}
 3016   interface(MEMORY_INTER) %{
 3017     base($reg);
 3018     index($lreg);
 3019     scale(0x0);
 3020     disp($off);
 3021   %}
 3022 %}
 3023 
 3024 // Indirect Memory Plus Index Register Plus Offset Operand
 3025 operand indIndex(any_RegP reg, rRegL lreg)
 3026 %{
 3027   constraint(ALLOC_IN_RC(ptr_reg));
 3028   match(AddP reg lreg);
 3029 
 3030   op_cost(10);
 3031   format %{"[$reg + $lreg]" %}
 3032   interface(MEMORY_INTER) %{
 3033     base($reg);
 3034     index($lreg);
 3035     scale(0x0);
 3036     disp(0x0);
 3037   %}
 3038 %}
 3039 
 3040 // Indirect Memory Times Scale Plus Index Register
 3041 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3042 %{
 3043   constraint(ALLOC_IN_RC(ptr_reg));
 3044   match(AddP reg (LShiftL lreg scale));
 3045 
 3046   op_cost(10);
 3047   format %{"[$reg + $lreg << $scale]" %}
 3048   interface(MEMORY_INTER) %{
 3049     base($reg);
 3050     index($lreg);
 3051     scale($scale);
 3052     disp(0x0);
 3053   %}
 3054 %}
 3055 
 3056 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3057 %{
 3058   constraint(ALLOC_IN_RC(ptr_reg));
 3059   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3060   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3061 
 3062   op_cost(10);
 3063   format %{"[$reg + pos $idx << $scale]" %}
 3064   interface(MEMORY_INTER) %{
 3065     base($reg);
 3066     index($idx);
 3067     scale($scale);
 3068     disp(0x0);
 3069   %}
 3070 %}
 3071 
 3072 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3073 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3074 %{
 3075   constraint(ALLOC_IN_RC(ptr_reg));
 3076   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3077 
 3078   op_cost(10);
 3079   format %{"[$reg + $off + $lreg << $scale]" %}
 3080   interface(MEMORY_INTER) %{
 3081     base($reg);
 3082     index($lreg);
 3083     scale($scale);
 3084     disp($off);
 3085   %}
 3086 %}
 3087 
 3088 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3089 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3090 %{
 3091   constraint(ALLOC_IN_RC(ptr_reg));
 3092   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3093   match(AddP (AddP reg (ConvI2L idx)) off);
 3094 
 3095   op_cost(10);
 3096   format %{"[$reg + $off + $idx]" %}
 3097   interface(MEMORY_INTER) %{
 3098     base($reg);
 3099     index($idx);
 3100     scale(0x0);
 3101     disp($off);
 3102   %}
 3103 %}
 3104 
 3105 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3106 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3107 %{
 3108   constraint(ALLOC_IN_RC(ptr_reg));
 3109   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3110   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3111 
 3112   op_cost(10);
 3113   format %{"[$reg + $off + $idx << $scale]" %}
 3114   interface(MEMORY_INTER) %{
 3115     base($reg);
 3116     index($idx);
 3117     scale($scale);
 3118     disp($off);
 3119   %}
 3120 %}
 3121 
 3122 // Indirect Narrow Oop Plus Offset Operand
 3123 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3124 // we can't free r12 even with CompressedOops::base() == NULL.
 3125 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3126   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3127   constraint(ALLOC_IN_RC(ptr_reg));
 3128   match(AddP (DecodeN reg) off);
 3129 
 3130   op_cost(10);
 3131   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3132   interface(MEMORY_INTER) %{
 3133     base(0xc); // R12
 3134     index($reg);
 3135     scale(0x3);
 3136     disp($off);
 3137   %}
 3138 %}
 3139 
 3140 // Indirect Memory Operand
 3141 operand indirectNarrow(rRegN reg)
 3142 %{
 3143   predicate(CompressedOops::shift() == 0);
 3144   constraint(ALLOC_IN_RC(ptr_reg));
 3145   match(DecodeN reg);
 3146 
 3147   format %{ "[$reg]" %}
 3148   interface(MEMORY_INTER) %{
 3149     base($reg);
 3150     index(0x4);
 3151     scale(0x0);
 3152     disp(0x0);
 3153   %}
 3154 %}
 3155 
 3156 // Indirect Memory Plus Short Offset Operand
 3157 operand indOffset8Narrow(rRegN reg, immL8 off)
 3158 %{
 3159   predicate(CompressedOops::shift() == 0);
 3160   constraint(ALLOC_IN_RC(ptr_reg));
 3161   match(AddP (DecodeN reg) off);
 3162 
 3163   format %{ "[$reg + $off (8-bit)]" %}
 3164   interface(MEMORY_INTER) %{
 3165     base($reg);
 3166     index(0x4);
 3167     scale(0x0);
 3168     disp($off);
 3169   %}
 3170 %}
 3171 
 3172 // Indirect Memory Plus Long Offset Operand
 3173 operand indOffset32Narrow(rRegN reg, immL32 off)
 3174 %{
 3175   predicate(CompressedOops::shift() == 0);
 3176   constraint(ALLOC_IN_RC(ptr_reg));
 3177   match(AddP (DecodeN reg) off);
 3178 
 3179   format %{ "[$reg + $off (32-bit)]" %}
 3180   interface(MEMORY_INTER) %{
 3181     base($reg);
 3182     index(0x4);
 3183     scale(0x0);
 3184     disp($off);
 3185   %}
 3186 %}
 3187 
 3188 // Indirect Memory Plus Index Register Plus Offset Operand
 3189 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3190 %{
 3191   predicate(CompressedOops::shift() == 0);
 3192   constraint(ALLOC_IN_RC(ptr_reg));
 3193   match(AddP (AddP (DecodeN reg) lreg) off);
 3194 
 3195   op_cost(10);
 3196   format %{"[$reg + $off + $lreg]" %}
 3197   interface(MEMORY_INTER) %{
 3198     base($reg);
 3199     index($lreg);
 3200     scale(0x0);
 3201     disp($off);
 3202   %}
 3203 %}
 3204 
 3205 // Indirect Memory Plus Index Register Plus Offset Operand
 3206 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3207 %{
 3208   predicate(CompressedOops::shift() == 0);
 3209   constraint(ALLOC_IN_RC(ptr_reg));
 3210   match(AddP (DecodeN reg) lreg);
 3211 
 3212   op_cost(10);
 3213   format %{"[$reg + $lreg]" %}
 3214   interface(MEMORY_INTER) %{
 3215     base($reg);
 3216     index($lreg);
 3217     scale(0x0);
 3218     disp(0x0);
 3219   %}
 3220 %}
 3221 
 3222 // Indirect Memory Times Scale Plus Index Register
 3223 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3224 %{
 3225   predicate(CompressedOops::shift() == 0);
 3226   constraint(ALLOC_IN_RC(ptr_reg));
 3227   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3228 
 3229   op_cost(10);
 3230   format %{"[$reg + $lreg << $scale]" %}
 3231   interface(MEMORY_INTER) %{
 3232     base($reg);
 3233     index($lreg);
 3234     scale($scale);
 3235     disp(0x0);
 3236   %}
 3237 %}
 3238 
 3239 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3240 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3241 %{
 3242   predicate(CompressedOops::shift() == 0);
 3243   constraint(ALLOC_IN_RC(ptr_reg));
 3244   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3245 
 3246   op_cost(10);
 3247   format %{"[$reg + $off + $lreg << $scale]" %}
 3248   interface(MEMORY_INTER) %{
 3249     base($reg);
 3250     index($lreg);
 3251     scale($scale);
 3252     disp($off);
 3253   %}
 3254 %}
 3255 
 3256 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3257 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3258 %{
 3259   constraint(ALLOC_IN_RC(ptr_reg));
 3260   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3261   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3262 
 3263   op_cost(10);
 3264   format %{"[$reg + $off + $idx]" %}
 3265   interface(MEMORY_INTER) %{
 3266     base($reg);
 3267     index($idx);
 3268     scale(0x0);
 3269     disp($off);
 3270   %}
 3271 %}
 3272 
 3273 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3274 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3275 %{
 3276   constraint(ALLOC_IN_RC(ptr_reg));
 3277   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3278   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3279 
 3280   op_cost(10);
 3281   format %{"[$reg + $off + $idx << $scale]" %}
 3282   interface(MEMORY_INTER) %{
 3283     base($reg);
 3284     index($idx);
 3285     scale($scale);
 3286     disp($off);
 3287   %}
 3288 %}
 3289 
 3290 //----------Special Memory Operands--------------------------------------------
 3291 // Stack Slot Operand - This operand is used for loading and storing temporary
 3292 //                      values on the stack where a match requires a value to
 3293 //                      flow through memory.
 3294 operand stackSlotP(sRegP reg)
 3295 %{
 3296   constraint(ALLOC_IN_RC(stack_slots));
 3297   // No match rule because this operand is only generated in matching
 3298 
 3299   format %{ "[$reg]" %}
 3300   interface(MEMORY_INTER) %{
 3301     base(0x4);   // RSP
 3302     index(0x4);  // No Index
 3303     scale(0x0);  // No Scale
 3304     disp($reg);  // Stack Offset
 3305   %}
 3306 %}
 3307 
 3308 operand stackSlotI(sRegI reg)
 3309 %{
 3310   constraint(ALLOC_IN_RC(stack_slots));
 3311   // No match rule because this operand is only generated in matching
 3312 
 3313   format %{ "[$reg]" %}
 3314   interface(MEMORY_INTER) %{
 3315     base(0x4);   // RSP
 3316     index(0x4);  // No Index
 3317     scale(0x0);  // No Scale
 3318     disp($reg);  // Stack Offset
 3319   %}
 3320 %}
 3321 
 3322 operand stackSlotF(sRegF reg)
 3323 %{
 3324   constraint(ALLOC_IN_RC(stack_slots));
 3325   // No match rule because this operand is only generated in matching
 3326 
 3327   format %{ "[$reg]" %}
 3328   interface(MEMORY_INTER) %{
 3329     base(0x4);   // RSP
 3330     index(0x4);  // No Index
 3331     scale(0x0);  // No Scale
 3332     disp($reg);  // Stack Offset
 3333   %}
 3334 %}
 3335 
 3336 operand stackSlotD(sRegD reg)
 3337 %{
 3338   constraint(ALLOC_IN_RC(stack_slots));
 3339   // No match rule because this operand is only generated in matching
 3340 
 3341   format %{ "[$reg]" %}
 3342   interface(MEMORY_INTER) %{
 3343     base(0x4);   // RSP
 3344     index(0x4);  // No Index
 3345     scale(0x0);  // No Scale
 3346     disp($reg);  // Stack Offset
 3347   %}
 3348 %}
 3349 operand stackSlotL(sRegL reg)
 3350 %{
 3351   constraint(ALLOC_IN_RC(stack_slots));
 3352   // No match rule because this operand is only generated in matching
 3353 
 3354   format %{ "[$reg]" %}
 3355   interface(MEMORY_INTER) %{
 3356     base(0x4);   // RSP
 3357     index(0x4);  // No Index
 3358     scale(0x0);  // No Scale
 3359     disp($reg);  // Stack Offset
 3360   %}
 3361 %}
 3362 
 3363 //----------Conditional Branch Operands----------------------------------------
 3364 // Comparison Op  - This is the operation of the comparison, and is limited to
 3365 //                  the following set of codes:
 3366 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3367 //
 3368 // Other attributes of the comparison, such as unsignedness, are specified
 3369 // by the comparison instruction that sets a condition code flags register.
 3370 // That result is represented by a flags operand whose subtype is appropriate
 3371 // to the unsignedness (etc.) of the comparison.
 3372 //
 3373 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3374 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3375 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3376 
 3377 // Comparison Code
 3378 operand cmpOp()
 3379 %{
 3380   match(Bool);
 3381 
 3382   format %{ "" %}
 3383   interface(COND_INTER) %{
 3384     equal(0x4, "e");
 3385     not_equal(0x5, "ne");
 3386     less(0xC, "l");
 3387     greater_equal(0xD, "ge");
 3388     less_equal(0xE, "le");
 3389     greater(0xF, "g");
 3390     overflow(0x0, "o");
 3391     no_overflow(0x1, "no");
 3392   %}
 3393 %}
 3394 
 3395 // Comparison Code, unsigned compare.  Used by FP also, with
 3396 // C2 (unordered) turned into GT or LT already.  The other bits
 3397 // C0 and C3 are turned into Carry & Zero flags.
 3398 operand cmpOpU()
 3399 %{
 3400   match(Bool);
 3401 
 3402   format %{ "" %}
 3403   interface(COND_INTER) %{
 3404     equal(0x4, "e");
 3405     not_equal(0x5, "ne");
 3406     less(0x2, "b");
 3407     greater_equal(0x3, "ae");
 3408     less_equal(0x6, "be");
 3409     greater(0x7, "a");
 3410     overflow(0x0, "o");
 3411     no_overflow(0x1, "no");
 3412   %}
 3413 %}
 3414 
 3415 
 3416 // Floating comparisons that don't require any fixup for the unordered case,
 3417 // If both inputs of the comparison are the same, ZF is always set so we
 3418 // don't need to use cmpOpUCF2 for eq/ne
 3419 operand cmpOpUCF() %{
 3420   match(Bool);
 3421   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3422             n->as_Bool()->_test._test == BoolTest::ge ||
 3423             n->as_Bool()->_test._test == BoolTest::le ||
 3424             n->as_Bool()->_test._test == BoolTest::gt ||
 3425             n->in(1)->in(1) == n->in(1)->in(2));
 3426   format %{ "" %}
 3427   interface(COND_INTER) %{
 3428     equal(0xb, "np");
 3429     not_equal(0xa, "p");
 3430     less(0x2, "b");
 3431     greater_equal(0x3, "ae");
 3432     less_equal(0x6, "be");
 3433     greater(0x7, "a");
 3434     overflow(0x0, "o");
 3435     no_overflow(0x1, "no");
 3436   %}
 3437 %}
 3438 
 3439 
 3440 // Floating comparisons that can be fixed up with extra conditional jumps
 3441 operand cmpOpUCF2() %{
 3442   match(Bool);
 3443   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3444              n->as_Bool()->_test._test == BoolTest::eq) &&
 3445             n->in(1)->in(1) != n->in(1)->in(2));
 3446   format %{ "" %}
 3447   interface(COND_INTER) %{
 3448     equal(0x4, "e");
 3449     not_equal(0x5, "ne");
 3450     less(0x2, "b");
 3451     greater_equal(0x3, "ae");
 3452     less_equal(0x6, "be");
 3453     greater(0x7, "a");
 3454     overflow(0x0, "o");
 3455     no_overflow(0x1, "no");
 3456   %}
 3457 %}
 3458 
 3459 //----------OPERAND CLASSES----------------------------------------------------
 3460 // Operand Classes are groups of operands that are used as to simplify
 3461 // instruction definitions by not requiring the AD writer to specify separate
 3462 // instructions for every form of operand when the instruction accepts
 3463 // multiple operand types with the same basic encoding and format.  The classic
 3464 // case of this is memory operands.
 3465 
 3466 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3467                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3468                indCompressedOopOffset,
 3469                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3470                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3471                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3472 
 3473 //----------PIPELINE-----------------------------------------------------------
 3474 // Rules which define the behavior of the target architectures pipeline.
 3475 pipeline %{
 3476 
 3477 //----------ATTRIBUTES---------------------------------------------------------
 3478 attributes %{
 3479   variable_size_instructions;        // Fixed size instructions
 3480   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3481   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3482   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3483   instruction_fetch_units = 1;       // of 16 bytes
 3484 
 3485   // List of nop instructions
 3486   nops( MachNop );
 3487 %}
 3488 
 3489 //----------RESOURCES----------------------------------------------------------
 3490 // Resources are the functional units available to the machine
 3491 
 3492 // Generic P2/P3 pipeline
 3493 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3494 // 3 instructions decoded per cycle.
 3495 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3496 // 3 ALU op, only ALU0 handles mul instructions.
 3497 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3498            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3499            BR, FPU,
 3500            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3501 
 3502 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3503 // Pipeline Description specifies the stages in the machine's pipeline
 3504 
 3505 // Generic P2/P3 pipeline
 3506 pipe_desc(S0, S1, S2, S3, S4, S5);
 3507 
 3508 //----------PIPELINE CLASSES---------------------------------------------------
 3509 // Pipeline Classes describe the stages in which input and output are
 3510 // referenced by the hardware pipeline.
 3511 
 3512 // Naming convention: ialu or fpu
 3513 // Then: _reg
 3514 // Then: _reg if there is a 2nd register
 3515 // Then: _long if it's a pair of instructions implementing a long
 3516 // Then: _fat if it requires the big decoder
 3517 //   Or: _mem if it requires the big decoder and a memory unit.
 3518 
 3519 // Integer ALU reg operation
 3520 pipe_class ialu_reg(rRegI dst)
 3521 %{
 3522     single_instruction;
 3523     dst    : S4(write);
 3524     dst    : S3(read);
 3525     DECODE : S0;        // any decoder
 3526     ALU    : S3;        // any alu
 3527 %}
 3528 
 3529 // Long ALU reg operation
 3530 pipe_class ialu_reg_long(rRegL dst)
 3531 %{
 3532     instruction_count(2);
 3533     dst    : S4(write);
 3534     dst    : S3(read);
 3535     DECODE : S0(2);     // any 2 decoders
 3536     ALU    : S3(2);     // both alus
 3537 %}
 3538 
 3539 // Integer ALU reg operation using big decoder
 3540 pipe_class ialu_reg_fat(rRegI dst)
 3541 %{
 3542     single_instruction;
 3543     dst    : S4(write);
 3544     dst    : S3(read);
 3545     D0     : S0;        // big decoder only
 3546     ALU    : S3;        // any alu
 3547 %}
 3548 
 3549 // Integer ALU reg-reg operation
 3550 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3551 %{
 3552     single_instruction;
 3553     dst    : S4(write);
 3554     src    : S3(read);
 3555     DECODE : S0;        // any decoder
 3556     ALU    : S3;        // any alu
 3557 %}
 3558 
 3559 // Integer ALU reg-reg operation
 3560 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3561 %{
 3562     single_instruction;
 3563     dst    : S4(write);
 3564     src    : S3(read);
 3565     D0     : S0;        // big decoder only
 3566     ALU    : S3;        // any alu
 3567 %}
 3568 
 3569 // Integer ALU reg-mem operation
 3570 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3571 %{
 3572     single_instruction;
 3573     dst    : S5(write);
 3574     mem    : S3(read);
 3575     D0     : S0;        // big decoder only
 3576     ALU    : S4;        // any alu
 3577     MEM    : S3;        // any mem
 3578 %}
 3579 
 3580 // Integer mem operation (prefetch)
 3581 pipe_class ialu_mem(memory mem)
 3582 %{
 3583     single_instruction;
 3584     mem    : S3(read);
 3585     D0     : S0;        // big decoder only
 3586     MEM    : S3;        // any mem
 3587 %}
 3588 
 3589 // Integer Store to Memory
 3590 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3591 %{
 3592     single_instruction;
 3593     mem    : S3(read);
 3594     src    : S5(read);
 3595     D0     : S0;        // big decoder only
 3596     ALU    : S4;        // any alu
 3597     MEM    : S3;
 3598 %}
 3599 
 3600 // // Long Store to Memory
 3601 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3602 // %{
 3603 //     instruction_count(2);
 3604 //     mem    : S3(read);
 3605 //     src    : S5(read);
 3606 //     D0     : S0(2);          // big decoder only; twice
 3607 //     ALU    : S4(2);     // any 2 alus
 3608 //     MEM    : S3(2);  // Both mems
 3609 // %}
 3610 
 3611 // Integer Store to Memory
 3612 pipe_class ialu_mem_imm(memory mem)
 3613 %{
 3614     single_instruction;
 3615     mem    : S3(read);
 3616     D0     : S0;        // big decoder only
 3617     ALU    : S4;        // any alu
 3618     MEM    : S3;
 3619 %}
 3620 
 3621 // Integer ALU0 reg-reg operation
 3622 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3623 %{
 3624     single_instruction;
 3625     dst    : S4(write);
 3626     src    : S3(read);
 3627     D0     : S0;        // Big decoder only
 3628     ALU0   : S3;        // only alu0
 3629 %}
 3630 
 3631 // Integer ALU0 reg-mem operation
 3632 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3633 %{
 3634     single_instruction;
 3635     dst    : S5(write);
 3636     mem    : S3(read);
 3637     D0     : S0;        // big decoder only
 3638     ALU0   : S4;        // ALU0 only
 3639     MEM    : S3;        // any mem
 3640 %}
 3641 
 3642 // Integer ALU reg-reg operation
 3643 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3644 %{
 3645     single_instruction;
 3646     cr     : S4(write);
 3647     src1   : S3(read);
 3648     src2   : S3(read);
 3649     DECODE : S0;        // any decoder
 3650     ALU    : S3;        // any alu
 3651 %}
 3652 
 3653 // Integer ALU reg-imm operation
 3654 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3655 %{
 3656     single_instruction;
 3657     cr     : S4(write);
 3658     src1   : S3(read);
 3659     DECODE : S0;        // any decoder
 3660     ALU    : S3;        // any alu
 3661 %}
 3662 
 3663 // Integer ALU reg-mem operation
 3664 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3665 %{
 3666     single_instruction;
 3667     cr     : S4(write);
 3668     src1   : S3(read);
 3669     src2   : S3(read);
 3670     D0     : S0;        // big decoder only
 3671     ALU    : S4;        // any alu
 3672     MEM    : S3;
 3673 %}
 3674 
 3675 // Conditional move reg-reg
 3676 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3677 %{
 3678     instruction_count(4);
 3679     y      : S4(read);
 3680     q      : S3(read);
 3681     p      : S3(read);
 3682     DECODE : S0(4);     // any decoder
 3683 %}
 3684 
 3685 // Conditional move reg-reg
 3686 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3687 %{
 3688     single_instruction;
 3689     dst    : S4(write);
 3690     src    : S3(read);
 3691     cr     : S3(read);
 3692     DECODE : S0;        // any decoder
 3693 %}
 3694 
 3695 // Conditional move reg-mem
 3696 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3697 %{
 3698     single_instruction;
 3699     dst    : S4(write);
 3700     src    : S3(read);
 3701     cr     : S3(read);
 3702     DECODE : S0;        // any decoder
 3703     MEM    : S3;
 3704 %}
 3705 
 3706 // Conditional move reg-reg long
 3707 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3708 %{
 3709     single_instruction;
 3710     dst    : S4(write);
 3711     src    : S3(read);
 3712     cr     : S3(read);
 3713     DECODE : S0(2);     // any 2 decoders
 3714 %}
 3715 
 3716 // Float reg-reg operation
 3717 pipe_class fpu_reg(regD dst)
 3718 %{
 3719     instruction_count(2);
 3720     dst    : S3(read);
 3721     DECODE : S0(2);     // any 2 decoders
 3722     FPU    : S3;
 3723 %}
 3724 
 3725 // Float reg-reg operation
 3726 pipe_class fpu_reg_reg(regD dst, regD src)
 3727 %{
 3728     instruction_count(2);
 3729     dst    : S4(write);
 3730     src    : S3(read);
 3731     DECODE : S0(2);     // any 2 decoders
 3732     FPU    : S3;
 3733 %}
 3734 
 3735 // Float reg-reg operation
 3736 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3737 %{
 3738     instruction_count(3);
 3739     dst    : S4(write);
 3740     src1   : S3(read);
 3741     src2   : S3(read);
 3742     DECODE : S0(3);     // any 3 decoders
 3743     FPU    : S3(2);
 3744 %}
 3745 
 3746 // Float reg-reg operation
 3747 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3748 %{
 3749     instruction_count(4);
 3750     dst    : S4(write);
 3751     src1   : S3(read);
 3752     src2   : S3(read);
 3753     src3   : S3(read);
 3754     DECODE : S0(4);     // any 3 decoders
 3755     FPU    : S3(2);
 3756 %}
 3757 
 3758 // Float reg-reg operation
 3759 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3760 %{
 3761     instruction_count(4);
 3762     dst    : S4(write);
 3763     src1   : S3(read);
 3764     src2   : S3(read);
 3765     src3   : S3(read);
 3766     DECODE : S1(3);     // any 3 decoders
 3767     D0     : S0;        // Big decoder only
 3768     FPU    : S3(2);
 3769     MEM    : S3;
 3770 %}
 3771 
 3772 // Float reg-mem operation
 3773 pipe_class fpu_reg_mem(regD dst, memory mem)
 3774 %{
 3775     instruction_count(2);
 3776     dst    : S5(write);
 3777     mem    : S3(read);
 3778     D0     : S0;        // big decoder only
 3779     DECODE : S1;        // any decoder for FPU POP
 3780     FPU    : S4;
 3781     MEM    : S3;        // any mem
 3782 %}
 3783 
 3784 // Float reg-mem operation
 3785 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3786 %{
 3787     instruction_count(3);
 3788     dst    : S5(write);
 3789     src1   : S3(read);
 3790     mem    : S3(read);
 3791     D0     : S0;        // big decoder only
 3792     DECODE : S1(2);     // any decoder for FPU POP
 3793     FPU    : S4;
 3794     MEM    : S3;        // any mem
 3795 %}
 3796 
 3797 // Float mem-reg operation
 3798 pipe_class fpu_mem_reg(memory mem, regD src)
 3799 %{
 3800     instruction_count(2);
 3801     src    : S5(read);
 3802     mem    : S3(read);
 3803     DECODE : S0;        // any decoder for FPU PUSH
 3804     D0     : S1;        // big decoder only
 3805     FPU    : S4;
 3806     MEM    : S3;        // any mem
 3807 %}
 3808 
 3809 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3810 %{
 3811     instruction_count(3);
 3812     src1   : S3(read);
 3813     src2   : S3(read);
 3814     mem    : S3(read);
 3815     DECODE : S0(2);     // any decoder for FPU PUSH
 3816     D0     : S1;        // big decoder only
 3817     FPU    : S4;
 3818     MEM    : S3;        // any mem
 3819 %}
 3820 
 3821 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3822 %{
 3823     instruction_count(3);
 3824     src1   : S3(read);
 3825     src2   : S3(read);
 3826     mem    : S4(read);
 3827     DECODE : S0;        // any decoder for FPU PUSH
 3828     D0     : S0(2);     // big decoder only
 3829     FPU    : S4;
 3830     MEM    : S3(2);     // any mem
 3831 %}
 3832 
 3833 pipe_class fpu_mem_mem(memory dst, memory src1)
 3834 %{
 3835     instruction_count(2);
 3836     src1   : S3(read);
 3837     dst    : S4(read);
 3838     D0     : S0(2);     // big decoder only
 3839     MEM    : S3(2);     // any mem
 3840 %}
 3841 
 3842 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3843 %{
 3844     instruction_count(3);
 3845     src1   : S3(read);
 3846     src2   : S3(read);
 3847     dst    : S4(read);
 3848     D0     : S0(3);     // big decoder only
 3849     FPU    : S4;
 3850     MEM    : S3(3);     // any mem
 3851 %}
 3852 
 3853 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3854 %{
 3855     instruction_count(3);
 3856     src1   : S4(read);
 3857     mem    : S4(read);
 3858     DECODE : S0;        // any decoder for FPU PUSH
 3859     D0     : S0(2);     // big decoder only
 3860     FPU    : S4;
 3861     MEM    : S3(2);     // any mem
 3862 %}
 3863 
 3864 // Float load constant
 3865 pipe_class fpu_reg_con(regD dst)
 3866 %{
 3867     instruction_count(2);
 3868     dst    : S5(write);
 3869     D0     : S0;        // big decoder only for the load
 3870     DECODE : S1;        // any decoder for FPU POP
 3871     FPU    : S4;
 3872     MEM    : S3;        // any mem
 3873 %}
 3874 
 3875 // Float load constant
 3876 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3877 %{
 3878     instruction_count(3);
 3879     dst    : S5(write);
 3880     src    : S3(read);
 3881     D0     : S0;        // big decoder only for the load
 3882     DECODE : S1(2);     // any decoder for FPU POP
 3883     FPU    : S4;
 3884     MEM    : S3;        // any mem
 3885 %}
 3886 
 3887 // UnConditional branch
 3888 pipe_class pipe_jmp(label labl)
 3889 %{
 3890     single_instruction;
 3891     BR   : S3;
 3892 %}
 3893 
 3894 // Conditional branch
 3895 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3896 %{
 3897     single_instruction;
 3898     cr    : S1(read);
 3899     BR    : S3;
 3900 %}
 3901 
 3902 // Allocation idiom
 3903 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3904 %{
 3905     instruction_count(1); force_serialization;
 3906     fixed_latency(6);
 3907     heap_ptr : S3(read);
 3908     DECODE   : S0(3);
 3909     D0       : S2;
 3910     MEM      : S3;
 3911     ALU      : S3(2);
 3912     dst      : S5(write);
 3913     BR       : S5;
 3914 %}
 3915 
 3916 // Generic big/slow expanded idiom
 3917 pipe_class pipe_slow()
 3918 %{
 3919     instruction_count(10); multiple_bundles; force_serialization;
 3920     fixed_latency(100);
 3921     D0  : S0(2);
 3922     MEM : S3(2);
 3923 %}
 3924 
 3925 // The real do-nothing guy
 3926 pipe_class empty()
 3927 %{
 3928     instruction_count(0);
 3929 %}
 3930 
 3931 // Define the class for the Nop node
 3932 define
 3933 %{
 3934    MachNop = empty;
 3935 %}
 3936 
 3937 %}
 3938 
 3939 //----------INSTRUCTIONS-------------------------------------------------------
 3940 //
 3941 // match      -- States which machine-independent subtree may be replaced
 3942 //               by this instruction.
 3943 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3944 //               selection to identify a minimum cost tree of machine
 3945 //               instructions that matches a tree of machine-independent
 3946 //               instructions.
 3947 // format     -- A string providing the disassembly for this instruction.
 3948 //               The value of an instruction's operand may be inserted
 3949 //               by referring to it with a '$' prefix.
 3950 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3951 //               to within an encode class as $primary, $secondary, and $tertiary
 3952 //               rrspectively.  The primary opcode is commonly used to
 3953 //               indicate the type of machine instruction, while secondary
 3954 //               and tertiary are often used for prefix options or addressing
 3955 //               modes.
 3956 // ins_encode -- A list of encode classes with parameters. The encode class
 3957 //               name must have been defined in an 'enc_class' specification
 3958 //               in the encode section of the architecture description.
 3959 
 3960 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3961 // Load Float
 3962 instruct MoveF2VL(vlRegF dst, regF src) %{
 3963   match(Set dst src);
 3964   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3965   ins_encode %{
 3966     ShouldNotReachHere();
 3967   %}
 3968   ins_pipe( fpu_reg_reg );
 3969 %}
 3970 
 3971 // Load Float
 3972 instruct MoveF2LEG(legRegF dst, regF src) %{
 3973   match(Set dst src);
 3974   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3975   ins_encode %{
 3976     ShouldNotReachHere();
 3977   %}
 3978   ins_pipe( fpu_reg_reg );
 3979 %}
 3980 
 3981 // Load Float
 3982 instruct MoveVL2F(regF dst, vlRegF src) %{
 3983   match(Set dst src);
 3984   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3985   ins_encode %{
 3986     ShouldNotReachHere();
 3987   %}
 3988   ins_pipe( fpu_reg_reg );
 3989 %}
 3990 
 3991 // Load Float
 3992 instruct MoveLEG2F(regF dst, legRegF src) %{
 3993   match(Set dst src);
 3994   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3995   ins_encode %{
 3996     ShouldNotReachHere();
 3997   %}
 3998   ins_pipe( fpu_reg_reg );
 3999 %}
 4000 
 4001 // Load Double
 4002 instruct MoveD2VL(vlRegD dst, regD src) %{
 4003   match(Set dst src);
 4004   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4005   ins_encode %{
 4006     ShouldNotReachHere();
 4007   %}
 4008   ins_pipe( fpu_reg_reg );
 4009 %}
 4010 
 4011 // Load Double
 4012 instruct MoveD2LEG(legRegD dst, regD src) %{
 4013   match(Set dst src);
 4014   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4015   ins_encode %{
 4016     ShouldNotReachHere();
 4017   %}
 4018   ins_pipe( fpu_reg_reg );
 4019 %}
 4020 
 4021 // Load Double
 4022 instruct MoveVL2D(regD dst, vlRegD src) %{
 4023   match(Set dst src);
 4024   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4025   ins_encode %{
 4026     ShouldNotReachHere();
 4027   %}
 4028   ins_pipe( fpu_reg_reg );
 4029 %}
 4030 
 4031 // Load Double
 4032 instruct MoveLEG2D(regD dst, legRegD src) %{
 4033   match(Set dst src);
 4034   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4035   ins_encode %{
 4036     ShouldNotReachHere();
 4037   %}
 4038   ins_pipe( fpu_reg_reg );
 4039 %}
 4040 
 4041 //----------Load/Store/Move Instructions---------------------------------------
 4042 //----------Load Instructions--------------------------------------------------
 4043 
 4044 // Load Byte (8 bit signed)
 4045 instruct loadB(rRegI dst, memory mem)
 4046 %{
 4047   match(Set dst (LoadB mem));
 4048 
 4049   ins_cost(125);
 4050   format %{ "movsbl  $dst, $mem\t# byte" %}
 4051 
 4052   ins_encode %{
 4053     __ movsbl($dst$$Register, $mem$$Address);
 4054   %}
 4055 
 4056   ins_pipe(ialu_reg_mem);
 4057 %}
 4058 
 4059 // Load Byte (8 bit signed) into Long Register
 4060 instruct loadB2L(rRegL dst, memory mem)
 4061 %{
 4062   match(Set dst (ConvI2L (LoadB mem)));
 4063 
 4064   ins_cost(125);
 4065   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4066 
 4067   ins_encode %{
 4068     __ movsbq($dst$$Register, $mem$$Address);
 4069   %}
 4070 
 4071   ins_pipe(ialu_reg_mem);
 4072 %}
 4073 
 4074 // Load Unsigned Byte (8 bit UNsigned)
 4075 instruct loadUB(rRegI dst, memory mem)
 4076 %{
 4077   match(Set dst (LoadUB mem));
 4078 
 4079   ins_cost(125);
 4080   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4081 
 4082   ins_encode %{
 4083     __ movzbl($dst$$Register, $mem$$Address);
 4084   %}
 4085 
 4086   ins_pipe(ialu_reg_mem);
 4087 %}
 4088 
 4089 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4090 instruct loadUB2L(rRegL dst, memory mem)
 4091 %{
 4092   match(Set dst (ConvI2L (LoadUB mem)));
 4093 
 4094   ins_cost(125);
 4095   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4096 
 4097   ins_encode %{
 4098     __ movzbq($dst$$Register, $mem$$Address);
 4099   %}
 4100 
 4101   ins_pipe(ialu_reg_mem);
 4102 %}
 4103 
 4104 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4105 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4106   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4107   effect(KILL cr);
 4108 
 4109   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4110             "andl    $dst, right_n_bits($mask, 8)" %}
 4111   ins_encode %{
 4112     Register Rdst = $dst$$Register;
 4113     __ movzbq(Rdst, $mem$$Address);
 4114     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4115   %}
 4116   ins_pipe(ialu_reg_mem);
 4117 %}
 4118 
 4119 // Load Short (16 bit signed)
 4120 instruct loadS(rRegI dst, memory mem)
 4121 %{
 4122   match(Set dst (LoadS mem));
 4123 
 4124   ins_cost(125);
 4125   format %{ "movswl $dst, $mem\t# short" %}
 4126 
 4127   ins_encode %{
 4128     __ movswl($dst$$Register, $mem$$Address);
 4129   %}
 4130 
 4131   ins_pipe(ialu_reg_mem);
 4132 %}
 4133 
 4134 // Load Short (16 bit signed) to Byte (8 bit signed)
 4135 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4136   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4137 
 4138   ins_cost(125);
 4139   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4140   ins_encode %{
 4141     __ movsbl($dst$$Register, $mem$$Address);
 4142   %}
 4143   ins_pipe(ialu_reg_mem);
 4144 %}
 4145 
 4146 // Load Short (16 bit signed) into Long Register
 4147 instruct loadS2L(rRegL dst, memory mem)
 4148 %{
 4149   match(Set dst (ConvI2L (LoadS mem)));
 4150 
 4151   ins_cost(125);
 4152   format %{ "movswq $dst, $mem\t# short -> long" %}
 4153 
 4154   ins_encode %{
 4155     __ movswq($dst$$Register, $mem$$Address);
 4156   %}
 4157 
 4158   ins_pipe(ialu_reg_mem);
 4159 %}
 4160 
 4161 // Load Unsigned Short/Char (16 bit UNsigned)
 4162 instruct loadUS(rRegI dst, memory mem)
 4163 %{
 4164   match(Set dst (LoadUS mem));
 4165 
 4166   ins_cost(125);
 4167   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4168 
 4169   ins_encode %{
 4170     __ movzwl($dst$$Register, $mem$$Address);
 4171   %}
 4172 
 4173   ins_pipe(ialu_reg_mem);
 4174 %}
 4175 
 4176 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4177 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4178   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4179 
 4180   ins_cost(125);
 4181   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4182   ins_encode %{
 4183     __ movsbl($dst$$Register, $mem$$Address);
 4184   %}
 4185   ins_pipe(ialu_reg_mem);
 4186 %}
 4187 
 4188 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4189 instruct loadUS2L(rRegL dst, memory mem)
 4190 %{
 4191   match(Set dst (ConvI2L (LoadUS mem)));
 4192 
 4193   ins_cost(125);
 4194   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4195 
 4196   ins_encode %{
 4197     __ movzwq($dst$$Register, $mem$$Address);
 4198   %}
 4199 
 4200   ins_pipe(ialu_reg_mem);
 4201 %}
 4202 
 4203 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4204 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4205   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4206 
 4207   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4208   ins_encode %{
 4209     __ movzbq($dst$$Register, $mem$$Address);
 4210   %}
 4211   ins_pipe(ialu_reg_mem);
 4212 %}
 4213 
 4214 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4215 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4216   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4217   effect(KILL cr);
 4218 
 4219   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4220             "andl    $dst, right_n_bits($mask, 16)" %}
 4221   ins_encode %{
 4222     Register Rdst = $dst$$Register;
 4223     __ movzwq(Rdst, $mem$$Address);
 4224     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4225   %}
 4226   ins_pipe(ialu_reg_mem);
 4227 %}
 4228 
 4229 // Load Integer
 4230 instruct loadI(rRegI dst, memory mem)
 4231 %{
 4232   match(Set dst (LoadI mem));
 4233 
 4234   ins_cost(125);
 4235   format %{ "movl    $dst, $mem\t# int" %}
 4236 
 4237   ins_encode %{
 4238     __ movl($dst$$Register, $mem$$Address);
 4239   %}
 4240 
 4241   ins_pipe(ialu_reg_mem);
 4242 %}
 4243 
 4244 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4245 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4246   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4247 
 4248   ins_cost(125);
 4249   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4250   ins_encode %{
 4251     __ movsbl($dst$$Register, $mem$$Address);
 4252   %}
 4253   ins_pipe(ialu_reg_mem);
 4254 %}
 4255 
 4256 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4257 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4258   match(Set dst (AndI (LoadI mem) mask));
 4259 
 4260   ins_cost(125);
 4261   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4262   ins_encode %{
 4263     __ movzbl($dst$$Register, $mem$$Address);
 4264   %}
 4265   ins_pipe(ialu_reg_mem);
 4266 %}
 4267 
 4268 // Load Integer (32 bit signed) to Short (16 bit signed)
 4269 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4270   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4271 
 4272   ins_cost(125);
 4273   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4274   ins_encode %{
 4275     __ movswl($dst$$Register, $mem$$Address);
 4276   %}
 4277   ins_pipe(ialu_reg_mem);
 4278 %}
 4279 
 4280 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4281 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4282   match(Set dst (AndI (LoadI mem) mask));
 4283 
 4284   ins_cost(125);
 4285   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4286   ins_encode %{
 4287     __ movzwl($dst$$Register, $mem$$Address);
 4288   %}
 4289   ins_pipe(ialu_reg_mem);
 4290 %}
 4291 
 4292 // Load Integer into Long Register
 4293 instruct loadI2L(rRegL dst, memory mem)
 4294 %{
 4295   match(Set dst (ConvI2L (LoadI mem)));
 4296 
 4297   ins_cost(125);
 4298   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4299 
 4300   ins_encode %{
 4301     __ movslq($dst$$Register, $mem$$Address);
 4302   %}
 4303 
 4304   ins_pipe(ialu_reg_mem);
 4305 %}
 4306 
 4307 // Load Integer with mask 0xFF into Long Register
 4308 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4309   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4310 
 4311   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4312   ins_encode %{
 4313     __ movzbq($dst$$Register, $mem$$Address);
 4314   %}
 4315   ins_pipe(ialu_reg_mem);
 4316 %}
 4317 
 4318 // Load Integer with mask 0xFFFF into Long Register
 4319 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4320   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4321 
 4322   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4323   ins_encode %{
 4324     __ movzwq($dst$$Register, $mem$$Address);
 4325   %}
 4326   ins_pipe(ialu_reg_mem);
 4327 %}
 4328 
 4329 // Load Integer with a 31-bit mask into Long Register
 4330 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4331   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4332   effect(KILL cr);
 4333 
 4334   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4335             "andl    $dst, $mask" %}
 4336   ins_encode %{
 4337     Register Rdst = $dst$$Register;
 4338     __ movl(Rdst, $mem$$Address);
 4339     __ andl(Rdst, $mask$$constant);
 4340   %}
 4341   ins_pipe(ialu_reg_mem);
 4342 %}
 4343 
 4344 // Load Unsigned Integer into Long Register
 4345 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4346 %{
 4347   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4348 
 4349   ins_cost(125);
 4350   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4351 
 4352   ins_encode %{
 4353     __ movl($dst$$Register, $mem$$Address);
 4354   %}
 4355 
 4356   ins_pipe(ialu_reg_mem);
 4357 %}
 4358 
 4359 // Load Long
 4360 instruct loadL(rRegL dst, memory mem)
 4361 %{
 4362   match(Set dst (LoadL mem));
 4363 
 4364   ins_cost(125);
 4365   format %{ "movq    $dst, $mem\t# long" %}
 4366 
 4367   ins_encode %{
 4368     __ movq($dst$$Register, $mem$$Address);
 4369   %}
 4370 
 4371   ins_pipe(ialu_reg_mem); // XXX
 4372 %}
 4373 
 4374 // Load Range
 4375 instruct loadRange(rRegI dst, memory mem)
 4376 %{
 4377   match(Set dst (LoadRange mem));
 4378 
 4379   ins_cost(125); // XXX
 4380   format %{ "movl    $dst, $mem\t# range" %}
 4381   ins_encode %{
 4382     __ movl($dst$$Register, $mem$$Address);
 4383   %}
 4384   ins_pipe(ialu_reg_mem);
 4385 %}
 4386 
 4387 // Load Pointer
 4388 instruct loadP(rRegP dst, memory mem)
 4389 %{
 4390   match(Set dst (LoadP mem));
 4391   predicate(n->as_Load()->barrier_data() == 0);
 4392 
 4393   ins_cost(125); // XXX
 4394   format %{ "movq    $dst, $mem\t# ptr" %}
 4395   ins_encode %{
 4396     __ movq($dst$$Register, $mem$$Address);
 4397   %}
 4398   ins_pipe(ialu_reg_mem); // XXX
 4399 %}
 4400 
 4401 // Load Compressed Pointer
 4402 instruct loadN(rRegN dst, memory mem)
 4403 %{
 4404    match(Set dst (LoadN mem));
 4405 
 4406    ins_cost(125); // XXX
 4407    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4408    ins_encode %{
 4409      __ movl($dst$$Register, $mem$$Address);
 4410    %}
 4411    ins_pipe(ialu_reg_mem); // XXX
 4412 %}
 4413 
 4414 
 4415 // Load Klass Pointer
 4416 instruct loadKlass(rRegP dst, memory mem)
 4417 %{
 4418   match(Set dst (LoadKlass mem));
 4419 
 4420   ins_cost(125); // XXX
 4421   format %{ "movq    $dst, $mem\t# class" %}
 4422   ins_encode %{
 4423     __ movq($dst$$Register, $mem$$Address);
 4424   %}
 4425   ins_pipe(ialu_reg_mem); // XXX
 4426 %}
 4427 
 4428 // Load narrow Klass Pointer
 4429 instruct loadNKlass(rRegN dst, memory mem)
 4430 %{
 4431   match(Set dst (LoadNKlass mem));
 4432 
 4433   ins_cost(125); // XXX
 4434   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4435   ins_encode %{
 4436     __ movl($dst$$Register, $mem$$Address);
 4437   %}
 4438   ins_pipe(ialu_reg_mem); // XXX
 4439 %}
 4440 
 4441 // Load Float
 4442 instruct loadF(regF dst, memory mem)
 4443 %{
 4444   match(Set dst (LoadF mem));
 4445 
 4446   ins_cost(145); // XXX
 4447   format %{ "movss   $dst, $mem\t# float" %}
 4448   ins_encode %{
 4449     __ movflt($dst$$XMMRegister, $mem$$Address);
 4450   %}
 4451   ins_pipe(pipe_slow); // XXX
 4452 %}
 4453 
 4454 // Load Double
 4455 instruct loadD_partial(regD dst, memory mem)
 4456 %{
 4457   predicate(!UseXmmLoadAndClearUpper);
 4458   match(Set dst (LoadD mem));
 4459 
 4460   ins_cost(145); // XXX
 4461   format %{ "movlpd  $dst, $mem\t# double" %}
 4462   ins_encode %{
 4463     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4464   %}
 4465   ins_pipe(pipe_slow); // XXX
 4466 %}
 4467 
 4468 instruct loadD(regD dst, memory mem)
 4469 %{
 4470   predicate(UseXmmLoadAndClearUpper);
 4471   match(Set dst (LoadD mem));
 4472 
 4473   ins_cost(145); // XXX
 4474   format %{ "movsd   $dst, $mem\t# double" %}
 4475   ins_encode %{
 4476     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4477   %}
 4478   ins_pipe(pipe_slow); // XXX
 4479 %}
 4480 
 4481 // max = java.lang.Math.max(float a, float b)
 4482 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4483   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4484   match(Set dst (MaxF a b));
 4485   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4486   format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4487   ins_encode %{
 4488     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4489   %}
 4490   ins_pipe( pipe_slow );
 4491 %}
 4492 
 4493 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4494   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4495   match(Set dst (MaxF a b));
 4496   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4497 
 4498   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 4499   ins_encode %{
 4500     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4501                     false /*min*/, true /*single*/);
 4502   %}
 4503   ins_pipe( pipe_slow );
 4504 %}
 4505 
 4506 // max = java.lang.Math.max(double a, double b)
 4507 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4508   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4509   match(Set dst (MaxD a b));
 4510   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4511   format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4512   ins_encode %{
 4513     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4514   %}
 4515   ins_pipe( pipe_slow );
 4516 %}
 4517 
 4518 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4519   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4520   match(Set dst (MaxD a b));
 4521   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4522 
 4523   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 4524   ins_encode %{
 4525     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4526                     false /*min*/, false /*single*/);
 4527   %}
 4528   ins_pipe( pipe_slow );
 4529 %}
 4530 
 4531 // min = java.lang.Math.min(float a, float b)
 4532 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4533   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4534   match(Set dst (MinF a b));
 4535   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4536   format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4537   ins_encode %{
 4538     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4539   %}
 4540   ins_pipe( pipe_slow );
 4541 %}
 4542 
 4543 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4544   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4545   match(Set dst (MinF a b));
 4546   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4547 
 4548   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 4549   ins_encode %{
 4550     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4551                     true /*min*/, true /*single*/);
 4552   %}
 4553   ins_pipe( pipe_slow );
 4554 %}
 4555 
 4556 // min = java.lang.Math.min(double a, double b)
 4557 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4558   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4559   match(Set dst (MinD a b));
 4560   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4561     format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4562   ins_encode %{
 4563     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4564   %}
 4565   ins_pipe( pipe_slow );
 4566 %}
 4567 
 4568 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4569   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4570   match(Set dst (MinD a b));
 4571   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4572 
 4573   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 4574   ins_encode %{
 4575     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4576                     true /*min*/, false /*single*/);
 4577   %}
 4578   ins_pipe( pipe_slow );
 4579 %}
 4580 
 4581 // Load Effective Address
 4582 instruct leaP8(rRegP dst, indOffset8 mem)
 4583 %{
 4584   match(Set dst mem);
 4585 
 4586   ins_cost(110); // XXX
 4587   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4588   ins_encode %{
 4589     __ leaq($dst$$Register, $mem$$Address);
 4590   %}
 4591   ins_pipe(ialu_reg_reg_fat);
 4592 %}
 4593 
 4594 instruct leaP32(rRegP dst, indOffset32 mem)
 4595 %{
 4596   match(Set dst mem);
 4597 
 4598   ins_cost(110);
 4599   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4600   ins_encode %{
 4601     __ leaq($dst$$Register, $mem$$Address);
 4602   %}
 4603   ins_pipe(ialu_reg_reg_fat);
 4604 %}
 4605 
 4606 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4607 %{
 4608   match(Set dst mem);
 4609 
 4610   ins_cost(110);
 4611   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4612   ins_encode %{
 4613     __ leaq($dst$$Register, $mem$$Address);
 4614   %}
 4615   ins_pipe(ialu_reg_reg_fat);
 4616 %}
 4617 
 4618 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4619 %{
 4620   match(Set dst mem);
 4621 
 4622   ins_cost(110);
 4623   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4624   ins_encode %{
 4625     __ leaq($dst$$Register, $mem$$Address);
 4626   %}
 4627   ins_pipe(ialu_reg_reg_fat);
 4628 %}
 4629 
 4630 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4631 %{
 4632   match(Set dst mem);
 4633 
 4634   ins_cost(110);
 4635   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4636   ins_encode %{
 4637     __ leaq($dst$$Register, $mem$$Address);
 4638   %}
 4639   ins_pipe(ialu_reg_reg_fat);
 4640 %}
 4641 
 4642 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4643 %{
 4644   match(Set dst mem);
 4645 
 4646   ins_cost(110);
 4647   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4648   ins_encode %{
 4649     __ leaq($dst$$Register, $mem$$Address);
 4650   %}
 4651   ins_pipe(ialu_reg_reg_fat);
 4652 %}
 4653 
 4654 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4655 %{
 4656   match(Set dst mem);
 4657 
 4658   ins_cost(110);
 4659   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4660   ins_encode %{
 4661     __ leaq($dst$$Register, $mem$$Address);
 4662   %}
 4663   ins_pipe(ialu_reg_reg_fat);
 4664 %}
 4665 
 4666 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4667 %{
 4668   match(Set dst mem);
 4669 
 4670   ins_cost(110);
 4671   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4672   ins_encode %{
 4673     __ leaq($dst$$Register, $mem$$Address);
 4674   %}
 4675   ins_pipe(ialu_reg_reg_fat);
 4676 %}
 4677 
 4678 // Load Effective Address which uses Narrow (32-bits) oop
 4679 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4680 %{
 4681   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4682   match(Set dst mem);
 4683 
 4684   ins_cost(110);
 4685   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4686   ins_encode %{
 4687     __ leaq($dst$$Register, $mem$$Address);
 4688   %}
 4689   ins_pipe(ialu_reg_reg_fat);
 4690 %}
 4691 
 4692 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4693 %{
 4694   predicate(CompressedOops::shift() == 0);
 4695   match(Set dst mem);
 4696 
 4697   ins_cost(110); // XXX
 4698   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4699   ins_encode %{
 4700     __ leaq($dst$$Register, $mem$$Address);
 4701   %}
 4702   ins_pipe(ialu_reg_reg_fat);
 4703 %}
 4704 
 4705 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4706 %{
 4707   predicate(CompressedOops::shift() == 0);
 4708   match(Set dst mem);
 4709 
 4710   ins_cost(110);
 4711   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4712   ins_encode %{
 4713     __ leaq($dst$$Register, $mem$$Address);
 4714   %}
 4715   ins_pipe(ialu_reg_reg_fat);
 4716 %}
 4717 
 4718 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4719 %{
 4720   predicate(CompressedOops::shift() == 0);
 4721   match(Set dst mem);
 4722 
 4723   ins_cost(110);
 4724   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4725   ins_encode %{
 4726     __ leaq($dst$$Register, $mem$$Address);
 4727   %}
 4728   ins_pipe(ialu_reg_reg_fat);
 4729 %}
 4730 
 4731 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4732 %{
 4733   predicate(CompressedOops::shift() == 0);
 4734   match(Set dst mem);
 4735 
 4736   ins_cost(110);
 4737   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4738   ins_encode %{
 4739     __ leaq($dst$$Register, $mem$$Address);
 4740   %}
 4741   ins_pipe(ialu_reg_reg_fat);
 4742 %}
 4743 
 4744 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4745 %{
 4746   predicate(CompressedOops::shift() == 0);
 4747   match(Set dst mem);
 4748 
 4749   ins_cost(110);
 4750   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4751   ins_encode %{
 4752     __ leaq($dst$$Register, $mem$$Address);
 4753   %}
 4754   ins_pipe(ialu_reg_reg_fat);
 4755 %}
 4756 
 4757 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4758 %{
 4759   predicate(CompressedOops::shift() == 0);
 4760   match(Set dst mem);
 4761 
 4762   ins_cost(110);
 4763   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4764   ins_encode %{
 4765     __ leaq($dst$$Register, $mem$$Address);
 4766   %}
 4767   ins_pipe(ialu_reg_reg_fat);
 4768 %}
 4769 
 4770 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4771 %{
 4772   predicate(CompressedOops::shift() == 0);
 4773   match(Set dst mem);
 4774 
 4775   ins_cost(110);
 4776   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4777   ins_encode %{
 4778     __ leaq($dst$$Register, $mem$$Address);
 4779   %}
 4780   ins_pipe(ialu_reg_reg_fat);
 4781 %}
 4782 
 4783 instruct loadConI(rRegI dst, immI src)
 4784 %{
 4785   match(Set dst src);
 4786 
 4787   format %{ "movl    $dst, $src\t# int" %}
 4788   ins_encode %{
 4789     __ movl($dst$$Register, $src$$constant);
 4790   %}
 4791   ins_pipe(ialu_reg_fat); // XXX
 4792 %}
 4793 
 4794 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4795 %{
 4796   match(Set dst src);
 4797   effect(KILL cr);
 4798 
 4799   ins_cost(50);
 4800   format %{ "xorl    $dst, $dst\t# int" %}
 4801   ins_encode %{
 4802     __ xorl($dst$$Register, $dst$$Register);
 4803   %}
 4804   ins_pipe(ialu_reg);
 4805 %}
 4806 
 4807 instruct loadConL(rRegL dst, immL src)
 4808 %{
 4809   match(Set dst src);
 4810 
 4811   ins_cost(150);
 4812   format %{ "movq    $dst, $src\t# long" %}
 4813   ins_encode %{
 4814     __ mov64($dst$$Register, $src$$constant);
 4815   %}
 4816   ins_pipe(ialu_reg);
 4817 %}
 4818 
 4819 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4820 %{
 4821   match(Set dst src);
 4822   effect(KILL cr);
 4823 
 4824   ins_cost(50);
 4825   format %{ "xorl    $dst, $dst\t# long" %}
 4826   ins_encode %{
 4827     __ xorl($dst$$Register, $dst$$Register);
 4828   %}
 4829   ins_pipe(ialu_reg); // XXX
 4830 %}
 4831 
 4832 instruct loadConUL32(rRegL dst, immUL32 src)
 4833 %{
 4834   match(Set dst src);
 4835 
 4836   ins_cost(60);
 4837   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4838   ins_encode %{
 4839     __ movl($dst$$Register, $src$$constant);
 4840   %}
 4841   ins_pipe(ialu_reg);
 4842 %}
 4843 
 4844 instruct loadConL32(rRegL dst, immL32 src)
 4845 %{
 4846   match(Set dst src);
 4847 
 4848   ins_cost(70);
 4849   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4850   ins_encode %{
 4851     __ movq($dst$$Register, $src$$constant);
 4852   %}
 4853   ins_pipe(ialu_reg);
 4854 %}
 4855 
 4856 instruct loadConP(rRegP dst, immP con) %{
 4857   match(Set dst con);
 4858 
 4859   format %{ "movq    $dst, $con\t# ptr" %}
 4860   ins_encode %{
 4861     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4862   %}
 4863   ins_pipe(ialu_reg_fat); // XXX
 4864 %}
 4865 
 4866 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4867 %{
 4868   match(Set dst src);
 4869   effect(KILL cr);
 4870 
 4871   ins_cost(50);
 4872   format %{ "xorl    $dst, $dst\t# ptr" %}
 4873   ins_encode %{
 4874     __ xorl($dst$$Register, $dst$$Register);
 4875   %}
 4876   ins_pipe(ialu_reg);
 4877 %}
 4878 
 4879 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4880 %{
 4881   match(Set dst src);
 4882   effect(KILL cr);
 4883 
 4884   ins_cost(60);
 4885   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4886   ins_encode %{
 4887     __ movl($dst$$Register, $src$$constant);
 4888   %}
 4889   ins_pipe(ialu_reg);
 4890 %}
 4891 
 4892 instruct loadConF(regF dst, immF con) %{
 4893   match(Set dst con);
 4894   ins_cost(125);
 4895   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4896   ins_encode %{
 4897     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4898   %}
 4899   ins_pipe(pipe_slow);
 4900 %}
 4901 
 4902 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4903   match(Set dst src);
 4904   effect(KILL cr);
 4905   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 4906   ins_encode %{
 4907     __ xorq($dst$$Register, $dst$$Register);
 4908   %}
 4909   ins_pipe(ialu_reg);
 4910 %}
 4911 
 4912 instruct loadConN(rRegN dst, immN src) %{
 4913   match(Set dst src);
 4914 
 4915   ins_cost(125);
 4916   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4917   ins_encode %{
 4918     address con = (address)$src$$constant;
 4919     if (con == NULL) {
 4920       ShouldNotReachHere();
 4921     } else {
 4922       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4923     }
 4924   %}
 4925   ins_pipe(ialu_reg_fat); // XXX
 4926 %}
 4927 
 4928 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4929   match(Set dst src);
 4930 
 4931   ins_cost(125);
 4932   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4933   ins_encode %{
 4934     address con = (address)$src$$constant;
 4935     if (con == NULL) {
 4936       ShouldNotReachHere();
 4937     } else {
 4938       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4939     }
 4940   %}
 4941   ins_pipe(ialu_reg_fat); // XXX
 4942 %}
 4943 
 4944 instruct loadConF0(regF dst, immF0 src)
 4945 %{
 4946   match(Set dst src);
 4947   ins_cost(100);
 4948 
 4949   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4950   ins_encode %{
 4951     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4952   %}
 4953   ins_pipe(pipe_slow);
 4954 %}
 4955 
 4956 // Use the same format since predicate() can not be used here.
 4957 instruct loadConD(regD dst, immD con) %{
 4958   match(Set dst con);
 4959   ins_cost(125);
 4960   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4961   ins_encode %{
 4962     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4963   %}
 4964   ins_pipe(pipe_slow);
 4965 %}
 4966 
 4967 instruct loadConD0(regD dst, immD0 src)
 4968 %{
 4969   match(Set dst src);
 4970   ins_cost(100);
 4971 
 4972   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4973   ins_encode %{
 4974     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4975   %}
 4976   ins_pipe(pipe_slow);
 4977 %}
 4978 
 4979 instruct loadSSI(rRegI dst, stackSlotI src)
 4980 %{
 4981   match(Set dst src);
 4982 
 4983   ins_cost(125);
 4984   format %{ "movl    $dst, $src\t# int stk" %}
 4985   ins_encode %{
 4986     __ movl($dst$$Register, $src$$Address);
 4987   %}
 4988   ins_pipe(ialu_reg_mem);
 4989 %}
 4990 
 4991 instruct loadSSL(rRegL dst, stackSlotL src)
 4992 %{
 4993   match(Set dst src);
 4994 
 4995   ins_cost(125);
 4996   format %{ "movq    $dst, $src\t# long stk" %}
 4997   ins_encode %{
 4998     __ movq($dst$$Register, $src$$Address);
 4999   %}
 5000   ins_pipe(ialu_reg_mem);
 5001 %}
 5002 
 5003 instruct loadSSP(rRegP dst, stackSlotP src)
 5004 %{
 5005   match(Set dst src);
 5006 
 5007   ins_cost(125);
 5008   format %{ "movq    $dst, $src\t# ptr stk" %}
 5009   ins_encode %{
 5010     __ movq($dst$$Register, $src$$Address);
 5011   %}
 5012   ins_pipe(ialu_reg_mem);
 5013 %}
 5014 
 5015 instruct loadSSF(regF dst, stackSlotF src)
 5016 %{
 5017   match(Set dst src);
 5018 
 5019   ins_cost(125);
 5020   format %{ "movss   $dst, $src\t# float stk" %}
 5021   ins_encode %{
 5022     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5023   %}
 5024   ins_pipe(pipe_slow); // XXX
 5025 %}
 5026 
 5027 // Use the same format since predicate() can not be used here.
 5028 instruct loadSSD(regD dst, stackSlotD src)
 5029 %{
 5030   match(Set dst src);
 5031 
 5032   ins_cost(125);
 5033   format %{ "movsd   $dst, $src\t# double stk" %}
 5034   ins_encode  %{
 5035     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5036   %}
 5037   ins_pipe(pipe_slow); // XXX
 5038 %}
 5039 
 5040 // Prefetch instructions for allocation.
 5041 // Must be safe to execute with invalid address (cannot fault).
 5042 
 5043 instruct prefetchAlloc( memory mem ) %{
 5044   predicate(AllocatePrefetchInstr==3);
 5045   match(PrefetchAllocation mem);
 5046   ins_cost(125);
 5047 
 5048   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5049   ins_encode %{
 5050     __ prefetchw($mem$$Address);
 5051   %}
 5052   ins_pipe(ialu_mem);
 5053 %}
 5054 
 5055 instruct prefetchAllocNTA( memory mem ) %{
 5056   predicate(AllocatePrefetchInstr==0);
 5057   match(PrefetchAllocation mem);
 5058   ins_cost(125);
 5059 
 5060   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5061   ins_encode %{
 5062     __ prefetchnta($mem$$Address);
 5063   %}
 5064   ins_pipe(ialu_mem);
 5065 %}
 5066 
 5067 instruct prefetchAllocT0( memory mem ) %{
 5068   predicate(AllocatePrefetchInstr==1);
 5069   match(PrefetchAllocation mem);
 5070   ins_cost(125);
 5071 
 5072   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5073   ins_encode %{
 5074     __ prefetcht0($mem$$Address);
 5075   %}
 5076   ins_pipe(ialu_mem);
 5077 %}
 5078 
 5079 instruct prefetchAllocT2( memory mem ) %{
 5080   predicate(AllocatePrefetchInstr==2);
 5081   match(PrefetchAllocation mem);
 5082   ins_cost(125);
 5083 
 5084   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5085   ins_encode %{
 5086     __ prefetcht2($mem$$Address);
 5087   %}
 5088   ins_pipe(ialu_mem);
 5089 %}
 5090 
 5091 //----------Store Instructions-------------------------------------------------
 5092 
 5093 // Store Byte
 5094 instruct storeB(memory mem, rRegI src)
 5095 %{
 5096   match(Set mem (StoreB mem src));
 5097 
 5098   ins_cost(125); // XXX
 5099   format %{ "movb    $mem, $src\t# byte" %}
 5100   ins_encode %{
 5101     __ movb($mem$$Address, $src$$Register);
 5102   %}
 5103   ins_pipe(ialu_mem_reg);
 5104 %}
 5105 
 5106 // Store Char/Short
 5107 instruct storeC(memory mem, rRegI src)
 5108 %{
 5109   match(Set mem (StoreC mem src));
 5110 
 5111   ins_cost(125); // XXX
 5112   format %{ "movw    $mem, $src\t# char/short" %}
 5113   ins_encode %{
 5114     __ movw($mem$$Address, $src$$Register);
 5115   %}
 5116   ins_pipe(ialu_mem_reg);
 5117 %}
 5118 
 5119 // Store Integer
 5120 instruct storeI(memory mem, rRegI src)
 5121 %{
 5122   match(Set mem (StoreI mem src));
 5123 
 5124   ins_cost(125); // XXX
 5125   format %{ "movl    $mem, $src\t# int" %}
 5126   ins_encode %{
 5127     __ movl($mem$$Address, $src$$Register);
 5128   %}
 5129   ins_pipe(ialu_mem_reg);
 5130 %}
 5131 
 5132 // Store Long
 5133 instruct storeL(memory mem, rRegL src)
 5134 %{
 5135   match(Set mem (StoreL mem src));
 5136 
 5137   ins_cost(125); // XXX
 5138   format %{ "movq    $mem, $src\t# long" %}
 5139   ins_encode %{
 5140     __ movq($mem$$Address, $src$$Register);
 5141   %}
 5142   ins_pipe(ialu_mem_reg); // XXX
 5143 %}
 5144 
 5145 // Store Pointer
 5146 instruct storeP(memory mem, any_RegP src)
 5147 %{
 5148   predicate(n->as_Store()->barrier_data() == 0);
 5149   match(Set mem (StoreP mem src));
 5150 
 5151   ins_cost(125); // XXX
 5152   format %{ "movq    $mem, $src\t# ptr" %}
 5153   ins_encode %{
 5154     __ movq($mem$$Address, $src$$Register);
 5155   %}
 5156   ins_pipe(ialu_mem_reg);
 5157 %}
 5158 
 5159 instruct storeImmP0(memory mem, immP0 zero)
 5160 %{
 5161   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && n->as_Store()->barrier_data() == 0);
 5162   match(Set mem (StoreP mem zero));
 5163 
 5164   ins_cost(125); // XXX
 5165   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5166   ins_encode %{
 5167     __ movq($mem$$Address, r12);
 5168   %}
 5169   ins_pipe(ialu_mem_reg);
 5170 %}
 5171 
 5172 // Store NULL Pointer, mark word, or other simple pointer constant.
 5173 instruct storeImmP(memory mem, immP31 src)
 5174 %{
 5175   predicate(n->as_Store()->barrier_data() == 0);
 5176   match(Set mem (StoreP mem src));
 5177 
 5178   ins_cost(150); // XXX
 5179   format %{ "movq    $mem, $src\t# ptr" %}
 5180   ins_encode %{
 5181     __ movq($mem$$Address, $src$$constant);
 5182   %}
 5183   ins_pipe(ialu_mem_imm);
 5184 %}
 5185 
 5186 // Store Compressed Pointer
 5187 instruct storeN(memory mem, rRegN src)
 5188 %{
 5189   match(Set mem (StoreN mem src));
 5190 
 5191   ins_cost(125); // XXX
 5192   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5193   ins_encode %{
 5194     __ movl($mem$$Address, $src$$Register);
 5195   %}
 5196   ins_pipe(ialu_mem_reg);
 5197 %}
 5198 
 5199 instruct storeNKlass(memory mem, rRegN src)
 5200 %{
 5201   match(Set mem (StoreNKlass mem src));
 5202 
 5203   ins_cost(125); // XXX
 5204   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5205   ins_encode %{
 5206     __ movl($mem$$Address, $src$$Register);
 5207   %}
 5208   ins_pipe(ialu_mem_reg);
 5209 %}
 5210 
 5211 instruct storeImmN0(memory mem, immN0 zero)
 5212 %{
 5213   predicate(CompressedOops::base() == NULL);
 5214   match(Set mem (StoreN mem zero));
 5215 
 5216   ins_cost(125); // XXX
 5217   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5218   ins_encode %{
 5219     __ movl($mem$$Address, r12);
 5220   %}
 5221   ins_pipe(ialu_mem_reg);
 5222 %}
 5223 
 5224 instruct storeImmN(memory mem, immN src)
 5225 %{
 5226   match(Set mem (StoreN mem src));
 5227 
 5228   ins_cost(150); // XXX
 5229   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5230   ins_encode %{
 5231     address con = (address)$src$$constant;
 5232     if (con == NULL) {
 5233       __ movl($mem$$Address, 0);
 5234     } else {
 5235       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5236     }
 5237   %}
 5238   ins_pipe(ialu_mem_imm);
 5239 %}
 5240 
 5241 instruct storeImmNKlass(memory mem, immNKlass src)
 5242 %{
 5243   match(Set mem (StoreNKlass mem src));
 5244 
 5245   ins_cost(150); // XXX
 5246   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5247   ins_encode %{
 5248     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5249   %}
 5250   ins_pipe(ialu_mem_imm);
 5251 %}
 5252 
 5253 // Store Integer Immediate
 5254 instruct storeImmI0(memory mem, immI_0 zero)
 5255 %{
 5256   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5257   match(Set mem (StoreI mem zero));
 5258 
 5259   ins_cost(125); // XXX
 5260   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5261   ins_encode %{
 5262     __ movl($mem$$Address, r12);
 5263   %}
 5264   ins_pipe(ialu_mem_reg);
 5265 %}
 5266 
 5267 instruct storeImmI(memory mem, immI src)
 5268 %{
 5269   match(Set mem (StoreI mem src));
 5270 
 5271   ins_cost(150);
 5272   format %{ "movl    $mem, $src\t# int" %}
 5273   ins_encode %{
 5274     __ movl($mem$$Address, $src$$constant);
 5275   %}
 5276   ins_pipe(ialu_mem_imm);
 5277 %}
 5278 
 5279 // Store Long Immediate
 5280 instruct storeImmL0(memory mem, immL0 zero)
 5281 %{
 5282   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5283   match(Set mem (StoreL mem zero));
 5284 
 5285   ins_cost(125); // XXX
 5286   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5287   ins_encode %{
 5288     __ movq($mem$$Address, r12);
 5289   %}
 5290   ins_pipe(ialu_mem_reg);
 5291 %}
 5292 
 5293 instruct storeImmL(memory mem, immL32 src)
 5294 %{
 5295   match(Set mem (StoreL mem src));
 5296 
 5297   ins_cost(150);
 5298   format %{ "movq    $mem, $src\t# long" %}
 5299   ins_encode %{
 5300     __ movq($mem$$Address, $src$$constant);
 5301   %}
 5302   ins_pipe(ialu_mem_imm);
 5303 %}
 5304 
 5305 // Store Short/Char Immediate
 5306 instruct storeImmC0(memory mem, immI_0 zero)
 5307 %{
 5308   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5309   match(Set mem (StoreC mem zero));
 5310 
 5311   ins_cost(125); // XXX
 5312   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5313   ins_encode %{
 5314     __ movw($mem$$Address, r12);
 5315   %}
 5316   ins_pipe(ialu_mem_reg);
 5317 %}
 5318 
 5319 instruct storeImmI16(memory mem, immI16 src)
 5320 %{
 5321   predicate(UseStoreImmI16);
 5322   match(Set mem (StoreC mem src));
 5323 
 5324   ins_cost(150);
 5325   format %{ "movw    $mem, $src\t# short/char" %}
 5326   ins_encode %{
 5327     __ movw($mem$$Address, $src$$constant);
 5328   %}
 5329   ins_pipe(ialu_mem_imm);
 5330 %}
 5331 
 5332 // Store Byte Immediate
 5333 instruct storeImmB0(memory mem, immI_0 zero)
 5334 %{
 5335   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5336   match(Set mem (StoreB mem zero));
 5337 
 5338   ins_cost(125); // XXX
 5339   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5340   ins_encode %{
 5341     __ movb($mem$$Address, r12);
 5342   %}
 5343   ins_pipe(ialu_mem_reg);
 5344 %}
 5345 
 5346 instruct storeImmB(memory mem, immI8 src)
 5347 %{
 5348   match(Set mem (StoreB mem src));
 5349 
 5350   ins_cost(150); // XXX
 5351   format %{ "movb    $mem, $src\t# byte" %}
 5352   ins_encode %{
 5353     __ movb($mem$$Address, $src$$constant);
 5354   %}
 5355   ins_pipe(ialu_mem_imm);
 5356 %}
 5357 
 5358 // Store CMS card-mark Immediate
 5359 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 5360 %{
 5361   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5362   match(Set mem (StoreCM mem zero));
 5363 
 5364   ins_cost(125); // XXX
 5365   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 5366   ins_encode %{
 5367     __ movb($mem$$Address, r12);
 5368   %}
 5369   ins_pipe(ialu_mem_reg);
 5370 %}
 5371 
 5372 instruct storeImmCM0(memory mem, immI_0 src)
 5373 %{
 5374   match(Set mem (StoreCM mem src));
 5375 
 5376   ins_cost(150); // XXX
 5377   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 5378   ins_encode %{
 5379     __ movb($mem$$Address, $src$$constant);
 5380   %}
 5381   ins_pipe(ialu_mem_imm);
 5382 %}
 5383 
 5384 // Store Float
 5385 instruct storeF(memory mem, regF src)
 5386 %{
 5387   match(Set mem (StoreF mem src));
 5388 
 5389   ins_cost(95); // XXX
 5390   format %{ "movss   $mem, $src\t# float" %}
 5391   ins_encode %{
 5392     __ movflt($mem$$Address, $src$$XMMRegister);
 5393   %}
 5394   ins_pipe(pipe_slow); // XXX
 5395 %}
 5396 
 5397 // Store immediate Float value (it is faster than store from XMM register)
 5398 instruct storeF0(memory mem, immF0 zero)
 5399 %{
 5400   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5401   match(Set mem (StoreF mem zero));
 5402 
 5403   ins_cost(25); // XXX
 5404   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5405   ins_encode %{
 5406     __ movl($mem$$Address, r12);
 5407   %}
 5408   ins_pipe(ialu_mem_reg);
 5409 %}
 5410 
 5411 instruct storeF_imm(memory mem, immF src)
 5412 %{
 5413   match(Set mem (StoreF mem src));
 5414 
 5415   ins_cost(50);
 5416   format %{ "movl    $mem, $src\t# float" %}
 5417   ins_encode %{
 5418     __ movl($mem$$Address, jint_cast($src$$constant));
 5419   %}
 5420   ins_pipe(ialu_mem_imm);
 5421 %}
 5422 
 5423 // Store Double
 5424 instruct storeD(memory mem, regD src)
 5425 %{
 5426   match(Set mem (StoreD mem src));
 5427 
 5428   ins_cost(95); // XXX
 5429   format %{ "movsd   $mem, $src\t# double" %}
 5430   ins_encode %{
 5431     __ movdbl($mem$$Address, $src$$XMMRegister);
 5432   %}
 5433   ins_pipe(pipe_slow); // XXX
 5434 %}
 5435 
 5436 // Store immediate double 0.0 (it is faster than store from XMM register)
 5437 instruct storeD0_imm(memory mem, immD0 src)
 5438 %{
 5439   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 5440   match(Set mem (StoreD mem src));
 5441 
 5442   ins_cost(50);
 5443   format %{ "movq    $mem, $src\t# double 0." %}
 5444   ins_encode %{
 5445     __ movq($mem$$Address, $src$$constant);
 5446   %}
 5447   ins_pipe(ialu_mem_imm);
 5448 %}
 5449 
 5450 instruct storeD0(memory mem, immD0 zero)
 5451 %{
 5452   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5453   match(Set mem (StoreD mem zero));
 5454 
 5455   ins_cost(25); // XXX
 5456   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5457   ins_encode %{
 5458     __ movq($mem$$Address, r12);
 5459   %}
 5460   ins_pipe(ialu_mem_reg);
 5461 %}
 5462 
 5463 instruct storeSSI(stackSlotI dst, rRegI src)
 5464 %{
 5465   match(Set dst src);
 5466 
 5467   ins_cost(100);
 5468   format %{ "movl    $dst, $src\t# int stk" %}
 5469   ins_encode %{
 5470     __ movl($dst$$Address, $src$$Register);
 5471   %}
 5472   ins_pipe( ialu_mem_reg );
 5473 %}
 5474 
 5475 instruct storeSSL(stackSlotL dst, rRegL src)
 5476 %{
 5477   match(Set dst src);
 5478 
 5479   ins_cost(100);
 5480   format %{ "movq    $dst, $src\t# long stk" %}
 5481   ins_encode %{
 5482     __ movq($dst$$Address, $src$$Register);
 5483   %}
 5484   ins_pipe(ialu_mem_reg);
 5485 %}
 5486 
 5487 instruct storeSSP(stackSlotP dst, rRegP src)
 5488 %{
 5489   match(Set dst src);
 5490 
 5491   ins_cost(100);
 5492   format %{ "movq    $dst, $src\t# ptr stk" %}
 5493   ins_encode %{
 5494     __ movq($dst$$Address, $src$$Register);
 5495   %}
 5496   ins_pipe(ialu_mem_reg);
 5497 %}
 5498 
 5499 instruct storeSSF(stackSlotF dst, regF src)
 5500 %{
 5501   match(Set dst src);
 5502 
 5503   ins_cost(95); // XXX
 5504   format %{ "movss   $dst, $src\t# float stk" %}
 5505   ins_encode %{
 5506     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5507   %}
 5508   ins_pipe(pipe_slow); // XXX
 5509 %}
 5510 
 5511 instruct storeSSD(stackSlotD dst, regD src)
 5512 %{
 5513   match(Set dst src);
 5514 
 5515   ins_cost(95); // XXX
 5516   format %{ "movsd   $dst, $src\t# double stk" %}
 5517   ins_encode %{
 5518     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5519   %}
 5520   ins_pipe(pipe_slow); // XXX
 5521 %}
 5522 
 5523 instruct cacheWB(indirect addr)
 5524 %{
 5525   predicate(VM_Version::supports_data_cache_line_flush());
 5526   match(CacheWB addr);
 5527 
 5528   ins_cost(100);
 5529   format %{"cache wb $addr" %}
 5530   ins_encode %{
 5531     assert($addr->index_position() < 0, "should be");
 5532     assert($addr$$disp == 0, "should be");
 5533     __ cache_wb(Address($addr$$base$$Register, 0));
 5534   %}
 5535   ins_pipe(pipe_slow); // XXX
 5536 %}
 5537 
 5538 instruct cacheWBPreSync()
 5539 %{
 5540   predicate(VM_Version::supports_data_cache_line_flush());
 5541   match(CacheWBPreSync);
 5542 
 5543   ins_cost(100);
 5544   format %{"cache wb presync" %}
 5545   ins_encode %{
 5546     __ cache_wbsync(true);
 5547   %}
 5548   ins_pipe(pipe_slow); // XXX
 5549 %}
 5550 
 5551 instruct cacheWBPostSync()
 5552 %{
 5553   predicate(VM_Version::supports_data_cache_line_flush());
 5554   match(CacheWBPostSync);
 5555 
 5556   ins_cost(100);
 5557   format %{"cache wb postsync" %}
 5558   ins_encode %{
 5559     __ cache_wbsync(false);
 5560   %}
 5561   ins_pipe(pipe_slow); // XXX
 5562 %}
 5563 
 5564 //----------BSWAP Instructions-------------------------------------------------
 5565 instruct bytes_reverse_int(rRegI dst) %{
 5566   match(Set dst (ReverseBytesI dst));
 5567 
 5568   format %{ "bswapl  $dst" %}
 5569   ins_encode %{
 5570     __ bswapl($dst$$Register);
 5571   %}
 5572   ins_pipe( ialu_reg );
 5573 %}
 5574 
 5575 instruct bytes_reverse_long(rRegL dst) %{
 5576   match(Set dst (ReverseBytesL dst));
 5577 
 5578   format %{ "bswapq  $dst" %}
 5579   ins_encode %{
 5580     __ bswapq($dst$$Register);
 5581   %}
 5582   ins_pipe( ialu_reg);
 5583 %}
 5584 
 5585 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5586   match(Set dst (ReverseBytesUS dst));
 5587   effect(KILL cr);
 5588 
 5589   format %{ "bswapl  $dst\n\t"
 5590             "shrl    $dst,16\n\t" %}
 5591   ins_encode %{
 5592     __ bswapl($dst$$Register);
 5593     __ shrl($dst$$Register, 16);
 5594   %}
 5595   ins_pipe( ialu_reg );
 5596 %}
 5597 
 5598 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5599   match(Set dst (ReverseBytesS dst));
 5600   effect(KILL cr);
 5601 
 5602   format %{ "bswapl  $dst\n\t"
 5603             "sar     $dst,16\n\t" %}
 5604   ins_encode %{
 5605     __ bswapl($dst$$Register);
 5606     __ sarl($dst$$Register, 16);
 5607   %}
 5608   ins_pipe( ialu_reg );
 5609 %}
 5610 
 5611 //---------- Zeros Count Instructions ------------------------------------------
 5612 
 5613 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5614   predicate(UseCountLeadingZerosInstruction);
 5615   match(Set dst (CountLeadingZerosI src));
 5616   effect(KILL cr);
 5617 
 5618   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5619   ins_encode %{
 5620     __ lzcntl($dst$$Register, $src$$Register);
 5621   %}
 5622   ins_pipe(ialu_reg);
 5623 %}
 5624 
 5625 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5626   predicate(UseCountLeadingZerosInstruction);
 5627   match(Set dst (CountLeadingZerosI (LoadI src)));
 5628   effect(KILL cr);
 5629   ins_cost(175);
 5630   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5631   ins_encode %{
 5632     __ lzcntl($dst$$Register, $src$$Address);
 5633   %}
 5634   ins_pipe(ialu_reg_mem);
 5635 %}
 5636 
 5637 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5638   predicate(!UseCountLeadingZerosInstruction);
 5639   match(Set dst (CountLeadingZerosI src));
 5640   effect(KILL cr);
 5641 
 5642   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5643             "jnz     skip\n\t"
 5644             "movl    $dst, -1\n"
 5645       "skip:\n\t"
 5646             "negl    $dst\n\t"
 5647             "addl    $dst, 31" %}
 5648   ins_encode %{
 5649     Register Rdst = $dst$$Register;
 5650     Register Rsrc = $src$$Register;
 5651     Label skip;
 5652     __ bsrl(Rdst, Rsrc);
 5653     __ jccb(Assembler::notZero, skip);
 5654     __ movl(Rdst, -1);
 5655     __ bind(skip);
 5656     __ negl(Rdst);
 5657     __ addl(Rdst, BitsPerInt - 1);
 5658   %}
 5659   ins_pipe(ialu_reg);
 5660 %}
 5661 
 5662 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5663   predicate(UseCountLeadingZerosInstruction);
 5664   match(Set dst (CountLeadingZerosL src));
 5665   effect(KILL cr);
 5666 
 5667   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5668   ins_encode %{
 5669     __ lzcntq($dst$$Register, $src$$Register);
 5670   %}
 5671   ins_pipe(ialu_reg);
 5672 %}
 5673 
 5674 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5675   predicate(UseCountLeadingZerosInstruction);
 5676   match(Set dst (CountLeadingZerosL (LoadL src)));
 5677   effect(KILL cr);
 5678   ins_cost(175);
 5679   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5680   ins_encode %{
 5681     __ lzcntq($dst$$Register, $src$$Address);
 5682   %}
 5683   ins_pipe(ialu_reg_mem);
 5684 %}
 5685 
 5686 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5687   predicate(!UseCountLeadingZerosInstruction);
 5688   match(Set dst (CountLeadingZerosL src));
 5689   effect(KILL cr);
 5690 
 5691   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5692             "jnz     skip\n\t"
 5693             "movl    $dst, -1\n"
 5694       "skip:\n\t"
 5695             "negl    $dst\n\t"
 5696             "addl    $dst, 63" %}
 5697   ins_encode %{
 5698     Register Rdst = $dst$$Register;
 5699     Register Rsrc = $src$$Register;
 5700     Label skip;
 5701     __ bsrq(Rdst, Rsrc);
 5702     __ jccb(Assembler::notZero, skip);
 5703     __ movl(Rdst, -1);
 5704     __ bind(skip);
 5705     __ negl(Rdst);
 5706     __ addl(Rdst, BitsPerLong - 1);
 5707   %}
 5708   ins_pipe(ialu_reg);
 5709 %}
 5710 
 5711 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5712   predicate(UseCountTrailingZerosInstruction);
 5713   match(Set dst (CountTrailingZerosI src));
 5714   effect(KILL cr);
 5715 
 5716   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5717   ins_encode %{
 5718     __ tzcntl($dst$$Register, $src$$Register);
 5719   %}
 5720   ins_pipe(ialu_reg);
 5721 %}
 5722 
 5723 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5724   predicate(UseCountTrailingZerosInstruction);
 5725   match(Set dst (CountTrailingZerosI (LoadI src)));
 5726   effect(KILL cr);
 5727   ins_cost(175);
 5728   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5729   ins_encode %{
 5730     __ tzcntl($dst$$Register, $src$$Address);
 5731   %}
 5732   ins_pipe(ialu_reg_mem);
 5733 %}
 5734 
 5735 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5736   predicate(!UseCountTrailingZerosInstruction);
 5737   match(Set dst (CountTrailingZerosI src));
 5738   effect(KILL cr);
 5739 
 5740   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5741             "jnz     done\n\t"
 5742             "movl    $dst, 32\n"
 5743       "done:" %}
 5744   ins_encode %{
 5745     Register Rdst = $dst$$Register;
 5746     Label done;
 5747     __ bsfl(Rdst, $src$$Register);
 5748     __ jccb(Assembler::notZero, done);
 5749     __ movl(Rdst, BitsPerInt);
 5750     __ bind(done);
 5751   %}
 5752   ins_pipe(ialu_reg);
 5753 %}
 5754 
 5755 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5756   predicate(UseCountTrailingZerosInstruction);
 5757   match(Set dst (CountTrailingZerosL src));
 5758   effect(KILL cr);
 5759 
 5760   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5761   ins_encode %{
 5762     __ tzcntq($dst$$Register, $src$$Register);
 5763   %}
 5764   ins_pipe(ialu_reg);
 5765 %}
 5766 
 5767 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5768   predicate(UseCountTrailingZerosInstruction);
 5769   match(Set dst (CountTrailingZerosL (LoadL src)));
 5770   effect(KILL cr);
 5771   ins_cost(175);
 5772   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5773   ins_encode %{
 5774     __ tzcntq($dst$$Register, $src$$Address);
 5775   %}
 5776   ins_pipe(ialu_reg_mem);
 5777 %}
 5778 
 5779 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5780   predicate(!UseCountTrailingZerosInstruction);
 5781   match(Set dst (CountTrailingZerosL src));
 5782   effect(KILL cr);
 5783 
 5784   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5785             "jnz     done\n\t"
 5786             "movl    $dst, 64\n"
 5787       "done:" %}
 5788   ins_encode %{
 5789     Register Rdst = $dst$$Register;
 5790     Label done;
 5791     __ bsfq(Rdst, $src$$Register);
 5792     __ jccb(Assembler::notZero, done);
 5793     __ movl(Rdst, BitsPerLong);
 5794     __ bind(done);
 5795   %}
 5796   ins_pipe(ialu_reg);
 5797 %}
 5798 
 5799 //--------------- Reverse Operation Instructions ----------------
 5800 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5801   predicate(!VM_Version::supports_gfni());
 5802   match(Set dst (ReverseI src));
 5803   effect(TEMP dst, TEMP rtmp, KILL cr);
 5804   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5805   ins_encode %{
 5806     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5807   %}
 5808   ins_pipe( ialu_reg );
 5809 %}
 5810 
 5811 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5812   predicate(VM_Version::supports_gfni());
 5813   match(Set dst (ReverseI src));
 5814   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5815   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5816   ins_encode %{
 5817     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5818   %}
 5819   ins_pipe( ialu_reg );
 5820 %}
 5821 
 5822 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5823   predicate(!VM_Version::supports_gfni());
 5824   match(Set dst (ReverseL src));
 5825   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5826   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5827   ins_encode %{
 5828     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5829   %}
 5830   ins_pipe( ialu_reg );
 5831 %}
 5832 
 5833 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5834   predicate(VM_Version::supports_gfni());
 5835   match(Set dst (ReverseL src));
 5836   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5837   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5838   ins_encode %{
 5839     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5840   %}
 5841   ins_pipe( ialu_reg );
 5842 %}
 5843 
 5844 //---------- Population Count Instructions -------------------------------------
 5845 
 5846 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5847   predicate(UsePopCountInstruction);
 5848   match(Set dst (PopCountI src));
 5849   effect(KILL cr);
 5850 
 5851   format %{ "popcnt  $dst, $src" %}
 5852   ins_encode %{
 5853     __ popcntl($dst$$Register, $src$$Register);
 5854   %}
 5855   ins_pipe(ialu_reg);
 5856 %}
 5857 
 5858 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5859   predicate(UsePopCountInstruction);
 5860   match(Set dst (PopCountI (LoadI mem)));
 5861   effect(KILL cr);
 5862 
 5863   format %{ "popcnt  $dst, $mem" %}
 5864   ins_encode %{
 5865     __ popcntl($dst$$Register, $mem$$Address);
 5866   %}
 5867   ins_pipe(ialu_reg);
 5868 %}
 5869 
 5870 // Note: Long.bitCount(long) returns an int.
 5871 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5872   predicate(UsePopCountInstruction);
 5873   match(Set dst (PopCountL src));
 5874   effect(KILL cr);
 5875 
 5876   format %{ "popcnt  $dst, $src" %}
 5877   ins_encode %{
 5878     __ popcntq($dst$$Register, $src$$Register);
 5879   %}
 5880   ins_pipe(ialu_reg);
 5881 %}
 5882 
 5883 // Note: Long.bitCount(long) returns an int.
 5884 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5885   predicate(UsePopCountInstruction);
 5886   match(Set dst (PopCountL (LoadL mem)));
 5887   effect(KILL cr);
 5888 
 5889   format %{ "popcnt  $dst, $mem" %}
 5890   ins_encode %{
 5891     __ popcntq($dst$$Register, $mem$$Address);
 5892   %}
 5893   ins_pipe(ialu_reg);
 5894 %}
 5895 
 5896 
 5897 //----------MemBar Instructions-----------------------------------------------
 5898 // Memory barrier flavors
 5899 
 5900 instruct membar_acquire()
 5901 %{
 5902   match(MemBarAcquire);
 5903   match(LoadFence);
 5904   ins_cost(0);
 5905 
 5906   size(0);
 5907   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5908   ins_encode();
 5909   ins_pipe(empty);
 5910 %}
 5911 
 5912 instruct membar_acquire_lock()
 5913 %{
 5914   match(MemBarAcquireLock);
 5915   ins_cost(0);
 5916 
 5917   size(0);
 5918   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5919   ins_encode();
 5920   ins_pipe(empty);
 5921 %}
 5922 
 5923 instruct membar_release()
 5924 %{
 5925   match(MemBarRelease);
 5926   match(StoreFence);
 5927   ins_cost(0);
 5928 
 5929   size(0);
 5930   format %{ "MEMBAR-release ! (empty encoding)" %}
 5931   ins_encode();
 5932   ins_pipe(empty);
 5933 %}
 5934 
 5935 instruct membar_release_lock()
 5936 %{
 5937   match(MemBarReleaseLock);
 5938   ins_cost(0);
 5939 
 5940   size(0);
 5941   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5942   ins_encode();
 5943   ins_pipe(empty);
 5944 %}
 5945 
 5946 instruct membar_volatile(rFlagsReg cr) %{
 5947   match(MemBarVolatile);
 5948   effect(KILL cr);
 5949   ins_cost(400);
 5950 
 5951   format %{
 5952     $$template
 5953     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5954   %}
 5955   ins_encode %{
 5956     __ membar(Assembler::StoreLoad);
 5957   %}
 5958   ins_pipe(pipe_slow);
 5959 %}
 5960 
 5961 instruct unnecessary_membar_volatile()
 5962 %{
 5963   match(MemBarVolatile);
 5964   predicate(Matcher::post_store_load_barrier(n));
 5965   ins_cost(0);
 5966 
 5967   size(0);
 5968   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5969   ins_encode();
 5970   ins_pipe(empty);
 5971 %}
 5972 
 5973 instruct membar_storestore() %{
 5974   match(MemBarStoreStore);
 5975   match(StoreStoreFence);
 5976   ins_cost(0);
 5977 
 5978   size(0);
 5979   format %{ "MEMBAR-storestore (empty encoding)" %}
 5980   ins_encode( );
 5981   ins_pipe(empty);
 5982 %}
 5983 
 5984 //----------Move Instructions--------------------------------------------------
 5985 
 5986 instruct castX2P(rRegP dst, rRegL src)
 5987 %{
 5988   match(Set dst (CastX2P src));
 5989 
 5990   format %{ "movq    $dst, $src\t# long->ptr" %}
 5991   ins_encode %{
 5992     if ($dst$$reg != $src$$reg) {
 5993       __ movptr($dst$$Register, $src$$Register);
 5994     }
 5995   %}
 5996   ins_pipe(ialu_reg_reg); // XXX
 5997 %}
 5998 
 5999 instruct castP2X(rRegL dst, rRegP src)
 6000 %{
 6001   match(Set dst (CastP2X src));
 6002 
 6003   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6004   ins_encode %{
 6005     if ($dst$$reg != $src$$reg) {
 6006       __ movptr($dst$$Register, $src$$Register);
 6007     }
 6008   %}
 6009   ins_pipe(ialu_reg_reg); // XXX
 6010 %}
 6011 
 6012 // Convert oop into int for vectors alignment masking
 6013 instruct convP2I(rRegI dst, rRegP src)
 6014 %{
 6015   match(Set dst (ConvL2I (CastP2X src)));
 6016 
 6017   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6018   ins_encode %{
 6019     __ movl($dst$$Register, $src$$Register);
 6020   %}
 6021   ins_pipe(ialu_reg_reg); // XXX
 6022 %}
 6023 
 6024 // Convert compressed oop into int for vectors alignment masking
 6025 // in case of 32bit oops (heap < 4Gb).
 6026 instruct convN2I(rRegI dst, rRegN src)
 6027 %{
 6028   predicate(CompressedOops::shift() == 0);
 6029   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6030 
 6031   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6032   ins_encode %{
 6033     __ movl($dst$$Register, $src$$Register);
 6034   %}
 6035   ins_pipe(ialu_reg_reg); // XXX
 6036 %}
 6037 
 6038 // Convert oop pointer into compressed form
 6039 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6040   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6041   match(Set dst (EncodeP src));
 6042   effect(KILL cr);
 6043   format %{ "encode_heap_oop $dst,$src" %}
 6044   ins_encode %{
 6045     Register s = $src$$Register;
 6046     Register d = $dst$$Register;
 6047     if (s != d) {
 6048       __ movq(d, s);
 6049     }
 6050     __ encode_heap_oop(d);
 6051   %}
 6052   ins_pipe(ialu_reg_long);
 6053 %}
 6054 
 6055 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6056   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6057   match(Set dst (EncodeP src));
 6058   effect(KILL cr);
 6059   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6060   ins_encode %{
 6061     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6062   %}
 6063   ins_pipe(ialu_reg_long);
 6064 %}
 6065 
 6066 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6067   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6068             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6069   match(Set dst (DecodeN src));
 6070   effect(KILL cr);
 6071   format %{ "decode_heap_oop $dst,$src" %}
 6072   ins_encode %{
 6073     Register s = $src$$Register;
 6074     Register d = $dst$$Register;
 6075     if (s != d) {
 6076       __ movq(d, s);
 6077     }
 6078     __ decode_heap_oop(d);
 6079   %}
 6080   ins_pipe(ialu_reg_long);
 6081 %}
 6082 
 6083 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6084   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6085             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6086   match(Set dst (DecodeN src));
 6087   effect(KILL cr);
 6088   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6089   ins_encode %{
 6090     Register s = $src$$Register;
 6091     Register d = $dst$$Register;
 6092     if (s != d) {
 6093       __ decode_heap_oop_not_null(d, s);
 6094     } else {
 6095       __ decode_heap_oop_not_null(d);
 6096     }
 6097   %}
 6098   ins_pipe(ialu_reg_long);
 6099 %}
 6100 
 6101 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6102   match(Set dst (EncodePKlass src));
 6103   effect(TEMP dst, KILL cr);
 6104   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6105   ins_encode %{
 6106     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6107   %}
 6108   ins_pipe(ialu_reg_long);
 6109 %}
 6110 
 6111 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6112   match(Set dst (DecodeNKlass src));
 6113   effect(TEMP dst, KILL cr);
 6114   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6115   ins_encode %{
 6116     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6117   %}
 6118   ins_pipe(ialu_reg_long);
 6119 %}
 6120 
 6121 //----------Conditional Move---------------------------------------------------
 6122 // Jump
 6123 // dummy instruction for generating temp registers
 6124 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6125   match(Jump (LShiftL switch_val shift));
 6126   ins_cost(350);
 6127   predicate(false);
 6128   effect(TEMP dest);
 6129 
 6130   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6131             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6132   ins_encode %{
 6133     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6134     // to do that and the compiler is using that register as one it can allocate.
 6135     // So we build it all by hand.
 6136     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6137     // ArrayAddress dispatch(table, index);
 6138     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6139     __ lea($dest$$Register, $constantaddress);
 6140     __ jmp(dispatch);
 6141   %}
 6142   ins_pipe(pipe_jmp);
 6143 %}
 6144 
 6145 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6146   match(Jump (AddL (LShiftL switch_val shift) offset));
 6147   ins_cost(350);
 6148   effect(TEMP dest);
 6149 
 6150   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6151             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6152   ins_encode %{
 6153     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6154     // to do that and the compiler is using that register as one it can allocate.
 6155     // So we build it all by hand.
 6156     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6157     // ArrayAddress dispatch(table, index);
 6158     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6159     __ lea($dest$$Register, $constantaddress);
 6160     __ jmp(dispatch);
 6161   %}
 6162   ins_pipe(pipe_jmp);
 6163 %}
 6164 
 6165 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6166   match(Jump switch_val);
 6167   ins_cost(350);
 6168   effect(TEMP dest);
 6169 
 6170   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6171             "jmp     [$dest + $switch_val]\n\t" %}
 6172   ins_encode %{
 6173     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6174     // to do that and the compiler is using that register as one it can allocate.
 6175     // So we build it all by hand.
 6176     // Address index(noreg, switch_reg, Address::times_1);
 6177     // ArrayAddress dispatch(table, index);
 6178     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6179     __ lea($dest$$Register, $constantaddress);
 6180     __ jmp(dispatch);
 6181   %}
 6182   ins_pipe(pipe_jmp);
 6183 %}
 6184 
 6185 // Conditional move
 6186 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6187 %{
 6188   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6189   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6190 
 6191   ins_cost(100); // XXX
 6192   format %{ "setbn$cop $dst\t# signed, int" %}
 6193   ins_encode %{
 6194     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6195     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6196   %}
 6197   ins_pipe(ialu_reg);
 6198 %}
 6199 
 6200 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6201 %{
 6202   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6203 
 6204   ins_cost(200); // XXX
 6205   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6206   ins_encode %{
 6207     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6208   %}
 6209   ins_pipe(pipe_cmov_reg);
 6210 %}
 6211 
 6212 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6213 %{
 6214   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6215   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6216 
 6217   ins_cost(100); // XXX
 6218   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6219   ins_encode %{
 6220     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6221     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6222   %}
 6223   ins_pipe(ialu_reg);
 6224 %}
 6225 
 6226 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6227   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6228 
 6229   ins_cost(200); // XXX
 6230   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6231   ins_encode %{
 6232     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6233   %}
 6234   ins_pipe(pipe_cmov_reg);
 6235 %}
 6236 
 6237 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6238 %{
 6239   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6240   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6241 
 6242   ins_cost(100); // XXX
 6243   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6244   ins_encode %{
 6245     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6246     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6247   %}
 6248   ins_pipe(ialu_reg);
 6249 %}
 6250 
 6251 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6252   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6253   ins_cost(200);
 6254   expand %{
 6255     cmovI_regU(cop, cr, dst, src);
 6256   %}
 6257 %}
 6258 
 6259 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6260   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6261   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6262 
 6263   ins_cost(200); // XXX
 6264   format %{ "cmovpl  $dst, $src\n\t"
 6265             "cmovnel $dst, $src" %}
 6266   ins_encode %{
 6267     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6268     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6269   %}
 6270   ins_pipe(pipe_cmov_reg);
 6271 %}
 6272 
 6273 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6274 // inputs of the CMove
 6275 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6276   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6277   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6278 
 6279   ins_cost(200); // XXX
 6280   format %{ "cmovpl  $dst, $src\n\t"
 6281             "cmovnel $dst, $src" %}
 6282   ins_encode %{
 6283     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6284     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6285   %}
 6286   ins_pipe(pipe_cmov_reg);
 6287 %}
 6288 
 6289 // Conditional move
 6290 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6291   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6292 
 6293   ins_cost(250); // XXX
 6294   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6295   ins_encode %{
 6296     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6297   %}
 6298   ins_pipe(pipe_cmov_mem);
 6299 %}
 6300 
 6301 // Conditional move
 6302 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6303 %{
 6304   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6305 
 6306   ins_cost(250); // XXX
 6307   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6308   ins_encode %{
 6309     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6310   %}
 6311   ins_pipe(pipe_cmov_mem);
 6312 %}
 6313 
 6314 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6315   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6316   ins_cost(250);
 6317   expand %{
 6318     cmovI_memU(cop, cr, dst, src);
 6319   %}
 6320 %}
 6321 
 6322 // Conditional move
 6323 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6324 %{
 6325   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6326 
 6327   ins_cost(200); // XXX
 6328   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6329   ins_encode %{
 6330     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6331   %}
 6332   ins_pipe(pipe_cmov_reg);
 6333 %}
 6334 
 6335 // Conditional move
 6336 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6337 %{
 6338   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6339 
 6340   ins_cost(200); // XXX
 6341   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6342   ins_encode %{
 6343     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6344   %}
 6345   ins_pipe(pipe_cmov_reg);
 6346 %}
 6347 
 6348 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6349   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6350   ins_cost(200);
 6351   expand %{
 6352     cmovN_regU(cop, cr, dst, src);
 6353   %}
 6354 %}
 6355 
 6356 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6357   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6358   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6359 
 6360   ins_cost(200); // XXX
 6361   format %{ "cmovpl  $dst, $src\n\t"
 6362             "cmovnel $dst, $src" %}
 6363   ins_encode %{
 6364     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6365     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6366   %}
 6367   ins_pipe(pipe_cmov_reg);
 6368 %}
 6369 
 6370 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6371 // inputs of the CMove
 6372 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6373   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6374   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6375 
 6376   ins_cost(200); // XXX
 6377   format %{ "cmovpl  $dst, $src\n\t"
 6378             "cmovnel $dst, $src" %}
 6379   ins_encode %{
 6380     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6381     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6382   %}
 6383   ins_pipe(pipe_cmov_reg);
 6384 %}
 6385 
 6386 // Conditional move
 6387 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6388 %{
 6389   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6390 
 6391   ins_cost(200); // XXX
 6392   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6393   ins_encode %{
 6394     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6395   %}
 6396   ins_pipe(pipe_cmov_reg);  // XXX
 6397 %}
 6398 
 6399 // Conditional move
 6400 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6401 %{
 6402   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6403 
 6404   ins_cost(200); // XXX
 6405   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6406   ins_encode %{
 6407     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6408   %}
 6409   ins_pipe(pipe_cmov_reg); // XXX
 6410 %}
 6411 
 6412 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6413   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6414   ins_cost(200);
 6415   expand %{
 6416     cmovP_regU(cop, cr, dst, src);
 6417   %}
 6418 %}
 6419 
 6420 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6421   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6422   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6423 
 6424   ins_cost(200); // XXX
 6425   format %{ "cmovpq  $dst, $src\n\t"
 6426             "cmovneq $dst, $src" %}
 6427   ins_encode %{
 6428     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6429     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6430   %}
 6431   ins_pipe(pipe_cmov_reg);
 6432 %}
 6433 
 6434 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6435 // inputs of the CMove
 6436 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6437   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6438   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6439 
 6440   ins_cost(200); // XXX
 6441   format %{ "cmovpq  $dst, $src\n\t"
 6442             "cmovneq $dst, $src" %}
 6443   ins_encode %{
 6444     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6445     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6446   %}
 6447   ins_pipe(pipe_cmov_reg);
 6448 %}
 6449 
 6450 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6451 %{
 6452   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6453   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6454 
 6455   ins_cost(100); // XXX
 6456   format %{ "setbn$cop $dst\t# signed, long" %}
 6457   ins_encode %{
 6458     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6459     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6460   %}
 6461   ins_pipe(ialu_reg);
 6462 %}
 6463 
 6464 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6465 %{
 6466   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6467 
 6468   ins_cost(200); // XXX
 6469   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6470   ins_encode %{
 6471     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6472   %}
 6473   ins_pipe(pipe_cmov_reg);  // XXX
 6474 %}
 6475 
 6476 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6477 %{
 6478   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6479 
 6480   ins_cost(200); // XXX
 6481   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6482   ins_encode %{
 6483     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6484   %}
 6485   ins_pipe(pipe_cmov_mem);  // XXX
 6486 %}
 6487 
 6488 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6489 %{
 6490   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6491   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6492 
 6493   ins_cost(100); // XXX
 6494   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6495   ins_encode %{
 6496     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6497     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6498   %}
 6499   ins_pipe(ialu_reg);
 6500 %}
 6501 
 6502 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6503 %{
 6504   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6505 
 6506   ins_cost(200); // XXX
 6507   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6508   ins_encode %{
 6509     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6510   %}
 6511   ins_pipe(pipe_cmov_reg); // XXX
 6512 %}
 6513 
 6514 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6515 %{
 6516   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6517   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6518 
 6519   ins_cost(100); // XXX
 6520   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6521   ins_encode %{
 6522     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6523     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6524   %}
 6525   ins_pipe(ialu_reg);
 6526 %}
 6527 
 6528 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6529   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6530   ins_cost(200);
 6531   expand %{
 6532     cmovL_regU(cop, cr, dst, src);
 6533   %}
 6534 %}
 6535 
 6536 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6537   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6538   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6539 
 6540   ins_cost(200); // XXX
 6541   format %{ "cmovpq  $dst, $src\n\t"
 6542             "cmovneq $dst, $src" %}
 6543   ins_encode %{
 6544     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6545     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6546   %}
 6547   ins_pipe(pipe_cmov_reg);
 6548 %}
 6549 
 6550 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6551 // inputs of the CMove
 6552 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6553   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6554   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6555 
 6556   ins_cost(200); // XXX
 6557   format %{ "cmovpq  $dst, $src\n\t"
 6558             "cmovneq $dst, $src" %}
 6559   ins_encode %{
 6560     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6561     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6562   %}
 6563   ins_pipe(pipe_cmov_reg);
 6564 %}
 6565 
 6566 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6567 %{
 6568   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6569 
 6570   ins_cost(200); // XXX
 6571   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6572   ins_encode %{
 6573     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6574   %}
 6575   ins_pipe(pipe_cmov_mem); // XXX
 6576 %}
 6577 
 6578 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6579   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6580   ins_cost(200);
 6581   expand %{
 6582     cmovL_memU(cop, cr, dst, src);
 6583   %}
 6584 %}
 6585 
 6586 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6587 %{
 6588   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6589 
 6590   ins_cost(200); // XXX
 6591   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6592             "movss     $dst, $src\n"
 6593     "skip:" %}
 6594   ins_encode %{
 6595     Label Lskip;
 6596     // Invert sense of branch from sense of CMOV
 6597     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6598     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6599     __ bind(Lskip);
 6600   %}
 6601   ins_pipe(pipe_slow);
 6602 %}
 6603 
 6604 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6605 %{
 6606   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6607 
 6608   ins_cost(200); // XXX
 6609   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6610             "movss     $dst, $src\n"
 6611     "skip:" %}
 6612   ins_encode %{
 6613     Label Lskip;
 6614     // Invert sense of branch from sense of CMOV
 6615     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6616     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6617     __ bind(Lskip);
 6618   %}
 6619   ins_pipe(pipe_slow);
 6620 %}
 6621 
 6622 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6623   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6624   ins_cost(200);
 6625   expand %{
 6626     cmovF_regU(cop, cr, dst, src);
 6627   %}
 6628 %}
 6629 
 6630 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6631 %{
 6632   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6633 
 6634   ins_cost(200); // XXX
 6635   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6636             "movsd     $dst, $src\n"
 6637     "skip:" %}
 6638   ins_encode %{
 6639     Label Lskip;
 6640     // Invert sense of branch from sense of CMOV
 6641     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6642     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6643     __ bind(Lskip);
 6644   %}
 6645   ins_pipe(pipe_slow);
 6646 %}
 6647 
 6648 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6649 %{
 6650   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6651 
 6652   ins_cost(200); // XXX
 6653   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6654             "movsd     $dst, $src\n"
 6655     "skip:" %}
 6656   ins_encode %{
 6657     Label Lskip;
 6658     // Invert sense of branch from sense of CMOV
 6659     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6660     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6661     __ bind(Lskip);
 6662   %}
 6663   ins_pipe(pipe_slow);
 6664 %}
 6665 
 6666 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6667   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6668   ins_cost(200);
 6669   expand %{
 6670     cmovD_regU(cop, cr, dst, src);
 6671   %}
 6672 %}
 6673 
 6674 //----------Arithmetic Instructions--------------------------------------------
 6675 //----------Addition Instructions----------------------------------------------
 6676 
 6677 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6678 %{
 6679   match(Set dst (AddI dst src));
 6680   effect(KILL cr);
 6681   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6682   format %{ "addl    $dst, $src\t# int" %}
 6683   ins_encode %{
 6684     __ addl($dst$$Register, $src$$Register);
 6685   %}
 6686   ins_pipe(ialu_reg_reg);
 6687 %}
 6688 
 6689 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 6690 %{
 6691   match(Set dst (AddI dst src));
 6692   effect(KILL cr);
 6693   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6694 
 6695   format %{ "addl    $dst, $src\t# int" %}
 6696   ins_encode %{
 6697     __ addl($dst$$Register, $src$$constant);
 6698   %}
 6699   ins_pipe( ialu_reg );
 6700 %}
 6701 
 6702 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 6703 %{
 6704   match(Set dst (AddI dst (LoadI src)));
 6705   effect(KILL cr);
 6706   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6707 
 6708   ins_cost(150); // XXX
 6709   format %{ "addl    $dst, $src\t# int" %}
 6710   ins_encode %{
 6711     __ addl($dst$$Register, $src$$Address);
 6712   %}
 6713   ins_pipe(ialu_reg_mem);
 6714 %}
 6715 
 6716 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 6717 %{
 6718   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6719   effect(KILL cr);
 6720   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6721 
 6722   ins_cost(150); // XXX
 6723   format %{ "addl    $dst, $src\t# int" %}
 6724   ins_encode %{
 6725     __ addl($dst$$Address, $src$$Register);
 6726   %}
 6727   ins_pipe(ialu_mem_reg);
 6728 %}
 6729 
 6730 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 6731 %{
 6732   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6733   effect(KILL cr);
 6734   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6735 
 6736 
 6737   ins_cost(125); // XXX
 6738   format %{ "addl    $dst, $src\t# int" %}
 6739   ins_encode %{
 6740     __ addl($dst$$Address, $src$$constant);
 6741   %}
 6742   ins_pipe(ialu_mem_imm);
 6743 %}
 6744 
 6745 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 6746 %{
 6747   predicate(UseIncDec);
 6748   match(Set dst (AddI dst src));
 6749   effect(KILL cr);
 6750 
 6751   format %{ "incl    $dst\t# int" %}
 6752   ins_encode %{
 6753     __ incrementl($dst$$Register);
 6754   %}
 6755   ins_pipe(ialu_reg);
 6756 %}
 6757 
 6758 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 6759 %{
 6760   predicate(UseIncDec);
 6761   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6762   effect(KILL cr);
 6763 
 6764   ins_cost(125); // XXX
 6765   format %{ "incl    $dst\t# int" %}
 6766   ins_encode %{
 6767     __ incrementl($dst$$Address);
 6768   %}
 6769   ins_pipe(ialu_mem_imm);
 6770 %}
 6771 
 6772 // XXX why does that use AddI
 6773 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 6774 %{
 6775   predicate(UseIncDec);
 6776   match(Set dst (AddI dst src));
 6777   effect(KILL cr);
 6778 
 6779   format %{ "decl    $dst\t# int" %}
 6780   ins_encode %{
 6781     __ decrementl($dst$$Register);
 6782   %}
 6783   ins_pipe(ialu_reg);
 6784 %}
 6785 
 6786 // XXX why does that use AddI
 6787 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 6788 %{
 6789   predicate(UseIncDec);
 6790   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6791   effect(KILL cr);
 6792 
 6793   ins_cost(125); // XXX
 6794   format %{ "decl    $dst\t# int" %}
 6795   ins_encode %{
 6796     __ decrementl($dst$$Address);
 6797   %}
 6798   ins_pipe(ialu_mem_imm);
 6799 %}
 6800 
 6801 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 6802 %{
 6803   predicate(VM_Version::supports_fast_2op_lea());
 6804   match(Set dst (AddI (LShiftI index scale) disp));
 6805 
 6806   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 6807   ins_encode %{
 6808     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6809     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6810   %}
 6811   ins_pipe(ialu_reg_reg);
 6812 %}
 6813 
 6814 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 6815 %{
 6816   predicate(VM_Version::supports_fast_3op_lea());
 6817   match(Set dst (AddI (AddI base index) disp));
 6818 
 6819   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 6820   ins_encode %{
 6821     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6822   %}
 6823   ins_pipe(ialu_reg_reg);
 6824 %}
 6825 
 6826 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 6827 %{
 6828   predicate(VM_Version::supports_fast_2op_lea());
 6829   match(Set dst (AddI base (LShiftI index scale)));
 6830 
 6831   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 6832   ins_encode %{
 6833     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6834     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6835   %}
 6836   ins_pipe(ialu_reg_reg);
 6837 %}
 6838 
 6839 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 6840 %{
 6841   predicate(VM_Version::supports_fast_3op_lea());
 6842   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 6843 
 6844   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 6845   ins_encode %{
 6846     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6847     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6848   %}
 6849   ins_pipe(ialu_reg_reg);
 6850 %}
 6851 
 6852 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 6853 %{
 6854   match(Set dst (AddL dst src));
 6855   effect(KILL cr);
 6856   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6857 
 6858   format %{ "addq    $dst, $src\t# long" %}
 6859   ins_encode %{
 6860     __ addq($dst$$Register, $src$$Register);
 6861   %}
 6862   ins_pipe(ialu_reg_reg);
 6863 %}
 6864 
 6865 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 6866 %{
 6867   match(Set dst (AddL dst src));
 6868   effect(KILL cr);
 6869   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6870 
 6871   format %{ "addq    $dst, $src\t# long" %}
 6872   ins_encode %{
 6873     __ addq($dst$$Register, $src$$constant);
 6874   %}
 6875   ins_pipe( ialu_reg );
 6876 %}
 6877 
 6878 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 6879 %{
 6880   match(Set dst (AddL dst (LoadL src)));
 6881   effect(KILL cr);
 6882   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6883 
 6884   ins_cost(150); // XXX
 6885   format %{ "addq    $dst, $src\t# long" %}
 6886   ins_encode %{
 6887     __ addq($dst$$Register, $src$$Address);
 6888   %}
 6889   ins_pipe(ialu_reg_mem);
 6890 %}
 6891 
 6892 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 6893 %{
 6894   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6895   effect(KILL cr);
 6896   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6897 
 6898   ins_cost(150); // XXX
 6899   format %{ "addq    $dst, $src\t# long" %}
 6900   ins_encode %{
 6901     __ addq($dst$$Address, $src$$Register);
 6902   %}
 6903   ins_pipe(ialu_mem_reg);
 6904 %}
 6905 
 6906 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 6907 %{
 6908   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6909   effect(KILL cr);
 6910   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6911 
 6912   ins_cost(125); // XXX
 6913   format %{ "addq    $dst, $src\t# long" %}
 6914   ins_encode %{
 6915     __ addq($dst$$Address, $src$$constant);
 6916   %}
 6917   ins_pipe(ialu_mem_imm);
 6918 %}
 6919 
 6920 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 6921 %{
 6922   predicate(UseIncDec);
 6923   match(Set dst (AddL dst src));
 6924   effect(KILL cr);
 6925 
 6926   format %{ "incq    $dst\t# long" %}
 6927   ins_encode %{
 6928     __ incrementq($dst$$Register);
 6929   %}
 6930   ins_pipe(ialu_reg);
 6931 %}
 6932 
 6933 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 6934 %{
 6935   predicate(UseIncDec);
 6936   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6937   effect(KILL cr);
 6938 
 6939   ins_cost(125); // XXX
 6940   format %{ "incq    $dst\t# long" %}
 6941   ins_encode %{
 6942     __ incrementq($dst$$Address);
 6943   %}
 6944   ins_pipe(ialu_mem_imm);
 6945 %}
 6946 
 6947 // XXX why does that use AddL
 6948 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 6949 %{
 6950   predicate(UseIncDec);
 6951   match(Set dst (AddL dst src));
 6952   effect(KILL cr);
 6953 
 6954   format %{ "decq    $dst\t# long" %}
 6955   ins_encode %{
 6956     __ decrementq($dst$$Register);
 6957   %}
 6958   ins_pipe(ialu_reg);
 6959 %}
 6960 
 6961 // XXX why does that use AddL
 6962 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 6963 %{
 6964   predicate(UseIncDec);
 6965   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6966   effect(KILL cr);
 6967 
 6968   ins_cost(125); // XXX
 6969   format %{ "decq    $dst\t# long" %}
 6970   ins_encode %{
 6971     __ decrementq($dst$$Address);
 6972   %}
 6973   ins_pipe(ialu_mem_imm);
 6974 %}
 6975 
 6976 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 6977 %{
 6978   predicate(VM_Version::supports_fast_2op_lea());
 6979   match(Set dst (AddL (LShiftL index scale) disp));
 6980 
 6981   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 6982   ins_encode %{
 6983     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6984     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6985   %}
 6986   ins_pipe(ialu_reg_reg);
 6987 %}
 6988 
 6989 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 6990 %{
 6991   predicate(VM_Version::supports_fast_3op_lea());
 6992   match(Set dst (AddL (AddL base index) disp));
 6993 
 6994   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 6995   ins_encode %{
 6996     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6997   %}
 6998   ins_pipe(ialu_reg_reg);
 6999 %}
 7000 
 7001 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7002 %{
 7003   predicate(VM_Version::supports_fast_2op_lea());
 7004   match(Set dst (AddL base (LShiftL index scale)));
 7005 
 7006   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7007   ins_encode %{
 7008     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7009     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7010   %}
 7011   ins_pipe(ialu_reg_reg);
 7012 %}
 7013 
 7014 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7015 %{
 7016   predicate(VM_Version::supports_fast_3op_lea());
 7017   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7018 
 7019   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7020   ins_encode %{
 7021     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7022     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7023   %}
 7024   ins_pipe(ialu_reg_reg);
 7025 %}
 7026 
 7027 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7028 %{
 7029   match(Set dst (AddP dst src));
 7030   effect(KILL cr);
 7031   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7032 
 7033   format %{ "addq    $dst, $src\t# ptr" %}
 7034   ins_encode %{
 7035     __ addq($dst$$Register, $src$$Register);
 7036   %}
 7037   ins_pipe(ialu_reg_reg);
 7038 %}
 7039 
 7040 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7041 %{
 7042   match(Set dst (AddP dst src));
 7043   effect(KILL cr);
 7044   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7045 
 7046   format %{ "addq    $dst, $src\t# ptr" %}
 7047   ins_encode %{
 7048     __ addq($dst$$Register, $src$$constant);
 7049   %}
 7050   ins_pipe( ialu_reg );
 7051 %}
 7052 
 7053 // XXX addP mem ops ????
 7054 
 7055 instruct checkCastPP(rRegP dst)
 7056 %{
 7057   match(Set dst (CheckCastPP dst));
 7058 
 7059   size(0);
 7060   format %{ "# checkcastPP of $dst" %}
 7061   ins_encode(/* empty encoding */);
 7062   ins_pipe(empty);
 7063 %}
 7064 
 7065 instruct castPP(rRegP dst)
 7066 %{
 7067   match(Set dst (CastPP dst));
 7068 
 7069   size(0);
 7070   format %{ "# castPP of $dst" %}
 7071   ins_encode(/* empty encoding */);
 7072   ins_pipe(empty);
 7073 %}
 7074 
 7075 instruct castII(rRegI dst)
 7076 %{
 7077   match(Set dst (CastII dst));
 7078 
 7079   size(0);
 7080   format %{ "# castII of $dst" %}
 7081   ins_encode(/* empty encoding */);
 7082   ins_cost(0);
 7083   ins_pipe(empty);
 7084 %}
 7085 
 7086 instruct castLL(rRegL dst)
 7087 %{
 7088   match(Set dst (CastLL dst));
 7089 
 7090   size(0);
 7091   format %{ "# castLL of $dst" %}
 7092   ins_encode(/* empty encoding */);
 7093   ins_cost(0);
 7094   ins_pipe(empty);
 7095 %}
 7096 
 7097 instruct castFF(regF dst)
 7098 %{
 7099   match(Set dst (CastFF dst));
 7100 
 7101   size(0);
 7102   format %{ "# castFF of $dst" %}
 7103   ins_encode(/* empty encoding */);
 7104   ins_cost(0);
 7105   ins_pipe(empty);
 7106 %}
 7107 
 7108 instruct castDD(regD dst)
 7109 %{
 7110   match(Set dst (CastDD dst));
 7111 
 7112   size(0);
 7113   format %{ "# castDD of $dst" %}
 7114   ins_encode(/* empty encoding */);
 7115   ins_cost(0);
 7116   ins_pipe(empty);
 7117 %}
 7118 
 7119 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7120 instruct compareAndSwapP(rRegI res,
 7121                          memory mem_ptr,
 7122                          rax_RegP oldval, rRegP newval,
 7123                          rFlagsReg cr)
 7124 %{
 7125   predicate(n->as_LoadStore()->barrier_data() == 0);
 7126   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7127   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7128   effect(KILL cr, KILL oldval);
 7129 
 7130   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7131             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7132             "sete    $res\n\t"
 7133             "movzbl  $res, $res" %}
 7134   ins_encode %{
 7135     __ lock();
 7136     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7137     __ setb(Assembler::equal, $res$$Register);
 7138     __ movzbl($res$$Register, $res$$Register);
 7139   %}
 7140   ins_pipe( pipe_cmpxchg );
 7141 %}
 7142 
 7143 instruct compareAndSwapL(rRegI res,
 7144                          memory mem_ptr,
 7145                          rax_RegL oldval, rRegL newval,
 7146                          rFlagsReg cr)
 7147 %{
 7148   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7149   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7150   effect(KILL cr, KILL oldval);
 7151 
 7152   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7153             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7154             "sete    $res\n\t"
 7155             "movzbl  $res, $res" %}
 7156   ins_encode %{
 7157     __ lock();
 7158     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7159     __ setb(Assembler::equal, $res$$Register);
 7160     __ movzbl($res$$Register, $res$$Register);
 7161   %}
 7162   ins_pipe( pipe_cmpxchg );
 7163 %}
 7164 
 7165 instruct compareAndSwapI(rRegI res,
 7166                          memory mem_ptr,
 7167                          rax_RegI oldval, rRegI newval,
 7168                          rFlagsReg cr)
 7169 %{
 7170   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7171   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7172   effect(KILL cr, KILL oldval);
 7173 
 7174   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7175             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7176             "sete    $res\n\t"
 7177             "movzbl  $res, $res" %}
 7178   ins_encode %{
 7179     __ lock();
 7180     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7181     __ setb(Assembler::equal, $res$$Register);
 7182     __ movzbl($res$$Register, $res$$Register);
 7183   %}
 7184   ins_pipe( pipe_cmpxchg );
 7185 %}
 7186 
 7187 instruct compareAndSwapB(rRegI res,
 7188                          memory mem_ptr,
 7189                          rax_RegI oldval, rRegI newval,
 7190                          rFlagsReg cr)
 7191 %{
 7192   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7193   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7194   effect(KILL cr, KILL oldval);
 7195 
 7196   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7197             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7198             "sete    $res\n\t"
 7199             "movzbl  $res, $res" %}
 7200   ins_encode %{
 7201     __ lock();
 7202     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7203     __ setb(Assembler::equal, $res$$Register);
 7204     __ movzbl($res$$Register, $res$$Register);
 7205   %}
 7206   ins_pipe( pipe_cmpxchg );
 7207 %}
 7208 
 7209 instruct compareAndSwapS(rRegI res,
 7210                          memory mem_ptr,
 7211                          rax_RegI oldval, rRegI newval,
 7212                          rFlagsReg cr)
 7213 %{
 7214   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7215   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7216   effect(KILL cr, KILL oldval);
 7217 
 7218   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7219             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7220             "sete    $res\n\t"
 7221             "movzbl  $res, $res" %}
 7222   ins_encode %{
 7223     __ lock();
 7224     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7225     __ setb(Assembler::equal, $res$$Register);
 7226     __ movzbl($res$$Register, $res$$Register);
 7227   %}
 7228   ins_pipe( pipe_cmpxchg );
 7229 %}
 7230 
 7231 instruct compareAndSwapN(rRegI res,
 7232                           memory mem_ptr,
 7233                           rax_RegN oldval, rRegN newval,
 7234                           rFlagsReg cr) %{
 7235   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7236   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7237   effect(KILL cr, KILL oldval);
 7238 
 7239   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7240             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7241             "sete    $res\n\t"
 7242             "movzbl  $res, $res" %}
 7243   ins_encode %{
 7244     __ lock();
 7245     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7246     __ setb(Assembler::equal, $res$$Register);
 7247     __ movzbl($res$$Register, $res$$Register);
 7248   %}
 7249   ins_pipe( pipe_cmpxchg );
 7250 %}
 7251 
 7252 instruct compareAndExchangeB(
 7253                          memory mem_ptr,
 7254                          rax_RegI oldval, rRegI newval,
 7255                          rFlagsReg cr)
 7256 %{
 7257   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7258   effect(KILL cr);
 7259 
 7260   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7261             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7262   ins_encode %{
 7263     __ lock();
 7264     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7265   %}
 7266   ins_pipe( pipe_cmpxchg );
 7267 %}
 7268 
 7269 instruct compareAndExchangeS(
 7270                          memory mem_ptr,
 7271                          rax_RegI oldval, rRegI newval,
 7272                          rFlagsReg cr)
 7273 %{
 7274   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7275   effect(KILL cr);
 7276 
 7277   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7278             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7279   ins_encode %{
 7280     __ lock();
 7281     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7282   %}
 7283   ins_pipe( pipe_cmpxchg );
 7284 %}
 7285 
 7286 instruct compareAndExchangeI(
 7287                          memory mem_ptr,
 7288                          rax_RegI oldval, rRegI newval,
 7289                          rFlagsReg cr)
 7290 %{
 7291   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7292   effect(KILL cr);
 7293 
 7294   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7295             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7296   ins_encode %{
 7297     __ lock();
 7298     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7299   %}
 7300   ins_pipe( pipe_cmpxchg );
 7301 %}
 7302 
 7303 instruct compareAndExchangeL(
 7304                          memory mem_ptr,
 7305                          rax_RegL oldval, rRegL newval,
 7306                          rFlagsReg cr)
 7307 %{
 7308   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7309   effect(KILL cr);
 7310 
 7311   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7312             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7313   ins_encode %{
 7314     __ lock();
 7315     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7316   %}
 7317   ins_pipe( pipe_cmpxchg );
 7318 %}
 7319 
 7320 instruct compareAndExchangeN(
 7321                           memory mem_ptr,
 7322                           rax_RegN oldval, rRegN newval,
 7323                           rFlagsReg cr) %{
 7324   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7325   effect(KILL cr);
 7326 
 7327   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7328             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7329   ins_encode %{
 7330     __ lock();
 7331     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7332   %}
 7333   ins_pipe( pipe_cmpxchg );
 7334 %}
 7335 
 7336 instruct compareAndExchangeP(
 7337                          memory mem_ptr,
 7338                          rax_RegP oldval, rRegP newval,
 7339                          rFlagsReg cr)
 7340 %{
 7341   predicate(n->as_LoadStore()->barrier_data() == 0);
 7342   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7343   effect(KILL cr);
 7344 
 7345   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7346             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7347   ins_encode %{
 7348     __ lock();
 7349     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7350   %}
 7351   ins_pipe( pipe_cmpxchg );
 7352 %}
 7353 
 7354 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7355   predicate(n->as_LoadStore()->result_not_used());
 7356   match(Set dummy (GetAndAddB mem add));
 7357   effect(KILL cr);
 7358   format %{ "addb_lock   $mem, $add" %}
 7359   ins_encode %{
 7360     __ lock();
 7361     __ addb($mem$$Address, $add$$Register);
 7362   %}
 7363   ins_pipe(pipe_cmpxchg);
 7364 %}
 7365 
 7366 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7367   predicate(n->as_LoadStore()->result_not_used());
 7368   match(Set dummy (GetAndAddB mem add));
 7369   effect(KILL cr);
 7370   format %{ "addb_lock   $mem, $add" %}
 7371   ins_encode %{
 7372     __ lock();
 7373     __ addb($mem$$Address, $add$$constant);
 7374   %}
 7375   ins_pipe(pipe_cmpxchg);
 7376 %}
 7377 
 7378 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7379   predicate(!n->as_LoadStore()->result_not_used());
 7380   match(Set newval (GetAndAddB mem newval));
 7381   effect(KILL cr);
 7382   format %{ "xaddb_lock  $mem, $newval" %}
 7383   ins_encode %{
 7384     __ lock();
 7385     __ xaddb($mem$$Address, $newval$$Register);
 7386   %}
 7387   ins_pipe(pipe_cmpxchg);
 7388 %}
 7389 
 7390 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7391   predicate(n->as_LoadStore()->result_not_used());
 7392   match(Set dummy (GetAndAddS mem add));
 7393   effect(KILL cr);
 7394   format %{ "addw_lock   $mem, $add" %}
 7395   ins_encode %{
 7396     __ lock();
 7397     __ addw($mem$$Address, $add$$Register);
 7398   %}
 7399   ins_pipe(pipe_cmpxchg);
 7400 %}
 7401 
 7402 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7403   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7404   match(Set dummy (GetAndAddS mem add));
 7405   effect(KILL cr);
 7406   format %{ "addw_lock   $mem, $add" %}
 7407   ins_encode %{
 7408     __ lock();
 7409     __ addw($mem$$Address, $add$$constant);
 7410   %}
 7411   ins_pipe(pipe_cmpxchg);
 7412 %}
 7413 
 7414 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7415   predicate(!n->as_LoadStore()->result_not_used());
 7416   match(Set newval (GetAndAddS mem newval));
 7417   effect(KILL cr);
 7418   format %{ "xaddw_lock  $mem, $newval" %}
 7419   ins_encode %{
 7420     __ lock();
 7421     __ xaddw($mem$$Address, $newval$$Register);
 7422   %}
 7423   ins_pipe(pipe_cmpxchg);
 7424 %}
 7425 
 7426 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7427   predicate(n->as_LoadStore()->result_not_used());
 7428   match(Set dummy (GetAndAddI mem add));
 7429   effect(KILL cr);
 7430   format %{ "addl_lock   $mem, $add" %}
 7431   ins_encode %{
 7432     __ lock();
 7433     __ addl($mem$$Address, $add$$Register);
 7434   %}
 7435   ins_pipe(pipe_cmpxchg);
 7436 %}
 7437 
 7438 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7439   predicate(n->as_LoadStore()->result_not_used());
 7440   match(Set dummy (GetAndAddI mem add));
 7441   effect(KILL cr);
 7442   format %{ "addl_lock   $mem, $add" %}
 7443   ins_encode %{
 7444     __ lock();
 7445     __ addl($mem$$Address, $add$$constant);
 7446   %}
 7447   ins_pipe(pipe_cmpxchg);
 7448 %}
 7449 
 7450 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 7451   predicate(!n->as_LoadStore()->result_not_used());
 7452   match(Set newval (GetAndAddI mem newval));
 7453   effect(KILL cr);
 7454   format %{ "xaddl_lock  $mem, $newval" %}
 7455   ins_encode %{
 7456     __ lock();
 7457     __ xaddl($mem$$Address, $newval$$Register);
 7458   %}
 7459   ins_pipe(pipe_cmpxchg);
 7460 %}
 7461 
 7462 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 7463   predicate(n->as_LoadStore()->result_not_used());
 7464   match(Set dummy (GetAndAddL mem add));
 7465   effect(KILL cr);
 7466   format %{ "addq_lock   $mem, $add" %}
 7467   ins_encode %{
 7468     __ lock();
 7469     __ addq($mem$$Address, $add$$Register);
 7470   %}
 7471   ins_pipe(pipe_cmpxchg);
 7472 %}
 7473 
 7474 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 7475   predicate(n->as_LoadStore()->result_not_used());
 7476   match(Set dummy (GetAndAddL mem add));
 7477   effect(KILL cr);
 7478   format %{ "addq_lock   $mem, $add" %}
 7479   ins_encode %{
 7480     __ lock();
 7481     __ addq($mem$$Address, $add$$constant);
 7482   %}
 7483   ins_pipe(pipe_cmpxchg);
 7484 %}
 7485 
 7486 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 7487   predicate(!n->as_LoadStore()->result_not_used());
 7488   match(Set newval (GetAndAddL mem newval));
 7489   effect(KILL cr);
 7490   format %{ "xaddq_lock  $mem, $newval" %}
 7491   ins_encode %{
 7492     __ lock();
 7493     __ xaddq($mem$$Address, $newval$$Register);
 7494   %}
 7495   ins_pipe(pipe_cmpxchg);
 7496 %}
 7497 
 7498 instruct xchgB( memory mem, rRegI newval) %{
 7499   match(Set newval (GetAndSetB mem newval));
 7500   format %{ "XCHGB  $newval,[$mem]" %}
 7501   ins_encode %{
 7502     __ xchgb($newval$$Register, $mem$$Address);
 7503   %}
 7504   ins_pipe( pipe_cmpxchg );
 7505 %}
 7506 
 7507 instruct xchgS( memory mem, rRegI newval) %{
 7508   match(Set newval (GetAndSetS mem newval));
 7509   format %{ "XCHGW  $newval,[$mem]" %}
 7510   ins_encode %{
 7511     __ xchgw($newval$$Register, $mem$$Address);
 7512   %}
 7513   ins_pipe( pipe_cmpxchg );
 7514 %}
 7515 
 7516 instruct xchgI( memory mem, rRegI newval) %{
 7517   match(Set newval (GetAndSetI mem newval));
 7518   format %{ "XCHGL  $newval,[$mem]" %}
 7519   ins_encode %{
 7520     __ xchgl($newval$$Register, $mem$$Address);
 7521   %}
 7522   ins_pipe( pipe_cmpxchg );
 7523 %}
 7524 
 7525 instruct xchgL( memory mem, rRegL newval) %{
 7526   match(Set newval (GetAndSetL mem newval));
 7527   format %{ "XCHGL  $newval,[$mem]" %}
 7528   ins_encode %{
 7529     __ xchgq($newval$$Register, $mem$$Address);
 7530   %}
 7531   ins_pipe( pipe_cmpxchg );
 7532 %}
 7533 
 7534 instruct xchgP( memory mem, rRegP newval) %{
 7535   match(Set newval (GetAndSetP mem newval));
 7536   predicate(n->as_LoadStore()->barrier_data() == 0);
 7537   format %{ "XCHGQ  $newval,[$mem]" %}
 7538   ins_encode %{
 7539     __ xchgq($newval$$Register, $mem$$Address);
 7540   %}
 7541   ins_pipe( pipe_cmpxchg );
 7542 %}
 7543 
 7544 instruct xchgN( memory mem, rRegN newval) %{
 7545   match(Set newval (GetAndSetN mem newval));
 7546   format %{ "XCHGL  $newval,$mem]" %}
 7547   ins_encode %{
 7548     __ xchgl($newval$$Register, $mem$$Address);
 7549   %}
 7550   ins_pipe( pipe_cmpxchg );
 7551 %}
 7552 
 7553 //----------Abs Instructions-------------------------------------------
 7554 
 7555 // Integer Absolute Instructions
 7556 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7557 %{
 7558   match(Set dst (AbsI src));
 7559   effect(TEMP dst, KILL cr);
 7560   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 7561             "subl    $dst, $src\n\t"
 7562             "cmovll  $dst, $src" %}
 7563   ins_encode %{
 7564     __ xorl($dst$$Register, $dst$$Register);
 7565     __ subl($dst$$Register, $src$$Register);
 7566     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 7567   %}
 7568 
 7569   ins_pipe(ialu_reg_reg);
 7570 %}
 7571 
 7572 // Long Absolute Instructions
 7573 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7574 %{
 7575   match(Set dst (AbsL src));
 7576   effect(TEMP dst, KILL cr);
 7577   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 7578             "subq    $dst, $src\n\t"
 7579             "cmovlq  $dst, $src" %}
 7580   ins_encode %{
 7581     __ xorl($dst$$Register, $dst$$Register);
 7582     __ subq($dst$$Register, $src$$Register);
 7583     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 7584   %}
 7585 
 7586   ins_pipe(ialu_reg_reg);
 7587 %}
 7588 
 7589 //----------Subtraction Instructions-------------------------------------------
 7590 
 7591 // Integer Subtraction Instructions
 7592 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7593 %{
 7594   match(Set dst (SubI dst src));
 7595   effect(KILL cr);
 7596   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7597 
 7598   format %{ "subl    $dst, $src\t# int" %}
 7599   ins_encode %{
 7600     __ subl($dst$$Register, $src$$Register);
 7601   %}
 7602   ins_pipe(ialu_reg_reg);
 7603 %}
 7604 
 7605 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7606 %{
 7607   match(Set dst (SubI dst (LoadI src)));
 7608   effect(KILL cr);
 7609   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7610 
 7611   ins_cost(150);
 7612   format %{ "subl    $dst, $src\t# int" %}
 7613   ins_encode %{
 7614     __ subl($dst$$Register, $src$$Address);
 7615   %}
 7616   ins_pipe(ialu_reg_mem);
 7617 %}
 7618 
 7619 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7620 %{
 7621   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 7622   effect(KILL cr);
 7623   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7624 
 7625   ins_cost(150);
 7626   format %{ "subl    $dst, $src\t# int" %}
 7627   ins_encode %{
 7628     __ subl($dst$$Address, $src$$Register);
 7629   %}
 7630   ins_pipe(ialu_mem_reg);
 7631 %}
 7632 
 7633 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7634 %{
 7635   match(Set dst (SubL dst src));
 7636   effect(KILL cr);
 7637   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7638 
 7639   format %{ "subq    $dst, $src\t# long" %}
 7640   ins_encode %{
 7641     __ subq($dst$$Register, $src$$Register);
 7642   %}
 7643   ins_pipe(ialu_reg_reg);
 7644 %}
 7645 
 7646 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7647 %{
 7648   match(Set dst (SubL dst (LoadL src)));
 7649   effect(KILL cr);
 7650   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7651 
 7652   ins_cost(150);
 7653   format %{ "subq    $dst, $src\t# long" %}
 7654   ins_encode %{
 7655     __ subq($dst$$Register, $src$$Address);
 7656   %}
 7657   ins_pipe(ialu_reg_mem);
 7658 %}
 7659 
 7660 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7661 %{
 7662   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 7663   effect(KILL cr);
 7664   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7665 
 7666   ins_cost(150);
 7667   format %{ "subq    $dst, $src\t# long" %}
 7668   ins_encode %{
 7669     __ subq($dst$$Address, $src$$Register);
 7670   %}
 7671   ins_pipe(ialu_mem_reg);
 7672 %}
 7673 
 7674 // Subtract from a pointer
 7675 // XXX hmpf???
 7676 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 7677 %{
 7678   match(Set dst (AddP dst (SubI zero src)));
 7679   effect(KILL cr);
 7680 
 7681   format %{ "subq    $dst, $src\t# ptr - int" %}
 7682   ins_encode %{
 7683     __ subq($dst$$Register, $src$$Register);
 7684   %}
 7685   ins_pipe(ialu_reg_reg);
 7686 %}
 7687 
 7688 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 7689 %{
 7690   match(Set dst (SubI zero dst));
 7691   effect(KILL cr);
 7692   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7693 
 7694   format %{ "negl    $dst\t# int" %}
 7695   ins_encode %{
 7696     __ negl($dst$$Register);
 7697   %}
 7698   ins_pipe(ialu_reg);
 7699 %}
 7700 
 7701 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 7702 %{
 7703   match(Set dst (NegI dst));
 7704   effect(KILL cr);
 7705   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7706 
 7707   format %{ "negl    $dst\t# int" %}
 7708   ins_encode %{
 7709     __ negl($dst$$Register);
 7710   %}
 7711   ins_pipe(ialu_reg);
 7712 %}
 7713 
 7714 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 7715 %{
 7716   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 7717   effect(KILL cr);
 7718   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7719 
 7720   format %{ "negl    $dst\t# int" %}
 7721   ins_encode %{
 7722     __ negl($dst$$Address);
 7723   %}
 7724   ins_pipe(ialu_reg);
 7725 %}
 7726 
 7727 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 7728 %{
 7729   match(Set dst (SubL zero dst));
 7730   effect(KILL cr);
 7731   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7732 
 7733   format %{ "negq    $dst\t# long" %}
 7734   ins_encode %{
 7735     __ negq($dst$$Register);
 7736   %}
 7737   ins_pipe(ialu_reg);
 7738 %}
 7739 
 7740 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 7741 %{
 7742   match(Set dst (NegL dst));
 7743   effect(KILL cr);
 7744   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7745 
 7746   format %{ "negq    $dst\t# int" %}
 7747   ins_encode %{
 7748     __ negq($dst$$Register);
 7749   %}
 7750   ins_pipe(ialu_reg);
 7751 %}
 7752 
 7753 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 7754 %{
 7755   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 7756   effect(KILL cr);
 7757   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7758 
 7759   format %{ "negq    $dst\t# long" %}
 7760   ins_encode %{
 7761     __ negq($dst$$Address);
 7762   %}
 7763   ins_pipe(ialu_reg);
 7764 %}
 7765 
 7766 //----------Multiplication/Division Instructions-------------------------------
 7767 // Integer Multiplication Instructions
 7768 // Multiply Register
 7769 
 7770 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7771 %{
 7772   match(Set dst (MulI dst src));
 7773   effect(KILL cr);
 7774 
 7775   ins_cost(300);
 7776   format %{ "imull   $dst, $src\t# int" %}
 7777   ins_encode %{
 7778     __ imull($dst$$Register, $src$$Register);
 7779   %}
 7780   ins_pipe(ialu_reg_reg_alu0);
 7781 %}
 7782 
 7783 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 7784 %{
 7785   match(Set dst (MulI src imm));
 7786   effect(KILL cr);
 7787 
 7788   ins_cost(300);
 7789   format %{ "imull   $dst, $src, $imm\t# int" %}
 7790   ins_encode %{
 7791     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 7792   %}
 7793   ins_pipe(ialu_reg_reg_alu0);
 7794 %}
 7795 
 7796 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 7797 %{
 7798   match(Set dst (MulI dst (LoadI src)));
 7799   effect(KILL cr);
 7800 
 7801   ins_cost(350);
 7802   format %{ "imull   $dst, $src\t# int" %}
 7803   ins_encode %{
 7804     __ imull($dst$$Register, $src$$Address);
 7805   %}
 7806   ins_pipe(ialu_reg_mem_alu0);
 7807 %}
 7808 
 7809 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 7810 %{
 7811   match(Set dst (MulI (LoadI src) imm));
 7812   effect(KILL cr);
 7813 
 7814   ins_cost(300);
 7815   format %{ "imull   $dst, $src, $imm\t# int" %}
 7816   ins_encode %{
 7817     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 7818   %}
 7819   ins_pipe(ialu_reg_mem_alu0);
 7820 %}
 7821 
 7822 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 7823 %{
 7824   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 7825   effect(KILL cr, KILL src2);
 7826 
 7827   expand %{ mulI_rReg(dst, src1, cr);
 7828            mulI_rReg(src2, src3, cr);
 7829            addI_rReg(dst, src2, cr); %}
 7830 %}
 7831 
 7832 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7833 %{
 7834   match(Set dst (MulL dst src));
 7835   effect(KILL cr);
 7836 
 7837   ins_cost(300);
 7838   format %{ "imulq   $dst, $src\t# long" %}
 7839   ins_encode %{
 7840     __ imulq($dst$$Register, $src$$Register);
 7841   %}
 7842   ins_pipe(ialu_reg_reg_alu0);
 7843 %}
 7844 
 7845 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 7846 %{
 7847   match(Set dst (MulL src imm));
 7848   effect(KILL cr);
 7849 
 7850   ins_cost(300);
 7851   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7852   ins_encode %{
 7853     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 7854   %}
 7855   ins_pipe(ialu_reg_reg_alu0);
 7856 %}
 7857 
 7858 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 7859 %{
 7860   match(Set dst (MulL dst (LoadL src)));
 7861   effect(KILL cr);
 7862 
 7863   ins_cost(350);
 7864   format %{ "imulq   $dst, $src\t# long" %}
 7865   ins_encode %{
 7866     __ imulq($dst$$Register, $src$$Address);
 7867   %}
 7868   ins_pipe(ialu_reg_mem_alu0);
 7869 %}
 7870 
 7871 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 7872 %{
 7873   match(Set dst (MulL (LoadL src) imm));
 7874   effect(KILL cr);
 7875 
 7876   ins_cost(300);
 7877   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7878   ins_encode %{
 7879     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 7880   %}
 7881   ins_pipe(ialu_reg_mem_alu0);
 7882 %}
 7883 
 7884 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7885 %{
 7886   match(Set dst (MulHiL src rax));
 7887   effect(USE_KILL rax, KILL cr);
 7888 
 7889   ins_cost(300);
 7890   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 7891   ins_encode %{
 7892     __ imulq($src$$Register);
 7893   %}
 7894   ins_pipe(ialu_reg_reg_alu0);
 7895 %}
 7896 
 7897 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7898 %{
 7899   match(Set dst (UMulHiL src rax));
 7900   effect(USE_KILL rax, KILL cr);
 7901 
 7902   ins_cost(300);
 7903   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 7904   ins_encode %{
 7905     __ mulq($src$$Register);
 7906   %}
 7907   ins_pipe(ialu_reg_reg_alu0);
 7908 %}
 7909 
 7910 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7911                    rFlagsReg cr)
 7912 %{
 7913   match(Set rax (DivI rax div));
 7914   effect(KILL rdx, KILL cr);
 7915 
 7916   ins_cost(30*100+10*100); // XXX
 7917   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7918             "jne,s   normal\n\t"
 7919             "xorl    rdx, rdx\n\t"
 7920             "cmpl    $div, -1\n\t"
 7921             "je,s    done\n"
 7922     "normal: cdql\n\t"
 7923             "idivl   $div\n"
 7924     "done:"        %}
 7925   ins_encode(cdql_enc(div));
 7926   ins_pipe(ialu_reg_reg_alu0);
 7927 %}
 7928 
 7929 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7930                    rFlagsReg cr)
 7931 %{
 7932   match(Set rax (DivL rax div));
 7933   effect(KILL rdx, KILL cr);
 7934 
 7935   ins_cost(30*100+10*100); // XXX
 7936   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7937             "cmpq    rax, rdx\n\t"
 7938             "jne,s   normal\n\t"
 7939             "xorl    rdx, rdx\n\t"
 7940             "cmpq    $div, -1\n\t"
 7941             "je,s    done\n"
 7942     "normal: cdqq\n\t"
 7943             "idivq   $div\n"
 7944     "done:"        %}
 7945   ins_encode(cdqq_enc(div));
 7946   ins_pipe(ialu_reg_reg_alu0);
 7947 %}
 7948 
 7949 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 7950 %{
 7951   match(Set rax (UDivI rax div));
 7952   effect(KILL rdx, KILL cr);
 7953 
 7954   ins_cost(300);
 7955   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 7956   ins_encode %{
 7957     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 7958   %}
 7959   ins_pipe(ialu_reg_reg_alu0);
 7960 %}
 7961 
 7962 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 7963 %{
 7964   match(Set rax (UDivL rax div));
 7965   effect(KILL rdx, KILL cr);
 7966 
 7967   ins_cost(300);
 7968   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 7969   ins_encode %{
 7970      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 7971   %}
 7972   ins_pipe(ialu_reg_reg_alu0);
 7973 %}
 7974 
 7975 // Integer DIVMOD with Register, both quotient and mod results
 7976 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7977                              rFlagsReg cr)
 7978 %{
 7979   match(DivModI rax div);
 7980   effect(KILL cr);
 7981 
 7982   ins_cost(30*100+10*100); // XXX
 7983   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7984             "jne,s   normal\n\t"
 7985             "xorl    rdx, rdx\n\t"
 7986             "cmpl    $div, -1\n\t"
 7987             "je,s    done\n"
 7988     "normal: cdql\n\t"
 7989             "idivl   $div\n"
 7990     "done:"        %}
 7991   ins_encode(cdql_enc(div));
 7992   ins_pipe(pipe_slow);
 7993 %}
 7994 
 7995 // Long DIVMOD with Register, both quotient and mod results
 7996 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7997                              rFlagsReg cr)
 7998 %{
 7999   match(DivModL rax div);
 8000   effect(KILL cr);
 8001 
 8002   ins_cost(30*100+10*100); // XXX
 8003   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8004             "cmpq    rax, rdx\n\t"
 8005             "jne,s   normal\n\t"
 8006             "xorl    rdx, rdx\n\t"
 8007             "cmpq    $div, -1\n\t"
 8008             "je,s    done\n"
 8009     "normal: cdqq\n\t"
 8010             "idivq   $div\n"
 8011     "done:"        %}
 8012   ins_encode(cdqq_enc(div));
 8013   ins_pipe(pipe_slow);
 8014 %}
 8015 
 8016 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8017 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8018                               no_rax_rdx_RegI div, rFlagsReg cr)
 8019 %{
 8020   match(UDivModI rax div);
 8021   effect(TEMP tmp, KILL cr);
 8022 
 8023   ins_cost(300);
 8024   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8025             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8026           %}
 8027   ins_encode %{
 8028     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8029   %}
 8030   ins_pipe(pipe_slow);
 8031 %}
 8032 
 8033 // Unsigned long DIVMOD with Register, both quotient and mod results
 8034 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8035                               no_rax_rdx_RegL div, rFlagsReg cr)
 8036 %{
 8037   match(UDivModL rax div);
 8038   effect(TEMP tmp, KILL cr);
 8039 
 8040   ins_cost(300);
 8041   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8042             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8043           %}
 8044   ins_encode %{
 8045     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8046   %}
 8047   ins_pipe(pipe_slow);
 8048 %}
 8049 
 8050 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8051                    rFlagsReg cr)
 8052 %{
 8053   match(Set rdx (ModI rax div));
 8054   effect(KILL rax, KILL cr);
 8055 
 8056   ins_cost(300); // XXX
 8057   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8058             "jne,s   normal\n\t"
 8059             "xorl    rdx, rdx\n\t"
 8060             "cmpl    $div, -1\n\t"
 8061             "je,s    done\n"
 8062     "normal: cdql\n\t"
 8063             "idivl   $div\n"
 8064     "done:"        %}
 8065   ins_encode(cdql_enc(div));
 8066   ins_pipe(ialu_reg_reg_alu0);
 8067 %}
 8068 
 8069 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8070                    rFlagsReg cr)
 8071 %{
 8072   match(Set rdx (ModL rax div));
 8073   effect(KILL rax, KILL cr);
 8074 
 8075   ins_cost(300); // XXX
 8076   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8077             "cmpq    rax, rdx\n\t"
 8078             "jne,s   normal\n\t"
 8079             "xorl    rdx, rdx\n\t"
 8080             "cmpq    $div, -1\n\t"
 8081             "je,s    done\n"
 8082     "normal: cdqq\n\t"
 8083             "idivq   $div\n"
 8084     "done:"        %}
 8085   ins_encode(cdqq_enc(div));
 8086   ins_pipe(ialu_reg_reg_alu0);
 8087 %}
 8088 
 8089 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8090 %{
 8091   match(Set rdx (UModI rax div));
 8092   effect(KILL rax, KILL cr);
 8093 
 8094   ins_cost(300);
 8095   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8096   ins_encode %{
 8097     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8098   %}
 8099   ins_pipe(ialu_reg_reg_alu0);
 8100 %}
 8101 
 8102 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8103 %{
 8104   match(Set rdx (UModL rax div));
 8105   effect(KILL rax, KILL cr);
 8106 
 8107   ins_cost(300);
 8108   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8109   ins_encode %{
 8110     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8111   %}
 8112   ins_pipe(ialu_reg_reg_alu0);
 8113 %}
 8114 
 8115 // Integer Shift Instructions
 8116 // Shift Left by one, two, three
 8117 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8118 %{
 8119   match(Set dst (LShiftI dst shift));
 8120   effect(KILL cr);
 8121 
 8122   format %{ "sall    $dst, $shift" %}
 8123   ins_encode %{
 8124     __ sall($dst$$Register, $shift$$constant);
 8125   %}
 8126   ins_pipe(ialu_reg);
 8127 %}
 8128 
 8129 // Shift Left by 8-bit immediate
 8130 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8131 %{
 8132   match(Set dst (LShiftI dst shift));
 8133   effect(KILL cr);
 8134 
 8135   format %{ "sall    $dst, $shift" %}
 8136   ins_encode %{
 8137     __ sall($dst$$Register, $shift$$constant);
 8138   %}
 8139   ins_pipe(ialu_reg);
 8140 %}
 8141 
 8142 // Shift Left by 8-bit immediate
 8143 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8144 %{
 8145   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8146   effect(KILL cr);
 8147 
 8148   format %{ "sall    $dst, $shift" %}
 8149   ins_encode %{
 8150     __ sall($dst$$Address, $shift$$constant);
 8151   %}
 8152   ins_pipe(ialu_mem_imm);
 8153 %}
 8154 
 8155 // Shift Left by variable
 8156 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8157 %{
 8158   predicate(!VM_Version::supports_bmi2());
 8159   match(Set dst (LShiftI dst shift));
 8160   effect(KILL cr);
 8161 
 8162   format %{ "sall    $dst, $shift" %}
 8163   ins_encode %{
 8164     __ sall($dst$$Register);
 8165   %}
 8166   ins_pipe(ialu_reg_reg);
 8167 %}
 8168 
 8169 // Shift Left by variable
 8170 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8171 %{
 8172   predicate(!VM_Version::supports_bmi2());
 8173   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8174   effect(KILL cr);
 8175 
 8176   format %{ "sall    $dst, $shift" %}
 8177   ins_encode %{
 8178     __ sall($dst$$Address);
 8179   %}
 8180   ins_pipe(ialu_mem_reg);
 8181 %}
 8182 
 8183 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8184 %{
 8185   predicate(VM_Version::supports_bmi2());
 8186   match(Set dst (LShiftI src shift));
 8187 
 8188   format %{ "shlxl   $dst, $src, $shift" %}
 8189   ins_encode %{
 8190     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 8191   %}
 8192   ins_pipe(ialu_reg_reg);
 8193 %}
 8194 
 8195 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8196 %{
 8197   predicate(VM_Version::supports_bmi2());
 8198   match(Set dst (LShiftI (LoadI src) shift));
 8199   ins_cost(175);
 8200   format %{ "shlxl   $dst, $src, $shift" %}
 8201   ins_encode %{
 8202     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 8203   %}
 8204   ins_pipe(ialu_reg_mem);
 8205 %}
 8206 
 8207 // Arithmetic Shift Right by 8-bit immediate
 8208 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8209 %{
 8210   match(Set dst (RShiftI dst shift));
 8211   effect(KILL cr);
 8212 
 8213   format %{ "sarl    $dst, $shift" %}
 8214   ins_encode %{
 8215     __ sarl($dst$$Register, $shift$$constant);
 8216   %}
 8217   ins_pipe(ialu_mem_imm);
 8218 %}
 8219 
 8220 // Arithmetic Shift Right by 8-bit immediate
 8221 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8222 %{
 8223   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8224   effect(KILL cr);
 8225 
 8226   format %{ "sarl    $dst, $shift" %}
 8227   ins_encode %{
 8228     __ sarl($dst$$Address, $shift$$constant);
 8229   %}
 8230   ins_pipe(ialu_mem_imm);
 8231 %}
 8232 
 8233 // Arithmetic Shift Right by variable
 8234 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8235 %{
 8236   predicate(!VM_Version::supports_bmi2());
 8237   match(Set dst (RShiftI dst shift));
 8238   effect(KILL cr);
 8239 
 8240   format %{ "sarl    $dst, $shift" %}
 8241   ins_encode %{
 8242     __ sarl($dst$$Register);
 8243   %}
 8244   ins_pipe(ialu_reg_reg);
 8245 %}
 8246 
 8247 // Arithmetic Shift Right by variable
 8248 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8249 %{
 8250   predicate(!VM_Version::supports_bmi2());
 8251   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8252   effect(KILL cr);
 8253 
 8254   format %{ "sarl    $dst, $shift" %}
 8255   ins_encode %{
 8256     __ sarl($dst$$Address);
 8257   %}
 8258   ins_pipe(ialu_mem_reg);
 8259 %}
 8260 
 8261 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8262 %{
 8263   predicate(VM_Version::supports_bmi2());
 8264   match(Set dst (RShiftI src shift));
 8265 
 8266   format %{ "sarxl   $dst, $src, $shift" %}
 8267   ins_encode %{
 8268     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 8269   %}
 8270   ins_pipe(ialu_reg_reg);
 8271 %}
 8272 
 8273 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8274 %{
 8275   predicate(VM_Version::supports_bmi2());
 8276   match(Set dst (RShiftI (LoadI src) shift));
 8277   ins_cost(175);
 8278   format %{ "sarxl   $dst, $src, $shift" %}
 8279   ins_encode %{
 8280     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 8281   %}
 8282   ins_pipe(ialu_reg_mem);
 8283 %}
 8284 
 8285 // Logical Shift Right by 8-bit immediate
 8286 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8287 %{
 8288   match(Set dst (URShiftI dst shift));
 8289   effect(KILL cr);
 8290 
 8291   format %{ "shrl    $dst, $shift" %}
 8292   ins_encode %{
 8293     __ shrl($dst$$Register, $shift$$constant);
 8294   %}
 8295   ins_pipe(ialu_reg);
 8296 %}
 8297 
 8298 // Logical Shift Right by 8-bit immediate
 8299 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8300 %{
 8301   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8302   effect(KILL cr);
 8303 
 8304   format %{ "shrl    $dst, $shift" %}
 8305   ins_encode %{
 8306     __ shrl($dst$$Address, $shift$$constant);
 8307   %}
 8308   ins_pipe(ialu_mem_imm);
 8309 %}
 8310 
 8311 // Logical Shift Right by variable
 8312 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8313 %{
 8314   predicate(!VM_Version::supports_bmi2());
 8315   match(Set dst (URShiftI dst shift));
 8316   effect(KILL cr);
 8317 
 8318   format %{ "shrl    $dst, $shift" %}
 8319   ins_encode %{
 8320     __ shrl($dst$$Register);
 8321   %}
 8322   ins_pipe(ialu_reg_reg);
 8323 %}
 8324 
 8325 // Logical Shift Right by variable
 8326 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8327 %{
 8328   predicate(!VM_Version::supports_bmi2());
 8329   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8330   effect(KILL cr);
 8331 
 8332   format %{ "shrl    $dst, $shift" %}
 8333   ins_encode %{
 8334     __ shrl($dst$$Address);
 8335   %}
 8336   ins_pipe(ialu_mem_reg);
 8337 %}
 8338 
 8339 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8340 %{
 8341   predicate(VM_Version::supports_bmi2());
 8342   match(Set dst (URShiftI src shift));
 8343 
 8344   format %{ "shrxl   $dst, $src, $shift" %}
 8345   ins_encode %{
 8346     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 8347   %}
 8348   ins_pipe(ialu_reg_reg);
 8349 %}
 8350 
 8351 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8352 %{
 8353   predicate(VM_Version::supports_bmi2());
 8354   match(Set dst (URShiftI (LoadI src) shift));
 8355   ins_cost(175);
 8356   format %{ "shrxl   $dst, $src, $shift" %}
 8357   ins_encode %{
 8358     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 8359   %}
 8360   ins_pipe(ialu_reg_mem);
 8361 %}
 8362 
 8363 // Long Shift Instructions
 8364 // Shift Left by one, two, three
 8365 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 8366 %{
 8367   match(Set dst (LShiftL dst shift));
 8368   effect(KILL cr);
 8369 
 8370   format %{ "salq    $dst, $shift" %}
 8371   ins_encode %{
 8372     __ salq($dst$$Register, $shift$$constant);
 8373   %}
 8374   ins_pipe(ialu_reg);
 8375 %}
 8376 
 8377 // Shift Left by 8-bit immediate
 8378 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8379 %{
 8380   match(Set dst (LShiftL dst shift));
 8381   effect(KILL cr);
 8382 
 8383   format %{ "salq    $dst, $shift" %}
 8384   ins_encode %{
 8385     __ salq($dst$$Register, $shift$$constant);
 8386   %}
 8387   ins_pipe(ialu_reg);
 8388 %}
 8389 
 8390 // Shift Left by 8-bit immediate
 8391 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8392 %{
 8393   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8394   effect(KILL cr);
 8395 
 8396   format %{ "salq    $dst, $shift" %}
 8397   ins_encode %{
 8398     __ salq($dst$$Address, $shift$$constant);
 8399   %}
 8400   ins_pipe(ialu_mem_imm);
 8401 %}
 8402 
 8403 // Shift Left by variable
 8404 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8405 %{
 8406   predicate(!VM_Version::supports_bmi2());
 8407   match(Set dst (LShiftL dst shift));
 8408   effect(KILL cr);
 8409 
 8410   format %{ "salq    $dst, $shift" %}
 8411   ins_encode %{
 8412     __ salq($dst$$Register);
 8413   %}
 8414   ins_pipe(ialu_reg_reg);
 8415 %}
 8416 
 8417 // Shift Left by variable
 8418 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8419 %{
 8420   predicate(!VM_Version::supports_bmi2());
 8421   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8422   effect(KILL cr);
 8423 
 8424   format %{ "salq    $dst, $shift" %}
 8425   ins_encode %{
 8426     __ salq($dst$$Address);
 8427   %}
 8428   ins_pipe(ialu_mem_reg);
 8429 %}
 8430 
 8431 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8432 %{
 8433   predicate(VM_Version::supports_bmi2());
 8434   match(Set dst (LShiftL src shift));
 8435 
 8436   format %{ "shlxq   $dst, $src, $shift" %}
 8437   ins_encode %{
 8438     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 8439   %}
 8440   ins_pipe(ialu_reg_reg);
 8441 %}
 8442 
 8443 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8444 %{
 8445   predicate(VM_Version::supports_bmi2());
 8446   match(Set dst (LShiftL (LoadL src) shift));
 8447   ins_cost(175);
 8448   format %{ "shlxq   $dst, $src, $shift" %}
 8449   ins_encode %{
 8450     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 8451   %}
 8452   ins_pipe(ialu_reg_mem);
 8453 %}
 8454 
 8455 // Arithmetic Shift Right by 8-bit immediate
 8456 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 8457 %{
 8458   match(Set dst (RShiftL dst shift));
 8459   effect(KILL cr);
 8460 
 8461   format %{ "sarq    $dst, $shift" %}
 8462   ins_encode %{
 8463     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 8464   %}
 8465   ins_pipe(ialu_mem_imm);
 8466 %}
 8467 
 8468 // Arithmetic Shift Right by 8-bit immediate
 8469 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 8470 %{
 8471   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8472   effect(KILL cr);
 8473 
 8474   format %{ "sarq    $dst, $shift" %}
 8475   ins_encode %{
 8476     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 8477   %}
 8478   ins_pipe(ialu_mem_imm);
 8479 %}
 8480 
 8481 // Arithmetic Shift Right by variable
 8482 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8483 %{
 8484   predicate(!VM_Version::supports_bmi2());
 8485   match(Set dst (RShiftL dst shift));
 8486   effect(KILL cr);
 8487 
 8488   format %{ "sarq    $dst, $shift" %}
 8489   ins_encode %{
 8490     __ sarq($dst$$Register);
 8491   %}
 8492   ins_pipe(ialu_reg_reg);
 8493 %}
 8494 
 8495 // Arithmetic Shift Right by variable
 8496 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8497 %{
 8498   predicate(!VM_Version::supports_bmi2());
 8499   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8500   effect(KILL cr);
 8501 
 8502   format %{ "sarq    $dst, $shift" %}
 8503   ins_encode %{
 8504     __ sarq($dst$$Address);
 8505   %}
 8506   ins_pipe(ialu_mem_reg);
 8507 %}
 8508 
 8509 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8510 %{
 8511   predicate(VM_Version::supports_bmi2());
 8512   match(Set dst (RShiftL src shift));
 8513 
 8514   format %{ "sarxq   $dst, $src, $shift" %}
 8515   ins_encode %{
 8516     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 8517   %}
 8518   ins_pipe(ialu_reg_reg);
 8519 %}
 8520 
 8521 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8522 %{
 8523   predicate(VM_Version::supports_bmi2());
 8524   match(Set dst (RShiftL (LoadL src) shift));
 8525   ins_cost(175);
 8526   format %{ "sarxq   $dst, $src, $shift" %}
 8527   ins_encode %{
 8528     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 8529   %}
 8530   ins_pipe(ialu_reg_mem);
 8531 %}
 8532 
 8533 // Logical Shift Right by 8-bit immediate
 8534 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8535 %{
 8536   match(Set dst (URShiftL dst shift));
 8537   effect(KILL cr);
 8538 
 8539   format %{ "shrq    $dst, $shift" %}
 8540   ins_encode %{
 8541     __ shrq($dst$$Register, $shift$$constant);
 8542   %}
 8543   ins_pipe(ialu_reg);
 8544 %}
 8545 
 8546 // Logical Shift Right by 8-bit immediate
 8547 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8548 %{
 8549   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8550   effect(KILL cr);
 8551 
 8552   format %{ "shrq    $dst, $shift" %}
 8553   ins_encode %{
 8554     __ shrq($dst$$Address, $shift$$constant);
 8555   %}
 8556   ins_pipe(ialu_mem_imm);
 8557 %}
 8558 
 8559 // Logical Shift Right by variable
 8560 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8561 %{
 8562   predicate(!VM_Version::supports_bmi2());
 8563   match(Set dst (URShiftL dst shift));
 8564   effect(KILL cr);
 8565 
 8566   format %{ "shrq    $dst, $shift" %}
 8567   ins_encode %{
 8568     __ shrq($dst$$Register);
 8569   %}
 8570   ins_pipe(ialu_reg_reg);
 8571 %}
 8572 
 8573 // Logical Shift Right by variable
 8574 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8575 %{
 8576   predicate(!VM_Version::supports_bmi2());
 8577   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8578   effect(KILL cr);
 8579 
 8580   format %{ "shrq    $dst, $shift" %}
 8581   ins_encode %{
 8582     __ shrq($dst$$Address);
 8583   %}
 8584   ins_pipe(ialu_mem_reg);
 8585 %}
 8586 
 8587 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8588 %{
 8589   predicate(VM_Version::supports_bmi2());
 8590   match(Set dst (URShiftL src shift));
 8591 
 8592   format %{ "shrxq   $dst, $src, $shift" %}
 8593   ins_encode %{
 8594     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 8595   %}
 8596   ins_pipe(ialu_reg_reg);
 8597 %}
 8598 
 8599 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8600 %{
 8601   predicate(VM_Version::supports_bmi2());
 8602   match(Set dst (URShiftL (LoadL src) shift));
 8603   ins_cost(175);
 8604   format %{ "shrxq   $dst, $src, $shift" %}
 8605   ins_encode %{
 8606     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 8607   %}
 8608   ins_pipe(ialu_reg_mem);
 8609 %}
 8610 
 8611 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 8612 // This idiom is used by the compiler for the i2b bytecode.
 8613 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 8614 %{
 8615   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 8616 
 8617   format %{ "movsbl  $dst, $src\t# i2b" %}
 8618   ins_encode %{
 8619     __ movsbl($dst$$Register, $src$$Register);
 8620   %}
 8621   ins_pipe(ialu_reg_reg);
 8622 %}
 8623 
 8624 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 8625 // This idiom is used by the compiler the i2s bytecode.
 8626 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 8627 %{
 8628   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 8629 
 8630   format %{ "movswl  $dst, $src\t# i2s" %}
 8631   ins_encode %{
 8632     __ movswl($dst$$Register, $src$$Register);
 8633   %}
 8634   ins_pipe(ialu_reg_reg);
 8635 %}
 8636 
 8637 // ROL/ROR instructions
 8638 
 8639 // Rotate left by constant.
 8640 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8641 %{
 8642   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8643   match(Set dst (RotateLeft dst shift));
 8644   effect(KILL cr);
 8645   format %{ "roll    $dst, $shift" %}
 8646   ins_encode %{
 8647     __ roll($dst$$Register, $shift$$constant);
 8648   %}
 8649   ins_pipe(ialu_reg);
 8650 %}
 8651 
 8652 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 8653 %{
 8654   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8655   match(Set dst (RotateLeft src shift));
 8656   format %{ "rolxl   $dst, $src, $shift" %}
 8657   ins_encode %{
 8658     int shift = 32 - ($shift$$constant & 31);
 8659     __ rorxl($dst$$Register, $src$$Register, shift);
 8660   %}
 8661   ins_pipe(ialu_reg_reg);
 8662 %}
 8663 
 8664 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8665 %{
 8666   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8667   match(Set dst (RotateLeft (LoadI src) shift));
 8668   ins_cost(175);
 8669   format %{ "rolxl   $dst, $src, $shift" %}
 8670   ins_encode %{
 8671     int shift = 32 - ($shift$$constant & 31);
 8672     __ rorxl($dst$$Register, $src$$Address, shift);
 8673   %}
 8674   ins_pipe(ialu_reg_mem);
 8675 %}
 8676 
 8677 // Rotate Left by variable
 8678 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8679 %{
 8680   predicate(n->bottom_type()->basic_type() == T_INT);
 8681   match(Set dst (RotateLeft dst shift));
 8682   effect(KILL cr);
 8683   format %{ "roll    $dst, $shift" %}
 8684   ins_encode %{
 8685     __ roll($dst$$Register);
 8686   %}
 8687   ins_pipe(ialu_reg_reg);
 8688 %}
 8689 
 8690 // Rotate Right by constant.
 8691 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8692 %{
 8693   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8694   match(Set dst (RotateRight dst shift));
 8695   effect(KILL cr);
 8696   format %{ "rorl    $dst, $shift" %}
 8697   ins_encode %{
 8698     __ rorl($dst$$Register, $shift$$constant);
 8699   %}
 8700   ins_pipe(ialu_reg);
 8701 %}
 8702 
 8703 // Rotate Right by constant.
 8704 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 8705 %{
 8706   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8707   match(Set dst (RotateRight src shift));
 8708   format %{ "rorxl   $dst, $src, $shift" %}
 8709   ins_encode %{
 8710     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 8711   %}
 8712   ins_pipe(ialu_reg_reg);
 8713 %}
 8714 
 8715 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8716 %{
 8717   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8718   match(Set dst (RotateRight (LoadI src) shift));
 8719   ins_cost(175);
 8720   format %{ "rorxl   $dst, $src, $shift" %}
 8721   ins_encode %{
 8722     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 8723   %}
 8724   ins_pipe(ialu_reg_mem);
 8725 %}
 8726 
 8727 // Rotate Right by variable
 8728 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8729 %{
 8730   predicate(n->bottom_type()->basic_type() == T_INT);
 8731   match(Set dst (RotateRight dst shift));
 8732   effect(KILL cr);
 8733   format %{ "rorl    $dst, $shift" %}
 8734   ins_encode %{
 8735     __ rorl($dst$$Register);
 8736   %}
 8737   ins_pipe(ialu_reg_reg);
 8738 %}
 8739 
 8740 // Rotate Left by constant.
 8741 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8742 %{
 8743   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8744   match(Set dst (RotateLeft dst shift));
 8745   effect(KILL cr);
 8746   format %{ "rolq    $dst, $shift" %}
 8747   ins_encode %{
 8748     __ rolq($dst$$Register, $shift$$constant);
 8749   %}
 8750   ins_pipe(ialu_reg);
 8751 %}
 8752 
 8753 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 8754 %{
 8755   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8756   match(Set dst (RotateLeft src shift));
 8757   format %{ "rolxq   $dst, $src, $shift" %}
 8758   ins_encode %{
 8759     int shift = 64 - ($shift$$constant & 63);
 8760     __ rorxq($dst$$Register, $src$$Register, shift);
 8761   %}
 8762   ins_pipe(ialu_reg_reg);
 8763 %}
 8764 
 8765 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8766 %{
 8767   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8768   match(Set dst (RotateLeft (LoadL src) shift));
 8769   ins_cost(175);
 8770   format %{ "rolxq   $dst, $src, $shift" %}
 8771   ins_encode %{
 8772     int shift = 64 - ($shift$$constant & 63);
 8773     __ rorxq($dst$$Register, $src$$Address, shift);
 8774   %}
 8775   ins_pipe(ialu_reg_mem);
 8776 %}
 8777 
 8778 // Rotate Left by variable
 8779 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8780 %{
 8781   predicate(n->bottom_type()->basic_type() == T_LONG);
 8782   match(Set dst (RotateLeft dst shift));
 8783   effect(KILL cr);
 8784   format %{ "rolq    $dst, $shift" %}
 8785   ins_encode %{
 8786     __ rolq($dst$$Register);
 8787   %}
 8788   ins_pipe(ialu_reg_reg);
 8789 %}
 8790 
 8791 // Rotate Right by constant.
 8792 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8793 %{
 8794   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8795   match(Set dst (RotateRight dst shift));
 8796   effect(KILL cr);
 8797   format %{ "rorq    $dst, $shift" %}
 8798   ins_encode %{
 8799     __ rorq($dst$$Register, $shift$$constant);
 8800   %}
 8801   ins_pipe(ialu_reg);
 8802 %}
 8803 
 8804 // Rotate Right by constant
 8805 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 8806 %{
 8807   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8808   match(Set dst (RotateRight src shift));
 8809   format %{ "rorxq   $dst, $src, $shift" %}
 8810   ins_encode %{
 8811     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 8812   %}
 8813   ins_pipe(ialu_reg_reg);
 8814 %}
 8815 
 8816 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8817 %{
 8818   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8819   match(Set dst (RotateRight (LoadL src) shift));
 8820   ins_cost(175);
 8821   format %{ "rorxq   $dst, $src, $shift" %}
 8822   ins_encode %{
 8823     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 8824   %}
 8825   ins_pipe(ialu_reg_mem);
 8826 %}
 8827 
 8828 // Rotate Right by variable
 8829 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8830 %{
 8831   predicate(n->bottom_type()->basic_type() == T_LONG);
 8832   match(Set dst (RotateRight dst shift));
 8833   effect(KILL cr);
 8834   format %{ "rorq    $dst, $shift" %}
 8835   ins_encode %{
 8836     __ rorq($dst$$Register);
 8837   %}
 8838   ins_pipe(ialu_reg_reg);
 8839 %}
 8840 
 8841 //----------------------------- CompressBits/ExpandBits ------------------------
 8842 
 8843 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8844   predicate(n->bottom_type()->isa_long());
 8845   match(Set dst (CompressBits src mask));
 8846   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8847   ins_encode %{
 8848     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 8849   %}
 8850   ins_pipe( pipe_slow );
 8851 %}
 8852 
 8853 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8854   predicate(n->bottom_type()->isa_long());
 8855   match(Set dst (ExpandBits src mask));
 8856   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8857   ins_encode %{
 8858     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 8859   %}
 8860   ins_pipe( pipe_slow );
 8861 %}
 8862 
 8863 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8864   predicate(n->bottom_type()->isa_long());
 8865   match(Set dst (CompressBits src (LoadL mask)));
 8866   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8867   ins_encode %{
 8868     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 8869   %}
 8870   ins_pipe( pipe_slow );
 8871 %}
 8872 
 8873 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8874   predicate(n->bottom_type()->isa_long());
 8875   match(Set dst (ExpandBits src (LoadL mask)));
 8876   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8877   ins_encode %{
 8878     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 8879   %}
 8880   ins_pipe( pipe_slow );
 8881 %}
 8882 
 8883 
 8884 // Logical Instructions
 8885 
 8886 // Integer Logical Instructions
 8887 
 8888 // And Instructions
 8889 // And Register with Register
 8890 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8891 %{
 8892   match(Set dst (AndI dst src));
 8893   effect(KILL cr);
 8894   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8895 
 8896   format %{ "andl    $dst, $src\t# int" %}
 8897   ins_encode %{
 8898     __ andl($dst$$Register, $src$$Register);
 8899   %}
 8900   ins_pipe(ialu_reg_reg);
 8901 %}
 8902 
 8903 // And Register with Immediate 255
 8904 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 8905 %{
 8906   match(Set dst (AndI src mask));
 8907 
 8908   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 8909   ins_encode %{
 8910     __ movzbl($dst$$Register, $src$$Register);
 8911   %}
 8912   ins_pipe(ialu_reg);
 8913 %}
 8914 
 8915 // And Register with Immediate 255 and promote to long
 8916 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 8917 %{
 8918   match(Set dst (ConvI2L (AndI src mask)));
 8919 
 8920   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 8921   ins_encode %{
 8922     __ movzbl($dst$$Register, $src$$Register);
 8923   %}
 8924   ins_pipe(ialu_reg);
 8925 %}
 8926 
 8927 // And Register with Immediate 65535
 8928 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 8929 %{
 8930   match(Set dst (AndI src mask));
 8931 
 8932   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 8933   ins_encode %{
 8934     __ movzwl($dst$$Register, $src$$Register);
 8935   %}
 8936   ins_pipe(ialu_reg);
 8937 %}
 8938 
 8939 // And Register with Immediate 65535 and promote to long
 8940 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 8941 %{
 8942   match(Set dst (ConvI2L (AndI src mask)));
 8943 
 8944   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 8945   ins_encode %{
 8946     __ movzwl($dst$$Register, $src$$Register);
 8947   %}
 8948   ins_pipe(ialu_reg);
 8949 %}
 8950 
 8951 // Can skip int2long conversions after AND with small bitmask
 8952 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 8953 %{
 8954   predicate(VM_Version::supports_bmi2());
 8955   ins_cost(125);
 8956   effect(TEMP tmp, KILL cr);
 8957   match(Set dst (ConvI2L (AndI src mask)));
 8958   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 8959   ins_encode %{
 8960     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 8961     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 8962   %}
 8963   ins_pipe(ialu_reg_reg);
 8964 %}
 8965 
 8966 // And Register with Immediate
 8967 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8968 %{
 8969   match(Set dst (AndI dst src));
 8970   effect(KILL cr);
 8971   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8972 
 8973   format %{ "andl    $dst, $src\t# int" %}
 8974   ins_encode %{
 8975     __ andl($dst$$Register, $src$$constant);
 8976   %}
 8977   ins_pipe(ialu_reg);
 8978 %}
 8979 
 8980 // And Register with Memory
 8981 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8982 %{
 8983   match(Set dst (AndI dst (LoadI src)));
 8984   effect(KILL cr);
 8985   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8986 
 8987   ins_cost(150);
 8988   format %{ "andl    $dst, $src\t# int" %}
 8989   ins_encode %{
 8990     __ andl($dst$$Register, $src$$Address);
 8991   %}
 8992   ins_pipe(ialu_reg_mem);
 8993 %}
 8994 
 8995 // And Memory with Register
 8996 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8997 %{
 8998   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 8999   effect(KILL cr);
 9000   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9001 
 9002   ins_cost(150);
 9003   format %{ "andb    $dst, $src\t# byte" %}
 9004   ins_encode %{
 9005     __ andb($dst$$Address, $src$$Register);
 9006   %}
 9007   ins_pipe(ialu_mem_reg);
 9008 %}
 9009 
 9010 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9011 %{
 9012   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9013   effect(KILL cr);
 9014   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9015 
 9016   ins_cost(150);
 9017   format %{ "andl    $dst, $src\t# int" %}
 9018   ins_encode %{
 9019     __ andl($dst$$Address, $src$$Register);
 9020   %}
 9021   ins_pipe(ialu_mem_reg);
 9022 %}
 9023 
 9024 // And Memory with Immediate
 9025 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9026 %{
 9027   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9028   effect(KILL cr);
 9029   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9030 
 9031   ins_cost(125);
 9032   format %{ "andl    $dst, $src\t# int" %}
 9033   ins_encode %{
 9034     __ andl($dst$$Address, $src$$constant);
 9035   %}
 9036   ins_pipe(ialu_mem_imm);
 9037 %}
 9038 
 9039 // BMI1 instructions
 9040 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9041   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9042   predicate(UseBMI1Instructions);
 9043   effect(KILL cr);
 9044   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9045 
 9046   ins_cost(125);
 9047   format %{ "andnl  $dst, $src1, $src2" %}
 9048 
 9049   ins_encode %{
 9050     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9051   %}
 9052   ins_pipe(ialu_reg_mem);
 9053 %}
 9054 
 9055 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9056   match(Set dst (AndI (XorI src1 minus_1) src2));
 9057   predicate(UseBMI1Instructions);
 9058   effect(KILL cr);
 9059   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9060 
 9061   format %{ "andnl  $dst, $src1, $src2" %}
 9062 
 9063   ins_encode %{
 9064     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9065   %}
 9066   ins_pipe(ialu_reg);
 9067 %}
 9068 
 9069 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9070   match(Set dst (AndI (SubI imm_zero src) src));
 9071   predicate(UseBMI1Instructions);
 9072   effect(KILL cr);
 9073   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9074 
 9075   format %{ "blsil  $dst, $src" %}
 9076 
 9077   ins_encode %{
 9078     __ blsil($dst$$Register, $src$$Register);
 9079   %}
 9080   ins_pipe(ialu_reg);
 9081 %}
 9082 
 9083 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9084   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9085   predicate(UseBMI1Instructions);
 9086   effect(KILL cr);
 9087   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9088 
 9089   ins_cost(125);
 9090   format %{ "blsil  $dst, $src" %}
 9091 
 9092   ins_encode %{
 9093     __ blsil($dst$$Register, $src$$Address);
 9094   %}
 9095   ins_pipe(ialu_reg_mem);
 9096 %}
 9097 
 9098 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9099 %{
 9100   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9101   predicate(UseBMI1Instructions);
 9102   effect(KILL cr);
 9103   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9104 
 9105   ins_cost(125);
 9106   format %{ "blsmskl $dst, $src" %}
 9107 
 9108   ins_encode %{
 9109     __ blsmskl($dst$$Register, $src$$Address);
 9110   %}
 9111   ins_pipe(ialu_reg_mem);
 9112 %}
 9113 
 9114 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9115 %{
 9116   match(Set dst (XorI (AddI src minus_1) src));
 9117   predicate(UseBMI1Instructions);
 9118   effect(KILL cr);
 9119   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9120 
 9121   format %{ "blsmskl $dst, $src" %}
 9122 
 9123   ins_encode %{
 9124     __ blsmskl($dst$$Register, $src$$Register);
 9125   %}
 9126 
 9127   ins_pipe(ialu_reg);
 9128 %}
 9129 
 9130 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9131 %{
 9132   match(Set dst (AndI (AddI src minus_1) src) );
 9133   predicate(UseBMI1Instructions);
 9134   effect(KILL cr);
 9135   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9136 
 9137   format %{ "blsrl  $dst, $src" %}
 9138 
 9139   ins_encode %{
 9140     __ blsrl($dst$$Register, $src$$Register);
 9141   %}
 9142 
 9143   ins_pipe(ialu_reg_mem);
 9144 %}
 9145 
 9146 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9147 %{
 9148   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9149   predicate(UseBMI1Instructions);
 9150   effect(KILL cr);
 9151   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9152 
 9153   ins_cost(125);
 9154   format %{ "blsrl  $dst, $src" %}
 9155 
 9156   ins_encode %{
 9157     __ blsrl($dst$$Register, $src$$Address);
 9158   %}
 9159 
 9160   ins_pipe(ialu_reg);
 9161 %}
 9162 
 9163 // Or Instructions
 9164 // Or Register with Register
 9165 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9166 %{
 9167   match(Set dst (OrI dst src));
 9168   effect(KILL cr);
 9169   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9170 
 9171   format %{ "orl     $dst, $src\t# int" %}
 9172   ins_encode %{
 9173     __ orl($dst$$Register, $src$$Register);
 9174   %}
 9175   ins_pipe(ialu_reg_reg);
 9176 %}
 9177 
 9178 // Or Register with Immediate
 9179 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9180 %{
 9181   match(Set dst (OrI dst src));
 9182   effect(KILL cr);
 9183   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9184 
 9185   format %{ "orl     $dst, $src\t# int" %}
 9186   ins_encode %{
 9187     __ orl($dst$$Register, $src$$constant);
 9188   %}
 9189   ins_pipe(ialu_reg);
 9190 %}
 9191 
 9192 // Or Register with Memory
 9193 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9194 %{
 9195   match(Set dst (OrI dst (LoadI src)));
 9196   effect(KILL cr);
 9197   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9198 
 9199   ins_cost(150);
 9200   format %{ "orl     $dst, $src\t# int" %}
 9201   ins_encode %{
 9202     __ orl($dst$$Register, $src$$Address);
 9203   %}
 9204   ins_pipe(ialu_reg_mem);
 9205 %}
 9206 
 9207 // Or Memory with Register
 9208 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9209 %{
 9210   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9211   effect(KILL cr);
 9212   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9213 
 9214   ins_cost(150);
 9215   format %{ "orb    $dst, $src\t# byte" %}
 9216   ins_encode %{
 9217     __ orb($dst$$Address, $src$$Register);
 9218   %}
 9219   ins_pipe(ialu_mem_reg);
 9220 %}
 9221 
 9222 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9223 %{
 9224   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9225   effect(KILL cr);
 9226   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9227 
 9228   ins_cost(150);
 9229   format %{ "orl     $dst, $src\t# int" %}
 9230   ins_encode %{
 9231     __ orl($dst$$Address, $src$$Register);
 9232   %}
 9233   ins_pipe(ialu_mem_reg);
 9234 %}
 9235 
 9236 // Or Memory with Immediate
 9237 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9238 %{
 9239   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9240   effect(KILL cr);
 9241   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9242 
 9243   ins_cost(125);
 9244   format %{ "orl     $dst, $src\t# int" %}
 9245   ins_encode %{
 9246     __ orl($dst$$Address, $src$$constant);
 9247   %}
 9248   ins_pipe(ialu_mem_imm);
 9249 %}
 9250 
 9251 // Xor Instructions
 9252 // Xor Register with Register
 9253 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9254 %{
 9255   match(Set dst (XorI dst src));
 9256   effect(KILL cr);
 9257   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9258 
 9259   format %{ "xorl    $dst, $src\t# int" %}
 9260   ins_encode %{
 9261     __ xorl($dst$$Register, $src$$Register);
 9262   %}
 9263   ins_pipe(ialu_reg_reg);
 9264 %}
 9265 
 9266 // Xor Register with Immediate -1
 9267 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9268   match(Set dst (XorI dst imm));
 9269 
 9270   format %{ "not    $dst" %}
 9271   ins_encode %{
 9272      __ notl($dst$$Register);
 9273   %}
 9274   ins_pipe(ialu_reg);
 9275 %}
 9276 
 9277 // Xor Register with Immediate
 9278 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9279 %{
 9280   match(Set dst (XorI dst src));
 9281   effect(KILL cr);
 9282   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9283 
 9284   format %{ "xorl    $dst, $src\t# int" %}
 9285   ins_encode %{
 9286     __ xorl($dst$$Register, $src$$constant);
 9287   %}
 9288   ins_pipe(ialu_reg);
 9289 %}
 9290 
 9291 // Xor Register with Memory
 9292 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9293 %{
 9294   match(Set dst (XorI dst (LoadI src)));
 9295   effect(KILL cr);
 9296   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9297 
 9298   ins_cost(150);
 9299   format %{ "xorl    $dst, $src\t# int" %}
 9300   ins_encode %{
 9301     __ xorl($dst$$Register, $src$$Address);
 9302   %}
 9303   ins_pipe(ialu_reg_mem);
 9304 %}
 9305 
 9306 // Xor Memory with Register
 9307 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9308 %{
 9309   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9310   effect(KILL cr);
 9311   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9312 
 9313   ins_cost(150);
 9314   format %{ "xorb    $dst, $src\t# byte" %}
 9315   ins_encode %{
 9316     __ xorb($dst$$Address, $src$$Register);
 9317   %}
 9318   ins_pipe(ialu_mem_reg);
 9319 %}
 9320 
 9321 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9322 %{
 9323   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9324   effect(KILL cr);
 9325   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9326 
 9327   ins_cost(150);
 9328   format %{ "xorl    $dst, $src\t# int" %}
 9329   ins_encode %{
 9330     __ xorl($dst$$Address, $src$$Register);
 9331   %}
 9332   ins_pipe(ialu_mem_reg);
 9333 %}
 9334 
 9335 // Xor Memory with Immediate
 9336 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9337 %{
 9338   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9339   effect(KILL cr);
 9340   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9341 
 9342   ins_cost(125);
 9343   format %{ "xorl    $dst, $src\t# int" %}
 9344   ins_encode %{
 9345     __ xorl($dst$$Address, $src$$constant);
 9346   %}
 9347   ins_pipe(ialu_mem_imm);
 9348 %}
 9349 
 9350 
 9351 // Long Logical Instructions
 9352 
 9353 // And Instructions
 9354 // And Register with Register
 9355 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9356 %{
 9357   match(Set dst (AndL dst src));
 9358   effect(KILL cr);
 9359   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9360 
 9361   format %{ "andq    $dst, $src\t# long" %}
 9362   ins_encode %{
 9363     __ andq($dst$$Register, $src$$Register);
 9364   %}
 9365   ins_pipe(ialu_reg_reg);
 9366 %}
 9367 
 9368 // And Register with Immediate 255
 9369 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
 9370 %{
 9371   match(Set dst (AndL src mask));
 9372 
 9373   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
 9374   ins_encode %{
 9375     // movzbl zeroes out the upper 32-bit and does not need REX.W
 9376     __ movzbl($dst$$Register, $src$$Register);
 9377   %}
 9378   ins_pipe(ialu_reg);
 9379 %}
 9380 
 9381 // And Register with Immediate 65535
 9382 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
 9383 %{
 9384   match(Set dst (AndL src mask));
 9385 
 9386   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
 9387   ins_encode %{
 9388     // movzwl zeroes out the upper 32-bit and does not need REX.W
 9389     __ movzwl($dst$$Register, $src$$Register);
 9390   %}
 9391   ins_pipe(ialu_reg);
 9392 %}
 9393 
 9394 // And Register with Immediate
 9395 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9396 %{
 9397   match(Set dst (AndL dst src));
 9398   effect(KILL cr);
 9399   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9400 
 9401   format %{ "andq    $dst, $src\t# long" %}
 9402   ins_encode %{
 9403     __ andq($dst$$Register, $src$$constant);
 9404   %}
 9405   ins_pipe(ialu_reg);
 9406 %}
 9407 
 9408 // And Register with Memory
 9409 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9410 %{
 9411   match(Set dst (AndL dst (LoadL src)));
 9412   effect(KILL cr);
 9413   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9414 
 9415   ins_cost(150);
 9416   format %{ "andq    $dst, $src\t# long" %}
 9417   ins_encode %{
 9418     __ andq($dst$$Register, $src$$Address);
 9419   %}
 9420   ins_pipe(ialu_reg_mem);
 9421 %}
 9422 
 9423 // And Memory with Register
 9424 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9425 %{
 9426   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9427   effect(KILL cr);
 9428   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9429 
 9430   ins_cost(150);
 9431   format %{ "andq    $dst, $src\t# long" %}
 9432   ins_encode %{
 9433     __ andq($dst$$Address, $src$$Register);
 9434   %}
 9435   ins_pipe(ialu_mem_reg);
 9436 %}
 9437 
 9438 // And Memory with Immediate
 9439 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9440 %{
 9441   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9442   effect(KILL cr);
 9443   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9444 
 9445   ins_cost(125);
 9446   format %{ "andq    $dst, $src\t# long" %}
 9447   ins_encode %{
 9448     __ andq($dst$$Address, $src$$constant);
 9449   %}
 9450   ins_pipe(ialu_mem_imm);
 9451 %}
 9452 
 9453 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9454 %{
 9455   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9456   // because AND/OR works well enough for 8/32-bit values.
 9457   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9458 
 9459   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9460   effect(KILL cr);
 9461 
 9462   ins_cost(125);
 9463   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9464   ins_encode %{
 9465     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9466   %}
 9467   ins_pipe(ialu_mem_imm);
 9468 %}
 9469 
 9470 // BMI1 instructions
 9471 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9472   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9473   predicate(UseBMI1Instructions);
 9474   effect(KILL cr);
 9475   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9476 
 9477   ins_cost(125);
 9478   format %{ "andnq  $dst, $src1, $src2" %}
 9479 
 9480   ins_encode %{
 9481     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9482   %}
 9483   ins_pipe(ialu_reg_mem);
 9484 %}
 9485 
 9486 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9487   match(Set dst (AndL (XorL src1 minus_1) src2));
 9488   predicate(UseBMI1Instructions);
 9489   effect(KILL cr);
 9490   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9491 
 9492   format %{ "andnq  $dst, $src1, $src2" %}
 9493 
 9494   ins_encode %{
 9495   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9496   %}
 9497   ins_pipe(ialu_reg_mem);
 9498 %}
 9499 
 9500 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9501   match(Set dst (AndL (SubL imm_zero src) src));
 9502   predicate(UseBMI1Instructions);
 9503   effect(KILL cr);
 9504   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9505 
 9506   format %{ "blsiq  $dst, $src" %}
 9507 
 9508   ins_encode %{
 9509     __ blsiq($dst$$Register, $src$$Register);
 9510   %}
 9511   ins_pipe(ialu_reg);
 9512 %}
 9513 
 9514 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9515   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9516   predicate(UseBMI1Instructions);
 9517   effect(KILL cr);
 9518   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9519 
 9520   ins_cost(125);
 9521   format %{ "blsiq  $dst, $src" %}
 9522 
 9523   ins_encode %{
 9524     __ blsiq($dst$$Register, $src$$Address);
 9525   %}
 9526   ins_pipe(ialu_reg_mem);
 9527 %}
 9528 
 9529 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9530 %{
 9531   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9532   predicate(UseBMI1Instructions);
 9533   effect(KILL cr);
 9534   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9535 
 9536   ins_cost(125);
 9537   format %{ "blsmskq $dst, $src" %}
 9538 
 9539   ins_encode %{
 9540     __ blsmskq($dst$$Register, $src$$Address);
 9541   %}
 9542   ins_pipe(ialu_reg_mem);
 9543 %}
 9544 
 9545 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9546 %{
 9547   match(Set dst (XorL (AddL src minus_1) src));
 9548   predicate(UseBMI1Instructions);
 9549   effect(KILL cr);
 9550   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9551 
 9552   format %{ "blsmskq $dst, $src" %}
 9553 
 9554   ins_encode %{
 9555     __ blsmskq($dst$$Register, $src$$Register);
 9556   %}
 9557 
 9558   ins_pipe(ialu_reg);
 9559 %}
 9560 
 9561 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9562 %{
 9563   match(Set dst (AndL (AddL src minus_1) src) );
 9564   predicate(UseBMI1Instructions);
 9565   effect(KILL cr);
 9566   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9567 
 9568   format %{ "blsrq  $dst, $src" %}
 9569 
 9570   ins_encode %{
 9571     __ blsrq($dst$$Register, $src$$Register);
 9572   %}
 9573 
 9574   ins_pipe(ialu_reg);
 9575 %}
 9576 
 9577 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9578 %{
 9579   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9580   predicate(UseBMI1Instructions);
 9581   effect(KILL cr);
 9582   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9583 
 9584   ins_cost(125);
 9585   format %{ "blsrq  $dst, $src" %}
 9586 
 9587   ins_encode %{
 9588     __ blsrq($dst$$Register, $src$$Address);
 9589   %}
 9590 
 9591   ins_pipe(ialu_reg);
 9592 %}
 9593 
 9594 // Or Instructions
 9595 // Or Register with Register
 9596 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9597 %{
 9598   match(Set dst (OrL dst src));
 9599   effect(KILL cr);
 9600   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9601 
 9602   format %{ "orq     $dst, $src\t# long" %}
 9603   ins_encode %{
 9604     __ orq($dst$$Register, $src$$Register);
 9605   %}
 9606   ins_pipe(ialu_reg_reg);
 9607 %}
 9608 
 9609 // Use any_RegP to match R15 (TLS register) without spilling.
 9610 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 9611   match(Set dst (OrL dst (CastP2X src)));
 9612   effect(KILL cr);
 9613   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9614 
 9615   format %{ "orq     $dst, $src\t# long" %}
 9616   ins_encode %{
 9617     __ orq($dst$$Register, $src$$Register);
 9618   %}
 9619   ins_pipe(ialu_reg_reg);
 9620 %}
 9621 
 9622 
 9623 // Or Register with Immediate
 9624 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9625 %{
 9626   match(Set dst (OrL dst src));
 9627   effect(KILL cr);
 9628   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9629 
 9630   format %{ "orq     $dst, $src\t# long" %}
 9631   ins_encode %{
 9632     __ orq($dst$$Register, $src$$constant);
 9633   %}
 9634   ins_pipe(ialu_reg);
 9635 %}
 9636 
 9637 // Or Register with Memory
 9638 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9639 %{
 9640   match(Set dst (OrL dst (LoadL src)));
 9641   effect(KILL cr);
 9642   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9643 
 9644   ins_cost(150);
 9645   format %{ "orq     $dst, $src\t# long" %}
 9646   ins_encode %{
 9647     __ orq($dst$$Register, $src$$Address);
 9648   %}
 9649   ins_pipe(ialu_reg_mem);
 9650 %}
 9651 
 9652 // Or Memory with Register
 9653 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9654 %{
 9655   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9656   effect(KILL cr);
 9657   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9658 
 9659   ins_cost(150);
 9660   format %{ "orq     $dst, $src\t# long" %}
 9661   ins_encode %{
 9662     __ orq($dst$$Address, $src$$Register);
 9663   %}
 9664   ins_pipe(ialu_mem_reg);
 9665 %}
 9666 
 9667 // Or Memory with Immediate
 9668 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9669 %{
 9670   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9671   effect(KILL cr);
 9672   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9673 
 9674   ins_cost(125);
 9675   format %{ "orq     $dst, $src\t# long" %}
 9676   ins_encode %{
 9677     __ orq($dst$$Address, $src$$constant);
 9678   %}
 9679   ins_pipe(ialu_mem_imm);
 9680 %}
 9681 
 9682 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
 9683 %{
 9684   // con should be a pure 64-bit power of 2 immediate
 9685   // because AND/OR works well enough for 8/32-bit values.
 9686   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
 9687 
 9688   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
 9689   effect(KILL cr);
 9690 
 9691   ins_cost(125);
 9692   format %{ "btsq    $dst, log2($con)\t# long" %}
 9693   ins_encode %{
 9694     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
 9695   %}
 9696   ins_pipe(ialu_mem_imm);
 9697 %}
 9698 
 9699 // Xor Instructions
 9700 // Xor Register with Register
 9701 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9702 %{
 9703   match(Set dst (XorL dst src));
 9704   effect(KILL cr);
 9705   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9706 
 9707   format %{ "xorq    $dst, $src\t# long" %}
 9708   ins_encode %{
 9709     __ xorq($dst$$Register, $src$$Register);
 9710   %}
 9711   ins_pipe(ialu_reg_reg);
 9712 %}
 9713 
 9714 // Xor Register with Immediate -1
 9715 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 9716   match(Set dst (XorL dst imm));
 9717 
 9718   format %{ "notq   $dst" %}
 9719   ins_encode %{
 9720      __ notq($dst$$Register);
 9721   %}
 9722   ins_pipe(ialu_reg);
 9723 %}
 9724 
 9725 // Xor Register with Immediate
 9726 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9727 %{
 9728   match(Set dst (XorL dst src));
 9729   effect(KILL cr);
 9730   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9731 
 9732   format %{ "xorq    $dst, $src\t# long" %}
 9733   ins_encode %{
 9734     __ xorq($dst$$Register, $src$$constant);
 9735   %}
 9736   ins_pipe(ialu_reg);
 9737 %}
 9738 
 9739 // Xor Register with Memory
 9740 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9741 %{
 9742   match(Set dst (XorL dst (LoadL src)));
 9743   effect(KILL cr);
 9744   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9745 
 9746   ins_cost(150);
 9747   format %{ "xorq    $dst, $src\t# long" %}
 9748   ins_encode %{
 9749     __ xorq($dst$$Register, $src$$Address);
 9750   %}
 9751   ins_pipe(ialu_reg_mem);
 9752 %}
 9753 
 9754 // Xor Memory with Register
 9755 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9756 %{
 9757   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9758   effect(KILL cr);
 9759   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9760 
 9761   ins_cost(150);
 9762   format %{ "xorq    $dst, $src\t# long" %}
 9763   ins_encode %{
 9764     __ xorq($dst$$Address, $src$$Register);
 9765   %}
 9766   ins_pipe(ialu_mem_reg);
 9767 %}
 9768 
 9769 // Xor Memory with Immediate
 9770 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9771 %{
 9772   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9773   effect(KILL cr);
 9774   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9775 
 9776   ins_cost(125);
 9777   format %{ "xorq    $dst, $src\t# long" %}
 9778   ins_encode %{
 9779     __ xorq($dst$$Address, $src$$constant);
 9780   %}
 9781   ins_pipe(ialu_mem_imm);
 9782 %}
 9783 
 9784 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 9785 %{
 9786   match(Set dst (CmpLTMask p q));
 9787   effect(KILL cr);
 9788 
 9789   ins_cost(400);
 9790   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 9791             "setlt   $dst\n\t"
 9792             "movzbl  $dst, $dst\n\t"
 9793             "negl    $dst" %}
 9794   ins_encode %{
 9795     __ cmpl($p$$Register, $q$$Register);
 9796     __ setb(Assembler::less, $dst$$Register);
 9797     __ movzbl($dst$$Register, $dst$$Register);
 9798     __ negl($dst$$Register);
 9799   %}
 9800   ins_pipe(pipe_slow);
 9801 %}
 9802 
 9803 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
 9804 %{
 9805   match(Set dst (CmpLTMask dst zero));
 9806   effect(KILL cr);
 9807 
 9808   ins_cost(100);
 9809   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 9810   ins_encode %{
 9811     __ sarl($dst$$Register, 31);
 9812   %}
 9813   ins_pipe(ialu_reg);
 9814 %}
 9815 
 9816 /* Better to save a register than avoid a branch */
 9817 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9818 %{
 9819   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 9820   effect(KILL cr);
 9821   ins_cost(300);
 9822   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
 9823             "jge     done\n\t"
 9824             "addl    $p,$y\n"
 9825             "done:   " %}
 9826   ins_encode %{
 9827     Register Rp = $p$$Register;
 9828     Register Rq = $q$$Register;
 9829     Register Ry = $y$$Register;
 9830     Label done;
 9831     __ subl(Rp, Rq);
 9832     __ jccb(Assembler::greaterEqual, done);
 9833     __ addl(Rp, Ry);
 9834     __ bind(done);
 9835   %}
 9836   ins_pipe(pipe_cmplt);
 9837 %}
 9838 
 9839 /* Better to save a register than avoid a branch */
 9840 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9841 %{
 9842   match(Set y (AndI (CmpLTMask p q) y));
 9843   effect(KILL cr);
 9844 
 9845   ins_cost(300);
 9846 
 9847   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
 9848             "jlt     done\n\t"
 9849             "xorl    $y, $y\n"
 9850             "done:   " %}
 9851   ins_encode %{
 9852     Register Rp = $p$$Register;
 9853     Register Rq = $q$$Register;
 9854     Register Ry = $y$$Register;
 9855     Label done;
 9856     __ cmpl(Rp, Rq);
 9857     __ jccb(Assembler::less, done);
 9858     __ xorl(Ry, Ry);
 9859     __ bind(done);
 9860   %}
 9861   ins_pipe(pipe_cmplt);
 9862 %}
 9863 
 9864 
 9865 //---------- FP Instructions------------------------------------------------
 9866 
 9867 // Really expensive, avoid
 9868 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 9869 %{
 9870   match(Set cr (CmpF src1 src2));
 9871 
 9872   ins_cost(500);
 9873   format %{ "ucomiss $src1, $src2\n\t"
 9874             "jnp,s   exit\n\t"
 9875             "pushfq\t# saw NaN, set CF\n\t"
 9876             "andq    [rsp], #0xffffff2b\n\t"
 9877             "popfq\n"
 9878     "exit:" %}
 9879   ins_encode %{
 9880     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9881     emit_cmpfp_fixup(_masm);
 9882   %}
 9883   ins_pipe(pipe_slow);
 9884 %}
 9885 
 9886 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 9887   match(Set cr (CmpF src1 src2));
 9888 
 9889   ins_cost(100);
 9890   format %{ "ucomiss $src1, $src2" %}
 9891   ins_encode %{
 9892     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9893   %}
 9894   ins_pipe(pipe_slow);
 9895 %}
 9896 
 9897 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 9898   match(Set cr (CmpF src1 (LoadF src2)));
 9899 
 9900   ins_cost(100);
 9901   format %{ "ucomiss $src1, $src2" %}
 9902   ins_encode %{
 9903     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9904   %}
 9905   ins_pipe(pipe_slow);
 9906 %}
 9907 
 9908 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
 9909   match(Set cr (CmpF src con));
 9910   ins_cost(100);
 9911   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
 9912   ins_encode %{
 9913     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9914   %}
 9915   ins_pipe(pipe_slow);
 9916 %}
 9917 
 9918 // Really expensive, avoid
 9919 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 9920 %{
 9921   match(Set cr (CmpD src1 src2));
 9922 
 9923   ins_cost(500);
 9924   format %{ "ucomisd $src1, $src2\n\t"
 9925             "jnp,s   exit\n\t"
 9926             "pushfq\t# saw NaN, set CF\n\t"
 9927             "andq    [rsp], #0xffffff2b\n\t"
 9928             "popfq\n"
 9929     "exit:" %}
 9930   ins_encode %{
 9931     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9932     emit_cmpfp_fixup(_masm);
 9933   %}
 9934   ins_pipe(pipe_slow);
 9935 %}
 9936 
 9937 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 9938   match(Set cr (CmpD src1 src2));
 9939 
 9940   ins_cost(100);
 9941   format %{ "ucomisd $src1, $src2 test" %}
 9942   ins_encode %{
 9943     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9944   %}
 9945   ins_pipe(pipe_slow);
 9946 %}
 9947 
 9948 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 9949   match(Set cr (CmpD src1 (LoadD src2)));
 9950 
 9951   ins_cost(100);
 9952   format %{ "ucomisd $src1, $src2" %}
 9953   ins_encode %{
 9954     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9955   %}
 9956   ins_pipe(pipe_slow);
 9957 %}
 9958 
 9959 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
 9960   match(Set cr (CmpD src con));
 9961   ins_cost(100);
 9962   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
 9963   ins_encode %{
 9964     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9965   %}
 9966   ins_pipe(pipe_slow);
 9967 %}
 9968 
 9969 // Compare into -1,0,1
 9970 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
 9971 %{
 9972   match(Set dst (CmpF3 src1 src2));
 9973   effect(KILL cr);
 9974 
 9975   ins_cost(275);
 9976   format %{ "ucomiss $src1, $src2\n\t"
 9977             "movl    $dst, #-1\n\t"
 9978             "jp,s    done\n\t"
 9979             "jb,s    done\n\t"
 9980             "setne   $dst\n\t"
 9981             "movzbl  $dst, $dst\n"
 9982     "done:" %}
 9983   ins_encode %{
 9984     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9985     emit_cmpfp3(_masm, $dst$$Register);
 9986   %}
 9987   ins_pipe(pipe_slow);
 9988 %}
 9989 
 9990 // Compare into -1,0,1
 9991 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 9992 %{
 9993   match(Set dst (CmpF3 src1 (LoadF src2)));
 9994   effect(KILL cr);
 9995 
 9996   ins_cost(275);
 9997   format %{ "ucomiss $src1, $src2\n\t"
 9998             "movl    $dst, #-1\n\t"
 9999             "jp,s    done\n\t"
10000             "jb,s    done\n\t"
10001             "setne   $dst\n\t"
10002             "movzbl  $dst, $dst\n"
10003     "done:" %}
10004   ins_encode %{
10005     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10006     emit_cmpfp3(_masm, $dst$$Register);
10007   %}
10008   ins_pipe(pipe_slow);
10009 %}
10010 
10011 // Compare into -1,0,1
10012 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10013   match(Set dst (CmpF3 src con));
10014   effect(KILL cr);
10015 
10016   ins_cost(275);
10017   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10018             "movl    $dst, #-1\n\t"
10019             "jp,s    done\n\t"
10020             "jb,s    done\n\t"
10021             "setne   $dst\n\t"
10022             "movzbl  $dst, $dst\n"
10023     "done:" %}
10024   ins_encode %{
10025     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10026     emit_cmpfp3(_masm, $dst$$Register);
10027   %}
10028   ins_pipe(pipe_slow);
10029 %}
10030 
10031 // Compare into -1,0,1
10032 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10033 %{
10034   match(Set dst (CmpD3 src1 src2));
10035   effect(KILL cr);
10036 
10037   ins_cost(275);
10038   format %{ "ucomisd $src1, $src2\n\t"
10039             "movl    $dst, #-1\n\t"
10040             "jp,s    done\n\t"
10041             "jb,s    done\n\t"
10042             "setne   $dst\n\t"
10043             "movzbl  $dst, $dst\n"
10044     "done:" %}
10045   ins_encode %{
10046     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10047     emit_cmpfp3(_masm, $dst$$Register);
10048   %}
10049   ins_pipe(pipe_slow);
10050 %}
10051 
10052 // Compare into -1,0,1
10053 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10054 %{
10055   match(Set dst (CmpD3 src1 (LoadD src2)));
10056   effect(KILL cr);
10057 
10058   ins_cost(275);
10059   format %{ "ucomisd $src1, $src2\n\t"
10060             "movl    $dst, #-1\n\t"
10061             "jp,s    done\n\t"
10062             "jb,s    done\n\t"
10063             "setne   $dst\n\t"
10064             "movzbl  $dst, $dst\n"
10065     "done:" %}
10066   ins_encode %{
10067     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10068     emit_cmpfp3(_masm, $dst$$Register);
10069   %}
10070   ins_pipe(pipe_slow);
10071 %}
10072 
10073 // Compare into -1,0,1
10074 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10075   match(Set dst (CmpD3 src con));
10076   effect(KILL cr);
10077 
10078   ins_cost(275);
10079   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10080             "movl    $dst, #-1\n\t"
10081             "jp,s    done\n\t"
10082             "jb,s    done\n\t"
10083             "setne   $dst\n\t"
10084             "movzbl  $dst, $dst\n"
10085     "done:" %}
10086   ins_encode %{
10087     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10088     emit_cmpfp3(_masm, $dst$$Register);
10089   %}
10090   ins_pipe(pipe_slow);
10091 %}
10092 
10093 //----------Arithmetic Conversion Instructions---------------------------------
10094 
10095 instruct convF2D_reg_reg(regD dst, regF src)
10096 %{
10097   match(Set dst (ConvF2D src));
10098   effect(TEMP dst);
10099   format %{ "cvtss2sd $dst, $src" %}
10100   ins_encode %{
10101     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10102   %}
10103   ins_pipe(pipe_slow); // XXX
10104 %}
10105 
10106 instruct convF2D_reg_mem(regD dst, memory src)
10107 %{
10108   match(Set dst (ConvF2D (LoadF src)));
10109 
10110   format %{ "cvtss2sd $dst, $src" %}
10111   ins_encode %{
10112     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10113   %}
10114   ins_pipe(pipe_slow); // XXX
10115 %}
10116 
10117 instruct convD2F_reg_reg(regF dst, regD src)
10118 %{
10119   match(Set dst (ConvD2F src));
10120   effect(TEMP dst);
10121   format %{ "cvtsd2ss $dst, $src" %}
10122   ins_encode %{
10123     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10124   %}
10125   ins_pipe(pipe_slow); // XXX
10126 %}
10127 
10128 instruct convD2F_reg_mem(regF dst, memory src)
10129 %{
10130   match(Set dst (ConvD2F (LoadD src)));
10131 
10132   format %{ "cvtsd2ss $dst, $src" %}
10133   ins_encode %{
10134     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10135   %}
10136   ins_pipe(pipe_slow); // XXX
10137 %}
10138 
10139 // XXX do mem variants
10140 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10141 %{
10142   match(Set dst (ConvF2I src));
10143   effect(KILL cr);
10144   format %{ "convert_f2i $dst, $src" %}
10145   ins_encode %{
10146     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10147   %}
10148   ins_pipe(pipe_slow);
10149 %}
10150 
10151 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10152 %{
10153   match(Set dst (ConvF2L src));
10154   effect(KILL cr);
10155   format %{ "convert_f2l $dst, $src"%}
10156   ins_encode %{
10157     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10158   %}
10159   ins_pipe(pipe_slow);
10160 %}
10161 
10162 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10163 %{
10164   match(Set dst (ConvD2I src));
10165   effect(KILL cr);
10166   format %{ "convert_d2i $dst, $src"%}
10167   ins_encode %{
10168     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10169   %}
10170   ins_pipe(pipe_slow);
10171 %}
10172 
10173 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10174 %{
10175   match(Set dst (ConvD2L src));
10176   effect(KILL cr);
10177   format %{ "convert_d2l $dst, $src"%}
10178   ins_encode %{
10179     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10180   %}
10181   ins_pipe(pipe_slow);
10182 %}
10183 
10184 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10185 %{
10186   match(Set dst (RoundD src));
10187   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10188   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10189   ins_encode %{
10190     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10191   %}
10192   ins_pipe(pipe_slow);
10193 %}
10194 
10195 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10196 %{
10197   match(Set dst (RoundF src));
10198   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10199   format %{ "round_float $dst,$src" %}
10200   ins_encode %{
10201     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10202   %}
10203   ins_pipe(pipe_slow);
10204 %}
10205 
10206 instruct convI2F_reg_reg(regF dst, rRegI src)
10207 %{
10208   predicate(!UseXmmI2F);
10209   match(Set dst (ConvI2F src));
10210 
10211   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10212   ins_encode %{
10213     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10214   %}
10215   ins_pipe(pipe_slow); // XXX
10216 %}
10217 
10218 instruct convI2F_reg_mem(regF dst, memory src)
10219 %{
10220   match(Set dst (ConvI2F (LoadI src)));
10221 
10222   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10223   ins_encode %{
10224     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10225   %}
10226   ins_pipe(pipe_slow); // XXX
10227 %}
10228 
10229 instruct convI2D_reg_reg(regD dst, rRegI src)
10230 %{
10231   predicate(!UseXmmI2D);
10232   match(Set dst (ConvI2D src));
10233 
10234   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10235   ins_encode %{
10236     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10237   %}
10238   ins_pipe(pipe_slow); // XXX
10239 %}
10240 
10241 instruct convI2D_reg_mem(regD dst, memory src)
10242 %{
10243   match(Set dst (ConvI2D (LoadI src)));
10244 
10245   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10246   ins_encode %{
10247     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10248   %}
10249   ins_pipe(pipe_slow); // XXX
10250 %}
10251 
10252 instruct convXI2F_reg(regF dst, rRegI src)
10253 %{
10254   predicate(UseXmmI2F);
10255   match(Set dst (ConvI2F src));
10256 
10257   format %{ "movdl $dst, $src\n\t"
10258             "cvtdq2psl $dst, $dst\t# i2f" %}
10259   ins_encode %{
10260     __ movdl($dst$$XMMRegister, $src$$Register);
10261     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10262   %}
10263   ins_pipe(pipe_slow); // XXX
10264 %}
10265 
10266 instruct convXI2D_reg(regD dst, rRegI src)
10267 %{
10268   predicate(UseXmmI2D);
10269   match(Set dst (ConvI2D src));
10270 
10271   format %{ "movdl $dst, $src\n\t"
10272             "cvtdq2pdl $dst, $dst\t# i2d" %}
10273   ins_encode %{
10274     __ movdl($dst$$XMMRegister, $src$$Register);
10275     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10276   %}
10277   ins_pipe(pipe_slow); // XXX
10278 %}
10279 
10280 instruct convL2F_reg_reg(regF dst, rRegL src)
10281 %{
10282   match(Set dst (ConvL2F src));
10283 
10284   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10285   ins_encode %{
10286     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10287   %}
10288   ins_pipe(pipe_slow); // XXX
10289 %}
10290 
10291 instruct convL2F_reg_mem(regF dst, memory src)
10292 %{
10293   match(Set dst (ConvL2F (LoadL src)));
10294 
10295   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10296   ins_encode %{
10297     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10298   %}
10299   ins_pipe(pipe_slow); // XXX
10300 %}
10301 
10302 instruct convL2D_reg_reg(regD dst, rRegL src)
10303 %{
10304   match(Set dst (ConvL2D src));
10305 
10306   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10307   ins_encode %{
10308     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10309   %}
10310   ins_pipe(pipe_slow); // XXX
10311 %}
10312 
10313 instruct convL2D_reg_mem(regD dst, memory src)
10314 %{
10315   match(Set dst (ConvL2D (LoadL src)));
10316 
10317   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10318   ins_encode %{
10319     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10320   %}
10321   ins_pipe(pipe_slow); // XXX
10322 %}
10323 
10324 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10325 %{
10326   match(Set dst (ConvI2L src));
10327 
10328   ins_cost(125);
10329   format %{ "movslq  $dst, $src\t# i2l" %}
10330   ins_encode %{
10331     __ movslq($dst$$Register, $src$$Register);
10332   %}
10333   ins_pipe(ialu_reg_reg);
10334 %}
10335 
10336 // Zero-extend convert int to long
10337 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10338 %{
10339   match(Set dst (AndL (ConvI2L src) mask));
10340 
10341   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10342   ins_encode %{
10343     if ($dst$$reg != $src$$reg) {
10344       __ movl($dst$$Register, $src$$Register);
10345     }
10346   %}
10347   ins_pipe(ialu_reg_reg);
10348 %}
10349 
10350 // Zero-extend convert int to long
10351 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10352 %{
10353   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10354 
10355   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10356   ins_encode %{
10357     __ movl($dst$$Register, $src$$Address);
10358   %}
10359   ins_pipe(ialu_reg_mem);
10360 %}
10361 
10362 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10363 %{
10364   match(Set dst (AndL src mask));
10365 
10366   format %{ "movl    $dst, $src\t# zero-extend long" %}
10367   ins_encode %{
10368     __ movl($dst$$Register, $src$$Register);
10369   %}
10370   ins_pipe(ialu_reg_reg);
10371 %}
10372 
10373 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10374 %{
10375   match(Set dst (ConvL2I src));
10376 
10377   format %{ "movl    $dst, $src\t# l2i" %}
10378   ins_encode %{
10379     __ movl($dst$$Register, $src$$Register);
10380   %}
10381   ins_pipe(ialu_reg_reg);
10382 %}
10383 
10384 
10385 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10386   match(Set dst (MoveF2I src));
10387   effect(DEF dst, USE src);
10388 
10389   ins_cost(125);
10390   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10391   ins_encode %{
10392     __ movl($dst$$Register, Address(rsp, $src$$disp));
10393   %}
10394   ins_pipe(ialu_reg_mem);
10395 %}
10396 
10397 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10398   match(Set dst (MoveI2F src));
10399   effect(DEF dst, USE src);
10400 
10401   ins_cost(125);
10402   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10403   ins_encode %{
10404     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10405   %}
10406   ins_pipe(pipe_slow);
10407 %}
10408 
10409 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10410   match(Set dst (MoveD2L src));
10411   effect(DEF dst, USE src);
10412 
10413   ins_cost(125);
10414   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10415   ins_encode %{
10416     __ movq($dst$$Register, Address(rsp, $src$$disp));
10417   %}
10418   ins_pipe(ialu_reg_mem);
10419 %}
10420 
10421 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10422   predicate(!UseXmmLoadAndClearUpper);
10423   match(Set dst (MoveL2D src));
10424   effect(DEF dst, USE src);
10425 
10426   ins_cost(125);
10427   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10428   ins_encode %{
10429     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10430   %}
10431   ins_pipe(pipe_slow);
10432 %}
10433 
10434 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10435   predicate(UseXmmLoadAndClearUpper);
10436   match(Set dst (MoveL2D src));
10437   effect(DEF dst, USE src);
10438 
10439   ins_cost(125);
10440   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10441   ins_encode %{
10442     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10443   %}
10444   ins_pipe(pipe_slow);
10445 %}
10446 
10447 
10448 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10449   match(Set dst (MoveF2I src));
10450   effect(DEF dst, USE src);
10451 
10452   ins_cost(95); // XXX
10453   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10454   ins_encode %{
10455     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10456   %}
10457   ins_pipe(pipe_slow);
10458 %}
10459 
10460 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10461   match(Set dst (MoveI2F src));
10462   effect(DEF dst, USE src);
10463 
10464   ins_cost(100);
10465   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10466   ins_encode %{
10467     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10468   %}
10469   ins_pipe( ialu_mem_reg );
10470 %}
10471 
10472 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10473   match(Set dst (MoveD2L src));
10474   effect(DEF dst, USE src);
10475 
10476   ins_cost(95); // XXX
10477   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10478   ins_encode %{
10479     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10480   %}
10481   ins_pipe(pipe_slow);
10482 %}
10483 
10484 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10485   match(Set dst (MoveL2D src));
10486   effect(DEF dst, USE src);
10487 
10488   ins_cost(100);
10489   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10490   ins_encode %{
10491     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10492   %}
10493   ins_pipe(ialu_mem_reg);
10494 %}
10495 
10496 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10497   match(Set dst (MoveF2I src));
10498   effect(DEF dst, USE src);
10499   ins_cost(85);
10500   format %{ "movd    $dst,$src\t# MoveF2I" %}
10501   ins_encode %{
10502     __ movdl($dst$$Register, $src$$XMMRegister);
10503   %}
10504   ins_pipe( pipe_slow );
10505 %}
10506 
10507 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10508   match(Set dst (MoveD2L src));
10509   effect(DEF dst, USE src);
10510   ins_cost(85);
10511   format %{ "movd    $dst,$src\t# MoveD2L" %}
10512   ins_encode %{
10513     __ movdq($dst$$Register, $src$$XMMRegister);
10514   %}
10515   ins_pipe( pipe_slow );
10516 %}
10517 
10518 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10519   match(Set dst (MoveI2F src));
10520   effect(DEF dst, USE src);
10521   ins_cost(100);
10522   format %{ "movd    $dst,$src\t# MoveI2F" %}
10523   ins_encode %{
10524     __ movdl($dst$$XMMRegister, $src$$Register);
10525   %}
10526   ins_pipe( pipe_slow );
10527 %}
10528 
10529 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10530   match(Set dst (MoveL2D src));
10531   effect(DEF dst, USE src);
10532   ins_cost(100);
10533   format %{ "movd    $dst,$src\t# MoveL2D" %}
10534   ins_encode %{
10535      __ movdq($dst$$XMMRegister, $src$$Register);
10536   %}
10537   ins_pipe( pipe_slow );
10538 %}
10539 
10540 // Fast clearing of an array
10541 // Small ClearArray non-AVX512.
10542 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10543                   Universe dummy, rFlagsReg cr)
10544 %{
10545   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
10546   match(Set dummy (ClearArray cnt base));
10547   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10548 
10549   format %{ $$template
10550     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10551     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10552     $$emit$$"jg      LARGE\n\t"
10553     $$emit$$"dec     rcx\n\t"
10554     $$emit$$"js      DONE\t# Zero length\n\t"
10555     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10556     $$emit$$"dec     rcx\n\t"
10557     $$emit$$"jge     LOOP\n\t"
10558     $$emit$$"jmp     DONE\n\t"
10559     $$emit$$"# LARGE:\n\t"
10560     if (UseFastStosb) {
10561        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10562        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10563     } else if (UseXMMForObjInit) {
10564        $$emit$$"mov     rdi,rax\n\t"
10565        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10566        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10567        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10568        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10569        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10570        $$emit$$"add     0x40,rax\n\t"
10571        $$emit$$"# L_zero_64_bytes:\n\t"
10572        $$emit$$"sub     0x8,rcx\n\t"
10573        $$emit$$"jge     L_loop\n\t"
10574        $$emit$$"add     0x4,rcx\n\t"
10575        $$emit$$"jl      L_tail\n\t"
10576        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10577        $$emit$$"add     0x20,rax\n\t"
10578        $$emit$$"sub     0x4,rcx\n\t"
10579        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10580        $$emit$$"add     0x4,rcx\n\t"
10581        $$emit$$"jle     L_end\n\t"
10582        $$emit$$"dec     rcx\n\t"
10583        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10584        $$emit$$"vmovq   xmm0,(rax)\n\t"
10585        $$emit$$"add     0x8,rax\n\t"
10586        $$emit$$"dec     rcx\n\t"
10587        $$emit$$"jge     L_sloop\n\t"
10588        $$emit$$"# L_end:\n\t"
10589     } else {
10590        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10591     }
10592     $$emit$$"# DONE"
10593   %}
10594   ins_encode %{
10595     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10596                  $tmp$$XMMRegister, false, knoreg);
10597   %}
10598   ins_pipe(pipe_slow);
10599 %}
10600 
10601 // Small ClearArray AVX512 non-constant length.
10602 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10603                        Universe dummy, rFlagsReg cr)
10604 %{
10605   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
10606   match(Set dummy (ClearArray cnt base));
10607   ins_cost(125);
10608   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10609 
10610   format %{ $$template
10611     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10612     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10613     $$emit$$"jg      LARGE\n\t"
10614     $$emit$$"dec     rcx\n\t"
10615     $$emit$$"js      DONE\t# Zero length\n\t"
10616     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10617     $$emit$$"dec     rcx\n\t"
10618     $$emit$$"jge     LOOP\n\t"
10619     $$emit$$"jmp     DONE\n\t"
10620     $$emit$$"# LARGE:\n\t"
10621     if (UseFastStosb) {
10622        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10623        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10624     } else if (UseXMMForObjInit) {
10625        $$emit$$"mov     rdi,rax\n\t"
10626        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10627        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10628        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10629        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10630        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10631        $$emit$$"add     0x40,rax\n\t"
10632        $$emit$$"# L_zero_64_bytes:\n\t"
10633        $$emit$$"sub     0x8,rcx\n\t"
10634        $$emit$$"jge     L_loop\n\t"
10635        $$emit$$"add     0x4,rcx\n\t"
10636        $$emit$$"jl      L_tail\n\t"
10637        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10638        $$emit$$"add     0x20,rax\n\t"
10639        $$emit$$"sub     0x4,rcx\n\t"
10640        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10641        $$emit$$"add     0x4,rcx\n\t"
10642        $$emit$$"jle     L_end\n\t"
10643        $$emit$$"dec     rcx\n\t"
10644        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10645        $$emit$$"vmovq   xmm0,(rax)\n\t"
10646        $$emit$$"add     0x8,rax\n\t"
10647        $$emit$$"dec     rcx\n\t"
10648        $$emit$$"jge     L_sloop\n\t"
10649        $$emit$$"# L_end:\n\t"
10650     } else {
10651        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10652     }
10653     $$emit$$"# DONE"
10654   %}
10655   ins_encode %{
10656     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10657                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
10658   %}
10659   ins_pipe(pipe_slow);
10660 %}
10661 
10662 // Large ClearArray non-AVX512.
10663 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10664                         Universe dummy, rFlagsReg cr)
10665 %{
10666   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
10667   match(Set dummy (ClearArray cnt base));
10668   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10669 
10670   format %{ $$template
10671     if (UseFastStosb) {
10672        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10673        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10674        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10675     } else if (UseXMMForObjInit) {
10676        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10677        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10678        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10679        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10680        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10681        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10682        $$emit$$"add     0x40,rax\n\t"
10683        $$emit$$"# L_zero_64_bytes:\n\t"
10684        $$emit$$"sub     0x8,rcx\n\t"
10685        $$emit$$"jge     L_loop\n\t"
10686        $$emit$$"add     0x4,rcx\n\t"
10687        $$emit$$"jl      L_tail\n\t"
10688        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10689        $$emit$$"add     0x20,rax\n\t"
10690        $$emit$$"sub     0x4,rcx\n\t"
10691        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10692        $$emit$$"add     0x4,rcx\n\t"
10693        $$emit$$"jle     L_end\n\t"
10694        $$emit$$"dec     rcx\n\t"
10695        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10696        $$emit$$"vmovq   xmm0,(rax)\n\t"
10697        $$emit$$"add     0x8,rax\n\t"
10698        $$emit$$"dec     rcx\n\t"
10699        $$emit$$"jge     L_sloop\n\t"
10700        $$emit$$"# L_end:\n\t"
10701     } else {
10702        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10703        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10704     }
10705   %}
10706   ins_encode %{
10707     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10708                  $tmp$$XMMRegister, true, knoreg);
10709   %}
10710   ins_pipe(pipe_slow);
10711 %}
10712 
10713 // Large ClearArray AVX512.
10714 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10715                              Universe dummy, rFlagsReg cr)
10716 %{
10717   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
10718   match(Set dummy (ClearArray cnt base));
10719   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10720 
10721   format %{ $$template
10722     if (UseFastStosb) {
10723        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10724        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10725        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10726     } else if (UseXMMForObjInit) {
10727        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10728        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10729        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10730        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10731        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10732        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10733        $$emit$$"add     0x40,rax\n\t"
10734        $$emit$$"# L_zero_64_bytes:\n\t"
10735        $$emit$$"sub     0x8,rcx\n\t"
10736        $$emit$$"jge     L_loop\n\t"
10737        $$emit$$"add     0x4,rcx\n\t"
10738        $$emit$$"jl      L_tail\n\t"
10739        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10740        $$emit$$"add     0x20,rax\n\t"
10741        $$emit$$"sub     0x4,rcx\n\t"
10742        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10743        $$emit$$"add     0x4,rcx\n\t"
10744        $$emit$$"jle     L_end\n\t"
10745        $$emit$$"dec     rcx\n\t"
10746        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10747        $$emit$$"vmovq   xmm0,(rax)\n\t"
10748        $$emit$$"add     0x8,rax\n\t"
10749        $$emit$$"dec     rcx\n\t"
10750        $$emit$$"jge     L_sloop\n\t"
10751        $$emit$$"# L_end:\n\t"
10752     } else {
10753        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10754        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10755     }
10756   %}
10757   ins_encode %{
10758     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10759                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
10760   %}
10761   ins_pipe(pipe_slow);
10762 %}
10763 
10764 // Small ClearArray AVX512 constant length.
10765 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
10766 %{
10767   predicate(!((ClearArrayNode*)n)->is_large() &&
10768               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
10769   match(Set dummy (ClearArray cnt base));
10770   ins_cost(100);
10771   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
10772   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
10773   ins_encode %{
10774    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10775   %}
10776   ins_pipe(pipe_slow);
10777 %}
10778 
10779 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10780                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10781 %{
10782   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10783   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10784   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10785 
10786   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10787   ins_encode %{
10788     __ string_compare($str1$$Register, $str2$$Register,
10789                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10790                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
10791   %}
10792   ins_pipe( pipe_slow );
10793 %}
10794 
10795 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10796                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10797 %{
10798   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10799   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10800   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10801 
10802   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10803   ins_encode %{
10804     __ string_compare($str1$$Register, $str2$$Register,
10805                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10806                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
10807   %}
10808   ins_pipe( pipe_slow );
10809 %}
10810 
10811 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10812                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10813 %{
10814   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10815   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10816   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10817 
10818   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10819   ins_encode %{
10820     __ string_compare($str1$$Register, $str2$$Register,
10821                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10822                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
10823   %}
10824   ins_pipe( pipe_slow );
10825 %}
10826 
10827 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10828                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10829 %{
10830   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10831   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10832   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10833 
10834   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10835   ins_encode %{
10836     __ string_compare($str1$$Register, $str2$$Register,
10837                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10838                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
10839   %}
10840   ins_pipe( pipe_slow );
10841 %}
10842 
10843 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10844                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10845 %{
10846   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10847   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10848   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10849 
10850   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10851   ins_encode %{
10852     __ string_compare($str1$$Register, $str2$$Register,
10853                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10854                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
10855   %}
10856   ins_pipe( pipe_slow );
10857 %}
10858 
10859 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10860                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10861 %{
10862   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10863   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10864   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10865 
10866   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10867   ins_encode %{
10868     __ string_compare($str1$$Register, $str2$$Register,
10869                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10870                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
10871   %}
10872   ins_pipe( pipe_slow );
10873 %}
10874 
10875 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10876                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10877 %{
10878   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10879   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10880   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10881 
10882   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10883   ins_encode %{
10884     __ string_compare($str2$$Register, $str1$$Register,
10885                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10886                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
10887   %}
10888   ins_pipe( pipe_slow );
10889 %}
10890 
10891 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10892                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10893 %{
10894   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10895   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10896   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10897 
10898   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10899   ins_encode %{
10900     __ string_compare($str2$$Register, $str1$$Register,
10901                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10902                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
10903   %}
10904   ins_pipe( pipe_slow );
10905 %}
10906 
10907 // fast search of substring with known size.
10908 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10909                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10910 %{
10911   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10912   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10913   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10914 
10915   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10916   ins_encode %{
10917     int icnt2 = (int)$int_cnt2$$constant;
10918     if (icnt2 >= 16) {
10919       // IndexOf for constant substrings with size >= 16 elements
10920       // which don't need to be loaded through stack.
10921       __ string_indexofC8($str1$$Register, $str2$$Register,
10922                           $cnt1$$Register, $cnt2$$Register,
10923                           icnt2, $result$$Register,
10924                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10925     } else {
10926       // Small strings are loaded through stack if they cross page boundary.
10927       __ string_indexof($str1$$Register, $str2$$Register,
10928                         $cnt1$$Register, $cnt2$$Register,
10929                         icnt2, $result$$Register,
10930                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10931     }
10932   %}
10933   ins_pipe( pipe_slow );
10934 %}
10935 
10936 // fast search of substring with known size.
10937 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10938                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10939 %{
10940   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10941   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10942   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10943 
10944   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10945   ins_encode %{
10946     int icnt2 = (int)$int_cnt2$$constant;
10947     if (icnt2 >= 8) {
10948       // IndexOf for constant substrings with size >= 8 elements
10949       // which don't need to be loaded through stack.
10950       __ string_indexofC8($str1$$Register, $str2$$Register,
10951                           $cnt1$$Register, $cnt2$$Register,
10952                           icnt2, $result$$Register,
10953                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10954     } else {
10955       // Small strings are loaded through stack if they cross page boundary.
10956       __ string_indexof($str1$$Register, $str2$$Register,
10957                         $cnt1$$Register, $cnt2$$Register,
10958                         icnt2, $result$$Register,
10959                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10960     }
10961   %}
10962   ins_pipe( pipe_slow );
10963 %}
10964 
10965 // fast search of substring with known size.
10966 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10967                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10968 %{
10969   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10970   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10971   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10972 
10973   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10974   ins_encode %{
10975     int icnt2 = (int)$int_cnt2$$constant;
10976     if (icnt2 >= 8) {
10977       // IndexOf for constant substrings with size >= 8 elements
10978       // which don't need to be loaded through stack.
10979       __ string_indexofC8($str1$$Register, $str2$$Register,
10980                           $cnt1$$Register, $cnt2$$Register,
10981                           icnt2, $result$$Register,
10982                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10983     } else {
10984       // Small strings are loaded through stack if they cross page boundary.
10985       __ string_indexof($str1$$Register, $str2$$Register,
10986                         $cnt1$$Register, $cnt2$$Register,
10987                         icnt2, $result$$Register,
10988                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10989     }
10990   %}
10991   ins_pipe( pipe_slow );
10992 %}
10993 
10994 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10995                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10996 %{
10997   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10998   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10999   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11000 
11001   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11002   ins_encode %{
11003     __ string_indexof($str1$$Register, $str2$$Register,
11004                       $cnt1$$Register, $cnt2$$Register,
11005                       (-1), $result$$Register,
11006                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11007   %}
11008   ins_pipe( pipe_slow );
11009 %}
11010 
11011 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11012                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11013 %{
11014   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11015   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11016   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11017 
11018   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11019   ins_encode %{
11020     __ string_indexof($str1$$Register, $str2$$Register,
11021                       $cnt1$$Register, $cnt2$$Register,
11022                       (-1), $result$$Register,
11023                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11024   %}
11025   ins_pipe( pipe_slow );
11026 %}
11027 
11028 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11029                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11030 %{
11031   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11032   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11033   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11034 
11035   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11036   ins_encode %{
11037     __ string_indexof($str1$$Register, $str2$$Register,
11038                       $cnt1$$Register, $cnt2$$Register,
11039                       (-1), $result$$Register,
11040                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11041   %}
11042   ins_pipe( pipe_slow );
11043 %}
11044 
11045 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11046                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11047 %{
11048   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11049   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11050   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11051   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11052   ins_encode %{
11053     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11054                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11055   %}
11056   ins_pipe( pipe_slow );
11057 %}
11058 
11059 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11060                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11061 %{
11062   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11063   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11064   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11065   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11066   ins_encode %{
11067     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11068                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11069   %}
11070   ins_pipe( pipe_slow );
11071 %}
11072 
11073 // fast string equals
11074 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11075                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11076 %{
11077   predicate(!VM_Version::supports_avx512vlbw());
11078   match(Set result (StrEquals (Binary str1 str2) cnt));
11079   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11080 
11081   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11082   ins_encode %{
11083     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11084                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11085                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11086   %}
11087   ins_pipe( pipe_slow );
11088 %}
11089 
11090 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11091                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11092 %{
11093   predicate(VM_Version::supports_avx512vlbw());
11094   match(Set result (StrEquals (Binary str1 str2) cnt));
11095   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11096 
11097   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11098   ins_encode %{
11099     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11100                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11101                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11102   %}
11103   ins_pipe( pipe_slow );
11104 %}
11105 
11106 // fast array equals
11107 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11108                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11109 %{
11110   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11111   match(Set result (AryEq ary1 ary2));
11112   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11113 
11114   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11115   ins_encode %{
11116     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11117                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11118                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11119   %}
11120   ins_pipe( pipe_slow );
11121 %}
11122 
11123 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11124                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11125 %{
11126   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11127   match(Set result (AryEq ary1 ary2));
11128   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11129 
11130   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11131   ins_encode %{
11132     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11133                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11134                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11135   %}
11136   ins_pipe( pipe_slow );
11137 %}
11138 
11139 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11140                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11141 %{
11142   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11143   match(Set result (AryEq ary1 ary2));
11144   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11145 
11146   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11147   ins_encode %{
11148     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11149                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11150                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11151   %}
11152   ins_pipe( pipe_slow );
11153 %}
11154 
11155 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11156                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11157 %{
11158   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11159   match(Set result (AryEq ary1 ary2));
11160   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11161 
11162   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11163   ins_encode %{
11164     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11165                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11166                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11167   %}
11168   ins_pipe( pipe_slow );
11169 %}
11170 
11171 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11172                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11173                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11174                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11175                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11176 %{
11177   predicate(UseAVX >= 2);
11178   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11179   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11180          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11181          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11182          USE basic_type, KILL cr);
11183 
11184   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
11185   ins_encode %{
11186     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11187                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11188                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11189                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11190                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11191                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11192                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11193   %}
11194   ins_pipe( pipe_slow );
11195 %}
11196 
11197 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11198                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11199 %{
11200   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11201   match(Set result (CountPositives ary1 len));
11202   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11203 
11204   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11205   ins_encode %{
11206     __ count_positives($ary1$$Register, $len$$Register,
11207                        $result$$Register, $tmp3$$Register,
11208                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11209   %}
11210   ins_pipe( pipe_slow );
11211 %}
11212 
11213 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11214                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11215 %{
11216   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11217   match(Set result (CountPositives ary1 len));
11218   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11219 
11220   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11221   ins_encode %{
11222     __ count_positives($ary1$$Register, $len$$Register,
11223                        $result$$Register, $tmp3$$Register,
11224                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11225   %}
11226   ins_pipe( pipe_slow );
11227 %}
11228 
11229 // fast char[] to byte[] compression
11230 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11231                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11232   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11233   match(Set result (StrCompressedCopy src (Binary dst len)));
11234   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11235          USE_KILL len, KILL tmp5, KILL cr);
11236 
11237   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11238   ins_encode %{
11239     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11240                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11241                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11242                            knoreg, knoreg);
11243   %}
11244   ins_pipe( pipe_slow );
11245 %}
11246 
11247 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11248                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11249   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11250   match(Set result (StrCompressedCopy src (Binary dst len)));
11251   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11252          USE_KILL len, KILL tmp5, KILL cr);
11253 
11254   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11255   ins_encode %{
11256     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11257                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11258                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11259                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11260   %}
11261   ins_pipe( pipe_slow );
11262 %}
11263 // fast byte[] to char[] inflation
11264 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11265                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11266   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11267   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11268   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11269 
11270   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11271   ins_encode %{
11272     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11273                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11274   %}
11275   ins_pipe( pipe_slow );
11276 %}
11277 
11278 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11279                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11280   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11281   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11282   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11283 
11284   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11285   ins_encode %{
11286     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11287                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11288   %}
11289   ins_pipe( pipe_slow );
11290 %}
11291 
11292 // encode char[] to byte[] in ISO_8859_1
11293 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11294                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11295                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11296   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11297   match(Set result (EncodeISOArray src (Binary dst len)));
11298   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11299 
11300   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11301   ins_encode %{
11302     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11303                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11304                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11305   %}
11306   ins_pipe( pipe_slow );
11307 %}
11308 
11309 // encode char[] to byte[] in ASCII
11310 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11311                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11312                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11313   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11314   match(Set result (EncodeISOArray src (Binary dst len)));
11315   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11316 
11317   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11318   ins_encode %{
11319     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11320                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11321                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11322   %}
11323   ins_pipe( pipe_slow );
11324 %}
11325 
11326 //----------Overflow Math Instructions-----------------------------------------
11327 
11328 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11329 %{
11330   match(Set cr (OverflowAddI op1 op2));
11331   effect(DEF cr, USE_KILL op1, USE op2);
11332 
11333   format %{ "addl    $op1, $op2\t# overflow check int" %}
11334 
11335   ins_encode %{
11336     __ addl($op1$$Register, $op2$$Register);
11337   %}
11338   ins_pipe(ialu_reg_reg);
11339 %}
11340 
11341 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11342 %{
11343   match(Set cr (OverflowAddI op1 op2));
11344   effect(DEF cr, USE_KILL op1, USE op2);
11345 
11346   format %{ "addl    $op1, $op2\t# overflow check int" %}
11347 
11348   ins_encode %{
11349     __ addl($op1$$Register, $op2$$constant);
11350   %}
11351   ins_pipe(ialu_reg_reg);
11352 %}
11353 
11354 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11355 %{
11356   match(Set cr (OverflowAddL op1 op2));
11357   effect(DEF cr, USE_KILL op1, USE op2);
11358 
11359   format %{ "addq    $op1, $op2\t# overflow check long" %}
11360   ins_encode %{
11361     __ addq($op1$$Register, $op2$$Register);
11362   %}
11363   ins_pipe(ialu_reg_reg);
11364 %}
11365 
11366 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11367 %{
11368   match(Set cr (OverflowAddL op1 op2));
11369   effect(DEF cr, USE_KILL op1, USE op2);
11370 
11371   format %{ "addq    $op1, $op2\t# overflow check long" %}
11372   ins_encode %{
11373     __ addq($op1$$Register, $op2$$constant);
11374   %}
11375   ins_pipe(ialu_reg_reg);
11376 %}
11377 
11378 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11379 %{
11380   match(Set cr (OverflowSubI op1 op2));
11381 
11382   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11383   ins_encode %{
11384     __ cmpl($op1$$Register, $op2$$Register);
11385   %}
11386   ins_pipe(ialu_reg_reg);
11387 %}
11388 
11389 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11390 %{
11391   match(Set cr (OverflowSubI op1 op2));
11392 
11393   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11394   ins_encode %{
11395     __ cmpl($op1$$Register, $op2$$constant);
11396   %}
11397   ins_pipe(ialu_reg_reg);
11398 %}
11399 
11400 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11401 %{
11402   match(Set cr (OverflowSubL op1 op2));
11403 
11404   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11405   ins_encode %{
11406     __ cmpq($op1$$Register, $op2$$Register);
11407   %}
11408   ins_pipe(ialu_reg_reg);
11409 %}
11410 
11411 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11412 %{
11413   match(Set cr (OverflowSubL op1 op2));
11414 
11415   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11416   ins_encode %{
11417     __ cmpq($op1$$Register, $op2$$constant);
11418   %}
11419   ins_pipe(ialu_reg_reg);
11420 %}
11421 
11422 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11423 %{
11424   match(Set cr (OverflowSubI zero op2));
11425   effect(DEF cr, USE_KILL op2);
11426 
11427   format %{ "negl    $op2\t# overflow check int" %}
11428   ins_encode %{
11429     __ negl($op2$$Register);
11430   %}
11431   ins_pipe(ialu_reg_reg);
11432 %}
11433 
11434 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11435 %{
11436   match(Set cr (OverflowSubL zero op2));
11437   effect(DEF cr, USE_KILL op2);
11438 
11439   format %{ "negq    $op2\t# overflow check long" %}
11440   ins_encode %{
11441     __ negq($op2$$Register);
11442   %}
11443   ins_pipe(ialu_reg_reg);
11444 %}
11445 
11446 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11447 %{
11448   match(Set cr (OverflowMulI op1 op2));
11449   effect(DEF cr, USE_KILL op1, USE op2);
11450 
11451   format %{ "imull    $op1, $op2\t# overflow check int" %}
11452   ins_encode %{
11453     __ imull($op1$$Register, $op2$$Register);
11454   %}
11455   ins_pipe(ialu_reg_reg_alu0);
11456 %}
11457 
11458 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11459 %{
11460   match(Set cr (OverflowMulI op1 op2));
11461   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11462 
11463   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11464   ins_encode %{
11465     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11466   %}
11467   ins_pipe(ialu_reg_reg_alu0);
11468 %}
11469 
11470 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11471 %{
11472   match(Set cr (OverflowMulL op1 op2));
11473   effect(DEF cr, USE_KILL op1, USE op2);
11474 
11475   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11476   ins_encode %{
11477     __ imulq($op1$$Register, $op2$$Register);
11478   %}
11479   ins_pipe(ialu_reg_reg_alu0);
11480 %}
11481 
11482 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11483 %{
11484   match(Set cr (OverflowMulL op1 op2));
11485   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11486 
11487   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11488   ins_encode %{
11489     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11490   %}
11491   ins_pipe(ialu_reg_reg_alu0);
11492 %}
11493 
11494 
11495 //----------Control Flow Instructions------------------------------------------
11496 // Signed compare Instructions
11497 
11498 // XXX more variants!!
11499 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11500 %{
11501   match(Set cr (CmpI op1 op2));
11502   effect(DEF cr, USE op1, USE op2);
11503 
11504   format %{ "cmpl    $op1, $op2" %}
11505   ins_encode %{
11506     __ cmpl($op1$$Register, $op2$$Register);
11507   %}
11508   ins_pipe(ialu_cr_reg_reg);
11509 %}
11510 
11511 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11512 %{
11513   match(Set cr (CmpI op1 op2));
11514 
11515   format %{ "cmpl    $op1, $op2" %}
11516   ins_encode %{
11517     __ cmpl($op1$$Register, $op2$$constant);
11518   %}
11519   ins_pipe(ialu_cr_reg_imm);
11520 %}
11521 
11522 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11523 %{
11524   match(Set cr (CmpI op1 (LoadI op2)));
11525 
11526   ins_cost(500); // XXX
11527   format %{ "cmpl    $op1, $op2" %}
11528   ins_encode %{
11529     __ cmpl($op1$$Register, $op2$$Address);
11530   %}
11531   ins_pipe(ialu_cr_reg_mem);
11532 %}
11533 
11534 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11535 %{
11536   match(Set cr (CmpI src zero));
11537 
11538   format %{ "testl   $src, $src" %}
11539   ins_encode %{
11540     __ testl($src$$Register, $src$$Register);
11541   %}
11542   ins_pipe(ialu_cr_reg_imm);
11543 %}
11544 
11545 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11546 %{
11547   match(Set cr (CmpI (AndI src con) zero));
11548 
11549   format %{ "testl   $src, $con" %}
11550   ins_encode %{
11551     __ testl($src$$Register, $con$$constant);
11552   %}
11553   ins_pipe(ialu_cr_reg_imm);
11554 %}
11555 
11556 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11557 %{
11558   match(Set cr (CmpI (AndI src1 src2) zero));
11559 
11560   format %{ "testl   $src1, $src2" %}
11561   ins_encode %{
11562     __ testl($src1$$Register, $src2$$Register);
11563   %}
11564   ins_pipe(ialu_cr_reg_imm);
11565 %}
11566 
11567 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11568 %{
11569   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11570 
11571   format %{ "testl   $src, $mem" %}
11572   ins_encode %{
11573     __ testl($src$$Register, $mem$$Address);
11574   %}
11575   ins_pipe(ialu_cr_reg_mem);
11576 %}
11577 
11578 // Unsigned compare Instructions; really, same as signed except they
11579 // produce an rFlagsRegU instead of rFlagsReg.
11580 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11581 %{
11582   match(Set cr (CmpU op1 op2));
11583 
11584   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11585   ins_encode %{
11586     __ cmpl($op1$$Register, $op2$$Register);
11587   %}
11588   ins_pipe(ialu_cr_reg_reg);
11589 %}
11590 
11591 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11592 %{
11593   match(Set cr (CmpU op1 op2));
11594 
11595   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11596   ins_encode %{
11597     __ cmpl($op1$$Register, $op2$$constant);
11598   %}
11599   ins_pipe(ialu_cr_reg_imm);
11600 %}
11601 
11602 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11603 %{
11604   match(Set cr (CmpU op1 (LoadI op2)));
11605 
11606   ins_cost(500); // XXX
11607   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11608   ins_encode %{
11609     __ cmpl($op1$$Register, $op2$$Address);
11610   %}
11611   ins_pipe(ialu_cr_reg_mem);
11612 %}
11613 
11614 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11615 %{
11616   match(Set cr (CmpU src zero));
11617 
11618   format %{ "testl   $src, $src\t# unsigned" %}
11619   ins_encode %{
11620     __ testl($src$$Register, $src$$Register);
11621   %}
11622   ins_pipe(ialu_cr_reg_imm);
11623 %}
11624 
11625 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11626 %{
11627   match(Set cr (CmpP op1 op2));
11628 
11629   format %{ "cmpq    $op1, $op2\t# ptr" %}
11630   ins_encode %{
11631     __ cmpq($op1$$Register, $op2$$Register);
11632   %}
11633   ins_pipe(ialu_cr_reg_reg);
11634 %}
11635 
11636 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11637 %{
11638   match(Set cr (CmpP op1 (LoadP op2)));
11639   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11640 
11641   ins_cost(500); // XXX
11642   format %{ "cmpq    $op1, $op2\t# ptr" %}
11643   ins_encode %{
11644     __ cmpq($op1$$Register, $op2$$Address);
11645   %}
11646   ins_pipe(ialu_cr_reg_mem);
11647 %}
11648 
11649 // XXX this is generalized by compP_rReg_mem???
11650 // Compare raw pointer (used in out-of-heap check).
11651 // Only works because non-oop pointers must be raw pointers
11652 // and raw pointers have no anti-dependencies.
11653 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11654 %{
11655   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11656             n->in(2)->as_Load()->barrier_data() == 0);
11657   match(Set cr (CmpP op1 (LoadP op2)));
11658 
11659   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11660   ins_encode %{
11661     __ cmpq($op1$$Register, $op2$$Address);
11662   %}
11663   ins_pipe(ialu_cr_reg_mem);
11664 %}
11665 
11666 // This will generate a signed flags result. This should be OK since
11667 // any compare to a zero should be eq/neq.
11668 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11669 %{
11670   match(Set cr (CmpP src zero));
11671 
11672   format %{ "testq   $src, $src\t# ptr" %}
11673   ins_encode %{
11674     __ testq($src$$Register, $src$$Register);
11675   %}
11676   ins_pipe(ialu_cr_reg_imm);
11677 %}
11678 
11679 // This will generate a signed flags result. This should be OK since
11680 // any compare to a zero should be eq/neq.
11681 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11682 %{
11683   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
11684             n->in(1)->as_Load()->barrier_data() == 0);
11685   match(Set cr (CmpP (LoadP op) zero));
11686 
11687   ins_cost(500); // XXX
11688   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11689   ins_encode %{
11690     __ testq($op$$Address, 0xFFFFFFFF);
11691   %}
11692   ins_pipe(ialu_cr_reg_imm);
11693 %}
11694 
11695 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11696 %{
11697   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
11698             n->in(1)->as_Load()->barrier_data() == 0);
11699   match(Set cr (CmpP (LoadP mem) zero));
11700 
11701   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11702   ins_encode %{
11703     __ cmpq(r12, $mem$$Address);
11704   %}
11705   ins_pipe(ialu_cr_reg_mem);
11706 %}
11707 
11708 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11709 %{
11710   match(Set cr (CmpN op1 op2));
11711 
11712   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11713   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11714   ins_pipe(ialu_cr_reg_reg);
11715 %}
11716 
11717 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11718 %{
11719   match(Set cr (CmpN src (LoadN mem)));
11720 
11721   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11722   ins_encode %{
11723     __ cmpl($src$$Register, $mem$$Address);
11724   %}
11725   ins_pipe(ialu_cr_reg_mem);
11726 %}
11727 
11728 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11729   match(Set cr (CmpN op1 op2));
11730 
11731   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11732   ins_encode %{
11733     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11734   %}
11735   ins_pipe(ialu_cr_reg_imm);
11736 %}
11737 
11738 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11739 %{
11740   match(Set cr (CmpN src (LoadN mem)));
11741 
11742   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11743   ins_encode %{
11744     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11745   %}
11746   ins_pipe(ialu_cr_reg_mem);
11747 %}
11748 
11749 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11750   match(Set cr (CmpN op1 op2));
11751 
11752   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11753   ins_encode %{
11754     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11755   %}
11756   ins_pipe(ialu_cr_reg_imm);
11757 %}
11758 
11759 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11760 %{
11761   match(Set cr (CmpN src (LoadNKlass mem)));
11762 
11763   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11764   ins_encode %{
11765     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11766   %}
11767   ins_pipe(ialu_cr_reg_mem);
11768 %}
11769 
11770 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11771   match(Set cr (CmpN src zero));
11772 
11773   format %{ "testl   $src, $src\t# compressed ptr" %}
11774   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11775   ins_pipe(ialu_cr_reg_imm);
11776 %}
11777 
11778 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11779 %{
11780   predicate(CompressedOops::base() != NULL);
11781   match(Set cr (CmpN (LoadN mem) zero));
11782 
11783   ins_cost(500); // XXX
11784   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11785   ins_encode %{
11786     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11787   %}
11788   ins_pipe(ialu_cr_reg_mem);
11789 %}
11790 
11791 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11792 %{
11793   predicate(CompressedOops::base() == NULL);
11794   match(Set cr (CmpN (LoadN mem) zero));
11795 
11796   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11797   ins_encode %{
11798     __ cmpl(r12, $mem$$Address);
11799   %}
11800   ins_pipe(ialu_cr_reg_mem);
11801 %}
11802 
11803 // Yanked all unsigned pointer compare operations.
11804 // Pointer compares are done with CmpP which is already unsigned.
11805 
11806 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11807 %{
11808   match(Set cr (CmpL op1 op2));
11809 
11810   format %{ "cmpq    $op1, $op2" %}
11811   ins_encode %{
11812     __ cmpq($op1$$Register, $op2$$Register);
11813   %}
11814   ins_pipe(ialu_cr_reg_reg);
11815 %}
11816 
11817 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11818 %{
11819   match(Set cr (CmpL op1 op2));
11820 
11821   format %{ "cmpq    $op1, $op2" %}
11822   ins_encode %{
11823     __ cmpq($op1$$Register, $op2$$constant);
11824   %}
11825   ins_pipe(ialu_cr_reg_imm);
11826 %}
11827 
11828 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11829 %{
11830   match(Set cr (CmpL op1 (LoadL op2)));
11831 
11832   format %{ "cmpq    $op1, $op2" %}
11833   ins_encode %{
11834     __ cmpq($op1$$Register, $op2$$Address);
11835   %}
11836   ins_pipe(ialu_cr_reg_mem);
11837 %}
11838 
11839 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11840 %{
11841   match(Set cr (CmpL src zero));
11842 
11843   format %{ "testq   $src, $src" %}
11844   ins_encode %{
11845     __ testq($src$$Register, $src$$Register);
11846   %}
11847   ins_pipe(ialu_cr_reg_imm);
11848 %}
11849 
11850 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11851 %{
11852   match(Set cr (CmpL (AndL src con) zero));
11853 
11854   format %{ "testq   $src, $con\t# long" %}
11855   ins_encode %{
11856     __ testq($src$$Register, $con$$constant);
11857   %}
11858   ins_pipe(ialu_cr_reg_imm);
11859 %}
11860 
11861 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
11862 %{
11863   match(Set cr (CmpL (AndL src1 src2) zero));
11864 
11865   format %{ "testq   $src1, $src2\t# long" %}
11866   ins_encode %{
11867     __ testq($src1$$Register, $src2$$Register);
11868   %}
11869   ins_pipe(ialu_cr_reg_imm);
11870 %}
11871 
11872 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11873 %{
11874   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11875 
11876   format %{ "testq   $src, $mem" %}
11877   ins_encode %{
11878     __ testq($src$$Register, $mem$$Address);
11879   %}
11880   ins_pipe(ialu_cr_reg_mem);
11881 %}
11882 
11883 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11884 %{
11885   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11886 
11887   format %{ "testq   $src, $mem" %}
11888   ins_encode %{
11889     __ testq($src$$Register, $mem$$Address);
11890   %}
11891   ins_pipe(ialu_cr_reg_mem);
11892 %}
11893 
11894 // Manifest a CmpU result in an integer register.  Very painful.
11895 // This is the test to avoid.
11896 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
11897 %{
11898   match(Set dst (CmpU3 src1 src2));
11899   effect(KILL flags);
11900 
11901   ins_cost(275); // XXX
11902   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
11903             "movl    $dst, -1\n\t"
11904             "jb,u    done\n\t"
11905             "setne   $dst\n\t"
11906             "movzbl  $dst, $dst\n\t"
11907     "done:" %}
11908   ins_encode %{
11909     Label done;
11910     __ cmpl($src1$$Register, $src2$$Register);
11911     __ movl($dst$$Register, -1);
11912     __ jccb(Assembler::below, done);
11913     __ setb(Assembler::notZero, $dst$$Register);
11914     __ movzbl($dst$$Register, $dst$$Register);
11915     __ bind(done);
11916   %}
11917   ins_pipe(pipe_slow);
11918 %}
11919 
11920 // Manifest a CmpL result in an integer register.  Very painful.
11921 // This is the test to avoid.
11922 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11923 %{
11924   match(Set dst (CmpL3 src1 src2));
11925   effect(KILL flags);
11926 
11927   ins_cost(275); // XXX
11928   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11929             "movl    $dst, -1\n\t"
11930             "jl,s    done\n\t"
11931             "setne   $dst\n\t"
11932             "movzbl  $dst, $dst\n\t"
11933     "done:" %}
11934   ins_encode %{
11935     Label done;
11936     __ cmpq($src1$$Register, $src2$$Register);
11937     __ movl($dst$$Register, -1);
11938     __ jccb(Assembler::less, done);
11939     __ setb(Assembler::notZero, $dst$$Register);
11940     __ movzbl($dst$$Register, $dst$$Register);
11941     __ bind(done);
11942   %}
11943   ins_pipe(pipe_slow);
11944 %}
11945 
11946 // Manifest a CmpUL result in an integer register.  Very painful.
11947 // This is the test to avoid.
11948 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11949 %{
11950   match(Set dst (CmpUL3 src1 src2));
11951   effect(KILL flags);
11952 
11953   ins_cost(275); // XXX
11954   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11955             "movl    $dst, -1\n\t"
11956             "jb,u    done\n\t"
11957             "setne   $dst\n\t"
11958             "movzbl  $dst, $dst\n\t"
11959     "done:" %}
11960   ins_encode %{
11961     Label done;
11962     __ cmpq($src1$$Register, $src2$$Register);
11963     __ movl($dst$$Register, -1);
11964     __ jccb(Assembler::below, done);
11965     __ setb(Assembler::notZero, $dst$$Register);
11966     __ movzbl($dst$$Register, $dst$$Register);
11967     __ bind(done);
11968   %}
11969   ins_pipe(pipe_slow);
11970 %}
11971 
11972 // Unsigned long compare Instructions; really, same as signed long except they
11973 // produce an rFlagsRegU instead of rFlagsReg.
11974 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11975 %{
11976   match(Set cr (CmpUL op1 op2));
11977 
11978   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11979   ins_encode %{
11980     __ cmpq($op1$$Register, $op2$$Register);
11981   %}
11982   ins_pipe(ialu_cr_reg_reg);
11983 %}
11984 
11985 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11986 %{
11987   match(Set cr (CmpUL op1 op2));
11988 
11989   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11990   ins_encode %{
11991     __ cmpq($op1$$Register, $op2$$constant);
11992   %}
11993   ins_pipe(ialu_cr_reg_imm);
11994 %}
11995 
11996 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11997 %{
11998   match(Set cr (CmpUL op1 (LoadL op2)));
11999 
12000   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12001   ins_encode %{
12002     __ cmpq($op1$$Register, $op2$$Address);
12003   %}
12004   ins_pipe(ialu_cr_reg_mem);
12005 %}
12006 
12007 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12008 %{
12009   match(Set cr (CmpUL src zero));
12010 
12011   format %{ "testq   $src, $src\t# unsigned" %}
12012   ins_encode %{
12013     __ testq($src$$Register, $src$$Register);
12014   %}
12015   ins_pipe(ialu_cr_reg_imm);
12016 %}
12017 
12018 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12019 %{
12020   match(Set cr (CmpI (LoadB mem) imm));
12021 
12022   ins_cost(125);
12023   format %{ "cmpb    $mem, $imm" %}
12024   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12025   ins_pipe(ialu_cr_reg_mem);
12026 %}
12027 
12028 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12029 %{
12030   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12031 
12032   ins_cost(125);
12033   format %{ "testb   $mem, $imm\t# ubyte" %}
12034   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12035   ins_pipe(ialu_cr_reg_mem);
12036 %}
12037 
12038 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12039 %{
12040   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12041 
12042   ins_cost(125);
12043   format %{ "testb   $mem, $imm\t# byte" %}
12044   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12045   ins_pipe(ialu_cr_reg_mem);
12046 %}
12047 
12048 //----------Max and Min--------------------------------------------------------
12049 // Min Instructions
12050 
12051 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12052 %{
12053   effect(USE_DEF dst, USE src, USE cr);
12054 
12055   format %{ "cmovlgt $dst, $src\t# min" %}
12056   ins_encode %{
12057     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12058   %}
12059   ins_pipe(pipe_cmov_reg);
12060 %}
12061 
12062 
12063 instruct minI_rReg(rRegI dst, rRegI src)
12064 %{
12065   match(Set dst (MinI dst src));
12066 
12067   ins_cost(200);
12068   expand %{
12069     rFlagsReg cr;
12070     compI_rReg(cr, dst, src);
12071     cmovI_reg_g(dst, src, cr);
12072   %}
12073 %}
12074 
12075 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12076 %{
12077   effect(USE_DEF dst, USE src, USE cr);
12078 
12079   format %{ "cmovllt $dst, $src\t# max" %}
12080   ins_encode %{
12081     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12082   %}
12083   ins_pipe(pipe_cmov_reg);
12084 %}
12085 
12086 
12087 instruct maxI_rReg(rRegI dst, rRegI src)
12088 %{
12089   match(Set dst (MaxI dst src));
12090 
12091   ins_cost(200);
12092   expand %{
12093     rFlagsReg cr;
12094     compI_rReg(cr, dst, src);
12095     cmovI_reg_l(dst, src, cr);
12096   %}
12097 %}
12098 
12099 // ============================================================================
12100 // Branch Instructions
12101 
12102 // Jump Direct - Label defines a relative address from JMP+1
12103 instruct jmpDir(label labl)
12104 %{
12105   match(Goto);
12106   effect(USE labl);
12107 
12108   ins_cost(300);
12109   format %{ "jmp     $labl" %}
12110   size(5);
12111   ins_encode %{
12112     Label* L = $labl$$label;
12113     __ jmp(*L, false); // Always long jump
12114   %}
12115   ins_pipe(pipe_jmp);
12116 %}
12117 
12118 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12119 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12120 %{
12121   match(If cop cr);
12122   effect(USE labl);
12123 
12124   ins_cost(300);
12125   format %{ "j$cop     $labl" %}
12126   size(6);
12127   ins_encode %{
12128     Label* L = $labl$$label;
12129     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12130   %}
12131   ins_pipe(pipe_jcc);
12132 %}
12133 
12134 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12135 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12136 %{
12137   match(CountedLoopEnd cop cr);
12138   effect(USE labl);
12139 
12140   ins_cost(300);
12141   format %{ "j$cop     $labl\t# loop end" %}
12142   size(6);
12143   ins_encode %{
12144     Label* L = $labl$$label;
12145     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12146   %}
12147   ins_pipe(pipe_jcc);
12148 %}
12149 
12150 // Jump Direct Conditional - using unsigned comparison
12151 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12152   match(If cop cmp);
12153   effect(USE labl);
12154 
12155   ins_cost(300);
12156   format %{ "j$cop,u   $labl" %}
12157   size(6);
12158   ins_encode %{
12159     Label* L = $labl$$label;
12160     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12161   %}
12162   ins_pipe(pipe_jcc);
12163 %}
12164 
12165 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12166   match(If cop cmp);
12167   effect(USE labl);
12168 
12169   ins_cost(200);
12170   format %{ "j$cop,u   $labl" %}
12171   size(6);
12172   ins_encode %{
12173     Label* L = $labl$$label;
12174     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12175   %}
12176   ins_pipe(pipe_jcc);
12177 %}
12178 
12179 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12180   match(If cop cmp);
12181   effect(USE labl);
12182 
12183   ins_cost(200);
12184   format %{ $$template
12185     if ($cop$$cmpcode == Assembler::notEqual) {
12186       $$emit$$"jp,u    $labl\n\t"
12187       $$emit$$"j$cop,u   $labl"
12188     } else {
12189       $$emit$$"jp,u    done\n\t"
12190       $$emit$$"j$cop,u   $labl\n\t"
12191       $$emit$$"done:"
12192     }
12193   %}
12194   ins_encode %{
12195     Label* l = $labl$$label;
12196     if ($cop$$cmpcode == Assembler::notEqual) {
12197       __ jcc(Assembler::parity, *l, false);
12198       __ jcc(Assembler::notEqual, *l, false);
12199     } else if ($cop$$cmpcode == Assembler::equal) {
12200       Label done;
12201       __ jccb(Assembler::parity, done);
12202       __ jcc(Assembler::equal, *l, false);
12203       __ bind(done);
12204     } else {
12205        ShouldNotReachHere();
12206     }
12207   %}
12208   ins_pipe(pipe_jcc);
12209 %}
12210 
12211 // ============================================================================
12212 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12213 // superklass array for an instance of the superklass.  Set a hidden
12214 // internal cache on a hit (cache is checked with exposed code in
12215 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12216 // encoding ALSO sets flags.
12217 
12218 instruct partialSubtypeCheck(rdi_RegP result,
12219                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12220                              rFlagsReg cr)
12221 %{
12222   match(Set result (PartialSubtypeCheck sub super));
12223   effect(KILL rcx, KILL cr);
12224 
12225   ins_cost(1100);  // slightly larger than the next version
12226   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12227             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12228             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12229             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12230             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12231             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12232             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12233     "miss:\t" %}
12234 
12235   opcode(0x1); // Force a XOR of RDI
12236   ins_encode(enc_PartialSubtypeCheck());
12237   ins_pipe(pipe_slow);
12238 %}
12239 
12240 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12241                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12242                                      immP0 zero,
12243                                      rdi_RegP result)
12244 %{
12245   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12246   effect(KILL rcx, KILL result);
12247 
12248   ins_cost(1000);
12249   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12250             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12251             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12252             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12253             "jne,s   miss\t\t# Missed: flags nz\n\t"
12254             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12255     "miss:\t" %}
12256 
12257   opcode(0x0); // No need to XOR RDI
12258   ins_encode(enc_PartialSubtypeCheck());
12259   ins_pipe(pipe_slow);
12260 %}
12261 
12262 // ============================================================================
12263 // Branch Instructions -- short offset versions
12264 //
12265 // These instructions are used to replace jumps of a long offset (the default
12266 // match) with jumps of a shorter offset.  These instructions are all tagged
12267 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12268 // match rules in general matching.  Instead, the ADLC generates a conversion
12269 // method in the MachNode which can be used to do in-place replacement of the
12270 // long variant with the shorter variant.  The compiler will determine if a
12271 // branch can be taken by the is_short_branch_offset() predicate in the machine
12272 // specific code section of the file.
12273 
12274 // Jump Direct - Label defines a relative address from JMP+1
12275 instruct jmpDir_short(label labl) %{
12276   match(Goto);
12277   effect(USE labl);
12278 
12279   ins_cost(300);
12280   format %{ "jmp,s   $labl" %}
12281   size(2);
12282   ins_encode %{
12283     Label* L = $labl$$label;
12284     __ jmpb(*L);
12285   %}
12286   ins_pipe(pipe_jmp);
12287   ins_short_branch(1);
12288 %}
12289 
12290 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12291 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12292   match(If cop cr);
12293   effect(USE labl);
12294 
12295   ins_cost(300);
12296   format %{ "j$cop,s   $labl" %}
12297   size(2);
12298   ins_encode %{
12299     Label* L = $labl$$label;
12300     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12301   %}
12302   ins_pipe(pipe_jcc);
12303   ins_short_branch(1);
12304 %}
12305 
12306 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12307 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12308   match(CountedLoopEnd cop cr);
12309   effect(USE labl);
12310 
12311   ins_cost(300);
12312   format %{ "j$cop,s   $labl\t# loop end" %}
12313   size(2);
12314   ins_encode %{
12315     Label* L = $labl$$label;
12316     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12317   %}
12318   ins_pipe(pipe_jcc);
12319   ins_short_branch(1);
12320 %}
12321 
12322 // Jump Direct Conditional - using unsigned comparison
12323 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12324   match(If cop cmp);
12325   effect(USE labl);
12326 
12327   ins_cost(300);
12328   format %{ "j$cop,us  $labl" %}
12329   size(2);
12330   ins_encode %{
12331     Label* L = $labl$$label;
12332     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12333   %}
12334   ins_pipe(pipe_jcc);
12335   ins_short_branch(1);
12336 %}
12337 
12338 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12339   match(If cop cmp);
12340   effect(USE labl);
12341 
12342   ins_cost(300);
12343   format %{ "j$cop,us  $labl" %}
12344   size(2);
12345   ins_encode %{
12346     Label* L = $labl$$label;
12347     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12348   %}
12349   ins_pipe(pipe_jcc);
12350   ins_short_branch(1);
12351 %}
12352 
12353 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12354   match(If cop cmp);
12355   effect(USE labl);
12356 
12357   ins_cost(300);
12358   format %{ $$template
12359     if ($cop$$cmpcode == Assembler::notEqual) {
12360       $$emit$$"jp,u,s  $labl\n\t"
12361       $$emit$$"j$cop,u,s  $labl"
12362     } else {
12363       $$emit$$"jp,u,s  done\n\t"
12364       $$emit$$"j$cop,u,s  $labl\n\t"
12365       $$emit$$"done:"
12366     }
12367   %}
12368   size(4);
12369   ins_encode %{
12370     Label* l = $labl$$label;
12371     if ($cop$$cmpcode == Assembler::notEqual) {
12372       __ jccb(Assembler::parity, *l);
12373       __ jccb(Assembler::notEqual, *l);
12374     } else if ($cop$$cmpcode == Assembler::equal) {
12375       Label done;
12376       __ jccb(Assembler::parity, done);
12377       __ jccb(Assembler::equal, *l);
12378       __ bind(done);
12379     } else {
12380        ShouldNotReachHere();
12381     }
12382   %}
12383   ins_pipe(pipe_jcc);
12384   ins_short_branch(1);
12385 %}
12386 
12387 // ============================================================================
12388 // inlined locking and unlocking
12389 
12390 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12391   predicate(Compile::current()->use_rtm());
12392   match(Set cr (FastLock object box));
12393   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12394   ins_cost(300);
12395   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12396   ins_encode %{
12397     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12398                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
12399                  _rtm_counters, _stack_rtm_counters,
12400                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12401                  true, ra_->C->profile_rtm());
12402   %}
12403   ins_pipe(pipe_slow);
12404 %}
12405 
12406 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12407   predicate(!Compile::current()->use_rtm());
12408   match(Set cr (FastLock object box));
12409   effect(TEMP tmp, TEMP scr, USE_KILL box);
12410   ins_cost(300);
12411   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12412   ins_encode %{
12413     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12414                  $scr$$Register, noreg, noreg, r15_thread, nullptr, nullptr, nullptr, false, false);
12415   %}
12416   ins_pipe(pipe_slow);
12417 %}
12418 
12419 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12420   match(Set cr (FastUnlock object box));
12421   effect(TEMP tmp, USE_KILL box);
12422   ins_cost(300);
12423   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12424   ins_encode %{
12425     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12426   %}
12427   ins_pipe(pipe_slow);
12428 %}
12429 
12430 
12431 // ============================================================================
12432 // Safepoint Instructions
12433 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12434 %{
12435   match(SafePoint poll);
12436   effect(KILL cr, USE poll);
12437 
12438   format %{ "testl   rax, [$poll]\t"
12439             "# Safepoint: poll for GC" %}
12440   ins_cost(125);
12441   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12442   ins_encode %{
12443     __ relocate(relocInfo::poll_type);
12444     address pre_pc = __ pc();
12445     __ testl(rax, Address($poll$$Register, 0));
12446     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12447   %}
12448   ins_pipe(ialu_reg_mem);
12449 %}
12450 
12451 instruct mask_all_evexL(kReg dst, rRegL src) %{
12452   match(Set dst (MaskAll src));
12453   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12454   ins_encode %{
12455     int mask_len = Matcher::vector_length(this);
12456     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12457   %}
12458   ins_pipe( pipe_slow );
12459 %}
12460 
12461 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12462   predicate(Matcher::vector_length(n) > 32);
12463   match(Set dst (MaskAll src));
12464   effect(TEMP tmp);
12465   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12466   ins_encode %{
12467     int mask_len = Matcher::vector_length(this);
12468     __ movslq($tmp$$Register, $src$$Register);
12469     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12470   %}
12471   ins_pipe( pipe_slow );
12472 %}
12473 
12474 // ============================================================================
12475 // Procedure Call/Return Instructions
12476 // Call Java Static Instruction
12477 // Note: If this code changes, the corresponding ret_addr_offset() and
12478 //       compute_padding() functions will have to be adjusted.
12479 instruct CallStaticJavaDirect(method meth) %{
12480   match(CallStaticJava);
12481   effect(USE meth);
12482 
12483   ins_cost(300);
12484   format %{ "call,static " %}
12485   opcode(0xE8); /* E8 cd */
12486   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12487   ins_pipe(pipe_slow);
12488   ins_alignment(4);
12489 %}
12490 
12491 // Call Java Dynamic Instruction
12492 // Note: If this code changes, the corresponding ret_addr_offset() and
12493 //       compute_padding() functions will have to be adjusted.
12494 instruct CallDynamicJavaDirect(method meth)
12495 %{
12496   match(CallDynamicJava);
12497   effect(USE meth);
12498 
12499   ins_cost(300);
12500   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12501             "call,dynamic " %}
12502   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12503   ins_pipe(pipe_slow);
12504   ins_alignment(4);
12505 %}
12506 
12507 // Call Runtime Instruction
12508 instruct CallRuntimeDirect(method meth)
12509 %{
12510   match(CallRuntime);
12511   effect(USE meth);
12512 
12513   ins_cost(300);
12514   format %{ "call,runtime " %}
12515   ins_encode(clear_avx, Java_To_Runtime(meth));
12516   ins_pipe(pipe_slow);
12517 %}
12518 
12519 // Call runtime without safepoint
12520 instruct CallLeafDirect(method meth)
12521 %{
12522   match(CallLeaf);
12523   effect(USE meth);
12524 
12525   ins_cost(300);
12526   format %{ "call_leaf,runtime " %}
12527   ins_encode(clear_avx, Java_To_Runtime(meth));
12528   ins_pipe(pipe_slow);
12529 %}
12530 
12531 // Call runtime without safepoint and with vector arguments
12532 instruct CallLeafDirectVector(method meth)
12533 %{
12534   match(CallLeafVector);
12535   effect(USE meth);
12536 
12537   ins_cost(300);
12538   format %{ "call_leaf,vector " %}
12539   ins_encode(Java_To_Runtime(meth));
12540   ins_pipe(pipe_slow);
12541 %}
12542 
12543 // Call runtime without safepoint
12544 instruct CallLeafNoFPDirect(method meth)
12545 %{
12546   match(CallLeafNoFP);
12547   effect(USE meth);
12548 
12549   ins_cost(300);
12550   format %{ "call_leaf_nofp,runtime " %}
12551   ins_encode(clear_avx, Java_To_Runtime(meth));
12552   ins_pipe(pipe_slow);
12553 %}
12554 
12555 // Return Instruction
12556 // Remove the return address & jump to it.
12557 // Notice: We always emit a nop after a ret to make sure there is room
12558 // for safepoint patching
12559 instruct Ret()
12560 %{
12561   match(Return);
12562 
12563   format %{ "ret" %}
12564   ins_encode %{
12565     __ ret(0);
12566   %}
12567   ins_pipe(pipe_jmp);
12568 %}
12569 
12570 // Tail Call; Jump from runtime stub to Java code.
12571 // Also known as an 'interprocedural jump'.
12572 // Target of jump will eventually return to caller.
12573 // TailJump below removes the return address.
12574 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12575 %{
12576   match(TailCall jump_target method_ptr);
12577 
12578   ins_cost(300);
12579   format %{ "jmp     $jump_target\t# rbx holds method" %}
12580   ins_encode %{
12581     __ jmp($jump_target$$Register);
12582   %}
12583   ins_pipe(pipe_jmp);
12584 %}
12585 
12586 // Tail Jump; remove the return address; jump to target.
12587 // TailCall above leaves the return address around.
12588 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12589 %{
12590   match(TailJump jump_target ex_oop);
12591 
12592   ins_cost(300);
12593   format %{ "popq    rdx\t# pop return address\n\t"
12594             "jmp     $jump_target" %}
12595   ins_encode %{
12596     __ popq(as_Register(RDX_enc));
12597     __ jmp($jump_target$$Register);
12598   %}
12599   ins_pipe(pipe_jmp);
12600 %}
12601 
12602 // Create exception oop: created by stack-crawling runtime code.
12603 // Created exception is now available to this handler, and is setup
12604 // just prior to jumping to this handler.  No code emitted.
12605 instruct CreateException(rax_RegP ex_oop)
12606 %{
12607   match(Set ex_oop (CreateEx));
12608 
12609   size(0);
12610   // use the following format syntax
12611   format %{ "# exception oop is in rax; no code emitted" %}
12612   ins_encode();
12613   ins_pipe(empty);
12614 %}
12615 
12616 // Rethrow exception:
12617 // The exception oop will come in the first argument position.
12618 // Then JUMP (not call) to the rethrow stub code.
12619 instruct RethrowException()
12620 %{
12621   match(Rethrow);
12622 
12623   // use the following format syntax
12624   format %{ "jmp     rethrow_stub" %}
12625   ins_encode %{
12626     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12627   %}
12628   ins_pipe(pipe_jmp);
12629 %}
12630 
12631 // ============================================================================
12632 // This name is KNOWN by the ADLC and cannot be changed.
12633 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12634 // for this guy.
12635 instruct tlsLoadP(r15_RegP dst) %{
12636   match(Set dst (ThreadLocal));
12637   effect(DEF dst);
12638 
12639   size(0);
12640   format %{ "# TLS is in R15" %}
12641   ins_encode( /*empty encoding*/ );
12642   ins_pipe(ialu_reg_reg);
12643 %}
12644 
12645 
12646 //----------PEEPHOLE RULES-----------------------------------------------------
12647 // These must follow all instruction definitions as they use the names
12648 // defined in the instructions definitions.
12649 //
12650 // peeppredicate ( rule_predicate );
12651 // // the predicate unless which the peephole rule will be ignored
12652 //
12653 // peepmatch ( root_instr_name [preceding_instruction]* );
12654 //
12655 // peepprocedure ( procedure_name );
12656 // // provide a procedure name to perform the optimization, the procedure should
12657 // // reside in the architecture dependent peephole file, the method has the
12658 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12659 // // with the arguments being the basic block, the current node index inside the
12660 // // block, the register allocator, the functions upon invoked return a new node
12661 // // defined in peepreplace, and the rules of the nodes appearing in the
12662 // // corresponding peepmatch, the function return true if successful, else
12663 // // return false
12664 //
12665 // peepconstraint %{
12666 // (instruction_number.operand_name relational_op instruction_number.operand_name
12667 //  [, ...] );
12668 // // instruction numbers are zero-based using left to right order in peepmatch
12669 //
12670 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12671 // // provide an instruction_number.operand_name for each operand that appears
12672 // // in the replacement instruction's match rule
12673 //
12674 // ---------VM FLAGS---------------------------------------------------------
12675 //
12676 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12677 //
12678 // Each peephole rule is given an identifying number starting with zero and
12679 // increasing by one in the order seen by the parser.  An individual peephole
12680 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12681 // on the command-line.
12682 //
12683 // ---------CURRENT LIMITATIONS----------------------------------------------
12684 //
12685 // Only transformations inside a basic block (do we need more for peephole)
12686 //
12687 // ---------EXAMPLE----------------------------------------------------------
12688 //
12689 // // pertinent parts of existing instructions in architecture description
12690 // instruct movI(rRegI dst, rRegI src)
12691 // %{
12692 //   match(Set dst (CopyI src));
12693 // %}
12694 //
12695 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12696 // %{
12697 //   match(Set dst (AddI dst src));
12698 //   effect(KILL cr);
12699 // %}
12700 //
12701 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12702 // %{
12703 //   match(Set dst (AddI dst src));
12704 // %}
12705 //
12706 // 1. Simple replacement
12707 // - Only match adjacent instructions in same basic block
12708 // - Only equality constraints
12709 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
12710 // - Only one replacement instruction
12711 //
12712 // // Change (inc mov) to lea
12713 // peephole %{
12714 //   // lea should only be emitted when beneficial
12715 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12716 //   // increment preceded by register-register move
12717 //   peepmatch ( incI_rReg movI );
12718 //   // require that the destination register of the increment
12719 //   // match the destination register of the move
12720 //   peepconstraint ( 0.dst == 1.dst );
12721 //   // construct a replacement instruction that sets
12722 //   // the destination to ( move's source register + one )
12723 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12724 // %}
12725 //
12726 // 2. Procedural replacement
12727 // - More flexible finding relevent nodes
12728 // - More flexible constraints
12729 // - More flexible transformations
12730 // - May utilise architecture-dependent API more effectively
12731 // - Currently only one replacement instruction due to adlc parsing capabilities
12732 //
12733 // // Change (inc mov) to lea
12734 // peephole %{
12735 //   // lea should only be emitted when beneficial
12736 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12737 //   // the rule numbers of these nodes inside are passed into the function below
12738 //   peepmatch ( incI_rReg movI );
12739 //   // the method that takes the responsibility of transformation
12740 //   peepprocedure ( inc_mov_to_lea );
12741 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
12742 //   // node is passed into the function above
12743 //   peepreplace ( leaI_rReg_immI() );
12744 // %}
12745 
12746 // These instructions is not matched by the matcher but used by the peephole
12747 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
12748 %{
12749   predicate(false);
12750   match(Set dst (AddI src1 src2));
12751   format %{ "leal    $dst, [$src1 + $src2]" %}
12752   ins_encode %{
12753     Register dst = $dst$$Register;
12754     Register src1 = $src1$$Register;
12755     Register src2 = $src2$$Register;
12756     if (src1 != rbp && src1 != r13) {
12757       __ leal(dst, Address(src1, src2, Address::times_1));
12758     } else {
12759       assert(src2 != rbp && src2 != r13, "");
12760       __ leal(dst, Address(src2, src1, Address::times_1));
12761     }
12762   %}
12763   ins_pipe(ialu_reg_reg);
12764 %}
12765 
12766 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
12767 %{
12768   predicate(false);
12769   match(Set dst (AddI src1 src2));
12770   format %{ "leal    $dst, [$src1 + $src2]" %}
12771   ins_encode %{
12772     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
12773   %}
12774   ins_pipe(ialu_reg_reg);
12775 %}
12776 
12777 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
12778 %{
12779   predicate(false);
12780   match(Set dst (LShiftI src shift));
12781   format %{ "leal    $dst, [$src << $shift]" %}
12782   ins_encode %{
12783     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12784     Register src = $src$$Register;
12785     if (scale == Address::times_2 && src != rbp && src != r13) {
12786       __ leal($dst$$Register, Address(src, src, Address::times_1));
12787     } else {
12788       __ leal($dst$$Register, Address(noreg, src, scale));
12789     }
12790   %}
12791   ins_pipe(ialu_reg_reg);
12792 %}
12793 
12794 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
12795 %{
12796   predicate(false);
12797   match(Set dst (AddL src1 src2));
12798   format %{ "leaq    $dst, [$src1 + $src2]" %}
12799   ins_encode %{
12800     Register dst = $dst$$Register;
12801     Register src1 = $src1$$Register;
12802     Register src2 = $src2$$Register;
12803     if (src1 != rbp && src1 != r13) {
12804       __ leaq(dst, Address(src1, src2, Address::times_1));
12805     } else {
12806       assert(src2 != rbp && src2 != r13, "");
12807       __ leaq(dst, Address(src2, src1, Address::times_1));
12808     }
12809   %}
12810   ins_pipe(ialu_reg_reg);
12811 %}
12812 
12813 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
12814 %{
12815   predicate(false);
12816   match(Set dst (AddL src1 src2));
12817   format %{ "leaq    $dst, [$src1 + $src2]" %}
12818   ins_encode %{
12819     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
12820   %}
12821   ins_pipe(ialu_reg_reg);
12822 %}
12823 
12824 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
12825 %{
12826   predicate(false);
12827   match(Set dst (LShiftL src shift));
12828   format %{ "leaq    $dst, [$src << $shift]" %}
12829   ins_encode %{
12830     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12831     Register src = $src$$Register;
12832     if (scale == Address::times_2 && src != rbp && src != r13) {
12833       __ leaq($dst$$Register, Address(src, src, Address::times_1));
12834     } else {
12835       __ leaq($dst$$Register, Address(noreg, src, scale));
12836     }
12837   %}
12838   ins_pipe(ialu_reg_reg);
12839 %}
12840 
12841 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
12842 // sal}) with lea instructions. The {add, sal} rules are beneficial in
12843 // processors with at least partial ALU support for lea
12844 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
12845 // beneficial for processors with full ALU support
12846 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
12847 
12848 peephole
12849 %{
12850   peeppredicate(VM_Version::supports_fast_2op_lea());
12851   peepmatch (addI_rReg);
12852   peepprocedure (lea_coalesce_reg);
12853   peepreplace (leaI_rReg_rReg_peep());
12854 %}
12855 
12856 peephole
12857 %{
12858   peeppredicate(VM_Version::supports_fast_2op_lea());
12859   peepmatch (addI_rReg_imm);
12860   peepprocedure (lea_coalesce_imm);
12861   peepreplace (leaI_rReg_immI_peep());
12862 %}
12863 
12864 peephole
12865 %{
12866   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12867                 VM_Version::is_intel_cascade_lake());
12868   peepmatch (incI_rReg);
12869   peepprocedure (lea_coalesce_imm);
12870   peepreplace (leaI_rReg_immI_peep());
12871 %}
12872 
12873 peephole
12874 %{
12875   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12876                 VM_Version::is_intel_cascade_lake());
12877   peepmatch (decI_rReg);
12878   peepprocedure (lea_coalesce_imm);
12879   peepreplace (leaI_rReg_immI_peep());
12880 %}
12881 
12882 peephole
12883 %{
12884   peeppredicate(VM_Version::supports_fast_2op_lea());
12885   peepmatch (salI_rReg_immI2);
12886   peepprocedure (lea_coalesce_imm);
12887   peepreplace (leaI_rReg_immI2_peep());
12888 %}
12889 
12890 peephole
12891 %{
12892   peeppredicate(VM_Version::supports_fast_2op_lea());
12893   peepmatch (addL_rReg);
12894   peepprocedure (lea_coalesce_reg);
12895   peepreplace (leaL_rReg_rReg_peep());
12896 %}
12897 
12898 peephole
12899 %{
12900   peeppredicate(VM_Version::supports_fast_2op_lea());
12901   peepmatch (addL_rReg_imm);
12902   peepprocedure (lea_coalesce_imm);
12903   peepreplace (leaL_rReg_immL32_peep());
12904 %}
12905 
12906 peephole
12907 %{
12908   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12909                 VM_Version::is_intel_cascade_lake());
12910   peepmatch (incL_rReg);
12911   peepprocedure (lea_coalesce_imm);
12912   peepreplace (leaL_rReg_immL32_peep());
12913 %}
12914 
12915 peephole
12916 %{
12917   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12918                 VM_Version::is_intel_cascade_lake());
12919   peepmatch (decL_rReg);
12920   peepprocedure (lea_coalesce_imm);
12921   peepreplace (leaL_rReg_immL32_peep());
12922 %}
12923 
12924 peephole
12925 %{
12926   peeppredicate(VM_Version::supports_fast_2op_lea());
12927   peepmatch (salL_rReg_immI2);
12928   peepprocedure (lea_coalesce_imm);
12929   peepreplace (leaL_rReg_immI2_peep());
12930 %}
12931 
12932 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
12933 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
12934 
12935 //int variant
12936 peephole
12937 %{
12938   peepmatch (testI_reg);
12939   peepprocedure (test_may_remove);
12940 %}
12941 
12942 //long variant
12943 peephole
12944 %{
12945   peepmatch (testL_reg);
12946   peepprocedure (test_may_remove);
12947 %}
12948 
12949 
12950 //----------SMARTSPILL RULES---------------------------------------------------
12951 // These must follow all instruction definitions as they use the names
12952 // defined in the instructions definitions.