1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RDX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RDI int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 
  323 source_hpp %{
  324 
  325 #include "peephole_x86_64.hpp"
  326 
  327 %}
  328 
  329 // Register masks
  330 source_hpp %{
  331 
  332 extern RegMask _ANY_REG_mask;
  333 extern RegMask _PTR_REG_mask;
  334 extern RegMask _PTR_REG_NO_RBP_mask;
  335 extern RegMask _PTR_NO_RAX_REG_mask;
  336 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  337 extern RegMask _LONG_REG_mask;
  338 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  339 extern RegMask _LONG_NO_RCX_REG_mask;
  340 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  341 extern RegMask _INT_REG_mask;
  342 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  343 extern RegMask _INT_NO_RCX_REG_mask;
  344 extern RegMask _INT_NO_RBP_R13_REG_mask;
  345 extern RegMask _FLOAT_REG_mask;
  346 
  347 extern RegMask _STACK_OR_PTR_REG_mask;
  348 extern RegMask _STACK_OR_LONG_REG_mask;
  349 extern RegMask _STACK_OR_INT_REG_mask;
  350 
  351 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  352 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  353 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_INT_REG_mask;
  380 
  381 static bool need_r12_heapbase() {
  382   return UseCompressedOops;
  383 }
  384 
  385 void reg_mask_init() {
  386   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  387   // We derive a number of subsets from it.
  388   _ANY_REG_mask = _ALL_REG_mask;
  389 
  390   if (PreserveFramePointer) {
  391     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  392     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  393   }
  394   if (need_r12_heapbase()) {
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  396     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  397   }
  398 
  399   _PTR_REG_mask = _ANY_REG_mask;
  400   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  401   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  404 
  405   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  406   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  407 
  408   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  409   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  410   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  411 
  412   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  413   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  414   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  415 
  416   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  417   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  418   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  419 
  420   _LONG_REG_mask = _PTR_REG_mask;
  421   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  422   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  423 
  424   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  425   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  426   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  429 
  430   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  431   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  432   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  433 
  434   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  435   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  436   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  439 
  440   _INT_REG_mask = _ALL_INT_REG_mask;
  441   if (PreserveFramePointer) {
  442     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  443   }
  444   if (need_r12_heapbase()) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  446   }
  447 
  448   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  449   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  450 
  451   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  452   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  453   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  454 
  455   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  456   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  457 
  458   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  459   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  460   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  461 
  462   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  463   // from the float_reg_legacy/float_reg_evex register class.
  464   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  465 }
  466 
  467 static bool generate_vzeroupper(Compile* C) {
  468   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  469 }
  470 
  471 static int clear_avx_size() {
  472   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  473 }
  474 
  475 // !!!!! Special hack to get all types of calls to specify the byte offset
  476 //       from the start of the call to the point where the return address
  477 //       will point.
  478 int MachCallStaticJavaNode::ret_addr_offset()
  479 {
  480   int offset = 5; // 5 bytes from start of call to where return address points
  481   offset += clear_avx_size();
  482   return offset;
  483 }
  484 
  485 int MachCallDynamicJavaNode::ret_addr_offset()
  486 {
  487   int offset = 15; // 15 bytes from start of call to where return address points
  488   offset += clear_avx_size();
  489   return offset;
  490 }
  491 
  492 int MachCallRuntimeNode::ret_addr_offset() {
  493   int offset = 13; // movq r10,#addr; callq (r10)
  494   if (this->ideal_Opcode() != Op_CallLeafVector) {
  495     offset += clear_avx_size();
  496   }
  497   return offset;
  498 }
  499 //
  500 // Compute padding required for nodes which need alignment
  501 //
  502 
  503 // The address of the call instruction needs to be 4-byte aligned to
  504 // ensure that it does not span a cache line so that it can be patched.
  505 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  506 {
  507   current_offset += clear_avx_size(); // skip vzeroupper
  508   current_offset += 1; // skip call opcode byte
  509   return align_up(current_offset, alignment_required()) - current_offset;
  510 }
  511 
  512 // The address of the call instruction needs to be 4-byte aligned to
  513 // ensure that it does not span a cache line so that it can be patched.
  514 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  515 {
  516   current_offset += clear_avx_size(); // skip vzeroupper
  517   current_offset += 11; // skip movq instruction + call opcode byte
  518   return align_up(current_offset, alignment_required()) - current_offset;
  519 }
  520 
  521 // This could be in MacroAssembler but it's fairly C2 specific
  522 static void emit_cmpfp_fixup(MacroAssembler& _masm) {
  523   Label exit;
  524   __ jccb(Assembler::noParity, exit);
  525   __ pushf();
  526   //
  527   // comiss/ucomiss instructions set ZF,PF,CF flags and
  528   // zero OF,AF,SF for NaN values.
  529   // Fixup flags by zeroing ZF,PF so that compare of NaN
  530   // values returns 'less than' result (CF is set).
  531   // Leave the rest of flags unchanged.
  532   //
  533   //    7 6 5 4 3 2 1 0
  534   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  535   //    0 0 1 0 1 0 1 1   (0x2B)
  536   //
  537   __ andq(Address(rsp, 0), 0xffffff2b);
  538   __ popf();
  539   __ bind(exit);
  540 }
  541 
  542 static void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  543   Label done;
  544   __ movl(dst, -1);
  545   __ jcc(Assembler::parity, done);
  546   __ jcc(Assembler::below, done);
  547   __ setb(Assembler::notEqual, dst);
  548   __ movzbl(dst, dst);
  549   __ bind(done);
  550 }
  551 
  552 // Math.min()    # Math.max()
  553 // --------------------------
  554 // ucomis[s/d]   #
  555 // ja   -> b     # a
  556 // jp   -> NaN   # NaN
  557 // jb   -> a     # b
  558 // je            #
  559 // |-jz -> a | b # a & b
  560 // |    -> a     #
  561 static void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  562                             XMMRegister a, XMMRegister b,
  563                             XMMRegister xmmt, Register rt,
  564                             bool min, bool single) {
  565 
  566   Label nan, zero, below, above, done;
  567 
  568   if (single)
  569     __ ucomiss(a, b);
  570   else
  571     __ ucomisd(a, b);
  572 
  573   if (dst->encoding() != (min ? b : a)->encoding())
  574     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  575   else
  576     __ jccb(Assembler::above, done);
  577 
  578   __ jccb(Assembler::parity, nan);  // PF=1
  579   __ jccb(Assembler::below, below); // CF=1
  580 
  581   // equal
  582   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  583   if (single) {
  584     __ ucomiss(a, xmmt);
  585     __ jccb(Assembler::equal, zero);
  586 
  587     __ movflt(dst, a);
  588     __ jmp(done);
  589   }
  590   else {
  591     __ ucomisd(a, xmmt);
  592     __ jccb(Assembler::equal, zero);
  593 
  594     __ movdbl(dst, a);
  595     __ jmp(done);
  596   }
  597 
  598   __ bind(zero);
  599   if (min)
  600     __ vpor(dst, a, b, Assembler::AVX_128bit);
  601   else
  602     __ vpand(dst, a, b, Assembler::AVX_128bit);
  603 
  604   __ jmp(done);
  605 
  606   __ bind(above);
  607   if (single)
  608     __ movflt(dst, min ? b : a);
  609   else
  610     __ movdbl(dst, min ? b : a);
  611 
  612   __ jmp(done);
  613 
  614   __ bind(nan);
  615   if (single) {
  616     __ movl(rt, 0x7fc00000); // Float.NaN
  617     __ movdl(dst, rt);
  618   }
  619   else {
  620     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  621     __ movdq(dst, rt);
  622   }
  623   __ jmp(done);
  624 
  625   __ bind(below);
  626   if (single)
  627     __ movflt(dst, min ? a : b);
  628   else
  629     __ movdbl(dst, min ? a : b);
  630 
  631   __ bind(done);
  632 }
  633 
  634 //=============================================================================
  635 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  636 
  637 int ConstantTable::calculate_table_base_offset() const {
  638   return 0;  // absolute addressing, no offset
  639 }
  640 
  641 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  642 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  643   ShouldNotReachHere();
  644 }
  645 
  646 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  647   // Empty encoding
  648 }
  649 
  650 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  651   return 0;
  652 }
  653 
  654 #ifndef PRODUCT
  655 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  656   st->print("# MachConstantBaseNode (empty encoding)");
  657 }
  658 #endif
  659 
  660 
  661 //=============================================================================
  662 #ifndef PRODUCT
  663 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  664   Compile* C = ra_->C;
  665 
  666   int framesize = C->output()->frame_size_in_bytes();
  667   int bangsize = C->output()->bang_size_in_bytes();
  668   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  669   // Remove wordSize for return addr which is already pushed.
  670   framesize -= wordSize;
  671 
  672   if (C->output()->need_stack_bang(bangsize)) {
  673     framesize -= wordSize;
  674     st->print("# stack bang (%d bytes)", bangsize);
  675     st->print("\n\t");
  676     st->print("pushq   rbp\t# Save rbp");
  677     if (PreserveFramePointer) {
  678         st->print("\n\t");
  679         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  680     }
  681     if (framesize) {
  682       st->print("\n\t");
  683       st->print("subq    rsp, #%d\t# Create frame",framesize);
  684     }
  685   } else {
  686     st->print("subq    rsp, #%d\t# Create frame",framesize);
  687     st->print("\n\t");
  688     framesize -= wordSize;
  689     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  690     if (PreserveFramePointer) {
  691       st->print("\n\t");
  692       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  693       if (framesize > 0) {
  694         st->print("\n\t");
  695         st->print("addq    rbp, #%d", framesize);
  696       }
  697     }
  698   }
  699 
  700   if (VerifyStackAtCalls) {
  701     st->print("\n\t");
  702     framesize -= wordSize;
  703     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  704 #ifdef ASSERT
  705     st->print("\n\t");
  706     st->print("# stack alignment check");
  707 #endif
  708   }
  709   if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
  710     st->print("\n\t");
  711     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  712     st->print("\n\t");
  713     st->print("je      fast_entry\t");
  714     st->print("\n\t");
  715     st->print("call    #nmethod_entry_barrier_stub\t");
  716     st->print("\n\tfast_entry:");
  717   }
  718   st->cr();
  719 }
  720 #endif
  721 
  722 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  723   Compile* C = ra_->C;
  724   C2_MacroAssembler _masm(&cbuf);
  725 
  726   int framesize = C->output()->frame_size_in_bytes();
  727   int bangsize = C->output()->bang_size_in_bytes();
  728 
  729   if (C->clinit_barrier_on_entry()) {
  730     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  731     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  732 
  733     Label L_skip_barrier;
  734     Register klass = rscratch1;
  735 
  736     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  737     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  738 
  739     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  740 
  741     __ bind(L_skip_barrier);
  742   }
  743 
  744   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  745 
  746   C->output()->set_frame_complete(cbuf.insts_size());
  747 
  748   if (C->has_mach_constant_base_node()) {
  749     // NOTE: We set the table base offset here because users might be
  750     // emitted before MachConstantBaseNode.
  751     ConstantTable& constant_table = C->output()->constant_table();
  752     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  753   }
  754 }
  755 
  756 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  757 {
  758   return MachNode::size(ra_); // too many variables; just compute it
  759                               // the hard way
  760 }
  761 
  762 int MachPrologNode::reloc() const
  763 {
  764   return 0; // a large enough number
  765 }
  766 
  767 //=============================================================================
  768 #ifndef PRODUCT
  769 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  770 {
  771   Compile* C = ra_->C;
  772   if (generate_vzeroupper(C)) {
  773     st->print("vzeroupper");
  774     st->cr(); st->print("\t");
  775   }
  776 
  777   int framesize = C->output()->frame_size_in_bytes();
  778   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  779   // Remove word for return adr already pushed
  780   // and RBP
  781   framesize -= 2*wordSize;
  782 
  783   if (framesize) {
  784     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  785     st->print("\t");
  786   }
  787 
  788   st->print_cr("popq    rbp");
  789   if (do_polling() && C->is_method_compilation()) {
  790     st->print("\t");
  791     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  792                  "ja      #safepoint_stub\t"
  793                  "# Safepoint: poll for GC");
  794   }
  795 }
  796 #endif
  797 
  798 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  799 {
  800   Compile* C = ra_->C;
  801   MacroAssembler _masm(&cbuf);
  802 
  803   if (generate_vzeroupper(C)) {
  804     // Clear upper bits of YMM registers when current compiled code uses
  805     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  806     __ vzeroupper();
  807   }
  808 
  809   int framesize = C->output()->frame_size_in_bytes();
  810   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  811   // Remove word for return adr already pushed
  812   // and RBP
  813   framesize -= 2*wordSize;
  814 
  815   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  816 
  817   if (framesize) {
  818     __ addq(rsp, framesize);
  819   }
  820 
  821   __ popq(rbp);
  822 
  823   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  824     __ reserved_stack_check();
  825   }
  826 
  827   if (do_polling() && C->is_method_compilation()) {
  828     MacroAssembler _masm(&cbuf);
  829     Label dummy_label;
  830     Label* code_stub = &dummy_label;
  831     if (!C->output()->in_scratch_emit_size()) {
  832       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  833       C->output()->add_stub(stub);
  834       code_stub = &stub->entry();
  835     }
  836     __ relocate(relocInfo::poll_return_type);
  837     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  838   }
  839 }
  840 
  841 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  842 {
  843   return MachNode::size(ra_); // too many variables; just compute it
  844                               // the hard way
  845 }
  846 
  847 int MachEpilogNode::reloc() const
  848 {
  849   return 2; // a large enough number
  850 }
  851 
  852 const Pipeline* MachEpilogNode::pipeline() const
  853 {
  854   return MachNode::pipeline_class();
  855 }
  856 
  857 //=============================================================================
  858 
  859 enum RC {
  860   rc_bad,
  861   rc_int,
  862   rc_kreg,
  863   rc_float,
  864   rc_stack
  865 };
  866 
  867 static enum RC rc_class(OptoReg::Name reg)
  868 {
  869   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  870 
  871   if (OptoReg::is_stack(reg)) return rc_stack;
  872 
  873   VMReg r = OptoReg::as_VMReg(reg);
  874 
  875   if (r->is_Register()) return rc_int;
  876 
  877   if (r->is_KRegister()) return rc_kreg;
  878 
  879   assert(r->is_XMMRegister(), "must be");
  880   return rc_float;
  881 }
  882 
  883 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  884 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
  885                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  886 
  887 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
  888                      int stack_offset, int reg, uint ireg, outputStream* st);
  889 
  890 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
  891                                       int dst_offset, uint ireg, outputStream* st) {
  892   if (cbuf) {
  893     MacroAssembler _masm(cbuf);
  894     switch (ireg) {
  895     case Op_VecS:
  896       __ movq(Address(rsp, -8), rax);
  897       __ movl(rax, Address(rsp, src_offset));
  898       __ movl(Address(rsp, dst_offset), rax);
  899       __ movq(rax, Address(rsp, -8));
  900       break;
  901     case Op_VecD:
  902       __ pushq(Address(rsp, src_offset));
  903       __ popq (Address(rsp, dst_offset));
  904       break;
  905     case Op_VecX:
  906       __ pushq(Address(rsp, src_offset));
  907       __ popq (Address(rsp, dst_offset));
  908       __ pushq(Address(rsp, src_offset+8));
  909       __ popq (Address(rsp, dst_offset+8));
  910       break;
  911     case Op_VecY:
  912       __ vmovdqu(Address(rsp, -32), xmm0);
  913       __ vmovdqu(xmm0, Address(rsp, src_offset));
  914       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  915       __ vmovdqu(xmm0, Address(rsp, -32));
  916       break;
  917     case Op_VecZ:
  918       __ evmovdquq(Address(rsp, -64), xmm0, 2);
  919       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
  920       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
  921       __ evmovdquq(xmm0, Address(rsp, -64), 2);
  922       break;
  923     default:
  924       ShouldNotReachHere();
  925     }
  926 #ifndef PRODUCT
  927   } else {
  928     switch (ireg) {
  929     case Op_VecS:
  930       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
  931                 "movl    rax, [rsp + #%d]\n\t"
  932                 "movl    [rsp + #%d], rax\n\t"
  933                 "movq    rax, [rsp - #8]",
  934                 src_offset, dst_offset);
  935       break;
  936     case Op_VecD:
  937       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
  938                 "popq    [rsp + #%d]",
  939                 src_offset, dst_offset);
  940       break;
  941      case Op_VecX:
  942       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
  943                 "popq    [rsp + #%d]\n\t"
  944                 "pushq   [rsp + #%d]\n\t"
  945                 "popq    [rsp + #%d]",
  946                 src_offset, dst_offset, src_offset+8, dst_offset+8);
  947       break;
  948     case Op_VecY:
  949       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
  950                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  951                 "vmovdqu [rsp + #%d], xmm0\n\t"
  952                 "vmovdqu xmm0, [rsp - #32]",
  953                 src_offset, dst_offset);
  954       break;
  955     case Op_VecZ:
  956       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
  957                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  958                 "vmovdqu [rsp + #%d], xmm0\n\t"
  959                 "vmovdqu xmm0, [rsp - #64]",
  960                 src_offset, dst_offset);
  961       break;
  962     default:
  963       ShouldNotReachHere();
  964     }
  965 #endif
  966   }
  967 }
  968 
  969 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
  970                                        PhaseRegAlloc* ra_,
  971                                        bool do_size,
  972                                        outputStream* st) const {
  973   assert(cbuf != nullptr || st  != nullptr, "sanity");
  974   // Get registers to move
  975   OptoReg::Name src_second = ra_->get_reg_second(in(1));
  976   OptoReg::Name src_first = ra_->get_reg_first(in(1));
  977   OptoReg::Name dst_second = ra_->get_reg_second(this);
  978   OptoReg::Name dst_first = ra_->get_reg_first(this);
  979 
  980   enum RC src_second_rc = rc_class(src_second);
  981   enum RC src_first_rc = rc_class(src_first);
  982   enum RC dst_second_rc = rc_class(dst_second);
  983   enum RC dst_first_rc = rc_class(dst_first);
  984 
  985   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
  986          "must move at least 1 register" );
  987 
  988   if (src_first == dst_first && src_second == dst_second) {
  989     // Self copy, no move
  990     return 0;
  991   }
  992   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
  993     uint ireg = ideal_reg();
  994     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
  995     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
  996     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
  997       // mem -> mem
  998       int src_offset = ra_->reg2offset(src_first);
  999       int dst_offset = ra_->reg2offset(dst_first);
 1000       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1001     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1002       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1003     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1004       int stack_offset = ra_->reg2offset(dst_first);
 1005       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1006     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1007       int stack_offset = ra_->reg2offset(src_first);
 1008       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1009     } else {
 1010       ShouldNotReachHere();
 1011     }
 1012     return 0;
 1013   }
 1014   if (src_first_rc == rc_stack) {
 1015     // mem ->
 1016     if (dst_first_rc == rc_stack) {
 1017       // mem -> mem
 1018       assert(src_second != dst_first, "overlap");
 1019       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1020           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1021         // 64-bit
 1022         int src_offset = ra_->reg2offset(src_first);
 1023         int dst_offset = ra_->reg2offset(dst_first);
 1024         if (cbuf) {
 1025           MacroAssembler _masm(cbuf);
 1026           __ pushq(Address(rsp, src_offset));
 1027           __ popq (Address(rsp, dst_offset));
 1028 #ifndef PRODUCT
 1029         } else {
 1030           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1031                     "popq    [rsp + #%d]",
 1032                      src_offset, dst_offset);
 1033 #endif
 1034         }
 1035       } else {
 1036         // 32-bit
 1037         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1038         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1039         // No pushl/popl, so:
 1040         int src_offset = ra_->reg2offset(src_first);
 1041         int dst_offset = ra_->reg2offset(dst_first);
 1042         if (cbuf) {
 1043           MacroAssembler _masm(cbuf);
 1044           __ movq(Address(rsp, -8), rax);
 1045           __ movl(rax, Address(rsp, src_offset));
 1046           __ movl(Address(rsp, dst_offset), rax);
 1047           __ movq(rax, Address(rsp, -8));
 1048 #ifndef PRODUCT
 1049         } else {
 1050           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                     "movl    rax, [rsp + #%d]\n\t"
 1052                     "movl    [rsp + #%d], rax\n\t"
 1053                     "movq    rax, [rsp - #8]",
 1054                      src_offset, dst_offset);
 1055 #endif
 1056         }
 1057       }
 1058       return 0;
 1059     } else if (dst_first_rc == rc_int) {
 1060       // mem -> gpr
 1061       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1062           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1063         // 64-bit
 1064         int offset = ra_->reg2offset(src_first);
 1065         if (cbuf) {
 1066           MacroAssembler _masm(cbuf);
 1067           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1068 #ifndef PRODUCT
 1069         } else {
 1070           st->print("movq    %s, [rsp + #%d]\t# spill",
 1071                      Matcher::regName[dst_first],
 1072                      offset);
 1073 #endif
 1074         }
 1075       } else {
 1076         // 32-bit
 1077         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1078         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1079         int offset = ra_->reg2offset(src_first);
 1080         if (cbuf) {
 1081           MacroAssembler _masm(cbuf);
 1082           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1083 #ifndef PRODUCT
 1084         } else {
 1085           st->print("movl    %s, [rsp + #%d]\t# spill",
 1086                      Matcher::regName[dst_first],
 1087                      offset);
 1088 #endif
 1089         }
 1090       }
 1091       return 0;
 1092     } else if (dst_first_rc == rc_float) {
 1093       // mem-> xmm
 1094       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1095           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1096         // 64-bit
 1097         int offset = ra_->reg2offset(src_first);
 1098         if (cbuf) {
 1099           MacroAssembler _masm(cbuf);
 1100           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1101 #ifndef PRODUCT
 1102         } else {
 1103           st->print("%s  %s, [rsp + #%d]\t# spill",
 1104                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1105                      Matcher::regName[dst_first],
 1106                      offset);
 1107 #endif
 1108         }
 1109       } else {
 1110         // 32-bit
 1111         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1112         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1113         int offset = ra_->reg2offset(src_first);
 1114         if (cbuf) {
 1115           MacroAssembler _masm(cbuf);
 1116           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1117 #ifndef PRODUCT
 1118         } else {
 1119           st->print("movss   %s, [rsp + #%d]\t# spill",
 1120                      Matcher::regName[dst_first],
 1121                      offset);
 1122 #endif
 1123         }
 1124       }
 1125       return 0;
 1126     } else if (dst_first_rc == rc_kreg) {
 1127       // mem -> kreg
 1128       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1129           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1130         // 64-bit
 1131         int offset = ra_->reg2offset(src_first);
 1132         if (cbuf) {
 1133           MacroAssembler _masm(cbuf);
 1134           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1135 #ifndef PRODUCT
 1136         } else {
 1137           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1138                      Matcher::regName[dst_first],
 1139                      offset);
 1140 #endif
 1141         }
 1142       }
 1143       return 0;
 1144     }
 1145   } else if (src_first_rc == rc_int) {
 1146     // gpr ->
 1147     if (dst_first_rc == rc_stack) {
 1148       // gpr -> mem
 1149       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1150           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1151         // 64-bit
 1152         int offset = ra_->reg2offset(dst_first);
 1153         if (cbuf) {
 1154           MacroAssembler _masm(cbuf);
 1155           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1156 #ifndef PRODUCT
 1157         } else {
 1158           st->print("movq    [rsp + #%d], %s\t# spill",
 1159                      offset,
 1160                      Matcher::regName[src_first]);
 1161 #endif
 1162         }
 1163       } else {
 1164         // 32-bit
 1165         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1166         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1167         int offset = ra_->reg2offset(dst_first);
 1168         if (cbuf) {
 1169           MacroAssembler _masm(cbuf);
 1170           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1171 #ifndef PRODUCT
 1172         } else {
 1173           st->print("movl    [rsp + #%d], %s\t# spill",
 1174                      offset,
 1175                      Matcher::regName[src_first]);
 1176 #endif
 1177         }
 1178       }
 1179       return 0;
 1180     } else if (dst_first_rc == rc_int) {
 1181       // gpr -> gpr
 1182       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1183           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1184         // 64-bit
 1185         if (cbuf) {
 1186           MacroAssembler _masm(cbuf);
 1187           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1188                   as_Register(Matcher::_regEncode[src_first]));
 1189 #ifndef PRODUCT
 1190         } else {
 1191           st->print("movq    %s, %s\t# spill",
 1192                      Matcher::regName[dst_first],
 1193                      Matcher::regName[src_first]);
 1194 #endif
 1195         }
 1196         return 0;
 1197       } else {
 1198         // 32-bit
 1199         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1200         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1201         if (cbuf) {
 1202           MacroAssembler _masm(cbuf);
 1203           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1204                   as_Register(Matcher::_regEncode[src_first]));
 1205 #ifndef PRODUCT
 1206         } else {
 1207           st->print("movl    %s, %s\t# spill",
 1208                      Matcher::regName[dst_first],
 1209                      Matcher::regName[src_first]);
 1210 #endif
 1211         }
 1212         return 0;
 1213       }
 1214     } else if (dst_first_rc == rc_float) {
 1215       // gpr -> xmm
 1216       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1217           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1218         // 64-bit
 1219         if (cbuf) {
 1220           MacroAssembler _masm(cbuf);
 1221           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1222 #ifndef PRODUCT
 1223         } else {
 1224           st->print("movdq   %s, %s\t# spill",
 1225                      Matcher::regName[dst_first],
 1226                      Matcher::regName[src_first]);
 1227 #endif
 1228         }
 1229       } else {
 1230         // 32-bit
 1231         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1232         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1233         if (cbuf) {
 1234           MacroAssembler _masm(cbuf);
 1235           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1236 #ifndef PRODUCT
 1237         } else {
 1238           st->print("movdl   %s, %s\t# spill",
 1239                      Matcher::regName[dst_first],
 1240                      Matcher::regName[src_first]);
 1241 #endif
 1242         }
 1243       }
 1244       return 0;
 1245     } else if (dst_first_rc == rc_kreg) {
 1246       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1247           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1248         // 64-bit
 1249         if (cbuf) {
 1250           MacroAssembler _masm(cbuf);
 1251           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1252   #ifndef PRODUCT
 1253         } else {
 1254            st->print("kmovq   %s, %s\t# spill",
 1255                        Matcher::regName[dst_first],
 1256                        Matcher::regName[src_first]);
 1257   #endif
 1258         }
 1259       }
 1260       Unimplemented();
 1261       return 0;
 1262     }
 1263   } else if (src_first_rc == rc_float) {
 1264     // xmm ->
 1265     if (dst_first_rc == rc_stack) {
 1266       // xmm -> mem
 1267       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1268           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1269         // 64-bit
 1270         int offset = ra_->reg2offset(dst_first);
 1271         if (cbuf) {
 1272           MacroAssembler _masm(cbuf);
 1273           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1274 #ifndef PRODUCT
 1275         } else {
 1276           st->print("movsd   [rsp + #%d], %s\t# spill",
 1277                      offset,
 1278                      Matcher::regName[src_first]);
 1279 #endif
 1280         }
 1281       } else {
 1282         // 32-bit
 1283         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1284         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1285         int offset = ra_->reg2offset(dst_first);
 1286         if (cbuf) {
 1287           MacroAssembler _masm(cbuf);
 1288           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1289 #ifndef PRODUCT
 1290         } else {
 1291           st->print("movss   [rsp + #%d], %s\t# spill",
 1292                      offset,
 1293                      Matcher::regName[src_first]);
 1294 #endif
 1295         }
 1296       }
 1297       return 0;
 1298     } else if (dst_first_rc == rc_int) {
 1299       // xmm -> gpr
 1300       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1301           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1302         // 64-bit
 1303         if (cbuf) {
 1304           MacroAssembler _masm(cbuf);
 1305           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1306 #ifndef PRODUCT
 1307         } else {
 1308           st->print("movdq   %s, %s\t# spill",
 1309                      Matcher::regName[dst_first],
 1310                      Matcher::regName[src_first]);
 1311 #endif
 1312         }
 1313       } else {
 1314         // 32-bit
 1315         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1316         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1317         if (cbuf) {
 1318           MacroAssembler _masm(cbuf);
 1319           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1320 #ifndef PRODUCT
 1321         } else {
 1322           st->print("movdl   %s, %s\t# spill",
 1323                      Matcher::regName[dst_first],
 1324                      Matcher::regName[src_first]);
 1325 #endif
 1326         }
 1327       }
 1328       return 0;
 1329     } else if (dst_first_rc == rc_float) {
 1330       // xmm -> xmm
 1331       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1332           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1333         // 64-bit
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("%s  %s, %s\t# spill",
 1340                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1341                      Matcher::regName[dst_first],
 1342                      Matcher::regName[src_first]);
 1343 #endif
 1344         }
 1345       } else {
 1346         // 32-bit
 1347         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1348         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("%s  %s, %s\t# spill",
 1355                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1356                      Matcher::regName[dst_first],
 1357                      Matcher::regName[src_first]);
 1358 #endif
 1359         }
 1360       }
 1361       return 0;
 1362     } else if (dst_first_rc == rc_kreg) {
 1363       assert(false, "Illegal spilling");
 1364       return 0;
 1365     }
 1366   } else if (src_first_rc == rc_kreg) {
 1367     if (dst_first_rc == rc_stack) {
 1368       // mem -> kreg
 1369       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1370           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1371         // 64-bit
 1372         int offset = ra_->reg2offset(dst_first);
 1373         if (cbuf) {
 1374           MacroAssembler _masm(cbuf);
 1375           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1376 #ifndef PRODUCT
 1377         } else {
 1378           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1379                      offset,
 1380                      Matcher::regName[src_first]);
 1381 #endif
 1382         }
 1383       }
 1384       return 0;
 1385     } else if (dst_first_rc == rc_int) {
 1386       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1387           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1388         // 64-bit
 1389         if (cbuf) {
 1390           MacroAssembler _masm(cbuf);
 1391           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1392 #ifndef PRODUCT
 1393         } else {
 1394          st->print("kmovq   %s, %s\t# spill",
 1395                      Matcher::regName[dst_first],
 1396                      Matcher::regName[src_first]);
 1397 #endif
 1398         }
 1399       }
 1400       Unimplemented();
 1401       return 0;
 1402     } else if (dst_first_rc == rc_kreg) {
 1403       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1404           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1405         // 64-bit
 1406         if (cbuf) {
 1407           MacroAssembler _masm(cbuf);
 1408           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411          st->print("kmovq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       }
 1417       return 0;
 1418     } else if (dst_first_rc == rc_float) {
 1419       assert(false, "Illegal spill");
 1420       return 0;
 1421     }
 1422   }
 1423 
 1424   assert(0," foo ");
 1425   Unimplemented();
 1426   return 0;
 1427 }
 1428 
 1429 #ifndef PRODUCT
 1430 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1431   implementation(nullptr, ra_, false, st);
 1432 }
 1433 #endif
 1434 
 1435 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1436   implementation(&cbuf, ra_, false, nullptr);
 1437 }
 1438 
 1439 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1440   return MachNode::size(ra_);
 1441 }
 1442 
 1443 //=============================================================================
 1444 #ifndef PRODUCT
 1445 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1446 {
 1447   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1448   int reg = ra_->get_reg_first(this);
 1449   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1450             Matcher::regName[reg], offset);
 1451 }
 1452 #endif
 1453 
 1454 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1455 {
 1456   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1457   int reg = ra_->get_encode(this);
 1458 
 1459   MacroAssembler masm(&cbuf);
 1460   masm.lea(as_Register(reg), Address(rsp, offset));
 1461 }
 1462 
 1463 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1464 {
 1465   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1466   return (offset < 0x80) ? 5 : 8; // REX
 1467 }
 1468 
 1469 //=============================================================================
 1470 #ifndef PRODUCT
 1471 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1472 {
 1473   if (UseCompressedClassPointers) {
 1474     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1475     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1476   } else {
 1477     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1478     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1479   }
 1480   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1481 }
 1482 #endif
 1483 
 1484 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1485 {
 1486   MacroAssembler masm(&cbuf);
 1487   masm.ic_check(InteriorEntryAlignment);
 1488 }
 1489 
 1490 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1491 {
 1492   return MachNode::size(ra_); // too many variables; just compute it
 1493                               // the hard way
 1494 }
 1495 
 1496 
 1497 //=============================================================================
 1498 
 1499 bool Matcher::supports_vector_calling_convention(void) {
 1500   if (EnableVectorSupport && UseVectorStubs) {
 1501     return true;
 1502   }
 1503   return false;
 1504 }
 1505 
 1506 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1507   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1508   int lo = XMM0_num;
 1509   int hi = XMM0b_num;
 1510   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1511   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1512   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1513   return OptoRegPair(hi, lo);
 1514 }
 1515 
 1516 // Is this branch offset short enough that a short branch can be used?
 1517 //
 1518 // NOTE: If the platform does not provide any short branch variants, then
 1519 //       this method should return false for offset 0.
 1520 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1521   // The passed offset is relative to address of the branch.
 1522   // On 86 a branch displacement is calculated relative to address
 1523   // of a next instruction.
 1524   offset -= br_size;
 1525 
 1526   // the short version of jmpConUCF2 contains multiple branches,
 1527   // making the reach slightly less
 1528   if (rule == jmpConUCF2_rule)
 1529     return (-126 <= offset && offset <= 125);
 1530   return (-128 <= offset && offset <= 127);
 1531 }
 1532 
 1533 // Return whether or not this register is ever used as an argument.
 1534 // This function is used on startup to build the trampoline stubs in
 1535 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1536 // call in the trampoline, and arguments in those registers not be
 1537 // available to the callee.
 1538 bool Matcher::can_be_java_arg(int reg)
 1539 {
 1540   return
 1541     reg ==  RDI_num || reg == RDI_H_num ||
 1542     reg ==  RSI_num || reg == RSI_H_num ||
 1543     reg ==  RDX_num || reg == RDX_H_num ||
 1544     reg ==  RCX_num || reg == RCX_H_num ||
 1545     reg ==   R8_num || reg ==  R8_H_num ||
 1546     reg ==   R9_num || reg ==  R9_H_num ||
 1547     reg ==  R12_num || reg == R12_H_num ||
 1548     reg == XMM0_num || reg == XMM0b_num ||
 1549     reg == XMM1_num || reg == XMM1b_num ||
 1550     reg == XMM2_num || reg == XMM2b_num ||
 1551     reg == XMM3_num || reg == XMM3b_num ||
 1552     reg == XMM4_num || reg == XMM4b_num ||
 1553     reg == XMM5_num || reg == XMM5b_num ||
 1554     reg == XMM6_num || reg == XMM6b_num ||
 1555     reg == XMM7_num || reg == XMM7b_num;
 1556 }
 1557 
 1558 bool Matcher::is_spillable_arg(int reg)
 1559 {
 1560   return can_be_java_arg(reg);
 1561 }
 1562 
 1563 uint Matcher::int_pressure_limit()
 1564 {
 1565   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1566 }
 1567 
 1568 uint Matcher::float_pressure_limit()
 1569 {
 1570   // After experiment around with different values, the following default threshold
 1571   // works best for LCM's register pressure scheduling on x64.
 1572   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1573   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1574   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1575 }
 1576 
 1577 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1578   // In 64 bit mode a code which use multiply when
 1579   // devisor is constant is faster than hardware
 1580   // DIV instruction (it uses MulHiL).
 1581   return false;
 1582 }
 1583 
 1584 // Register for DIVI projection of divmodI
 1585 RegMask Matcher::divI_proj_mask() {
 1586   return INT_RAX_REG_mask();
 1587 }
 1588 
 1589 // Register for MODI projection of divmodI
 1590 RegMask Matcher::modI_proj_mask() {
 1591   return INT_RDX_REG_mask();
 1592 }
 1593 
 1594 // Register for DIVL projection of divmodL
 1595 RegMask Matcher::divL_proj_mask() {
 1596   return LONG_RAX_REG_mask();
 1597 }
 1598 
 1599 // Register for MODL projection of divmodL
 1600 RegMask Matcher::modL_proj_mask() {
 1601   return LONG_RDX_REG_mask();
 1602 }
 1603 
 1604 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1605 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1606     return NO_REG_mask();
 1607 }
 1608 
 1609 %}
 1610 
 1611 //----------ENCODING BLOCK-----------------------------------------------------
 1612 // This block specifies the encoding classes used by the compiler to
 1613 // output byte streams.  Encoding classes are parameterized macros
 1614 // used by Machine Instruction Nodes in order to generate the bit
 1615 // encoding of the instruction.  Operands specify their base encoding
 1616 // interface with the interface keyword.  There are currently
 1617 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1618 // COND_INTER.  REG_INTER causes an operand to generate a function
 1619 // which returns its register number when queried.  CONST_INTER causes
 1620 // an operand to generate a function which returns the value of the
 1621 // constant when queried.  MEMORY_INTER causes an operand to generate
 1622 // four functions which return the Base Register, the Index Register,
 1623 // the Scale Value, and the Offset Value of the operand when queried.
 1624 // COND_INTER causes an operand to generate six functions which return
 1625 // the encoding code (ie - encoding bits for the instruction)
 1626 // associated with each basic boolean condition for a conditional
 1627 // instruction.
 1628 //
 1629 // Instructions specify two basic values for encoding.  Again, a
 1630 // function is available to check if the constant displacement is an
 1631 // oop. They use the ins_encode keyword to specify their encoding
 1632 // classes (which must be a sequence of enc_class names, and their
 1633 // parameters, specified in the encoding block), and they use the
 1634 // opcode keyword to specify, in order, their primary, secondary, and
 1635 // tertiary opcode.  Only the opcode sections which a particular
 1636 // instruction needs for encoding need to be specified.
 1637 encode %{
 1638   enc_class cdql_enc(no_rax_rdx_RegI div)
 1639   %{
 1640     // Full implementation of Java idiv and irem; checks for
 1641     // special case as described in JVM spec., p.243 & p.271.
 1642     //
 1643     //         normal case                           special case
 1644     //
 1645     // input : rax: dividend                         min_int
 1646     //         reg: divisor                          -1
 1647     //
 1648     // output: rax: quotient  (= rax idiv reg)       min_int
 1649     //         rdx: remainder (= rax irem reg)       0
 1650     //
 1651     //  Code sequnce:
 1652     //
 1653     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1654     //    5:   75 07/08                jne    e <normal>
 1655     //    7:   33 d2                   xor    %edx,%edx
 1656     //  [div >= 8 -> offset + 1]
 1657     //  [REX_B]
 1658     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1659     //    c:   74 03/04                je     11 <done>
 1660     // 000000000000000e <normal>:
 1661     //    e:   99                      cltd
 1662     //  [div >= 8 -> offset + 1]
 1663     //  [REX_B]
 1664     //    f:   f7 f9                   idiv   $div
 1665     // 0000000000000011 <done>:
 1666     MacroAssembler _masm(&cbuf);
 1667     Label normal;
 1668     Label done;
 1669 
 1670     // cmp    $0x80000000,%eax
 1671     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1672 
 1673     // jne    e <normal>
 1674     __ jccb(Assembler::notEqual, normal);
 1675 
 1676     // xor    %edx,%edx
 1677     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1678 
 1679     // cmp    $0xffffffffffffffff,%ecx
 1680     __ cmpl($div$$Register, -1);
 1681 
 1682     // je     11 <done>
 1683     __ jccb(Assembler::equal, done);
 1684 
 1685     // <normal>
 1686     // cltd
 1687     __ bind(normal);
 1688     __ cdql();
 1689 
 1690     // idivl
 1691     // <done>
 1692     __ idivl($div$$Register);
 1693     __ bind(done);
 1694   %}
 1695 
 1696   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1697   %{
 1698     // Full implementation of Java ldiv and lrem; checks for
 1699     // special case as described in JVM spec., p.243 & p.271.
 1700     //
 1701     //         normal case                           special case
 1702     //
 1703     // input : rax: dividend                         min_long
 1704     //         reg: divisor                          -1
 1705     //
 1706     // output: rax: quotient  (= rax idiv reg)       min_long
 1707     //         rdx: remainder (= rax irem reg)       0
 1708     //
 1709     //  Code sequnce:
 1710     //
 1711     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1712     //    7:   00 00 80
 1713     //    a:   48 39 d0                cmp    %rdx,%rax
 1714     //    d:   75 08                   jne    17 <normal>
 1715     //    f:   33 d2                   xor    %edx,%edx
 1716     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1717     //   15:   74 05                   je     1c <done>
 1718     // 0000000000000017 <normal>:
 1719     //   17:   48 99                   cqto
 1720     //   19:   48 f7 f9                idiv   $div
 1721     // 000000000000001c <done>:
 1722     MacroAssembler _masm(&cbuf);
 1723     Label normal;
 1724     Label done;
 1725 
 1726     // mov    $0x8000000000000000,%rdx
 1727     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1728 
 1729     // cmp    %rdx,%rax
 1730     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1731 
 1732     // jne    17 <normal>
 1733     __ jccb(Assembler::notEqual, normal);
 1734 
 1735     // xor    %edx,%edx
 1736     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1737 
 1738     // cmp    $0xffffffffffffffff,$div
 1739     __ cmpq($div$$Register, -1);
 1740 
 1741     // je     1e <done>
 1742     __ jccb(Assembler::equal, done);
 1743 
 1744     // <normal>
 1745     // cqto
 1746     __ bind(normal);
 1747     __ cdqq();
 1748 
 1749     // idivq (note: must be emitted by the user of this rule)
 1750     // <done>
 1751     __ idivq($div$$Register);
 1752     __ bind(done);
 1753   %}
 1754 
 1755   enc_class enc_PartialSubtypeCheck()
 1756   %{
 1757     Register Rrdi = as_Register(RDI_enc); // result register
 1758     Register Rrax = as_Register(RAX_enc); // super class
 1759     Register Rrcx = as_Register(RCX_enc); // killed
 1760     Register Rrsi = as_Register(RSI_enc); // sub class
 1761     Label miss;
 1762     const bool set_cond_codes = true;
 1763 
 1764     MacroAssembler _masm(&cbuf);
 1765     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 1766                                      nullptr, &miss,
 1767                                      /*set_cond_codes:*/ true);
 1768     if ($primary) {
 1769       __ xorptr(Rrdi, Rrdi);
 1770     }
 1771     __ bind(miss);
 1772   %}
 1773 
 1774   enc_class clear_avx %{
 1775     debug_only(int off0 = cbuf.insts_size());
 1776     if (generate_vzeroupper(Compile::current())) {
 1777       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1778       // Clear upper bits of YMM registers when current compiled code uses
 1779       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1780       MacroAssembler _masm(&cbuf);
 1781       __ vzeroupper();
 1782     }
 1783     debug_only(int off1 = cbuf.insts_size());
 1784     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1785   %}
 1786 
 1787   enc_class Java_To_Runtime(method meth) %{
 1788     // No relocation needed
 1789     MacroAssembler _masm(&cbuf);
 1790     __ mov64(r10, (int64_t) $meth$$method);
 1791     __ call(r10);
 1792     __ post_call_nop();
 1793   %}
 1794 
 1795   enc_class Java_Static_Call(method meth)
 1796   %{
 1797     // JAVA STATIC CALL
 1798     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1799     // determine who we intended to call.
 1800     MacroAssembler _masm(&cbuf);
 1801 
 1802     if (!_method) {
 1803       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1804     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1805       // The NOP here is purely to ensure that eliding a call to
 1806       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1807       __ addr_nop_5();
 1808       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1809     } else {
 1810       int method_index = resolved_method_index(cbuf);
 1811       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1812                                                   : static_call_Relocation::spec(method_index);
 1813       address mark = __ pc();
 1814       int call_offset = __ offset();
 1815       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1816       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1817         // Calls of the same statically bound method can share
 1818         // a stub to the interpreter.
 1819         cbuf.shared_stub_to_interp_for(_method, call_offset);
 1820       } else {
 1821         // Emit stubs for static call.
 1822         address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, mark);
 1823         if (stub == nullptr) {
 1824           ciEnv::current()->record_failure("CodeCache is full");
 1825           return;
 1826         }
 1827       }
 1828     }
 1829     __ post_call_nop();
 1830   %}
 1831 
 1832   enc_class Java_Dynamic_Call(method meth) %{
 1833     MacroAssembler _masm(&cbuf);
 1834     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 1835     __ post_call_nop();
 1836   %}
 1837 
 1838 %}
 1839 
 1840 
 1841 
 1842 //----------FRAME--------------------------------------------------------------
 1843 // Definition of frame structure and management information.
 1844 //
 1845 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1846 //                             |   (to get allocators register number
 1847 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1848 //  r   CALLER     |        |
 1849 //  o     |        +--------+      pad to even-align allocators stack-slot
 1850 //  w     V        |  pad0  |        numbers; owned by CALLER
 1851 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1852 //  h     ^        |   in   |  5
 1853 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1854 //  |     |        |        |  3
 1855 //  |     |        +--------+
 1856 //  V     |        | old out|      Empty on Intel, window on Sparc
 1857 //        |    old |preserve|      Must be even aligned.
 1858 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1859 //        |        |   in   |  3   area for Intel ret address
 1860 //     Owned by    |preserve|      Empty on Sparc.
 1861 //       SELF      +--------+
 1862 //        |        |  pad2  |  2   pad to align old SP
 1863 //        |        +--------+  1
 1864 //        |        | locks  |  0
 1865 //        |        +--------+----> OptoReg::stack0(), even aligned
 1866 //        |        |  pad1  | 11   pad to align new SP
 1867 //        |        +--------+
 1868 //        |        |        | 10
 1869 //        |        | spills |  9   spills
 1870 //        V        |        |  8   (pad0 slot for callee)
 1871 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1872 //        ^        |  out   |  7
 1873 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1874 //     Owned by    +--------+
 1875 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1876 //        |    new |preserve|      Must be even-aligned.
 1877 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1878 //        |        |        |
 1879 //
 1880 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1881 //         known from SELF's arguments and the Java calling convention.
 1882 //         Region 6-7 is determined per call site.
 1883 // Note 2: If the calling convention leaves holes in the incoming argument
 1884 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1885 //         are owned by the CALLEE.  Holes should not be necessary in the
 1886 //         incoming area, as the Java calling convention is completely under
 1887 //         the control of the AD file.  Doubles can be sorted and packed to
 1888 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1889 //         varargs C calling conventions.
 1890 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1891 //         even aligned with pad0 as needed.
 1892 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1893 //         region 6-11 is even aligned; it may be padded out more so that
 1894 //         the region from SP to FP meets the minimum stack alignment.
 1895 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1896 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1897 //         SP meets the minimum alignment.
 1898 
 1899 frame
 1900 %{
 1901   // These three registers define part of the calling convention
 1902   // between compiled code and the interpreter.
 1903   inline_cache_reg(RAX);                // Inline Cache Register
 1904 
 1905   // Optional: name the operand used by cisc-spilling to access
 1906   // [stack_pointer + offset]
 1907   cisc_spilling_operand_name(indOffset32);
 1908 
 1909   // Number of stack slots consumed by locking an object
 1910   sync_stack_slots(2);
 1911 
 1912   // Compiled code's Frame Pointer
 1913   frame_pointer(RSP);
 1914 
 1915   // Interpreter stores its frame pointer in a register which is
 1916   // stored to the stack by I2CAdaptors.
 1917   // I2CAdaptors convert from interpreted java to compiled java.
 1918   interpreter_frame_pointer(RBP);
 1919 
 1920   // Stack alignment requirement
 1921   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1922 
 1923   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1924   // for calls to C.  Supports the var-args backing area for register parms.
 1925   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1926 
 1927   // The after-PROLOG location of the return address.  Location of
 1928   // return address specifies a type (REG or STACK) and a number
 1929   // representing the register number (i.e. - use a register name) or
 1930   // stack slot.
 1931   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1932   // Otherwise, it is above the locks and verification slot and alignment word
 1933   return_addr(STACK - 2 +
 1934               align_up((Compile::current()->in_preserve_stack_slots() +
 1935                         Compile::current()->fixed_slots()),
 1936                        stack_alignment_in_slots()));
 1937 
 1938   // Location of compiled Java return values.  Same as C for now.
 1939   return_value
 1940   %{
 1941     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 1942            "only return normal values");
 1943 
 1944     static const int lo[Op_RegL + 1] = {
 1945       0,
 1946       0,
 1947       RAX_num,  // Op_RegN
 1948       RAX_num,  // Op_RegI
 1949       RAX_num,  // Op_RegP
 1950       XMM0_num, // Op_RegF
 1951       XMM0_num, // Op_RegD
 1952       RAX_num   // Op_RegL
 1953     };
 1954     static const int hi[Op_RegL + 1] = {
 1955       0,
 1956       0,
 1957       OptoReg::Bad, // Op_RegN
 1958       OptoReg::Bad, // Op_RegI
 1959       RAX_H_num,    // Op_RegP
 1960       OptoReg::Bad, // Op_RegF
 1961       XMM0b_num,    // Op_RegD
 1962       RAX_H_num     // Op_RegL
 1963     };
 1964     // Excluded flags and vector registers.
 1965     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 1966     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 1967   %}
 1968 %}
 1969 
 1970 //----------ATTRIBUTES---------------------------------------------------------
 1971 //----------Operand Attributes-------------------------------------------------
 1972 op_attrib op_cost(0);        // Required cost attribute
 1973 
 1974 //----------Instruction Attributes---------------------------------------------
 1975 ins_attrib ins_cost(100);       // Required cost attribute
 1976 ins_attrib ins_size(8);         // Required size attribute (in bits)
 1977 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 1978                                 // a non-matching short branch variant
 1979                                 // of some long branch?
 1980 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 1981                                 // be a power of 2) specifies the
 1982                                 // alignment that some part of the
 1983                                 // instruction (not necessarily the
 1984                                 // start) requires.  If > 1, a
 1985                                 // compute_padding() function must be
 1986                                 // provided for the instruction
 1987 
 1988 //----------OPERANDS-----------------------------------------------------------
 1989 // Operand definitions must precede instruction definitions for correct parsing
 1990 // in the ADLC because operands constitute user defined types which are used in
 1991 // instruction definitions.
 1992 
 1993 //----------Simple Operands----------------------------------------------------
 1994 // Immediate Operands
 1995 // Integer Immediate
 1996 operand immI()
 1997 %{
 1998   match(ConI);
 1999 
 2000   op_cost(10);
 2001   format %{ %}
 2002   interface(CONST_INTER);
 2003 %}
 2004 
 2005 // Constant for test vs zero
 2006 operand immI_0()
 2007 %{
 2008   predicate(n->get_int() == 0);
 2009   match(ConI);
 2010 
 2011   op_cost(0);
 2012   format %{ %}
 2013   interface(CONST_INTER);
 2014 %}
 2015 
 2016 // Constant for increment
 2017 operand immI_1()
 2018 %{
 2019   predicate(n->get_int() == 1);
 2020   match(ConI);
 2021 
 2022   op_cost(0);
 2023   format %{ %}
 2024   interface(CONST_INTER);
 2025 %}
 2026 
 2027 // Constant for decrement
 2028 operand immI_M1()
 2029 %{
 2030   predicate(n->get_int() == -1);
 2031   match(ConI);
 2032 
 2033   op_cost(0);
 2034   format %{ %}
 2035   interface(CONST_INTER);
 2036 %}
 2037 
 2038 operand immI_2()
 2039 %{
 2040   predicate(n->get_int() == 2);
 2041   match(ConI);
 2042 
 2043   op_cost(0);
 2044   format %{ %}
 2045   interface(CONST_INTER);
 2046 %}
 2047 
 2048 operand immI_4()
 2049 %{
 2050   predicate(n->get_int() == 4);
 2051   match(ConI);
 2052 
 2053   op_cost(0);
 2054   format %{ %}
 2055   interface(CONST_INTER);
 2056 %}
 2057 
 2058 operand immI_8()
 2059 %{
 2060   predicate(n->get_int() == 8);
 2061   match(ConI);
 2062 
 2063   op_cost(0);
 2064   format %{ %}
 2065   interface(CONST_INTER);
 2066 %}
 2067 
 2068 // Valid scale values for addressing modes
 2069 operand immI2()
 2070 %{
 2071   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2072   match(ConI);
 2073 
 2074   format %{ %}
 2075   interface(CONST_INTER);
 2076 %}
 2077 
 2078 operand immU7()
 2079 %{
 2080   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2081   match(ConI);
 2082 
 2083   op_cost(5);
 2084   format %{ %}
 2085   interface(CONST_INTER);
 2086 %}
 2087 
 2088 operand immI8()
 2089 %{
 2090   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2091   match(ConI);
 2092 
 2093   op_cost(5);
 2094   format %{ %}
 2095   interface(CONST_INTER);
 2096 %}
 2097 
 2098 operand immU8()
 2099 %{
 2100   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2101   match(ConI);
 2102 
 2103   op_cost(5);
 2104   format %{ %}
 2105   interface(CONST_INTER);
 2106 %}
 2107 
 2108 operand immI16()
 2109 %{
 2110   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2111   match(ConI);
 2112 
 2113   op_cost(10);
 2114   format %{ %}
 2115   interface(CONST_INTER);
 2116 %}
 2117 
 2118 // Int Immediate non-negative
 2119 operand immU31()
 2120 %{
 2121   predicate(n->get_int() >= 0);
 2122   match(ConI);
 2123 
 2124   op_cost(0);
 2125   format %{ %}
 2126   interface(CONST_INTER);
 2127 %}
 2128 
 2129 // Constant for long shifts
 2130 operand immI_32()
 2131 %{
 2132   predicate( n->get_int() == 32 );
 2133   match(ConI);
 2134 
 2135   op_cost(0);
 2136   format %{ %}
 2137   interface(CONST_INTER);
 2138 %}
 2139 
 2140 // Constant for long shifts
 2141 operand immI_64()
 2142 %{
 2143   predicate( n->get_int() == 64 );
 2144   match(ConI);
 2145 
 2146   op_cost(0);
 2147   format %{ %}
 2148   interface(CONST_INTER);
 2149 %}
 2150 
 2151 // Pointer Immediate
 2152 operand immP()
 2153 %{
 2154   match(ConP);
 2155 
 2156   op_cost(10);
 2157   format %{ %}
 2158   interface(CONST_INTER);
 2159 %}
 2160 
 2161 // Null Pointer Immediate
 2162 operand immP0()
 2163 %{
 2164   predicate(n->get_ptr() == 0);
 2165   match(ConP);
 2166 
 2167   op_cost(5);
 2168   format %{ %}
 2169   interface(CONST_INTER);
 2170 %}
 2171 
 2172 // Pointer Immediate
 2173 operand immN() %{
 2174   match(ConN);
 2175 
 2176   op_cost(10);
 2177   format %{ %}
 2178   interface(CONST_INTER);
 2179 %}
 2180 
 2181 operand immNKlass() %{
 2182   match(ConNKlass);
 2183 
 2184   op_cost(10);
 2185   format %{ %}
 2186   interface(CONST_INTER);
 2187 %}
 2188 
 2189 // Null Pointer Immediate
 2190 operand immN0() %{
 2191   predicate(n->get_narrowcon() == 0);
 2192   match(ConN);
 2193 
 2194   op_cost(5);
 2195   format %{ %}
 2196   interface(CONST_INTER);
 2197 %}
 2198 
 2199 operand immP31()
 2200 %{
 2201   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2202             && (n->get_ptr() >> 31) == 0);
 2203   match(ConP);
 2204 
 2205   op_cost(5);
 2206   format %{ %}
 2207   interface(CONST_INTER);
 2208 %}
 2209 
 2210 
 2211 // Long Immediate
 2212 operand immL()
 2213 %{
 2214   match(ConL);
 2215 
 2216   op_cost(20);
 2217   format %{ %}
 2218   interface(CONST_INTER);
 2219 %}
 2220 
 2221 // Long Immediate 8-bit
 2222 operand immL8()
 2223 %{
 2224   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2225   match(ConL);
 2226 
 2227   op_cost(5);
 2228   format %{ %}
 2229   interface(CONST_INTER);
 2230 %}
 2231 
 2232 // Long Immediate 32-bit unsigned
 2233 operand immUL32()
 2234 %{
 2235   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2236   match(ConL);
 2237 
 2238   op_cost(10);
 2239   format %{ %}
 2240   interface(CONST_INTER);
 2241 %}
 2242 
 2243 // Long Immediate 32-bit signed
 2244 operand immL32()
 2245 %{
 2246   predicate(n->get_long() == (int) (n->get_long()));
 2247   match(ConL);
 2248 
 2249   op_cost(15);
 2250   format %{ %}
 2251   interface(CONST_INTER);
 2252 %}
 2253 
 2254 operand immL_Pow2()
 2255 %{
 2256   predicate(is_power_of_2((julong)n->get_long()));
 2257   match(ConL);
 2258 
 2259   op_cost(15);
 2260   format %{ %}
 2261   interface(CONST_INTER);
 2262 %}
 2263 
 2264 operand immL_NotPow2()
 2265 %{
 2266   predicate(is_power_of_2((julong)~n->get_long()));
 2267   match(ConL);
 2268 
 2269   op_cost(15);
 2270   format %{ %}
 2271   interface(CONST_INTER);
 2272 %}
 2273 
 2274 // Long Immediate zero
 2275 operand immL0()
 2276 %{
 2277   predicate(n->get_long() == 0L);
 2278   match(ConL);
 2279 
 2280   op_cost(10);
 2281   format %{ %}
 2282   interface(CONST_INTER);
 2283 %}
 2284 
 2285 // Constant for increment
 2286 operand immL1()
 2287 %{
 2288   predicate(n->get_long() == 1);
 2289   match(ConL);
 2290 
 2291   format %{ %}
 2292   interface(CONST_INTER);
 2293 %}
 2294 
 2295 // Constant for decrement
 2296 operand immL_M1()
 2297 %{
 2298   predicate(n->get_long() == -1);
 2299   match(ConL);
 2300 
 2301   format %{ %}
 2302   interface(CONST_INTER);
 2303 %}
 2304 
 2305 // Long Immediate: the value 10
 2306 operand immL10()
 2307 %{
 2308   predicate(n->get_long() == 10);
 2309   match(ConL);
 2310 
 2311   format %{ %}
 2312   interface(CONST_INTER);
 2313 %}
 2314 
 2315 // Long immediate from 0 to 127.
 2316 // Used for a shorter form of long mul by 10.
 2317 operand immL_127()
 2318 %{
 2319   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 2320   match(ConL);
 2321 
 2322   op_cost(10);
 2323   format %{ %}
 2324   interface(CONST_INTER);
 2325 %}
 2326 
 2327 // Long Immediate: low 32-bit mask
 2328 operand immL_32bits()
 2329 %{
 2330   predicate(n->get_long() == 0xFFFFFFFFL);
 2331   match(ConL);
 2332   op_cost(20);
 2333 
 2334   format %{ %}
 2335   interface(CONST_INTER);
 2336 %}
 2337 
 2338 // Int Immediate: 2^n-1, positive
 2339 operand immI_Pow2M1()
 2340 %{
 2341   predicate((n->get_int() > 0)
 2342             && is_power_of_2((juint)n->get_int() + 1));
 2343   match(ConI);
 2344 
 2345   op_cost(20);
 2346   format %{ %}
 2347   interface(CONST_INTER);
 2348 %}
 2349 
 2350 // Float Immediate zero
 2351 operand immF0()
 2352 %{
 2353   predicate(jint_cast(n->getf()) == 0);
 2354   match(ConF);
 2355 
 2356   op_cost(5);
 2357   format %{ %}
 2358   interface(CONST_INTER);
 2359 %}
 2360 
 2361 // Float Immediate
 2362 operand immF()
 2363 %{
 2364   match(ConF);
 2365 
 2366   op_cost(15);
 2367   format %{ %}
 2368   interface(CONST_INTER);
 2369 %}
 2370 
 2371 // Double Immediate zero
 2372 operand immD0()
 2373 %{
 2374   predicate(jlong_cast(n->getd()) == 0);
 2375   match(ConD);
 2376 
 2377   op_cost(5);
 2378   format %{ %}
 2379   interface(CONST_INTER);
 2380 %}
 2381 
 2382 // Double Immediate
 2383 operand immD()
 2384 %{
 2385   match(ConD);
 2386 
 2387   op_cost(15);
 2388   format %{ %}
 2389   interface(CONST_INTER);
 2390 %}
 2391 
 2392 // Immediates for special shifts (sign extend)
 2393 
 2394 // Constants for increment
 2395 operand immI_16()
 2396 %{
 2397   predicate(n->get_int() == 16);
 2398   match(ConI);
 2399 
 2400   format %{ %}
 2401   interface(CONST_INTER);
 2402 %}
 2403 
 2404 operand immI_24()
 2405 %{
 2406   predicate(n->get_int() == 24);
 2407   match(ConI);
 2408 
 2409   format %{ %}
 2410   interface(CONST_INTER);
 2411 %}
 2412 
 2413 // Constant for byte-wide masking
 2414 operand immI_255()
 2415 %{
 2416   predicate(n->get_int() == 255);
 2417   match(ConI);
 2418 
 2419   format %{ %}
 2420   interface(CONST_INTER);
 2421 %}
 2422 
 2423 // Constant for short-wide masking
 2424 operand immI_65535()
 2425 %{
 2426   predicate(n->get_int() == 65535);
 2427   match(ConI);
 2428 
 2429   format %{ %}
 2430   interface(CONST_INTER);
 2431 %}
 2432 
 2433 // Constant for byte-wide masking
 2434 operand immL_255()
 2435 %{
 2436   predicate(n->get_long() == 255);
 2437   match(ConL);
 2438 
 2439   format %{ %}
 2440   interface(CONST_INTER);
 2441 %}
 2442 
 2443 // Constant for short-wide masking
 2444 operand immL_65535()
 2445 %{
 2446   predicate(n->get_long() == 65535);
 2447   match(ConL);
 2448 
 2449   format %{ %}
 2450   interface(CONST_INTER);
 2451 %}
 2452 
 2453 operand kReg()
 2454 %{
 2455   constraint(ALLOC_IN_RC(vectmask_reg));
 2456   match(RegVectMask);
 2457   format %{%}
 2458   interface(REG_INTER);
 2459 %}
 2460 
 2461 operand kReg_K1()
 2462 %{
 2463   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 2464   match(RegVectMask);
 2465   format %{%}
 2466   interface(REG_INTER);
 2467 %}
 2468 
 2469 operand kReg_K2()
 2470 %{
 2471   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 2472   match(RegVectMask);
 2473   format %{%}
 2474   interface(REG_INTER);
 2475 %}
 2476 
 2477 // Special Registers
 2478 operand kReg_K3()
 2479 %{
 2480   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 2481   match(RegVectMask);
 2482   format %{%}
 2483   interface(REG_INTER);
 2484 %}
 2485 
 2486 operand kReg_K4()
 2487 %{
 2488   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 2489   match(RegVectMask);
 2490   format %{%}
 2491   interface(REG_INTER);
 2492 %}
 2493 
 2494 operand kReg_K5()
 2495 %{
 2496   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 2497   match(RegVectMask);
 2498   format %{%}
 2499   interface(REG_INTER);
 2500 %}
 2501 
 2502 operand kReg_K6()
 2503 %{
 2504   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 2505   match(RegVectMask);
 2506   format %{%}
 2507   interface(REG_INTER);
 2508 %}
 2509 
 2510 // Special Registers
 2511 operand kReg_K7()
 2512 %{
 2513   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 2514   match(RegVectMask);
 2515   format %{%}
 2516   interface(REG_INTER);
 2517 %}
 2518 
 2519 // Register Operands
 2520 // Integer Register
 2521 operand rRegI()
 2522 %{
 2523   constraint(ALLOC_IN_RC(int_reg));
 2524   match(RegI);
 2525 
 2526   match(rax_RegI);
 2527   match(rbx_RegI);
 2528   match(rcx_RegI);
 2529   match(rdx_RegI);
 2530   match(rdi_RegI);
 2531 
 2532   format %{ %}
 2533   interface(REG_INTER);
 2534 %}
 2535 
 2536 // Special Registers
 2537 operand rax_RegI()
 2538 %{
 2539   constraint(ALLOC_IN_RC(int_rax_reg));
 2540   match(RegI);
 2541   match(rRegI);
 2542 
 2543   format %{ "RAX" %}
 2544   interface(REG_INTER);
 2545 %}
 2546 
 2547 // Special Registers
 2548 operand rbx_RegI()
 2549 %{
 2550   constraint(ALLOC_IN_RC(int_rbx_reg));
 2551   match(RegI);
 2552   match(rRegI);
 2553 
 2554   format %{ "RBX" %}
 2555   interface(REG_INTER);
 2556 %}
 2557 
 2558 operand rcx_RegI()
 2559 %{
 2560   constraint(ALLOC_IN_RC(int_rcx_reg));
 2561   match(RegI);
 2562   match(rRegI);
 2563 
 2564   format %{ "RCX" %}
 2565   interface(REG_INTER);
 2566 %}
 2567 
 2568 operand rdx_RegI()
 2569 %{
 2570   constraint(ALLOC_IN_RC(int_rdx_reg));
 2571   match(RegI);
 2572   match(rRegI);
 2573 
 2574   format %{ "RDX" %}
 2575   interface(REG_INTER);
 2576 %}
 2577 
 2578 operand rdi_RegI()
 2579 %{
 2580   constraint(ALLOC_IN_RC(int_rdi_reg));
 2581   match(RegI);
 2582   match(rRegI);
 2583 
 2584   format %{ "RDI" %}
 2585   interface(REG_INTER);
 2586 %}
 2587 
 2588 operand no_rax_rdx_RegI()
 2589 %{
 2590   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2591   match(RegI);
 2592   match(rbx_RegI);
 2593   match(rcx_RegI);
 2594   match(rdi_RegI);
 2595 
 2596   format %{ %}
 2597   interface(REG_INTER);
 2598 %}
 2599 
 2600 operand no_rbp_r13_RegI()
 2601 %{
 2602   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2603   match(RegI);
 2604   match(rRegI);
 2605   match(rax_RegI);
 2606   match(rbx_RegI);
 2607   match(rcx_RegI);
 2608   match(rdx_RegI);
 2609   match(rdi_RegI);
 2610 
 2611   format %{ %}
 2612   interface(REG_INTER);
 2613 %}
 2614 
 2615 // Pointer Register
 2616 operand any_RegP()
 2617 %{
 2618   constraint(ALLOC_IN_RC(any_reg));
 2619   match(RegP);
 2620   match(rax_RegP);
 2621   match(rbx_RegP);
 2622   match(rdi_RegP);
 2623   match(rsi_RegP);
 2624   match(rbp_RegP);
 2625   match(r15_RegP);
 2626   match(rRegP);
 2627 
 2628   format %{ %}
 2629   interface(REG_INTER);
 2630 %}
 2631 
 2632 operand rRegP()
 2633 %{
 2634   constraint(ALLOC_IN_RC(ptr_reg));
 2635   match(RegP);
 2636   match(rax_RegP);
 2637   match(rbx_RegP);
 2638   match(rdi_RegP);
 2639   match(rsi_RegP);
 2640   match(rbp_RegP);  // See Q&A below about
 2641   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2642 
 2643   format %{ %}
 2644   interface(REG_INTER);
 2645 %}
 2646 
 2647 operand rRegN() %{
 2648   constraint(ALLOC_IN_RC(int_reg));
 2649   match(RegN);
 2650 
 2651   format %{ %}
 2652   interface(REG_INTER);
 2653 %}
 2654 
 2655 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2656 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2657 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2658 // The output of an instruction is controlled by the allocator, which respects
 2659 // register class masks, not match rules.  Unless an instruction mentions
 2660 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2661 // by the allocator as an input.
 2662 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2663 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2664 // result, RBP is not included in the output of the instruction either.
 2665 
 2666 operand no_rax_RegP()
 2667 %{
 2668   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 2669   match(RegP);
 2670   match(rbx_RegP);
 2671   match(rsi_RegP);
 2672   match(rdi_RegP);
 2673 
 2674   format %{ %}
 2675   interface(REG_INTER);
 2676 %}
 2677 
 2678 // This operand is not allowed to use RBP even if
 2679 // RBP is not used to hold the frame pointer.
 2680 operand no_rbp_RegP()
 2681 %{
 2682   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2683   match(RegP);
 2684   match(rbx_RegP);
 2685   match(rsi_RegP);
 2686   match(rdi_RegP);
 2687 
 2688   format %{ %}
 2689   interface(REG_INTER);
 2690 %}
 2691 
 2692 operand no_rax_rbx_RegP()
 2693 %{
 2694   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 2695   match(RegP);
 2696   match(rsi_RegP);
 2697   match(rdi_RegP);
 2698 
 2699   format %{ %}
 2700   interface(REG_INTER);
 2701 %}
 2702 
 2703 // Special Registers
 2704 // Return a pointer value
 2705 operand rax_RegP()
 2706 %{
 2707   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2708   match(RegP);
 2709   match(rRegP);
 2710 
 2711   format %{ %}
 2712   interface(REG_INTER);
 2713 %}
 2714 
 2715 // Special Registers
 2716 // Return a compressed pointer value
 2717 operand rax_RegN()
 2718 %{
 2719   constraint(ALLOC_IN_RC(int_rax_reg));
 2720   match(RegN);
 2721   match(rRegN);
 2722 
 2723   format %{ %}
 2724   interface(REG_INTER);
 2725 %}
 2726 
 2727 // Used in AtomicAdd
 2728 operand rbx_RegP()
 2729 %{
 2730   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2731   match(RegP);
 2732   match(rRegP);
 2733 
 2734   format %{ %}
 2735   interface(REG_INTER);
 2736 %}
 2737 
 2738 operand rsi_RegP()
 2739 %{
 2740   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2741   match(RegP);
 2742   match(rRegP);
 2743 
 2744   format %{ %}
 2745   interface(REG_INTER);
 2746 %}
 2747 
 2748 operand rbp_RegP()
 2749 %{
 2750   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2751   match(RegP);
 2752   match(rRegP);
 2753 
 2754   format %{ %}
 2755   interface(REG_INTER);
 2756 %}
 2757 
 2758 // Used in rep stosq
 2759 operand rdi_RegP()
 2760 %{
 2761   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2762   match(RegP);
 2763   match(rRegP);
 2764 
 2765   format %{ %}
 2766   interface(REG_INTER);
 2767 %}
 2768 
 2769 operand r15_RegP()
 2770 %{
 2771   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2772   match(RegP);
 2773   match(rRegP);
 2774 
 2775   format %{ %}
 2776   interface(REG_INTER);
 2777 %}
 2778 
 2779 operand rRegL()
 2780 %{
 2781   constraint(ALLOC_IN_RC(long_reg));
 2782   match(RegL);
 2783   match(rax_RegL);
 2784   match(rdx_RegL);
 2785 
 2786   format %{ %}
 2787   interface(REG_INTER);
 2788 %}
 2789 
 2790 // Special Registers
 2791 operand no_rax_rdx_RegL()
 2792 %{
 2793   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2794   match(RegL);
 2795   match(rRegL);
 2796 
 2797   format %{ %}
 2798   interface(REG_INTER);
 2799 %}
 2800 
 2801 operand rax_RegL()
 2802 %{
 2803   constraint(ALLOC_IN_RC(long_rax_reg));
 2804   match(RegL);
 2805   match(rRegL);
 2806 
 2807   format %{ "RAX" %}
 2808   interface(REG_INTER);
 2809 %}
 2810 
 2811 operand rcx_RegL()
 2812 %{
 2813   constraint(ALLOC_IN_RC(long_rcx_reg));
 2814   match(RegL);
 2815   match(rRegL);
 2816 
 2817   format %{ %}
 2818   interface(REG_INTER);
 2819 %}
 2820 
 2821 operand rdx_RegL()
 2822 %{
 2823   constraint(ALLOC_IN_RC(long_rdx_reg));
 2824   match(RegL);
 2825   match(rRegL);
 2826 
 2827   format %{ %}
 2828   interface(REG_INTER);
 2829 %}
 2830 
 2831 operand no_rbp_r13_RegL()
 2832 %{
 2833   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2834   match(RegL);
 2835   match(rRegL);
 2836   match(rax_RegL);
 2837   match(rcx_RegL);
 2838   match(rdx_RegL);
 2839 
 2840   format %{ %}
 2841   interface(REG_INTER);
 2842 %}
 2843 
 2844 // Flags register, used as output of compare instructions
 2845 operand rFlagsReg()
 2846 %{
 2847   constraint(ALLOC_IN_RC(int_flags));
 2848   match(RegFlags);
 2849 
 2850   format %{ "RFLAGS" %}
 2851   interface(REG_INTER);
 2852 %}
 2853 
 2854 // Flags register, used as output of FLOATING POINT compare instructions
 2855 operand rFlagsRegU()
 2856 %{
 2857   constraint(ALLOC_IN_RC(int_flags));
 2858   match(RegFlags);
 2859 
 2860   format %{ "RFLAGS_U" %}
 2861   interface(REG_INTER);
 2862 %}
 2863 
 2864 operand rFlagsRegUCF() %{
 2865   constraint(ALLOC_IN_RC(int_flags));
 2866   match(RegFlags);
 2867   predicate(false);
 2868 
 2869   format %{ "RFLAGS_U_CF" %}
 2870   interface(REG_INTER);
 2871 %}
 2872 
 2873 // Float register operands
 2874 operand regF() %{
 2875    constraint(ALLOC_IN_RC(float_reg));
 2876    match(RegF);
 2877 
 2878    format %{ %}
 2879    interface(REG_INTER);
 2880 %}
 2881 
 2882 // Float register operands
 2883 operand legRegF() %{
 2884    constraint(ALLOC_IN_RC(float_reg_legacy));
 2885    match(RegF);
 2886 
 2887    format %{ %}
 2888    interface(REG_INTER);
 2889 %}
 2890 
 2891 // Float register operands
 2892 operand vlRegF() %{
 2893    constraint(ALLOC_IN_RC(float_reg_vl));
 2894    match(RegF);
 2895 
 2896    format %{ %}
 2897    interface(REG_INTER);
 2898 %}
 2899 
 2900 // Double register operands
 2901 operand regD() %{
 2902    constraint(ALLOC_IN_RC(double_reg));
 2903    match(RegD);
 2904 
 2905    format %{ %}
 2906    interface(REG_INTER);
 2907 %}
 2908 
 2909 // Double register operands
 2910 operand legRegD() %{
 2911    constraint(ALLOC_IN_RC(double_reg_legacy));
 2912    match(RegD);
 2913 
 2914    format %{ %}
 2915    interface(REG_INTER);
 2916 %}
 2917 
 2918 // Double register operands
 2919 operand vlRegD() %{
 2920    constraint(ALLOC_IN_RC(double_reg_vl));
 2921    match(RegD);
 2922 
 2923    format %{ %}
 2924    interface(REG_INTER);
 2925 %}
 2926 
 2927 //----------Memory Operands----------------------------------------------------
 2928 // Direct Memory Operand
 2929 // operand direct(immP addr)
 2930 // %{
 2931 //   match(addr);
 2932 
 2933 //   format %{ "[$addr]" %}
 2934 //   interface(MEMORY_INTER) %{
 2935 //     base(0xFFFFFFFF);
 2936 //     index(0x4);
 2937 //     scale(0x0);
 2938 //     disp($addr);
 2939 //   %}
 2940 // %}
 2941 
 2942 // Indirect Memory Operand
 2943 operand indirect(any_RegP reg)
 2944 %{
 2945   constraint(ALLOC_IN_RC(ptr_reg));
 2946   match(reg);
 2947 
 2948   format %{ "[$reg]" %}
 2949   interface(MEMORY_INTER) %{
 2950     base($reg);
 2951     index(0x4);
 2952     scale(0x0);
 2953     disp(0x0);
 2954   %}
 2955 %}
 2956 
 2957 // Indirect Memory Plus Short Offset Operand
 2958 operand indOffset8(any_RegP reg, immL8 off)
 2959 %{
 2960   constraint(ALLOC_IN_RC(ptr_reg));
 2961   match(AddP reg off);
 2962 
 2963   format %{ "[$reg + $off (8-bit)]" %}
 2964   interface(MEMORY_INTER) %{
 2965     base($reg);
 2966     index(0x4);
 2967     scale(0x0);
 2968     disp($off);
 2969   %}
 2970 %}
 2971 
 2972 // Indirect Memory Plus Long Offset Operand
 2973 operand indOffset32(any_RegP reg, immL32 off)
 2974 %{
 2975   constraint(ALLOC_IN_RC(ptr_reg));
 2976   match(AddP reg off);
 2977 
 2978   format %{ "[$reg + $off (32-bit)]" %}
 2979   interface(MEMORY_INTER) %{
 2980     base($reg);
 2981     index(0x4);
 2982     scale(0x0);
 2983     disp($off);
 2984   %}
 2985 %}
 2986 
 2987 // Indirect Memory Plus Index Register Plus Offset Operand
 2988 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2989 %{
 2990   constraint(ALLOC_IN_RC(ptr_reg));
 2991   match(AddP (AddP reg lreg) off);
 2992 
 2993   op_cost(10);
 2994   format %{"[$reg + $off + $lreg]" %}
 2995   interface(MEMORY_INTER) %{
 2996     base($reg);
 2997     index($lreg);
 2998     scale(0x0);
 2999     disp($off);
 3000   %}
 3001 %}
 3002 
 3003 // Indirect Memory Plus Index Register Plus Offset Operand
 3004 operand indIndex(any_RegP reg, rRegL lreg)
 3005 %{
 3006   constraint(ALLOC_IN_RC(ptr_reg));
 3007   match(AddP reg lreg);
 3008 
 3009   op_cost(10);
 3010   format %{"[$reg + $lreg]" %}
 3011   interface(MEMORY_INTER) %{
 3012     base($reg);
 3013     index($lreg);
 3014     scale(0x0);
 3015     disp(0x0);
 3016   %}
 3017 %}
 3018 
 3019 // Indirect Memory Times Scale Plus Index Register
 3020 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3021 %{
 3022   constraint(ALLOC_IN_RC(ptr_reg));
 3023   match(AddP reg (LShiftL lreg scale));
 3024 
 3025   op_cost(10);
 3026   format %{"[$reg + $lreg << $scale]" %}
 3027   interface(MEMORY_INTER) %{
 3028     base($reg);
 3029     index($lreg);
 3030     scale($scale);
 3031     disp(0x0);
 3032   %}
 3033 %}
 3034 
 3035 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3036 %{
 3037   constraint(ALLOC_IN_RC(ptr_reg));
 3038   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3039   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3040 
 3041   op_cost(10);
 3042   format %{"[$reg + pos $idx << $scale]" %}
 3043   interface(MEMORY_INTER) %{
 3044     base($reg);
 3045     index($idx);
 3046     scale($scale);
 3047     disp(0x0);
 3048   %}
 3049 %}
 3050 
 3051 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3052 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3053 %{
 3054   constraint(ALLOC_IN_RC(ptr_reg));
 3055   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3056 
 3057   op_cost(10);
 3058   format %{"[$reg + $off + $lreg << $scale]" %}
 3059   interface(MEMORY_INTER) %{
 3060     base($reg);
 3061     index($lreg);
 3062     scale($scale);
 3063     disp($off);
 3064   %}
 3065 %}
 3066 
 3067 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3068 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3069 %{
 3070   constraint(ALLOC_IN_RC(ptr_reg));
 3071   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3072   match(AddP (AddP reg (ConvI2L idx)) off);
 3073 
 3074   op_cost(10);
 3075   format %{"[$reg + $off + $idx]" %}
 3076   interface(MEMORY_INTER) %{
 3077     base($reg);
 3078     index($idx);
 3079     scale(0x0);
 3080     disp($off);
 3081   %}
 3082 %}
 3083 
 3084 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3085 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3086 %{
 3087   constraint(ALLOC_IN_RC(ptr_reg));
 3088   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3089   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3090 
 3091   op_cost(10);
 3092   format %{"[$reg + $off + $idx << $scale]" %}
 3093   interface(MEMORY_INTER) %{
 3094     base($reg);
 3095     index($idx);
 3096     scale($scale);
 3097     disp($off);
 3098   %}
 3099 %}
 3100 
 3101 // Indirect Narrow Oop Plus Offset Operand
 3102 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3103 // we can't free r12 even with CompressedOops::base() == nullptr.
 3104 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3105   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3106   constraint(ALLOC_IN_RC(ptr_reg));
 3107   match(AddP (DecodeN reg) off);
 3108 
 3109   op_cost(10);
 3110   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3111   interface(MEMORY_INTER) %{
 3112     base(0xc); // R12
 3113     index($reg);
 3114     scale(0x3);
 3115     disp($off);
 3116   %}
 3117 %}
 3118 
 3119 // Indirect Memory Operand
 3120 operand indirectNarrow(rRegN reg)
 3121 %{
 3122   predicate(CompressedOops::shift() == 0);
 3123   constraint(ALLOC_IN_RC(ptr_reg));
 3124   match(DecodeN reg);
 3125 
 3126   format %{ "[$reg]" %}
 3127   interface(MEMORY_INTER) %{
 3128     base($reg);
 3129     index(0x4);
 3130     scale(0x0);
 3131     disp(0x0);
 3132   %}
 3133 %}
 3134 
 3135 // Indirect Memory Plus Short Offset Operand
 3136 operand indOffset8Narrow(rRegN reg, immL8 off)
 3137 %{
 3138   predicate(CompressedOops::shift() == 0);
 3139   constraint(ALLOC_IN_RC(ptr_reg));
 3140   match(AddP (DecodeN reg) off);
 3141 
 3142   format %{ "[$reg + $off (8-bit)]" %}
 3143   interface(MEMORY_INTER) %{
 3144     base($reg);
 3145     index(0x4);
 3146     scale(0x0);
 3147     disp($off);
 3148   %}
 3149 %}
 3150 
 3151 // Indirect Memory Plus Long Offset Operand
 3152 operand indOffset32Narrow(rRegN reg, immL32 off)
 3153 %{
 3154   predicate(CompressedOops::shift() == 0);
 3155   constraint(ALLOC_IN_RC(ptr_reg));
 3156   match(AddP (DecodeN reg) off);
 3157 
 3158   format %{ "[$reg + $off (32-bit)]" %}
 3159   interface(MEMORY_INTER) %{
 3160     base($reg);
 3161     index(0x4);
 3162     scale(0x0);
 3163     disp($off);
 3164   %}
 3165 %}
 3166 
 3167 // Indirect Memory Plus Index Register Plus Offset Operand
 3168 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3169 %{
 3170   predicate(CompressedOops::shift() == 0);
 3171   constraint(ALLOC_IN_RC(ptr_reg));
 3172   match(AddP (AddP (DecodeN reg) lreg) off);
 3173 
 3174   op_cost(10);
 3175   format %{"[$reg + $off + $lreg]" %}
 3176   interface(MEMORY_INTER) %{
 3177     base($reg);
 3178     index($lreg);
 3179     scale(0x0);
 3180     disp($off);
 3181   %}
 3182 %}
 3183 
 3184 // Indirect Memory Plus Index Register Plus Offset Operand
 3185 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3186 %{
 3187   predicate(CompressedOops::shift() == 0);
 3188   constraint(ALLOC_IN_RC(ptr_reg));
 3189   match(AddP (DecodeN reg) lreg);
 3190 
 3191   op_cost(10);
 3192   format %{"[$reg + $lreg]" %}
 3193   interface(MEMORY_INTER) %{
 3194     base($reg);
 3195     index($lreg);
 3196     scale(0x0);
 3197     disp(0x0);
 3198   %}
 3199 %}
 3200 
 3201 // Indirect Memory Times Scale Plus Index Register
 3202 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3203 %{
 3204   predicate(CompressedOops::shift() == 0);
 3205   constraint(ALLOC_IN_RC(ptr_reg));
 3206   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3207 
 3208   op_cost(10);
 3209   format %{"[$reg + $lreg << $scale]" %}
 3210   interface(MEMORY_INTER) %{
 3211     base($reg);
 3212     index($lreg);
 3213     scale($scale);
 3214     disp(0x0);
 3215   %}
 3216 %}
 3217 
 3218 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3219 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3220 %{
 3221   predicate(CompressedOops::shift() == 0);
 3222   constraint(ALLOC_IN_RC(ptr_reg));
 3223   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3224 
 3225   op_cost(10);
 3226   format %{"[$reg + $off + $lreg << $scale]" %}
 3227   interface(MEMORY_INTER) %{
 3228     base($reg);
 3229     index($lreg);
 3230     scale($scale);
 3231     disp($off);
 3232   %}
 3233 %}
 3234 
 3235 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3236 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3237 %{
 3238   constraint(ALLOC_IN_RC(ptr_reg));
 3239   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3240   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3241 
 3242   op_cost(10);
 3243   format %{"[$reg + $off + $idx]" %}
 3244   interface(MEMORY_INTER) %{
 3245     base($reg);
 3246     index($idx);
 3247     scale(0x0);
 3248     disp($off);
 3249   %}
 3250 %}
 3251 
 3252 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3253 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3254 %{
 3255   constraint(ALLOC_IN_RC(ptr_reg));
 3256   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3257   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3258 
 3259   op_cost(10);
 3260   format %{"[$reg + $off + $idx << $scale]" %}
 3261   interface(MEMORY_INTER) %{
 3262     base($reg);
 3263     index($idx);
 3264     scale($scale);
 3265     disp($off);
 3266   %}
 3267 %}
 3268 
 3269 //----------Special Memory Operands--------------------------------------------
 3270 // Stack Slot Operand - This operand is used for loading and storing temporary
 3271 //                      values on the stack where a match requires a value to
 3272 //                      flow through memory.
 3273 operand stackSlotP(sRegP reg)
 3274 %{
 3275   constraint(ALLOC_IN_RC(stack_slots));
 3276   // No match rule because this operand is only generated in matching
 3277 
 3278   format %{ "[$reg]" %}
 3279   interface(MEMORY_INTER) %{
 3280     base(0x4);   // RSP
 3281     index(0x4);  // No Index
 3282     scale(0x0);  // No Scale
 3283     disp($reg);  // Stack Offset
 3284   %}
 3285 %}
 3286 
 3287 operand stackSlotI(sRegI reg)
 3288 %{
 3289   constraint(ALLOC_IN_RC(stack_slots));
 3290   // No match rule because this operand is only generated in matching
 3291 
 3292   format %{ "[$reg]" %}
 3293   interface(MEMORY_INTER) %{
 3294     base(0x4);   // RSP
 3295     index(0x4);  // No Index
 3296     scale(0x0);  // No Scale
 3297     disp($reg);  // Stack Offset
 3298   %}
 3299 %}
 3300 
 3301 operand stackSlotF(sRegF reg)
 3302 %{
 3303   constraint(ALLOC_IN_RC(stack_slots));
 3304   // No match rule because this operand is only generated in matching
 3305 
 3306   format %{ "[$reg]" %}
 3307   interface(MEMORY_INTER) %{
 3308     base(0x4);   // RSP
 3309     index(0x4);  // No Index
 3310     scale(0x0);  // No Scale
 3311     disp($reg);  // Stack Offset
 3312   %}
 3313 %}
 3314 
 3315 operand stackSlotD(sRegD reg)
 3316 %{
 3317   constraint(ALLOC_IN_RC(stack_slots));
 3318   // No match rule because this operand is only generated in matching
 3319 
 3320   format %{ "[$reg]" %}
 3321   interface(MEMORY_INTER) %{
 3322     base(0x4);   // RSP
 3323     index(0x4);  // No Index
 3324     scale(0x0);  // No Scale
 3325     disp($reg);  // Stack Offset
 3326   %}
 3327 %}
 3328 operand stackSlotL(sRegL reg)
 3329 %{
 3330   constraint(ALLOC_IN_RC(stack_slots));
 3331   // No match rule because this operand is only generated in matching
 3332 
 3333   format %{ "[$reg]" %}
 3334   interface(MEMORY_INTER) %{
 3335     base(0x4);   // RSP
 3336     index(0x4);  // No Index
 3337     scale(0x0);  // No Scale
 3338     disp($reg);  // Stack Offset
 3339   %}
 3340 %}
 3341 
 3342 //----------Conditional Branch Operands----------------------------------------
 3343 // Comparison Op  - This is the operation of the comparison, and is limited to
 3344 //                  the following set of codes:
 3345 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3346 //
 3347 // Other attributes of the comparison, such as unsignedness, are specified
 3348 // by the comparison instruction that sets a condition code flags register.
 3349 // That result is represented by a flags operand whose subtype is appropriate
 3350 // to the unsignedness (etc.) of the comparison.
 3351 //
 3352 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3353 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3354 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3355 
 3356 // Comparison Code
 3357 operand cmpOp()
 3358 %{
 3359   match(Bool);
 3360 
 3361   format %{ "" %}
 3362   interface(COND_INTER) %{
 3363     equal(0x4, "e");
 3364     not_equal(0x5, "ne");
 3365     less(0xC, "l");
 3366     greater_equal(0xD, "ge");
 3367     less_equal(0xE, "le");
 3368     greater(0xF, "g");
 3369     overflow(0x0, "o");
 3370     no_overflow(0x1, "no");
 3371   %}
 3372 %}
 3373 
 3374 // Comparison Code, unsigned compare.  Used by FP also, with
 3375 // C2 (unordered) turned into GT or LT already.  The other bits
 3376 // C0 and C3 are turned into Carry & Zero flags.
 3377 operand cmpOpU()
 3378 %{
 3379   match(Bool);
 3380 
 3381   format %{ "" %}
 3382   interface(COND_INTER) %{
 3383     equal(0x4, "e");
 3384     not_equal(0x5, "ne");
 3385     less(0x2, "b");
 3386     greater_equal(0x3, "ae");
 3387     less_equal(0x6, "be");
 3388     greater(0x7, "a");
 3389     overflow(0x0, "o");
 3390     no_overflow(0x1, "no");
 3391   %}
 3392 %}
 3393 
 3394 
 3395 // Floating comparisons that don't require any fixup for the unordered case,
 3396 // If both inputs of the comparison are the same, ZF is always set so we
 3397 // don't need to use cmpOpUCF2 for eq/ne
 3398 operand cmpOpUCF() %{
 3399   match(Bool);
 3400   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3401             n->as_Bool()->_test._test == BoolTest::ge ||
 3402             n->as_Bool()->_test._test == BoolTest::le ||
 3403             n->as_Bool()->_test._test == BoolTest::gt ||
 3404             n->in(1)->in(1) == n->in(1)->in(2));
 3405   format %{ "" %}
 3406   interface(COND_INTER) %{
 3407     equal(0xb, "np");
 3408     not_equal(0xa, "p");
 3409     less(0x2, "b");
 3410     greater_equal(0x3, "ae");
 3411     less_equal(0x6, "be");
 3412     greater(0x7, "a");
 3413     overflow(0x0, "o");
 3414     no_overflow(0x1, "no");
 3415   %}
 3416 %}
 3417 
 3418 
 3419 // Floating comparisons that can be fixed up with extra conditional jumps
 3420 operand cmpOpUCF2() %{
 3421   match(Bool);
 3422   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3423              n->as_Bool()->_test._test == BoolTest::eq) &&
 3424             n->in(1)->in(1) != n->in(1)->in(2));
 3425   format %{ "" %}
 3426   interface(COND_INTER) %{
 3427     equal(0x4, "e");
 3428     not_equal(0x5, "ne");
 3429     less(0x2, "b");
 3430     greater_equal(0x3, "ae");
 3431     less_equal(0x6, "be");
 3432     greater(0x7, "a");
 3433     overflow(0x0, "o");
 3434     no_overflow(0x1, "no");
 3435   %}
 3436 %}
 3437 
 3438 //----------OPERAND CLASSES----------------------------------------------------
 3439 // Operand Classes are groups of operands that are used as to simplify
 3440 // instruction definitions by not requiring the AD writer to specify separate
 3441 // instructions for every form of operand when the instruction accepts
 3442 // multiple operand types with the same basic encoding and format.  The classic
 3443 // case of this is memory operands.
 3444 
 3445 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3446                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3447                indCompressedOopOffset,
 3448                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3449                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3450                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3451 
 3452 //----------PIPELINE-----------------------------------------------------------
 3453 // Rules which define the behavior of the target architectures pipeline.
 3454 pipeline %{
 3455 
 3456 //----------ATTRIBUTES---------------------------------------------------------
 3457 attributes %{
 3458   variable_size_instructions;        // Fixed size instructions
 3459   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3460   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3461   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3462   instruction_fetch_units = 1;       // of 16 bytes
 3463 
 3464   // List of nop instructions
 3465   nops( MachNop );
 3466 %}
 3467 
 3468 //----------RESOURCES----------------------------------------------------------
 3469 // Resources are the functional units available to the machine
 3470 
 3471 // Generic P2/P3 pipeline
 3472 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3473 // 3 instructions decoded per cycle.
 3474 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3475 // 3 ALU op, only ALU0 handles mul instructions.
 3476 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3477            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3478            BR, FPU,
 3479            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3480 
 3481 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3482 // Pipeline Description specifies the stages in the machine's pipeline
 3483 
 3484 // Generic P2/P3 pipeline
 3485 pipe_desc(S0, S1, S2, S3, S4, S5);
 3486 
 3487 //----------PIPELINE CLASSES---------------------------------------------------
 3488 // Pipeline Classes describe the stages in which input and output are
 3489 // referenced by the hardware pipeline.
 3490 
 3491 // Naming convention: ialu or fpu
 3492 // Then: _reg
 3493 // Then: _reg if there is a 2nd register
 3494 // Then: _long if it's a pair of instructions implementing a long
 3495 // Then: _fat if it requires the big decoder
 3496 //   Or: _mem if it requires the big decoder and a memory unit.
 3497 
 3498 // Integer ALU reg operation
 3499 pipe_class ialu_reg(rRegI dst)
 3500 %{
 3501     single_instruction;
 3502     dst    : S4(write);
 3503     dst    : S3(read);
 3504     DECODE : S0;        // any decoder
 3505     ALU    : S3;        // any alu
 3506 %}
 3507 
 3508 // Long ALU reg operation
 3509 pipe_class ialu_reg_long(rRegL dst)
 3510 %{
 3511     instruction_count(2);
 3512     dst    : S4(write);
 3513     dst    : S3(read);
 3514     DECODE : S0(2);     // any 2 decoders
 3515     ALU    : S3(2);     // both alus
 3516 %}
 3517 
 3518 // Integer ALU reg operation using big decoder
 3519 pipe_class ialu_reg_fat(rRegI dst)
 3520 %{
 3521     single_instruction;
 3522     dst    : S4(write);
 3523     dst    : S3(read);
 3524     D0     : S0;        // big decoder only
 3525     ALU    : S3;        // any alu
 3526 %}
 3527 
 3528 // Integer ALU reg-reg operation
 3529 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3530 %{
 3531     single_instruction;
 3532     dst    : S4(write);
 3533     src    : S3(read);
 3534     DECODE : S0;        // any decoder
 3535     ALU    : S3;        // any alu
 3536 %}
 3537 
 3538 // Integer ALU reg-reg operation
 3539 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3540 %{
 3541     single_instruction;
 3542     dst    : S4(write);
 3543     src    : S3(read);
 3544     D0     : S0;        // big decoder only
 3545     ALU    : S3;        // any alu
 3546 %}
 3547 
 3548 // Integer ALU reg-mem operation
 3549 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3550 %{
 3551     single_instruction;
 3552     dst    : S5(write);
 3553     mem    : S3(read);
 3554     D0     : S0;        // big decoder only
 3555     ALU    : S4;        // any alu
 3556     MEM    : S3;        // any mem
 3557 %}
 3558 
 3559 // Integer mem operation (prefetch)
 3560 pipe_class ialu_mem(memory mem)
 3561 %{
 3562     single_instruction;
 3563     mem    : S3(read);
 3564     D0     : S0;        // big decoder only
 3565     MEM    : S3;        // any mem
 3566 %}
 3567 
 3568 // Integer Store to Memory
 3569 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3570 %{
 3571     single_instruction;
 3572     mem    : S3(read);
 3573     src    : S5(read);
 3574     D0     : S0;        // big decoder only
 3575     ALU    : S4;        // any alu
 3576     MEM    : S3;
 3577 %}
 3578 
 3579 // // Long Store to Memory
 3580 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3581 // %{
 3582 //     instruction_count(2);
 3583 //     mem    : S3(read);
 3584 //     src    : S5(read);
 3585 //     D0     : S0(2);          // big decoder only; twice
 3586 //     ALU    : S4(2);     // any 2 alus
 3587 //     MEM    : S3(2);  // Both mems
 3588 // %}
 3589 
 3590 // Integer Store to Memory
 3591 pipe_class ialu_mem_imm(memory mem)
 3592 %{
 3593     single_instruction;
 3594     mem    : S3(read);
 3595     D0     : S0;        // big decoder only
 3596     ALU    : S4;        // any alu
 3597     MEM    : S3;
 3598 %}
 3599 
 3600 // Integer ALU0 reg-reg operation
 3601 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3602 %{
 3603     single_instruction;
 3604     dst    : S4(write);
 3605     src    : S3(read);
 3606     D0     : S0;        // Big decoder only
 3607     ALU0   : S3;        // only alu0
 3608 %}
 3609 
 3610 // Integer ALU0 reg-mem operation
 3611 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3612 %{
 3613     single_instruction;
 3614     dst    : S5(write);
 3615     mem    : S3(read);
 3616     D0     : S0;        // big decoder only
 3617     ALU0   : S4;        // ALU0 only
 3618     MEM    : S3;        // any mem
 3619 %}
 3620 
 3621 // Integer ALU reg-reg operation
 3622 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3623 %{
 3624     single_instruction;
 3625     cr     : S4(write);
 3626     src1   : S3(read);
 3627     src2   : S3(read);
 3628     DECODE : S0;        // any decoder
 3629     ALU    : S3;        // any alu
 3630 %}
 3631 
 3632 // Integer ALU reg-imm operation
 3633 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3634 %{
 3635     single_instruction;
 3636     cr     : S4(write);
 3637     src1   : S3(read);
 3638     DECODE : S0;        // any decoder
 3639     ALU    : S3;        // any alu
 3640 %}
 3641 
 3642 // Integer ALU reg-mem operation
 3643 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3644 %{
 3645     single_instruction;
 3646     cr     : S4(write);
 3647     src1   : S3(read);
 3648     src2   : S3(read);
 3649     D0     : S0;        // big decoder only
 3650     ALU    : S4;        // any alu
 3651     MEM    : S3;
 3652 %}
 3653 
 3654 // Conditional move reg-reg
 3655 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3656 %{
 3657     instruction_count(4);
 3658     y      : S4(read);
 3659     q      : S3(read);
 3660     p      : S3(read);
 3661     DECODE : S0(4);     // any decoder
 3662 %}
 3663 
 3664 // Conditional move reg-reg
 3665 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3666 %{
 3667     single_instruction;
 3668     dst    : S4(write);
 3669     src    : S3(read);
 3670     cr     : S3(read);
 3671     DECODE : S0;        // any decoder
 3672 %}
 3673 
 3674 // Conditional move reg-mem
 3675 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3676 %{
 3677     single_instruction;
 3678     dst    : S4(write);
 3679     src    : S3(read);
 3680     cr     : S3(read);
 3681     DECODE : S0;        // any decoder
 3682     MEM    : S3;
 3683 %}
 3684 
 3685 // Conditional move reg-reg long
 3686 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3687 %{
 3688     single_instruction;
 3689     dst    : S4(write);
 3690     src    : S3(read);
 3691     cr     : S3(read);
 3692     DECODE : S0(2);     // any 2 decoders
 3693 %}
 3694 
 3695 // Float reg-reg operation
 3696 pipe_class fpu_reg(regD dst)
 3697 %{
 3698     instruction_count(2);
 3699     dst    : S3(read);
 3700     DECODE : S0(2);     // any 2 decoders
 3701     FPU    : S3;
 3702 %}
 3703 
 3704 // Float reg-reg operation
 3705 pipe_class fpu_reg_reg(regD dst, regD src)
 3706 %{
 3707     instruction_count(2);
 3708     dst    : S4(write);
 3709     src    : S3(read);
 3710     DECODE : S0(2);     // any 2 decoders
 3711     FPU    : S3;
 3712 %}
 3713 
 3714 // Float reg-reg operation
 3715 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3716 %{
 3717     instruction_count(3);
 3718     dst    : S4(write);
 3719     src1   : S3(read);
 3720     src2   : S3(read);
 3721     DECODE : S0(3);     // any 3 decoders
 3722     FPU    : S3(2);
 3723 %}
 3724 
 3725 // Float reg-reg operation
 3726 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3727 %{
 3728     instruction_count(4);
 3729     dst    : S4(write);
 3730     src1   : S3(read);
 3731     src2   : S3(read);
 3732     src3   : S3(read);
 3733     DECODE : S0(4);     // any 3 decoders
 3734     FPU    : S3(2);
 3735 %}
 3736 
 3737 // Float reg-reg operation
 3738 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3739 %{
 3740     instruction_count(4);
 3741     dst    : S4(write);
 3742     src1   : S3(read);
 3743     src2   : S3(read);
 3744     src3   : S3(read);
 3745     DECODE : S1(3);     // any 3 decoders
 3746     D0     : S0;        // Big decoder only
 3747     FPU    : S3(2);
 3748     MEM    : S3;
 3749 %}
 3750 
 3751 // Float reg-mem operation
 3752 pipe_class fpu_reg_mem(regD dst, memory mem)
 3753 %{
 3754     instruction_count(2);
 3755     dst    : S5(write);
 3756     mem    : S3(read);
 3757     D0     : S0;        // big decoder only
 3758     DECODE : S1;        // any decoder for FPU POP
 3759     FPU    : S4;
 3760     MEM    : S3;        // any mem
 3761 %}
 3762 
 3763 // Float reg-mem operation
 3764 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3765 %{
 3766     instruction_count(3);
 3767     dst    : S5(write);
 3768     src1   : S3(read);
 3769     mem    : S3(read);
 3770     D0     : S0;        // big decoder only
 3771     DECODE : S1(2);     // any decoder for FPU POP
 3772     FPU    : S4;
 3773     MEM    : S3;        // any mem
 3774 %}
 3775 
 3776 // Float mem-reg operation
 3777 pipe_class fpu_mem_reg(memory mem, regD src)
 3778 %{
 3779     instruction_count(2);
 3780     src    : S5(read);
 3781     mem    : S3(read);
 3782     DECODE : S0;        // any decoder for FPU PUSH
 3783     D0     : S1;        // big decoder only
 3784     FPU    : S4;
 3785     MEM    : S3;        // any mem
 3786 %}
 3787 
 3788 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3789 %{
 3790     instruction_count(3);
 3791     src1   : S3(read);
 3792     src2   : S3(read);
 3793     mem    : S3(read);
 3794     DECODE : S0(2);     // any decoder for FPU PUSH
 3795     D0     : S1;        // big decoder only
 3796     FPU    : S4;
 3797     MEM    : S3;        // any mem
 3798 %}
 3799 
 3800 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3801 %{
 3802     instruction_count(3);
 3803     src1   : S3(read);
 3804     src2   : S3(read);
 3805     mem    : S4(read);
 3806     DECODE : S0;        // any decoder for FPU PUSH
 3807     D0     : S0(2);     // big decoder only
 3808     FPU    : S4;
 3809     MEM    : S3(2);     // any mem
 3810 %}
 3811 
 3812 pipe_class fpu_mem_mem(memory dst, memory src1)
 3813 %{
 3814     instruction_count(2);
 3815     src1   : S3(read);
 3816     dst    : S4(read);
 3817     D0     : S0(2);     // big decoder only
 3818     MEM    : S3(2);     // any mem
 3819 %}
 3820 
 3821 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3822 %{
 3823     instruction_count(3);
 3824     src1   : S3(read);
 3825     src2   : S3(read);
 3826     dst    : S4(read);
 3827     D0     : S0(3);     // big decoder only
 3828     FPU    : S4;
 3829     MEM    : S3(3);     // any mem
 3830 %}
 3831 
 3832 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3833 %{
 3834     instruction_count(3);
 3835     src1   : S4(read);
 3836     mem    : S4(read);
 3837     DECODE : S0;        // any decoder for FPU PUSH
 3838     D0     : S0(2);     // big decoder only
 3839     FPU    : S4;
 3840     MEM    : S3(2);     // any mem
 3841 %}
 3842 
 3843 // Float load constant
 3844 pipe_class fpu_reg_con(regD dst)
 3845 %{
 3846     instruction_count(2);
 3847     dst    : S5(write);
 3848     D0     : S0;        // big decoder only for the load
 3849     DECODE : S1;        // any decoder for FPU POP
 3850     FPU    : S4;
 3851     MEM    : S3;        // any mem
 3852 %}
 3853 
 3854 // Float load constant
 3855 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3856 %{
 3857     instruction_count(3);
 3858     dst    : S5(write);
 3859     src    : S3(read);
 3860     D0     : S0;        // big decoder only for the load
 3861     DECODE : S1(2);     // any decoder for FPU POP
 3862     FPU    : S4;
 3863     MEM    : S3;        // any mem
 3864 %}
 3865 
 3866 // UnConditional branch
 3867 pipe_class pipe_jmp(label labl)
 3868 %{
 3869     single_instruction;
 3870     BR   : S3;
 3871 %}
 3872 
 3873 // Conditional branch
 3874 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3875 %{
 3876     single_instruction;
 3877     cr    : S1(read);
 3878     BR    : S3;
 3879 %}
 3880 
 3881 // Allocation idiom
 3882 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3883 %{
 3884     instruction_count(1); force_serialization;
 3885     fixed_latency(6);
 3886     heap_ptr : S3(read);
 3887     DECODE   : S0(3);
 3888     D0       : S2;
 3889     MEM      : S3;
 3890     ALU      : S3(2);
 3891     dst      : S5(write);
 3892     BR       : S5;
 3893 %}
 3894 
 3895 // Generic big/slow expanded idiom
 3896 pipe_class pipe_slow()
 3897 %{
 3898     instruction_count(10); multiple_bundles; force_serialization;
 3899     fixed_latency(100);
 3900     D0  : S0(2);
 3901     MEM : S3(2);
 3902 %}
 3903 
 3904 // The real do-nothing guy
 3905 pipe_class empty()
 3906 %{
 3907     instruction_count(0);
 3908 %}
 3909 
 3910 // Define the class for the Nop node
 3911 define
 3912 %{
 3913    MachNop = empty;
 3914 %}
 3915 
 3916 %}
 3917 
 3918 //----------INSTRUCTIONS-------------------------------------------------------
 3919 //
 3920 // match      -- States which machine-independent subtree may be replaced
 3921 //               by this instruction.
 3922 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3923 //               selection to identify a minimum cost tree of machine
 3924 //               instructions that matches a tree of machine-independent
 3925 //               instructions.
 3926 // format     -- A string providing the disassembly for this instruction.
 3927 //               The value of an instruction's operand may be inserted
 3928 //               by referring to it with a '$' prefix.
 3929 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3930 //               to within an encode class as $primary, $secondary, and $tertiary
 3931 //               rrspectively.  The primary opcode is commonly used to
 3932 //               indicate the type of machine instruction, while secondary
 3933 //               and tertiary are often used for prefix options or addressing
 3934 //               modes.
 3935 // ins_encode -- A list of encode classes with parameters. The encode class
 3936 //               name must have been defined in an 'enc_class' specification
 3937 //               in the encode section of the architecture description.
 3938 
 3939 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3940 // Load Float
 3941 instruct MoveF2VL(vlRegF dst, regF src) %{
 3942   match(Set dst src);
 3943   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3944   ins_encode %{
 3945     ShouldNotReachHere();
 3946   %}
 3947   ins_pipe( fpu_reg_reg );
 3948 %}
 3949 
 3950 // Load Float
 3951 instruct MoveF2LEG(legRegF dst, regF src) %{
 3952   match(Set dst src);
 3953   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3954   ins_encode %{
 3955     ShouldNotReachHere();
 3956   %}
 3957   ins_pipe( fpu_reg_reg );
 3958 %}
 3959 
 3960 // Load Float
 3961 instruct MoveVL2F(regF dst, vlRegF src) %{
 3962   match(Set dst src);
 3963   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3964   ins_encode %{
 3965     ShouldNotReachHere();
 3966   %}
 3967   ins_pipe( fpu_reg_reg );
 3968 %}
 3969 
 3970 // Load Float
 3971 instruct MoveLEG2F(regF dst, legRegF src) %{
 3972   match(Set dst src);
 3973   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3974   ins_encode %{
 3975     ShouldNotReachHere();
 3976   %}
 3977   ins_pipe( fpu_reg_reg );
 3978 %}
 3979 
 3980 // Load Double
 3981 instruct MoveD2VL(vlRegD dst, regD src) %{
 3982   match(Set dst src);
 3983   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3984   ins_encode %{
 3985     ShouldNotReachHere();
 3986   %}
 3987   ins_pipe( fpu_reg_reg );
 3988 %}
 3989 
 3990 // Load Double
 3991 instruct MoveD2LEG(legRegD dst, regD src) %{
 3992   match(Set dst src);
 3993   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3994   ins_encode %{
 3995     ShouldNotReachHere();
 3996   %}
 3997   ins_pipe( fpu_reg_reg );
 3998 %}
 3999 
 4000 // Load Double
 4001 instruct MoveVL2D(regD dst, vlRegD src) %{
 4002   match(Set dst src);
 4003   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4004   ins_encode %{
 4005     ShouldNotReachHere();
 4006   %}
 4007   ins_pipe( fpu_reg_reg );
 4008 %}
 4009 
 4010 // Load Double
 4011 instruct MoveLEG2D(regD dst, legRegD src) %{
 4012   match(Set dst src);
 4013   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4014   ins_encode %{
 4015     ShouldNotReachHere();
 4016   %}
 4017   ins_pipe( fpu_reg_reg );
 4018 %}
 4019 
 4020 //----------Load/Store/Move Instructions---------------------------------------
 4021 //----------Load Instructions--------------------------------------------------
 4022 
 4023 // Load Byte (8 bit signed)
 4024 instruct loadB(rRegI dst, memory mem)
 4025 %{
 4026   match(Set dst (LoadB mem));
 4027 
 4028   ins_cost(125);
 4029   format %{ "movsbl  $dst, $mem\t# byte" %}
 4030 
 4031   ins_encode %{
 4032     __ movsbl($dst$$Register, $mem$$Address);
 4033   %}
 4034 
 4035   ins_pipe(ialu_reg_mem);
 4036 %}
 4037 
 4038 // Load Byte (8 bit signed) into Long Register
 4039 instruct loadB2L(rRegL dst, memory mem)
 4040 %{
 4041   match(Set dst (ConvI2L (LoadB mem)));
 4042 
 4043   ins_cost(125);
 4044   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4045 
 4046   ins_encode %{
 4047     __ movsbq($dst$$Register, $mem$$Address);
 4048   %}
 4049 
 4050   ins_pipe(ialu_reg_mem);
 4051 %}
 4052 
 4053 // Load Unsigned Byte (8 bit UNsigned)
 4054 instruct loadUB(rRegI dst, memory mem)
 4055 %{
 4056   match(Set dst (LoadUB mem));
 4057 
 4058   ins_cost(125);
 4059   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4060 
 4061   ins_encode %{
 4062     __ movzbl($dst$$Register, $mem$$Address);
 4063   %}
 4064 
 4065   ins_pipe(ialu_reg_mem);
 4066 %}
 4067 
 4068 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4069 instruct loadUB2L(rRegL dst, memory mem)
 4070 %{
 4071   match(Set dst (ConvI2L (LoadUB mem)));
 4072 
 4073   ins_cost(125);
 4074   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4075 
 4076   ins_encode %{
 4077     __ movzbq($dst$$Register, $mem$$Address);
 4078   %}
 4079 
 4080   ins_pipe(ialu_reg_mem);
 4081 %}
 4082 
 4083 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4084 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4085   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4086   effect(KILL cr);
 4087 
 4088   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4089             "andl    $dst, right_n_bits($mask, 8)" %}
 4090   ins_encode %{
 4091     Register Rdst = $dst$$Register;
 4092     __ movzbq(Rdst, $mem$$Address);
 4093     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4094   %}
 4095   ins_pipe(ialu_reg_mem);
 4096 %}
 4097 
 4098 // Load Short (16 bit signed)
 4099 instruct loadS(rRegI dst, memory mem)
 4100 %{
 4101   match(Set dst (LoadS mem));
 4102 
 4103   ins_cost(125);
 4104   format %{ "movswl $dst, $mem\t# short" %}
 4105 
 4106   ins_encode %{
 4107     __ movswl($dst$$Register, $mem$$Address);
 4108   %}
 4109 
 4110   ins_pipe(ialu_reg_mem);
 4111 %}
 4112 
 4113 // Load Short (16 bit signed) to Byte (8 bit signed)
 4114 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4115   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4116 
 4117   ins_cost(125);
 4118   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4119   ins_encode %{
 4120     __ movsbl($dst$$Register, $mem$$Address);
 4121   %}
 4122   ins_pipe(ialu_reg_mem);
 4123 %}
 4124 
 4125 // Load Short (16 bit signed) into Long Register
 4126 instruct loadS2L(rRegL dst, memory mem)
 4127 %{
 4128   match(Set dst (ConvI2L (LoadS mem)));
 4129 
 4130   ins_cost(125);
 4131   format %{ "movswq $dst, $mem\t# short -> long" %}
 4132 
 4133   ins_encode %{
 4134     __ movswq($dst$$Register, $mem$$Address);
 4135   %}
 4136 
 4137   ins_pipe(ialu_reg_mem);
 4138 %}
 4139 
 4140 // Load Unsigned Short/Char (16 bit UNsigned)
 4141 instruct loadUS(rRegI dst, memory mem)
 4142 %{
 4143   match(Set dst (LoadUS mem));
 4144 
 4145   ins_cost(125);
 4146   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4147 
 4148   ins_encode %{
 4149     __ movzwl($dst$$Register, $mem$$Address);
 4150   %}
 4151 
 4152   ins_pipe(ialu_reg_mem);
 4153 %}
 4154 
 4155 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4156 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4157   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4158 
 4159   ins_cost(125);
 4160   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4161   ins_encode %{
 4162     __ movsbl($dst$$Register, $mem$$Address);
 4163   %}
 4164   ins_pipe(ialu_reg_mem);
 4165 %}
 4166 
 4167 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4168 instruct loadUS2L(rRegL dst, memory mem)
 4169 %{
 4170   match(Set dst (ConvI2L (LoadUS mem)));
 4171 
 4172   ins_cost(125);
 4173   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4174 
 4175   ins_encode %{
 4176     __ movzwq($dst$$Register, $mem$$Address);
 4177   %}
 4178 
 4179   ins_pipe(ialu_reg_mem);
 4180 %}
 4181 
 4182 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4183 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4184   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4185 
 4186   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4187   ins_encode %{
 4188     __ movzbq($dst$$Register, $mem$$Address);
 4189   %}
 4190   ins_pipe(ialu_reg_mem);
 4191 %}
 4192 
 4193 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4194 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4195   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4196   effect(KILL cr);
 4197 
 4198   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4199             "andl    $dst, right_n_bits($mask, 16)" %}
 4200   ins_encode %{
 4201     Register Rdst = $dst$$Register;
 4202     __ movzwq(Rdst, $mem$$Address);
 4203     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4204   %}
 4205   ins_pipe(ialu_reg_mem);
 4206 %}
 4207 
 4208 // Load Integer
 4209 instruct loadI(rRegI dst, memory mem)
 4210 %{
 4211   match(Set dst (LoadI mem));
 4212 
 4213   ins_cost(125);
 4214   format %{ "movl    $dst, $mem\t# int" %}
 4215 
 4216   ins_encode %{
 4217     __ movl($dst$$Register, $mem$$Address);
 4218   %}
 4219 
 4220   ins_pipe(ialu_reg_mem);
 4221 %}
 4222 
 4223 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4224 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4225   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4226 
 4227   ins_cost(125);
 4228   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4229   ins_encode %{
 4230     __ movsbl($dst$$Register, $mem$$Address);
 4231   %}
 4232   ins_pipe(ialu_reg_mem);
 4233 %}
 4234 
 4235 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4236 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4237   match(Set dst (AndI (LoadI mem) mask));
 4238 
 4239   ins_cost(125);
 4240   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4241   ins_encode %{
 4242     __ movzbl($dst$$Register, $mem$$Address);
 4243   %}
 4244   ins_pipe(ialu_reg_mem);
 4245 %}
 4246 
 4247 // Load Integer (32 bit signed) to Short (16 bit signed)
 4248 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4249   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4250 
 4251   ins_cost(125);
 4252   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4253   ins_encode %{
 4254     __ movswl($dst$$Register, $mem$$Address);
 4255   %}
 4256   ins_pipe(ialu_reg_mem);
 4257 %}
 4258 
 4259 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4260 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4261   match(Set dst (AndI (LoadI mem) mask));
 4262 
 4263   ins_cost(125);
 4264   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4265   ins_encode %{
 4266     __ movzwl($dst$$Register, $mem$$Address);
 4267   %}
 4268   ins_pipe(ialu_reg_mem);
 4269 %}
 4270 
 4271 // Load Integer into Long Register
 4272 instruct loadI2L(rRegL dst, memory mem)
 4273 %{
 4274   match(Set dst (ConvI2L (LoadI mem)));
 4275 
 4276   ins_cost(125);
 4277   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4278 
 4279   ins_encode %{
 4280     __ movslq($dst$$Register, $mem$$Address);
 4281   %}
 4282 
 4283   ins_pipe(ialu_reg_mem);
 4284 %}
 4285 
 4286 // Load Integer with mask 0xFF into Long Register
 4287 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4288   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4289 
 4290   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4291   ins_encode %{
 4292     __ movzbq($dst$$Register, $mem$$Address);
 4293   %}
 4294   ins_pipe(ialu_reg_mem);
 4295 %}
 4296 
 4297 // Load Integer with mask 0xFFFF into Long Register
 4298 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4299   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4300 
 4301   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4302   ins_encode %{
 4303     __ movzwq($dst$$Register, $mem$$Address);
 4304   %}
 4305   ins_pipe(ialu_reg_mem);
 4306 %}
 4307 
 4308 // Load Integer with a 31-bit mask into Long Register
 4309 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4310   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4311   effect(KILL cr);
 4312 
 4313   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4314             "andl    $dst, $mask" %}
 4315   ins_encode %{
 4316     Register Rdst = $dst$$Register;
 4317     __ movl(Rdst, $mem$$Address);
 4318     __ andl(Rdst, $mask$$constant);
 4319   %}
 4320   ins_pipe(ialu_reg_mem);
 4321 %}
 4322 
 4323 // Load Unsigned Integer into Long Register
 4324 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4325 %{
 4326   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4327 
 4328   ins_cost(125);
 4329   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4330 
 4331   ins_encode %{
 4332     __ movl($dst$$Register, $mem$$Address);
 4333   %}
 4334 
 4335   ins_pipe(ialu_reg_mem);
 4336 %}
 4337 
 4338 // Load Long
 4339 instruct loadL(rRegL dst, memory mem)
 4340 %{
 4341   match(Set dst (LoadL mem));
 4342 
 4343   ins_cost(125);
 4344   format %{ "movq    $dst, $mem\t# long" %}
 4345 
 4346   ins_encode %{
 4347     __ movq($dst$$Register, $mem$$Address);
 4348   %}
 4349 
 4350   ins_pipe(ialu_reg_mem); // XXX
 4351 %}
 4352 
 4353 // Load Range
 4354 instruct loadRange(rRegI dst, memory mem)
 4355 %{
 4356   match(Set dst (LoadRange mem));
 4357 
 4358   ins_cost(125); // XXX
 4359   format %{ "movl    $dst, $mem\t# range" %}
 4360   ins_encode %{
 4361     __ movl($dst$$Register, $mem$$Address);
 4362   %}
 4363   ins_pipe(ialu_reg_mem);
 4364 %}
 4365 
 4366 // Load Pointer
 4367 instruct loadP(rRegP dst, memory mem)
 4368 %{
 4369   match(Set dst (LoadP mem));
 4370   predicate(n->as_Load()->barrier_data() == 0);
 4371 
 4372   ins_cost(125); // XXX
 4373   format %{ "movq    $dst, $mem\t# ptr" %}
 4374   ins_encode %{
 4375     __ movq($dst$$Register, $mem$$Address);
 4376   %}
 4377   ins_pipe(ialu_reg_mem); // XXX
 4378 %}
 4379 
 4380 // Load Compressed Pointer
 4381 instruct loadN(rRegN dst, memory mem)
 4382 %{
 4383    match(Set dst (LoadN mem));
 4384 
 4385    ins_cost(125); // XXX
 4386    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4387    ins_encode %{
 4388      __ movl($dst$$Register, $mem$$Address);
 4389    %}
 4390    ins_pipe(ialu_reg_mem); // XXX
 4391 %}
 4392 
 4393 
 4394 // Load Klass Pointer
 4395 instruct loadKlass(rRegP dst, memory mem)
 4396 %{
 4397   match(Set dst (LoadKlass mem));
 4398 
 4399   ins_cost(125); // XXX
 4400   format %{ "movq    $dst, $mem\t# class" %}
 4401   ins_encode %{
 4402     __ movq($dst$$Register, $mem$$Address);
 4403   %}
 4404   ins_pipe(ialu_reg_mem); // XXX
 4405 %}
 4406 
 4407 // Load narrow Klass Pointer
 4408 instruct loadNKlass(rRegN dst, memory mem)
 4409 %{
 4410   predicate(!UseCompactObjectHeaders);
 4411   match(Set dst (LoadNKlass mem));
 4412 
 4413   ins_cost(125); // XXX
 4414   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4415   ins_encode %{
 4416     __ movl($dst$$Register, $mem$$Address);
 4417   %}
 4418   ins_pipe(ialu_reg_mem); // XXX
 4419 %}
 4420 
 4421 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4422 %{
 4423   predicate(UseCompactObjectHeaders);
 4424   match(Set dst (LoadNKlass mem));
 4425   effect(KILL cr);
 4426   ins_cost(125); // XXX
 4427   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4428   ins_encode %{
 4429     Register index = $mem$$index != 4 ? $mem$$index$$Register : noreg;
 4430     Address::ScaleFactor sf = (index != noreg) ? static_cast<Address::ScaleFactor>($mem$$scale) : Address::no_scale;
 4431     __ load_nklass_compact_c2($dst$$Register, $mem$$base$$Register, index, sf, $mem$$disp);
 4432   %}
 4433   ins_pipe(pipe_slow); // XXX
 4434 %}
 4435 
 4436 // Load Float
 4437 instruct loadF(regF dst, memory mem)
 4438 %{
 4439   match(Set dst (LoadF mem));
 4440 
 4441   ins_cost(145); // XXX
 4442   format %{ "movss   $dst, $mem\t# float" %}
 4443   ins_encode %{
 4444     __ movflt($dst$$XMMRegister, $mem$$Address);
 4445   %}
 4446   ins_pipe(pipe_slow); // XXX
 4447 %}
 4448 
 4449 // Load Double
 4450 instruct loadD_partial(regD dst, memory mem)
 4451 %{
 4452   predicate(!UseXmmLoadAndClearUpper);
 4453   match(Set dst (LoadD mem));
 4454 
 4455   ins_cost(145); // XXX
 4456   format %{ "movlpd  $dst, $mem\t# double" %}
 4457   ins_encode %{
 4458     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4459   %}
 4460   ins_pipe(pipe_slow); // XXX
 4461 %}
 4462 
 4463 instruct loadD(regD dst, memory mem)
 4464 %{
 4465   predicate(UseXmmLoadAndClearUpper);
 4466   match(Set dst (LoadD mem));
 4467 
 4468   ins_cost(145); // XXX
 4469   format %{ "movsd   $dst, $mem\t# double" %}
 4470   ins_encode %{
 4471     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4472   %}
 4473   ins_pipe(pipe_slow); // XXX
 4474 %}
 4475 
 4476 // max = java.lang.Math.max(float a, float b)
 4477 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4478   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4479   match(Set dst (MaxF a b));
 4480   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4481   format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4482   ins_encode %{
 4483     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4484   %}
 4485   ins_pipe( pipe_slow );
 4486 %}
 4487 
 4488 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4489   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4490   match(Set dst (MaxF a b));
 4491   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4492 
 4493   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 4494   ins_encode %{
 4495     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4496                     false /*min*/, true /*single*/);
 4497   %}
 4498   ins_pipe( pipe_slow );
 4499 %}
 4500 
 4501 // max = java.lang.Math.max(double a, double b)
 4502 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4503   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4504   match(Set dst (MaxD a b));
 4505   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4506   format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4507   ins_encode %{
 4508     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4509   %}
 4510   ins_pipe( pipe_slow );
 4511 %}
 4512 
 4513 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4514   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4515   match(Set dst (MaxD a b));
 4516   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4517 
 4518   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 4519   ins_encode %{
 4520     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4521                     false /*min*/, false /*single*/);
 4522   %}
 4523   ins_pipe( pipe_slow );
 4524 %}
 4525 
 4526 // min = java.lang.Math.min(float a, float b)
 4527 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4528   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4529   match(Set dst (MinF a b));
 4530   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4531   format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4532   ins_encode %{
 4533     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4534   %}
 4535   ins_pipe( pipe_slow );
 4536 %}
 4537 
 4538 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4539   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4540   match(Set dst (MinF a b));
 4541   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4542 
 4543   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 4544   ins_encode %{
 4545     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4546                     true /*min*/, true /*single*/);
 4547   %}
 4548   ins_pipe( pipe_slow );
 4549 %}
 4550 
 4551 // min = java.lang.Math.min(double a, double b)
 4552 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4553   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4554   match(Set dst (MinD a b));
 4555   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4556     format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4557   ins_encode %{
 4558     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4559   %}
 4560   ins_pipe( pipe_slow );
 4561 %}
 4562 
 4563 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4564   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4565   match(Set dst (MinD a b));
 4566   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4567 
 4568   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 4569   ins_encode %{
 4570     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4571                     true /*min*/, false /*single*/);
 4572   %}
 4573   ins_pipe( pipe_slow );
 4574 %}
 4575 
 4576 // Load Effective Address
 4577 instruct leaP8(rRegP dst, indOffset8 mem)
 4578 %{
 4579   match(Set dst mem);
 4580 
 4581   ins_cost(110); // XXX
 4582   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4583   ins_encode %{
 4584     __ leaq($dst$$Register, $mem$$Address);
 4585   %}
 4586   ins_pipe(ialu_reg_reg_fat);
 4587 %}
 4588 
 4589 instruct leaP32(rRegP dst, indOffset32 mem)
 4590 %{
 4591   match(Set dst mem);
 4592 
 4593   ins_cost(110);
 4594   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4595   ins_encode %{
 4596     __ leaq($dst$$Register, $mem$$Address);
 4597   %}
 4598   ins_pipe(ialu_reg_reg_fat);
 4599 %}
 4600 
 4601 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4602 %{
 4603   match(Set dst mem);
 4604 
 4605   ins_cost(110);
 4606   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4607   ins_encode %{
 4608     __ leaq($dst$$Register, $mem$$Address);
 4609   %}
 4610   ins_pipe(ialu_reg_reg_fat);
 4611 %}
 4612 
 4613 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4614 %{
 4615   match(Set dst mem);
 4616 
 4617   ins_cost(110);
 4618   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4619   ins_encode %{
 4620     __ leaq($dst$$Register, $mem$$Address);
 4621   %}
 4622   ins_pipe(ialu_reg_reg_fat);
 4623 %}
 4624 
 4625 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4626 %{
 4627   match(Set dst mem);
 4628 
 4629   ins_cost(110);
 4630   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4631   ins_encode %{
 4632     __ leaq($dst$$Register, $mem$$Address);
 4633   %}
 4634   ins_pipe(ialu_reg_reg_fat);
 4635 %}
 4636 
 4637 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4638 %{
 4639   match(Set dst mem);
 4640 
 4641   ins_cost(110);
 4642   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4643   ins_encode %{
 4644     __ leaq($dst$$Register, $mem$$Address);
 4645   %}
 4646   ins_pipe(ialu_reg_reg_fat);
 4647 %}
 4648 
 4649 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4650 %{
 4651   match(Set dst mem);
 4652 
 4653   ins_cost(110);
 4654   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4655   ins_encode %{
 4656     __ leaq($dst$$Register, $mem$$Address);
 4657   %}
 4658   ins_pipe(ialu_reg_reg_fat);
 4659 %}
 4660 
 4661 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4662 %{
 4663   match(Set dst mem);
 4664 
 4665   ins_cost(110);
 4666   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4667   ins_encode %{
 4668     __ leaq($dst$$Register, $mem$$Address);
 4669   %}
 4670   ins_pipe(ialu_reg_reg_fat);
 4671 %}
 4672 
 4673 // Load Effective Address which uses Narrow (32-bits) oop
 4674 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4675 %{
 4676   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4677   match(Set dst mem);
 4678 
 4679   ins_cost(110);
 4680   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4681   ins_encode %{
 4682     __ leaq($dst$$Register, $mem$$Address);
 4683   %}
 4684   ins_pipe(ialu_reg_reg_fat);
 4685 %}
 4686 
 4687 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4688 %{
 4689   predicate(CompressedOops::shift() == 0);
 4690   match(Set dst mem);
 4691 
 4692   ins_cost(110); // XXX
 4693   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4694   ins_encode %{
 4695     __ leaq($dst$$Register, $mem$$Address);
 4696   %}
 4697   ins_pipe(ialu_reg_reg_fat);
 4698 %}
 4699 
 4700 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4701 %{
 4702   predicate(CompressedOops::shift() == 0);
 4703   match(Set dst mem);
 4704 
 4705   ins_cost(110);
 4706   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4707   ins_encode %{
 4708     __ leaq($dst$$Register, $mem$$Address);
 4709   %}
 4710   ins_pipe(ialu_reg_reg_fat);
 4711 %}
 4712 
 4713 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4714 %{
 4715   predicate(CompressedOops::shift() == 0);
 4716   match(Set dst mem);
 4717 
 4718   ins_cost(110);
 4719   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4720   ins_encode %{
 4721     __ leaq($dst$$Register, $mem$$Address);
 4722   %}
 4723   ins_pipe(ialu_reg_reg_fat);
 4724 %}
 4725 
 4726 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4727 %{
 4728   predicate(CompressedOops::shift() == 0);
 4729   match(Set dst mem);
 4730 
 4731   ins_cost(110);
 4732   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4733   ins_encode %{
 4734     __ leaq($dst$$Register, $mem$$Address);
 4735   %}
 4736   ins_pipe(ialu_reg_reg_fat);
 4737 %}
 4738 
 4739 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4740 %{
 4741   predicate(CompressedOops::shift() == 0);
 4742   match(Set dst mem);
 4743 
 4744   ins_cost(110);
 4745   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4746   ins_encode %{
 4747     __ leaq($dst$$Register, $mem$$Address);
 4748   %}
 4749   ins_pipe(ialu_reg_reg_fat);
 4750 %}
 4751 
 4752 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4753 %{
 4754   predicate(CompressedOops::shift() == 0);
 4755   match(Set dst mem);
 4756 
 4757   ins_cost(110);
 4758   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4759   ins_encode %{
 4760     __ leaq($dst$$Register, $mem$$Address);
 4761   %}
 4762   ins_pipe(ialu_reg_reg_fat);
 4763 %}
 4764 
 4765 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4766 %{
 4767   predicate(CompressedOops::shift() == 0);
 4768   match(Set dst mem);
 4769 
 4770   ins_cost(110);
 4771   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4772   ins_encode %{
 4773     __ leaq($dst$$Register, $mem$$Address);
 4774   %}
 4775   ins_pipe(ialu_reg_reg_fat);
 4776 %}
 4777 
 4778 instruct loadConI(rRegI dst, immI src)
 4779 %{
 4780   match(Set dst src);
 4781 
 4782   format %{ "movl    $dst, $src\t# int" %}
 4783   ins_encode %{
 4784     __ movl($dst$$Register, $src$$constant);
 4785   %}
 4786   ins_pipe(ialu_reg_fat); // XXX
 4787 %}
 4788 
 4789 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4790 %{
 4791   match(Set dst src);
 4792   effect(KILL cr);
 4793 
 4794   ins_cost(50);
 4795   format %{ "xorl    $dst, $dst\t# int" %}
 4796   ins_encode %{
 4797     __ xorl($dst$$Register, $dst$$Register);
 4798   %}
 4799   ins_pipe(ialu_reg);
 4800 %}
 4801 
 4802 instruct loadConL(rRegL dst, immL src)
 4803 %{
 4804   match(Set dst src);
 4805 
 4806   ins_cost(150);
 4807   format %{ "movq    $dst, $src\t# long" %}
 4808   ins_encode %{
 4809     __ mov64($dst$$Register, $src$$constant);
 4810   %}
 4811   ins_pipe(ialu_reg);
 4812 %}
 4813 
 4814 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4815 %{
 4816   match(Set dst src);
 4817   effect(KILL cr);
 4818 
 4819   ins_cost(50);
 4820   format %{ "xorl    $dst, $dst\t# long" %}
 4821   ins_encode %{
 4822     __ xorl($dst$$Register, $dst$$Register);
 4823   %}
 4824   ins_pipe(ialu_reg); // XXX
 4825 %}
 4826 
 4827 instruct loadConUL32(rRegL dst, immUL32 src)
 4828 %{
 4829   match(Set dst src);
 4830 
 4831   ins_cost(60);
 4832   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4833   ins_encode %{
 4834     __ movl($dst$$Register, $src$$constant);
 4835   %}
 4836   ins_pipe(ialu_reg);
 4837 %}
 4838 
 4839 instruct loadConL32(rRegL dst, immL32 src)
 4840 %{
 4841   match(Set dst src);
 4842 
 4843   ins_cost(70);
 4844   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4845   ins_encode %{
 4846     __ movq($dst$$Register, $src$$constant);
 4847   %}
 4848   ins_pipe(ialu_reg);
 4849 %}
 4850 
 4851 instruct loadConP(rRegP dst, immP con) %{
 4852   match(Set dst con);
 4853 
 4854   format %{ "movq    $dst, $con\t# ptr" %}
 4855   ins_encode %{
 4856     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4857   %}
 4858   ins_pipe(ialu_reg_fat); // XXX
 4859 %}
 4860 
 4861 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4862 %{
 4863   match(Set dst src);
 4864   effect(KILL cr);
 4865 
 4866   ins_cost(50);
 4867   format %{ "xorl    $dst, $dst\t# ptr" %}
 4868   ins_encode %{
 4869     __ xorl($dst$$Register, $dst$$Register);
 4870   %}
 4871   ins_pipe(ialu_reg);
 4872 %}
 4873 
 4874 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4875 %{
 4876   match(Set dst src);
 4877   effect(KILL cr);
 4878 
 4879   ins_cost(60);
 4880   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4881   ins_encode %{
 4882     __ movl($dst$$Register, $src$$constant);
 4883   %}
 4884   ins_pipe(ialu_reg);
 4885 %}
 4886 
 4887 instruct loadConF(regF dst, immF con) %{
 4888   match(Set dst con);
 4889   ins_cost(125);
 4890   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4891   ins_encode %{
 4892     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4893   %}
 4894   ins_pipe(pipe_slow);
 4895 %}
 4896 
 4897 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4898   match(Set dst src);
 4899   effect(KILL cr);
 4900   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4901   ins_encode %{
 4902     __ xorq($dst$$Register, $dst$$Register);
 4903   %}
 4904   ins_pipe(ialu_reg);
 4905 %}
 4906 
 4907 instruct loadConN(rRegN dst, immN src) %{
 4908   match(Set dst src);
 4909 
 4910   ins_cost(125);
 4911   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4912   ins_encode %{
 4913     address con = (address)$src$$constant;
 4914     if (con == nullptr) {
 4915       ShouldNotReachHere();
 4916     } else {
 4917       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4918     }
 4919   %}
 4920   ins_pipe(ialu_reg_fat); // XXX
 4921 %}
 4922 
 4923 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4924   match(Set dst src);
 4925 
 4926   ins_cost(125);
 4927   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4928   ins_encode %{
 4929     address con = (address)$src$$constant;
 4930     if (con == nullptr) {
 4931       ShouldNotReachHere();
 4932     } else {
 4933       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4934     }
 4935   %}
 4936   ins_pipe(ialu_reg_fat); // XXX
 4937 %}
 4938 
 4939 instruct loadConF0(regF dst, immF0 src)
 4940 %{
 4941   match(Set dst src);
 4942   ins_cost(100);
 4943 
 4944   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4945   ins_encode %{
 4946     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4947   %}
 4948   ins_pipe(pipe_slow);
 4949 %}
 4950 
 4951 // Use the same format since predicate() can not be used here.
 4952 instruct loadConD(regD dst, immD con) %{
 4953   match(Set dst con);
 4954   ins_cost(125);
 4955   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4956   ins_encode %{
 4957     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4958   %}
 4959   ins_pipe(pipe_slow);
 4960 %}
 4961 
 4962 instruct loadConD0(regD dst, immD0 src)
 4963 %{
 4964   match(Set dst src);
 4965   ins_cost(100);
 4966 
 4967   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4968   ins_encode %{
 4969     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4970   %}
 4971   ins_pipe(pipe_slow);
 4972 %}
 4973 
 4974 instruct loadSSI(rRegI dst, stackSlotI src)
 4975 %{
 4976   match(Set dst src);
 4977 
 4978   ins_cost(125);
 4979   format %{ "movl    $dst, $src\t# int stk" %}
 4980   ins_encode %{
 4981     __ movl($dst$$Register, $src$$Address);
 4982   %}
 4983   ins_pipe(ialu_reg_mem);
 4984 %}
 4985 
 4986 instruct loadSSL(rRegL dst, stackSlotL src)
 4987 %{
 4988   match(Set dst src);
 4989 
 4990   ins_cost(125);
 4991   format %{ "movq    $dst, $src\t# long stk" %}
 4992   ins_encode %{
 4993     __ movq($dst$$Register, $src$$Address);
 4994   %}
 4995   ins_pipe(ialu_reg_mem);
 4996 %}
 4997 
 4998 instruct loadSSP(rRegP dst, stackSlotP src)
 4999 %{
 5000   match(Set dst src);
 5001 
 5002   ins_cost(125);
 5003   format %{ "movq    $dst, $src\t# ptr stk" %}
 5004   ins_encode %{
 5005     __ movq($dst$$Register, $src$$Address);
 5006   %}
 5007   ins_pipe(ialu_reg_mem);
 5008 %}
 5009 
 5010 instruct loadSSF(regF dst, stackSlotF src)
 5011 %{
 5012   match(Set dst src);
 5013 
 5014   ins_cost(125);
 5015   format %{ "movss   $dst, $src\t# float stk" %}
 5016   ins_encode %{
 5017     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5018   %}
 5019   ins_pipe(pipe_slow); // XXX
 5020 %}
 5021 
 5022 // Use the same format since predicate() can not be used here.
 5023 instruct loadSSD(regD dst, stackSlotD src)
 5024 %{
 5025   match(Set dst src);
 5026 
 5027   ins_cost(125);
 5028   format %{ "movsd   $dst, $src\t# double stk" %}
 5029   ins_encode  %{
 5030     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5031   %}
 5032   ins_pipe(pipe_slow); // XXX
 5033 %}
 5034 
 5035 // Prefetch instructions for allocation.
 5036 // Must be safe to execute with invalid address (cannot fault).
 5037 
 5038 instruct prefetchAlloc( memory mem ) %{
 5039   predicate(AllocatePrefetchInstr==3);
 5040   match(PrefetchAllocation mem);
 5041   ins_cost(125);
 5042 
 5043   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5044   ins_encode %{
 5045     __ prefetchw($mem$$Address);
 5046   %}
 5047   ins_pipe(ialu_mem);
 5048 %}
 5049 
 5050 instruct prefetchAllocNTA( memory mem ) %{
 5051   predicate(AllocatePrefetchInstr==0);
 5052   match(PrefetchAllocation mem);
 5053   ins_cost(125);
 5054 
 5055   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5056   ins_encode %{
 5057     __ prefetchnta($mem$$Address);
 5058   %}
 5059   ins_pipe(ialu_mem);
 5060 %}
 5061 
 5062 instruct prefetchAllocT0( memory mem ) %{
 5063   predicate(AllocatePrefetchInstr==1);
 5064   match(PrefetchAllocation mem);
 5065   ins_cost(125);
 5066 
 5067   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5068   ins_encode %{
 5069     __ prefetcht0($mem$$Address);
 5070   %}
 5071   ins_pipe(ialu_mem);
 5072 %}
 5073 
 5074 instruct prefetchAllocT2( memory mem ) %{
 5075   predicate(AllocatePrefetchInstr==2);
 5076   match(PrefetchAllocation mem);
 5077   ins_cost(125);
 5078 
 5079   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5080   ins_encode %{
 5081     __ prefetcht2($mem$$Address);
 5082   %}
 5083   ins_pipe(ialu_mem);
 5084 %}
 5085 
 5086 //----------Store Instructions-------------------------------------------------
 5087 
 5088 // Store Byte
 5089 instruct storeB(memory mem, rRegI src)
 5090 %{
 5091   match(Set mem (StoreB mem src));
 5092 
 5093   ins_cost(125); // XXX
 5094   format %{ "movb    $mem, $src\t# byte" %}
 5095   ins_encode %{
 5096     __ movb($mem$$Address, $src$$Register);
 5097   %}
 5098   ins_pipe(ialu_mem_reg);
 5099 %}
 5100 
 5101 // Store Char/Short
 5102 instruct storeC(memory mem, rRegI src)
 5103 %{
 5104   match(Set mem (StoreC mem src));
 5105 
 5106   ins_cost(125); // XXX
 5107   format %{ "movw    $mem, $src\t# char/short" %}
 5108   ins_encode %{
 5109     __ movw($mem$$Address, $src$$Register);
 5110   %}
 5111   ins_pipe(ialu_mem_reg);
 5112 %}
 5113 
 5114 // Store Integer
 5115 instruct storeI(memory mem, rRegI src)
 5116 %{
 5117   match(Set mem (StoreI mem src));
 5118 
 5119   ins_cost(125); // XXX
 5120   format %{ "movl    $mem, $src\t# int" %}
 5121   ins_encode %{
 5122     __ movl($mem$$Address, $src$$Register);
 5123   %}
 5124   ins_pipe(ialu_mem_reg);
 5125 %}
 5126 
 5127 // Store Long
 5128 instruct storeL(memory mem, rRegL src)
 5129 %{
 5130   match(Set mem (StoreL mem src));
 5131 
 5132   ins_cost(125); // XXX
 5133   format %{ "movq    $mem, $src\t# long" %}
 5134   ins_encode %{
 5135     __ movq($mem$$Address, $src$$Register);
 5136   %}
 5137   ins_pipe(ialu_mem_reg); // XXX
 5138 %}
 5139 
 5140 // Store Pointer
 5141 instruct storeP(memory mem, any_RegP src)
 5142 %{
 5143   predicate(n->as_Store()->barrier_data() == 0);
 5144   match(Set mem (StoreP mem src));
 5145 
 5146   ins_cost(125); // XXX
 5147   format %{ "movq    $mem, $src\t# ptr" %}
 5148   ins_encode %{
 5149     __ movq($mem$$Address, $src$$Register);
 5150   %}
 5151   ins_pipe(ialu_mem_reg);
 5152 %}
 5153 
 5154 instruct storeImmP0(memory mem, immP0 zero)
 5155 %{
 5156   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5157   match(Set mem (StoreP mem zero));
 5158 
 5159   ins_cost(125); // XXX
 5160   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5161   ins_encode %{
 5162     __ movq($mem$$Address, r12);
 5163   %}
 5164   ins_pipe(ialu_mem_reg);
 5165 %}
 5166 
 5167 // Store Null Pointer, mark word, or other simple pointer constant.
 5168 instruct storeImmP(memory mem, immP31 src)
 5169 %{
 5170   predicate(n->as_Store()->barrier_data() == 0);
 5171   match(Set mem (StoreP mem src));
 5172 
 5173   ins_cost(150); // XXX
 5174   format %{ "movq    $mem, $src\t# ptr" %}
 5175   ins_encode %{
 5176     __ movq($mem$$Address, $src$$constant);
 5177   %}
 5178   ins_pipe(ialu_mem_imm);
 5179 %}
 5180 
 5181 // Store Compressed Pointer
 5182 instruct storeN(memory mem, rRegN src)
 5183 %{
 5184   match(Set mem (StoreN mem src));
 5185 
 5186   ins_cost(125); // XXX
 5187   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5188   ins_encode %{
 5189     __ movl($mem$$Address, $src$$Register);
 5190   %}
 5191   ins_pipe(ialu_mem_reg);
 5192 %}
 5193 
 5194 instruct storeNKlass(memory mem, rRegN src)
 5195 %{
 5196   match(Set mem (StoreNKlass mem src));
 5197 
 5198   ins_cost(125); // XXX
 5199   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5200   ins_encode %{
 5201     __ movl($mem$$Address, $src$$Register);
 5202   %}
 5203   ins_pipe(ialu_mem_reg);
 5204 %}
 5205 
 5206 instruct storeImmN0(memory mem, immN0 zero)
 5207 %{
 5208   predicate(CompressedOops::base() == nullptr);
 5209   match(Set mem (StoreN mem zero));
 5210 
 5211   ins_cost(125); // XXX
 5212   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5213   ins_encode %{
 5214     __ movl($mem$$Address, r12);
 5215   %}
 5216   ins_pipe(ialu_mem_reg);
 5217 %}
 5218 
 5219 instruct storeImmN(memory mem, immN src)
 5220 %{
 5221   match(Set mem (StoreN mem src));
 5222 
 5223   ins_cost(150); // XXX
 5224   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5225   ins_encode %{
 5226     address con = (address)$src$$constant;
 5227     if (con == nullptr) {
 5228       __ movl($mem$$Address, 0);
 5229     } else {
 5230       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5231     }
 5232   %}
 5233   ins_pipe(ialu_mem_imm);
 5234 %}
 5235 
 5236 instruct storeImmNKlass(memory mem, immNKlass src)
 5237 %{
 5238   match(Set mem (StoreNKlass mem src));
 5239 
 5240   ins_cost(150); // XXX
 5241   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5242   ins_encode %{
 5243     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5244   %}
 5245   ins_pipe(ialu_mem_imm);
 5246 %}
 5247 
 5248 // Store Integer Immediate
 5249 instruct storeImmI0(memory mem, immI_0 zero)
 5250 %{
 5251   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5252   match(Set mem (StoreI mem zero));
 5253 
 5254   ins_cost(125); // XXX
 5255   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5256   ins_encode %{
 5257     __ movl($mem$$Address, r12);
 5258   %}
 5259   ins_pipe(ialu_mem_reg);
 5260 %}
 5261 
 5262 instruct storeImmI(memory mem, immI src)
 5263 %{
 5264   match(Set mem (StoreI mem src));
 5265 
 5266   ins_cost(150);
 5267   format %{ "movl    $mem, $src\t# int" %}
 5268   ins_encode %{
 5269     __ movl($mem$$Address, $src$$constant);
 5270   %}
 5271   ins_pipe(ialu_mem_imm);
 5272 %}
 5273 
 5274 // Store Long Immediate
 5275 instruct storeImmL0(memory mem, immL0 zero)
 5276 %{
 5277   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5278   match(Set mem (StoreL mem zero));
 5279 
 5280   ins_cost(125); // XXX
 5281   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5282   ins_encode %{
 5283     __ movq($mem$$Address, r12);
 5284   %}
 5285   ins_pipe(ialu_mem_reg);
 5286 %}
 5287 
 5288 instruct storeImmL(memory mem, immL32 src)
 5289 %{
 5290   match(Set mem (StoreL mem src));
 5291 
 5292   ins_cost(150);
 5293   format %{ "movq    $mem, $src\t# long" %}
 5294   ins_encode %{
 5295     __ movq($mem$$Address, $src$$constant);
 5296   %}
 5297   ins_pipe(ialu_mem_imm);
 5298 %}
 5299 
 5300 // Store Short/Char Immediate
 5301 instruct storeImmC0(memory mem, immI_0 zero)
 5302 %{
 5303   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5304   match(Set mem (StoreC mem zero));
 5305 
 5306   ins_cost(125); // XXX
 5307   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5308   ins_encode %{
 5309     __ movw($mem$$Address, r12);
 5310   %}
 5311   ins_pipe(ialu_mem_reg);
 5312 %}
 5313 
 5314 instruct storeImmI16(memory mem, immI16 src)
 5315 %{
 5316   predicate(UseStoreImmI16);
 5317   match(Set mem (StoreC mem src));
 5318 
 5319   ins_cost(150);
 5320   format %{ "movw    $mem, $src\t# short/char" %}
 5321   ins_encode %{
 5322     __ movw($mem$$Address, $src$$constant);
 5323   %}
 5324   ins_pipe(ialu_mem_imm);
 5325 %}
 5326 
 5327 // Store Byte Immediate
 5328 instruct storeImmB0(memory mem, immI_0 zero)
 5329 %{
 5330   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5331   match(Set mem (StoreB mem zero));
 5332 
 5333   ins_cost(125); // XXX
 5334   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5335   ins_encode %{
 5336     __ movb($mem$$Address, r12);
 5337   %}
 5338   ins_pipe(ialu_mem_reg);
 5339 %}
 5340 
 5341 instruct storeImmB(memory mem, immI8 src)
 5342 %{
 5343   match(Set mem (StoreB mem src));
 5344 
 5345   ins_cost(150); // XXX
 5346   format %{ "movb    $mem, $src\t# byte" %}
 5347   ins_encode %{
 5348     __ movb($mem$$Address, $src$$constant);
 5349   %}
 5350   ins_pipe(ialu_mem_imm);
 5351 %}
 5352 
 5353 // Store CMS card-mark Immediate
 5354 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 5355 %{
 5356   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5357   match(Set mem (StoreCM mem zero));
 5358 
 5359   ins_cost(125); // XXX
 5360   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 5361   ins_encode %{
 5362     __ movb($mem$$Address, r12);
 5363   %}
 5364   ins_pipe(ialu_mem_reg);
 5365 %}
 5366 
 5367 instruct storeImmCM0(memory mem, immI_0 src)
 5368 %{
 5369   match(Set mem (StoreCM mem src));
 5370 
 5371   ins_cost(150); // XXX
 5372   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 5373   ins_encode %{
 5374     __ movb($mem$$Address, $src$$constant);
 5375   %}
 5376   ins_pipe(ialu_mem_imm);
 5377 %}
 5378 
 5379 // Store Float
 5380 instruct storeF(memory mem, regF src)
 5381 %{
 5382   match(Set mem (StoreF mem src));
 5383 
 5384   ins_cost(95); // XXX
 5385   format %{ "movss   $mem, $src\t# float" %}
 5386   ins_encode %{
 5387     __ movflt($mem$$Address, $src$$XMMRegister);
 5388   %}
 5389   ins_pipe(pipe_slow); // XXX
 5390 %}
 5391 
 5392 // Store immediate Float value (it is faster than store from XMM register)
 5393 instruct storeF0(memory mem, immF0 zero)
 5394 %{
 5395   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5396   match(Set mem (StoreF mem zero));
 5397 
 5398   ins_cost(25); // XXX
 5399   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5400   ins_encode %{
 5401     __ movl($mem$$Address, r12);
 5402   %}
 5403   ins_pipe(ialu_mem_reg);
 5404 %}
 5405 
 5406 instruct storeF_imm(memory mem, immF src)
 5407 %{
 5408   match(Set mem (StoreF mem src));
 5409 
 5410   ins_cost(50);
 5411   format %{ "movl    $mem, $src\t# float" %}
 5412   ins_encode %{
 5413     __ movl($mem$$Address, jint_cast($src$$constant));
 5414   %}
 5415   ins_pipe(ialu_mem_imm);
 5416 %}
 5417 
 5418 // Store Double
 5419 instruct storeD(memory mem, regD src)
 5420 %{
 5421   match(Set mem (StoreD mem src));
 5422 
 5423   ins_cost(95); // XXX
 5424   format %{ "movsd   $mem, $src\t# double" %}
 5425   ins_encode %{
 5426     __ movdbl($mem$$Address, $src$$XMMRegister);
 5427   %}
 5428   ins_pipe(pipe_slow); // XXX
 5429 %}
 5430 
 5431 // Store immediate double 0.0 (it is faster than store from XMM register)
 5432 instruct storeD0_imm(memory mem, immD0 src)
 5433 %{
 5434   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5435   match(Set mem (StoreD mem src));
 5436 
 5437   ins_cost(50);
 5438   format %{ "movq    $mem, $src\t# double 0." %}
 5439   ins_encode %{
 5440     __ movq($mem$$Address, $src$$constant);
 5441   %}
 5442   ins_pipe(ialu_mem_imm);
 5443 %}
 5444 
 5445 instruct storeD0(memory mem, immD0 zero)
 5446 %{
 5447   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5448   match(Set mem (StoreD mem zero));
 5449 
 5450   ins_cost(25); // XXX
 5451   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5452   ins_encode %{
 5453     __ movq($mem$$Address, r12);
 5454   %}
 5455   ins_pipe(ialu_mem_reg);
 5456 %}
 5457 
 5458 instruct storeSSI(stackSlotI dst, rRegI src)
 5459 %{
 5460   match(Set dst src);
 5461 
 5462   ins_cost(100);
 5463   format %{ "movl    $dst, $src\t# int stk" %}
 5464   ins_encode %{
 5465     __ movl($dst$$Address, $src$$Register);
 5466   %}
 5467   ins_pipe( ialu_mem_reg );
 5468 %}
 5469 
 5470 instruct storeSSL(stackSlotL dst, rRegL src)
 5471 %{
 5472   match(Set dst src);
 5473 
 5474   ins_cost(100);
 5475   format %{ "movq    $dst, $src\t# long stk" %}
 5476   ins_encode %{
 5477     __ movq($dst$$Address, $src$$Register);
 5478   %}
 5479   ins_pipe(ialu_mem_reg);
 5480 %}
 5481 
 5482 instruct storeSSP(stackSlotP dst, rRegP src)
 5483 %{
 5484   match(Set dst src);
 5485 
 5486   ins_cost(100);
 5487   format %{ "movq    $dst, $src\t# ptr stk" %}
 5488   ins_encode %{
 5489     __ movq($dst$$Address, $src$$Register);
 5490   %}
 5491   ins_pipe(ialu_mem_reg);
 5492 %}
 5493 
 5494 instruct storeSSF(stackSlotF dst, regF src)
 5495 %{
 5496   match(Set dst src);
 5497 
 5498   ins_cost(95); // XXX
 5499   format %{ "movss   $dst, $src\t# float stk" %}
 5500   ins_encode %{
 5501     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5502   %}
 5503   ins_pipe(pipe_slow); // XXX
 5504 %}
 5505 
 5506 instruct storeSSD(stackSlotD dst, regD src)
 5507 %{
 5508   match(Set dst src);
 5509 
 5510   ins_cost(95); // XXX
 5511   format %{ "movsd   $dst, $src\t# double stk" %}
 5512   ins_encode %{
 5513     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5514   %}
 5515   ins_pipe(pipe_slow); // XXX
 5516 %}
 5517 
 5518 instruct cacheWB(indirect addr)
 5519 %{
 5520   predicate(VM_Version::supports_data_cache_line_flush());
 5521   match(CacheWB addr);
 5522 
 5523   ins_cost(100);
 5524   format %{"cache wb $addr" %}
 5525   ins_encode %{
 5526     assert($addr->index_position() < 0, "should be");
 5527     assert($addr$$disp == 0, "should be");
 5528     __ cache_wb(Address($addr$$base$$Register, 0));
 5529   %}
 5530   ins_pipe(pipe_slow); // XXX
 5531 %}
 5532 
 5533 instruct cacheWBPreSync()
 5534 %{
 5535   predicate(VM_Version::supports_data_cache_line_flush());
 5536   match(CacheWBPreSync);
 5537 
 5538   ins_cost(100);
 5539   format %{"cache wb presync" %}
 5540   ins_encode %{
 5541     __ cache_wbsync(true);
 5542   %}
 5543   ins_pipe(pipe_slow); // XXX
 5544 %}
 5545 
 5546 instruct cacheWBPostSync()
 5547 %{
 5548   predicate(VM_Version::supports_data_cache_line_flush());
 5549   match(CacheWBPostSync);
 5550 
 5551   ins_cost(100);
 5552   format %{"cache wb postsync" %}
 5553   ins_encode %{
 5554     __ cache_wbsync(false);
 5555   %}
 5556   ins_pipe(pipe_slow); // XXX
 5557 %}
 5558 
 5559 //----------BSWAP Instructions-------------------------------------------------
 5560 instruct bytes_reverse_int(rRegI dst) %{
 5561   match(Set dst (ReverseBytesI dst));
 5562 
 5563   format %{ "bswapl  $dst" %}
 5564   ins_encode %{
 5565     __ bswapl($dst$$Register);
 5566   %}
 5567   ins_pipe( ialu_reg );
 5568 %}
 5569 
 5570 instruct bytes_reverse_long(rRegL dst) %{
 5571   match(Set dst (ReverseBytesL dst));
 5572 
 5573   format %{ "bswapq  $dst" %}
 5574   ins_encode %{
 5575     __ bswapq($dst$$Register);
 5576   %}
 5577   ins_pipe( ialu_reg);
 5578 %}
 5579 
 5580 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5581   match(Set dst (ReverseBytesUS dst));
 5582   effect(KILL cr);
 5583 
 5584   format %{ "bswapl  $dst\n\t"
 5585             "shrl    $dst,16\n\t" %}
 5586   ins_encode %{
 5587     __ bswapl($dst$$Register);
 5588     __ shrl($dst$$Register, 16);
 5589   %}
 5590   ins_pipe( ialu_reg );
 5591 %}
 5592 
 5593 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5594   match(Set dst (ReverseBytesS dst));
 5595   effect(KILL cr);
 5596 
 5597   format %{ "bswapl  $dst\n\t"
 5598             "sar     $dst,16\n\t" %}
 5599   ins_encode %{
 5600     __ bswapl($dst$$Register);
 5601     __ sarl($dst$$Register, 16);
 5602   %}
 5603   ins_pipe( ialu_reg );
 5604 %}
 5605 
 5606 //---------- Zeros Count Instructions ------------------------------------------
 5607 
 5608 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5609   predicate(UseCountLeadingZerosInstruction);
 5610   match(Set dst (CountLeadingZerosI src));
 5611   effect(KILL cr);
 5612 
 5613   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5614   ins_encode %{
 5615     __ lzcntl($dst$$Register, $src$$Register);
 5616   %}
 5617   ins_pipe(ialu_reg);
 5618 %}
 5619 
 5620 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5621   predicate(UseCountLeadingZerosInstruction);
 5622   match(Set dst (CountLeadingZerosI (LoadI src)));
 5623   effect(KILL cr);
 5624   ins_cost(175);
 5625   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5626   ins_encode %{
 5627     __ lzcntl($dst$$Register, $src$$Address);
 5628   %}
 5629   ins_pipe(ialu_reg_mem);
 5630 %}
 5631 
 5632 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5633   predicate(!UseCountLeadingZerosInstruction);
 5634   match(Set dst (CountLeadingZerosI src));
 5635   effect(KILL cr);
 5636 
 5637   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5638             "jnz     skip\n\t"
 5639             "movl    $dst, -1\n"
 5640       "skip:\n\t"
 5641             "negl    $dst\n\t"
 5642             "addl    $dst, 31" %}
 5643   ins_encode %{
 5644     Register Rdst = $dst$$Register;
 5645     Register Rsrc = $src$$Register;
 5646     Label skip;
 5647     __ bsrl(Rdst, Rsrc);
 5648     __ jccb(Assembler::notZero, skip);
 5649     __ movl(Rdst, -1);
 5650     __ bind(skip);
 5651     __ negl(Rdst);
 5652     __ addl(Rdst, BitsPerInt - 1);
 5653   %}
 5654   ins_pipe(ialu_reg);
 5655 %}
 5656 
 5657 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5658   predicate(UseCountLeadingZerosInstruction);
 5659   match(Set dst (CountLeadingZerosL src));
 5660   effect(KILL cr);
 5661 
 5662   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5663   ins_encode %{
 5664     __ lzcntq($dst$$Register, $src$$Register);
 5665   %}
 5666   ins_pipe(ialu_reg);
 5667 %}
 5668 
 5669 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5670   predicate(UseCountLeadingZerosInstruction);
 5671   match(Set dst (CountLeadingZerosL (LoadL src)));
 5672   effect(KILL cr);
 5673   ins_cost(175);
 5674   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5675   ins_encode %{
 5676     __ lzcntq($dst$$Register, $src$$Address);
 5677   %}
 5678   ins_pipe(ialu_reg_mem);
 5679 %}
 5680 
 5681 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5682   predicate(!UseCountLeadingZerosInstruction);
 5683   match(Set dst (CountLeadingZerosL src));
 5684   effect(KILL cr);
 5685 
 5686   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5687             "jnz     skip\n\t"
 5688             "movl    $dst, -1\n"
 5689       "skip:\n\t"
 5690             "negl    $dst\n\t"
 5691             "addl    $dst, 63" %}
 5692   ins_encode %{
 5693     Register Rdst = $dst$$Register;
 5694     Register Rsrc = $src$$Register;
 5695     Label skip;
 5696     __ bsrq(Rdst, Rsrc);
 5697     __ jccb(Assembler::notZero, skip);
 5698     __ movl(Rdst, -1);
 5699     __ bind(skip);
 5700     __ negl(Rdst);
 5701     __ addl(Rdst, BitsPerLong - 1);
 5702   %}
 5703   ins_pipe(ialu_reg);
 5704 %}
 5705 
 5706 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5707   predicate(UseCountTrailingZerosInstruction);
 5708   match(Set dst (CountTrailingZerosI src));
 5709   effect(KILL cr);
 5710 
 5711   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5712   ins_encode %{
 5713     __ tzcntl($dst$$Register, $src$$Register);
 5714   %}
 5715   ins_pipe(ialu_reg);
 5716 %}
 5717 
 5718 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5719   predicate(UseCountTrailingZerosInstruction);
 5720   match(Set dst (CountTrailingZerosI (LoadI src)));
 5721   effect(KILL cr);
 5722   ins_cost(175);
 5723   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5724   ins_encode %{
 5725     __ tzcntl($dst$$Register, $src$$Address);
 5726   %}
 5727   ins_pipe(ialu_reg_mem);
 5728 %}
 5729 
 5730 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5731   predicate(!UseCountTrailingZerosInstruction);
 5732   match(Set dst (CountTrailingZerosI src));
 5733   effect(KILL cr);
 5734 
 5735   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5736             "jnz     done\n\t"
 5737             "movl    $dst, 32\n"
 5738       "done:" %}
 5739   ins_encode %{
 5740     Register Rdst = $dst$$Register;
 5741     Label done;
 5742     __ bsfl(Rdst, $src$$Register);
 5743     __ jccb(Assembler::notZero, done);
 5744     __ movl(Rdst, BitsPerInt);
 5745     __ bind(done);
 5746   %}
 5747   ins_pipe(ialu_reg);
 5748 %}
 5749 
 5750 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5751   predicate(UseCountTrailingZerosInstruction);
 5752   match(Set dst (CountTrailingZerosL src));
 5753   effect(KILL cr);
 5754 
 5755   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5756   ins_encode %{
 5757     __ tzcntq($dst$$Register, $src$$Register);
 5758   %}
 5759   ins_pipe(ialu_reg);
 5760 %}
 5761 
 5762 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5763   predicate(UseCountTrailingZerosInstruction);
 5764   match(Set dst (CountTrailingZerosL (LoadL src)));
 5765   effect(KILL cr);
 5766   ins_cost(175);
 5767   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5768   ins_encode %{
 5769     __ tzcntq($dst$$Register, $src$$Address);
 5770   %}
 5771   ins_pipe(ialu_reg_mem);
 5772 %}
 5773 
 5774 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5775   predicate(!UseCountTrailingZerosInstruction);
 5776   match(Set dst (CountTrailingZerosL src));
 5777   effect(KILL cr);
 5778 
 5779   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5780             "jnz     done\n\t"
 5781             "movl    $dst, 64\n"
 5782       "done:" %}
 5783   ins_encode %{
 5784     Register Rdst = $dst$$Register;
 5785     Label done;
 5786     __ bsfq(Rdst, $src$$Register);
 5787     __ jccb(Assembler::notZero, done);
 5788     __ movl(Rdst, BitsPerLong);
 5789     __ bind(done);
 5790   %}
 5791   ins_pipe(ialu_reg);
 5792 %}
 5793 
 5794 //--------------- Reverse Operation Instructions ----------------
 5795 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5796   predicate(!VM_Version::supports_gfni());
 5797   match(Set dst (ReverseI src));
 5798   effect(TEMP dst, TEMP rtmp, KILL cr);
 5799   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5800   ins_encode %{
 5801     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5802   %}
 5803   ins_pipe( ialu_reg );
 5804 %}
 5805 
 5806 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5807   predicate(VM_Version::supports_gfni());
 5808   match(Set dst (ReverseI src));
 5809   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5810   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5811   ins_encode %{
 5812     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5813   %}
 5814   ins_pipe( ialu_reg );
 5815 %}
 5816 
 5817 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5818   predicate(!VM_Version::supports_gfni());
 5819   match(Set dst (ReverseL src));
 5820   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5821   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5822   ins_encode %{
 5823     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5824   %}
 5825   ins_pipe( ialu_reg );
 5826 %}
 5827 
 5828 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5829   predicate(VM_Version::supports_gfni());
 5830   match(Set dst (ReverseL src));
 5831   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5832   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5833   ins_encode %{
 5834     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5835   %}
 5836   ins_pipe( ialu_reg );
 5837 %}
 5838 
 5839 //---------- Population Count Instructions -------------------------------------
 5840 
 5841 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5842   predicate(UsePopCountInstruction);
 5843   match(Set dst (PopCountI src));
 5844   effect(KILL cr);
 5845 
 5846   format %{ "popcnt  $dst, $src" %}
 5847   ins_encode %{
 5848     __ popcntl($dst$$Register, $src$$Register);
 5849   %}
 5850   ins_pipe(ialu_reg);
 5851 %}
 5852 
 5853 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5854   predicate(UsePopCountInstruction);
 5855   match(Set dst (PopCountI (LoadI mem)));
 5856   effect(KILL cr);
 5857 
 5858   format %{ "popcnt  $dst, $mem" %}
 5859   ins_encode %{
 5860     __ popcntl($dst$$Register, $mem$$Address);
 5861   %}
 5862   ins_pipe(ialu_reg);
 5863 %}
 5864 
 5865 // Note: Long.bitCount(long) returns an int.
 5866 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5867   predicate(UsePopCountInstruction);
 5868   match(Set dst (PopCountL src));
 5869   effect(KILL cr);
 5870 
 5871   format %{ "popcnt  $dst, $src" %}
 5872   ins_encode %{
 5873     __ popcntq($dst$$Register, $src$$Register);
 5874   %}
 5875   ins_pipe(ialu_reg);
 5876 %}
 5877 
 5878 // Note: Long.bitCount(long) returns an int.
 5879 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5880   predicate(UsePopCountInstruction);
 5881   match(Set dst (PopCountL (LoadL mem)));
 5882   effect(KILL cr);
 5883 
 5884   format %{ "popcnt  $dst, $mem" %}
 5885   ins_encode %{
 5886     __ popcntq($dst$$Register, $mem$$Address);
 5887   %}
 5888   ins_pipe(ialu_reg);
 5889 %}
 5890 
 5891 
 5892 //----------MemBar Instructions-----------------------------------------------
 5893 // Memory barrier flavors
 5894 
 5895 instruct membar_acquire()
 5896 %{
 5897   match(MemBarAcquire);
 5898   match(LoadFence);
 5899   ins_cost(0);
 5900 
 5901   size(0);
 5902   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5903   ins_encode();
 5904   ins_pipe(empty);
 5905 %}
 5906 
 5907 instruct membar_acquire_lock()
 5908 %{
 5909   match(MemBarAcquireLock);
 5910   ins_cost(0);
 5911 
 5912   size(0);
 5913   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5914   ins_encode();
 5915   ins_pipe(empty);
 5916 %}
 5917 
 5918 instruct membar_release()
 5919 %{
 5920   match(MemBarRelease);
 5921   match(StoreFence);
 5922   ins_cost(0);
 5923 
 5924   size(0);
 5925   format %{ "MEMBAR-release ! (empty encoding)" %}
 5926   ins_encode();
 5927   ins_pipe(empty);
 5928 %}
 5929 
 5930 instruct membar_release_lock()
 5931 %{
 5932   match(MemBarReleaseLock);
 5933   ins_cost(0);
 5934 
 5935   size(0);
 5936   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5937   ins_encode();
 5938   ins_pipe(empty);
 5939 %}
 5940 
 5941 instruct membar_volatile(rFlagsReg cr) %{
 5942   match(MemBarVolatile);
 5943   effect(KILL cr);
 5944   ins_cost(400);
 5945 
 5946   format %{
 5947     $$template
 5948     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5949   %}
 5950   ins_encode %{
 5951     __ membar(Assembler::StoreLoad);
 5952   %}
 5953   ins_pipe(pipe_slow);
 5954 %}
 5955 
 5956 instruct unnecessary_membar_volatile()
 5957 %{
 5958   match(MemBarVolatile);
 5959   predicate(Matcher::post_store_load_barrier(n));
 5960   ins_cost(0);
 5961 
 5962   size(0);
 5963   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5964   ins_encode();
 5965   ins_pipe(empty);
 5966 %}
 5967 
 5968 instruct membar_storestore() %{
 5969   match(MemBarStoreStore);
 5970   match(StoreStoreFence);
 5971   ins_cost(0);
 5972 
 5973   size(0);
 5974   format %{ "MEMBAR-storestore (empty encoding)" %}
 5975   ins_encode( );
 5976   ins_pipe(empty);
 5977 %}
 5978 
 5979 //----------Move Instructions--------------------------------------------------
 5980 
 5981 instruct castX2P(rRegP dst, rRegL src)
 5982 %{
 5983   match(Set dst (CastX2P src));
 5984 
 5985   format %{ "movq    $dst, $src\t# long->ptr" %}
 5986   ins_encode %{
 5987     if ($dst$$reg != $src$$reg) {
 5988       __ movptr($dst$$Register, $src$$Register);
 5989     }
 5990   %}
 5991   ins_pipe(ialu_reg_reg); // XXX
 5992 %}
 5993 
 5994 instruct castP2X(rRegL dst, rRegP src)
 5995 %{
 5996   match(Set dst (CastP2X src));
 5997 
 5998   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5999   ins_encode %{
 6000     if ($dst$$reg != $src$$reg) {
 6001       __ movptr($dst$$Register, $src$$Register);
 6002     }
 6003   %}
 6004   ins_pipe(ialu_reg_reg); // XXX
 6005 %}
 6006 
 6007 // Convert oop into int for vectors alignment masking
 6008 instruct convP2I(rRegI dst, rRegP src)
 6009 %{
 6010   match(Set dst (ConvL2I (CastP2X src)));
 6011 
 6012   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6013   ins_encode %{
 6014     __ movl($dst$$Register, $src$$Register);
 6015   %}
 6016   ins_pipe(ialu_reg_reg); // XXX
 6017 %}
 6018 
 6019 // Convert compressed oop into int for vectors alignment masking
 6020 // in case of 32bit oops (heap < 4Gb).
 6021 instruct convN2I(rRegI dst, rRegN src)
 6022 %{
 6023   predicate(CompressedOops::shift() == 0);
 6024   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6025 
 6026   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6027   ins_encode %{
 6028     __ movl($dst$$Register, $src$$Register);
 6029   %}
 6030   ins_pipe(ialu_reg_reg); // XXX
 6031 %}
 6032 
 6033 // Convert oop pointer into compressed form
 6034 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6035   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6036   match(Set dst (EncodeP src));
 6037   effect(KILL cr);
 6038   format %{ "encode_heap_oop $dst,$src" %}
 6039   ins_encode %{
 6040     Register s = $src$$Register;
 6041     Register d = $dst$$Register;
 6042     if (s != d) {
 6043       __ movq(d, s);
 6044     }
 6045     __ encode_heap_oop(d);
 6046   %}
 6047   ins_pipe(ialu_reg_long);
 6048 %}
 6049 
 6050 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6051   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6052   match(Set dst (EncodeP src));
 6053   effect(KILL cr);
 6054   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6055   ins_encode %{
 6056     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6057   %}
 6058   ins_pipe(ialu_reg_long);
 6059 %}
 6060 
 6061 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6062   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6063             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6064   match(Set dst (DecodeN src));
 6065   effect(KILL cr);
 6066   format %{ "decode_heap_oop $dst,$src" %}
 6067   ins_encode %{
 6068     Register s = $src$$Register;
 6069     Register d = $dst$$Register;
 6070     if (s != d) {
 6071       __ movq(d, s);
 6072     }
 6073     __ decode_heap_oop(d);
 6074   %}
 6075   ins_pipe(ialu_reg_long);
 6076 %}
 6077 
 6078 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6079   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6080             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6081   match(Set dst (DecodeN src));
 6082   effect(KILL cr);
 6083   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6084   ins_encode %{
 6085     Register s = $src$$Register;
 6086     Register d = $dst$$Register;
 6087     if (s != d) {
 6088       __ decode_heap_oop_not_null(d, s);
 6089     } else {
 6090       __ decode_heap_oop_not_null(d);
 6091     }
 6092   %}
 6093   ins_pipe(ialu_reg_long);
 6094 %}
 6095 
 6096 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6097   match(Set dst (EncodePKlass src));
 6098   effect(TEMP dst, KILL cr);
 6099   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6100   ins_encode %{
 6101     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6102   %}
 6103   ins_pipe(ialu_reg_long);
 6104 %}
 6105 
 6106 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6107   match(Set dst (DecodeNKlass src));
 6108   effect(TEMP dst, KILL cr);
 6109   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6110   ins_encode %{
 6111     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6112   %}
 6113   ins_pipe(ialu_reg_long);
 6114 %}
 6115 
 6116 //----------Conditional Move---------------------------------------------------
 6117 // Jump
 6118 // dummy instruction for generating temp registers
 6119 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6120   match(Jump (LShiftL switch_val shift));
 6121   ins_cost(350);
 6122   predicate(false);
 6123   effect(TEMP dest);
 6124 
 6125   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6126             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6127   ins_encode %{
 6128     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6129     // to do that and the compiler is using that register as one it can allocate.
 6130     // So we build it all by hand.
 6131     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6132     // ArrayAddress dispatch(table, index);
 6133     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6134     __ lea($dest$$Register, $constantaddress);
 6135     __ jmp(dispatch);
 6136   %}
 6137   ins_pipe(pipe_jmp);
 6138 %}
 6139 
 6140 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6141   match(Jump (AddL (LShiftL switch_val shift) offset));
 6142   ins_cost(350);
 6143   effect(TEMP dest);
 6144 
 6145   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6146             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6147   ins_encode %{
 6148     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6149     // to do that and the compiler is using that register as one it can allocate.
 6150     // So we build it all by hand.
 6151     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6152     // ArrayAddress dispatch(table, index);
 6153     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6154     __ lea($dest$$Register, $constantaddress);
 6155     __ jmp(dispatch);
 6156   %}
 6157   ins_pipe(pipe_jmp);
 6158 %}
 6159 
 6160 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6161   match(Jump switch_val);
 6162   ins_cost(350);
 6163   effect(TEMP dest);
 6164 
 6165   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6166             "jmp     [$dest + $switch_val]\n\t" %}
 6167   ins_encode %{
 6168     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6169     // to do that and the compiler is using that register as one it can allocate.
 6170     // So we build it all by hand.
 6171     // Address index(noreg, switch_reg, Address::times_1);
 6172     // ArrayAddress dispatch(table, index);
 6173     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6174     __ lea($dest$$Register, $constantaddress);
 6175     __ jmp(dispatch);
 6176   %}
 6177   ins_pipe(pipe_jmp);
 6178 %}
 6179 
 6180 // Conditional move
 6181 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6182 %{
 6183   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6184   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6185 
 6186   ins_cost(100); // XXX
 6187   format %{ "setbn$cop $dst\t# signed, int" %}
 6188   ins_encode %{
 6189     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6190     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6191   %}
 6192   ins_pipe(ialu_reg);
 6193 %}
 6194 
 6195 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6196 %{
 6197   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6198 
 6199   ins_cost(200); // XXX
 6200   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6201   ins_encode %{
 6202     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6203   %}
 6204   ins_pipe(pipe_cmov_reg);
 6205 %}
 6206 
 6207 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6208 %{
 6209   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6210   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6211 
 6212   ins_cost(100); // XXX
 6213   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6214   ins_encode %{
 6215     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6216     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6217   %}
 6218   ins_pipe(ialu_reg);
 6219 %}
 6220 
 6221 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6222   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6223 
 6224   ins_cost(200); // XXX
 6225   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6226   ins_encode %{
 6227     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6228   %}
 6229   ins_pipe(pipe_cmov_reg);
 6230 %}
 6231 
 6232 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6233 %{
 6234   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6235   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6236 
 6237   ins_cost(100); // XXX
 6238   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6239   ins_encode %{
 6240     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6241     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6242   %}
 6243   ins_pipe(ialu_reg);
 6244 %}
 6245 
 6246 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6247   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6248   ins_cost(200);
 6249   expand %{
 6250     cmovI_regU(cop, cr, dst, src);
 6251   %}
 6252 %}
 6253 
 6254 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6255   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6256   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6257 
 6258   ins_cost(200); // XXX
 6259   format %{ "cmovpl  $dst, $src\n\t"
 6260             "cmovnel $dst, $src" %}
 6261   ins_encode %{
 6262     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6263     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6264   %}
 6265   ins_pipe(pipe_cmov_reg);
 6266 %}
 6267 
 6268 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6269 // inputs of the CMove
 6270 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6271   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6272   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6273 
 6274   ins_cost(200); // XXX
 6275   format %{ "cmovpl  $dst, $src\n\t"
 6276             "cmovnel $dst, $src" %}
 6277   ins_encode %{
 6278     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6279     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6280   %}
 6281   ins_pipe(pipe_cmov_reg);
 6282 %}
 6283 
 6284 // Conditional move
 6285 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6286   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6287 
 6288   ins_cost(250); // XXX
 6289   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6290   ins_encode %{
 6291     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6292   %}
 6293   ins_pipe(pipe_cmov_mem);
 6294 %}
 6295 
 6296 // Conditional move
 6297 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6298 %{
 6299   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6300 
 6301   ins_cost(250); // XXX
 6302   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6303   ins_encode %{
 6304     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6305   %}
 6306   ins_pipe(pipe_cmov_mem);
 6307 %}
 6308 
 6309 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6310   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6311   ins_cost(250);
 6312   expand %{
 6313     cmovI_memU(cop, cr, dst, src);
 6314   %}
 6315 %}
 6316 
 6317 // Conditional move
 6318 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6319 %{
 6320   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6321 
 6322   ins_cost(200); // XXX
 6323   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6324   ins_encode %{
 6325     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6326   %}
 6327   ins_pipe(pipe_cmov_reg);
 6328 %}
 6329 
 6330 // Conditional move
 6331 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6332 %{
 6333   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6334 
 6335   ins_cost(200); // XXX
 6336   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6337   ins_encode %{
 6338     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6339   %}
 6340   ins_pipe(pipe_cmov_reg);
 6341 %}
 6342 
 6343 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6344   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6345   ins_cost(200);
 6346   expand %{
 6347     cmovN_regU(cop, cr, dst, src);
 6348   %}
 6349 %}
 6350 
 6351 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6352   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6353   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6354 
 6355   ins_cost(200); // XXX
 6356   format %{ "cmovpl  $dst, $src\n\t"
 6357             "cmovnel $dst, $src" %}
 6358   ins_encode %{
 6359     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6360     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6361   %}
 6362   ins_pipe(pipe_cmov_reg);
 6363 %}
 6364 
 6365 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6366 // inputs of the CMove
 6367 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6368   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6369   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6370 
 6371   ins_cost(200); // XXX
 6372   format %{ "cmovpl  $dst, $src\n\t"
 6373             "cmovnel $dst, $src" %}
 6374   ins_encode %{
 6375     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6376     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6377   %}
 6378   ins_pipe(pipe_cmov_reg);
 6379 %}
 6380 
 6381 // Conditional move
 6382 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6383 %{
 6384   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6385 
 6386   ins_cost(200); // XXX
 6387   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6388   ins_encode %{
 6389     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6390   %}
 6391   ins_pipe(pipe_cmov_reg);  // XXX
 6392 %}
 6393 
 6394 // Conditional move
 6395 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6396 %{
 6397   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6398 
 6399   ins_cost(200); // XXX
 6400   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6401   ins_encode %{
 6402     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6403   %}
 6404   ins_pipe(pipe_cmov_reg); // XXX
 6405 %}
 6406 
 6407 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6408   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6409   ins_cost(200);
 6410   expand %{
 6411     cmovP_regU(cop, cr, dst, src);
 6412   %}
 6413 %}
 6414 
 6415 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6416   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6417   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6418 
 6419   ins_cost(200); // XXX
 6420   format %{ "cmovpq  $dst, $src\n\t"
 6421             "cmovneq $dst, $src" %}
 6422   ins_encode %{
 6423     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6424     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6425   %}
 6426   ins_pipe(pipe_cmov_reg);
 6427 %}
 6428 
 6429 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6430 // inputs of the CMove
 6431 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6432   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6433   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6434 
 6435   ins_cost(200); // XXX
 6436   format %{ "cmovpq  $dst, $src\n\t"
 6437             "cmovneq $dst, $src" %}
 6438   ins_encode %{
 6439     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6440     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6441   %}
 6442   ins_pipe(pipe_cmov_reg);
 6443 %}
 6444 
 6445 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6446 %{
 6447   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6448   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6449 
 6450   ins_cost(100); // XXX
 6451   format %{ "setbn$cop $dst\t# signed, long" %}
 6452   ins_encode %{
 6453     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6454     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6455   %}
 6456   ins_pipe(ialu_reg);
 6457 %}
 6458 
 6459 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6460 %{
 6461   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6462 
 6463   ins_cost(200); // XXX
 6464   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6465   ins_encode %{
 6466     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6467   %}
 6468   ins_pipe(pipe_cmov_reg);  // XXX
 6469 %}
 6470 
 6471 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6472 %{
 6473   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6474 
 6475   ins_cost(200); // XXX
 6476   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6477   ins_encode %{
 6478     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6479   %}
 6480   ins_pipe(pipe_cmov_mem);  // XXX
 6481 %}
 6482 
 6483 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6484 %{
 6485   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6486   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6487 
 6488   ins_cost(100); // XXX
 6489   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6490   ins_encode %{
 6491     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6492     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6493   %}
 6494   ins_pipe(ialu_reg);
 6495 %}
 6496 
 6497 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6498 %{
 6499   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6500 
 6501   ins_cost(200); // XXX
 6502   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6503   ins_encode %{
 6504     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6505   %}
 6506   ins_pipe(pipe_cmov_reg); // XXX
 6507 %}
 6508 
 6509 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6510 %{
 6511   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6512   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6513 
 6514   ins_cost(100); // XXX
 6515   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6516   ins_encode %{
 6517     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6518     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6519   %}
 6520   ins_pipe(ialu_reg);
 6521 %}
 6522 
 6523 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6524   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6525   ins_cost(200);
 6526   expand %{
 6527     cmovL_regU(cop, cr, dst, src);
 6528   %}
 6529 %}
 6530 
 6531 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6532   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6533   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6534 
 6535   ins_cost(200); // XXX
 6536   format %{ "cmovpq  $dst, $src\n\t"
 6537             "cmovneq $dst, $src" %}
 6538   ins_encode %{
 6539     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6540     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6541   %}
 6542   ins_pipe(pipe_cmov_reg);
 6543 %}
 6544 
 6545 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6546 // inputs of the CMove
 6547 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6548   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6549   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6550 
 6551   ins_cost(200); // XXX
 6552   format %{ "cmovpq  $dst, $src\n\t"
 6553             "cmovneq $dst, $src" %}
 6554   ins_encode %{
 6555     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6556     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6557   %}
 6558   ins_pipe(pipe_cmov_reg);
 6559 %}
 6560 
 6561 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6562 %{
 6563   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6564 
 6565   ins_cost(200); // XXX
 6566   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6567   ins_encode %{
 6568     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6569   %}
 6570   ins_pipe(pipe_cmov_mem); // XXX
 6571 %}
 6572 
 6573 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6574   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6575   ins_cost(200);
 6576   expand %{
 6577     cmovL_memU(cop, cr, dst, src);
 6578   %}
 6579 %}
 6580 
 6581 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6582 %{
 6583   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6584 
 6585   ins_cost(200); // XXX
 6586   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6587             "movss     $dst, $src\n"
 6588     "skip:" %}
 6589   ins_encode %{
 6590     Label Lskip;
 6591     // Invert sense of branch from sense of CMOV
 6592     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6593     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6594     __ bind(Lskip);
 6595   %}
 6596   ins_pipe(pipe_slow);
 6597 %}
 6598 
 6599 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6600 %{
 6601   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6602 
 6603   ins_cost(200); // XXX
 6604   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6605             "movss     $dst, $src\n"
 6606     "skip:" %}
 6607   ins_encode %{
 6608     Label Lskip;
 6609     // Invert sense of branch from sense of CMOV
 6610     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6611     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6612     __ bind(Lskip);
 6613   %}
 6614   ins_pipe(pipe_slow);
 6615 %}
 6616 
 6617 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6618   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6619   ins_cost(200);
 6620   expand %{
 6621     cmovF_regU(cop, cr, dst, src);
 6622   %}
 6623 %}
 6624 
 6625 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6626 %{
 6627   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6628 
 6629   ins_cost(200); // XXX
 6630   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6631             "movsd     $dst, $src\n"
 6632     "skip:" %}
 6633   ins_encode %{
 6634     Label Lskip;
 6635     // Invert sense of branch from sense of CMOV
 6636     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6637     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6638     __ bind(Lskip);
 6639   %}
 6640   ins_pipe(pipe_slow);
 6641 %}
 6642 
 6643 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6644 %{
 6645   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6646 
 6647   ins_cost(200); // XXX
 6648   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6649             "movsd     $dst, $src\n"
 6650     "skip:" %}
 6651   ins_encode %{
 6652     Label Lskip;
 6653     // Invert sense of branch from sense of CMOV
 6654     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6655     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6656     __ bind(Lskip);
 6657   %}
 6658   ins_pipe(pipe_slow);
 6659 %}
 6660 
 6661 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6662   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6663   ins_cost(200);
 6664   expand %{
 6665     cmovD_regU(cop, cr, dst, src);
 6666   %}
 6667 %}
 6668 
 6669 //----------Arithmetic Instructions--------------------------------------------
 6670 //----------Addition Instructions----------------------------------------------
 6671 
 6672 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6673 %{
 6674   match(Set dst (AddI dst src));
 6675   effect(KILL cr);
 6676   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6677   format %{ "addl    $dst, $src\t# int" %}
 6678   ins_encode %{
 6679     __ addl($dst$$Register, $src$$Register);
 6680   %}
 6681   ins_pipe(ialu_reg_reg);
 6682 %}
 6683 
 6684 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 6685 %{
 6686   match(Set dst (AddI dst src));
 6687   effect(KILL cr);
 6688   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6689 
 6690   format %{ "addl    $dst, $src\t# int" %}
 6691   ins_encode %{
 6692     __ addl($dst$$Register, $src$$constant);
 6693   %}
 6694   ins_pipe( ialu_reg );
 6695 %}
 6696 
 6697 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 6698 %{
 6699   match(Set dst (AddI dst (LoadI src)));
 6700   effect(KILL cr);
 6701   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6702 
 6703   ins_cost(150); // XXX
 6704   format %{ "addl    $dst, $src\t# int" %}
 6705   ins_encode %{
 6706     __ addl($dst$$Register, $src$$Address);
 6707   %}
 6708   ins_pipe(ialu_reg_mem);
 6709 %}
 6710 
 6711 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 6712 %{
 6713   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6714   effect(KILL cr);
 6715   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6716 
 6717   ins_cost(150); // XXX
 6718   format %{ "addl    $dst, $src\t# int" %}
 6719   ins_encode %{
 6720     __ addl($dst$$Address, $src$$Register);
 6721   %}
 6722   ins_pipe(ialu_mem_reg);
 6723 %}
 6724 
 6725 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 6726 %{
 6727   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6728   effect(KILL cr);
 6729   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6730 
 6731 
 6732   ins_cost(125); // XXX
 6733   format %{ "addl    $dst, $src\t# int" %}
 6734   ins_encode %{
 6735     __ addl($dst$$Address, $src$$constant);
 6736   %}
 6737   ins_pipe(ialu_mem_imm);
 6738 %}
 6739 
 6740 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 6741 %{
 6742   predicate(UseIncDec);
 6743   match(Set dst (AddI dst src));
 6744   effect(KILL cr);
 6745 
 6746   format %{ "incl    $dst\t# int" %}
 6747   ins_encode %{
 6748     __ incrementl($dst$$Register);
 6749   %}
 6750   ins_pipe(ialu_reg);
 6751 %}
 6752 
 6753 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 6754 %{
 6755   predicate(UseIncDec);
 6756   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6757   effect(KILL cr);
 6758 
 6759   ins_cost(125); // XXX
 6760   format %{ "incl    $dst\t# int" %}
 6761   ins_encode %{
 6762     __ incrementl($dst$$Address);
 6763   %}
 6764   ins_pipe(ialu_mem_imm);
 6765 %}
 6766 
 6767 // XXX why does that use AddI
 6768 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 6769 %{
 6770   predicate(UseIncDec);
 6771   match(Set dst (AddI dst src));
 6772   effect(KILL cr);
 6773 
 6774   format %{ "decl    $dst\t# int" %}
 6775   ins_encode %{
 6776     __ decrementl($dst$$Register);
 6777   %}
 6778   ins_pipe(ialu_reg);
 6779 %}
 6780 
 6781 // XXX why does that use AddI
 6782 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 6783 %{
 6784   predicate(UseIncDec);
 6785   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6786   effect(KILL cr);
 6787 
 6788   ins_cost(125); // XXX
 6789   format %{ "decl    $dst\t# int" %}
 6790   ins_encode %{
 6791     __ decrementl($dst$$Address);
 6792   %}
 6793   ins_pipe(ialu_mem_imm);
 6794 %}
 6795 
 6796 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 6797 %{
 6798   predicate(VM_Version::supports_fast_2op_lea());
 6799   match(Set dst (AddI (LShiftI index scale) disp));
 6800 
 6801   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 6802   ins_encode %{
 6803     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6804     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6805   %}
 6806   ins_pipe(ialu_reg_reg);
 6807 %}
 6808 
 6809 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 6810 %{
 6811   predicate(VM_Version::supports_fast_3op_lea());
 6812   match(Set dst (AddI (AddI base index) disp));
 6813 
 6814   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 6815   ins_encode %{
 6816     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6817   %}
 6818   ins_pipe(ialu_reg_reg);
 6819 %}
 6820 
 6821 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 6822 %{
 6823   predicate(VM_Version::supports_fast_2op_lea());
 6824   match(Set dst (AddI base (LShiftI index scale)));
 6825 
 6826   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 6827   ins_encode %{
 6828     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6829     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6830   %}
 6831   ins_pipe(ialu_reg_reg);
 6832 %}
 6833 
 6834 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 6835 %{
 6836   predicate(VM_Version::supports_fast_3op_lea());
 6837   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 6838 
 6839   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 6840   ins_encode %{
 6841     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6842     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6843   %}
 6844   ins_pipe(ialu_reg_reg);
 6845 %}
 6846 
 6847 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 6848 %{
 6849   match(Set dst (AddL dst src));
 6850   effect(KILL cr);
 6851   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6852 
 6853   format %{ "addq    $dst, $src\t# long" %}
 6854   ins_encode %{
 6855     __ addq($dst$$Register, $src$$Register);
 6856   %}
 6857   ins_pipe(ialu_reg_reg);
 6858 %}
 6859 
 6860 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 6861 %{
 6862   match(Set dst (AddL dst src));
 6863   effect(KILL cr);
 6864   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6865 
 6866   format %{ "addq    $dst, $src\t# long" %}
 6867   ins_encode %{
 6868     __ addq($dst$$Register, $src$$constant);
 6869   %}
 6870   ins_pipe( ialu_reg );
 6871 %}
 6872 
 6873 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 6874 %{
 6875   match(Set dst (AddL dst (LoadL src)));
 6876   effect(KILL cr);
 6877   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6878 
 6879   ins_cost(150); // XXX
 6880   format %{ "addq    $dst, $src\t# long" %}
 6881   ins_encode %{
 6882     __ addq($dst$$Register, $src$$Address);
 6883   %}
 6884   ins_pipe(ialu_reg_mem);
 6885 %}
 6886 
 6887 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 6888 %{
 6889   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6890   effect(KILL cr);
 6891   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6892 
 6893   ins_cost(150); // XXX
 6894   format %{ "addq    $dst, $src\t# long" %}
 6895   ins_encode %{
 6896     __ addq($dst$$Address, $src$$Register);
 6897   %}
 6898   ins_pipe(ialu_mem_reg);
 6899 %}
 6900 
 6901 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 6902 %{
 6903   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6904   effect(KILL cr);
 6905   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6906 
 6907   ins_cost(125); // XXX
 6908   format %{ "addq    $dst, $src\t# long" %}
 6909   ins_encode %{
 6910     __ addq($dst$$Address, $src$$constant);
 6911   %}
 6912   ins_pipe(ialu_mem_imm);
 6913 %}
 6914 
 6915 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 6916 %{
 6917   predicate(UseIncDec);
 6918   match(Set dst (AddL dst src));
 6919   effect(KILL cr);
 6920 
 6921   format %{ "incq    $dst\t# long" %}
 6922   ins_encode %{
 6923     __ incrementq($dst$$Register);
 6924   %}
 6925   ins_pipe(ialu_reg);
 6926 %}
 6927 
 6928 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 6929 %{
 6930   predicate(UseIncDec);
 6931   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6932   effect(KILL cr);
 6933 
 6934   ins_cost(125); // XXX
 6935   format %{ "incq    $dst\t# long" %}
 6936   ins_encode %{
 6937     __ incrementq($dst$$Address);
 6938   %}
 6939   ins_pipe(ialu_mem_imm);
 6940 %}
 6941 
 6942 // XXX why does that use AddL
 6943 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 6944 %{
 6945   predicate(UseIncDec);
 6946   match(Set dst (AddL dst src));
 6947   effect(KILL cr);
 6948 
 6949   format %{ "decq    $dst\t# long" %}
 6950   ins_encode %{
 6951     __ decrementq($dst$$Register);
 6952   %}
 6953   ins_pipe(ialu_reg);
 6954 %}
 6955 
 6956 // XXX why does that use AddL
 6957 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 6958 %{
 6959   predicate(UseIncDec);
 6960   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6961   effect(KILL cr);
 6962 
 6963   ins_cost(125); // XXX
 6964   format %{ "decq    $dst\t# long" %}
 6965   ins_encode %{
 6966     __ decrementq($dst$$Address);
 6967   %}
 6968   ins_pipe(ialu_mem_imm);
 6969 %}
 6970 
 6971 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 6972 %{
 6973   predicate(VM_Version::supports_fast_2op_lea());
 6974   match(Set dst (AddL (LShiftL index scale) disp));
 6975 
 6976   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 6977   ins_encode %{
 6978     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6979     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6980   %}
 6981   ins_pipe(ialu_reg_reg);
 6982 %}
 6983 
 6984 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 6985 %{
 6986   predicate(VM_Version::supports_fast_3op_lea());
 6987   match(Set dst (AddL (AddL base index) disp));
 6988 
 6989   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 6990   ins_encode %{
 6991     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6992   %}
 6993   ins_pipe(ialu_reg_reg);
 6994 %}
 6995 
 6996 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 6997 %{
 6998   predicate(VM_Version::supports_fast_2op_lea());
 6999   match(Set dst (AddL base (LShiftL index scale)));
 7000 
 7001   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7002   ins_encode %{
 7003     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7004     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7005   %}
 7006   ins_pipe(ialu_reg_reg);
 7007 %}
 7008 
 7009 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7010 %{
 7011   predicate(VM_Version::supports_fast_3op_lea());
 7012   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7013 
 7014   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7015   ins_encode %{
 7016     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7017     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7018   %}
 7019   ins_pipe(ialu_reg_reg);
 7020 %}
 7021 
 7022 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7023 %{
 7024   match(Set dst (AddP dst src));
 7025   effect(KILL cr);
 7026   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7027 
 7028   format %{ "addq    $dst, $src\t# ptr" %}
 7029   ins_encode %{
 7030     __ addq($dst$$Register, $src$$Register);
 7031   %}
 7032   ins_pipe(ialu_reg_reg);
 7033 %}
 7034 
 7035 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7036 %{
 7037   match(Set dst (AddP dst src));
 7038   effect(KILL cr);
 7039   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7040 
 7041   format %{ "addq    $dst, $src\t# ptr" %}
 7042   ins_encode %{
 7043     __ addq($dst$$Register, $src$$constant);
 7044   %}
 7045   ins_pipe( ialu_reg );
 7046 %}
 7047 
 7048 // XXX addP mem ops ????
 7049 
 7050 instruct checkCastPP(rRegP dst)
 7051 %{
 7052   match(Set dst (CheckCastPP dst));
 7053 
 7054   size(0);
 7055   format %{ "# checkcastPP of $dst" %}
 7056   ins_encode(/* empty encoding */);
 7057   ins_pipe(empty);
 7058 %}
 7059 
 7060 instruct castPP(rRegP dst)
 7061 %{
 7062   match(Set dst (CastPP dst));
 7063 
 7064   size(0);
 7065   format %{ "# castPP of $dst" %}
 7066   ins_encode(/* empty encoding */);
 7067   ins_pipe(empty);
 7068 %}
 7069 
 7070 instruct castII(rRegI dst)
 7071 %{
 7072   match(Set dst (CastII dst));
 7073 
 7074   size(0);
 7075   format %{ "# castII of $dst" %}
 7076   ins_encode(/* empty encoding */);
 7077   ins_cost(0);
 7078   ins_pipe(empty);
 7079 %}
 7080 
 7081 instruct castLL(rRegL dst)
 7082 %{
 7083   match(Set dst (CastLL dst));
 7084 
 7085   size(0);
 7086   format %{ "# castLL of $dst" %}
 7087   ins_encode(/* empty encoding */);
 7088   ins_cost(0);
 7089   ins_pipe(empty);
 7090 %}
 7091 
 7092 instruct castFF(regF dst)
 7093 %{
 7094   match(Set dst (CastFF dst));
 7095 
 7096   size(0);
 7097   format %{ "# castFF of $dst" %}
 7098   ins_encode(/* empty encoding */);
 7099   ins_cost(0);
 7100   ins_pipe(empty);
 7101 %}
 7102 
 7103 instruct castDD(regD dst)
 7104 %{
 7105   match(Set dst (CastDD dst));
 7106 
 7107   size(0);
 7108   format %{ "# castDD of $dst" %}
 7109   ins_encode(/* empty encoding */);
 7110   ins_cost(0);
 7111   ins_pipe(empty);
 7112 %}
 7113 
 7114 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7115 instruct compareAndSwapP(rRegI res,
 7116                          memory mem_ptr,
 7117                          rax_RegP oldval, rRegP newval,
 7118                          rFlagsReg cr)
 7119 %{
 7120   predicate(n->as_LoadStore()->barrier_data() == 0);
 7121   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7122   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7123   effect(KILL cr, KILL oldval);
 7124 
 7125   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7126             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7127             "sete    $res\n\t"
 7128             "movzbl  $res, $res" %}
 7129   ins_encode %{
 7130     __ lock();
 7131     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7132     __ setb(Assembler::equal, $res$$Register);
 7133     __ movzbl($res$$Register, $res$$Register);
 7134   %}
 7135   ins_pipe( pipe_cmpxchg );
 7136 %}
 7137 
 7138 instruct compareAndSwapL(rRegI res,
 7139                          memory mem_ptr,
 7140                          rax_RegL oldval, rRegL newval,
 7141                          rFlagsReg cr)
 7142 %{
 7143   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7144   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7145   effect(KILL cr, KILL oldval);
 7146 
 7147   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7148             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7149             "sete    $res\n\t"
 7150             "movzbl  $res, $res" %}
 7151   ins_encode %{
 7152     __ lock();
 7153     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7154     __ setb(Assembler::equal, $res$$Register);
 7155     __ movzbl($res$$Register, $res$$Register);
 7156   %}
 7157   ins_pipe( pipe_cmpxchg );
 7158 %}
 7159 
 7160 instruct compareAndSwapI(rRegI res,
 7161                          memory mem_ptr,
 7162                          rax_RegI oldval, rRegI newval,
 7163                          rFlagsReg cr)
 7164 %{
 7165   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7166   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7167   effect(KILL cr, KILL oldval);
 7168 
 7169   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7170             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7171             "sete    $res\n\t"
 7172             "movzbl  $res, $res" %}
 7173   ins_encode %{
 7174     __ lock();
 7175     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7176     __ setb(Assembler::equal, $res$$Register);
 7177     __ movzbl($res$$Register, $res$$Register);
 7178   %}
 7179   ins_pipe( pipe_cmpxchg );
 7180 %}
 7181 
 7182 instruct compareAndSwapB(rRegI res,
 7183                          memory mem_ptr,
 7184                          rax_RegI oldval, rRegI newval,
 7185                          rFlagsReg cr)
 7186 %{
 7187   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7188   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7189   effect(KILL cr, KILL oldval);
 7190 
 7191   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7192             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7193             "sete    $res\n\t"
 7194             "movzbl  $res, $res" %}
 7195   ins_encode %{
 7196     __ lock();
 7197     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7198     __ setb(Assembler::equal, $res$$Register);
 7199     __ movzbl($res$$Register, $res$$Register);
 7200   %}
 7201   ins_pipe( pipe_cmpxchg );
 7202 %}
 7203 
 7204 instruct compareAndSwapS(rRegI res,
 7205                          memory mem_ptr,
 7206                          rax_RegI oldval, rRegI newval,
 7207                          rFlagsReg cr)
 7208 %{
 7209   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7210   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7211   effect(KILL cr, KILL oldval);
 7212 
 7213   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7214             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7215             "sete    $res\n\t"
 7216             "movzbl  $res, $res" %}
 7217   ins_encode %{
 7218     __ lock();
 7219     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7220     __ setb(Assembler::equal, $res$$Register);
 7221     __ movzbl($res$$Register, $res$$Register);
 7222   %}
 7223   ins_pipe( pipe_cmpxchg );
 7224 %}
 7225 
 7226 instruct compareAndSwapN(rRegI res,
 7227                           memory mem_ptr,
 7228                           rax_RegN oldval, rRegN newval,
 7229                           rFlagsReg cr) %{
 7230   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7231   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7232   effect(KILL cr, KILL oldval);
 7233 
 7234   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7235             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7236             "sete    $res\n\t"
 7237             "movzbl  $res, $res" %}
 7238   ins_encode %{
 7239     __ lock();
 7240     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7241     __ setb(Assembler::equal, $res$$Register);
 7242     __ movzbl($res$$Register, $res$$Register);
 7243   %}
 7244   ins_pipe( pipe_cmpxchg );
 7245 %}
 7246 
 7247 instruct compareAndExchangeB(
 7248                          memory mem_ptr,
 7249                          rax_RegI oldval, rRegI newval,
 7250                          rFlagsReg cr)
 7251 %{
 7252   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7253   effect(KILL cr);
 7254 
 7255   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7256             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7257   ins_encode %{
 7258     __ lock();
 7259     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7260   %}
 7261   ins_pipe( pipe_cmpxchg );
 7262 %}
 7263 
 7264 instruct compareAndExchangeS(
 7265                          memory mem_ptr,
 7266                          rax_RegI oldval, rRegI newval,
 7267                          rFlagsReg cr)
 7268 %{
 7269   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7270   effect(KILL cr);
 7271 
 7272   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7273             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7274   ins_encode %{
 7275     __ lock();
 7276     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7277   %}
 7278   ins_pipe( pipe_cmpxchg );
 7279 %}
 7280 
 7281 instruct compareAndExchangeI(
 7282                          memory mem_ptr,
 7283                          rax_RegI oldval, rRegI newval,
 7284                          rFlagsReg cr)
 7285 %{
 7286   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7287   effect(KILL cr);
 7288 
 7289   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7290             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7291   ins_encode %{
 7292     __ lock();
 7293     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7294   %}
 7295   ins_pipe( pipe_cmpxchg );
 7296 %}
 7297 
 7298 instruct compareAndExchangeL(
 7299                          memory mem_ptr,
 7300                          rax_RegL oldval, rRegL newval,
 7301                          rFlagsReg cr)
 7302 %{
 7303   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7304   effect(KILL cr);
 7305 
 7306   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7307             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7308   ins_encode %{
 7309     __ lock();
 7310     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7311   %}
 7312   ins_pipe( pipe_cmpxchg );
 7313 %}
 7314 
 7315 instruct compareAndExchangeN(
 7316                           memory mem_ptr,
 7317                           rax_RegN oldval, rRegN newval,
 7318                           rFlagsReg cr) %{
 7319   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7320   effect(KILL cr);
 7321 
 7322   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7323             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7324   ins_encode %{
 7325     __ lock();
 7326     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7327   %}
 7328   ins_pipe( pipe_cmpxchg );
 7329 %}
 7330 
 7331 instruct compareAndExchangeP(
 7332                          memory mem_ptr,
 7333                          rax_RegP oldval, rRegP newval,
 7334                          rFlagsReg cr)
 7335 %{
 7336   predicate(n->as_LoadStore()->barrier_data() == 0);
 7337   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7338   effect(KILL cr);
 7339 
 7340   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7341             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7342   ins_encode %{
 7343     __ lock();
 7344     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7345   %}
 7346   ins_pipe( pipe_cmpxchg );
 7347 %}
 7348 
 7349 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7350   predicate(n->as_LoadStore()->result_not_used());
 7351   match(Set dummy (GetAndAddB mem add));
 7352   effect(KILL cr);
 7353   format %{ "addb_lock   $mem, $add" %}
 7354   ins_encode %{
 7355     __ lock();
 7356     __ addb($mem$$Address, $add$$Register);
 7357   %}
 7358   ins_pipe(pipe_cmpxchg);
 7359 %}
 7360 
 7361 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7362   predicate(n->as_LoadStore()->result_not_used());
 7363   match(Set dummy (GetAndAddB mem add));
 7364   effect(KILL cr);
 7365   format %{ "addb_lock   $mem, $add" %}
 7366   ins_encode %{
 7367     __ lock();
 7368     __ addb($mem$$Address, $add$$constant);
 7369   %}
 7370   ins_pipe(pipe_cmpxchg);
 7371 %}
 7372 
 7373 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7374   predicate(!n->as_LoadStore()->result_not_used());
 7375   match(Set newval (GetAndAddB mem newval));
 7376   effect(KILL cr);
 7377   format %{ "xaddb_lock  $mem, $newval" %}
 7378   ins_encode %{
 7379     __ lock();
 7380     __ xaddb($mem$$Address, $newval$$Register);
 7381   %}
 7382   ins_pipe(pipe_cmpxchg);
 7383 %}
 7384 
 7385 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7386   predicate(n->as_LoadStore()->result_not_used());
 7387   match(Set dummy (GetAndAddS mem add));
 7388   effect(KILL cr);
 7389   format %{ "addw_lock   $mem, $add" %}
 7390   ins_encode %{
 7391     __ lock();
 7392     __ addw($mem$$Address, $add$$Register);
 7393   %}
 7394   ins_pipe(pipe_cmpxchg);
 7395 %}
 7396 
 7397 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7398   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7399   match(Set dummy (GetAndAddS mem add));
 7400   effect(KILL cr);
 7401   format %{ "addw_lock   $mem, $add" %}
 7402   ins_encode %{
 7403     __ lock();
 7404     __ addw($mem$$Address, $add$$constant);
 7405   %}
 7406   ins_pipe(pipe_cmpxchg);
 7407 %}
 7408 
 7409 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7410   predicate(!n->as_LoadStore()->result_not_used());
 7411   match(Set newval (GetAndAddS mem newval));
 7412   effect(KILL cr);
 7413   format %{ "xaddw_lock  $mem, $newval" %}
 7414   ins_encode %{
 7415     __ lock();
 7416     __ xaddw($mem$$Address, $newval$$Register);
 7417   %}
 7418   ins_pipe(pipe_cmpxchg);
 7419 %}
 7420 
 7421 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7422   predicate(n->as_LoadStore()->result_not_used());
 7423   match(Set dummy (GetAndAddI mem add));
 7424   effect(KILL cr);
 7425   format %{ "addl_lock   $mem, $add" %}
 7426   ins_encode %{
 7427     __ lock();
 7428     __ addl($mem$$Address, $add$$Register);
 7429   %}
 7430   ins_pipe(pipe_cmpxchg);
 7431 %}
 7432 
 7433 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7434   predicate(n->as_LoadStore()->result_not_used());
 7435   match(Set dummy (GetAndAddI mem add));
 7436   effect(KILL cr);
 7437   format %{ "addl_lock   $mem, $add" %}
 7438   ins_encode %{
 7439     __ lock();
 7440     __ addl($mem$$Address, $add$$constant);
 7441   %}
 7442   ins_pipe(pipe_cmpxchg);
 7443 %}
 7444 
 7445 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 7446   predicate(!n->as_LoadStore()->result_not_used());
 7447   match(Set newval (GetAndAddI mem newval));
 7448   effect(KILL cr);
 7449   format %{ "xaddl_lock  $mem, $newval" %}
 7450   ins_encode %{
 7451     __ lock();
 7452     __ xaddl($mem$$Address, $newval$$Register);
 7453   %}
 7454   ins_pipe(pipe_cmpxchg);
 7455 %}
 7456 
 7457 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 7458   predicate(n->as_LoadStore()->result_not_used());
 7459   match(Set dummy (GetAndAddL mem add));
 7460   effect(KILL cr);
 7461   format %{ "addq_lock   $mem, $add" %}
 7462   ins_encode %{
 7463     __ lock();
 7464     __ addq($mem$$Address, $add$$Register);
 7465   %}
 7466   ins_pipe(pipe_cmpxchg);
 7467 %}
 7468 
 7469 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 7470   predicate(n->as_LoadStore()->result_not_used());
 7471   match(Set dummy (GetAndAddL mem add));
 7472   effect(KILL cr);
 7473   format %{ "addq_lock   $mem, $add" %}
 7474   ins_encode %{
 7475     __ lock();
 7476     __ addq($mem$$Address, $add$$constant);
 7477   %}
 7478   ins_pipe(pipe_cmpxchg);
 7479 %}
 7480 
 7481 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 7482   predicate(!n->as_LoadStore()->result_not_used());
 7483   match(Set newval (GetAndAddL mem newval));
 7484   effect(KILL cr);
 7485   format %{ "xaddq_lock  $mem, $newval" %}
 7486   ins_encode %{
 7487     __ lock();
 7488     __ xaddq($mem$$Address, $newval$$Register);
 7489   %}
 7490   ins_pipe(pipe_cmpxchg);
 7491 %}
 7492 
 7493 instruct xchgB( memory mem, rRegI newval) %{
 7494   match(Set newval (GetAndSetB mem newval));
 7495   format %{ "XCHGB  $newval,[$mem]" %}
 7496   ins_encode %{
 7497     __ xchgb($newval$$Register, $mem$$Address);
 7498   %}
 7499   ins_pipe( pipe_cmpxchg );
 7500 %}
 7501 
 7502 instruct xchgS( memory mem, rRegI newval) %{
 7503   match(Set newval (GetAndSetS mem newval));
 7504   format %{ "XCHGW  $newval,[$mem]" %}
 7505   ins_encode %{
 7506     __ xchgw($newval$$Register, $mem$$Address);
 7507   %}
 7508   ins_pipe( pipe_cmpxchg );
 7509 %}
 7510 
 7511 instruct xchgI( memory mem, rRegI newval) %{
 7512   match(Set newval (GetAndSetI mem newval));
 7513   format %{ "XCHGL  $newval,[$mem]" %}
 7514   ins_encode %{
 7515     __ xchgl($newval$$Register, $mem$$Address);
 7516   %}
 7517   ins_pipe( pipe_cmpxchg );
 7518 %}
 7519 
 7520 instruct xchgL( memory mem, rRegL newval) %{
 7521   match(Set newval (GetAndSetL mem newval));
 7522   format %{ "XCHGL  $newval,[$mem]" %}
 7523   ins_encode %{
 7524     __ xchgq($newval$$Register, $mem$$Address);
 7525   %}
 7526   ins_pipe( pipe_cmpxchg );
 7527 %}
 7528 
 7529 instruct xchgP( memory mem, rRegP newval) %{
 7530   match(Set newval (GetAndSetP mem newval));
 7531   predicate(n->as_LoadStore()->barrier_data() == 0);
 7532   format %{ "XCHGQ  $newval,[$mem]" %}
 7533   ins_encode %{
 7534     __ xchgq($newval$$Register, $mem$$Address);
 7535   %}
 7536   ins_pipe( pipe_cmpxchg );
 7537 %}
 7538 
 7539 instruct xchgN( memory mem, rRegN newval) %{
 7540   match(Set newval (GetAndSetN mem newval));
 7541   format %{ "XCHGL  $newval,$mem]" %}
 7542   ins_encode %{
 7543     __ xchgl($newval$$Register, $mem$$Address);
 7544   %}
 7545   ins_pipe( pipe_cmpxchg );
 7546 %}
 7547 
 7548 //----------Abs Instructions-------------------------------------------
 7549 
 7550 // Integer Absolute Instructions
 7551 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7552 %{
 7553   match(Set dst (AbsI src));
 7554   effect(TEMP dst, KILL cr);
 7555   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 7556             "subl    $dst, $src\n\t"
 7557             "cmovll  $dst, $src" %}
 7558   ins_encode %{
 7559     __ xorl($dst$$Register, $dst$$Register);
 7560     __ subl($dst$$Register, $src$$Register);
 7561     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 7562   %}
 7563 
 7564   ins_pipe(ialu_reg_reg);
 7565 %}
 7566 
 7567 // Long Absolute Instructions
 7568 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7569 %{
 7570   match(Set dst (AbsL src));
 7571   effect(TEMP dst, KILL cr);
 7572   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 7573             "subq    $dst, $src\n\t"
 7574             "cmovlq  $dst, $src" %}
 7575   ins_encode %{
 7576     __ xorl($dst$$Register, $dst$$Register);
 7577     __ subq($dst$$Register, $src$$Register);
 7578     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 7579   %}
 7580 
 7581   ins_pipe(ialu_reg_reg);
 7582 %}
 7583 
 7584 //----------Subtraction Instructions-------------------------------------------
 7585 
 7586 // Integer Subtraction Instructions
 7587 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7588 %{
 7589   match(Set dst (SubI dst src));
 7590   effect(KILL cr);
 7591   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7592 
 7593   format %{ "subl    $dst, $src\t# int" %}
 7594   ins_encode %{
 7595     __ subl($dst$$Register, $src$$Register);
 7596   %}
 7597   ins_pipe(ialu_reg_reg);
 7598 %}
 7599 
 7600 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7601 %{
 7602   match(Set dst (SubI dst (LoadI src)));
 7603   effect(KILL cr);
 7604   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7605 
 7606   ins_cost(150);
 7607   format %{ "subl    $dst, $src\t# int" %}
 7608   ins_encode %{
 7609     __ subl($dst$$Register, $src$$Address);
 7610   %}
 7611   ins_pipe(ialu_reg_mem);
 7612 %}
 7613 
 7614 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7615 %{
 7616   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 7617   effect(KILL cr);
 7618   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7619 
 7620   ins_cost(150);
 7621   format %{ "subl    $dst, $src\t# int" %}
 7622   ins_encode %{
 7623     __ subl($dst$$Address, $src$$Register);
 7624   %}
 7625   ins_pipe(ialu_mem_reg);
 7626 %}
 7627 
 7628 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7629 %{
 7630   match(Set dst (SubL dst src));
 7631   effect(KILL cr);
 7632   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7633 
 7634   format %{ "subq    $dst, $src\t# long" %}
 7635   ins_encode %{
 7636     __ subq($dst$$Register, $src$$Register);
 7637   %}
 7638   ins_pipe(ialu_reg_reg);
 7639 %}
 7640 
 7641 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7642 %{
 7643   match(Set dst (SubL dst (LoadL src)));
 7644   effect(KILL cr);
 7645   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7646 
 7647   ins_cost(150);
 7648   format %{ "subq    $dst, $src\t# long" %}
 7649   ins_encode %{
 7650     __ subq($dst$$Register, $src$$Address);
 7651   %}
 7652   ins_pipe(ialu_reg_mem);
 7653 %}
 7654 
 7655 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7656 %{
 7657   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 7658   effect(KILL cr);
 7659   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7660 
 7661   ins_cost(150);
 7662   format %{ "subq    $dst, $src\t# long" %}
 7663   ins_encode %{
 7664     __ subq($dst$$Address, $src$$Register);
 7665   %}
 7666   ins_pipe(ialu_mem_reg);
 7667 %}
 7668 
 7669 // Subtract from a pointer
 7670 // XXX hmpf???
 7671 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 7672 %{
 7673   match(Set dst (AddP dst (SubI zero src)));
 7674   effect(KILL cr);
 7675 
 7676   format %{ "subq    $dst, $src\t# ptr - int" %}
 7677   ins_encode %{
 7678     __ subq($dst$$Register, $src$$Register);
 7679   %}
 7680   ins_pipe(ialu_reg_reg);
 7681 %}
 7682 
 7683 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 7684 %{
 7685   match(Set dst (SubI zero dst));
 7686   effect(KILL cr);
 7687   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7688 
 7689   format %{ "negl    $dst\t# int" %}
 7690   ins_encode %{
 7691     __ negl($dst$$Register);
 7692   %}
 7693   ins_pipe(ialu_reg);
 7694 %}
 7695 
 7696 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 7697 %{
 7698   match(Set dst (NegI dst));
 7699   effect(KILL cr);
 7700   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7701 
 7702   format %{ "negl    $dst\t# int" %}
 7703   ins_encode %{
 7704     __ negl($dst$$Register);
 7705   %}
 7706   ins_pipe(ialu_reg);
 7707 %}
 7708 
 7709 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 7710 %{
 7711   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 7712   effect(KILL cr);
 7713   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7714 
 7715   format %{ "negl    $dst\t# int" %}
 7716   ins_encode %{
 7717     __ negl($dst$$Address);
 7718   %}
 7719   ins_pipe(ialu_reg);
 7720 %}
 7721 
 7722 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 7723 %{
 7724   match(Set dst (SubL zero dst));
 7725   effect(KILL cr);
 7726   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7727 
 7728   format %{ "negq    $dst\t# long" %}
 7729   ins_encode %{
 7730     __ negq($dst$$Register);
 7731   %}
 7732   ins_pipe(ialu_reg);
 7733 %}
 7734 
 7735 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 7736 %{
 7737   match(Set dst (NegL dst));
 7738   effect(KILL cr);
 7739   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7740 
 7741   format %{ "negq    $dst\t# int" %}
 7742   ins_encode %{
 7743     __ negq($dst$$Register);
 7744   %}
 7745   ins_pipe(ialu_reg);
 7746 %}
 7747 
 7748 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 7749 %{
 7750   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 7751   effect(KILL cr);
 7752   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7753 
 7754   format %{ "negq    $dst\t# long" %}
 7755   ins_encode %{
 7756     __ negq($dst$$Address);
 7757   %}
 7758   ins_pipe(ialu_reg);
 7759 %}
 7760 
 7761 //----------Multiplication/Division Instructions-------------------------------
 7762 // Integer Multiplication Instructions
 7763 // Multiply Register
 7764 
 7765 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7766 %{
 7767   match(Set dst (MulI dst src));
 7768   effect(KILL cr);
 7769 
 7770   ins_cost(300);
 7771   format %{ "imull   $dst, $src\t# int" %}
 7772   ins_encode %{
 7773     __ imull($dst$$Register, $src$$Register);
 7774   %}
 7775   ins_pipe(ialu_reg_reg_alu0);
 7776 %}
 7777 
 7778 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 7779 %{
 7780   match(Set dst (MulI src imm));
 7781   effect(KILL cr);
 7782 
 7783   ins_cost(300);
 7784   format %{ "imull   $dst, $src, $imm\t# int" %}
 7785   ins_encode %{
 7786     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 7787   %}
 7788   ins_pipe(ialu_reg_reg_alu0);
 7789 %}
 7790 
 7791 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 7792 %{
 7793   match(Set dst (MulI dst (LoadI src)));
 7794   effect(KILL cr);
 7795 
 7796   ins_cost(350);
 7797   format %{ "imull   $dst, $src\t# int" %}
 7798   ins_encode %{
 7799     __ imull($dst$$Register, $src$$Address);
 7800   %}
 7801   ins_pipe(ialu_reg_mem_alu0);
 7802 %}
 7803 
 7804 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 7805 %{
 7806   match(Set dst (MulI (LoadI src) imm));
 7807   effect(KILL cr);
 7808 
 7809   ins_cost(300);
 7810   format %{ "imull   $dst, $src, $imm\t# int" %}
 7811   ins_encode %{
 7812     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 7813   %}
 7814   ins_pipe(ialu_reg_mem_alu0);
 7815 %}
 7816 
 7817 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 7818 %{
 7819   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 7820   effect(KILL cr, KILL src2);
 7821 
 7822   expand %{ mulI_rReg(dst, src1, cr);
 7823            mulI_rReg(src2, src3, cr);
 7824            addI_rReg(dst, src2, cr); %}
 7825 %}
 7826 
 7827 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7828 %{
 7829   match(Set dst (MulL dst src));
 7830   effect(KILL cr);
 7831 
 7832   ins_cost(300);
 7833   format %{ "imulq   $dst, $src\t# long" %}
 7834   ins_encode %{
 7835     __ imulq($dst$$Register, $src$$Register);
 7836   %}
 7837   ins_pipe(ialu_reg_reg_alu0);
 7838 %}
 7839 
 7840 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 7841 %{
 7842   match(Set dst (MulL src imm));
 7843   effect(KILL cr);
 7844 
 7845   ins_cost(300);
 7846   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7847   ins_encode %{
 7848     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 7849   %}
 7850   ins_pipe(ialu_reg_reg_alu0);
 7851 %}
 7852 
 7853 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 7854 %{
 7855   match(Set dst (MulL dst (LoadL src)));
 7856   effect(KILL cr);
 7857 
 7858   ins_cost(350);
 7859   format %{ "imulq   $dst, $src\t# long" %}
 7860   ins_encode %{
 7861     __ imulq($dst$$Register, $src$$Address);
 7862   %}
 7863   ins_pipe(ialu_reg_mem_alu0);
 7864 %}
 7865 
 7866 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 7867 %{
 7868   match(Set dst (MulL (LoadL src) imm));
 7869   effect(KILL cr);
 7870 
 7871   ins_cost(300);
 7872   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7873   ins_encode %{
 7874     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 7875   %}
 7876   ins_pipe(ialu_reg_mem_alu0);
 7877 %}
 7878 
 7879 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7880 %{
 7881   match(Set dst (MulHiL src rax));
 7882   effect(USE_KILL rax, KILL cr);
 7883 
 7884   ins_cost(300);
 7885   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 7886   ins_encode %{
 7887     __ imulq($src$$Register);
 7888   %}
 7889   ins_pipe(ialu_reg_reg_alu0);
 7890 %}
 7891 
 7892 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7893 %{
 7894   match(Set dst (UMulHiL src rax));
 7895   effect(USE_KILL rax, KILL cr);
 7896 
 7897   ins_cost(300);
 7898   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 7899   ins_encode %{
 7900     __ mulq($src$$Register);
 7901   %}
 7902   ins_pipe(ialu_reg_reg_alu0);
 7903 %}
 7904 
 7905 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7906                    rFlagsReg cr)
 7907 %{
 7908   match(Set rax (DivI rax div));
 7909   effect(KILL rdx, KILL cr);
 7910 
 7911   ins_cost(30*100+10*100); // XXX
 7912   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7913             "jne,s   normal\n\t"
 7914             "xorl    rdx, rdx\n\t"
 7915             "cmpl    $div, -1\n\t"
 7916             "je,s    done\n"
 7917     "normal: cdql\n\t"
 7918             "idivl   $div\n"
 7919     "done:"        %}
 7920   ins_encode(cdql_enc(div));
 7921   ins_pipe(ialu_reg_reg_alu0);
 7922 %}
 7923 
 7924 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7925                    rFlagsReg cr)
 7926 %{
 7927   match(Set rax (DivL rax div));
 7928   effect(KILL rdx, KILL cr);
 7929 
 7930   ins_cost(30*100+10*100); // XXX
 7931   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7932             "cmpq    rax, rdx\n\t"
 7933             "jne,s   normal\n\t"
 7934             "xorl    rdx, rdx\n\t"
 7935             "cmpq    $div, -1\n\t"
 7936             "je,s    done\n"
 7937     "normal: cdqq\n\t"
 7938             "idivq   $div\n"
 7939     "done:"        %}
 7940   ins_encode(cdqq_enc(div));
 7941   ins_pipe(ialu_reg_reg_alu0);
 7942 %}
 7943 
 7944 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 7945 %{
 7946   match(Set rax (UDivI rax div));
 7947   effect(KILL rdx, KILL cr);
 7948 
 7949   ins_cost(300);
 7950   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 7951   ins_encode %{
 7952     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 7953   %}
 7954   ins_pipe(ialu_reg_reg_alu0);
 7955 %}
 7956 
 7957 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 7958 %{
 7959   match(Set rax (UDivL rax div));
 7960   effect(KILL rdx, KILL cr);
 7961 
 7962   ins_cost(300);
 7963   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 7964   ins_encode %{
 7965      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 7966   %}
 7967   ins_pipe(ialu_reg_reg_alu0);
 7968 %}
 7969 
 7970 // Integer DIVMOD with Register, both quotient and mod results
 7971 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7972                              rFlagsReg cr)
 7973 %{
 7974   match(DivModI rax div);
 7975   effect(KILL cr);
 7976 
 7977   ins_cost(30*100+10*100); // XXX
 7978   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7979             "jne,s   normal\n\t"
 7980             "xorl    rdx, rdx\n\t"
 7981             "cmpl    $div, -1\n\t"
 7982             "je,s    done\n"
 7983     "normal: cdql\n\t"
 7984             "idivl   $div\n"
 7985     "done:"        %}
 7986   ins_encode(cdql_enc(div));
 7987   ins_pipe(pipe_slow);
 7988 %}
 7989 
 7990 // Long DIVMOD with Register, both quotient and mod results
 7991 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7992                              rFlagsReg cr)
 7993 %{
 7994   match(DivModL rax div);
 7995   effect(KILL cr);
 7996 
 7997   ins_cost(30*100+10*100); // XXX
 7998   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7999             "cmpq    rax, rdx\n\t"
 8000             "jne,s   normal\n\t"
 8001             "xorl    rdx, rdx\n\t"
 8002             "cmpq    $div, -1\n\t"
 8003             "je,s    done\n"
 8004     "normal: cdqq\n\t"
 8005             "idivq   $div\n"
 8006     "done:"        %}
 8007   ins_encode(cdqq_enc(div));
 8008   ins_pipe(pipe_slow);
 8009 %}
 8010 
 8011 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8012 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8013                               no_rax_rdx_RegI div, rFlagsReg cr)
 8014 %{
 8015   match(UDivModI rax div);
 8016   effect(TEMP tmp, KILL cr);
 8017 
 8018   ins_cost(300);
 8019   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8020             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8021           %}
 8022   ins_encode %{
 8023     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8024   %}
 8025   ins_pipe(pipe_slow);
 8026 %}
 8027 
 8028 // Unsigned long DIVMOD with Register, both quotient and mod results
 8029 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8030                               no_rax_rdx_RegL div, rFlagsReg cr)
 8031 %{
 8032   match(UDivModL rax div);
 8033   effect(TEMP tmp, KILL cr);
 8034 
 8035   ins_cost(300);
 8036   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8037             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8038           %}
 8039   ins_encode %{
 8040     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8041   %}
 8042   ins_pipe(pipe_slow);
 8043 %}
 8044 
 8045 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8046                    rFlagsReg cr)
 8047 %{
 8048   match(Set rdx (ModI rax div));
 8049   effect(KILL rax, KILL cr);
 8050 
 8051   ins_cost(300); // XXX
 8052   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8053             "jne,s   normal\n\t"
 8054             "xorl    rdx, rdx\n\t"
 8055             "cmpl    $div, -1\n\t"
 8056             "je,s    done\n"
 8057     "normal: cdql\n\t"
 8058             "idivl   $div\n"
 8059     "done:"        %}
 8060   ins_encode(cdql_enc(div));
 8061   ins_pipe(ialu_reg_reg_alu0);
 8062 %}
 8063 
 8064 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8065                    rFlagsReg cr)
 8066 %{
 8067   match(Set rdx (ModL rax div));
 8068   effect(KILL rax, KILL cr);
 8069 
 8070   ins_cost(300); // XXX
 8071   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8072             "cmpq    rax, rdx\n\t"
 8073             "jne,s   normal\n\t"
 8074             "xorl    rdx, rdx\n\t"
 8075             "cmpq    $div, -1\n\t"
 8076             "je,s    done\n"
 8077     "normal: cdqq\n\t"
 8078             "idivq   $div\n"
 8079     "done:"        %}
 8080   ins_encode(cdqq_enc(div));
 8081   ins_pipe(ialu_reg_reg_alu0);
 8082 %}
 8083 
 8084 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8085 %{
 8086   match(Set rdx (UModI rax div));
 8087   effect(KILL rax, KILL cr);
 8088 
 8089   ins_cost(300);
 8090   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8091   ins_encode %{
 8092     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8093   %}
 8094   ins_pipe(ialu_reg_reg_alu0);
 8095 %}
 8096 
 8097 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8098 %{
 8099   match(Set rdx (UModL rax div));
 8100   effect(KILL rax, KILL cr);
 8101 
 8102   ins_cost(300);
 8103   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8104   ins_encode %{
 8105     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8106   %}
 8107   ins_pipe(ialu_reg_reg_alu0);
 8108 %}
 8109 
 8110 // Integer Shift Instructions
 8111 // Shift Left by one, two, three
 8112 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8113 %{
 8114   match(Set dst (LShiftI dst shift));
 8115   effect(KILL cr);
 8116 
 8117   format %{ "sall    $dst, $shift" %}
 8118   ins_encode %{
 8119     __ sall($dst$$Register, $shift$$constant);
 8120   %}
 8121   ins_pipe(ialu_reg);
 8122 %}
 8123 
 8124 // Shift Left by 8-bit immediate
 8125 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8126 %{
 8127   match(Set dst (LShiftI dst shift));
 8128   effect(KILL cr);
 8129 
 8130   format %{ "sall    $dst, $shift" %}
 8131   ins_encode %{
 8132     __ sall($dst$$Register, $shift$$constant);
 8133   %}
 8134   ins_pipe(ialu_reg);
 8135 %}
 8136 
 8137 // Shift Left by 8-bit immediate
 8138 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8139 %{
 8140   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8141   effect(KILL cr);
 8142 
 8143   format %{ "sall    $dst, $shift" %}
 8144   ins_encode %{
 8145     __ sall($dst$$Address, $shift$$constant);
 8146   %}
 8147   ins_pipe(ialu_mem_imm);
 8148 %}
 8149 
 8150 // Shift Left by variable
 8151 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8152 %{
 8153   predicate(!VM_Version::supports_bmi2());
 8154   match(Set dst (LShiftI dst shift));
 8155   effect(KILL cr);
 8156 
 8157   format %{ "sall    $dst, $shift" %}
 8158   ins_encode %{
 8159     __ sall($dst$$Register);
 8160   %}
 8161   ins_pipe(ialu_reg_reg);
 8162 %}
 8163 
 8164 // Shift Left by variable
 8165 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8166 %{
 8167   predicate(!VM_Version::supports_bmi2());
 8168   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8169   effect(KILL cr);
 8170 
 8171   format %{ "sall    $dst, $shift" %}
 8172   ins_encode %{
 8173     __ sall($dst$$Address);
 8174   %}
 8175   ins_pipe(ialu_mem_reg);
 8176 %}
 8177 
 8178 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8179 %{
 8180   predicate(VM_Version::supports_bmi2());
 8181   match(Set dst (LShiftI src shift));
 8182 
 8183   format %{ "shlxl   $dst, $src, $shift" %}
 8184   ins_encode %{
 8185     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 8186   %}
 8187   ins_pipe(ialu_reg_reg);
 8188 %}
 8189 
 8190 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8191 %{
 8192   predicate(VM_Version::supports_bmi2());
 8193   match(Set dst (LShiftI (LoadI src) shift));
 8194   ins_cost(175);
 8195   format %{ "shlxl   $dst, $src, $shift" %}
 8196   ins_encode %{
 8197     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 8198   %}
 8199   ins_pipe(ialu_reg_mem);
 8200 %}
 8201 
 8202 // Arithmetic Shift Right by 8-bit immediate
 8203 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8204 %{
 8205   match(Set dst (RShiftI dst shift));
 8206   effect(KILL cr);
 8207 
 8208   format %{ "sarl    $dst, $shift" %}
 8209   ins_encode %{
 8210     __ sarl($dst$$Register, $shift$$constant);
 8211   %}
 8212   ins_pipe(ialu_mem_imm);
 8213 %}
 8214 
 8215 // Arithmetic Shift Right by 8-bit immediate
 8216 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8217 %{
 8218   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8219   effect(KILL cr);
 8220 
 8221   format %{ "sarl    $dst, $shift" %}
 8222   ins_encode %{
 8223     __ sarl($dst$$Address, $shift$$constant);
 8224   %}
 8225   ins_pipe(ialu_mem_imm);
 8226 %}
 8227 
 8228 // Arithmetic Shift Right by variable
 8229 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8230 %{
 8231   predicate(!VM_Version::supports_bmi2());
 8232   match(Set dst (RShiftI dst shift));
 8233   effect(KILL cr);
 8234 
 8235   format %{ "sarl    $dst, $shift" %}
 8236   ins_encode %{
 8237     __ sarl($dst$$Register);
 8238   %}
 8239   ins_pipe(ialu_reg_reg);
 8240 %}
 8241 
 8242 // Arithmetic Shift Right by variable
 8243 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8244 %{
 8245   predicate(!VM_Version::supports_bmi2());
 8246   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8247   effect(KILL cr);
 8248 
 8249   format %{ "sarl    $dst, $shift" %}
 8250   ins_encode %{
 8251     __ sarl($dst$$Address);
 8252   %}
 8253   ins_pipe(ialu_mem_reg);
 8254 %}
 8255 
 8256 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8257 %{
 8258   predicate(VM_Version::supports_bmi2());
 8259   match(Set dst (RShiftI src shift));
 8260 
 8261   format %{ "sarxl   $dst, $src, $shift" %}
 8262   ins_encode %{
 8263     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 8264   %}
 8265   ins_pipe(ialu_reg_reg);
 8266 %}
 8267 
 8268 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8269 %{
 8270   predicate(VM_Version::supports_bmi2());
 8271   match(Set dst (RShiftI (LoadI src) shift));
 8272   ins_cost(175);
 8273   format %{ "sarxl   $dst, $src, $shift" %}
 8274   ins_encode %{
 8275     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 8276   %}
 8277   ins_pipe(ialu_reg_mem);
 8278 %}
 8279 
 8280 // Logical Shift Right by 8-bit immediate
 8281 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8282 %{
 8283   match(Set dst (URShiftI dst shift));
 8284   effect(KILL cr);
 8285 
 8286   format %{ "shrl    $dst, $shift" %}
 8287   ins_encode %{
 8288     __ shrl($dst$$Register, $shift$$constant);
 8289   %}
 8290   ins_pipe(ialu_reg);
 8291 %}
 8292 
 8293 // Logical Shift Right by 8-bit immediate
 8294 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8295 %{
 8296   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8297   effect(KILL cr);
 8298 
 8299   format %{ "shrl    $dst, $shift" %}
 8300   ins_encode %{
 8301     __ shrl($dst$$Address, $shift$$constant);
 8302   %}
 8303   ins_pipe(ialu_mem_imm);
 8304 %}
 8305 
 8306 // Logical Shift Right by variable
 8307 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8308 %{
 8309   predicate(!VM_Version::supports_bmi2());
 8310   match(Set dst (URShiftI dst shift));
 8311   effect(KILL cr);
 8312 
 8313   format %{ "shrl    $dst, $shift" %}
 8314   ins_encode %{
 8315     __ shrl($dst$$Register);
 8316   %}
 8317   ins_pipe(ialu_reg_reg);
 8318 %}
 8319 
 8320 // Logical Shift Right by variable
 8321 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8322 %{
 8323   predicate(!VM_Version::supports_bmi2());
 8324   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8325   effect(KILL cr);
 8326 
 8327   format %{ "shrl    $dst, $shift" %}
 8328   ins_encode %{
 8329     __ shrl($dst$$Address);
 8330   %}
 8331   ins_pipe(ialu_mem_reg);
 8332 %}
 8333 
 8334 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8335 %{
 8336   predicate(VM_Version::supports_bmi2());
 8337   match(Set dst (URShiftI src shift));
 8338 
 8339   format %{ "shrxl   $dst, $src, $shift" %}
 8340   ins_encode %{
 8341     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 8342   %}
 8343   ins_pipe(ialu_reg_reg);
 8344 %}
 8345 
 8346 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8347 %{
 8348   predicate(VM_Version::supports_bmi2());
 8349   match(Set dst (URShiftI (LoadI src) shift));
 8350   ins_cost(175);
 8351   format %{ "shrxl   $dst, $src, $shift" %}
 8352   ins_encode %{
 8353     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 8354   %}
 8355   ins_pipe(ialu_reg_mem);
 8356 %}
 8357 
 8358 // Long Shift Instructions
 8359 // Shift Left by one, two, three
 8360 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 8361 %{
 8362   match(Set dst (LShiftL dst shift));
 8363   effect(KILL cr);
 8364 
 8365   format %{ "salq    $dst, $shift" %}
 8366   ins_encode %{
 8367     __ salq($dst$$Register, $shift$$constant);
 8368   %}
 8369   ins_pipe(ialu_reg);
 8370 %}
 8371 
 8372 // Shift Left by 8-bit immediate
 8373 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8374 %{
 8375   match(Set dst (LShiftL dst shift));
 8376   effect(KILL cr);
 8377 
 8378   format %{ "salq    $dst, $shift" %}
 8379   ins_encode %{
 8380     __ salq($dst$$Register, $shift$$constant);
 8381   %}
 8382   ins_pipe(ialu_reg);
 8383 %}
 8384 
 8385 // Shift Left by 8-bit immediate
 8386 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8387 %{
 8388   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8389   effect(KILL cr);
 8390 
 8391   format %{ "salq    $dst, $shift" %}
 8392   ins_encode %{
 8393     __ salq($dst$$Address, $shift$$constant);
 8394   %}
 8395   ins_pipe(ialu_mem_imm);
 8396 %}
 8397 
 8398 // Shift Left by variable
 8399 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8400 %{
 8401   predicate(!VM_Version::supports_bmi2());
 8402   match(Set dst (LShiftL dst shift));
 8403   effect(KILL cr);
 8404 
 8405   format %{ "salq    $dst, $shift" %}
 8406   ins_encode %{
 8407     __ salq($dst$$Register);
 8408   %}
 8409   ins_pipe(ialu_reg_reg);
 8410 %}
 8411 
 8412 // Shift Left by variable
 8413 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8414 %{
 8415   predicate(!VM_Version::supports_bmi2());
 8416   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8417   effect(KILL cr);
 8418 
 8419   format %{ "salq    $dst, $shift" %}
 8420   ins_encode %{
 8421     __ salq($dst$$Address);
 8422   %}
 8423   ins_pipe(ialu_mem_reg);
 8424 %}
 8425 
 8426 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8427 %{
 8428   predicate(VM_Version::supports_bmi2());
 8429   match(Set dst (LShiftL src shift));
 8430 
 8431   format %{ "shlxq   $dst, $src, $shift" %}
 8432   ins_encode %{
 8433     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 8434   %}
 8435   ins_pipe(ialu_reg_reg);
 8436 %}
 8437 
 8438 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8439 %{
 8440   predicate(VM_Version::supports_bmi2());
 8441   match(Set dst (LShiftL (LoadL src) shift));
 8442   ins_cost(175);
 8443   format %{ "shlxq   $dst, $src, $shift" %}
 8444   ins_encode %{
 8445     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 8446   %}
 8447   ins_pipe(ialu_reg_mem);
 8448 %}
 8449 
 8450 // Arithmetic Shift Right by 8-bit immediate
 8451 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 8452 %{
 8453   match(Set dst (RShiftL dst shift));
 8454   effect(KILL cr);
 8455 
 8456   format %{ "sarq    $dst, $shift" %}
 8457   ins_encode %{
 8458     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 8459   %}
 8460   ins_pipe(ialu_mem_imm);
 8461 %}
 8462 
 8463 // Arithmetic Shift Right by 8-bit immediate
 8464 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 8465 %{
 8466   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8467   effect(KILL cr);
 8468 
 8469   format %{ "sarq    $dst, $shift" %}
 8470   ins_encode %{
 8471     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 8472   %}
 8473   ins_pipe(ialu_mem_imm);
 8474 %}
 8475 
 8476 // Arithmetic Shift Right by variable
 8477 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8478 %{
 8479   predicate(!VM_Version::supports_bmi2());
 8480   match(Set dst (RShiftL dst shift));
 8481   effect(KILL cr);
 8482 
 8483   format %{ "sarq    $dst, $shift" %}
 8484   ins_encode %{
 8485     __ sarq($dst$$Register);
 8486   %}
 8487   ins_pipe(ialu_reg_reg);
 8488 %}
 8489 
 8490 // Arithmetic Shift Right by variable
 8491 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8492 %{
 8493   predicate(!VM_Version::supports_bmi2());
 8494   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8495   effect(KILL cr);
 8496 
 8497   format %{ "sarq    $dst, $shift" %}
 8498   ins_encode %{
 8499     __ sarq($dst$$Address);
 8500   %}
 8501   ins_pipe(ialu_mem_reg);
 8502 %}
 8503 
 8504 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8505 %{
 8506   predicate(VM_Version::supports_bmi2());
 8507   match(Set dst (RShiftL src shift));
 8508 
 8509   format %{ "sarxq   $dst, $src, $shift" %}
 8510   ins_encode %{
 8511     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 8512   %}
 8513   ins_pipe(ialu_reg_reg);
 8514 %}
 8515 
 8516 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8517 %{
 8518   predicate(VM_Version::supports_bmi2());
 8519   match(Set dst (RShiftL (LoadL src) shift));
 8520   ins_cost(175);
 8521   format %{ "sarxq   $dst, $src, $shift" %}
 8522   ins_encode %{
 8523     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 8524   %}
 8525   ins_pipe(ialu_reg_mem);
 8526 %}
 8527 
 8528 // Logical Shift Right by 8-bit immediate
 8529 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8530 %{
 8531   match(Set dst (URShiftL dst shift));
 8532   effect(KILL cr);
 8533 
 8534   format %{ "shrq    $dst, $shift" %}
 8535   ins_encode %{
 8536     __ shrq($dst$$Register, $shift$$constant);
 8537   %}
 8538   ins_pipe(ialu_reg);
 8539 %}
 8540 
 8541 // Logical Shift Right by 8-bit immediate
 8542 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8543 %{
 8544   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8545   effect(KILL cr);
 8546 
 8547   format %{ "shrq    $dst, $shift" %}
 8548   ins_encode %{
 8549     __ shrq($dst$$Address, $shift$$constant);
 8550   %}
 8551   ins_pipe(ialu_mem_imm);
 8552 %}
 8553 
 8554 // Logical Shift Right by variable
 8555 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8556 %{
 8557   predicate(!VM_Version::supports_bmi2());
 8558   match(Set dst (URShiftL dst shift));
 8559   effect(KILL cr);
 8560 
 8561   format %{ "shrq    $dst, $shift" %}
 8562   ins_encode %{
 8563     __ shrq($dst$$Register);
 8564   %}
 8565   ins_pipe(ialu_reg_reg);
 8566 %}
 8567 
 8568 // Logical Shift Right by variable
 8569 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8570 %{
 8571   predicate(!VM_Version::supports_bmi2());
 8572   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8573   effect(KILL cr);
 8574 
 8575   format %{ "shrq    $dst, $shift" %}
 8576   ins_encode %{
 8577     __ shrq($dst$$Address);
 8578   %}
 8579   ins_pipe(ialu_mem_reg);
 8580 %}
 8581 
 8582 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8583 %{
 8584   predicate(VM_Version::supports_bmi2());
 8585   match(Set dst (URShiftL src shift));
 8586 
 8587   format %{ "shrxq   $dst, $src, $shift" %}
 8588   ins_encode %{
 8589     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 8590   %}
 8591   ins_pipe(ialu_reg_reg);
 8592 %}
 8593 
 8594 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8595 %{
 8596   predicate(VM_Version::supports_bmi2());
 8597   match(Set dst (URShiftL (LoadL src) shift));
 8598   ins_cost(175);
 8599   format %{ "shrxq   $dst, $src, $shift" %}
 8600   ins_encode %{
 8601     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 8602   %}
 8603   ins_pipe(ialu_reg_mem);
 8604 %}
 8605 
 8606 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 8607 // This idiom is used by the compiler for the i2b bytecode.
 8608 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 8609 %{
 8610   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 8611 
 8612   format %{ "movsbl  $dst, $src\t# i2b" %}
 8613   ins_encode %{
 8614     __ movsbl($dst$$Register, $src$$Register);
 8615   %}
 8616   ins_pipe(ialu_reg_reg);
 8617 %}
 8618 
 8619 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 8620 // This idiom is used by the compiler the i2s bytecode.
 8621 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 8622 %{
 8623   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 8624 
 8625   format %{ "movswl  $dst, $src\t# i2s" %}
 8626   ins_encode %{
 8627     __ movswl($dst$$Register, $src$$Register);
 8628   %}
 8629   ins_pipe(ialu_reg_reg);
 8630 %}
 8631 
 8632 // ROL/ROR instructions
 8633 
 8634 // Rotate left by constant.
 8635 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8636 %{
 8637   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8638   match(Set dst (RotateLeft dst shift));
 8639   effect(KILL cr);
 8640   format %{ "roll    $dst, $shift" %}
 8641   ins_encode %{
 8642     __ roll($dst$$Register, $shift$$constant);
 8643   %}
 8644   ins_pipe(ialu_reg);
 8645 %}
 8646 
 8647 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 8648 %{
 8649   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8650   match(Set dst (RotateLeft src shift));
 8651   format %{ "rolxl   $dst, $src, $shift" %}
 8652   ins_encode %{
 8653     int shift = 32 - ($shift$$constant & 31);
 8654     __ rorxl($dst$$Register, $src$$Register, shift);
 8655   %}
 8656   ins_pipe(ialu_reg_reg);
 8657 %}
 8658 
 8659 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8660 %{
 8661   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8662   match(Set dst (RotateLeft (LoadI src) shift));
 8663   ins_cost(175);
 8664   format %{ "rolxl   $dst, $src, $shift" %}
 8665   ins_encode %{
 8666     int shift = 32 - ($shift$$constant & 31);
 8667     __ rorxl($dst$$Register, $src$$Address, shift);
 8668   %}
 8669   ins_pipe(ialu_reg_mem);
 8670 %}
 8671 
 8672 // Rotate Left by variable
 8673 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8674 %{
 8675   predicate(n->bottom_type()->basic_type() == T_INT);
 8676   match(Set dst (RotateLeft dst shift));
 8677   effect(KILL cr);
 8678   format %{ "roll    $dst, $shift" %}
 8679   ins_encode %{
 8680     __ roll($dst$$Register);
 8681   %}
 8682   ins_pipe(ialu_reg_reg);
 8683 %}
 8684 
 8685 // Rotate Right by constant.
 8686 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8687 %{
 8688   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8689   match(Set dst (RotateRight dst shift));
 8690   effect(KILL cr);
 8691   format %{ "rorl    $dst, $shift" %}
 8692   ins_encode %{
 8693     __ rorl($dst$$Register, $shift$$constant);
 8694   %}
 8695   ins_pipe(ialu_reg);
 8696 %}
 8697 
 8698 // Rotate Right by constant.
 8699 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 8700 %{
 8701   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8702   match(Set dst (RotateRight src shift));
 8703   format %{ "rorxl   $dst, $src, $shift" %}
 8704   ins_encode %{
 8705     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 8706   %}
 8707   ins_pipe(ialu_reg_reg);
 8708 %}
 8709 
 8710 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8711 %{
 8712   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8713   match(Set dst (RotateRight (LoadI src) shift));
 8714   ins_cost(175);
 8715   format %{ "rorxl   $dst, $src, $shift" %}
 8716   ins_encode %{
 8717     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 8718   %}
 8719   ins_pipe(ialu_reg_mem);
 8720 %}
 8721 
 8722 // Rotate Right by variable
 8723 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8724 %{
 8725   predicate(n->bottom_type()->basic_type() == T_INT);
 8726   match(Set dst (RotateRight dst shift));
 8727   effect(KILL cr);
 8728   format %{ "rorl    $dst, $shift" %}
 8729   ins_encode %{
 8730     __ rorl($dst$$Register);
 8731   %}
 8732   ins_pipe(ialu_reg_reg);
 8733 %}
 8734 
 8735 // Rotate Left by constant.
 8736 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8737 %{
 8738   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8739   match(Set dst (RotateLeft dst shift));
 8740   effect(KILL cr);
 8741   format %{ "rolq    $dst, $shift" %}
 8742   ins_encode %{
 8743     __ rolq($dst$$Register, $shift$$constant);
 8744   %}
 8745   ins_pipe(ialu_reg);
 8746 %}
 8747 
 8748 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 8749 %{
 8750   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8751   match(Set dst (RotateLeft src shift));
 8752   format %{ "rolxq   $dst, $src, $shift" %}
 8753   ins_encode %{
 8754     int shift = 64 - ($shift$$constant & 63);
 8755     __ rorxq($dst$$Register, $src$$Register, shift);
 8756   %}
 8757   ins_pipe(ialu_reg_reg);
 8758 %}
 8759 
 8760 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8761 %{
 8762   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8763   match(Set dst (RotateLeft (LoadL src) shift));
 8764   ins_cost(175);
 8765   format %{ "rolxq   $dst, $src, $shift" %}
 8766   ins_encode %{
 8767     int shift = 64 - ($shift$$constant & 63);
 8768     __ rorxq($dst$$Register, $src$$Address, shift);
 8769   %}
 8770   ins_pipe(ialu_reg_mem);
 8771 %}
 8772 
 8773 // Rotate Left by variable
 8774 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8775 %{
 8776   predicate(n->bottom_type()->basic_type() == T_LONG);
 8777   match(Set dst (RotateLeft dst shift));
 8778   effect(KILL cr);
 8779   format %{ "rolq    $dst, $shift" %}
 8780   ins_encode %{
 8781     __ rolq($dst$$Register);
 8782   %}
 8783   ins_pipe(ialu_reg_reg);
 8784 %}
 8785 
 8786 // Rotate Right by constant.
 8787 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8788 %{
 8789   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8790   match(Set dst (RotateRight dst shift));
 8791   effect(KILL cr);
 8792   format %{ "rorq    $dst, $shift" %}
 8793   ins_encode %{
 8794     __ rorq($dst$$Register, $shift$$constant);
 8795   %}
 8796   ins_pipe(ialu_reg);
 8797 %}
 8798 
 8799 // Rotate Right by constant
 8800 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 8801 %{
 8802   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8803   match(Set dst (RotateRight src shift));
 8804   format %{ "rorxq   $dst, $src, $shift" %}
 8805   ins_encode %{
 8806     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 8807   %}
 8808   ins_pipe(ialu_reg_reg);
 8809 %}
 8810 
 8811 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8812 %{
 8813   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8814   match(Set dst (RotateRight (LoadL src) shift));
 8815   ins_cost(175);
 8816   format %{ "rorxq   $dst, $src, $shift" %}
 8817   ins_encode %{
 8818     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 8819   %}
 8820   ins_pipe(ialu_reg_mem);
 8821 %}
 8822 
 8823 // Rotate Right by variable
 8824 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8825 %{
 8826   predicate(n->bottom_type()->basic_type() == T_LONG);
 8827   match(Set dst (RotateRight dst shift));
 8828   effect(KILL cr);
 8829   format %{ "rorq    $dst, $shift" %}
 8830   ins_encode %{
 8831     __ rorq($dst$$Register);
 8832   %}
 8833   ins_pipe(ialu_reg_reg);
 8834 %}
 8835 
 8836 //----------------------------- CompressBits/ExpandBits ------------------------
 8837 
 8838 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8839   predicate(n->bottom_type()->isa_long());
 8840   match(Set dst (CompressBits src mask));
 8841   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8842   ins_encode %{
 8843     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 8844   %}
 8845   ins_pipe( pipe_slow );
 8846 %}
 8847 
 8848 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8849   predicate(n->bottom_type()->isa_long());
 8850   match(Set dst (ExpandBits src mask));
 8851   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8852   ins_encode %{
 8853     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 8854   %}
 8855   ins_pipe( pipe_slow );
 8856 %}
 8857 
 8858 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8859   predicate(n->bottom_type()->isa_long());
 8860   match(Set dst (CompressBits src (LoadL mask)));
 8861   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8862   ins_encode %{
 8863     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 8864   %}
 8865   ins_pipe( pipe_slow );
 8866 %}
 8867 
 8868 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8869   predicate(n->bottom_type()->isa_long());
 8870   match(Set dst (ExpandBits src (LoadL mask)));
 8871   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8872   ins_encode %{
 8873     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 8874   %}
 8875   ins_pipe( pipe_slow );
 8876 %}
 8877 
 8878 
 8879 // Logical Instructions
 8880 
 8881 // Integer Logical Instructions
 8882 
 8883 // And Instructions
 8884 // And Register with Register
 8885 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8886 %{
 8887   match(Set dst (AndI dst src));
 8888   effect(KILL cr);
 8889   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8890 
 8891   format %{ "andl    $dst, $src\t# int" %}
 8892   ins_encode %{
 8893     __ andl($dst$$Register, $src$$Register);
 8894   %}
 8895   ins_pipe(ialu_reg_reg);
 8896 %}
 8897 
 8898 // And Register with Immediate 255
 8899 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 8900 %{
 8901   match(Set dst (AndI src mask));
 8902 
 8903   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 8904   ins_encode %{
 8905     __ movzbl($dst$$Register, $src$$Register);
 8906   %}
 8907   ins_pipe(ialu_reg);
 8908 %}
 8909 
 8910 // And Register with Immediate 255 and promote to long
 8911 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 8912 %{
 8913   match(Set dst (ConvI2L (AndI src mask)));
 8914 
 8915   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 8916   ins_encode %{
 8917     __ movzbl($dst$$Register, $src$$Register);
 8918   %}
 8919   ins_pipe(ialu_reg);
 8920 %}
 8921 
 8922 // And Register with Immediate 65535
 8923 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 8924 %{
 8925   match(Set dst (AndI src mask));
 8926 
 8927   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 8928   ins_encode %{
 8929     __ movzwl($dst$$Register, $src$$Register);
 8930   %}
 8931   ins_pipe(ialu_reg);
 8932 %}
 8933 
 8934 // And Register with Immediate 65535 and promote to long
 8935 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 8936 %{
 8937   match(Set dst (ConvI2L (AndI src mask)));
 8938 
 8939   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 8940   ins_encode %{
 8941     __ movzwl($dst$$Register, $src$$Register);
 8942   %}
 8943   ins_pipe(ialu_reg);
 8944 %}
 8945 
 8946 // Can skip int2long conversions after AND with small bitmask
 8947 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 8948 %{
 8949   predicate(VM_Version::supports_bmi2());
 8950   ins_cost(125);
 8951   effect(TEMP tmp, KILL cr);
 8952   match(Set dst (ConvI2L (AndI src mask)));
 8953   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 8954   ins_encode %{
 8955     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 8956     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 8957   %}
 8958   ins_pipe(ialu_reg_reg);
 8959 %}
 8960 
 8961 // And Register with Immediate
 8962 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8963 %{
 8964   match(Set dst (AndI dst src));
 8965   effect(KILL cr);
 8966   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8967 
 8968   format %{ "andl    $dst, $src\t# int" %}
 8969   ins_encode %{
 8970     __ andl($dst$$Register, $src$$constant);
 8971   %}
 8972   ins_pipe(ialu_reg);
 8973 %}
 8974 
 8975 // And Register with Memory
 8976 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8977 %{
 8978   match(Set dst (AndI dst (LoadI src)));
 8979   effect(KILL cr);
 8980   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8981 
 8982   ins_cost(150);
 8983   format %{ "andl    $dst, $src\t# int" %}
 8984   ins_encode %{
 8985     __ andl($dst$$Register, $src$$Address);
 8986   %}
 8987   ins_pipe(ialu_reg_mem);
 8988 %}
 8989 
 8990 // And Memory with Register
 8991 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8992 %{
 8993   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 8994   effect(KILL cr);
 8995   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8996 
 8997   ins_cost(150);
 8998   format %{ "andb    $dst, $src\t# byte" %}
 8999   ins_encode %{
 9000     __ andb($dst$$Address, $src$$Register);
 9001   %}
 9002   ins_pipe(ialu_mem_reg);
 9003 %}
 9004 
 9005 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9006 %{
 9007   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9008   effect(KILL cr);
 9009   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9010 
 9011   ins_cost(150);
 9012   format %{ "andl    $dst, $src\t# int" %}
 9013   ins_encode %{
 9014     __ andl($dst$$Address, $src$$Register);
 9015   %}
 9016   ins_pipe(ialu_mem_reg);
 9017 %}
 9018 
 9019 // And Memory with Immediate
 9020 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9021 %{
 9022   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9023   effect(KILL cr);
 9024   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9025 
 9026   ins_cost(125);
 9027   format %{ "andl    $dst, $src\t# int" %}
 9028   ins_encode %{
 9029     __ andl($dst$$Address, $src$$constant);
 9030   %}
 9031   ins_pipe(ialu_mem_imm);
 9032 %}
 9033 
 9034 // BMI1 instructions
 9035 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9036   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9037   predicate(UseBMI1Instructions);
 9038   effect(KILL cr);
 9039   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9040 
 9041   ins_cost(125);
 9042   format %{ "andnl  $dst, $src1, $src2" %}
 9043 
 9044   ins_encode %{
 9045     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9046   %}
 9047   ins_pipe(ialu_reg_mem);
 9048 %}
 9049 
 9050 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9051   match(Set dst (AndI (XorI src1 minus_1) src2));
 9052   predicate(UseBMI1Instructions);
 9053   effect(KILL cr);
 9054   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9055 
 9056   format %{ "andnl  $dst, $src1, $src2" %}
 9057 
 9058   ins_encode %{
 9059     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9060   %}
 9061   ins_pipe(ialu_reg);
 9062 %}
 9063 
 9064 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9065   match(Set dst (AndI (SubI imm_zero src) src));
 9066   predicate(UseBMI1Instructions);
 9067   effect(KILL cr);
 9068   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9069 
 9070   format %{ "blsil  $dst, $src" %}
 9071 
 9072   ins_encode %{
 9073     __ blsil($dst$$Register, $src$$Register);
 9074   %}
 9075   ins_pipe(ialu_reg);
 9076 %}
 9077 
 9078 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9079   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9080   predicate(UseBMI1Instructions);
 9081   effect(KILL cr);
 9082   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9083 
 9084   ins_cost(125);
 9085   format %{ "blsil  $dst, $src" %}
 9086 
 9087   ins_encode %{
 9088     __ blsil($dst$$Register, $src$$Address);
 9089   %}
 9090   ins_pipe(ialu_reg_mem);
 9091 %}
 9092 
 9093 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9094 %{
 9095   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9096   predicate(UseBMI1Instructions);
 9097   effect(KILL cr);
 9098   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9099 
 9100   ins_cost(125);
 9101   format %{ "blsmskl $dst, $src" %}
 9102 
 9103   ins_encode %{
 9104     __ blsmskl($dst$$Register, $src$$Address);
 9105   %}
 9106   ins_pipe(ialu_reg_mem);
 9107 %}
 9108 
 9109 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9110 %{
 9111   match(Set dst (XorI (AddI src minus_1) src));
 9112   predicate(UseBMI1Instructions);
 9113   effect(KILL cr);
 9114   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9115 
 9116   format %{ "blsmskl $dst, $src" %}
 9117 
 9118   ins_encode %{
 9119     __ blsmskl($dst$$Register, $src$$Register);
 9120   %}
 9121 
 9122   ins_pipe(ialu_reg);
 9123 %}
 9124 
 9125 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9126 %{
 9127   match(Set dst (AndI (AddI src minus_1) src) );
 9128   predicate(UseBMI1Instructions);
 9129   effect(KILL cr);
 9130   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9131 
 9132   format %{ "blsrl  $dst, $src" %}
 9133 
 9134   ins_encode %{
 9135     __ blsrl($dst$$Register, $src$$Register);
 9136   %}
 9137 
 9138   ins_pipe(ialu_reg_mem);
 9139 %}
 9140 
 9141 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9142 %{
 9143   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9144   predicate(UseBMI1Instructions);
 9145   effect(KILL cr);
 9146   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9147 
 9148   ins_cost(125);
 9149   format %{ "blsrl  $dst, $src" %}
 9150 
 9151   ins_encode %{
 9152     __ blsrl($dst$$Register, $src$$Address);
 9153   %}
 9154 
 9155   ins_pipe(ialu_reg);
 9156 %}
 9157 
 9158 // Or Instructions
 9159 // Or Register with Register
 9160 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9161 %{
 9162   match(Set dst (OrI dst src));
 9163   effect(KILL cr);
 9164   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9165 
 9166   format %{ "orl     $dst, $src\t# int" %}
 9167   ins_encode %{
 9168     __ orl($dst$$Register, $src$$Register);
 9169   %}
 9170   ins_pipe(ialu_reg_reg);
 9171 %}
 9172 
 9173 // Or Register with Immediate
 9174 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9175 %{
 9176   match(Set dst (OrI dst src));
 9177   effect(KILL cr);
 9178   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9179 
 9180   format %{ "orl     $dst, $src\t# int" %}
 9181   ins_encode %{
 9182     __ orl($dst$$Register, $src$$constant);
 9183   %}
 9184   ins_pipe(ialu_reg);
 9185 %}
 9186 
 9187 // Or Register with Memory
 9188 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9189 %{
 9190   match(Set dst (OrI dst (LoadI src)));
 9191   effect(KILL cr);
 9192   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9193 
 9194   ins_cost(150);
 9195   format %{ "orl     $dst, $src\t# int" %}
 9196   ins_encode %{
 9197     __ orl($dst$$Register, $src$$Address);
 9198   %}
 9199   ins_pipe(ialu_reg_mem);
 9200 %}
 9201 
 9202 // Or Memory with Register
 9203 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9204 %{
 9205   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9206   effect(KILL cr);
 9207   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9208 
 9209   ins_cost(150);
 9210   format %{ "orb    $dst, $src\t# byte" %}
 9211   ins_encode %{
 9212     __ orb($dst$$Address, $src$$Register);
 9213   %}
 9214   ins_pipe(ialu_mem_reg);
 9215 %}
 9216 
 9217 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9218 %{
 9219   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9220   effect(KILL cr);
 9221   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9222 
 9223   ins_cost(150);
 9224   format %{ "orl     $dst, $src\t# int" %}
 9225   ins_encode %{
 9226     __ orl($dst$$Address, $src$$Register);
 9227   %}
 9228   ins_pipe(ialu_mem_reg);
 9229 %}
 9230 
 9231 // Or Memory with Immediate
 9232 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9233 %{
 9234   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9235   effect(KILL cr);
 9236   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9237 
 9238   ins_cost(125);
 9239   format %{ "orl     $dst, $src\t# int" %}
 9240   ins_encode %{
 9241     __ orl($dst$$Address, $src$$constant);
 9242   %}
 9243   ins_pipe(ialu_mem_imm);
 9244 %}
 9245 
 9246 // Xor Instructions
 9247 // Xor Register with Register
 9248 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9249 %{
 9250   match(Set dst (XorI dst src));
 9251   effect(KILL cr);
 9252   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9253 
 9254   format %{ "xorl    $dst, $src\t# int" %}
 9255   ins_encode %{
 9256     __ xorl($dst$$Register, $src$$Register);
 9257   %}
 9258   ins_pipe(ialu_reg_reg);
 9259 %}
 9260 
 9261 // Xor Register with Immediate -1
 9262 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9263   match(Set dst (XorI dst imm));
 9264 
 9265   format %{ "not    $dst" %}
 9266   ins_encode %{
 9267      __ notl($dst$$Register);
 9268   %}
 9269   ins_pipe(ialu_reg);
 9270 %}
 9271 
 9272 // Xor Register with Immediate
 9273 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9274 %{
 9275   match(Set dst (XorI dst src));
 9276   effect(KILL cr);
 9277   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9278 
 9279   format %{ "xorl    $dst, $src\t# int" %}
 9280   ins_encode %{
 9281     __ xorl($dst$$Register, $src$$constant);
 9282   %}
 9283   ins_pipe(ialu_reg);
 9284 %}
 9285 
 9286 // Xor Register with Memory
 9287 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9288 %{
 9289   match(Set dst (XorI dst (LoadI src)));
 9290   effect(KILL cr);
 9291   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9292 
 9293   ins_cost(150);
 9294   format %{ "xorl    $dst, $src\t# int" %}
 9295   ins_encode %{
 9296     __ xorl($dst$$Register, $src$$Address);
 9297   %}
 9298   ins_pipe(ialu_reg_mem);
 9299 %}
 9300 
 9301 // Xor Memory with Register
 9302 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9303 %{
 9304   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9305   effect(KILL cr);
 9306   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9307 
 9308   ins_cost(150);
 9309   format %{ "xorb    $dst, $src\t# byte" %}
 9310   ins_encode %{
 9311     __ xorb($dst$$Address, $src$$Register);
 9312   %}
 9313   ins_pipe(ialu_mem_reg);
 9314 %}
 9315 
 9316 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9317 %{
 9318   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9319   effect(KILL cr);
 9320   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9321 
 9322   ins_cost(150);
 9323   format %{ "xorl    $dst, $src\t# int" %}
 9324   ins_encode %{
 9325     __ xorl($dst$$Address, $src$$Register);
 9326   %}
 9327   ins_pipe(ialu_mem_reg);
 9328 %}
 9329 
 9330 // Xor Memory with Immediate
 9331 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9332 %{
 9333   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9334   effect(KILL cr);
 9335   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9336 
 9337   ins_cost(125);
 9338   format %{ "xorl    $dst, $src\t# int" %}
 9339   ins_encode %{
 9340     __ xorl($dst$$Address, $src$$constant);
 9341   %}
 9342   ins_pipe(ialu_mem_imm);
 9343 %}
 9344 
 9345 
 9346 // Long Logical Instructions
 9347 
 9348 // And Instructions
 9349 // And Register with Register
 9350 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9351 %{
 9352   match(Set dst (AndL dst src));
 9353   effect(KILL cr);
 9354   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9355 
 9356   format %{ "andq    $dst, $src\t# long" %}
 9357   ins_encode %{
 9358     __ andq($dst$$Register, $src$$Register);
 9359   %}
 9360   ins_pipe(ialu_reg_reg);
 9361 %}
 9362 
 9363 // And Register with Immediate 255
 9364 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
 9365 %{
 9366   match(Set dst (AndL src mask));
 9367 
 9368   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
 9369   ins_encode %{
 9370     // movzbl zeroes out the upper 32-bit and does not need REX.W
 9371     __ movzbl($dst$$Register, $src$$Register);
 9372   %}
 9373   ins_pipe(ialu_reg);
 9374 %}
 9375 
 9376 // And Register with Immediate 65535
 9377 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
 9378 %{
 9379   match(Set dst (AndL src mask));
 9380 
 9381   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
 9382   ins_encode %{
 9383     // movzwl zeroes out the upper 32-bit and does not need REX.W
 9384     __ movzwl($dst$$Register, $src$$Register);
 9385   %}
 9386   ins_pipe(ialu_reg);
 9387 %}
 9388 
 9389 // And Register with Immediate
 9390 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9391 %{
 9392   match(Set dst (AndL dst src));
 9393   effect(KILL cr);
 9394   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9395 
 9396   format %{ "andq    $dst, $src\t# long" %}
 9397   ins_encode %{
 9398     __ andq($dst$$Register, $src$$constant);
 9399   %}
 9400   ins_pipe(ialu_reg);
 9401 %}
 9402 
 9403 // And Register with Memory
 9404 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9405 %{
 9406   match(Set dst (AndL dst (LoadL src)));
 9407   effect(KILL cr);
 9408   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9409 
 9410   ins_cost(150);
 9411   format %{ "andq    $dst, $src\t# long" %}
 9412   ins_encode %{
 9413     __ andq($dst$$Register, $src$$Address);
 9414   %}
 9415   ins_pipe(ialu_reg_mem);
 9416 %}
 9417 
 9418 // And Memory with Register
 9419 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9420 %{
 9421   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9422   effect(KILL cr);
 9423   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9424 
 9425   ins_cost(150);
 9426   format %{ "andq    $dst, $src\t# long" %}
 9427   ins_encode %{
 9428     __ andq($dst$$Address, $src$$Register);
 9429   %}
 9430   ins_pipe(ialu_mem_reg);
 9431 %}
 9432 
 9433 // And Memory with Immediate
 9434 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9435 %{
 9436   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9437   effect(KILL cr);
 9438   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9439 
 9440   ins_cost(125);
 9441   format %{ "andq    $dst, $src\t# long" %}
 9442   ins_encode %{
 9443     __ andq($dst$$Address, $src$$constant);
 9444   %}
 9445   ins_pipe(ialu_mem_imm);
 9446 %}
 9447 
 9448 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9449 %{
 9450   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9451   // because AND/OR works well enough for 8/32-bit values.
 9452   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9453 
 9454   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9455   effect(KILL cr);
 9456 
 9457   ins_cost(125);
 9458   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9459   ins_encode %{
 9460     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9461   %}
 9462   ins_pipe(ialu_mem_imm);
 9463 %}
 9464 
 9465 // BMI1 instructions
 9466 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9467   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9468   predicate(UseBMI1Instructions);
 9469   effect(KILL cr);
 9470   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9471 
 9472   ins_cost(125);
 9473   format %{ "andnq  $dst, $src1, $src2" %}
 9474 
 9475   ins_encode %{
 9476     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9477   %}
 9478   ins_pipe(ialu_reg_mem);
 9479 %}
 9480 
 9481 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9482   match(Set dst (AndL (XorL src1 minus_1) src2));
 9483   predicate(UseBMI1Instructions);
 9484   effect(KILL cr);
 9485   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9486 
 9487   format %{ "andnq  $dst, $src1, $src2" %}
 9488 
 9489   ins_encode %{
 9490   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9491   %}
 9492   ins_pipe(ialu_reg_mem);
 9493 %}
 9494 
 9495 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9496   match(Set dst (AndL (SubL imm_zero src) src));
 9497   predicate(UseBMI1Instructions);
 9498   effect(KILL cr);
 9499   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9500 
 9501   format %{ "blsiq  $dst, $src" %}
 9502 
 9503   ins_encode %{
 9504     __ blsiq($dst$$Register, $src$$Register);
 9505   %}
 9506   ins_pipe(ialu_reg);
 9507 %}
 9508 
 9509 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9510   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9511   predicate(UseBMI1Instructions);
 9512   effect(KILL cr);
 9513   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9514 
 9515   ins_cost(125);
 9516   format %{ "blsiq  $dst, $src" %}
 9517 
 9518   ins_encode %{
 9519     __ blsiq($dst$$Register, $src$$Address);
 9520   %}
 9521   ins_pipe(ialu_reg_mem);
 9522 %}
 9523 
 9524 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9525 %{
 9526   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9527   predicate(UseBMI1Instructions);
 9528   effect(KILL cr);
 9529   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9530 
 9531   ins_cost(125);
 9532   format %{ "blsmskq $dst, $src" %}
 9533 
 9534   ins_encode %{
 9535     __ blsmskq($dst$$Register, $src$$Address);
 9536   %}
 9537   ins_pipe(ialu_reg_mem);
 9538 %}
 9539 
 9540 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9541 %{
 9542   match(Set dst (XorL (AddL src minus_1) src));
 9543   predicate(UseBMI1Instructions);
 9544   effect(KILL cr);
 9545   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9546 
 9547   format %{ "blsmskq $dst, $src" %}
 9548 
 9549   ins_encode %{
 9550     __ blsmskq($dst$$Register, $src$$Register);
 9551   %}
 9552 
 9553   ins_pipe(ialu_reg);
 9554 %}
 9555 
 9556 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9557 %{
 9558   match(Set dst (AndL (AddL src minus_1) src) );
 9559   predicate(UseBMI1Instructions);
 9560   effect(KILL cr);
 9561   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9562 
 9563   format %{ "blsrq  $dst, $src" %}
 9564 
 9565   ins_encode %{
 9566     __ blsrq($dst$$Register, $src$$Register);
 9567   %}
 9568 
 9569   ins_pipe(ialu_reg);
 9570 %}
 9571 
 9572 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9573 %{
 9574   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9575   predicate(UseBMI1Instructions);
 9576   effect(KILL cr);
 9577   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9578 
 9579   ins_cost(125);
 9580   format %{ "blsrq  $dst, $src" %}
 9581 
 9582   ins_encode %{
 9583     __ blsrq($dst$$Register, $src$$Address);
 9584   %}
 9585 
 9586   ins_pipe(ialu_reg);
 9587 %}
 9588 
 9589 // Or Instructions
 9590 // Or Register with Register
 9591 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9592 %{
 9593   match(Set dst (OrL dst src));
 9594   effect(KILL cr);
 9595   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9596 
 9597   format %{ "orq     $dst, $src\t# long" %}
 9598   ins_encode %{
 9599     __ orq($dst$$Register, $src$$Register);
 9600   %}
 9601   ins_pipe(ialu_reg_reg);
 9602 %}
 9603 
 9604 // Use any_RegP to match R15 (TLS register) without spilling.
 9605 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 9606   match(Set dst (OrL dst (CastP2X src)));
 9607   effect(KILL cr);
 9608   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9609 
 9610   format %{ "orq     $dst, $src\t# long" %}
 9611   ins_encode %{
 9612     __ orq($dst$$Register, $src$$Register);
 9613   %}
 9614   ins_pipe(ialu_reg_reg);
 9615 %}
 9616 
 9617 
 9618 // Or Register with Immediate
 9619 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9620 %{
 9621   match(Set dst (OrL dst src));
 9622   effect(KILL cr);
 9623   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9624 
 9625   format %{ "orq     $dst, $src\t# long" %}
 9626   ins_encode %{
 9627     __ orq($dst$$Register, $src$$constant);
 9628   %}
 9629   ins_pipe(ialu_reg);
 9630 %}
 9631 
 9632 // Or Register with Memory
 9633 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9634 %{
 9635   match(Set dst (OrL dst (LoadL src)));
 9636   effect(KILL cr);
 9637   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9638 
 9639   ins_cost(150);
 9640   format %{ "orq     $dst, $src\t# long" %}
 9641   ins_encode %{
 9642     __ orq($dst$$Register, $src$$Address);
 9643   %}
 9644   ins_pipe(ialu_reg_mem);
 9645 %}
 9646 
 9647 // Or Memory with Register
 9648 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9649 %{
 9650   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9651   effect(KILL cr);
 9652   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9653 
 9654   ins_cost(150);
 9655   format %{ "orq     $dst, $src\t# long" %}
 9656   ins_encode %{
 9657     __ orq($dst$$Address, $src$$Register);
 9658   %}
 9659   ins_pipe(ialu_mem_reg);
 9660 %}
 9661 
 9662 // Or Memory with Immediate
 9663 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9664 %{
 9665   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9666   effect(KILL cr);
 9667   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9668 
 9669   ins_cost(125);
 9670   format %{ "orq     $dst, $src\t# long" %}
 9671   ins_encode %{
 9672     __ orq($dst$$Address, $src$$constant);
 9673   %}
 9674   ins_pipe(ialu_mem_imm);
 9675 %}
 9676 
 9677 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
 9678 %{
 9679   // con should be a pure 64-bit power of 2 immediate
 9680   // because AND/OR works well enough for 8/32-bit values.
 9681   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
 9682 
 9683   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
 9684   effect(KILL cr);
 9685 
 9686   ins_cost(125);
 9687   format %{ "btsq    $dst, log2($con)\t# long" %}
 9688   ins_encode %{
 9689     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
 9690   %}
 9691   ins_pipe(ialu_mem_imm);
 9692 %}
 9693 
 9694 // Xor Instructions
 9695 // Xor Register with Register
 9696 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9697 %{
 9698   match(Set dst (XorL dst src));
 9699   effect(KILL cr);
 9700   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9701 
 9702   format %{ "xorq    $dst, $src\t# long" %}
 9703   ins_encode %{
 9704     __ xorq($dst$$Register, $src$$Register);
 9705   %}
 9706   ins_pipe(ialu_reg_reg);
 9707 %}
 9708 
 9709 // Xor Register with Immediate -1
 9710 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 9711   match(Set dst (XorL dst imm));
 9712 
 9713   format %{ "notq   $dst" %}
 9714   ins_encode %{
 9715      __ notq($dst$$Register);
 9716   %}
 9717   ins_pipe(ialu_reg);
 9718 %}
 9719 
 9720 // Xor Register with Immediate
 9721 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9722 %{
 9723   match(Set dst (XorL dst src));
 9724   effect(KILL cr);
 9725   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9726 
 9727   format %{ "xorq    $dst, $src\t# long" %}
 9728   ins_encode %{
 9729     __ xorq($dst$$Register, $src$$constant);
 9730   %}
 9731   ins_pipe(ialu_reg);
 9732 %}
 9733 
 9734 // Xor Register with Memory
 9735 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9736 %{
 9737   match(Set dst (XorL dst (LoadL src)));
 9738   effect(KILL cr);
 9739   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9740 
 9741   ins_cost(150);
 9742   format %{ "xorq    $dst, $src\t# long" %}
 9743   ins_encode %{
 9744     __ xorq($dst$$Register, $src$$Address);
 9745   %}
 9746   ins_pipe(ialu_reg_mem);
 9747 %}
 9748 
 9749 // Xor Memory with Register
 9750 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9751 %{
 9752   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9753   effect(KILL cr);
 9754   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9755 
 9756   ins_cost(150);
 9757   format %{ "xorq    $dst, $src\t# long" %}
 9758   ins_encode %{
 9759     __ xorq($dst$$Address, $src$$Register);
 9760   %}
 9761   ins_pipe(ialu_mem_reg);
 9762 %}
 9763 
 9764 // Xor Memory with Immediate
 9765 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9766 %{
 9767   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9768   effect(KILL cr);
 9769   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9770 
 9771   ins_cost(125);
 9772   format %{ "xorq    $dst, $src\t# long" %}
 9773   ins_encode %{
 9774     __ xorq($dst$$Address, $src$$constant);
 9775   %}
 9776   ins_pipe(ialu_mem_imm);
 9777 %}
 9778 
 9779 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 9780 %{
 9781   match(Set dst (CmpLTMask p q));
 9782   effect(KILL cr);
 9783 
 9784   ins_cost(400);
 9785   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 9786             "setlt   $dst\n\t"
 9787             "movzbl  $dst, $dst\n\t"
 9788             "negl    $dst" %}
 9789   ins_encode %{
 9790     __ cmpl($p$$Register, $q$$Register);
 9791     __ setb(Assembler::less, $dst$$Register);
 9792     __ movzbl($dst$$Register, $dst$$Register);
 9793     __ negl($dst$$Register);
 9794   %}
 9795   ins_pipe(pipe_slow);
 9796 %}
 9797 
 9798 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
 9799 %{
 9800   match(Set dst (CmpLTMask dst zero));
 9801   effect(KILL cr);
 9802 
 9803   ins_cost(100);
 9804   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 9805   ins_encode %{
 9806     __ sarl($dst$$Register, 31);
 9807   %}
 9808   ins_pipe(ialu_reg);
 9809 %}
 9810 
 9811 /* Better to save a register than avoid a branch */
 9812 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9813 %{
 9814   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 9815   effect(KILL cr);
 9816   ins_cost(300);
 9817   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
 9818             "jge     done\n\t"
 9819             "addl    $p,$y\n"
 9820             "done:   " %}
 9821   ins_encode %{
 9822     Register Rp = $p$$Register;
 9823     Register Rq = $q$$Register;
 9824     Register Ry = $y$$Register;
 9825     Label done;
 9826     __ subl(Rp, Rq);
 9827     __ jccb(Assembler::greaterEqual, done);
 9828     __ addl(Rp, Ry);
 9829     __ bind(done);
 9830   %}
 9831   ins_pipe(pipe_cmplt);
 9832 %}
 9833 
 9834 /* Better to save a register than avoid a branch */
 9835 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9836 %{
 9837   match(Set y (AndI (CmpLTMask p q) y));
 9838   effect(KILL cr);
 9839 
 9840   ins_cost(300);
 9841 
 9842   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
 9843             "jlt     done\n\t"
 9844             "xorl    $y, $y\n"
 9845             "done:   " %}
 9846   ins_encode %{
 9847     Register Rp = $p$$Register;
 9848     Register Rq = $q$$Register;
 9849     Register Ry = $y$$Register;
 9850     Label done;
 9851     __ cmpl(Rp, Rq);
 9852     __ jccb(Assembler::less, done);
 9853     __ xorl(Ry, Ry);
 9854     __ bind(done);
 9855   %}
 9856   ins_pipe(pipe_cmplt);
 9857 %}
 9858 
 9859 
 9860 //---------- FP Instructions------------------------------------------------
 9861 
 9862 // Really expensive, avoid
 9863 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 9864 %{
 9865   match(Set cr (CmpF src1 src2));
 9866 
 9867   ins_cost(500);
 9868   format %{ "ucomiss $src1, $src2\n\t"
 9869             "jnp,s   exit\n\t"
 9870             "pushfq\t# saw NaN, set CF\n\t"
 9871             "andq    [rsp], #0xffffff2b\n\t"
 9872             "popfq\n"
 9873     "exit:" %}
 9874   ins_encode %{
 9875     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9876     emit_cmpfp_fixup(_masm);
 9877   %}
 9878   ins_pipe(pipe_slow);
 9879 %}
 9880 
 9881 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 9882   match(Set cr (CmpF src1 src2));
 9883 
 9884   ins_cost(100);
 9885   format %{ "ucomiss $src1, $src2" %}
 9886   ins_encode %{
 9887     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9888   %}
 9889   ins_pipe(pipe_slow);
 9890 %}
 9891 
 9892 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 9893   match(Set cr (CmpF src1 (LoadF src2)));
 9894 
 9895   ins_cost(100);
 9896   format %{ "ucomiss $src1, $src2" %}
 9897   ins_encode %{
 9898     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9899   %}
 9900   ins_pipe(pipe_slow);
 9901 %}
 9902 
 9903 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
 9904   match(Set cr (CmpF src con));
 9905   ins_cost(100);
 9906   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
 9907   ins_encode %{
 9908     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9909   %}
 9910   ins_pipe(pipe_slow);
 9911 %}
 9912 
 9913 // Really expensive, avoid
 9914 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 9915 %{
 9916   match(Set cr (CmpD src1 src2));
 9917 
 9918   ins_cost(500);
 9919   format %{ "ucomisd $src1, $src2\n\t"
 9920             "jnp,s   exit\n\t"
 9921             "pushfq\t# saw NaN, set CF\n\t"
 9922             "andq    [rsp], #0xffffff2b\n\t"
 9923             "popfq\n"
 9924     "exit:" %}
 9925   ins_encode %{
 9926     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9927     emit_cmpfp_fixup(_masm);
 9928   %}
 9929   ins_pipe(pipe_slow);
 9930 %}
 9931 
 9932 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 9933   match(Set cr (CmpD src1 src2));
 9934 
 9935   ins_cost(100);
 9936   format %{ "ucomisd $src1, $src2 test" %}
 9937   ins_encode %{
 9938     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9939   %}
 9940   ins_pipe(pipe_slow);
 9941 %}
 9942 
 9943 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 9944   match(Set cr (CmpD src1 (LoadD src2)));
 9945 
 9946   ins_cost(100);
 9947   format %{ "ucomisd $src1, $src2" %}
 9948   ins_encode %{
 9949     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9950   %}
 9951   ins_pipe(pipe_slow);
 9952 %}
 9953 
 9954 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
 9955   match(Set cr (CmpD src con));
 9956   ins_cost(100);
 9957   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
 9958   ins_encode %{
 9959     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9960   %}
 9961   ins_pipe(pipe_slow);
 9962 %}
 9963 
 9964 // Compare into -1,0,1
 9965 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
 9966 %{
 9967   match(Set dst (CmpF3 src1 src2));
 9968   effect(KILL cr);
 9969 
 9970   ins_cost(275);
 9971   format %{ "ucomiss $src1, $src2\n\t"
 9972             "movl    $dst, #-1\n\t"
 9973             "jp,s    done\n\t"
 9974             "jb,s    done\n\t"
 9975             "setne   $dst\n\t"
 9976             "movzbl  $dst, $dst\n"
 9977     "done:" %}
 9978   ins_encode %{
 9979     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9980     emit_cmpfp3(_masm, $dst$$Register);
 9981   %}
 9982   ins_pipe(pipe_slow);
 9983 %}
 9984 
 9985 // Compare into -1,0,1
 9986 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 9987 %{
 9988   match(Set dst (CmpF3 src1 (LoadF src2)));
 9989   effect(KILL cr);
 9990 
 9991   ins_cost(275);
 9992   format %{ "ucomiss $src1, $src2\n\t"
 9993             "movl    $dst, #-1\n\t"
 9994             "jp,s    done\n\t"
 9995             "jb,s    done\n\t"
 9996             "setne   $dst\n\t"
 9997             "movzbl  $dst, $dst\n"
 9998     "done:" %}
 9999   ins_encode %{
10000     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10001     emit_cmpfp3(_masm, $dst$$Register);
10002   %}
10003   ins_pipe(pipe_slow);
10004 %}
10005 
10006 // Compare into -1,0,1
10007 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10008   match(Set dst (CmpF3 src con));
10009   effect(KILL cr);
10010 
10011   ins_cost(275);
10012   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10013             "movl    $dst, #-1\n\t"
10014             "jp,s    done\n\t"
10015             "jb,s    done\n\t"
10016             "setne   $dst\n\t"
10017             "movzbl  $dst, $dst\n"
10018     "done:" %}
10019   ins_encode %{
10020     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10021     emit_cmpfp3(_masm, $dst$$Register);
10022   %}
10023   ins_pipe(pipe_slow);
10024 %}
10025 
10026 // Compare into -1,0,1
10027 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10028 %{
10029   match(Set dst (CmpD3 src1 src2));
10030   effect(KILL cr);
10031 
10032   ins_cost(275);
10033   format %{ "ucomisd $src1, $src2\n\t"
10034             "movl    $dst, #-1\n\t"
10035             "jp,s    done\n\t"
10036             "jb,s    done\n\t"
10037             "setne   $dst\n\t"
10038             "movzbl  $dst, $dst\n"
10039     "done:" %}
10040   ins_encode %{
10041     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10042     emit_cmpfp3(_masm, $dst$$Register);
10043   %}
10044   ins_pipe(pipe_slow);
10045 %}
10046 
10047 // Compare into -1,0,1
10048 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10049 %{
10050   match(Set dst (CmpD3 src1 (LoadD src2)));
10051   effect(KILL cr);
10052 
10053   ins_cost(275);
10054   format %{ "ucomisd $src1, $src2\n\t"
10055             "movl    $dst, #-1\n\t"
10056             "jp,s    done\n\t"
10057             "jb,s    done\n\t"
10058             "setne   $dst\n\t"
10059             "movzbl  $dst, $dst\n"
10060     "done:" %}
10061   ins_encode %{
10062     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10063     emit_cmpfp3(_masm, $dst$$Register);
10064   %}
10065   ins_pipe(pipe_slow);
10066 %}
10067 
10068 // Compare into -1,0,1
10069 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10070   match(Set dst (CmpD3 src con));
10071   effect(KILL cr);
10072 
10073   ins_cost(275);
10074   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10075             "movl    $dst, #-1\n\t"
10076             "jp,s    done\n\t"
10077             "jb,s    done\n\t"
10078             "setne   $dst\n\t"
10079             "movzbl  $dst, $dst\n"
10080     "done:" %}
10081   ins_encode %{
10082     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10083     emit_cmpfp3(_masm, $dst$$Register);
10084   %}
10085   ins_pipe(pipe_slow);
10086 %}
10087 
10088 //----------Arithmetic Conversion Instructions---------------------------------
10089 
10090 instruct convF2D_reg_reg(regD dst, regF src)
10091 %{
10092   match(Set dst (ConvF2D src));
10093 
10094   format %{ "cvtss2sd $dst, $src" %}
10095   ins_encode %{
10096     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10097   %}
10098   ins_pipe(pipe_slow); // XXX
10099 %}
10100 
10101 instruct convF2D_reg_mem(regD dst, memory src)
10102 %{
10103   match(Set dst (ConvF2D (LoadF src)));
10104 
10105   format %{ "cvtss2sd $dst, $src" %}
10106   ins_encode %{
10107     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10108   %}
10109   ins_pipe(pipe_slow); // XXX
10110 %}
10111 
10112 instruct convD2F_reg_reg(regF dst, regD src)
10113 %{
10114   match(Set dst (ConvD2F src));
10115 
10116   format %{ "cvtsd2ss $dst, $src" %}
10117   ins_encode %{
10118     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10119   %}
10120   ins_pipe(pipe_slow); // XXX
10121 %}
10122 
10123 instruct convD2F_reg_mem(regF dst, memory src)
10124 %{
10125   match(Set dst (ConvD2F (LoadD src)));
10126 
10127   format %{ "cvtsd2ss $dst, $src" %}
10128   ins_encode %{
10129     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10130   %}
10131   ins_pipe(pipe_slow); // XXX
10132 %}
10133 
10134 // XXX do mem variants
10135 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10136 %{
10137   match(Set dst (ConvF2I src));
10138   effect(KILL cr);
10139   format %{ "convert_f2i $dst, $src" %}
10140   ins_encode %{
10141     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10142   %}
10143   ins_pipe(pipe_slow);
10144 %}
10145 
10146 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10147 %{
10148   match(Set dst (ConvF2L src));
10149   effect(KILL cr);
10150   format %{ "convert_f2l $dst, $src"%}
10151   ins_encode %{
10152     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10153   %}
10154   ins_pipe(pipe_slow);
10155 %}
10156 
10157 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10158 %{
10159   match(Set dst (ConvD2I src));
10160   effect(KILL cr);
10161   format %{ "convert_d2i $dst, $src"%}
10162   ins_encode %{
10163     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10164   %}
10165   ins_pipe(pipe_slow);
10166 %}
10167 
10168 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10169 %{
10170   match(Set dst (ConvD2L src));
10171   effect(KILL cr);
10172   format %{ "convert_d2l $dst, $src"%}
10173   ins_encode %{
10174     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10175   %}
10176   ins_pipe(pipe_slow);
10177 %}
10178 
10179 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10180 %{
10181   match(Set dst (RoundD src));
10182   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10183   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10184   ins_encode %{
10185     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10186   %}
10187   ins_pipe(pipe_slow);
10188 %}
10189 
10190 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10191 %{
10192   match(Set dst (RoundF src));
10193   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10194   format %{ "round_float $dst,$src" %}
10195   ins_encode %{
10196     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10197   %}
10198   ins_pipe(pipe_slow);
10199 %}
10200 
10201 instruct convI2F_reg_reg(regF dst, rRegI src)
10202 %{
10203   predicate(!UseXmmI2F);
10204   match(Set dst (ConvI2F src));
10205 
10206   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10207   ins_encode %{
10208     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10209   %}
10210   ins_pipe(pipe_slow); // XXX
10211 %}
10212 
10213 instruct convI2F_reg_mem(regF dst, memory src)
10214 %{
10215   match(Set dst (ConvI2F (LoadI src)));
10216 
10217   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10218   ins_encode %{
10219     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10220   %}
10221   ins_pipe(pipe_slow); // XXX
10222 %}
10223 
10224 instruct convI2D_reg_reg(regD dst, rRegI src)
10225 %{
10226   predicate(!UseXmmI2D);
10227   match(Set dst (ConvI2D src));
10228 
10229   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10230   ins_encode %{
10231     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10232   %}
10233   ins_pipe(pipe_slow); // XXX
10234 %}
10235 
10236 instruct convI2D_reg_mem(regD dst, memory src)
10237 %{
10238   match(Set dst (ConvI2D (LoadI src)));
10239 
10240   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10241   ins_encode %{
10242     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10243   %}
10244   ins_pipe(pipe_slow); // XXX
10245 %}
10246 
10247 instruct convXI2F_reg(regF dst, rRegI src)
10248 %{
10249   predicate(UseXmmI2F);
10250   match(Set dst (ConvI2F src));
10251 
10252   format %{ "movdl $dst, $src\n\t"
10253             "cvtdq2psl $dst, $dst\t# i2f" %}
10254   ins_encode %{
10255     __ movdl($dst$$XMMRegister, $src$$Register);
10256     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10257   %}
10258   ins_pipe(pipe_slow); // XXX
10259 %}
10260 
10261 instruct convXI2D_reg(regD dst, rRegI src)
10262 %{
10263   predicate(UseXmmI2D);
10264   match(Set dst (ConvI2D src));
10265 
10266   format %{ "movdl $dst, $src\n\t"
10267             "cvtdq2pdl $dst, $dst\t# i2d" %}
10268   ins_encode %{
10269     __ movdl($dst$$XMMRegister, $src$$Register);
10270     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10271   %}
10272   ins_pipe(pipe_slow); // XXX
10273 %}
10274 
10275 instruct convL2F_reg_reg(regF dst, rRegL src)
10276 %{
10277   match(Set dst (ConvL2F src));
10278 
10279   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10280   ins_encode %{
10281     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10282   %}
10283   ins_pipe(pipe_slow); // XXX
10284 %}
10285 
10286 instruct convL2F_reg_mem(regF dst, memory src)
10287 %{
10288   match(Set dst (ConvL2F (LoadL src)));
10289 
10290   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10291   ins_encode %{
10292     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10293   %}
10294   ins_pipe(pipe_slow); // XXX
10295 %}
10296 
10297 instruct convL2D_reg_reg(regD dst, rRegL src)
10298 %{
10299   match(Set dst (ConvL2D src));
10300 
10301   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10302   ins_encode %{
10303     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10304   %}
10305   ins_pipe(pipe_slow); // XXX
10306 %}
10307 
10308 instruct convL2D_reg_mem(regD dst, memory src)
10309 %{
10310   match(Set dst (ConvL2D (LoadL src)));
10311 
10312   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10313   ins_encode %{
10314     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10315   %}
10316   ins_pipe(pipe_slow); // XXX
10317 %}
10318 
10319 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10320 %{
10321   match(Set dst (ConvI2L src));
10322 
10323   ins_cost(125);
10324   format %{ "movslq  $dst, $src\t# i2l" %}
10325   ins_encode %{
10326     __ movslq($dst$$Register, $src$$Register);
10327   %}
10328   ins_pipe(ialu_reg_reg);
10329 %}
10330 
10331 // Zero-extend convert int to long
10332 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10333 %{
10334   match(Set dst (AndL (ConvI2L src) mask));
10335 
10336   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10337   ins_encode %{
10338     if ($dst$$reg != $src$$reg) {
10339       __ movl($dst$$Register, $src$$Register);
10340     }
10341   %}
10342   ins_pipe(ialu_reg_reg);
10343 %}
10344 
10345 // Zero-extend convert int to long
10346 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10347 %{
10348   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10349 
10350   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10351   ins_encode %{
10352     __ movl($dst$$Register, $src$$Address);
10353   %}
10354   ins_pipe(ialu_reg_mem);
10355 %}
10356 
10357 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10358 %{
10359   match(Set dst (AndL src mask));
10360 
10361   format %{ "movl    $dst, $src\t# zero-extend long" %}
10362   ins_encode %{
10363     __ movl($dst$$Register, $src$$Register);
10364   %}
10365   ins_pipe(ialu_reg_reg);
10366 %}
10367 
10368 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10369 %{
10370   match(Set dst (ConvL2I src));
10371 
10372   format %{ "movl    $dst, $src\t# l2i" %}
10373   ins_encode %{
10374     __ movl($dst$$Register, $src$$Register);
10375   %}
10376   ins_pipe(ialu_reg_reg);
10377 %}
10378 
10379 
10380 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10381   match(Set dst (MoveF2I src));
10382   effect(DEF dst, USE src);
10383 
10384   ins_cost(125);
10385   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10386   ins_encode %{
10387     __ movl($dst$$Register, Address(rsp, $src$$disp));
10388   %}
10389   ins_pipe(ialu_reg_mem);
10390 %}
10391 
10392 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10393   match(Set dst (MoveI2F src));
10394   effect(DEF dst, USE src);
10395 
10396   ins_cost(125);
10397   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10398   ins_encode %{
10399     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10400   %}
10401   ins_pipe(pipe_slow);
10402 %}
10403 
10404 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10405   match(Set dst (MoveD2L src));
10406   effect(DEF dst, USE src);
10407 
10408   ins_cost(125);
10409   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10410   ins_encode %{
10411     __ movq($dst$$Register, Address(rsp, $src$$disp));
10412   %}
10413   ins_pipe(ialu_reg_mem);
10414 %}
10415 
10416 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10417   predicate(!UseXmmLoadAndClearUpper);
10418   match(Set dst (MoveL2D src));
10419   effect(DEF dst, USE src);
10420 
10421   ins_cost(125);
10422   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10423   ins_encode %{
10424     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10425   %}
10426   ins_pipe(pipe_slow);
10427 %}
10428 
10429 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10430   predicate(UseXmmLoadAndClearUpper);
10431   match(Set dst (MoveL2D src));
10432   effect(DEF dst, USE src);
10433 
10434   ins_cost(125);
10435   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10436   ins_encode %{
10437     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10438   %}
10439   ins_pipe(pipe_slow);
10440 %}
10441 
10442 
10443 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10444   match(Set dst (MoveF2I src));
10445   effect(DEF dst, USE src);
10446 
10447   ins_cost(95); // XXX
10448   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10449   ins_encode %{
10450     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10451   %}
10452   ins_pipe(pipe_slow);
10453 %}
10454 
10455 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10456   match(Set dst (MoveI2F src));
10457   effect(DEF dst, USE src);
10458 
10459   ins_cost(100);
10460   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10461   ins_encode %{
10462     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10463   %}
10464   ins_pipe( ialu_mem_reg );
10465 %}
10466 
10467 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10468   match(Set dst (MoveD2L src));
10469   effect(DEF dst, USE src);
10470 
10471   ins_cost(95); // XXX
10472   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10473   ins_encode %{
10474     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10475   %}
10476   ins_pipe(pipe_slow);
10477 %}
10478 
10479 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10480   match(Set dst (MoveL2D src));
10481   effect(DEF dst, USE src);
10482 
10483   ins_cost(100);
10484   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10485   ins_encode %{
10486     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10487   %}
10488   ins_pipe(ialu_mem_reg);
10489 %}
10490 
10491 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10492   match(Set dst (MoveF2I src));
10493   effect(DEF dst, USE src);
10494   ins_cost(85);
10495   format %{ "movd    $dst,$src\t# MoveF2I" %}
10496   ins_encode %{
10497     __ movdl($dst$$Register, $src$$XMMRegister);
10498   %}
10499   ins_pipe( pipe_slow );
10500 %}
10501 
10502 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10503   match(Set dst (MoveD2L src));
10504   effect(DEF dst, USE src);
10505   ins_cost(85);
10506   format %{ "movd    $dst,$src\t# MoveD2L" %}
10507   ins_encode %{
10508     __ movdq($dst$$Register, $src$$XMMRegister);
10509   %}
10510   ins_pipe( pipe_slow );
10511 %}
10512 
10513 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10514   match(Set dst (MoveI2F src));
10515   effect(DEF dst, USE src);
10516   ins_cost(100);
10517   format %{ "movd    $dst,$src\t# MoveI2F" %}
10518   ins_encode %{
10519     __ movdl($dst$$XMMRegister, $src$$Register);
10520   %}
10521   ins_pipe( pipe_slow );
10522 %}
10523 
10524 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10525   match(Set dst (MoveL2D src));
10526   effect(DEF dst, USE src);
10527   ins_cost(100);
10528   format %{ "movd    $dst,$src\t# MoveL2D" %}
10529   ins_encode %{
10530      __ movdq($dst$$XMMRegister, $src$$Register);
10531   %}
10532   ins_pipe( pipe_slow );
10533 %}
10534 
10535 // Fast clearing of an array
10536 // Small ClearArray non-AVX512.
10537 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10538                   Universe dummy, rFlagsReg cr)
10539 %{
10540   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
10541   match(Set dummy (ClearArray cnt base));
10542   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10543 
10544   format %{ $$template
10545     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10546     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10547     $$emit$$"jg      LARGE\n\t"
10548     $$emit$$"dec     rcx\n\t"
10549     $$emit$$"js      DONE\t# Zero length\n\t"
10550     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10551     $$emit$$"dec     rcx\n\t"
10552     $$emit$$"jge     LOOP\n\t"
10553     $$emit$$"jmp     DONE\n\t"
10554     $$emit$$"# LARGE:\n\t"
10555     if (UseFastStosb) {
10556        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10557        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10558     } else if (UseXMMForObjInit) {
10559        $$emit$$"mov     rdi,rax\n\t"
10560        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10561        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10562        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10563        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10564        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10565        $$emit$$"add     0x40,rax\n\t"
10566        $$emit$$"# L_zero_64_bytes:\n\t"
10567        $$emit$$"sub     0x8,rcx\n\t"
10568        $$emit$$"jge     L_loop\n\t"
10569        $$emit$$"add     0x4,rcx\n\t"
10570        $$emit$$"jl      L_tail\n\t"
10571        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10572        $$emit$$"add     0x20,rax\n\t"
10573        $$emit$$"sub     0x4,rcx\n\t"
10574        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10575        $$emit$$"add     0x4,rcx\n\t"
10576        $$emit$$"jle     L_end\n\t"
10577        $$emit$$"dec     rcx\n\t"
10578        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10579        $$emit$$"vmovq   xmm0,(rax)\n\t"
10580        $$emit$$"add     0x8,rax\n\t"
10581        $$emit$$"dec     rcx\n\t"
10582        $$emit$$"jge     L_sloop\n\t"
10583        $$emit$$"# L_end:\n\t"
10584     } else {
10585        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10586     }
10587     $$emit$$"# DONE"
10588   %}
10589   ins_encode %{
10590     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10591                  $tmp$$XMMRegister, false, knoreg);
10592   %}
10593   ins_pipe(pipe_slow);
10594 %}
10595 
10596 // Small ClearArray AVX512 non-constant length.
10597 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10598                        Universe dummy, rFlagsReg cr)
10599 %{
10600   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
10601   match(Set dummy (ClearArray cnt base));
10602   ins_cost(125);
10603   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10604 
10605   format %{ $$template
10606     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10607     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10608     $$emit$$"jg      LARGE\n\t"
10609     $$emit$$"dec     rcx\n\t"
10610     $$emit$$"js      DONE\t# Zero length\n\t"
10611     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10612     $$emit$$"dec     rcx\n\t"
10613     $$emit$$"jge     LOOP\n\t"
10614     $$emit$$"jmp     DONE\n\t"
10615     $$emit$$"# LARGE:\n\t"
10616     if (UseFastStosb) {
10617        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10618        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10619     } else if (UseXMMForObjInit) {
10620        $$emit$$"mov     rdi,rax\n\t"
10621        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10622        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10623        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10624        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10625        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10626        $$emit$$"add     0x40,rax\n\t"
10627        $$emit$$"# L_zero_64_bytes:\n\t"
10628        $$emit$$"sub     0x8,rcx\n\t"
10629        $$emit$$"jge     L_loop\n\t"
10630        $$emit$$"add     0x4,rcx\n\t"
10631        $$emit$$"jl      L_tail\n\t"
10632        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10633        $$emit$$"add     0x20,rax\n\t"
10634        $$emit$$"sub     0x4,rcx\n\t"
10635        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10636        $$emit$$"add     0x4,rcx\n\t"
10637        $$emit$$"jle     L_end\n\t"
10638        $$emit$$"dec     rcx\n\t"
10639        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10640        $$emit$$"vmovq   xmm0,(rax)\n\t"
10641        $$emit$$"add     0x8,rax\n\t"
10642        $$emit$$"dec     rcx\n\t"
10643        $$emit$$"jge     L_sloop\n\t"
10644        $$emit$$"# L_end:\n\t"
10645     } else {
10646        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10647     }
10648     $$emit$$"# DONE"
10649   %}
10650   ins_encode %{
10651     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10652                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
10653   %}
10654   ins_pipe(pipe_slow);
10655 %}
10656 
10657 // Large ClearArray non-AVX512.
10658 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10659                         Universe dummy, rFlagsReg cr)
10660 %{
10661   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
10662   match(Set dummy (ClearArray cnt base));
10663   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10664 
10665   format %{ $$template
10666     if (UseFastStosb) {
10667        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10668        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10669        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10670     } else if (UseXMMForObjInit) {
10671        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10672        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10673        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10674        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10675        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10676        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10677        $$emit$$"add     0x40,rax\n\t"
10678        $$emit$$"# L_zero_64_bytes:\n\t"
10679        $$emit$$"sub     0x8,rcx\n\t"
10680        $$emit$$"jge     L_loop\n\t"
10681        $$emit$$"add     0x4,rcx\n\t"
10682        $$emit$$"jl      L_tail\n\t"
10683        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10684        $$emit$$"add     0x20,rax\n\t"
10685        $$emit$$"sub     0x4,rcx\n\t"
10686        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10687        $$emit$$"add     0x4,rcx\n\t"
10688        $$emit$$"jle     L_end\n\t"
10689        $$emit$$"dec     rcx\n\t"
10690        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10691        $$emit$$"vmovq   xmm0,(rax)\n\t"
10692        $$emit$$"add     0x8,rax\n\t"
10693        $$emit$$"dec     rcx\n\t"
10694        $$emit$$"jge     L_sloop\n\t"
10695        $$emit$$"# L_end:\n\t"
10696     } else {
10697        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10698        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10699     }
10700   %}
10701   ins_encode %{
10702     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10703                  $tmp$$XMMRegister, true, knoreg);
10704   %}
10705   ins_pipe(pipe_slow);
10706 %}
10707 
10708 // Large ClearArray AVX512.
10709 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10710                              Universe dummy, rFlagsReg cr)
10711 %{
10712   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
10713   match(Set dummy (ClearArray cnt base));
10714   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10715 
10716   format %{ $$template
10717     if (UseFastStosb) {
10718        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10719        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10720        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10721     } else if (UseXMMForObjInit) {
10722        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10723        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10724        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10725        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10726        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10727        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10728        $$emit$$"add     0x40,rax\n\t"
10729        $$emit$$"# L_zero_64_bytes:\n\t"
10730        $$emit$$"sub     0x8,rcx\n\t"
10731        $$emit$$"jge     L_loop\n\t"
10732        $$emit$$"add     0x4,rcx\n\t"
10733        $$emit$$"jl      L_tail\n\t"
10734        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10735        $$emit$$"add     0x20,rax\n\t"
10736        $$emit$$"sub     0x4,rcx\n\t"
10737        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10738        $$emit$$"add     0x4,rcx\n\t"
10739        $$emit$$"jle     L_end\n\t"
10740        $$emit$$"dec     rcx\n\t"
10741        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10742        $$emit$$"vmovq   xmm0,(rax)\n\t"
10743        $$emit$$"add     0x8,rax\n\t"
10744        $$emit$$"dec     rcx\n\t"
10745        $$emit$$"jge     L_sloop\n\t"
10746        $$emit$$"# L_end:\n\t"
10747     } else {
10748        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10749        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10750     }
10751   %}
10752   ins_encode %{
10753     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10754                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
10755   %}
10756   ins_pipe(pipe_slow);
10757 %}
10758 
10759 // Small ClearArray AVX512 constant length.
10760 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
10761 %{
10762   predicate(!((ClearArrayNode*)n)->is_large() &&
10763               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
10764   match(Set dummy (ClearArray cnt base));
10765   ins_cost(100);
10766   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
10767   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
10768   ins_encode %{
10769    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10770   %}
10771   ins_pipe(pipe_slow);
10772 %}
10773 
10774 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10775                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10776 %{
10777   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10778   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10779   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10780 
10781   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10782   ins_encode %{
10783     __ string_compare($str1$$Register, $str2$$Register,
10784                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10785                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
10786   %}
10787   ins_pipe( pipe_slow );
10788 %}
10789 
10790 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10791                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10792 %{
10793   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10794   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10795   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10796 
10797   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10798   ins_encode %{
10799     __ string_compare($str1$$Register, $str2$$Register,
10800                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10801                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
10802   %}
10803   ins_pipe( pipe_slow );
10804 %}
10805 
10806 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10807                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10808 %{
10809   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10810   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10811   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10812 
10813   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10814   ins_encode %{
10815     __ string_compare($str1$$Register, $str2$$Register,
10816                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10817                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
10818   %}
10819   ins_pipe( pipe_slow );
10820 %}
10821 
10822 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10823                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10824 %{
10825   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10826   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10827   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10828 
10829   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10830   ins_encode %{
10831     __ string_compare($str1$$Register, $str2$$Register,
10832                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10833                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
10834   %}
10835   ins_pipe( pipe_slow );
10836 %}
10837 
10838 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10839                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10840 %{
10841   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10842   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10843   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10844 
10845   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10846   ins_encode %{
10847     __ string_compare($str1$$Register, $str2$$Register,
10848                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10849                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
10850   %}
10851   ins_pipe( pipe_slow );
10852 %}
10853 
10854 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10855                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10856 %{
10857   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10858   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10859   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10860 
10861   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10862   ins_encode %{
10863     __ string_compare($str1$$Register, $str2$$Register,
10864                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10865                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
10866   %}
10867   ins_pipe( pipe_slow );
10868 %}
10869 
10870 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10871                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10872 %{
10873   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10874   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10875   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10876 
10877   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10878   ins_encode %{
10879     __ string_compare($str2$$Register, $str1$$Register,
10880                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10881                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
10882   %}
10883   ins_pipe( pipe_slow );
10884 %}
10885 
10886 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10887                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10888 %{
10889   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10890   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10891   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10892 
10893   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10894   ins_encode %{
10895     __ string_compare($str2$$Register, $str1$$Register,
10896                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10897                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
10898   %}
10899   ins_pipe( pipe_slow );
10900 %}
10901 
10902 // fast search of substring with known size.
10903 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10904                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10905 %{
10906   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10907   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10908   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10909 
10910   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10911   ins_encode %{
10912     int icnt2 = (int)$int_cnt2$$constant;
10913     if (icnt2 >= 16) {
10914       // IndexOf for constant substrings with size >= 16 elements
10915       // which don't need to be loaded through stack.
10916       __ string_indexofC8($str1$$Register, $str2$$Register,
10917                           $cnt1$$Register, $cnt2$$Register,
10918                           icnt2, $result$$Register,
10919                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10920     } else {
10921       // Small strings are loaded through stack if they cross page boundary.
10922       __ string_indexof($str1$$Register, $str2$$Register,
10923                         $cnt1$$Register, $cnt2$$Register,
10924                         icnt2, $result$$Register,
10925                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10926     }
10927   %}
10928   ins_pipe( pipe_slow );
10929 %}
10930 
10931 // fast search of substring with known size.
10932 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10933                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10934 %{
10935   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10936   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10937   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10938 
10939   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10940   ins_encode %{
10941     int icnt2 = (int)$int_cnt2$$constant;
10942     if (icnt2 >= 8) {
10943       // IndexOf for constant substrings with size >= 8 elements
10944       // which don't need to be loaded through stack.
10945       __ string_indexofC8($str1$$Register, $str2$$Register,
10946                           $cnt1$$Register, $cnt2$$Register,
10947                           icnt2, $result$$Register,
10948                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10949     } else {
10950       // Small strings are loaded through stack if they cross page boundary.
10951       __ string_indexof($str1$$Register, $str2$$Register,
10952                         $cnt1$$Register, $cnt2$$Register,
10953                         icnt2, $result$$Register,
10954                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10955     }
10956   %}
10957   ins_pipe( pipe_slow );
10958 %}
10959 
10960 // fast search of substring with known size.
10961 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10962                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10963 %{
10964   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10965   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10966   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10967 
10968   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10969   ins_encode %{
10970     int icnt2 = (int)$int_cnt2$$constant;
10971     if (icnt2 >= 8) {
10972       // IndexOf for constant substrings with size >= 8 elements
10973       // which don't need to be loaded through stack.
10974       __ string_indexofC8($str1$$Register, $str2$$Register,
10975                           $cnt1$$Register, $cnt2$$Register,
10976                           icnt2, $result$$Register,
10977                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10978     } else {
10979       // Small strings are loaded through stack if they cross page boundary.
10980       __ string_indexof($str1$$Register, $str2$$Register,
10981                         $cnt1$$Register, $cnt2$$Register,
10982                         icnt2, $result$$Register,
10983                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10984     }
10985   %}
10986   ins_pipe( pipe_slow );
10987 %}
10988 
10989 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10990                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10991 %{
10992   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10993   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10994   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10995 
10996   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10997   ins_encode %{
10998     __ string_indexof($str1$$Register, $str2$$Register,
10999                       $cnt1$$Register, $cnt2$$Register,
11000                       (-1), $result$$Register,
11001                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11002   %}
11003   ins_pipe( pipe_slow );
11004 %}
11005 
11006 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11007                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11008 %{
11009   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11010   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11011   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11012 
11013   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11014   ins_encode %{
11015     __ string_indexof($str1$$Register, $str2$$Register,
11016                       $cnt1$$Register, $cnt2$$Register,
11017                       (-1), $result$$Register,
11018                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11019   %}
11020   ins_pipe( pipe_slow );
11021 %}
11022 
11023 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11024                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11025 %{
11026   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11027   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11028   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11029 
11030   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11031   ins_encode %{
11032     __ string_indexof($str1$$Register, $str2$$Register,
11033                       $cnt1$$Register, $cnt2$$Register,
11034                       (-1), $result$$Register,
11035                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11036   %}
11037   ins_pipe( pipe_slow );
11038 %}
11039 
11040 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11041                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11042 %{
11043   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11044   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11045   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11046   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11047   ins_encode %{
11048     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11049                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11050   %}
11051   ins_pipe( pipe_slow );
11052 %}
11053 
11054 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11055                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11056 %{
11057   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11058   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11059   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11060   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11061   ins_encode %{
11062     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11063                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11064   %}
11065   ins_pipe( pipe_slow );
11066 %}
11067 
11068 // fast string equals
11069 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11070                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11071 %{
11072   predicate(!VM_Version::supports_avx512vlbw());
11073   match(Set result (StrEquals (Binary str1 str2) cnt));
11074   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11075 
11076   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11077   ins_encode %{
11078     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11079                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11080                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11081   %}
11082   ins_pipe( pipe_slow );
11083 %}
11084 
11085 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11086                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11087 %{
11088   predicate(VM_Version::supports_avx512vlbw());
11089   match(Set result (StrEquals (Binary str1 str2) cnt));
11090   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11091 
11092   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11093   ins_encode %{
11094     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11095                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11096                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11097   %}
11098   ins_pipe( pipe_slow );
11099 %}
11100 
11101 // fast array equals
11102 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11103                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11104 %{
11105   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11106   match(Set result (AryEq ary1 ary2));
11107   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11108 
11109   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11110   ins_encode %{
11111     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11112                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11113                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11114   %}
11115   ins_pipe( pipe_slow );
11116 %}
11117 
11118 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11119                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11120 %{
11121   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11122   match(Set result (AryEq ary1 ary2));
11123   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11124 
11125   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11126   ins_encode %{
11127     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11128                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11129                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11130   %}
11131   ins_pipe( pipe_slow );
11132 %}
11133 
11134 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11135                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11136 %{
11137   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11138   match(Set result (AryEq ary1 ary2));
11139   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11140 
11141   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11142   ins_encode %{
11143     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11144                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11145                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11146   %}
11147   ins_pipe( pipe_slow );
11148 %}
11149 
11150 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11151                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11152 %{
11153   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11154   match(Set result (AryEq ary1 ary2));
11155   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11156 
11157   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11158   ins_encode %{
11159     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11160                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11161                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11162   %}
11163   ins_pipe( pipe_slow );
11164 %}
11165 
11166 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11167                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11168                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11169                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11170                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11171 %{
11172   predicate(UseAVX >= 2);
11173   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11174   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11175          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11176          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11177          USE basic_type, KILL cr);
11178 
11179   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
11180   ins_encode %{
11181     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11182                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11183                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11184                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11185                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11186                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11187                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11188   %}
11189   ins_pipe( pipe_slow );
11190 %}
11191 
11192 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11193                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11194 %{
11195   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11196   match(Set result (CountPositives ary1 len));
11197   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11198 
11199   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11200   ins_encode %{
11201     __ count_positives($ary1$$Register, $len$$Register,
11202                        $result$$Register, $tmp3$$Register,
11203                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11204   %}
11205   ins_pipe( pipe_slow );
11206 %}
11207 
11208 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11209                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11210 %{
11211   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11212   match(Set result (CountPositives ary1 len));
11213   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11214 
11215   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11216   ins_encode %{
11217     __ count_positives($ary1$$Register, $len$$Register,
11218                        $result$$Register, $tmp3$$Register,
11219                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11220   %}
11221   ins_pipe( pipe_slow );
11222 %}
11223 
11224 // fast char[] to byte[] compression
11225 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11226                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11227   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11228   match(Set result (StrCompressedCopy src (Binary dst len)));
11229   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11230          USE_KILL len, KILL tmp5, KILL cr);
11231 
11232   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11233   ins_encode %{
11234     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11235                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11236                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11237                            knoreg, knoreg);
11238   %}
11239   ins_pipe( pipe_slow );
11240 %}
11241 
11242 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11243                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11244   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11245   match(Set result (StrCompressedCopy src (Binary dst len)));
11246   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11247          USE_KILL len, KILL tmp5, KILL cr);
11248 
11249   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11250   ins_encode %{
11251     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11252                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11253                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11254                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11255   %}
11256   ins_pipe( pipe_slow );
11257 %}
11258 // fast byte[] to char[] inflation
11259 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11260                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11261   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11262   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11263   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11264 
11265   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11266   ins_encode %{
11267     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11268                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11269   %}
11270   ins_pipe( pipe_slow );
11271 %}
11272 
11273 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11274                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11275   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11276   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11277   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11278 
11279   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11280   ins_encode %{
11281     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11282                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11283   %}
11284   ins_pipe( pipe_slow );
11285 %}
11286 
11287 // encode char[] to byte[] in ISO_8859_1
11288 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11289                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11290                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11291   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11292   match(Set result (EncodeISOArray src (Binary dst len)));
11293   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11294 
11295   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11296   ins_encode %{
11297     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11298                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11299                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11300   %}
11301   ins_pipe( pipe_slow );
11302 %}
11303 
11304 // encode char[] to byte[] in ASCII
11305 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11306                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11307                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11308   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11309   match(Set result (EncodeISOArray src (Binary dst len)));
11310   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11311 
11312   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11313   ins_encode %{
11314     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11315                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11316                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11317   %}
11318   ins_pipe( pipe_slow );
11319 %}
11320 
11321 //----------Overflow Math Instructions-----------------------------------------
11322 
11323 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11324 %{
11325   match(Set cr (OverflowAddI op1 op2));
11326   effect(DEF cr, USE_KILL op1, USE op2);
11327 
11328   format %{ "addl    $op1, $op2\t# overflow check int" %}
11329 
11330   ins_encode %{
11331     __ addl($op1$$Register, $op2$$Register);
11332   %}
11333   ins_pipe(ialu_reg_reg);
11334 %}
11335 
11336 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11337 %{
11338   match(Set cr (OverflowAddI op1 op2));
11339   effect(DEF cr, USE_KILL op1, USE op2);
11340 
11341   format %{ "addl    $op1, $op2\t# overflow check int" %}
11342 
11343   ins_encode %{
11344     __ addl($op1$$Register, $op2$$constant);
11345   %}
11346   ins_pipe(ialu_reg_reg);
11347 %}
11348 
11349 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11350 %{
11351   match(Set cr (OverflowAddL op1 op2));
11352   effect(DEF cr, USE_KILL op1, USE op2);
11353 
11354   format %{ "addq    $op1, $op2\t# overflow check long" %}
11355   ins_encode %{
11356     __ addq($op1$$Register, $op2$$Register);
11357   %}
11358   ins_pipe(ialu_reg_reg);
11359 %}
11360 
11361 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11362 %{
11363   match(Set cr (OverflowAddL op1 op2));
11364   effect(DEF cr, USE_KILL op1, USE op2);
11365 
11366   format %{ "addq    $op1, $op2\t# overflow check long" %}
11367   ins_encode %{
11368     __ addq($op1$$Register, $op2$$constant);
11369   %}
11370   ins_pipe(ialu_reg_reg);
11371 %}
11372 
11373 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11374 %{
11375   match(Set cr (OverflowSubI op1 op2));
11376 
11377   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11378   ins_encode %{
11379     __ cmpl($op1$$Register, $op2$$Register);
11380   %}
11381   ins_pipe(ialu_reg_reg);
11382 %}
11383 
11384 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11385 %{
11386   match(Set cr (OverflowSubI op1 op2));
11387 
11388   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11389   ins_encode %{
11390     __ cmpl($op1$$Register, $op2$$constant);
11391   %}
11392   ins_pipe(ialu_reg_reg);
11393 %}
11394 
11395 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11396 %{
11397   match(Set cr (OverflowSubL op1 op2));
11398 
11399   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11400   ins_encode %{
11401     __ cmpq($op1$$Register, $op2$$Register);
11402   %}
11403   ins_pipe(ialu_reg_reg);
11404 %}
11405 
11406 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11407 %{
11408   match(Set cr (OverflowSubL op1 op2));
11409 
11410   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11411   ins_encode %{
11412     __ cmpq($op1$$Register, $op2$$constant);
11413   %}
11414   ins_pipe(ialu_reg_reg);
11415 %}
11416 
11417 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11418 %{
11419   match(Set cr (OverflowSubI zero op2));
11420   effect(DEF cr, USE_KILL op2);
11421 
11422   format %{ "negl    $op2\t# overflow check int" %}
11423   ins_encode %{
11424     __ negl($op2$$Register);
11425   %}
11426   ins_pipe(ialu_reg_reg);
11427 %}
11428 
11429 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11430 %{
11431   match(Set cr (OverflowSubL zero op2));
11432   effect(DEF cr, USE_KILL op2);
11433 
11434   format %{ "negq    $op2\t# overflow check long" %}
11435   ins_encode %{
11436     __ negq($op2$$Register);
11437   %}
11438   ins_pipe(ialu_reg_reg);
11439 %}
11440 
11441 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11442 %{
11443   match(Set cr (OverflowMulI op1 op2));
11444   effect(DEF cr, USE_KILL op1, USE op2);
11445 
11446   format %{ "imull    $op1, $op2\t# overflow check int" %}
11447   ins_encode %{
11448     __ imull($op1$$Register, $op2$$Register);
11449   %}
11450   ins_pipe(ialu_reg_reg_alu0);
11451 %}
11452 
11453 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11454 %{
11455   match(Set cr (OverflowMulI op1 op2));
11456   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11457 
11458   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11459   ins_encode %{
11460     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11461   %}
11462   ins_pipe(ialu_reg_reg_alu0);
11463 %}
11464 
11465 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11466 %{
11467   match(Set cr (OverflowMulL op1 op2));
11468   effect(DEF cr, USE_KILL op1, USE op2);
11469 
11470   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11471   ins_encode %{
11472     __ imulq($op1$$Register, $op2$$Register);
11473   %}
11474   ins_pipe(ialu_reg_reg_alu0);
11475 %}
11476 
11477 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11478 %{
11479   match(Set cr (OverflowMulL op1 op2));
11480   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11481 
11482   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11483   ins_encode %{
11484     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11485   %}
11486   ins_pipe(ialu_reg_reg_alu0);
11487 %}
11488 
11489 
11490 //----------Control Flow Instructions------------------------------------------
11491 // Signed compare Instructions
11492 
11493 // XXX more variants!!
11494 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11495 %{
11496   match(Set cr (CmpI op1 op2));
11497   effect(DEF cr, USE op1, USE op2);
11498 
11499   format %{ "cmpl    $op1, $op2" %}
11500   ins_encode %{
11501     __ cmpl($op1$$Register, $op2$$Register);
11502   %}
11503   ins_pipe(ialu_cr_reg_reg);
11504 %}
11505 
11506 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11507 %{
11508   match(Set cr (CmpI op1 op2));
11509 
11510   format %{ "cmpl    $op1, $op2" %}
11511   ins_encode %{
11512     __ cmpl($op1$$Register, $op2$$constant);
11513   %}
11514   ins_pipe(ialu_cr_reg_imm);
11515 %}
11516 
11517 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11518 %{
11519   match(Set cr (CmpI op1 (LoadI op2)));
11520 
11521   ins_cost(500); // XXX
11522   format %{ "cmpl    $op1, $op2" %}
11523   ins_encode %{
11524     __ cmpl($op1$$Register, $op2$$Address);
11525   %}
11526   ins_pipe(ialu_cr_reg_mem);
11527 %}
11528 
11529 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11530 %{
11531   match(Set cr (CmpI src zero));
11532 
11533   format %{ "testl   $src, $src" %}
11534   ins_encode %{
11535     __ testl($src$$Register, $src$$Register);
11536   %}
11537   ins_pipe(ialu_cr_reg_imm);
11538 %}
11539 
11540 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11541 %{
11542   match(Set cr (CmpI (AndI src con) zero));
11543 
11544   format %{ "testl   $src, $con" %}
11545   ins_encode %{
11546     __ testl($src$$Register, $con$$constant);
11547   %}
11548   ins_pipe(ialu_cr_reg_imm);
11549 %}
11550 
11551 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11552 %{
11553   match(Set cr (CmpI (AndI src1 src2) zero));
11554 
11555   format %{ "testl   $src1, $src2" %}
11556   ins_encode %{
11557     __ testl($src1$$Register, $src2$$Register);
11558   %}
11559   ins_pipe(ialu_cr_reg_imm);
11560 %}
11561 
11562 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11563 %{
11564   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11565 
11566   format %{ "testl   $src, $mem" %}
11567   ins_encode %{
11568     __ testl($src$$Register, $mem$$Address);
11569   %}
11570   ins_pipe(ialu_cr_reg_mem);
11571 %}
11572 
11573 // Unsigned compare Instructions; really, same as signed except they
11574 // produce an rFlagsRegU instead of rFlagsReg.
11575 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11576 %{
11577   match(Set cr (CmpU op1 op2));
11578 
11579   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11580   ins_encode %{
11581     __ cmpl($op1$$Register, $op2$$Register);
11582   %}
11583   ins_pipe(ialu_cr_reg_reg);
11584 %}
11585 
11586 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11587 %{
11588   match(Set cr (CmpU op1 op2));
11589 
11590   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11591   ins_encode %{
11592     __ cmpl($op1$$Register, $op2$$constant);
11593   %}
11594   ins_pipe(ialu_cr_reg_imm);
11595 %}
11596 
11597 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11598 %{
11599   match(Set cr (CmpU op1 (LoadI op2)));
11600 
11601   ins_cost(500); // XXX
11602   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11603   ins_encode %{
11604     __ cmpl($op1$$Register, $op2$$Address);
11605   %}
11606   ins_pipe(ialu_cr_reg_mem);
11607 %}
11608 
11609 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11610 %{
11611   match(Set cr (CmpU src zero));
11612 
11613   format %{ "testl   $src, $src\t# unsigned" %}
11614   ins_encode %{
11615     __ testl($src$$Register, $src$$Register);
11616   %}
11617   ins_pipe(ialu_cr_reg_imm);
11618 %}
11619 
11620 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11621 %{
11622   match(Set cr (CmpP op1 op2));
11623 
11624   format %{ "cmpq    $op1, $op2\t# ptr" %}
11625   ins_encode %{
11626     __ cmpq($op1$$Register, $op2$$Register);
11627   %}
11628   ins_pipe(ialu_cr_reg_reg);
11629 %}
11630 
11631 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11632 %{
11633   match(Set cr (CmpP op1 (LoadP op2)));
11634   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11635 
11636   ins_cost(500); // XXX
11637   format %{ "cmpq    $op1, $op2\t# ptr" %}
11638   ins_encode %{
11639     __ cmpq($op1$$Register, $op2$$Address);
11640   %}
11641   ins_pipe(ialu_cr_reg_mem);
11642 %}
11643 
11644 // XXX this is generalized by compP_rReg_mem???
11645 // Compare raw pointer (used in out-of-heap check).
11646 // Only works because non-oop pointers must be raw pointers
11647 // and raw pointers have no anti-dependencies.
11648 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11649 %{
11650   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11651             n->in(2)->as_Load()->barrier_data() == 0);
11652   match(Set cr (CmpP op1 (LoadP op2)));
11653 
11654   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11655   ins_encode %{
11656     __ cmpq($op1$$Register, $op2$$Address);
11657   %}
11658   ins_pipe(ialu_cr_reg_mem);
11659 %}
11660 
11661 // This will generate a signed flags result. This should be OK since
11662 // any compare to a zero should be eq/neq.
11663 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11664 %{
11665   match(Set cr (CmpP src zero));
11666 
11667   format %{ "testq   $src, $src\t# ptr" %}
11668   ins_encode %{
11669     __ testq($src$$Register, $src$$Register);
11670   %}
11671   ins_pipe(ialu_cr_reg_imm);
11672 %}
11673 
11674 // This will generate a signed flags result. This should be OK since
11675 // any compare to a zero should be eq/neq.
11676 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11677 %{
11678   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
11679             n->in(1)->as_Load()->barrier_data() == 0);
11680   match(Set cr (CmpP (LoadP op) zero));
11681 
11682   ins_cost(500); // XXX
11683   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11684   ins_encode %{
11685     __ testq($op$$Address, 0xFFFFFFFF);
11686   %}
11687   ins_pipe(ialu_cr_reg_imm);
11688 %}
11689 
11690 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11691 %{
11692   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
11693             n->in(1)->as_Load()->barrier_data() == 0);
11694   match(Set cr (CmpP (LoadP mem) zero));
11695 
11696   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11697   ins_encode %{
11698     __ cmpq(r12, $mem$$Address);
11699   %}
11700   ins_pipe(ialu_cr_reg_mem);
11701 %}
11702 
11703 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11704 %{
11705   match(Set cr (CmpN op1 op2));
11706 
11707   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11708   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11709   ins_pipe(ialu_cr_reg_reg);
11710 %}
11711 
11712 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11713 %{
11714   match(Set cr (CmpN src (LoadN mem)));
11715 
11716   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11717   ins_encode %{
11718     __ cmpl($src$$Register, $mem$$Address);
11719   %}
11720   ins_pipe(ialu_cr_reg_mem);
11721 %}
11722 
11723 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11724   match(Set cr (CmpN op1 op2));
11725 
11726   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11727   ins_encode %{
11728     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11729   %}
11730   ins_pipe(ialu_cr_reg_imm);
11731 %}
11732 
11733 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11734 %{
11735   match(Set cr (CmpN src (LoadN mem)));
11736 
11737   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11738   ins_encode %{
11739     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11740   %}
11741   ins_pipe(ialu_cr_reg_mem);
11742 %}
11743 
11744 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11745   match(Set cr (CmpN op1 op2));
11746 
11747   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11748   ins_encode %{
11749     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11750   %}
11751   ins_pipe(ialu_cr_reg_imm);
11752 %}
11753 
11754 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11755 %{
11756   predicate(!UseCompactObjectHeaders);
11757   match(Set cr (CmpN src (LoadNKlass mem)));
11758 
11759   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11760   ins_encode %{
11761     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11762   %}
11763   ins_pipe(ialu_cr_reg_mem);
11764 %}
11765 
11766 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11767   match(Set cr (CmpN src zero));
11768 
11769   format %{ "testl   $src, $src\t# compressed ptr" %}
11770   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11771   ins_pipe(ialu_cr_reg_imm);
11772 %}
11773 
11774 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11775 %{
11776   predicate(CompressedOops::base() != nullptr);
11777   match(Set cr (CmpN (LoadN mem) zero));
11778 
11779   ins_cost(500); // XXX
11780   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11781   ins_encode %{
11782     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11783   %}
11784   ins_pipe(ialu_cr_reg_mem);
11785 %}
11786 
11787 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11788 %{
11789   predicate(CompressedOops::base() == nullptr);
11790   match(Set cr (CmpN (LoadN mem) zero));
11791 
11792   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11793   ins_encode %{
11794     __ cmpl(r12, $mem$$Address);
11795   %}
11796   ins_pipe(ialu_cr_reg_mem);
11797 %}
11798 
11799 // Yanked all unsigned pointer compare operations.
11800 // Pointer compares are done with CmpP which is already unsigned.
11801 
11802 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11803 %{
11804   match(Set cr (CmpL op1 op2));
11805 
11806   format %{ "cmpq    $op1, $op2" %}
11807   ins_encode %{
11808     __ cmpq($op1$$Register, $op2$$Register);
11809   %}
11810   ins_pipe(ialu_cr_reg_reg);
11811 %}
11812 
11813 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11814 %{
11815   match(Set cr (CmpL op1 op2));
11816 
11817   format %{ "cmpq    $op1, $op2" %}
11818   ins_encode %{
11819     __ cmpq($op1$$Register, $op2$$constant);
11820   %}
11821   ins_pipe(ialu_cr_reg_imm);
11822 %}
11823 
11824 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11825 %{
11826   match(Set cr (CmpL op1 (LoadL op2)));
11827 
11828   format %{ "cmpq    $op1, $op2" %}
11829   ins_encode %{
11830     __ cmpq($op1$$Register, $op2$$Address);
11831   %}
11832   ins_pipe(ialu_cr_reg_mem);
11833 %}
11834 
11835 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11836 %{
11837   match(Set cr (CmpL src zero));
11838 
11839   format %{ "testq   $src, $src" %}
11840   ins_encode %{
11841     __ testq($src$$Register, $src$$Register);
11842   %}
11843   ins_pipe(ialu_cr_reg_imm);
11844 %}
11845 
11846 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11847 %{
11848   match(Set cr (CmpL (AndL src con) zero));
11849 
11850   format %{ "testq   $src, $con\t# long" %}
11851   ins_encode %{
11852     __ testq($src$$Register, $con$$constant);
11853   %}
11854   ins_pipe(ialu_cr_reg_imm);
11855 %}
11856 
11857 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
11858 %{
11859   match(Set cr (CmpL (AndL src1 src2) zero));
11860 
11861   format %{ "testq   $src1, $src2\t# long" %}
11862   ins_encode %{
11863     __ testq($src1$$Register, $src2$$Register);
11864   %}
11865   ins_pipe(ialu_cr_reg_imm);
11866 %}
11867 
11868 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11869 %{
11870   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11871 
11872   format %{ "testq   $src, $mem" %}
11873   ins_encode %{
11874     __ testq($src$$Register, $mem$$Address);
11875   %}
11876   ins_pipe(ialu_cr_reg_mem);
11877 %}
11878 
11879 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11880 %{
11881   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11882 
11883   format %{ "testq   $src, $mem" %}
11884   ins_encode %{
11885     __ testq($src$$Register, $mem$$Address);
11886   %}
11887   ins_pipe(ialu_cr_reg_mem);
11888 %}
11889 
11890 // Manifest a CmpU result in an integer register.  Very painful.
11891 // This is the test to avoid.
11892 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
11893 %{
11894   match(Set dst (CmpU3 src1 src2));
11895   effect(KILL flags);
11896 
11897   ins_cost(275); // XXX
11898   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
11899             "movl    $dst, -1\n\t"
11900             "jb,u    done\n\t"
11901             "setne   $dst\n\t"
11902             "movzbl  $dst, $dst\n\t"
11903     "done:" %}
11904   ins_encode %{
11905     Label done;
11906     __ cmpl($src1$$Register, $src2$$Register);
11907     __ movl($dst$$Register, -1);
11908     __ jccb(Assembler::below, done);
11909     __ setb(Assembler::notZero, $dst$$Register);
11910     __ movzbl($dst$$Register, $dst$$Register);
11911     __ bind(done);
11912   %}
11913   ins_pipe(pipe_slow);
11914 %}
11915 
11916 // Manifest a CmpL result in an integer register.  Very painful.
11917 // This is the test to avoid.
11918 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11919 %{
11920   match(Set dst (CmpL3 src1 src2));
11921   effect(KILL flags);
11922 
11923   ins_cost(275); // XXX
11924   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11925             "movl    $dst, -1\n\t"
11926             "jl,s    done\n\t"
11927             "setne   $dst\n\t"
11928             "movzbl  $dst, $dst\n\t"
11929     "done:" %}
11930   ins_encode %{
11931     Label done;
11932     __ cmpq($src1$$Register, $src2$$Register);
11933     __ movl($dst$$Register, -1);
11934     __ jccb(Assembler::less, done);
11935     __ setb(Assembler::notZero, $dst$$Register);
11936     __ movzbl($dst$$Register, $dst$$Register);
11937     __ bind(done);
11938   %}
11939   ins_pipe(pipe_slow);
11940 %}
11941 
11942 // Manifest a CmpUL result in an integer register.  Very painful.
11943 // This is the test to avoid.
11944 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11945 %{
11946   match(Set dst (CmpUL3 src1 src2));
11947   effect(KILL flags);
11948 
11949   ins_cost(275); // XXX
11950   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11951             "movl    $dst, -1\n\t"
11952             "jb,u    done\n\t"
11953             "setne   $dst\n\t"
11954             "movzbl  $dst, $dst\n\t"
11955     "done:" %}
11956   ins_encode %{
11957     Label done;
11958     __ cmpq($src1$$Register, $src2$$Register);
11959     __ movl($dst$$Register, -1);
11960     __ jccb(Assembler::below, done);
11961     __ setb(Assembler::notZero, $dst$$Register);
11962     __ movzbl($dst$$Register, $dst$$Register);
11963     __ bind(done);
11964   %}
11965   ins_pipe(pipe_slow);
11966 %}
11967 
11968 // Unsigned long compare Instructions; really, same as signed long except they
11969 // produce an rFlagsRegU instead of rFlagsReg.
11970 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11971 %{
11972   match(Set cr (CmpUL op1 op2));
11973 
11974   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11975   ins_encode %{
11976     __ cmpq($op1$$Register, $op2$$Register);
11977   %}
11978   ins_pipe(ialu_cr_reg_reg);
11979 %}
11980 
11981 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11982 %{
11983   match(Set cr (CmpUL op1 op2));
11984 
11985   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11986   ins_encode %{
11987     __ cmpq($op1$$Register, $op2$$constant);
11988   %}
11989   ins_pipe(ialu_cr_reg_imm);
11990 %}
11991 
11992 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11993 %{
11994   match(Set cr (CmpUL op1 (LoadL op2)));
11995 
11996   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11997   ins_encode %{
11998     __ cmpq($op1$$Register, $op2$$Address);
11999   %}
12000   ins_pipe(ialu_cr_reg_mem);
12001 %}
12002 
12003 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12004 %{
12005   match(Set cr (CmpUL src zero));
12006 
12007   format %{ "testq   $src, $src\t# unsigned" %}
12008   ins_encode %{
12009     __ testq($src$$Register, $src$$Register);
12010   %}
12011   ins_pipe(ialu_cr_reg_imm);
12012 %}
12013 
12014 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12015 %{
12016   match(Set cr (CmpI (LoadB mem) imm));
12017 
12018   ins_cost(125);
12019   format %{ "cmpb    $mem, $imm" %}
12020   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12021   ins_pipe(ialu_cr_reg_mem);
12022 %}
12023 
12024 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12025 %{
12026   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12027 
12028   ins_cost(125);
12029   format %{ "testb   $mem, $imm\t# ubyte" %}
12030   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12031   ins_pipe(ialu_cr_reg_mem);
12032 %}
12033 
12034 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12035 %{
12036   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12037 
12038   ins_cost(125);
12039   format %{ "testb   $mem, $imm\t# byte" %}
12040   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12041   ins_pipe(ialu_cr_reg_mem);
12042 %}
12043 
12044 //----------Max and Min--------------------------------------------------------
12045 // Min Instructions
12046 
12047 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12048 %{
12049   effect(USE_DEF dst, USE src, USE cr);
12050 
12051   format %{ "cmovlgt $dst, $src\t# min" %}
12052   ins_encode %{
12053     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12054   %}
12055   ins_pipe(pipe_cmov_reg);
12056 %}
12057 
12058 
12059 instruct minI_rReg(rRegI dst, rRegI src)
12060 %{
12061   match(Set dst (MinI dst src));
12062 
12063   ins_cost(200);
12064   expand %{
12065     rFlagsReg cr;
12066     compI_rReg(cr, dst, src);
12067     cmovI_reg_g(dst, src, cr);
12068   %}
12069 %}
12070 
12071 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12072 %{
12073   effect(USE_DEF dst, USE src, USE cr);
12074 
12075   format %{ "cmovllt $dst, $src\t# max" %}
12076   ins_encode %{
12077     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12078   %}
12079   ins_pipe(pipe_cmov_reg);
12080 %}
12081 
12082 
12083 instruct maxI_rReg(rRegI dst, rRegI src)
12084 %{
12085   match(Set dst (MaxI dst src));
12086 
12087   ins_cost(200);
12088   expand %{
12089     rFlagsReg cr;
12090     compI_rReg(cr, dst, src);
12091     cmovI_reg_l(dst, src, cr);
12092   %}
12093 %}
12094 
12095 // ============================================================================
12096 // Branch Instructions
12097 
12098 // Jump Direct - Label defines a relative address from JMP+1
12099 instruct jmpDir(label labl)
12100 %{
12101   match(Goto);
12102   effect(USE labl);
12103 
12104   ins_cost(300);
12105   format %{ "jmp     $labl" %}
12106   size(5);
12107   ins_encode %{
12108     Label* L = $labl$$label;
12109     __ jmp(*L, false); // Always long jump
12110   %}
12111   ins_pipe(pipe_jmp);
12112 %}
12113 
12114 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12115 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12116 %{
12117   match(If cop cr);
12118   effect(USE labl);
12119 
12120   ins_cost(300);
12121   format %{ "j$cop     $labl" %}
12122   size(6);
12123   ins_encode %{
12124     Label* L = $labl$$label;
12125     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12126   %}
12127   ins_pipe(pipe_jcc);
12128 %}
12129 
12130 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12131 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12132 %{
12133   match(CountedLoopEnd cop cr);
12134   effect(USE labl);
12135 
12136   ins_cost(300);
12137   format %{ "j$cop     $labl\t# loop end" %}
12138   size(6);
12139   ins_encode %{
12140     Label* L = $labl$$label;
12141     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12142   %}
12143   ins_pipe(pipe_jcc);
12144 %}
12145 
12146 // Jump Direct Conditional - using unsigned comparison
12147 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12148   match(If cop cmp);
12149   effect(USE labl);
12150 
12151   ins_cost(300);
12152   format %{ "j$cop,u   $labl" %}
12153   size(6);
12154   ins_encode %{
12155     Label* L = $labl$$label;
12156     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12157   %}
12158   ins_pipe(pipe_jcc);
12159 %}
12160 
12161 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12162   match(If cop cmp);
12163   effect(USE labl);
12164 
12165   ins_cost(200);
12166   format %{ "j$cop,u   $labl" %}
12167   size(6);
12168   ins_encode %{
12169     Label* L = $labl$$label;
12170     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12171   %}
12172   ins_pipe(pipe_jcc);
12173 %}
12174 
12175 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12176   match(If cop cmp);
12177   effect(USE labl);
12178 
12179   ins_cost(200);
12180   format %{ $$template
12181     if ($cop$$cmpcode == Assembler::notEqual) {
12182       $$emit$$"jp,u    $labl\n\t"
12183       $$emit$$"j$cop,u   $labl"
12184     } else {
12185       $$emit$$"jp,u    done\n\t"
12186       $$emit$$"j$cop,u   $labl\n\t"
12187       $$emit$$"done:"
12188     }
12189   %}
12190   ins_encode %{
12191     Label* l = $labl$$label;
12192     if ($cop$$cmpcode == Assembler::notEqual) {
12193       __ jcc(Assembler::parity, *l, false);
12194       __ jcc(Assembler::notEqual, *l, false);
12195     } else if ($cop$$cmpcode == Assembler::equal) {
12196       Label done;
12197       __ jccb(Assembler::parity, done);
12198       __ jcc(Assembler::equal, *l, false);
12199       __ bind(done);
12200     } else {
12201        ShouldNotReachHere();
12202     }
12203   %}
12204   ins_pipe(pipe_jcc);
12205 %}
12206 
12207 // ============================================================================
12208 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12209 // superklass array for an instance of the superklass.  Set a hidden
12210 // internal cache on a hit (cache is checked with exposed code in
12211 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12212 // encoding ALSO sets flags.
12213 
12214 instruct partialSubtypeCheck(rdi_RegP result,
12215                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12216                              rFlagsReg cr)
12217 %{
12218   match(Set result (PartialSubtypeCheck sub super));
12219   effect(KILL rcx, KILL cr);
12220 
12221   ins_cost(1100);  // slightly larger than the next version
12222   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12223             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12224             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12225             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12226             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12227             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12228             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12229     "miss:\t" %}
12230 
12231   opcode(0x1); // Force a XOR of RDI
12232   ins_encode(enc_PartialSubtypeCheck());
12233   ins_pipe(pipe_slow);
12234 %}
12235 
12236 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12237                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12238                                      immP0 zero,
12239                                      rdi_RegP result)
12240 %{
12241   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12242   effect(KILL rcx, KILL result);
12243 
12244   ins_cost(1000);
12245   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12246             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12247             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12248             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12249             "jne,s   miss\t\t# Missed: flags nz\n\t"
12250             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12251     "miss:\t" %}
12252 
12253   opcode(0x0); // No need to XOR RDI
12254   ins_encode(enc_PartialSubtypeCheck());
12255   ins_pipe(pipe_slow);
12256 %}
12257 
12258 // ============================================================================
12259 // Branch Instructions -- short offset versions
12260 //
12261 // These instructions are used to replace jumps of a long offset (the default
12262 // match) with jumps of a shorter offset.  These instructions are all tagged
12263 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12264 // match rules in general matching.  Instead, the ADLC generates a conversion
12265 // method in the MachNode which can be used to do in-place replacement of the
12266 // long variant with the shorter variant.  The compiler will determine if a
12267 // branch can be taken by the is_short_branch_offset() predicate in the machine
12268 // specific code section of the file.
12269 
12270 // Jump Direct - Label defines a relative address from JMP+1
12271 instruct jmpDir_short(label labl) %{
12272   match(Goto);
12273   effect(USE labl);
12274 
12275   ins_cost(300);
12276   format %{ "jmp,s   $labl" %}
12277   size(2);
12278   ins_encode %{
12279     Label* L = $labl$$label;
12280     __ jmpb(*L);
12281   %}
12282   ins_pipe(pipe_jmp);
12283   ins_short_branch(1);
12284 %}
12285 
12286 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12287 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12288   match(If cop cr);
12289   effect(USE labl);
12290 
12291   ins_cost(300);
12292   format %{ "j$cop,s   $labl" %}
12293   size(2);
12294   ins_encode %{
12295     Label* L = $labl$$label;
12296     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12297   %}
12298   ins_pipe(pipe_jcc);
12299   ins_short_branch(1);
12300 %}
12301 
12302 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12303 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12304   match(CountedLoopEnd cop cr);
12305   effect(USE labl);
12306 
12307   ins_cost(300);
12308   format %{ "j$cop,s   $labl\t# loop end" %}
12309   size(2);
12310   ins_encode %{
12311     Label* L = $labl$$label;
12312     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12313   %}
12314   ins_pipe(pipe_jcc);
12315   ins_short_branch(1);
12316 %}
12317 
12318 // Jump Direct Conditional - using unsigned comparison
12319 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12320   match(If cop cmp);
12321   effect(USE labl);
12322 
12323   ins_cost(300);
12324   format %{ "j$cop,us  $labl" %}
12325   size(2);
12326   ins_encode %{
12327     Label* L = $labl$$label;
12328     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12329   %}
12330   ins_pipe(pipe_jcc);
12331   ins_short_branch(1);
12332 %}
12333 
12334 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12335   match(If cop cmp);
12336   effect(USE labl);
12337 
12338   ins_cost(300);
12339   format %{ "j$cop,us  $labl" %}
12340   size(2);
12341   ins_encode %{
12342     Label* L = $labl$$label;
12343     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12344   %}
12345   ins_pipe(pipe_jcc);
12346   ins_short_branch(1);
12347 %}
12348 
12349 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12350   match(If cop cmp);
12351   effect(USE labl);
12352 
12353   ins_cost(300);
12354   format %{ $$template
12355     if ($cop$$cmpcode == Assembler::notEqual) {
12356       $$emit$$"jp,u,s  $labl\n\t"
12357       $$emit$$"j$cop,u,s  $labl"
12358     } else {
12359       $$emit$$"jp,u,s  done\n\t"
12360       $$emit$$"j$cop,u,s  $labl\n\t"
12361       $$emit$$"done:"
12362     }
12363   %}
12364   size(4);
12365   ins_encode %{
12366     Label* l = $labl$$label;
12367     if ($cop$$cmpcode == Assembler::notEqual) {
12368       __ jccb(Assembler::parity, *l);
12369       __ jccb(Assembler::notEqual, *l);
12370     } else if ($cop$$cmpcode == Assembler::equal) {
12371       Label done;
12372       __ jccb(Assembler::parity, done);
12373       __ jccb(Assembler::equal, *l);
12374       __ bind(done);
12375     } else {
12376        ShouldNotReachHere();
12377     }
12378   %}
12379   ins_pipe(pipe_jcc);
12380   ins_short_branch(1);
12381 %}
12382 
12383 // ============================================================================
12384 // inlined locking and unlocking
12385 
12386 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12387   predicate(Compile::current()->use_rtm());
12388   match(Set cr (FastLock object box));
12389   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12390   ins_cost(300);
12391   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12392   ins_encode %{
12393     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12394                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
12395                  _rtm_counters, _stack_rtm_counters,
12396                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12397                  true, ra_->C->profile_rtm());
12398   %}
12399   ins_pipe(pipe_slow);
12400 %}
12401 
12402 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12403   predicate(LockingMode != LM_LIGHTWEIGHT && !Compile::current()->use_rtm());
12404   match(Set cr (FastLock object box));
12405   effect(TEMP tmp, TEMP scr, USE_KILL box);
12406   ins_cost(300);
12407   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12408   ins_encode %{
12409     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12410                  $scr$$Register, noreg, noreg, r15_thread, nullptr, nullptr, nullptr, false, false);
12411   %}
12412   ins_pipe(pipe_slow);
12413 %}
12414 
12415 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12416   predicate(LockingMode != LM_LIGHTWEIGHT);
12417   match(Set cr (FastUnlock object box));
12418   effect(TEMP tmp, USE_KILL box);
12419   ins_cost(300);
12420   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12421   ins_encode %{
12422     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12423   %}
12424   ins_pipe(pipe_slow);
12425 %}
12426 
12427 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
12428   predicate(LockingMode == LM_LIGHTWEIGHT);
12429   match(Set cr (FastLock object box));
12430   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
12431   ins_cost(300);
12432   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
12433   ins_encode %{
12434     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12435   %}
12436   ins_pipe(pipe_slow);
12437 %}
12438 
12439 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
12440   predicate(LockingMode == LM_LIGHTWEIGHT);
12441   match(Set cr (FastUnlock object rax_reg));
12442   effect(TEMP tmp, USE_KILL rax_reg);
12443   ins_cost(300);
12444   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
12445   ins_encode %{
12446     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12447   %}
12448   ins_pipe(pipe_slow);
12449 %}
12450 
12451 
12452 // ============================================================================
12453 // Safepoint Instructions
12454 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12455 %{
12456   match(SafePoint poll);
12457   effect(KILL cr, USE poll);
12458 
12459   format %{ "testl   rax, [$poll]\t"
12460             "# Safepoint: poll for GC" %}
12461   ins_cost(125);
12462   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12463   ins_encode %{
12464     __ relocate(relocInfo::poll_type);
12465     address pre_pc = __ pc();
12466     __ testl(rax, Address($poll$$Register, 0));
12467     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12468   %}
12469   ins_pipe(ialu_reg_mem);
12470 %}
12471 
12472 instruct mask_all_evexL(kReg dst, rRegL src) %{
12473   match(Set dst (MaskAll src));
12474   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12475   ins_encode %{
12476     int mask_len = Matcher::vector_length(this);
12477     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12478   %}
12479   ins_pipe( pipe_slow );
12480 %}
12481 
12482 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12483   predicate(Matcher::vector_length(n) > 32);
12484   match(Set dst (MaskAll src));
12485   effect(TEMP tmp);
12486   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12487   ins_encode %{
12488     int mask_len = Matcher::vector_length(this);
12489     __ movslq($tmp$$Register, $src$$Register);
12490     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12491   %}
12492   ins_pipe( pipe_slow );
12493 %}
12494 
12495 // ============================================================================
12496 // Procedure Call/Return Instructions
12497 // Call Java Static Instruction
12498 // Note: If this code changes, the corresponding ret_addr_offset() and
12499 //       compute_padding() functions will have to be adjusted.
12500 instruct CallStaticJavaDirect(method meth) %{
12501   match(CallStaticJava);
12502   effect(USE meth);
12503 
12504   ins_cost(300);
12505   format %{ "call,static " %}
12506   opcode(0xE8); /* E8 cd */
12507   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12508   ins_pipe(pipe_slow);
12509   ins_alignment(4);
12510 %}
12511 
12512 // Call Java Dynamic Instruction
12513 // Note: If this code changes, the corresponding ret_addr_offset() and
12514 //       compute_padding() functions will have to be adjusted.
12515 instruct CallDynamicJavaDirect(method meth)
12516 %{
12517   match(CallDynamicJava);
12518   effect(USE meth);
12519 
12520   ins_cost(300);
12521   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12522             "call,dynamic " %}
12523   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12524   ins_pipe(pipe_slow);
12525   ins_alignment(4);
12526 %}
12527 
12528 // Call Runtime Instruction
12529 instruct CallRuntimeDirect(method meth)
12530 %{
12531   match(CallRuntime);
12532   effect(USE meth);
12533 
12534   ins_cost(300);
12535   format %{ "call,runtime " %}
12536   ins_encode(clear_avx, Java_To_Runtime(meth));
12537   ins_pipe(pipe_slow);
12538 %}
12539 
12540 // Call runtime without safepoint
12541 instruct CallLeafDirect(method meth)
12542 %{
12543   match(CallLeaf);
12544   effect(USE meth);
12545 
12546   ins_cost(300);
12547   format %{ "call_leaf,runtime " %}
12548   ins_encode(clear_avx, Java_To_Runtime(meth));
12549   ins_pipe(pipe_slow);
12550 %}
12551 
12552 // Call runtime without safepoint and with vector arguments
12553 instruct CallLeafDirectVector(method meth)
12554 %{
12555   match(CallLeafVector);
12556   effect(USE meth);
12557 
12558   ins_cost(300);
12559   format %{ "call_leaf,vector " %}
12560   ins_encode(Java_To_Runtime(meth));
12561   ins_pipe(pipe_slow);
12562 %}
12563 
12564 // Call runtime without safepoint
12565 instruct CallLeafNoFPDirect(method meth)
12566 %{
12567   match(CallLeafNoFP);
12568   effect(USE meth);
12569 
12570   ins_cost(300);
12571   format %{ "call_leaf_nofp,runtime " %}
12572   ins_encode(clear_avx, Java_To_Runtime(meth));
12573   ins_pipe(pipe_slow);
12574 %}
12575 
12576 // Return Instruction
12577 // Remove the return address & jump to it.
12578 // Notice: We always emit a nop after a ret to make sure there is room
12579 // for safepoint patching
12580 instruct Ret()
12581 %{
12582   match(Return);
12583 
12584   format %{ "ret" %}
12585   ins_encode %{
12586     __ ret(0);
12587   %}
12588   ins_pipe(pipe_jmp);
12589 %}
12590 
12591 // Tail Call; Jump from runtime stub to Java code.
12592 // Also known as an 'interprocedural jump'.
12593 // Target of jump will eventually return to caller.
12594 // TailJump below removes the return address.
12595 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12596 %{
12597   match(TailCall jump_target method_ptr);
12598 
12599   ins_cost(300);
12600   format %{ "jmp     $jump_target\t# rbx holds method" %}
12601   ins_encode %{
12602     __ jmp($jump_target$$Register);
12603   %}
12604   ins_pipe(pipe_jmp);
12605 %}
12606 
12607 // Tail Jump; remove the return address; jump to target.
12608 // TailCall above leaves the return address around.
12609 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12610 %{
12611   match(TailJump jump_target ex_oop);
12612 
12613   ins_cost(300);
12614   format %{ "popq    rdx\t# pop return address\n\t"
12615             "jmp     $jump_target" %}
12616   ins_encode %{
12617     __ popq(as_Register(RDX_enc));
12618     __ jmp($jump_target$$Register);
12619   %}
12620   ins_pipe(pipe_jmp);
12621 %}
12622 
12623 // Create exception oop: created by stack-crawling runtime code.
12624 // Created exception is now available to this handler, and is setup
12625 // just prior to jumping to this handler.  No code emitted.
12626 instruct CreateException(rax_RegP ex_oop)
12627 %{
12628   match(Set ex_oop (CreateEx));
12629 
12630   size(0);
12631   // use the following format syntax
12632   format %{ "# exception oop is in rax; no code emitted" %}
12633   ins_encode();
12634   ins_pipe(empty);
12635 %}
12636 
12637 // Rethrow exception:
12638 // The exception oop will come in the first argument position.
12639 // Then JUMP (not call) to the rethrow stub code.
12640 instruct RethrowException()
12641 %{
12642   match(Rethrow);
12643 
12644   // use the following format syntax
12645   format %{ "jmp     rethrow_stub" %}
12646   ins_encode %{
12647     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12648   %}
12649   ins_pipe(pipe_jmp);
12650 %}
12651 
12652 // ============================================================================
12653 // This name is KNOWN by the ADLC and cannot be changed.
12654 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12655 // for this guy.
12656 instruct tlsLoadP(r15_RegP dst) %{
12657   match(Set dst (ThreadLocal));
12658   effect(DEF dst);
12659 
12660   size(0);
12661   format %{ "# TLS is in R15" %}
12662   ins_encode( /*empty encoding*/ );
12663   ins_pipe(ialu_reg_reg);
12664 %}
12665 
12666 
12667 //----------PEEPHOLE RULES-----------------------------------------------------
12668 // These must follow all instruction definitions as they use the names
12669 // defined in the instructions definitions.
12670 //
12671 // peeppredicate ( rule_predicate );
12672 // // the predicate unless which the peephole rule will be ignored
12673 //
12674 // peepmatch ( root_instr_name [preceding_instruction]* );
12675 //
12676 // peepprocedure ( procedure_name );
12677 // // provide a procedure name to perform the optimization, the procedure should
12678 // // reside in the architecture dependent peephole file, the method has the
12679 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12680 // // with the arguments being the basic block, the current node index inside the
12681 // // block, the register allocator, the functions upon invoked return a new node
12682 // // defined in peepreplace, and the rules of the nodes appearing in the
12683 // // corresponding peepmatch, the function return true if successful, else
12684 // // return false
12685 //
12686 // peepconstraint %{
12687 // (instruction_number.operand_name relational_op instruction_number.operand_name
12688 //  [, ...] );
12689 // // instruction numbers are zero-based using left to right order in peepmatch
12690 //
12691 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12692 // // provide an instruction_number.operand_name for each operand that appears
12693 // // in the replacement instruction's match rule
12694 //
12695 // ---------VM FLAGS---------------------------------------------------------
12696 //
12697 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12698 //
12699 // Each peephole rule is given an identifying number starting with zero and
12700 // increasing by one in the order seen by the parser.  An individual peephole
12701 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12702 // on the command-line.
12703 //
12704 // ---------CURRENT LIMITATIONS----------------------------------------------
12705 //
12706 // Only transformations inside a basic block (do we need more for peephole)
12707 //
12708 // ---------EXAMPLE----------------------------------------------------------
12709 //
12710 // // pertinent parts of existing instructions in architecture description
12711 // instruct movI(rRegI dst, rRegI src)
12712 // %{
12713 //   match(Set dst (CopyI src));
12714 // %}
12715 //
12716 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12717 // %{
12718 //   match(Set dst (AddI dst src));
12719 //   effect(KILL cr);
12720 // %}
12721 //
12722 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12723 // %{
12724 //   match(Set dst (AddI dst src));
12725 // %}
12726 //
12727 // 1. Simple replacement
12728 // - Only match adjacent instructions in same basic block
12729 // - Only equality constraints
12730 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
12731 // - Only one replacement instruction
12732 //
12733 // // Change (inc mov) to lea
12734 // peephole %{
12735 //   // lea should only be emitted when beneficial
12736 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12737 //   // increment preceded by register-register move
12738 //   peepmatch ( incI_rReg movI );
12739 //   // require that the destination register of the increment
12740 //   // match the destination register of the move
12741 //   peepconstraint ( 0.dst == 1.dst );
12742 //   // construct a replacement instruction that sets
12743 //   // the destination to ( move's source register + one )
12744 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12745 // %}
12746 //
12747 // 2. Procedural replacement
12748 // - More flexible finding relevent nodes
12749 // - More flexible constraints
12750 // - More flexible transformations
12751 // - May utilise architecture-dependent API more effectively
12752 // - Currently only one replacement instruction due to adlc parsing capabilities
12753 //
12754 // // Change (inc mov) to lea
12755 // peephole %{
12756 //   // lea should only be emitted when beneficial
12757 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12758 //   // the rule numbers of these nodes inside are passed into the function below
12759 //   peepmatch ( incI_rReg movI );
12760 //   // the method that takes the responsibility of transformation
12761 //   peepprocedure ( inc_mov_to_lea );
12762 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
12763 //   // node is passed into the function above
12764 //   peepreplace ( leaI_rReg_immI() );
12765 // %}
12766 
12767 // These instructions is not matched by the matcher but used by the peephole
12768 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
12769 %{
12770   predicate(false);
12771   match(Set dst (AddI src1 src2));
12772   format %{ "leal    $dst, [$src1 + $src2]" %}
12773   ins_encode %{
12774     Register dst = $dst$$Register;
12775     Register src1 = $src1$$Register;
12776     Register src2 = $src2$$Register;
12777     if (src1 != rbp && src1 != r13) {
12778       __ leal(dst, Address(src1, src2, Address::times_1));
12779     } else {
12780       assert(src2 != rbp && src2 != r13, "");
12781       __ leal(dst, Address(src2, src1, Address::times_1));
12782     }
12783   %}
12784   ins_pipe(ialu_reg_reg);
12785 %}
12786 
12787 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
12788 %{
12789   predicate(false);
12790   match(Set dst (AddI src1 src2));
12791   format %{ "leal    $dst, [$src1 + $src2]" %}
12792   ins_encode %{
12793     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
12794   %}
12795   ins_pipe(ialu_reg_reg);
12796 %}
12797 
12798 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
12799 %{
12800   predicate(false);
12801   match(Set dst (LShiftI src shift));
12802   format %{ "leal    $dst, [$src << $shift]" %}
12803   ins_encode %{
12804     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12805     Register src = $src$$Register;
12806     if (scale == Address::times_2 && src != rbp && src != r13) {
12807       __ leal($dst$$Register, Address(src, src, Address::times_1));
12808     } else {
12809       __ leal($dst$$Register, Address(noreg, src, scale));
12810     }
12811   %}
12812   ins_pipe(ialu_reg_reg);
12813 %}
12814 
12815 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
12816 %{
12817   predicate(false);
12818   match(Set dst (AddL src1 src2));
12819   format %{ "leaq    $dst, [$src1 + $src2]" %}
12820   ins_encode %{
12821     Register dst = $dst$$Register;
12822     Register src1 = $src1$$Register;
12823     Register src2 = $src2$$Register;
12824     if (src1 != rbp && src1 != r13) {
12825       __ leaq(dst, Address(src1, src2, Address::times_1));
12826     } else {
12827       assert(src2 != rbp && src2 != r13, "");
12828       __ leaq(dst, Address(src2, src1, Address::times_1));
12829     }
12830   %}
12831   ins_pipe(ialu_reg_reg);
12832 %}
12833 
12834 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
12835 %{
12836   predicate(false);
12837   match(Set dst (AddL src1 src2));
12838   format %{ "leaq    $dst, [$src1 + $src2]" %}
12839   ins_encode %{
12840     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
12841   %}
12842   ins_pipe(ialu_reg_reg);
12843 %}
12844 
12845 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
12846 %{
12847   predicate(false);
12848   match(Set dst (LShiftL src shift));
12849   format %{ "leaq    $dst, [$src << $shift]" %}
12850   ins_encode %{
12851     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12852     Register src = $src$$Register;
12853     if (scale == Address::times_2 && src != rbp && src != r13) {
12854       __ leaq($dst$$Register, Address(src, src, Address::times_1));
12855     } else {
12856       __ leaq($dst$$Register, Address(noreg, src, scale));
12857     }
12858   %}
12859   ins_pipe(ialu_reg_reg);
12860 %}
12861 
12862 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
12863 // sal}) with lea instructions. The {add, sal} rules are beneficial in
12864 // processors with at least partial ALU support for lea
12865 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
12866 // beneficial for processors with full ALU support
12867 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
12868 
12869 peephole
12870 %{
12871   peeppredicate(VM_Version::supports_fast_2op_lea());
12872   peepmatch (addI_rReg);
12873   peepprocedure (lea_coalesce_reg);
12874   peepreplace (leaI_rReg_rReg_peep());
12875 %}
12876 
12877 peephole
12878 %{
12879   peeppredicate(VM_Version::supports_fast_2op_lea());
12880   peepmatch (addI_rReg_imm);
12881   peepprocedure (lea_coalesce_imm);
12882   peepreplace (leaI_rReg_immI_peep());
12883 %}
12884 
12885 peephole
12886 %{
12887   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12888                 VM_Version::is_intel_cascade_lake());
12889   peepmatch (incI_rReg);
12890   peepprocedure (lea_coalesce_imm);
12891   peepreplace (leaI_rReg_immI_peep());
12892 %}
12893 
12894 peephole
12895 %{
12896   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12897                 VM_Version::is_intel_cascade_lake());
12898   peepmatch (decI_rReg);
12899   peepprocedure (lea_coalesce_imm);
12900   peepreplace (leaI_rReg_immI_peep());
12901 %}
12902 
12903 peephole
12904 %{
12905   peeppredicate(VM_Version::supports_fast_2op_lea());
12906   peepmatch (salI_rReg_immI2);
12907   peepprocedure (lea_coalesce_imm);
12908   peepreplace (leaI_rReg_immI2_peep());
12909 %}
12910 
12911 peephole
12912 %{
12913   peeppredicate(VM_Version::supports_fast_2op_lea());
12914   peepmatch (addL_rReg);
12915   peepprocedure (lea_coalesce_reg);
12916   peepreplace (leaL_rReg_rReg_peep());
12917 %}
12918 
12919 peephole
12920 %{
12921   peeppredicate(VM_Version::supports_fast_2op_lea());
12922   peepmatch (addL_rReg_imm);
12923   peepprocedure (lea_coalesce_imm);
12924   peepreplace (leaL_rReg_immL32_peep());
12925 %}
12926 
12927 peephole
12928 %{
12929   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12930                 VM_Version::is_intel_cascade_lake());
12931   peepmatch (incL_rReg);
12932   peepprocedure (lea_coalesce_imm);
12933   peepreplace (leaL_rReg_immL32_peep());
12934 %}
12935 
12936 peephole
12937 %{
12938   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12939                 VM_Version::is_intel_cascade_lake());
12940   peepmatch (decL_rReg);
12941   peepprocedure (lea_coalesce_imm);
12942   peepreplace (leaL_rReg_immL32_peep());
12943 %}
12944 
12945 peephole
12946 %{
12947   peeppredicate(VM_Version::supports_fast_2op_lea());
12948   peepmatch (salL_rReg_immI2);
12949   peepprocedure (lea_coalesce_imm);
12950   peepreplace (leaL_rReg_immI2_peep());
12951 %}
12952 
12953 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
12954 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
12955 
12956 //int variant
12957 peephole
12958 %{
12959   peepmatch (testI_reg);
12960   peepprocedure (test_may_remove);
12961 %}
12962 
12963 //long variant
12964 peephole
12965 %{
12966   peepmatch (testL_reg);
12967   peepprocedure (test_may_remove);
12968 %}
12969 
12970 
12971 //----------SMARTSPILL RULES---------------------------------------------------
12972 // These must follow all instruction definitions as they use the names
12973 // defined in the instructions definitions.