1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RDX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RDI int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 
  323 source_hpp %{
  324 
  325 #include "peephole_x86_64.hpp"
  326 
  327 %}
  328 
  329 // Register masks
  330 source_hpp %{
  331 
  332 extern RegMask _ANY_REG_mask;
  333 extern RegMask _PTR_REG_mask;
  334 extern RegMask _PTR_REG_NO_RBP_mask;
  335 extern RegMask _PTR_NO_RAX_REG_mask;
  336 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  337 extern RegMask _LONG_REG_mask;
  338 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  339 extern RegMask _LONG_NO_RCX_REG_mask;
  340 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  341 extern RegMask _INT_REG_mask;
  342 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  343 extern RegMask _INT_NO_RCX_REG_mask;
  344 extern RegMask _INT_NO_RBP_R13_REG_mask;
  345 extern RegMask _FLOAT_REG_mask;
  346 
  347 extern RegMask _STACK_OR_PTR_REG_mask;
  348 extern RegMask _STACK_OR_LONG_REG_mask;
  349 extern RegMask _STACK_OR_INT_REG_mask;
  350 
  351 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  352 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  353 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ masm->
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_INT_REG_mask;
  380 
  381 static bool need_r12_heapbase() {
  382   return UseCompressedOops;
  383 }
  384 
  385 void reg_mask_init() {
  386   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  387   // We derive a number of subsets from it.
  388   _ANY_REG_mask = _ALL_REG_mask;
  389 
  390   if (PreserveFramePointer) {
  391     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  392     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  393   }
  394   if (need_r12_heapbase()) {
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  396     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  397   }
  398 
  399   _PTR_REG_mask = _ANY_REG_mask;
  400   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  401   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  404 
  405   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  406   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  407 
  408   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  409   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  410   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  411 
  412   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  413   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  414   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  415 
  416   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  417   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  418   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  419 
  420   _LONG_REG_mask = _PTR_REG_mask;
  421   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  422   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  423 
  424   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  425   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  426   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  429 
  430   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  431   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  432   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  433 
  434   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  435   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  436   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  439 
  440   _INT_REG_mask = _ALL_INT_REG_mask;
  441   if (PreserveFramePointer) {
  442     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  443   }
  444   if (need_r12_heapbase()) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  446   }
  447 
  448   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  449   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  450 
  451   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  452   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  453   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  454 
  455   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  456   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  457 
  458   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  459   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  460   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  461 
  462   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  463   // from the float_reg_legacy/float_reg_evex register class.
  464   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  465 }
  466 
  467 static bool generate_vzeroupper(Compile* C) {
  468   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  469 }
  470 
  471 static int clear_avx_size() {
  472   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  473 }
  474 
  475 // !!!!! Special hack to get all types of calls to specify the byte offset
  476 //       from the start of the call to the point where the return address
  477 //       will point.
  478 int MachCallStaticJavaNode::ret_addr_offset()
  479 {
  480   int offset = 5; // 5 bytes from start of call to where return address points
  481   offset += clear_avx_size();
  482   return offset;
  483 }
  484 
  485 int MachCallDynamicJavaNode::ret_addr_offset()
  486 {
  487   int offset = 15; // 15 bytes from start of call to where return address points
  488   offset += clear_avx_size();
  489   return offset;
  490 }
  491 
  492 int MachCallRuntimeNode::ret_addr_offset() {
  493   int offset = 13; // movq r10,#addr; callq (r10)
  494   if (this->ideal_Opcode() != Op_CallLeafVector) {
  495     offset += clear_avx_size();
  496   }
  497   return offset;
  498 }
  499 //
  500 // Compute padding required for nodes which need alignment
  501 //
  502 
  503 // The address of the call instruction needs to be 4-byte aligned to
  504 // ensure that it does not span a cache line so that it can be patched.
  505 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  506 {
  507   current_offset += clear_avx_size(); // skip vzeroupper
  508   current_offset += 1; // skip call opcode byte
  509   return align_up(current_offset, alignment_required()) - current_offset;
  510 }
  511 
  512 // The address of the call instruction needs to be 4-byte aligned to
  513 // ensure that it does not span a cache line so that it can be patched.
  514 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  515 {
  516   current_offset += clear_avx_size(); // skip vzeroupper
  517   current_offset += 11; // skip movq instruction + call opcode byte
  518   return align_up(current_offset, alignment_required()) - current_offset;
  519 }
  520 
  521 // This could be in MacroAssembler but it's fairly C2 specific
  522 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  523   Label exit;
  524   __ jccb(Assembler::noParity, exit);
  525   __ pushf();
  526   //
  527   // comiss/ucomiss instructions set ZF,PF,CF flags and
  528   // zero OF,AF,SF for NaN values.
  529   // Fixup flags by zeroing ZF,PF so that compare of NaN
  530   // values returns 'less than' result (CF is set).
  531   // Leave the rest of flags unchanged.
  532   //
  533   //    7 6 5 4 3 2 1 0
  534   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  535   //    0 0 1 0 1 0 1 1   (0x2B)
  536   //
  537   __ andq(Address(rsp, 0), 0xffffff2b);
  538   __ popf();
  539   __ bind(exit);
  540 }
  541 
  542 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  543   Label done;
  544   __ movl(dst, -1);
  545   __ jcc(Assembler::parity, done);
  546   __ jcc(Assembler::below, done);
  547   __ setb(Assembler::notEqual, dst);
  548   __ movzbl(dst, dst);
  549   __ bind(done);
  550 }
  551 
  552 // Math.min()    # Math.max()
  553 // --------------------------
  554 // ucomis[s/d]   #
  555 // ja   -> b     # a
  556 // jp   -> NaN   # NaN
  557 // jb   -> a     # b
  558 // je            #
  559 // |-jz -> a | b # a & b
  560 // |    -> a     #
  561 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  562                             XMMRegister a, XMMRegister b,
  563                             XMMRegister xmmt, Register rt,
  564                             bool min, bool single) {
  565 
  566   Label nan, zero, below, above, done;
  567 
  568   if (single)
  569     __ ucomiss(a, b);
  570   else
  571     __ ucomisd(a, b);
  572 
  573   if (dst->encoding() != (min ? b : a)->encoding())
  574     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  575   else
  576     __ jccb(Assembler::above, done);
  577 
  578   __ jccb(Assembler::parity, nan);  // PF=1
  579   __ jccb(Assembler::below, below); // CF=1
  580 
  581   // equal
  582   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  583   if (single) {
  584     __ ucomiss(a, xmmt);
  585     __ jccb(Assembler::equal, zero);
  586 
  587     __ movflt(dst, a);
  588     __ jmp(done);
  589   }
  590   else {
  591     __ ucomisd(a, xmmt);
  592     __ jccb(Assembler::equal, zero);
  593 
  594     __ movdbl(dst, a);
  595     __ jmp(done);
  596   }
  597 
  598   __ bind(zero);
  599   if (min)
  600     __ vpor(dst, a, b, Assembler::AVX_128bit);
  601   else
  602     __ vpand(dst, a, b, Assembler::AVX_128bit);
  603 
  604   __ jmp(done);
  605 
  606   __ bind(above);
  607   if (single)
  608     __ movflt(dst, min ? b : a);
  609   else
  610     __ movdbl(dst, min ? b : a);
  611 
  612   __ jmp(done);
  613 
  614   __ bind(nan);
  615   if (single) {
  616     __ movl(rt, 0x7fc00000); // Float.NaN
  617     __ movdl(dst, rt);
  618   }
  619   else {
  620     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  621     __ movdq(dst, rt);
  622   }
  623   __ jmp(done);
  624 
  625   __ bind(below);
  626   if (single)
  627     __ movflt(dst, min ? a : b);
  628   else
  629     __ movdbl(dst, min ? a : b);
  630 
  631   __ bind(done);
  632 }
  633 
  634 //=============================================================================
  635 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  636 
  637 int ConstantTable::calculate_table_base_offset() const {
  638   return 0;  // absolute addressing, no offset
  639 }
  640 
  641 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  642 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  643   ShouldNotReachHere();
  644 }
  645 
  646 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  647   // Empty encoding
  648 }
  649 
  650 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  651   return 0;
  652 }
  653 
  654 #ifndef PRODUCT
  655 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  656   st->print("# MachConstantBaseNode (empty encoding)");
  657 }
  658 #endif
  659 
  660 
  661 //=============================================================================
  662 #ifndef PRODUCT
  663 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  664   Compile* C = ra_->C;
  665 
  666   int framesize = C->output()->frame_size_in_bytes();
  667   int bangsize = C->output()->bang_size_in_bytes();
  668   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  669   // Remove wordSize for return addr which is already pushed.
  670   framesize -= wordSize;
  671 
  672   if (C->output()->need_stack_bang(bangsize)) {
  673     framesize -= wordSize;
  674     st->print("# stack bang (%d bytes)", bangsize);
  675     st->print("\n\t");
  676     st->print("pushq   rbp\t# Save rbp");
  677     if (PreserveFramePointer) {
  678         st->print("\n\t");
  679         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  680     }
  681     if (framesize) {
  682       st->print("\n\t");
  683       st->print("subq    rsp, #%d\t# Create frame",framesize);
  684     }
  685   } else {
  686     st->print("subq    rsp, #%d\t# Create frame",framesize);
  687     st->print("\n\t");
  688     framesize -= wordSize;
  689     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  690     if (PreserveFramePointer) {
  691       st->print("\n\t");
  692       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  693       if (framesize > 0) {
  694         st->print("\n\t");
  695         st->print("addq    rbp, #%d", framesize);
  696       }
  697     }
  698   }
  699 
  700   if (VerifyStackAtCalls) {
  701     st->print("\n\t");
  702     framesize -= wordSize;
  703     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  704 #ifdef ASSERT
  705     st->print("\n\t");
  706     st->print("# stack alignment check");
  707 #endif
  708   }
  709   if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
  710     st->print("\n\t");
  711     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  712     st->print("\n\t");
  713     st->print("je      fast_entry\t");
  714     st->print("\n\t");
  715     st->print("call    #nmethod_entry_barrier_stub\t");
  716     st->print("\n\tfast_entry:");
  717   }
  718   st->cr();
  719 }
  720 #endif
  721 
  722 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  723   Compile* C = ra_->C;
  724 
  725   int framesize = C->output()->frame_size_in_bytes();
  726   int bangsize = C->output()->bang_size_in_bytes();
  727 
  728   if (C->clinit_barrier_on_entry()) {
  729     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  730     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  731 
  732     Label L_skip_barrier;
  733     Register klass = rscratch1;
  734 
  735     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  736     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  737 
  738     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  739 
  740     __ bind(L_skip_barrier);
  741   }
  742 
  743   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  744 
  745   C->output()->set_frame_complete(__ offset());
  746 
  747   if (C->has_mach_constant_base_node()) {
  748     // NOTE: We set the table base offset here because users might be
  749     // emitted before MachConstantBaseNode.
  750     ConstantTable& constant_table = C->output()->constant_table();
  751     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  752   }
  753 }
  754 
  755 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  756 {
  757   return MachNode::size(ra_); // too many variables; just compute it
  758                               // the hard way
  759 }
  760 
  761 int MachPrologNode::reloc() const
  762 {
  763   return 0; // a large enough number
  764 }
  765 
  766 //=============================================================================
  767 #ifndef PRODUCT
  768 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  769 {
  770   Compile* C = ra_->C;
  771   if (generate_vzeroupper(C)) {
  772     st->print("vzeroupper");
  773     st->cr(); st->print("\t");
  774   }
  775 
  776   int framesize = C->output()->frame_size_in_bytes();
  777   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  778   // Remove word for return adr already pushed
  779   // and RBP
  780   framesize -= 2*wordSize;
  781 
  782   if (framesize) {
  783     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  784     st->print("\t");
  785   }
  786 
  787   st->print_cr("popq    rbp");
  788   if (do_polling() && C->is_method_compilation()) {
  789     st->print("\t");
  790     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  791                  "ja      #safepoint_stub\t"
  792                  "# Safepoint: poll for GC");
  793   }
  794 }
  795 #endif
  796 
  797 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  798 {
  799   Compile* C = ra_->C;
  800 
  801   if (generate_vzeroupper(C)) {
  802     // Clear upper bits of YMM registers when current compiled code uses
  803     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  804     __ vzeroupper();
  805   }
  806 
  807   int framesize = C->output()->frame_size_in_bytes();
  808   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  809   // Remove word for return adr already pushed
  810   // and RBP
  811   framesize -= 2*wordSize;
  812 
  813   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  814 
  815   if (framesize) {
  816     __ addq(rsp, framesize);
  817   }
  818 
  819   __ popq(rbp);
  820 
  821   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  822     __ reserved_stack_check();
  823   }
  824 
  825   if (do_polling() && C->is_method_compilation()) {
  826     Label dummy_label;
  827     Label* code_stub = &dummy_label;
  828     if (!C->output()->in_scratch_emit_size()) {
  829       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  830       C->output()->add_stub(stub);
  831       code_stub = &stub->entry();
  832     }
  833     __ relocate(relocInfo::poll_return_type);
  834     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  835   }
  836 }
  837 
  838 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  839 {
  840   return MachNode::size(ra_); // too many variables; just compute it
  841                               // the hard way
  842 }
  843 
  844 int MachEpilogNode::reloc() const
  845 {
  846   return 2; // a large enough number
  847 }
  848 
  849 const Pipeline* MachEpilogNode::pipeline() const
  850 {
  851   return MachNode::pipeline_class();
  852 }
  853 
  854 //=============================================================================
  855 
  856 enum RC {
  857   rc_bad,
  858   rc_int,
  859   rc_kreg,
  860   rc_float,
  861   rc_stack
  862 };
  863 
  864 static enum RC rc_class(OptoReg::Name reg)
  865 {
  866   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  867 
  868   if (OptoReg::is_stack(reg)) return rc_stack;
  869 
  870   VMReg r = OptoReg::as_VMReg(reg);
  871 
  872   if (r->is_Register()) return rc_int;
  873 
  874   if (r->is_KRegister()) return rc_kreg;
  875 
  876   assert(r->is_XMMRegister(), "must be");
  877   return rc_float;
  878 }
  879 
  880 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  881 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
  882                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  883 
  884 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
  885                      int stack_offset, int reg, uint ireg, outputStream* st);
  886 
  887 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
  888                                       int dst_offset, uint ireg, outputStream* st) {
  889   if (masm) {
  890     switch (ireg) {
  891     case Op_VecS:
  892       __ movq(Address(rsp, -8), rax);
  893       __ movl(rax, Address(rsp, src_offset));
  894       __ movl(Address(rsp, dst_offset), rax);
  895       __ movq(rax, Address(rsp, -8));
  896       break;
  897     case Op_VecD:
  898       __ pushq(Address(rsp, src_offset));
  899       __ popq (Address(rsp, dst_offset));
  900       break;
  901     case Op_VecX:
  902       __ pushq(Address(rsp, src_offset));
  903       __ popq (Address(rsp, dst_offset));
  904       __ pushq(Address(rsp, src_offset+8));
  905       __ popq (Address(rsp, dst_offset+8));
  906       break;
  907     case Op_VecY:
  908       __ vmovdqu(Address(rsp, -32), xmm0);
  909       __ vmovdqu(xmm0, Address(rsp, src_offset));
  910       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  911       __ vmovdqu(xmm0, Address(rsp, -32));
  912       break;
  913     case Op_VecZ:
  914       __ evmovdquq(Address(rsp, -64), xmm0, 2);
  915       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
  916       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
  917       __ evmovdquq(xmm0, Address(rsp, -64), 2);
  918       break;
  919     default:
  920       ShouldNotReachHere();
  921     }
  922 #ifndef PRODUCT
  923   } else {
  924     switch (ireg) {
  925     case Op_VecS:
  926       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
  927                 "movl    rax, [rsp + #%d]\n\t"
  928                 "movl    [rsp + #%d], rax\n\t"
  929                 "movq    rax, [rsp - #8]",
  930                 src_offset, dst_offset);
  931       break;
  932     case Op_VecD:
  933       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
  934                 "popq    [rsp + #%d]",
  935                 src_offset, dst_offset);
  936       break;
  937      case Op_VecX:
  938       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
  939                 "popq    [rsp + #%d]\n\t"
  940                 "pushq   [rsp + #%d]\n\t"
  941                 "popq    [rsp + #%d]",
  942                 src_offset, dst_offset, src_offset+8, dst_offset+8);
  943       break;
  944     case Op_VecY:
  945       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
  946                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  947                 "vmovdqu [rsp + #%d], xmm0\n\t"
  948                 "vmovdqu xmm0, [rsp - #32]",
  949                 src_offset, dst_offset);
  950       break;
  951     case Op_VecZ:
  952       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
  953                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  954                 "vmovdqu [rsp + #%d], xmm0\n\t"
  955                 "vmovdqu xmm0, [rsp - #64]",
  956                 src_offset, dst_offset);
  957       break;
  958     default:
  959       ShouldNotReachHere();
  960     }
  961 #endif
  962   }
  963 }
  964 
  965 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
  966                                        PhaseRegAlloc* ra_,
  967                                        bool do_size,
  968                                        outputStream* st) const {
  969   assert(masm != nullptr || st  != nullptr, "sanity");
  970   // Get registers to move
  971   OptoReg::Name src_second = ra_->get_reg_second(in(1));
  972   OptoReg::Name src_first = ra_->get_reg_first(in(1));
  973   OptoReg::Name dst_second = ra_->get_reg_second(this);
  974   OptoReg::Name dst_first = ra_->get_reg_first(this);
  975 
  976   enum RC src_second_rc = rc_class(src_second);
  977   enum RC src_first_rc = rc_class(src_first);
  978   enum RC dst_second_rc = rc_class(dst_second);
  979   enum RC dst_first_rc = rc_class(dst_first);
  980 
  981   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
  982          "must move at least 1 register" );
  983 
  984   if (src_first == dst_first && src_second == dst_second) {
  985     // Self copy, no move
  986     return 0;
  987   }
  988   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
  989     uint ireg = ideal_reg();
  990     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
  991     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
  992     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
  993       // mem -> mem
  994       int src_offset = ra_->reg2offset(src_first);
  995       int dst_offset = ra_->reg2offset(dst_first);
  996       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
  997     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
  998       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
  999     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1000       int stack_offset = ra_->reg2offset(dst_first);
 1001       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1002     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1003       int stack_offset = ra_->reg2offset(src_first);
 1004       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1005     } else {
 1006       ShouldNotReachHere();
 1007     }
 1008     return 0;
 1009   }
 1010   if (src_first_rc == rc_stack) {
 1011     // mem ->
 1012     if (dst_first_rc == rc_stack) {
 1013       // mem -> mem
 1014       assert(src_second != dst_first, "overlap");
 1015       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1016           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1017         // 64-bit
 1018         int src_offset = ra_->reg2offset(src_first);
 1019         int dst_offset = ra_->reg2offset(dst_first);
 1020         if (masm) {
 1021           __ pushq(Address(rsp, src_offset));
 1022           __ popq (Address(rsp, dst_offset));
 1023 #ifndef PRODUCT
 1024         } else {
 1025           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1026                     "popq    [rsp + #%d]",
 1027                      src_offset, dst_offset);
 1028 #endif
 1029         }
 1030       } else {
 1031         // 32-bit
 1032         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1033         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1034         // No pushl/popl, so:
 1035         int src_offset = ra_->reg2offset(src_first);
 1036         int dst_offset = ra_->reg2offset(dst_first);
 1037         if (masm) {
 1038           __ movq(Address(rsp, -8), rax);
 1039           __ movl(rax, Address(rsp, src_offset));
 1040           __ movl(Address(rsp, dst_offset), rax);
 1041           __ movq(rax, Address(rsp, -8));
 1042 #ifndef PRODUCT
 1043         } else {
 1044           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1045                     "movl    rax, [rsp + #%d]\n\t"
 1046                     "movl    [rsp + #%d], rax\n\t"
 1047                     "movq    rax, [rsp - #8]",
 1048                      src_offset, dst_offset);
 1049 #endif
 1050         }
 1051       }
 1052       return 0;
 1053     } else if (dst_first_rc == rc_int) {
 1054       // mem -> gpr
 1055       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1056           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1057         // 64-bit
 1058         int offset = ra_->reg2offset(src_first);
 1059         if (masm) {
 1060           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1061 #ifndef PRODUCT
 1062         } else {
 1063           st->print("movq    %s, [rsp + #%d]\t# spill",
 1064                      Matcher::regName[dst_first],
 1065                      offset);
 1066 #endif
 1067         }
 1068       } else {
 1069         // 32-bit
 1070         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1071         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1072         int offset = ra_->reg2offset(src_first);
 1073         if (masm) {
 1074           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1075 #ifndef PRODUCT
 1076         } else {
 1077           st->print("movl    %s, [rsp + #%d]\t# spill",
 1078                      Matcher::regName[dst_first],
 1079                      offset);
 1080 #endif
 1081         }
 1082       }
 1083       return 0;
 1084     } else if (dst_first_rc == rc_float) {
 1085       // mem-> xmm
 1086       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1087           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1088         // 64-bit
 1089         int offset = ra_->reg2offset(src_first);
 1090         if (masm) {
 1091           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1092 #ifndef PRODUCT
 1093         } else {
 1094           st->print("%s  %s, [rsp + #%d]\t# spill",
 1095                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1096                      Matcher::regName[dst_first],
 1097                      offset);
 1098 #endif
 1099         }
 1100       } else {
 1101         // 32-bit
 1102         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1103         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1104         int offset = ra_->reg2offset(src_first);
 1105         if (masm) {
 1106           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1107 #ifndef PRODUCT
 1108         } else {
 1109           st->print("movss   %s, [rsp + #%d]\t# spill",
 1110                      Matcher::regName[dst_first],
 1111                      offset);
 1112 #endif
 1113         }
 1114       }
 1115       return 0;
 1116     } else if (dst_first_rc == rc_kreg) {
 1117       // mem -> kreg
 1118       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1119           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1120         // 64-bit
 1121         int offset = ra_->reg2offset(src_first);
 1122         if (masm) {
 1123           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1124 #ifndef PRODUCT
 1125         } else {
 1126           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1127                      Matcher::regName[dst_first],
 1128                      offset);
 1129 #endif
 1130         }
 1131       }
 1132       return 0;
 1133     }
 1134   } else if (src_first_rc == rc_int) {
 1135     // gpr ->
 1136     if (dst_first_rc == rc_stack) {
 1137       // gpr -> mem
 1138       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1139           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1140         // 64-bit
 1141         int offset = ra_->reg2offset(dst_first);
 1142         if (masm) {
 1143           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1144 #ifndef PRODUCT
 1145         } else {
 1146           st->print("movq    [rsp + #%d], %s\t# spill",
 1147                      offset,
 1148                      Matcher::regName[src_first]);
 1149 #endif
 1150         }
 1151       } else {
 1152         // 32-bit
 1153         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1154         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1155         int offset = ra_->reg2offset(dst_first);
 1156         if (masm) {
 1157           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1158 #ifndef PRODUCT
 1159         } else {
 1160           st->print("movl    [rsp + #%d], %s\t# spill",
 1161                      offset,
 1162                      Matcher::regName[src_first]);
 1163 #endif
 1164         }
 1165       }
 1166       return 0;
 1167     } else if (dst_first_rc == rc_int) {
 1168       // gpr -> gpr
 1169       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1170           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1171         // 64-bit
 1172         if (masm) {
 1173           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1174                   as_Register(Matcher::_regEncode[src_first]));
 1175 #ifndef PRODUCT
 1176         } else {
 1177           st->print("movq    %s, %s\t# spill",
 1178                      Matcher::regName[dst_first],
 1179                      Matcher::regName[src_first]);
 1180 #endif
 1181         }
 1182         return 0;
 1183       } else {
 1184         // 32-bit
 1185         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1186         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1187         if (masm) {
 1188           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1189                   as_Register(Matcher::_regEncode[src_first]));
 1190 #ifndef PRODUCT
 1191         } else {
 1192           st->print("movl    %s, %s\t# spill",
 1193                      Matcher::regName[dst_first],
 1194                      Matcher::regName[src_first]);
 1195 #endif
 1196         }
 1197         return 0;
 1198       }
 1199     } else if (dst_first_rc == rc_float) {
 1200       // gpr -> xmm
 1201       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1202           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1203         // 64-bit
 1204         if (masm) {
 1205           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1206 #ifndef PRODUCT
 1207         } else {
 1208           st->print("movdq   %s, %s\t# spill",
 1209                      Matcher::regName[dst_first],
 1210                      Matcher::regName[src_first]);
 1211 #endif
 1212         }
 1213       } else {
 1214         // 32-bit
 1215         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1216         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1217         if (masm) {
 1218           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1219 #ifndef PRODUCT
 1220         } else {
 1221           st->print("movdl   %s, %s\t# spill",
 1222                      Matcher::regName[dst_first],
 1223                      Matcher::regName[src_first]);
 1224 #endif
 1225         }
 1226       }
 1227       return 0;
 1228     } else if (dst_first_rc == rc_kreg) {
 1229       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1230           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1231         // 64-bit
 1232         if (masm) {
 1233           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1234   #ifndef PRODUCT
 1235         } else {
 1236            st->print("kmovq   %s, %s\t# spill",
 1237                        Matcher::regName[dst_first],
 1238                        Matcher::regName[src_first]);
 1239   #endif
 1240         }
 1241       }
 1242       Unimplemented();
 1243       return 0;
 1244     }
 1245   } else if (src_first_rc == rc_float) {
 1246     // xmm ->
 1247     if (dst_first_rc == rc_stack) {
 1248       // xmm -> mem
 1249       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1250           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1251         // 64-bit
 1252         int offset = ra_->reg2offset(dst_first);
 1253         if (masm) {
 1254           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1255 #ifndef PRODUCT
 1256         } else {
 1257           st->print("movsd   [rsp + #%d], %s\t# spill",
 1258                      offset,
 1259                      Matcher::regName[src_first]);
 1260 #endif
 1261         }
 1262       } else {
 1263         // 32-bit
 1264         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1265         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1266         int offset = ra_->reg2offset(dst_first);
 1267         if (masm) {
 1268           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1269 #ifndef PRODUCT
 1270         } else {
 1271           st->print("movss   [rsp + #%d], %s\t# spill",
 1272                      offset,
 1273                      Matcher::regName[src_first]);
 1274 #endif
 1275         }
 1276       }
 1277       return 0;
 1278     } else if (dst_first_rc == rc_int) {
 1279       // xmm -> gpr
 1280       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1281           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1282         // 64-bit
 1283         if (masm) {
 1284           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1285 #ifndef PRODUCT
 1286         } else {
 1287           st->print("movdq   %s, %s\t# spill",
 1288                      Matcher::regName[dst_first],
 1289                      Matcher::regName[src_first]);
 1290 #endif
 1291         }
 1292       } else {
 1293         // 32-bit
 1294         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1295         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1296         if (masm) {
 1297           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1298 #ifndef PRODUCT
 1299         } else {
 1300           st->print("movdl   %s, %s\t# spill",
 1301                      Matcher::regName[dst_first],
 1302                      Matcher::regName[src_first]);
 1303 #endif
 1304         }
 1305       }
 1306       return 0;
 1307     } else if (dst_first_rc == rc_float) {
 1308       // xmm -> xmm
 1309       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1310           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1311         // 64-bit
 1312         if (masm) {
 1313           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1314 #ifndef PRODUCT
 1315         } else {
 1316           st->print("%s  %s, %s\t# spill",
 1317                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1318                      Matcher::regName[dst_first],
 1319                      Matcher::regName[src_first]);
 1320 #endif
 1321         }
 1322       } else {
 1323         // 32-bit
 1324         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1325         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1326         if (masm) {
 1327           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1328 #ifndef PRODUCT
 1329         } else {
 1330           st->print("%s  %s, %s\t# spill",
 1331                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1332                      Matcher::regName[dst_first],
 1333                      Matcher::regName[src_first]);
 1334 #endif
 1335         }
 1336       }
 1337       return 0;
 1338     } else if (dst_first_rc == rc_kreg) {
 1339       assert(false, "Illegal spilling");
 1340       return 0;
 1341     }
 1342   } else if (src_first_rc == rc_kreg) {
 1343     if (dst_first_rc == rc_stack) {
 1344       // mem -> kreg
 1345       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1346           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1347         // 64-bit
 1348         int offset = ra_->reg2offset(dst_first);
 1349         if (masm) {
 1350           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1351 #ifndef PRODUCT
 1352         } else {
 1353           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1354                      offset,
 1355                      Matcher::regName[src_first]);
 1356 #endif
 1357         }
 1358       }
 1359       return 0;
 1360     } else if (dst_first_rc == rc_int) {
 1361       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1362           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1363         // 64-bit
 1364         if (masm) {
 1365           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1366 #ifndef PRODUCT
 1367         } else {
 1368          st->print("kmovq   %s, %s\t# spill",
 1369                      Matcher::regName[dst_first],
 1370                      Matcher::regName[src_first]);
 1371 #endif
 1372         }
 1373       }
 1374       Unimplemented();
 1375       return 0;
 1376     } else if (dst_first_rc == rc_kreg) {
 1377       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1378           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1379         // 64-bit
 1380         if (masm) {
 1381           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1382 #ifndef PRODUCT
 1383         } else {
 1384          st->print("kmovq   %s, %s\t# spill",
 1385                      Matcher::regName[dst_first],
 1386                      Matcher::regName[src_first]);
 1387 #endif
 1388         }
 1389       }
 1390       return 0;
 1391     } else if (dst_first_rc == rc_float) {
 1392       assert(false, "Illegal spill");
 1393       return 0;
 1394     }
 1395   }
 1396 
 1397   assert(0," foo ");
 1398   Unimplemented();
 1399   return 0;
 1400 }
 1401 
 1402 #ifndef PRODUCT
 1403 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1404   implementation(nullptr, ra_, false, st);
 1405 }
 1406 #endif
 1407 
 1408 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1409   implementation(masm, ra_, false, nullptr);
 1410 }
 1411 
 1412 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1413   return MachNode::size(ra_);
 1414 }
 1415 
 1416 //=============================================================================
 1417 #ifndef PRODUCT
 1418 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1419 {
 1420   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1421   int reg = ra_->get_reg_first(this);
 1422   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1423             Matcher::regName[reg], offset);
 1424 }
 1425 #endif
 1426 
 1427 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1428 {
 1429   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1430   int reg = ra_->get_encode(this);
 1431 
 1432   __ lea(as_Register(reg), Address(rsp, offset));
 1433 }
 1434 
 1435 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1436 {
 1437   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1438   return (offset < 0x80) ? 5 : 8; // REX
 1439 }
 1440 
 1441 //=============================================================================
 1442 #ifndef PRODUCT
 1443 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1444 {
 1445   if (UseCompressedClassPointers) {
 1446     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1447     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1448   } else {
 1449     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1450     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1451   }
 1452   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1453 }
 1454 #endif
 1455 
 1456 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1457 {
 1458   __ ic_check(InteriorEntryAlignment);
 1459 }
 1460 
 1461 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1462 {
 1463   return MachNode::size(ra_); // too many variables; just compute it
 1464                               // the hard way
 1465 }
 1466 
 1467 
 1468 //=============================================================================
 1469 
 1470 bool Matcher::supports_vector_calling_convention(void) {
 1471   if (EnableVectorSupport && UseVectorStubs) {
 1472     return true;
 1473   }
 1474   return false;
 1475 }
 1476 
 1477 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1478   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1479   int lo = XMM0_num;
 1480   int hi = XMM0b_num;
 1481   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1482   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1483   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1484   return OptoRegPair(hi, lo);
 1485 }
 1486 
 1487 // Is this branch offset short enough that a short branch can be used?
 1488 //
 1489 // NOTE: If the platform does not provide any short branch variants, then
 1490 //       this method should return false for offset 0.
 1491 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1492   // The passed offset is relative to address of the branch.
 1493   // On 86 a branch displacement is calculated relative to address
 1494   // of a next instruction.
 1495   offset -= br_size;
 1496 
 1497   // the short version of jmpConUCF2 contains multiple branches,
 1498   // making the reach slightly less
 1499   if (rule == jmpConUCF2_rule)
 1500     return (-126 <= offset && offset <= 125);
 1501   return (-128 <= offset && offset <= 127);
 1502 }
 1503 
 1504 // Return whether or not this register is ever used as an argument.
 1505 // This function is used on startup to build the trampoline stubs in
 1506 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1507 // call in the trampoline, and arguments in those registers not be
 1508 // available to the callee.
 1509 bool Matcher::can_be_java_arg(int reg)
 1510 {
 1511   return
 1512     reg ==  RDI_num || reg == RDI_H_num ||
 1513     reg ==  RSI_num || reg == RSI_H_num ||
 1514     reg ==  RDX_num || reg == RDX_H_num ||
 1515     reg ==  RCX_num || reg == RCX_H_num ||
 1516     reg ==   R8_num || reg ==  R8_H_num ||
 1517     reg ==   R9_num || reg ==  R9_H_num ||
 1518     reg ==  R12_num || reg == R12_H_num ||
 1519     reg == XMM0_num || reg == XMM0b_num ||
 1520     reg == XMM1_num || reg == XMM1b_num ||
 1521     reg == XMM2_num || reg == XMM2b_num ||
 1522     reg == XMM3_num || reg == XMM3b_num ||
 1523     reg == XMM4_num || reg == XMM4b_num ||
 1524     reg == XMM5_num || reg == XMM5b_num ||
 1525     reg == XMM6_num || reg == XMM6b_num ||
 1526     reg == XMM7_num || reg == XMM7b_num;
 1527 }
 1528 
 1529 bool Matcher::is_spillable_arg(int reg)
 1530 {
 1531   return can_be_java_arg(reg);
 1532 }
 1533 
 1534 uint Matcher::int_pressure_limit()
 1535 {
 1536   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1537 }
 1538 
 1539 uint Matcher::float_pressure_limit()
 1540 {
 1541   // After experiment around with different values, the following default threshold
 1542   // works best for LCM's register pressure scheduling on x64.
 1543   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1544   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1545   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1546 }
 1547 
 1548 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1549   // In 64 bit mode a code which use multiply when
 1550   // devisor is constant is faster than hardware
 1551   // DIV instruction (it uses MulHiL).
 1552   return false;
 1553 }
 1554 
 1555 // Register for DIVI projection of divmodI
 1556 RegMask Matcher::divI_proj_mask() {
 1557   return INT_RAX_REG_mask();
 1558 }
 1559 
 1560 // Register for MODI projection of divmodI
 1561 RegMask Matcher::modI_proj_mask() {
 1562   return INT_RDX_REG_mask();
 1563 }
 1564 
 1565 // Register for DIVL projection of divmodL
 1566 RegMask Matcher::divL_proj_mask() {
 1567   return LONG_RAX_REG_mask();
 1568 }
 1569 
 1570 // Register for MODL projection of divmodL
 1571 RegMask Matcher::modL_proj_mask() {
 1572   return LONG_RDX_REG_mask();
 1573 }
 1574 
 1575 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1576 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1577     return NO_REG_mask();
 1578 }
 1579 
 1580 %}
 1581 
 1582 //----------ENCODING BLOCK-----------------------------------------------------
 1583 // This block specifies the encoding classes used by the compiler to
 1584 // output byte streams.  Encoding classes are parameterized macros
 1585 // used by Machine Instruction Nodes in order to generate the bit
 1586 // encoding of the instruction.  Operands specify their base encoding
 1587 // interface with the interface keyword.  There are currently
 1588 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1589 // COND_INTER.  REG_INTER causes an operand to generate a function
 1590 // which returns its register number when queried.  CONST_INTER causes
 1591 // an operand to generate a function which returns the value of the
 1592 // constant when queried.  MEMORY_INTER causes an operand to generate
 1593 // four functions which return the Base Register, the Index Register,
 1594 // the Scale Value, and the Offset Value of the operand when queried.
 1595 // COND_INTER causes an operand to generate six functions which return
 1596 // the encoding code (ie - encoding bits for the instruction)
 1597 // associated with each basic boolean condition for a conditional
 1598 // instruction.
 1599 //
 1600 // Instructions specify two basic values for encoding.  Again, a
 1601 // function is available to check if the constant displacement is an
 1602 // oop. They use the ins_encode keyword to specify their encoding
 1603 // classes (which must be a sequence of enc_class names, and their
 1604 // parameters, specified in the encoding block), and they use the
 1605 // opcode keyword to specify, in order, their primary, secondary, and
 1606 // tertiary opcode.  Only the opcode sections which a particular
 1607 // instruction needs for encoding need to be specified.
 1608 encode %{
 1609   enc_class cdql_enc(no_rax_rdx_RegI div)
 1610   %{
 1611     // Full implementation of Java idiv and irem; checks for
 1612     // special case as described in JVM spec., p.243 & p.271.
 1613     //
 1614     //         normal case                           special case
 1615     //
 1616     // input : rax: dividend                         min_int
 1617     //         reg: divisor                          -1
 1618     //
 1619     // output: rax: quotient  (= rax idiv reg)       min_int
 1620     //         rdx: remainder (= rax irem reg)       0
 1621     //
 1622     //  Code sequnce:
 1623     //
 1624     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1625     //    5:   75 07/08                jne    e <normal>
 1626     //    7:   33 d2                   xor    %edx,%edx
 1627     //  [div >= 8 -> offset + 1]
 1628     //  [REX_B]
 1629     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1630     //    c:   74 03/04                je     11 <done>
 1631     // 000000000000000e <normal>:
 1632     //    e:   99                      cltd
 1633     //  [div >= 8 -> offset + 1]
 1634     //  [REX_B]
 1635     //    f:   f7 f9                   idiv   $div
 1636     // 0000000000000011 <done>:
 1637     Label normal;
 1638     Label done;
 1639 
 1640     // cmp    $0x80000000,%eax
 1641     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1642 
 1643     // jne    e <normal>
 1644     __ jccb(Assembler::notEqual, normal);
 1645 
 1646     // xor    %edx,%edx
 1647     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1648 
 1649     // cmp    $0xffffffffffffffff,%ecx
 1650     __ cmpl($div$$Register, -1);
 1651 
 1652     // je     11 <done>
 1653     __ jccb(Assembler::equal, done);
 1654 
 1655     // <normal>
 1656     // cltd
 1657     __ bind(normal);
 1658     __ cdql();
 1659 
 1660     // idivl
 1661     // <done>
 1662     __ idivl($div$$Register);
 1663     __ bind(done);
 1664   %}
 1665 
 1666   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1667   %{
 1668     // Full implementation of Java ldiv and lrem; checks for
 1669     // special case as described in JVM spec., p.243 & p.271.
 1670     //
 1671     //         normal case                           special case
 1672     //
 1673     // input : rax: dividend                         min_long
 1674     //         reg: divisor                          -1
 1675     //
 1676     // output: rax: quotient  (= rax idiv reg)       min_long
 1677     //         rdx: remainder (= rax irem reg)       0
 1678     //
 1679     //  Code sequnce:
 1680     //
 1681     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1682     //    7:   00 00 80
 1683     //    a:   48 39 d0                cmp    %rdx,%rax
 1684     //    d:   75 08                   jne    17 <normal>
 1685     //    f:   33 d2                   xor    %edx,%edx
 1686     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1687     //   15:   74 05                   je     1c <done>
 1688     // 0000000000000017 <normal>:
 1689     //   17:   48 99                   cqto
 1690     //   19:   48 f7 f9                idiv   $div
 1691     // 000000000000001c <done>:
 1692     Label normal;
 1693     Label done;
 1694 
 1695     // mov    $0x8000000000000000,%rdx
 1696     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1697 
 1698     // cmp    %rdx,%rax
 1699     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1700 
 1701     // jne    17 <normal>
 1702     __ jccb(Assembler::notEqual, normal);
 1703 
 1704     // xor    %edx,%edx
 1705     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1706 
 1707     // cmp    $0xffffffffffffffff,$div
 1708     __ cmpq($div$$Register, -1);
 1709 
 1710     // je     1e <done>
 1711     __ jccb(Assembler::equal, done);
 1712 
 1713     // <normal>
 1714     // cqto
 1715     __ bind(normal);
 1716     __ cdqq();
 1717 
 1718     // idivq (note: must be emitted by the user of this rule)
 1719     // <done>
 1720     __ idivq($div$$Register);
 1721     __ bind(done);
 1722   %}
 1723 
 1724   enc_class enc_PartialSubtypeCheck()
 1725   %{
 1726     Register Rrdi = as_Register(RDI_enc); // result register
 1727     Register Rrax = as_Register(RAX_enc); // super class
 1728     Register Rrcx = as_Register(RCX_enc); // killed
 1729     Register Rrsi = as_Register(RSI_enc); // sub class
 1730     Label miss;
 1731     const bool set_cond_codes = true;
 1732 
 1733     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 1734                                      nullptr, &miss,
 1735                                      /*set_cond_codes:*/ true);
 1736     if ($primary) {
 1737       __ xorptr(Rrdi, Rrdi);
 1738     }
 1739     __ bind(miss);
 1740   %}
 1741 
 1742   enc_class clear_avx %{
 1743     debug_only(int off0 = __ offset());
 1744     if (generate_vzeroupper(Compile::current())) {
 1745       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1746       // Clear upper bits of YMM registers when current compiled code uses
 1747       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1748       __ vzeroupper();
 1749     }
 1750     debug_only(int off1 = __ offset());
 1751     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1752   %}
 1753 
 1754   enc_class Java_To_Runtime(method meth) %{
 1755     // No relocation needed
 1756     __ mov64(r10, (int64_t) $meth$$method);
 1757     __ call(r10);
 1758     __ post_call_nop();
 1759   %}
 1760 
 1761   enc_class Java_Static_Call(method meth)
 1762   %{
 1763     // JAVA STATIC CALL
 1764     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1765     // determine who we intended to call.
 1766     if (!_method) {
 1767       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1768     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1769       // The NOP here is purely to ensure that eliding a call to
 1770       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1771       __ addr_nop_5();
 1772       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1773     } else {
 1774       int method_index = resolved_method_index(masm);
 1775       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1776                                                   : static_call_Relocation::spec(method_index);
 1777       address mark = __ pc();
 1778       int call_offset = __ offset();
 1779       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1780       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1781         // Calls of the same statically bound method can share
 1782         // a stub to the interpreter.
 1783         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1784       } else {
 1785         // Emit stubs for static call.
 1786         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1787         __ clear_inst_mark();
 1788         if (stub == nullptr) {
 1789           ciEnv::current()->record_failure("CodeCache is full");
 1790           return;
 1791         }
 1792       }
 1793     }
 1794     __ post_call_nop();
 1795   %}
 1796 
 1797   enc_class Java_Dynamic_Call(method meth) %{
 1798     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1799     __ post_call_nop();
 1800   %}
 1801 
 1802 %}
 1803 
 1804 
 1805 
 1806 //----------FRAME--------------------------------------------------------------
 1807 // Definition of frame structure and management information.
 1808 //
 1809 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1810 //                             |   (to get allocators register number
 1811 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1812 //  r   CALLER     |        |
 1813 //  o     |        +--------+      pad to even-align allocators stack-slot
 1814 //  w     V        |  pad0  |        numbers; owned by CALLER
 1815 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1816 //  h     ^        |   in   |  5
 1817 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1818 //  |     |        |        |  3
 1819 //  |     |        +--------+
 1820 //  V     |        | old out|      Empty on Intel, window on Sparc
 1821 //        |    old |preserve|      Must be even aligned.
 1822 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1823 //        |        |   in   |  3   area for Intel ret address
 1824 //     Owned by    |preserve|      Empty on Sparc.
 1825 //       SELF      +--------+
 1826 //        |        |  pad2  |  2   pad to align old SP
 1827 //        |        +--------+  1
 1828 //        |        | locks  |  0
 1829 //        |        +--------+----> OptoReg::stack0(), even aligned
 1830 //        |        |  pad1  | 11   pad to align new SP
 1831 //        |        +--------+
 1832 //        |        |        | 10
 1833 //        |        | spills |  9   spills
 1834 //        V        |        |  8   (pad0 slot for callee)
 1835 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1836 //        ^        |  out   |  7
 1837 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1838 //     Owned by    +--------+
 1839 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1840 //        |    new |preserve|      Must be even-aligned.
 1841 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1842 //        |        |        |
 1843 //
 1844 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1845 //         known from SELF's arguments and the Java calling convention.
 1846 //         Region 6-7 is determined per call site.
 1847 // Note 2: If the calling convention leaves holes in the incoming argument
 1848 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1849 //         are owned by the CALLEE.  Holes should not be necessary in the
 1850 //         incoming area, as the Java calling convention is completely under
 1851 //         the control of the AD file.  Doubles can be sorted and packed to
 1852 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1853 //         varargs C calling conventions.
 1854 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1855 //         even aligned with pad0 as needed.
 1856 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1857 //         region 6-11 is even aligned; it may be padded out more so that
 1858 //         the region from SP to FP meets the minimum stack alignment.
 1859 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1860 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1861 //         SP meets the minimum alignment.
 1862 
 1863 frame
 1864 %{
 1865   // These three registers define part of the calling convention
 1866   // between compiled code and the interpreter.
 1867   inline_cache_reg(RAX);                // Inline Cache Register
 1868 
 1869   // Optional: name the operand used by cisc-spilling to access
 1870   // [stack_pointer + offset]
 1871   cisc_spilling_operand_name(indOffset32);
 1872 
 1873   // Number of stack slots consumed by locking an object
 1874   sync_stack_slots(2);
 1875 
 1876   // Compiled code's Frame Pointer
 1877   frame_pointer(RSP);
 1878 
 1879   // Interpreter stores its frame pointer in a register which is
 1880   // stored to the stack by I2CAdaptors.
 1881   // I2CAdaptors convert from interpreted java to compiled java.
 1882   interpreter_frame_pointer(RBP);
 1883 
 1884   // Stack alignment requirement
 1885   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1886 
 1887   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1888   // for calls to C.  Supports the var-args backing area for register parms.
 1889   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1890 
 1891   // The after-PROLOG location of the return address.  Location of
 1892   // return address specifies a type (REG or STACK) and a number
 1893   // representing the register number (i.e. - use a register name) or
 1894   // stack slot.
 1895   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1896   // Otherwise, it is above the locks and verification slot and alignment word
 1897   return_addr(STACK - 2 +
 1898               align_up((Compile::current()->in_preserve_stack_slots() +
 1899                         Compile::current()->fixed_slots()),
 1900                        stack_alignment_in_slots()));
 1901 
 1902   // Location of compiled Java return values.  Same as C for now.
 1903   return_value
 1904   %{
 1905     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 1906            "only return normal values");
 1907 
 1908     static const int lo[Op_RegL + 1] = {
 1909       0,
 1910       0,
 1911       RAX_num,  // Op_RegN
 1912       RAX_num,  // Op_RegI
 1913       RAX_num,  // Op_RegP
 1914       XMM0_num, // Op_RegF
 1915       XMM0_num, // Op_RegD
 1916       RAX_num   // Op_RegL
 1917     };
 1918     static const int hi[Op_RegL + 1] = {
 1919       0,
 1920       0,
 1921       OptoReg::Bad, // Op_RegN
 1922       OptoReg::Bad, // Op_RegI
 1923       RAX_H_num,    // Op_RegP
 1924       OptoReg::Bad, // Op_RegF
 1925       XMM0b_num,    // Op_RegD
 1926       RAX_H_num     // Op_RegL
 1927     };
 1928     // Excluded flags and vector registers.
 1929     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 1930     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 1931   %}
 1932 %}
 1933 
 1934 //----------ATTRIBUTES---------------------------------------------------------
 1935 //----------Operand Attributes-------------------------------------------------
 1936 op_attrib op_cost(0);        // Required cost attribute
 1937 
 1938 //----------Instruction Attributes---------------------------------------------
 1939 ins_attrib ins_cost(100);       // Required cost attribute
 1940 ins_attrib ins_size(8);         // Required size attribute (in bits)
 1941 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 1942                                 // a non-matching short branch variant
 1943                                 // of some long branch?
 1944 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 1945                                 // be a power of 2) specifies the
 1946                                 // alignment that some part of the
 1947                                 // instruction (not necessarily the
 1948                                 // start) requires.  If > 1, a
 1949                                 // compute_padding() function must be
 1950                                 // provided for the instruction
 1951 
 1952 //----------OPERANDS-----------------------------------------------------------
 1953 // Operand definitions must precede instruction definitions for correct parsing
 1954 // in the ADLC because operands constitute user defined types which are used in
 1955 // instruction definitions.
 1956 
 1957 //----------Simple Operands----------------------------------------------------
 1958 // Immediate Operands
 1959 // Integer Immediate
 1960 operand immI()
 1961 %{
 1962   match(ConI);
 1963 
 1964   op_cost(10);
 1965   format %{ %}
 1966   interface(CONST_INTER);
 1967 %}
 1968 
 1969 // Constant for test vs zero
 1970 operand immI_0()
 1971 %{
 1972   predicate(n->get_int() == 0);
 1973   match(ConI);
 1974 
 1975   op_cost(0);
 1976   format %{ %}
 1977   interface(CONST_INTER);
 1978 %}
 1979 
 1980 // Constant for increment
 1981 operand immI_1()
 1982 %{
 1983   predicate(n->get_int() == 1);
 1984   match(ConI);
 1985 
 1986   op_cost(0);
 1987   format %{ %}
 1988   interface(CONST_INTER);
 1989 %}
 1990 
 1991 // Constant for decrement
 1992 operand immI_M1()
 1993 %{
 1994   predicate(n->get_int() == -1);
 1995   match(ConI);
 1996 
 1997   op_cost(0);
 1998   format %{ %}
 1999   interface(CONST_INTER);
 2000 %}
 2001 
 2002 operand immI_2()
 2003 %{
 2004   predicate(n->get_int() == 2);
 2005   match(ConI);
 2006 
 2007   op_cost(0);
 2008   format %{ %}
 2009   interface(CONST_INTER);
 2010 %}
 2011 
 2012 operand immI_4()
 2013 %{
 2014   predicate(n->get_int() == 4);
 2015   match(ConI);
 2016 
 2017   op_cost(0);
 2018   format %{ %}
 2019   interface(CONST_INTER);
 2020 %}
 2021 
 2022 operand immI_8()
 2023 %{
 2024   predicate(n->get_int() == 8);
 2025   match(ConI);
 2026 
 2027   op_cost(0);
 2028   format %{ %}
 2029   interface(CONST_INTER);
 2030 %}
 2031 
 2032 // Valid scale values for addressing modes
 2033 operand immI2()
 2034 %{
 2035   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2036   match(ConI);
 2037 
 2038   format %{ %}
 2039   interface(CONST_INTER);
 2040 %}
 2041 
 2042 operand immU7()
 2043 %{
 2044   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2045   match(ConI);
 2046 
 2047   op_cost(5);
 2048   format %{ %}
 2049   interface(CONST_INTER);
 2050 %}
 2051 
 2052 operand immI8()
 2053 %{
 2054   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2055   match(ConI);
 2056 
 2057   op_cost(5);
 2058   format %{ %}
 2059   interface(CONST_INTER);
 2060 %}
 2061 
 2062 operand immU8()
 2063 %{
 2064   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2065   match(ConI);
 2066 
 2067   op_cost(5);
 2068   format %{ %}
 2069   interface(CONST_INTER);
 2070 %}
 2071 
 2072 operand immI16()
 2073 %{
 2074   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2075   match(ConI);
 2076 
 2077   op_cost(10);
 2078   format %{ %}
 2079   interface(CONST_INTER);
 2080 %}
 2081 
 2082 // Int Immediate non-negative
 2083 operand immU31()
 2084 %{
 2085   predicate(n->get_int() >= 0);
 2086   match(ConI);
 2087 
 2088   op_cost(0);
 2089   format %{ %}
 2090   interface(CONST_INTER);
 2091 %}
 2092 
 2093 // Pointer Immediate
 2094 operand immP()
 2095 %{
 2096   match(ConP);
 2097 
 2098   op_cost(10);
 2099   format %{ %}
 2100   interface(CONST_INTER);
 2101 %}
 2102 
 2103 // Null Pointer Immediate
 2104 operand immP0()
 2105 %{
 2106   predicate(n->get_ptr() == 0);
 2107   match(ConP);
 2108 
 2109   op_cost(5);
 2110   format %{ %}
 2111   interface(CONST_INTER);
 2112 %}
 2113 
 2114 // Pointer Immediate
 2115 operand immN() %{
 2116   match(ConN);
 2117 
 2118   op_cost(10);
 2119   format %{ %}
 2120   interface(CONST_INTER);
 2121 %}
 2122 
 2123 operand immNKlass() %{
 2124   match(ConNKlass);
 2125 
 2126   op_cost(10);
 2127   format %{ %}
 2128   interface(CONST_INTER);
 2129 %}
 2130 
 2131 // Null Pointer Immediate
 2132 operand immN0() %{
 2133   predicate(n->get_narrowcon() == 0);
 2134   match(ConN);
 2135 
 2136   op_cost(5);
 2137   format %{ %}
 2138   interface(CONST_INTER);
 2139 %}
 2140 
 2141 operand immP31()
 2142 %{
 2143   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2144             && (n->get_ptr() >> 31) == 0);
 2145   match(ConP);
 2146 
 2147   op_cost(5);
 2148   format %{ %}
 2149   interface(CONST_INTER);
 2150 %}
 2151 
 2152 
 2153 // Long Immediate
 2154 operand immL()
 2155 %{
 2156   match(ConL);
 2157 
 2158   op_cost(20);
 2159   format %{ %}
 2160   interface(CONST_INTER);
 2161 %}
 2162 
 2163 // Long Immediate 8-bit
 2164 operand immL8()
 2165 %{
 2166   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2167   match(ConL);
 2168 
 2169   op_cost(5);
 2170   format %{ %}
 2171   interface(CONST_INTER);
 2172 %}
 2173 
 2174 // Long Immediate 32-bit unsigned
 2175 operand immUL32()
 2176 %{
 2177   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2178   match(ConL);
 2179 
 2180   op_cost(10);
 2181   format %{ %}
 2182   interface(CONST_INTER);
 2183 %}
 2184 
 2185 // Long Immediate 32-bit signed
 2186 operand immL32()
 2187 %{
 2188   predicate(n->get_long() == (int) (n->get_long()));
 2189   match(ConL);
 2190 
 2191   op_cost(15);
 2192   format %{ %}
 2193   interface(CONST_INTER);
 2194 %}
 2195 
 2196 operand immL_Pow2()
 2197 %{
 2198   predicate(is_power_of_2((julong)n->get_long()));
 2199   match(ConL);
 2200 
 2201   op_cost(15);
 2202   format %{ %}
 2203   interface(CONST_INTER);
 2204 %}
 2205 
 2206 operand immL_NotPow2()
 2207 %{
 2208   predicate(is_power_of_2((julong)~n->get_long()));
 2209   match(ConL);
 2210 
 2211   op_cost(15);
 2212   format %{ %}
 2213   interface(CONST_INTER);
 2214 %}
 2215 
 2216 // Long Immediate zero
 2217 operand immL0()
 2218 %{
 2219   predicate(n->get_long() == 0L);
 2220   match(ConL);
 2221 
 2222   op_cost(10);
 2223   format %{ %}
 2224   interface(CONST_INTER);
 2225 %}
 2226 
 2227 // Constant for increment
 2228 operand immL1()
 2229 %{
 2230   predicate(n->get_long() == 1);
 2231   match(ConL);
 2232 
 2233   format %{ %}
 2234   interface(CONST_INTER);
 2235 %}
 2236 
 2237 // Constant for decrement
 2238 operand immL_M1()
 2239 %{
 2240   predicate(n->get_long() == -1);
 2241   match(ConL);
 2242 
 2243   format %{ %}
 2244   interface(CONST_INTER);
 2245 %}
 2246 
 2247 // Long Immediate: low 32-bit mask
 2248 operand immL_32bits()
 2249 %{
 2250   predicate(n->get_long() == 0xFFFFFFFFL);
 2251   match(ConL);
 2252   op_cost(20);
 2253 
 2254   format %{ %}
 2255   interface(CONST_INTER);
 2256 %}
 2257 
 2258 // Int Immediate: 2^n-1, positive
 2259 operand immI_Pow2M1()
 2260 %{
 2261   predicate((n->get_int() > 0)
 2262             && is_power_of_2((juint)n->get_int() + 1));
 2263   match(ConI);
 2264 
 2265   op_cost(20);
 2266   format %{ %}
 2267   interface(CONST_INTER);
 2268 %}
 2269 
 2270 // Float Immediate zero
 2271 operand immF0()
 2272 %{
 2273   predicate(jint_cast(n->getf()) == 0);
 2274   match(ConF);
 2275 
 2276   op_cost(5);
 2277   format %{ %}
 2278   interface(CONST_INTER);
 2279 %}
 2280 
 2281 // Float Immediate
 2282 operand immF()
 2283 %{
 2284   match(ConF);
 2285 
 2286   op_cost(15);
 2287   format %{ %}
 2288   interface(CONST_INTER);
 2289 %}
 2290 
 2291 // Double Immediate zero
 2292 operand immD0()
 2293 %{
 2294   predicate(jlong_cast(n->getd()) == 0);
 2295   match(ConD);
 2296 
 2297   op_cost(5);
 2298   format %{ %}
 2299   interface(CONST_INTER);
 2300 %}
 2301 
 2302 // Double Immediate
 2303 operand immD()
 2304 %{
 2305   match(ConD);
 2306 
 2307   op_cost(15);
 2308   format %{ %}
 2309   interface(CONST_INTER);
 2310 %}
 2311 
 2312 // Immediates for special shifts (sign extend)
 2313 
 2314 // Constants for increment
 2315 operand immI_16()
 2316 %{
 2317   predicate(n->get_int() == 16);
 2318   match(ConI);
 2319 
 2320   format %{ %}
 2321   interface(CONST_INTER);
 2322 %}
 2323 
 2324 operand immI_24()
 2325 %{
 2326   predicate(n->get_int() == 24);
 2327   match(ConI);
 2328 
 2329   format %{ %}
 2330   interface(CONST_INTER);
 2331 %}
 2332 
 2333 // Constant for byte-wide masking
 2334 operand immI_255()
 2335 %{
 2336   predicate(n->get_int() == 255);
 2337   match(ConI);
 2338 
 2339   format %{ %}
 2340   interface(CONST_INTER);
 2341 %}
 2342 
 2343 // Constant for short-wide masking
 2344 operand immI_65535()
 2345 %{
 2346   predicate(n->get_int() == 65535);
 2347   match(ConI);
 2348 
 2349   format %{ %}
 2350   interface(CONST_INTER);
 2351 %}
 2352 
 2353 // Constant for byte-wide masking
 2354 operand immL_255()
 2355 %{
 2356   predicate(n->get_long() == 255);
 2357   match(ConL);
 2358 
 2359   format %{ %}
 2360   interface(CONST_INTER);
 2361 %}
 2362 
 2363 // Constant for short-wide masking
 2364 operand immL_65535()
 2365 %{
 2366   predicate(n->get_long() == 65535);
 2367   match(ConL);
 2368 
 2369   format %{ %}
 2370   interface(CONST_INTER);
 2371 %}
 2372 
 2373 operand kReg()
 2374 %{
 2375   constraint(ALLOC_IN_RC(vectmask_reg));
 2376   match(RegVectMask);
 2377   format %{%}
 2378   interface(REG_INTER);
 2379 %}
 2380 
 2381 // Register Operands
 2382 // Integer Register
 2383 operand rRegI()
 2384 %{
 2385   constraint(ALLOC_IN_RC(int_reg));
 2386   match(RegI);
 2387 
 2388   match(rax_RegI);
 2389   match(rbx_RegI);
 2390   match(rcx_RegI);
 2391   match(rdx_RegI);
 2392   match(rdi_RegI);
 2393 
 2394   format %{ %}
 2395   interface(REG_INTER);
 2396 %}
 2397 
 2398 // Special Registers
 2399 operand rax_RegI()
 2400 %{
 2401   constraint(ALLOC_IN_RC(int_rax_reg));
 2402   match(RegI);
 2403   match(rRegI);
 2404 
 2405   format %{ "RAX" %}
 2406   interface(REG_INTER);
 2407 %}
 2408 
 2409 // Special Registers
 2410 operand rbx_RegI()
 2411 %{
 2412   constraint(ALLOC_IN_RC(int_rbx_reg));
 2413   match(RegI);
 2414   match(rRegI);
 2415 
 2416   format %{ "RBX" %}
 2417   interface(REG_INTER);
 2418 %}
 2419 
 2420 operand rcx_RegI()
 2421 %{
 2422   constraint(ALLOC_IN_RC(int_rcx_reg));
 2423   match(RegI);
 2424   match(rRegI);
 2425 
 2426   format %{ "RCX" %}
 2427   interface(REG_INTER);
 2428 %}
 2429 
 2430 operand rdx_RegI()
 2431 %{
 2432   constraint(ALLOC_IN_RC(int_rdx_reg));
 2433   match(RegI);
 2434   match(rRegI);
 2435 
 2436   format %{ "RDX" %}
 2437   interface(REG_INTER);
 2438 %}
 2439 
 2440 operand rdi_RegI()
 2441 %{
 2442   constraint(ALLOC_IN_RC(int_rdi_reg));
 2443   match(RegI);
 2444   match(rRegI);
 2445 
 2446   format %{ "RDI" %}
 2447   interface(REG_INTER);
 2448 %}
 2449 
 2450 operand no_rax_rdx_RegI()
 2451 %{
 2452   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2453   match(RegI);
 2454   match(rbx_RegI);
 2455   match(rcx_RegI);
 2456   match(rdi_RegI);
 2457 
 2458   format %{ %}
 2459   interface(REG_INTER);
 2460 %}
 2461 
 2462 operand no_rbp_r13_RegI()
 2463 %{
 2464   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2465   match(RegI);
 2466   match(rRegI);
 2467   match(rax_RegI);
 2468   match(rbx_RegI);
 2469   match(rcx_RegI);
 2470   match(rdx_RegI);
 2471   match(rdi_RegI);
 2472 
 2473   format %{ %}
 2474   interface(REG_INTER);
 2475 %}
 2476 
 2477 // Pointer Register
 2478 operand any_RegP()
 2479 %{
 2480   constraint(ALLOC_IN_RC(any_reg));
 2481   match(RegP);
 2482   match(rax_RegP);
 2483   match(rbx_RegP);
 2484   match(rdi_RegP);
 2485   match(rsi_RegP);
 2486   match(rbp_RegP);
 2487   match(r15_RegP);
 2488   match(rRegP);
 2489 
 2490   format %{ %}
 2491   interface(REG_INTER);
 2492 %}
 2493 
 2494 operand rRegP()
 2495 %{
 2496   constraint(ALLOC_IN_RC(ptr_reg));
 2497   match(RegP);
 2498   match(rax_RegP);
 2499   match(rbx_RegP);
 2500   match(rdi_RegP);
 2501   match(rsi_RegP);
 2502   match(rbp_RegP);  // See Q&A below about
 2503   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2504 
 2505   format %{ %}
 2506   interface(REG_INTER);
 2507 %}
 2508 
 2509 operand rRegN() %{
 2510   constraint(ALLOC_IN_RC(int_reg));
 2511   match(RegN);
 2512 
 2513   format %{ %}
 2514   interface(REG_INTER);
 2515 %}
 2516 
 2517 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2518 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2519 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2520 // The output of an instruction is controlled by the allocator, which respects
 2521 // register class masks, not match rules.  Unless an instruction mentions
 2522 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2523 // by the allocator as an input.
 2524 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2525 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2526 // result, RBP is not included in the output of the instruction either.
 2527 
 2528 // This operand is not allowed to use RBP even if
 2529 // RBP is not used to hold the frame pointer.
 2530 operand no_rbp_RegP()
 2531 %{
 2532   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2533   match(RegP);
 2534   match(rbx_RegP);
 2535   match(rsi_RegP);
 2536   match(rdi_RegP);
 2537 
 2538   format %{ %}
 2539   interface(REG_INTER);
 2540 %}
 2541 
 2542 // Special Registers
 2543 // Return a pointer value
 2544 operand rax_RegP()
 2545 %{
 2546   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2547   match(RegP);
 2548   match(rRegP);
 2549 
 2550   format %{ %}
 2551   interface(REG_INTER);
 2552 %}
 2553 
 2554 // Special Registers
 2555 // Return a compressed pointer value
 2556 operand rax_RegN()
 2557 %{
 2558   constraint(ALLOC_IN_RC(int_rax_reg));
 2559   match(RegN);
 2560   match(rRegN);
 2561 
 2562   format %{ %}
 2563   interface(REG_INTER);
 2564 %}
 2565 
 2566 // Used in AtomicAdd
 2567 operand rbx_RegP()
 2568 %{
 2569   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2570   match(RegP);
 2571   match(rRegP);
 2572 
 2573   format %{ %}
 2574   interface(REG_INTER);
 2575 %}
 2576 
 2577 operand rsi_RegP()
 2578 %{
 2579   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2580   match(RegP);
 2581   match(rRegP);
 2582 
 2583   format %{ %}
 2584   interface(REG_INTER);
 2585 %}
 2586 
 2587 operand rbp_RegP()
 2588 %{
 2589   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2590   match(RegP);
 2591   match(rRegP);
 2592 
 2593   format %{ %}
 2594   interface(REG_INTER);
 2595 %}
 2596 
 2597 // Used in rep stosq
 2598 operand rdi_RegP()
 2599 %{
 2600   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2601   match(RegP);
 2602   match(rRegP);
 2603 
 2604   format %{ %}
 2605   interface(REG_INTER);
 2606 %}
 2607 
 2608 operand r15_RegP()
 2609 %{
 2610   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2611   match(RegP);
 2612   match(rRegP);
 2613 
 2614   format %{ %}
 2615   interface(REG_INTER);
 2616 %}
 2617 
 2618 operand rRegL()
 2619 %{
 2620   constraint(ALLOC_IN_RC(long_reg));
 2621   match(RegL);
 2622   match(rax_RegL);
 2623   match(rdx_RegL);
 2624 
 2625   format %{ %}
 2626   interface(REG_INTER);
 2627 %}
 2628 
 2629 // Special Registers
 2630 operand no_rax_rdx_RegL()
 2631 %{
 2632   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2633   match(RegL);
 2634   match(rRegL);
 2635 
 2636   format %{ %}
 2637   interface(REG_INTER);
 2638 %}
 2639 
 2640 operand rax_RegL()
 2641 %{
 2642   constraint(ALLOC_IN_RC(long_rax_reg));
 2643   match(RegL);
 2644   match(rRegL);
 2645 
 2646   format %{ "RAX" %}
 2647   interface(REG_INTER);
 2648 %}
 2649 
 2650 operand rcx_RegL()
 2651 %{
 2652   constraint(ALLOC_IN_RC(long_rcx_reg));
 2653   match(RegL);
 2654   match(rRegL);
 2655 
 2656   format %{ %}
 2657   interface(REG_INTER);
 2658 %}
 2659 
 2660 operand rdx_RegL()
 2661 %{
 2662   constraint(ALLOC_IN_RC(long_rdx_reg));
 2663   match(RegL);
 2664   match(rRegL);
 2665 
 2666   format %{ %}
 2667   interface(REG_INTER);
 2668 %}
 2669 
 2670 operand no_rbp_r13_RegL()
 2671 %{
 2672   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2673   match(RegL);
 2674   match(rRegL);
 2675   match(rax_RegL);
 2676   match(rcx_RegL);
 2677   match(rdx_RegL);
 2678 
 2679   format %{ %}
 2680   interface(REG_INTER);
 2681 %}
 2682 
 2683 // Flags register, used as output of compare instructions
 2684 operand rFlagsReg()
 2685 %{
 2686   constraint(ALLOC_IN_RC(int_flags));
 2687   match(RegFlags);
 2688 
 2689   format %{ "RFLAGS" %}
 2690   interface(REG_INTER);
 2691 %}
 2692 
 2693 // Flags register, used as output of FLOATING POINT compare instructions
 2694 operand rFlagsRegU()
 2695 %{
 2696   constraint(ALLOC_IN_RC(int_flags));
 2697   match(RegFlags);
 2698 
 2699   format %{ "RFLAGS_U" %}
 2700   interface(REG_INTER);
 2701 %}
 2702 
 2703 operand rFlagsRegUCF() %{
 2704   constraint(ALLOC_IN_RC(int_flags));
 2705   match(RegFlags);
 2706   predicate(false);
 2707 
 2708   format %{ "RFLAGS_U_CF" %}
 2709   interface(REG_INTER);
 2710 %}
 2711 
 2712 // Float register operands
 2713 operand regF() %{
 2714    constraint(ALLOC_IN_RC(float_reg));
 2715    match(RegF);
 2716 
 2717    format %{ %}
 2718    interface(REG_INTER);
 2719 %}
 2720 
 2721 // Float register operands
 2722 operand legRegF() %{
 2723    constraint(ALLOC_IN_RC(float_reg_legacy));
 2724    match(RegF);
 2725 
 2726    format %{ %}
 2727    interface(REG_INTER);
 2728 %}
 2729 
 2730 // Float register operands
 2731 operand vlRegF() %{
 2732    constraint(ALLOC_IN_RC(float_reg_vl));
 2733    match(RegF);
 2734 
 2735    format %{ %}
 2736    interface(REG_INTER);
 2737 %}
 2738 
 2739 // Double register operands
 2740 operand regD() %{
 2741    constraint(ALLOC_IN_RC(double_reg));
 2742    match(RegD);
 2743 
 2744    format %{ %}
 2745    interface(REG_INTER);
 2746 %}
 2747 
 2748 // Double register operands
 2749 operand legRegD() %{
 2750    constraint(ALLOC_IN_RC(double_reg_legacy));
 2751    match(RegD);
 2752 
 2753    format %{ %}
 2754    interface(REG_INTER);
 2755 %}
 2756 
 2757 // Double register operands
 2758 operand vlRegD() %{
 2759    constraint(ALLOC_IN_RC(double_reg_vl));
 2760    match(RegD);
 2761 
 2762    format %{ %}
 2763    interface(REG_INTER);
 2764 %}
 2765 
 2766 //----------Memory Operands----------------------------------------------------
 2767 // Direct Memory Operand
 2768 // operand direct(immP addr)
 2769 // %{
 2770 //   match(addr);
 2771 
 2772 //   format %{ "[$addr]" %}
 2773 //   interface(MEMORY_INTER) %{
 2774 //     base(0xFFFFFFFF);
 2775 //     index(0x4);
 2776 //     scale(0x0);
 2777 //     disp($addr);
 2778 //   %}
 2779 // %}
 2780 
 2781 // Indirect Memory Operand
 2782 operand indirect(any_RegP reg)
 2783 %{
 2784   constraint(ALLOC_IN_RC(ptr_reg));
 2785   match(reg);
 2786 
 2787   format %{ "[$reg]" %}
 2788   interface(MEMORY_INTER) %{
 2789     base($reg);
 2790     index(0x4);
 2791     scale(0x0);
 2792     disp(0x0);
 2793   %}
 2794 %}
 2795 
 2796 // Indirect Memory Plus Short Offset Operand
 2797 operand indOffset8(any_RegP reg, immL8 off)
 2798 %{
 2799   constraint(ALLOC_IN_RC(ptr_reg));
 2800   match(AddP reg off);
 2801 
 2802   format %{ "[$reg + $off (8-bit)]" %}
 2803   interface(MEMORY_INTER) %{
 2804     base($reg);
 2805     index(0x4);
 2806     scale(0x0);
 2807     disp($off);
 2808   %}
 2809 %}
 2810 
 2811 // Indirect Memory Plus Long Offset Operand
 2812 operand indOffset32(any_RegP reg, immL32 off)
 2813 %{
 2814   constraint(ALLOC_IN_RC(ptr_reg));
 2815   match(AddP reg off);
 2816 
 2817   format %{ "[$reg + $off (32-bit)]" %}
 2818   interface(MEMORY_INTER) %{
 2819     base($reg);
 2820     index(0x4);
 2821     scale(0x0);
 2822     disp($off);
 2823   %}
 2824 %}
 2825 
 2826 // Indirect Memory Plus Index Register Plus Offset Operand
 2827 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2828 %{
 2829   constraint(ALLOC_IN_RC(ptr_reg));
 2830   match(AddP (AddP reg lreg) off);
 2831 
 2832   op_cost(10);
 2833   format %{"[$reg + $off + $lreg]" %}
 2834   interface(MEMORY_INTER) %{
 2835     base($reg);
 2836     index($lreg);
 2837     scale(0x0);
 2838     disp($off);
 2839   %}
 2840 %}
 2841 
 2842 // Indirect Memory Plus Index Register Plus Offset Operand
 2843 operand indIndex(any_RegP reg, rRegL lreg)
 2844 %{
 2845   constraint(ALLOC_IN_RC(ptr_reg));
 2846   match(AddP reg lreg);
 2847 
 2848   op_cost(10);
 2849   format %{"[$reg + $lreg]" %}
 2850   interface(MEMORY_INTER) %{
 2851     base($reg);
 2852     index($lreg);
 2853     scale(0x0);
 2854     disp(0x0);
 2855   %}
 2856 %}
 2857 
 2858 // Indirect Memory Times Scale Plus Index Register
 2859 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2860 %{
 2861   constraint(ALLOC_IN_RC(ptr_reg));
 2862   match(AddP reg (LShiftL lreg scale));
 2863 
 2864   op_cost(10);
 2865   format %{"[$reg + $lreg << $scale]" %}
 2866   interface(MEMORY_INTER) %{
 2867     base($reg);
 2868     index($lreg);
 2869     scale($scale);
 2870     disp(0x0);
 2871   %}
 2872 %}
 2873 
 2874 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 2875 %{
 2876   constraint(ALLOC_IN_RC(ptr_reg));
 2877   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 2878   match(AddP reg (LShiftL (ConvI2L idx) scale));
 2879 
 2880   op_cost(10);
 2881   format %{"[$reg + pos $idx << $scale]" %}
 2882   interface(MEMORY_INTER) %{
 2883     base($reg);
 2884     index($idx);
 2885     scale($scale);
 2886     disp(0x0);
 2887   %}
 2888 %}
 2889 
 2890 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 2891 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 2892 %{
 2893   constraint(ALLOC_IN_RC(ptr_reg));
 2894   match(AddP (AddP reg (LShiftL lreg scale)) off);
 2895 
 2896   op_cost(10);
 2897   format %{"[$reg + $off + $lreg << $scale]" %}
 2898   interface(MEMORY_INTER) %{
 2899     base($reg);
 2900     index($lreg);
 2901     scale($scale);
 2902     disp($off);
 2903   %}
 2904 %}
 2905 
 2906 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 2907 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 2908 %{
 2909   constraint(ALLOC_IN_RC(ptr_reg));
 2910   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 2911   match(AddP (AddP reg (ConvI2L idx)) off);
 2912 
 2913   op_cost(10);
 2914   format %{"[$reg + $off + $idx]" %}
 2915   interface(MEMORY_INTER) %{
 2916     base($reg);
 2917     index($idx);
 2918     scale(0x0);
 2919     disp($off);
 2920   %}
 2921 %}
 2922 
 2923 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 2924 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 2925 %{
 2926   constraint(ALLOC_IN_RC(ptr_reg));
 2927   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 2928   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 2929 
 2930   op_cost(10);
 2931   format %{"[$reg + $off + $idx << $scale]" %}
 2932   interface(MEMORY_INTER) %{
 2933     base($reg);
 2934     index($idx);
 2935     scale($scale);
 2936     disp($off);
 2937   %}
 2938 %}
 2939 
 2940 // Indirect Narrow Oop Plus Offset Operand
 2941 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 2942 // we can't free r12 even with CompressedOops::base() == nullptr.
 2943 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 2944   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 2945   constraint(ALLOC_IN_RC(ptr_reg));
 2946   match(AddP (DecodeN reg) off);
 2947 
 2948   op_cost(10);
 2949   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 2950   interface(MEMORY_INTER) %{
 2951     base(0xc); // R12
 2952     index($reg);
 2953     scale(0x3);
 2954     disp($off);
 2955   %}
 2956 %}
 2957 
 2958 // Indirect Memory Operand
 2959 operand indirectNarrow(rRegN reg)
 2960 %{
 2961   predicate(CompressedOops::shift() == 0);
 2962   constraint(ALLOC_IN_RC(ptr_reg));
 2963   match(DecodeN reg);
 2964 
 2965   format %{ "[$reg]" %}
 2966   interface(MEMORY_INTER) %{
 2967     base($reg);
 2968     index(0x4);
 2969     scale(0x0);
 2970     disp(0x0);
 2971   %}
 2972 %}
 2973 
 2974 // Indirect Memory Plus Short Offset Operand
 2975 operand indOffset8Narrow(rRegN reg, immL8 off)
 2976 %{
 2977   predicate(CompressedOops::shift() == 0);
 2978   constraint(ALLOC_IN_RC(ptr_reg));
 2979   match(AddP (DecodeN reg) off);
 2980 
 2981   format %{ "[$reg + $off (8-bit)]" %}
 2982   interface(MEMORY_INTER) %{
 2983     base($reg);
 2984     index(0x4);
 2985     scale(0x0);
 2986     disp($off);
 2987   %}
 2988 %}
 2989 
 2990 // Indirect Memory Plus Long Offset Operand
 2991 operand indOffset32Narrow(rRegN reg, immL32 off)
 2992 %{
 2993   predicate(CompressedOops::shift() == 0);
 2994   constraint(ALLOC_IN_RC(ptr_reg));
 2995   match(AddP (DecodeN reg) off);
 2996 
 2997   format %{ "[$reg + $off (32-bit)]" %}
 2998   interface(MEMORY_INTER) %{
 2999     base($reg);
 3000     index(0x4);
 3001     scale(0x0);
 3002     disp($off);
 3003   %}
 3004 %}
 3005 
 3006 // Indirect Memory Plus Index Register Plus Offset Operand
 3007 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3008 %{
 3009   predicate(CompressedOops::shift() == 0);
 3010   constraint(ALLOC_IN_RC(ptr_reg));
 3011   match(AddP (AddP (DecodeN reg) lreg) off);
 3012 
 3013   op_cost(10);
 3014   format %{"[$reg + $off + $lreg]" %}
 3015   interface(MEMORY_INTER) %{
 3016     base($reg);
 3017     index($lreg);
 3018     scale(0x0);
 3019     disp($off);
 3020   %}
 3021 %}
 3022 
 3023 // Indirect Memory Plus Index Register Plus Offset Operand
 3024 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3025 %{
 3026   predicate(CompressedOops::shift() == 0);
 3027   constraint(ALLOC_IN_RC(ptr_reg));
 3028   match(AddP (DecodeN reg) lreg);
 3029 
 3030   op_cost(10);
 3031   format %{"[$reg + $lreg]" %}
 3032   interface(MEMORY_INTER) %{
 3033     base($reg);
 3034     index($lreg);
 3035     scale(0x0);
 3036     disp(0x0);
 3037   %}
 3038 %}
 3039 
 3040 // Indirect Memory Times Scale Plus Index Register
 3041 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3042 %{
 3043   predicate(CompressedOops::shift() == 0);
 3044   constraint(ALLOC_IN_RC(ptr_reg));
 3045   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3046 
 3047   op_cost(10);
 3048   format %{"[$reg + $lreg << $scale]" %}
 3049   interface(MEMORY_INTER) %{
 3050     base($reg);
 3051     index($lreg);
 3052     scale($scale);
 3053     disp(0x0);
 3054   %}
 3055 %}
 3056 
 3057 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3058 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3059 %{
 3060   predicate(CompressedOops::shift() == 0);
 3061   constraint(ALLOC_IN_RC(ptr_reg));
 3062   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3063 
 3064   op_cost(10);
 3065   format %{"[$reg + $off + $lreg << $scale]" %}
 3066   interface(MEMORY_INTER) %{
 3067     base($reg);
 3068     index($lreg);
 3069     scale($scale);
 3070     disp($off);
 3071   %}
 3072 %}
 3073 
 3074 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3075 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3076 %{
 3077   constraint(ALLOC_IN_RC(ptr_reg));
 3078   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3079   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3080 
 3081   op_cost(10);
 3082   format %{"[$reg + $off + $idx]" %}
 3083   interface(MEMORY_INTER) %{
 3084     base($reg);
 3085     index($idx);
 3086     scale(0x0);
 3087     disp($off);
 3088   %}
 3089 %}
 3090 
 3091 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3092 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3093 %{
 3094   constraint(ALLOC_IN_RC(ptr_reg));
 3095   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3096   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3097 
 3098   op_cost(10);
 3099   format %{"[$reg + $off + $idx << $scale]" %}
 3100   interface(MEMORY_INTER) %{
 3101     base($reg);
 3102     index($idx);
 3103     scale($scale);
 3104     disp($off);
 3105   %}
 3106 %}
 3107 
 3108 //----------Special Memory Operands--------------------------------------------
 3109 // Stack Slot Operand - This operand is used for loading and storing temporary
 3110 //                      values on the stack where a match requires a value to
 3111 //                      flow through memory.
 3112 operand stackSlotP(sRegP reg)
 3113 %{
 3114   constraint(ALLOC_IN_RC(stack_slots));
 3115   // No match rule because this operand is only generated in matching
 3116 
 3117   format %{ "[$reg]" %}
 3118   interface(MEMORY_INTER) %{
 3119     base(0x4);   // RSP
 3120     index(0x4);  // No Index
 3121     scale(0x0);  // No Scale
 3122     disp($reg);  // Stack Offset
 3123   %}
 3124 %}
 3125 
 3126 operand stackSlotI(sRegI reg)
 3127 %{
 3128   constraint(ALLOC_IN_RC(stack_slots));
 3129   // No match rule because this operand is only generated in matching
 3130 
 3131   format %{ "[$reg]" %}
 3132   interface(MEMORY_INTER) %{
 3133     base(0x4);   // RSP
 3134     index(0x4);  // No Index
 3135     scale(0x0);  // No Scale
 3136     disp($reg);  // Stack Offset
 3137   %}
 3138 %}
 3139 
 3140 operand stackSlotF(sRegF reg)
 3141 %{
 3142   constraint(ALLOC_IN_RC(stack_slots));
 3143   // No match rule because this operand is only generated in matching
 3144 
 3145   format %{ "[$reg]" %}
 3146   interface(MEMORY_INTER) %{
 3147     base(0x4);   // RSP
 3148     index(0x4);  // No Index
 3149     scale(0x0);  // No Scale
 3150     disp($reg);  // Stack Offset
 3151   %}
 3152 %}
 3153 
 3154 operand stackSlotD(sRegD reg)
 3155 %{
 3156   constraint(ALLOC_IN_RC(stack_slots));
 3157   // No match rule because this operand is only generated in matching
 3158 
 3159   format %{ "[$reg]" %}
 3160   interface(MEMORY_INTER) %{
 3161     base(0x4);   // RSP
 3162     index(0x4);  // No Index
 3163     scale(0x0);  // No Scale
 3164     disp($reg);  // Stack Offset
 3165   %}
 3166 %}
 3167 operand stackSlotL(sRegL reg)
 3168 %{
 3169   constraint(ALLOC_IN_RC(stack_slots));
 3170   // No match rule because this operand is only generated in matching
 3171 
 3172   format %{ "[$reg]" %}
 3173   interface(MEMORY_INTER) %{
 3174     base(0x4);   // RSP
 3175     index(0x4);  // No Index
 3176     scale(0x0);  // No Scale
 3177     disp($reg);  // Stack Offset
 3178   %}
 3179 %}
 3180 
 3181 //----------Conditional Branch Operands----------------------------------------
 3182 // Comparison Op  - This is the operation of the comparison, and is limited to
 3183 //                  the following set of codes:
 3184 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3185 //
 3186 // Other attributes of the comparison, such as unsignedness, are specified
 3187 // by the comparison instruction that sets a condition code flags register.
 3188 // That result is represented by a flags operand whose subtype is appropriate
 3189 // to the unsignedness (etc.) of the comparison.
 3190 //
 3191 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3192 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3193 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3194 
 3195 // Comparison Code
 3196 operand cmpOp()
 3197 %{
 3198   match(Bool);
 3199 
 3200   format %{ "" %}
 3201   interface(COND_INTER) %{
 3202     equal(0x4, "e");
 3203     not_equal(0x5, "ne");
 3204     less(0xC, "l");
 3205     greater_equal(0xD, "ge");
 3206     less_equal(0xE, "le");
 3207     greater(0xF, "g");
 3208     overflow(0x0, "o");
 3209     no_overflow(0x1, "no");
 3210   %}
 3211 %}
 3212 
 3213 // Comparison Code, unsigned compare.  Used by FP also, with
 3214 // C2 (unordered) turned into GT or LT already.  The other bits
 3215 // C0 and C3 are turned into Carry & Zero flags.
 3216 operand cmpOpU()
 3217 %{
 3218   match(Bool);
 3219 
 3220   format %{ "" %}
 3221   interface(COND_INTER) %{
 3222     equal(0x4, "e");
 3223     not_equal(0x5, "ne");
 3224     less(0x2, "b");
 3225     greater_equal(0x3, "ae");
 3226     less_equal(0x6, "be");
 3227     greater(0x7, "a");
 3228     overflow(0x0, "o");
 3229     no_overflow(0x1, "no");
 3230   %}
 3231 %}
 3232 
 3233 
 3234 // Floating comparisons that don't require any fixup for the unordered case,
 3235 // If both inputs of the comparison are the same, ZF is always set so we
 3236 // don't need to use cmpOpUCF2 for eq/ne
 3237 operand cmpOpUCF() %{
 3238   match(Bool);
 3239   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3240             n->as_Bool()->_test._test == BoolTest::ge ||
 3241             n->as_Bool()->_test._test == BoolTest::le ||
 3242             n->as_Bool()->_test._test == BoolTest::gt ||
 3243             n->in(1)->in(1) == n->in(1)->in(2));
 3244   format %{ "" %}
 3245   interface(COND_INTER) %{
 3246     equal(0xb, "np");
 3247     not_equal(0xa, "p");
 3248     less(0x2, "b");
 3249     greater_equal(0x3, "ae");
 3250     less_equal(0x6, "be");
 3251     greater(0x7, "a");
 3252     overflow(0x0, "o");
 3253     no_overflow(0x1, "no");
 3254   %}
 3255 %}
 3256 
 3257 
 3258 // Floating comparisons that can be fixed up with extra conditional jumps
 3259 operand cmpOpUCF2() %{
 3260   match(Bool);
 3261   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3262              n->as_Bool()->_test._test == BoolTest::eq) &&
 3263             n->in(1)->in(1) != n->in(1)->in(2));
 3264   format %{ "" %}
 3265   interface(COND_INTER) %{
 3266     equal(0x4, "e");
 3267     not_equal(0x5, "ne");
 3268     less(0x2, "b");
 3269     greater_equal(0x3, "ae");
 3270     less_equal(0x6, "be");
 3271     greater(0x7, "a");
 3272     overflow(0x0, "o");
 3273     no_overflow(0x1, "no");
 3274   %}
 3275 %}
 3276 
 3277 //----------OPERAND CLASSES----------------------------------------------------
 3278 // Operand Classes are groups of operands that are used as to simplify
 3279 // instruction definitions by not requiring the AD writer to specify separate
 3280 // instructions for every form of operand when the instruction accepts
 3281 // multiple operand types with the same basic encoding and format.  The classic
 3282 // case of this is memory operands.
 3283 
 3284 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3285                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3286                indCompressedOopOffset,
 3287                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3288                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3289                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3290 
 3291 //----------PIPELINE-----------------------------------------------------------
 3292 // Rules which define the behavior of the target architectures pipeline.
 3293 pipeline %{
 3294 
 3295 //----------ATTRIBUTES---------------------------------------------------------
 3296 attributes %{
 3297   variable_size_instructions;        // Fixed size instructions
 3298   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3299   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3300   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3301   instruction_fetch_units = 1;       // of 16 bytes
 3302 
 3303   // List of nop instructions
 3304   nops( MachNop );
 3305 %}
 3306 
 3307 //----------RESOURCES----------------------------------------------------------
 3308 // Resources are the functional units available to the machine
 3309 
 3310 // Generic P2/P3 pipeline
 3311 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3312 // 3 instructions decoded per cycle.
 3313 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3314 // 3 ALU op, only ALU0 handles mul instructions.
 3315 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3316            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3317            BR, FPU,
 3318            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3319 
 3320 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3321 // Pipeline Description specifies the stages in the machine's pipeline
 3322 
 3323 // Generic P2/P3 pipeline
 3324 pipe_desc(S0, S1, S2, S3, S4, S5);
 3325 
 3326 //----------PIPELINE CLASSES---------------------------------------------------
 3327 // Pipeline Classes describe the stages in which input and output are
 3328 // referenced by the hardware pipeline.
 3329 
 3330 // Naming convention: ialu or fpu
 3331 // Then: _reg
 3332 // Then: _reg if there is a 2nd register
 3333 // Then: _long if it's a pair of instructions implementing a long
 3334 // Then: _fat if it requires the big decoder
 3335 //   Or: _mem if it requires the big decoder and a memory unit.
 3336 
 3337 // Integer ALU reg operation
 3338 pipe_class ialu_reg(rRegI dst)
 3339 %{
 3340     single_instruction;
 3341     dst    : S4(write);
 3342     dst    : S3(read);
 3343     DECODE : S0;        // any decoder
 3344     ALU    : S3;        // any alu
 3345 %}
 3346 
 3347 // Long ALU reg operation
 3348 pipe_class ialu_reg_long(rRegL dst)
 3349 %{
 3350     instruction_count(2);
 3351     dst    : S4(write);
 3352     dst    : S3(read);
 3353     DECODE : S0(2);     // any 2 decoders
 3354     ALU    : S3(2);     // both alus
 3355 %}
 3356 
 3357 // Integer ALU reg operation using big decoder
 3358 pipe_class ialu_reg_fat(rRegI dst)
 3359 %{
 3360     single_instruction;
 3361     dst    : S4(write);
 3362     dst    : S3(read);
 3363     D0     : S0;        // big decoder only
 3364     ALU    : S3;        // any alu
 3365 %}
 3366 
 3367 // Integer ALU reg-reg operation
 3368 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3369 %{
 3370     single_instruction;
 3371     dst    : S4(write);
 3372     src    : S3(read);
 3373     DECODE : S0;        // any decoder
 3374     ALU    : S3;        // any alu
 3375 %}
 3376 
 3377 // Integer ALU reg-reg operation
 3378 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3379 %{
 3380     single_instruction;
 3381     dst    : S4(write);
 3382     src    : S3(read);
 3383     D0     : S0;        // big decoder only
 3384     ALU    : S3;        // any alu
 3385 %}
 3386 
 3387 // Integer ALU reg-mem operation
 3388 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3389 %{
 3390     single_instruction;
 3391     dst    : S5(write);
 3392     mem    : S3(read);
 3393     D0     : S0;        // big decoder only
 3394     ALU    : S4;        // any alu
 3395     MEM    : S3;        // any mem
 3396 %}
 3397 
 3398 // Integer mem operation (prefetch)
 3399 pipe_class ialu_mem(memory mem)
 3400 %{
 3401     single_instruction;
 3402     mem    : S3(read);
 3403     D0     : S0;        // big decoder only
 3404     MEM    : S3;        // any mem
 3405 %}
 3406 
 3407 // Integer Store to Memory
 3408 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3409 %{
 3410     single_instruction;
 3411     mem    : S3(read);
 3412     src    : S5(read);
 3413     D0     : S0;        // big decoder only
 3414     ALU    : S4;        // any alu
 3415     MEM    : S3;
 3416 %}
 3417 
 3418 // // Long Store to Memory
 3419 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3420 // %{
 3421 //     instruction_count(2);
 3422 //     mem    : S3(read);
 3423 //     src    : S5(read);
 3424 //     D0     : S0(2);          // big decoder only; twice
 3425 //     ALU    : S4(2);     // any 2 alus
 3426 //     MEM    : S3(2);  // Both mems
 3427 // %}
 3428 
 3429 // Integer Store to Memory
 3430 pipe_class ialu_mem_imm(memory mem)
 3431 %{
 3432     single_instruction;
 3433     mem    : S3(read);
 3434     D0     : S0;        // big decoder only
 3435     ALU    : S4;        // any alu
 3436     MEM    : S3;
 3437 %}
 3438 
 3439 // Integer ALU0 reg-reg operation
 3440 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3441 %{
 3442     single_instruction;
 3443     dst    : S4(write);
 3444     src    : S3(read);
 3445     D0     : S0;        // Big decoder only
 3446     ALU0   : S3;        // only alu0
 3447 %}
 3448 
 3449 // Integer ALU0 reg-mem operation
 3450 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3451 %{
 3452     single_instruction;
 3453     dst    : S5(write);
 3454     mem    : S3(read);
 3455     D0     : S0;        // big decoder only
 3456     ALU0   : S4;        // ALU0 only
 3457     MEM    : S3;        // any mem
 3458 %}
 3459 
 3460 // Integer ALU reg-reg operation
 3461 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3462 %{
 3463     single_instruction;
 3464     cr     : S4(write);
 3465     src1   : S3(read);
 3466     src2   : S3(read);
 3467     DECODE : S0;        // any decoder
 3468     ALU    : S3;        // any alu
 3469 %}
 3470 
 3471 // Integer ALU reg-imm operation
 3472 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3473 %{
 3474     single_instruction;
 3475     cr     : S4(write);
 3476     src1   : S3(read);
 3477     DECODE : S0;        // any decoder
 3478     ALU    : S3;        // any alu
 3479 %}
 3480 
 3481 // Integer ALU reg-mem operation
 3482 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3483 %{
 3484     single_instruction;
 3485     cr     : S4(write);
 3486     src1   : S3(read);
 3487     src2   : S3(read);
 3488     D0     : S0;        // big decoder only
 3489     ALU    : S4;        // any alu
 3490     MEM    : S3;
 3491 %}
 3492 
 3493 // Conditional move reg-reg
 3494 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3495 %{
 3496     instruction_count(4);
 3497     y      : S4(read);
 3498     q      : S3(read);
 3499     p      : S3(read);
 3500     DECODE : S0(4);     // any decoder
 3501 %}
 3502 
 3503 // Conditional move reg-reg
 3504 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3505 %{
 3506     single_instruction;
 3507     dst    : S4(write);
 3508     src    : S3(read);
 3509     cr     : S3(read);
 3510     DECODE : S0;        // any decoder
 3511 %}
 3512 
 3513 // Conditional move reg-mem
 3514 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3515 %{
 3516     single_instruction;
 3517     dst    : S4(write);
 3518     src    : S3(read);
 3519     cr     : S3(read);
 3520     DECODE : S0;        // any decoder
 3521     MEM    : S3;
 3522 %}
 3523 
 3524 // Conditional move reg-reg long
 3525 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3526 %{
 3527     single_instruction;
 3528     dst    : S4(write);
 3529     src    : S3(read);
 3530     cr     : S3(read);
 3531     DECODE : S0(2);     // any 2 decoders
 3532 %}
 3533 
 3534 // Float reg-reg operation
 3535 pipe_class fpu_reg(regD dst)
 3536 %{
 3537     instruction_count(2);
 3538     dst    : S3(read);
 3539     DECODE : S0(2);     // any 2 decoders
 3540     FPU    : S3;
 3541 %}
 3542 
 3543 // Float reg-reg operation
 3544 pipe_class fpu_reg_reg(regD dst, regD src)
 3545 %{
 3546     instruction_count(2);
 3547     dst    : S4(write);
 3548     src    : S3(read);
 3549     DECODE : S0(2);     // any 2 decoders
 3550     FPU    : S3;
 3551 %}
 3552 
 3553 // Float reg-reg operation
 3554 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3555 %{
 3556     instruction_count(3);
 3557     dst    : S4(write);
 3558     src1   : S3(read);
 3559     src2   : S3(read);
 3560     DECODE : S0(3);     // any 3 decoders
 3561     FPU    : S3(2);
 3562 %}
 3563 
 3564 // Float reg-reg operation
 3565 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3566 %{
 3567     instruction_count(4);
 3568     dst    : S4(write);
 3569     src1   : S3(read);
 3570     src2   : S3(read);
 3571     src3   : S3(read);
 3572     DECODE : S0(4);     // any 3 decoders
 3573     FPU    : S3(2);
 3574 %}
 3575 
 3576 // Float reg-reg operation
 3577 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3578 %{
 3579     instruction_count(4);
 3580     dst    : S4(write);
 3581     src1   : S3(read);
 3582     src2   : S3(read);
 3583     src3   : S3(read);
 3584     DECODE : S1(3);     // any 3 decoders
 3585     D0     : S0;        // Big decoder only
 3586     FPU    : S3(2);
 3587     MEM    : S3;
 3588 %}
 3589 
 3590 // Float reg-mem operation
 3591 pipe_class fpu_reg_mem(regD dst, memory mem)
 3592 %{
 3593     instruction_count(2);
 3594     dst    : S5(write);
 3595     mem    : S3(read);
 3596     D0     : S0;        // big decoder only
 3597     DECODE : S1;        // any decoder for FPU POP
 3598     FPU    : S4;
 3599     MEM    : S3;        // any mem
 3600 %}
 3601 
 3602 // Float reg-mem operation
 3603 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3604 %{
 3605     instruction_count(3);
 3606     dst    : S5(write);
 3607     src1   : S3(read);
 3608     mem    : S3(read);
 3609     D0     : S0;        // big decoder only
 3610     DECODE : S1(2);     // any decoder for FPU POP
 3611     FPU    : S4;
 3612     MEM    : S3;        // any mem
 3613 %}
 3614 
 3615 // Float mem-reg operation
 3616 pipe_class fpu_mem_reg(memory mem, regD src)
 3617 %{
 3618     instruction_count(2);
 3619     src    : S5(read);
 3620     mem    : S3(read);
 3621     DECODE : S0;        // any decoder for FPU PUSH
 3622     D0     : S1;        // big decoder only
 3623     FPU    : S4;
 3624     MEM    : S3;        // any mem
 3625 %}
 3626 
 3627 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3628 %{
 3629     instruction_count(3);
 3630     src1   : S3(read);
 3631     src2   : S3(read);
 3632     mem    : S3(read);
 3633     DECODE : S0(2);     // any decoder for FPU PUSH
 3634     D0     : S1;        // big decoder only
 3635     FPU    : S4;
 3636     MEM    : S3;        // any mem
 3637 %}
 3638 
 3639 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3640 %{
 3641     instruction_count(3);
 3642     src1   : S3(read);
 3643     src2   : S3(read);
 3644     mem    : S4(read);
 3645     DECODE : S0;        // any decoder for FPU PUSH
 3646     D0     : S0(2);     // big decoder only
 3647     FPU    : S4;
 3648     MEM    : S3(2);     // any mem
 3649 %}
 3650 
 3651 pipe_class fpu_mem_mem(memory dst, memory src1)
 3652 %{
 3653     instruction_count(2);
 3654     src1   : S3(read);
 3655     dst    : S4(read);
 3656     D0     : S0(2);     // big decoder only
 3657     MEM    : S3(2);     // any mem
 3658 %}
 3659 
 3660 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3661 %{
 3662     instruction_count(3);
 3663     src1   : S3(read);
 3664     src2   : S3(read);
 3665     dst    : S4(read);
 3666     D0     : S0(3);     // big decoder only
 3667     FPU    : S4;
 3668     MEM    : S3(3);     // any mem
 3669 %}
 3670 
 3671 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3672 %{
 3673     instruction_count(3);
 3674     src1   : S4(read);
 3675     mem    : S4(read);
 3676     DECODE : S0;        // any decoder for FPU PUSH
 3677     D0     : S0(2);     // big decoder only
 3678     FPU    : S4;
 3679     MEM    : S3(2);     // any mem
 3680 %}
 3681 
 3682 // Float load constant
 3683 pipe_class fpu_reg_con(regD dst)
 3684 %{
 3685     instruction_count(2);
 3686     dst    : S5(write);
 3687     D0     : S0;        // big decoder only for the load
 3688     DECODE : S1;        // any decoder for FPU POP
 3689     FPU    : S4;
 3690     MEM    : S3;        // any mem
 3691 %}
 3692 
 3693 // Float load constant
 3694 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3695 %{
 3696     instruction_count(3);
 3697     dst    : S5(write);
 3698     src    : S3(read);
 3699     D0     : S0;        // big decoder only for the load
 3700     DECODE : S1(2);     // any decoder for FPU POP
 3701     FPU    : S4;
 3702     MEM    : S3;        // any mem
 3703 %}
 3704 
 3705 // UnConditional branch
 3706 pipe_class pipe_jmp(label labl)
 3707 %{
 3708     single_instruction;
 3709     BR   : S3;
 3710 %}
 3711 
 3712 // Conditional branch
 3713 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3714 %{
 3715     single_instruction;
 3716     cr    : S1(read);
 3717     BR    : S3;
 3718 %}
 3719 
 3720 // Allocation idiom
 3721 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3722 %{
 3723     instruction_count(1); force_serialization;
 3724     fixed_latency(6);
 3725     heap_ptr : S3(read);
 3726     DECODE   : S0(3);
 3727     D0       : S2;
 3728     MEM      : S3;
 3729     ALU      : S3(2);
 3730     dst      : S5(write);
 3731     BR       : S5;
 3732 %}
 3733 
 3734 // Generic big/slow expanded idiom
 3735 pipe_class pipe_slow()
 3736 %{
 3737     instruction_count(10); multiple_bundles; force_serialization;
 3738     fixed_latency(100);
 3739     D0  : S0(2);
 3740     MEM : S3(2);
 3741 %}
 3742 
 3743 // The real do-nothing guy
 3744 pipe_class empty()
 3745 %{
 3746     instruction_count(0);
 3747 %}
 3748 
 3749 // Define the class for the Nop node
 3750 define
 3751 %{
 3752    MachNop = empty;
 3753 %}
 3754 
 3755 %}
 3756 
 3757 //----------INSTRUCTIONS-------------------------------------------------------
 3758 //
 3759 // match      -- States which machine-independent subtree may be replaced
 3760 //               by this instruction.
 3761 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3762 //               selection to identify a minimum cost tree of machine
 3763 //               instructions that matches a tree of machine-independent
 3764 //               instructions.
 3765 // format     -- A string providing the disassembly for this instruction.
 3766 //               The value of an instruction's operand may be inserted
 3767 //               by referring to it with a '$' prefix.
 3768 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3769 //               to within an encode class as $primary, $secondary, and $tertiary
 3770 //               rrspectively.  The primary opcode is commonly used to
 3771 //               indicate the type of machine instruction, while secondary
 3772 //               and tertiary are often used for prefix options or addressing
 3773 //               modes.
 3774 // ins_encode -- A list of encode classes with parameters. The encode class
 3775 //               name must have been defined in an 'enc_class' specification
 3776 //               in the encode section of the architecture description.
 3777 
 3778 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3779 // Load Float
 3780 instruct MoveF2VL(vlRegF dst, regF src) %{
 3781   match(Set dst src);
 3782   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3783   ins_encode %{
 3784     ShouldNotReachHere();
 3785   %}
 3786   ins_pipe( fpu_reg_reg );
 3787 %}
 3788 
 3789 // Load Float
 3790 instruct MoveF2LEG(legRegF dst, regF src) %{
 3791   match(Set dst src);
 3792   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3793   ins_encode %{
 3794     ShouldNotReachHere();
 3795   %}
 3796   ins_pipe( fpu_reg_reg );
 3797 %}
 3798 
 3799 // Load Float
 3800 instruct MoveVL2F(regF dst, vlRegF src) %{
 3801   match(Set dst src);
 3802   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3803   ins_encode %{
 3804     ShouldNotReachHere();
 3805   %}
 3806   ins_pipe( fpu_reg_reg );
 3807 %}
 3808 
 3809 // Load Float
 3810 instruct MoveLEG2F(regF dst, legRegF src) %{
 3811   match(Set dst src);
 3812   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3813   ins_encode %{
 3814     ShouldNotReachHere();
 3815   %}
 3816   ins_pipe( fpu_reg_reg );
 3817 %}
 3818 
 3819 // Load Double
 3820 instruct MoveD2VL(vlRegD dst, regD src) %{
 3821   match(Set dst src);
 3822   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3823   ins_encode %{
 3824     ShouldNotReachHere();
 3825   %}
 3826   ins_pipe( fpu_reg_reg );
 3827 %}
 3828 
 3829 // Load Double
 3830 instruct MoveD2LEG(legRegD dst, regD src) %{
 3831   match(Set dst src);
 3832   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3833   ins_encode %{
 3834     ShouldNotReachHere();
 3835   %}
 3836   ins_pipe( fpu_reg_reg );
 3837 %}
 3838 
 3839 // Load Double
 3840 instruct MoveVL2D(regD dst, vlRegD src) %{
 3841   match(Set dst src);
 3842   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3843   ins_encode %{
 3844     ShouldNotReachHere();
 3845   %}
 3846   ins_pipe( fpu_reg_reg );
 3847 %}
 3848 
 3849 // Load Double
 3850 instruct MoveLEG2D(regD dst, legRegD src) %{
 3851   match(Set dst src);
 3852   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3853   ins_encode %{
 3854     ShouldNotReachHere();
 3855   %}
 3856   ins_pipe( fpu_reg_reg );
 3857 %}
 3858 
 3859 //----------Load/Store/Move Instructions---------------------------------------
 3860 //----------Load Instructions--------------------------------------------------
 3861 
 3862 // Load Byte (8 bit signed)
 3863 instruct loadB(rRegI dst, memory mem)
 3864 %{
 3865   match(Set dst (LoadB mem));
 3866 
 3867   ins_cost(125);
 3868   format %{ "movsbl  $dst, $mem\t# byte" %}
 3869 
 3870   ins_encode %{
 3871     __ movsbl($dst$$Register, $mem$$Address);
 3872   %}
 3873 
 3874   ins_pipe(ialu_reg_mem);
 3875 %}
 3876 
 3877 // Load Byte (8 bit signed) into Long Register
 3878 instruct loadB2L(rRegL dst, memory mem)
 3879 %{
 3880   match(Set dst (ConvI2L (LoadB mem)));
 3881 
 3882   ins_cost(125);
 3883   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 3884 
 3885   ins_encode %{
 3886     __ movsbq($dst$$Register, $mem$$Address);
 3887   %}
 3888 
 3889   ins_pipe(ialu_reg_mem);
 3890 %}
 3891 
 3892 // Load Unsigned Byte (8 bit UNsigned)
 3893 instruct loadUB(rRegI dst, memory mem)
 3894 %{
 3895   match(Set dst (LoadUB mem));
 3896 
 3897   ins_cost(125);
 3898   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 3899 
 3900   ins_encode %{
 3901     __ movzbl($dst$$Register, $mem$$Address);
 3902   %}
 3903 
 3904   ins_pipe(ialu_reg_mem);
 3905 %}
 3906 
 3907 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 3908 instruct loadUB2L(rRegL dst, memory mem)
 3909 %{
 3910   match(Set dst (ConvI2L (LoadUB mem)));
 3911 
 3912   ins_cost(125);
 3913   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 3914 
 3915   ins_encode %{
 3916     __ movzbq($dst$$Register, $mem$$Address);
 3917   %}
 3918 
 3919   ins_pipe(ialu_reg_mem);
 3920 %}
 3921 
 3922 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 3923 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 3924   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 3925   effect(KILL cr);
 3926 
 3927   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 3928             "andl    $dst, right_n_bits($mask, 8)" %}
 3929   ins_encode %{
 3930     Register Rdst = $dst$$Register;
 3931     __ movzbq(Rdst, $mem$$Address);
 3932     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 3933   %}
 3934   ins_pipe(ialu_reg_mem);
 3935 %}
 3936 
 3937 // Load Short (16 bit signed)
 3938 instruct loadS(rRegI dst, memory mem)
 3939 %{
 3940   match(Set dst (LoadS mem));
 3941 
 3942   ins_cost(125);
 3943   format %{ "movswl $dst, $mem\t# short" %}
 3944 
 3945   ins_encode %{
 3946     __ movswl($dst$$Register, $mem$$Address);
 3947   %}
 3948 
 3949   ins_pipe(ialu_reg_mem);
 3950 %}
 3951 
 3952 // Load Short (16 bit signed) to Byte (8 bit signed)
 3953 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 3954   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 3955 
 3956   ins_cost(125);
 3957   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 3958   ins_encode %{
 3959     __ movsbl($dst$$Register, $mem$$Address);
 3960   %}
 3961   ins_pipe(ialu_reg_mem);
 3962 %}
 3963 
 3964 // Load Short (16 bit signed) into Long Register
 3965 instruct loadS2L(rRegL dst, memory mem)
 3966 %{
 3967   match(Set dst (ConvI2L (LoadS mem)));
 3968 
 3969   ins_cost(125);
 3970   format %{ "movswq $dst, $mem\t# short -> long" %}
 3971 
 3972   ins_encode %{
 3973     __ movswq($dst$$Register, $mem$$Address);
 3974   %}
 3975 
 3976   ins_pipe(ialu_reg_mem);
 3977 %}
 3978 
 3979 // Load Unsigned Short/Char (16 bit UNsigned)
 3980 instruct loadUS(rRegI dst, memory mem)
 3981 %{
 3982   match(Set dst (LoadUS mem));
 3983 
 3984   ins_cost(125);
 3985   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 3986 
 3987   ins_encode %{
 3988     __ movzwl($dst$$Register, $mem$$Address);
 3989   %}
 3990 
 3991   ins_pipe(ialu_reg_mem);
 3992 %}
 3993 
 3994 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 3995 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 3996   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 3997 
 3998   ins_cost(125);
 3999   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4000   ins_encode %{
 4001     __ movsbl($dst$$Register, $mem$$Address);
 4002   %}
 4003   ins_pipe(ialu_reg_mem);
 4004 %}
 4005 
 4006 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4007 instruct loadUS2L(rRegL dst, memory mem)
 4008 %{
 4009   match(Set dst (ConvI2L (LoadUS mem)));
 4010 
 4011   ins_cost(125);
 4012   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4013 
 4014   ins_encode %{
 4015     __ movzwq($dst$$Register, $mem$$Address);
 4016   %}
 4017 
 4018   ins_pipe(ialu_reg_mem);
 4019 %}
 4020 
 4021 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4022 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4023   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4024 
 4025   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4026   ins_encode %{
 4027     __ movzbq($dst$$Register, $mem$$Address);
 4028   %}
 4029   ins_pipe(ialu_reg_mem);
 4030 %}
 4031 
 4032 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4033 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4034   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4035   effect(KILL cr);
 4036 
 4037   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4038             "andl    $dst, right_n_bits($mask, 16)" %}
 4039   ins_encode %{
 4040     Register Rdst = $dst$$Register;
 4041     __ movzwq(Rdst, $mem$$Address);
 4042     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4043   %}
 4044   ins_pipe(ialu_reg_mem);
 4045 %}
 4046 
 4047 // Load Integer
 4048 instruct loadI(rRegI dst, memory mem)
 4049 %{
 4050   match(Set dst (LoadI mem));
 4051 
 4052   ins_cost(125);
 4053   format %{ "movl    $dst, $mem\t# int" %}
 4054 
 4055   ins_encode %{
 4056     __ movl($dst$$Register, $mem$$Address);
 4057   %}
 4058 
 4059   ins_pipe(ialu_reg_mem);
 4060 %}
 4061 
 4062 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4063 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4064   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4065 
 4066   ins_cost(125);
 4067   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4068   ins_encode %{
 4069     __ movsbl($dst$$Register, $mem$$Address);
 4070   %}
 4071   ins_pipe(ialu_reg_mem);
 4072 %}
 4073 
 4074 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4075 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4076   match(Set dst (AndI (LoadI mem) mask));
 4077 
 4078   ins_cost(125);
 4079   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4080   ins_encode %{
 4081     __ movzbl($dst$$Register, $mem$$Address);
 4082   %}
 4083   ins_pipe(ialu_reg_mem);
 4084 %}
 4085 
 4086 // Load Integer (32 bit signed) to Short (16 bit signed)
 4087 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4088   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4089 
 4090   ins_cost(125);
 4091   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4092   ins_encode %{
 4093     __ movswl($dst$$Register, $mem$$Address);
 4094   %}
 4095   ins_pipe(ialu_reg_mem);
 4096 %}
 4097 
 4098 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4099 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4100   match(Set dst (AndI (LoadI mem) mask));
 4101 
 4102   ins_cost(125);
 4103   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4104   ins_encode %{
 4105     __ movzwl($dst$$Register, $mem$$Address);
 4106   %}
 4107   ins_pipe(ialu_reg_mem);
 4108 %}
 4109 
 4110 // Load Integer into Long Register
 4111 instruct loadI2L(rRegL dst, memory mem)
 4112 %{
 4113   match(Set dst (ConvI2L (LoadI mem)));
 4114 
 4115   ins_cost(125);
 4116   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4117 
 4118   ins_encode %{
 4119     __ movslq($dst$$Register, $mem$$Address);
 4120   %}
 4121 
 4122   ins_pipe(ialu_reg_mem);
 4123 %}
 4124 
 4125 // Load Integer with mask 0xFF into Long Register
 4126 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4127   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4128 
 4129   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4130   ins_encode %{
 4131     __ movzbq($dst$$Register, $mem$$Address);
 4132   %}
 4133   ins_pipe(ialu_reg_mem);
 4134 %}
 4135 
 4136 // Load Integer with mask 0xFFFF into Long Register
 4137 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4138   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4139 
 4140   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4141   ins_encode %{
 4142     __ movzwq($dst$$Register, $mem$$Address);
 4143   %}
 4144   ins_pipe(ialu_reg_mem);
 4145 %}
 4146 
 4147 // Load Integer with a 31-bit mask into Long Register
 4148 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4149   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4150   effect(KILL cr);
 4151 
 4152   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4153             "andl    $dst, $mask" %}
 4154   ins_encode %{
 4155     Register Rdst = $dst$$Register;
 4156     __ movl(Rdst, $mem$$Address);
 4157     __ andl(Rdst, $mask$$constant);
 4158   %}
 4159   ins_pipe(ialu_reg_mem);
 4160 %}
 4161 
 4162 // Load Unsigned Integer into Long Register
 4163 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4164 %{
 4165   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4166 
 4167   ins_cost(125);
 4168   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4169 
 4170   ins_encode %{
 4171     __ movl($dst$$Register, $mem$$Address);
 4172   %}
 4173 
 4174   ins_pipe(ialu_reg_mem);
 4175 %}
 4176 
 4177 // Load Long
 4178 instruct loadL(rRegL dst, memory mem)
 4179 %{
 4180   match(Set dst (LoadL mem));
 4181 
 4182   ins_cost(125);
 4183   format %{ "movq    $dst, $mem\t# long" %}
 4184 
 4185   ins_encode %{
 4186     __ movq($dst$$Register, $mem$$Address);
 4187   %}
 4188 
 4189   ins_pipe(ialu_reg_mem); // XXX
 4190 %}
 4191 
 4192 // Load Range
 4193 instruct loadRange(rRegI dst, memory mem)
 4194 %{
 4195   match(Set dst (LoadRange mem));
 4196 
 4197   ins_cost(125); // XXX
 4198   format %{ "movl    $dst, $mem\t# range" %}
 4199   ins_encode %{
 4200     __ movl($dst$$Register, $mem$$Address);
 4201   %}
 4202   ins_pipe(ialu_reg_mem);
 4203 %}
 4204 
 4205 // Load Pointer
 4206 instruct loadP(rRegP dst, memory mem)
 4207 %{
 4208   match(Set dst (LoadP mem));
 4209   predicate(n->as_Load()->barrier_data() == 0);
 4210 
 4211   ins_cost(125); // XXX
 4212   format %{ "movq    $dst, $mem\t# ptr" %}
 4213   ins_encode %{
 4214     __ movq($dst$$Register, $mem$$Address);
 4215   %}
 4216   ins_pipe(ialu_reg_mem); // XXX
 4217 %}
 4218 
 4219 // Load Compressed Pointer
 4220 instruct loadN(rRegN dst, memory mem)
 4221 %{
 4222    match(Set dst (LoadN mem));
 4223 
 4224    ins_cost(125); // XXX
 4225    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4226    ins_encode %{
 4227      __ movl($dst$$Register, $mem$$Address);
 4228    %}
 4229    ins_pipe(ialu_reg_mem); // XXX
 4230 %}
 4231 
 4232 
 4233 // Load Klass Pointer
 4234 instruct loadKlass(rRegP dst, memory mem)
 4235 %{
 4236   match(Set dst (LoadKlass mem));
 4237 
 4238   ins_cost(125); // XXX
 4239   format %{ "movq    $dst, $mem\t# class" %}
 4240   ins_encode %{
 4241     __ movq($dst$$Register, $mem$$Address);
 4242   %}
 4243   ins_pipe(ialu_reg_mem); // XXX
 4244 %}
 4245 
 4246 // Load narrow Klass Pointer
 4247 instruct loadNKlass(rRegN dst, memory mem)
 4248 %{
 4249   match(Set dst (LoadNKlass mem));
 4250 
 4251   ins_cost(125); // XXX
 4252   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4253   ins_encode %{
 4254     __ movl($dst$$Register, $mem$$Address);
 4255   %}
 4256   ins_pipe(ialu_reg_mem); // XXX
 4257 %}
 4258 
 4259 // Load Float
 4260 instruct loadF(regF dst, memory mem)
 4261 %{
 4262   match(Set dst (LoadF mem));
 4263 
 4264   ins_cost(145); // XXX
 4265   format %{ "movss   $dst, $mem\t# float" %}
 4266   ins_encode %{
 4267     __ movflt($dst$$XMMRegister, $mem$$Address);
 4268   %}
 4269   ins_pipe(pipe_slow); // XXX
 4270 %}
 4271 
 4272 // Load Double
 4273 instruct loadD_partial(regD dst, memory mem)
 4274 %{
 4275   predicate(!UseXmmLoadAndClearUpper);
 4276   match(Set dst (LoadD mem));
 4277 
 4278   ins_cost(145); // XXX
 4279   format %{ "movlpd  $dst, $mem\t# double" %}
 4280   ins_encode %{
 4281     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4282   %}
 4283   ins_pipe(pipe_slow); // XXX
 4284 %}
 4285 
 4286 instruct loadD(regD dst, memory mem)
 4287 %{
 4288   predicate(UseXmmLoadAndClearUpper);
 4289   match(Set dst (LoadD mem));
 4290 
 4291   ins_cost(145); // XXX
 4292   format %{ "movsd   $dst, $mem\t# double" %}
 4293   ins_encode %{
 4294     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4295   %}
 4296   ins_pipe(pipe_slow); // XXX
 4297 %}
 4298 
 4299 // max = java.lang.Math.max(float a, float b)
 4300 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4301   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4302   match(Set dst (MaxF a b));
 4303   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4304   format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4305   ins_encode %{
 4306     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4307   %}
 4308   ins_pipe( pipe_slow );
 4309 %}
 4310 
 4311 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4312   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4313   match(Set dst (MaxF a b));
 4314   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4315 
 4316   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 4317   ins_encode %{
 4318     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4319                     false /*min*/, true /*single*/);
 4320   %}
 4321   ins_pipe( pipe_slow );
 4322 %}
 4323 
 4324 // max = java.lang.Math.max(double a, double b)
 4325 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4326   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4327   match(Set dst (MaxD a b));
 4328   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4329   format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4330   ins_encode %{
 4331     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4332   %}
 4333   ins_pipe( pipe_slow );
 4334 %}
 4335 
 4336 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4337   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4338   match(Set dst (MaxD a b));
 4339   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4340 
 4341   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 4342   ins_encode %{
 4343     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4344                     false /*min*/, false /*single*/);
 4345   %}
 4346   ins_pipe( pipe_slow );
 4347 %}
 4348 
 4349 // min = java.lang.Math.min(float a, float b)
 4350 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4351   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4352   match(Set dst (MinF a b));
 4353   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4354   format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4355   ins_encode %{
 4356     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4357   %}
 4358   ins_pipe( pipe_slow );
 4359 %}
 4360 
 4361 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4362   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4363   match(Set dst (MinF a b));
 4364   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4365 
 4366   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 4367   ins_encode %{
 4368     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4369                     true /*min*/, true /*single*/);
 4370   %}
 4371   ins_pipe( pipe_slow );
 4372 %}
 4373 
 4374 // min = java.lang.Math.min(double a, double b)
 4375 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4376   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4377   match(Set dst (MinD a b));
 4378   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4379     format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4380   ins_encode %{
 4381     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4382   %}
 4383   ins_pipe( pipe_slow );
 4384 %}
 4385 
 4386 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4387   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4388   match(Set dst (MinD a b));
 4389   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4390 
 4391   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 4392   ins_encode %{
 4393     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4394                     true /*min*/, false /*single*/);
 4395   %}
 4396   ins_pipe( pipe_slow );
 4397 %}
 4398 
 4399 // Load Effective Address
 4400 instruct leaP8(rRegP dst, indOffset8 mem)
 4401 %{
 4402   match(Set dst mem);
 4403 
 4404   ins_cost(110); // XXX
 4405   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4406   ins_encode %{
 4407     __ leaq($dst$$Register, $mem$$Address);
 4408   %}
 4409   ins_pipe(ialu_reg_reg_fat);
 4410 %}
 4411 
 4412 instruct leaP32(rRegP dst, indOffset32 mem)
 4413 %{
 4414   match(Set dst mem);
 4415 
 4416   ins_cost(110);
 4417   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4418   ins_encode %{
 4419     __ leaq($dst$$Register, $mem$$Address);
 4420   %}
 4421   ins_pipe(ialu_reg_reg_fat);
 4422 %}
 4423 
 4424 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4425 %{
 4426   match(Set dst mem);
 4427 
 4428   ins_cost(110);
 4429   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4430   ins_encode %{
 4431     __ leaq($dst$$Register, $mem$$Address);
 4432   %}
 4433   ins_pipe(ialu_reg_reg_fat);
 4434 %}
 4435 
 4436 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4437 %{
 4438   match(Set dst mem);
 4439 
 4440   ins_cost(110);
 4441   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4442   ins_encode %{
 4443     __ leaq($dst$$Register, $mem$$Address);
 4444   %}
 4445   ins_pipe(ialu_reg_reg_fat);
 4446 %}
 4447 
 4448 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4449 %{
 4450   match(Set dst mem);
 4451 
 4452   ins_cost(110);
 4453   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4454   ins_encode %{
 4455     __ leaq($dst$$Register, $mem$$Address);
 4456   %}
 4457   ins_pipe(ialu_reg_reg_fat);
 4458 %}
 4459 
 4460 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4461 %{
 4462   match(Set dst mem);
 4463 
 4464   ins_cost(110);
 4465   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4466   ins_encode %{
 4467     __ leaq($dst$$Register, $mem$$Address);
 4468   %}
 4469   ins_pipe(ialu_reg_reg_fat);
 4470 %}
 4471 
 4472 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4473 %{
 4474   match(Set dst mem);
 4475 
 4476   ins_cost(110);
 4477   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4478   ins_encode %{
 4479     __ leaq($dst$$Register, $mem$$Address);
 4480   %}
 4481   ins_pipe(ialu_reg_reg_fat);
 4482 %}
 4483 
 4484 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4485 %{
 4486   match(Set dst mem);
 4487 
 4488   ins_cost(110);
 4489   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4490   ins_encode %{
 4491     __ leaq($dst$$Register, $mem$$Address);
 4492   %}
 4493   ins_pipe(ialu_reg_reg_fat);
 4494 %}
 4495 
 4496 // Load Effective Address which uses Narrow (32-bits) oop
 4497 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4498 %{
 4499   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4500   match(Set dst mem);
 4501 
 4502   ins_cost(110);
 4503   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4504   ins_encode %{
 4505     __ leaq($dst$$Register, $mem$$Address);
 4506   %}
 4507   ins_pipe(ialu_reg_reg_fat);
 4508 %}
 4509 
 4510 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4511 %{
 4512   predicate(CompressedOops::shift() == 0);
 4513   match(Set dst mem);
 4514 
 4515   ins_cost(110); // XXX
 4516   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4517   ins_encode %{
 4518     __ leaq($dst$$Register, $mem$$Address);
 4519   %}
 4520   ins_pipe(ialu_reg_reg_fat);
 4521 %}
 4522 
 4523 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4524 %{
 4525   predicate(CompressedOops::shift() == 0);
 4526   match(Set dst mem);
 4527 
 4528   ins_cost(110);
 4529   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4530   ins_encode %{
 4531     __ leaq($dst$$Register, $mem$$Address);
 4532   %}
 4533   ins_pipe(ialu_reg_reg_fat);
 4534 %}
 4535 
 4536 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4537 %{
 4538   predicate(CompressedOops::shift() == 0);
 4539   match(Set dst mem);
 4540 
 4541   ins_cost(110);
 4542   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4543   ins_encode %{
 4544     __ leaq($dst$$Register, $mem$$Address);
 4545   %}
 4546   ins_pipe(ialu_reg_reg_fat);
 4547 %}
 4548 
 4549 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4550 %{
 4551   predicate(CompressedOops::shift() == 0);
 4552   match(Set dst mem);
 4553 
 4554   ins_cost(110);
 4555   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4556   ins_encode %{
 4557     __ leaq($dst$$Register, $mem$$Address);
 4558   %}
 4559   ins_pipe(ialu_reg_reg_fat);
 4560 %}
 4561 
 4562 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4563 %{
 4564   predicate(CompressedOops::shift() == 0);
 4565   match(Set dst mem);
 4566 
 4567   ins_cost(110);
 4568   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4569   ins_encode %{
 4570     __ leaq($dst$$Register, $mem$$Address);
 4571   %}
 4572   ins_pipe(ialu_reg_reg_fat);
 4573 %}
 4574 
 4575 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4576 %{
 4577   predicate(CompressedOops::shift() == 0);
 4578   match(Set dst mem);
 4579 
 4580   ins_cost(110);
 4581   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4582   ins_encode %{
 4583     __ leaq($dst$$Register, $mem$$Address);
 4584   %}
 4585   ins_pipe(ialu_reg_reg_fat);
 4586 %}
 4587 
 4588 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4589 %{
 4590   predicate(CompressedOops::shift() == 0);
 4591   match(Set dst mem);
 4592 
 4593   ins_cost(110);
 4594   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4595   ins_encode %{
 4596     __ leaq($dst$$Register, $mem$$Address);
 4597   %}
 4598   ins_pipe(ialu_reg_reg_fat);
 4599 %}
 4600 
 4601 instruct loadConI(rRegI dst, immI src)
 4602 %{
 4603   match(Set dst src);
 4604 
 4605   format %{ "movl    $dst, $src\t# int" %}
 4606   ins_encode %{
 4607     __ movl($dst$$Register, $src$$constant);
 4608   %}
 4609   ins_pipe(ialu_reg_fat); // XXX
 4610 %}
 4611 
 4612 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4613 %{
 4614   match(Set dst src);
 4615   effect(KILL cr);
 4616 
 4617   ins_cost(50);
 4618   format %{ "xorl    $dst, $dst\t# int" %}
 4619   ins_encode %{
 4620     __ xorl($dst$$Register, $dst$$Register);
 4621   %}
 4622   ins_pipe(ialu_reg);
 4623 %}
 4624 
 4625 instruct loadConL(rRegL dst, immL src)
 4626 %{
 4627   match(Set dst src);
 4628 
 4629   ins_cost(150);
 4630   format %{ "movq    $dst, $src\t# long" %}
 4631   ins_encode %{
 4632     __ mov64($dst$$Register, $src$$constant);
 4633   %}
 4634   ins_pipe(ialu_reg);
 4635 %}
 4636 
 4637 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4638 %{
 4639   match(Set dst src);
 4640   effect(KILL cr);
 4641 
 4642   ins_cost(50);
 4643   format %{ "xorl    $dst, $dst\t# long" %}
 4644   ins_encode %{
 4645     __ xorl($dst$$Register, $dst$$Register);
 4646   %}
 4647   ins_pipe(ialu_reg); // XXX
 4648 %}
 4649 
 4650 instruct loadConUL32(rRegL dst, immUL32 src)
 4651 %{
 4652   match(Set dst src);
 4653 
 4654   ins_cost(60);
 4655   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4656   ins_encode %{
 4657     __ movl($dst$$Register, $src$$constant);
 4658   %}
 4659   ins_pipe(ialu_reg);
 4660 %}
 4661 
 4662 instruct loadConL32(rRegL dst, immL32 src)
 4663 %{
 4664   match(Set dst src);
 4665 
 4666   ins_cost(70);
 4667   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4668   ins_encode %{
 4669     __ movq($dst$$Register, $src$$constant);
 4670   %}
 4671   ins_pipe(ialu_reg);
 4672 %}
 4673 
 4674 instruct loadConP(rRegP dst, immP con) %{
 4675   match(Set dst con);
 4676 
 4677   format %{ "movq    $dst, $con\t# ptr" %}
 4678   ins_encode %{
 4679     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4680   %}
 4681   ins_pipe(ialu_reg_fat); // XXX
 4682 %}
 4683 
 4684 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4685 %{
 4686   match(Set dst src);
 4687   effect(KILL cr);
 4688 
 4689   ins_cost(50);
 4690   format %{ "xorl    $dst, $dst\t# ptr" %}
 4691   ins_encode %{
 4692     __ xorl($dst$$Register, $dst$$Register);
 4693   %}
 4694   ins_pipe(ialu_reg);
 4695 %}
 4696 
 4697 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4698 %{
 4699   match(Set dst src);
 4700   effect(KILL cr);
 4701 
 4702   ins_cost(60);
 4703   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4704   ins_encode %{
 4705     __ movl($dst$$Register, $src$$constant);
 4706   %}
 4707   ins_pipe(ialu_reg);
 4708 %}
 4709 
 4710 instruct loadConF(regF dst, immF con) %{
 4711   match(Set dst con);
 4712   ins_cost(125);
 4713   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4714   ins_encode %{
 4715     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4716   %}
 4717   ins_pipe(pipe_slow);
 4718 %}
 4719 
 4720 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4721   match(Set dst src);
 4722   effect(KILL cr);
 4723   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4724   ins_encode %{
 4725     __ xorq($dst$$Register, $dst$$Register);
 4726   %}
 4727   ins_pipe(ialu_reg);
 4728 %}
 4729 
 4730 instruct loadConN(rRegN dst, immN src) %{
 4731   match(Set dst src);
 4732 
 4733   ins_cost(125);
 4734   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4735   ins_encode %{
 4736     address con = (address)$src$$constant;
 4737     if (con == nullptr) {
 4738       ShouldNotReachHere();
 4739     } else {
 4740       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4741     }
 4742   %}
 4743   ins_pipe(ialu_reg_fat); // XXX
 4744 %}
 4745 
 4746 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4747   match(Set dst src);
 4748 
 4749   ins_cost(125);
 4750   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4751   ins_encode %{
 4752     address con = (address)$src$$constant;
 4753     if (con == nullptr) {
 4754       ShouldNotReachHere();
 4755     } else {
 4756       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4757     }
 4758   %}
 4759   ins_pipe(ialu_reg_fat); // XXX
 4760 %}
 4761 
 4762 instruct loadConF0(regF dst, immF0 src)
 4763 %{
 4764   match(Set dst src);
 4765   ins_cost(100);
 4766 
 4767   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4768   ins_encode %{
 4769     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4770   %}
 4771   ins_pipe(pipe_slow);
 4772 %}
 4773 
 4774 // Use the same format since predicate() can not be used here.
 4775 instruct loadConD(regD dst, immD con) %{
 4776   match(Set dst con);
 4777   ins_cost(125);
 4778   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4779   ins_encode %{
 4780     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4781   %}
 4782   ins_pipe(pipe_slow);
 4783 %}
 4784 
 4785 instruct loadConD0(regD dst, immD0 src)
 4786 %{
 4787   match(Set dst src);
 4788   ins_cost(100);
 4789 
 4790   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4791   ins_encode %{
 4792     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4793   %}
 4794   ins_pipe(pipe_slow);
 4795 %}
 4796 
 4797 instruct loadSSI(rRegI dst, stackSlotI src)
 4798 %{
 4799   match(Set dst src);
 4800 
 4801   ins_cost(125);
 4802   format %{ "movl    $dst, $src\t# int stk" %}
 4803   ins_encode %{
 4804     __ movl($dst$$Register, $src$$Address);
 4805   %}
 4806   ins_pipe(ialu_reg_mem);
 4807 %}
 4808 
 4809 instruct loadSSL(rRegL dst, stackSlotL src)
 4810 %{
 4811   match(Set dst src);
 4812 
 4813   ins_cost(125);
 4814   format %{ "movq    $dst, $src\t# long stk" %}
 4815   ins_encode %{
 4816     __ movq($dst$$Register, $src$$Address);
 4817   %}
 4818   ins_pipe(ialu_reg_mem);
 4819 %}
 4820 
 4821 instruct loadSSP(rRegP dst, stackSlotP src)
 4822 %{
 4823   match(Set dst src);
 4824 
 4825   ins_cost(125);
 4826   format %{ "movq    $dst, $src\t# ptr stk" %}
 4827   ins_encode %{
 4828     __ movq($dst$$Register, $src$$Address);
 4829   %}
 4830   ins_pipe(ialu_reg_mem);
 4831 %}
 4832 
 4833 instruct loadSSF(regF dst, stackSlotF src)
 4834 %{
 4835   match(Set dst src);
 4836 
 4837   ins_cost(125);
 4838   format %{ "movss   $dst, $src\t# float stk" %}
 4839   ins_encode %{
 4840     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 4841   %}
 4842   ins_pipe(pipe_slow); // XXX
 4843 %}
 4844 
 4845 // Use the same format since predicate() can not be used here.
 4846 instruct loadSSD(regD dst, stackSlotD src)
 4847 %{
 4848   match(Set dst src);
 4849 
 4850   ins_cost(125);
 4851   format %{ "movsd   $dst, $src\t# double stk" %}
 4852   ins_encode  %{
 4853     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 4854   %}
 4855   ins_pipe(pipe_slow); // XXX
 4856 %}
 4857 
 4858 // Prefetch instructions for allocation.
 4859 // Must be safe to execute with invalid address (cannot fault).
 4860 
 4861 instruct prefetchAlloc( memory mem ) %{
 4862   predicate(AllocatePrefetchInstr==3);
 4863   match(PrefetchAllocation mem);
 4864   ins_cost(125);
 4865 
 4866   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 4867   ins_encode %{
 4868     __ prefetchw($mem$$Address);
 4869   %}
 4870   ins_pipe(ialu_mem);
 4871 %}
 4872 
 4873 instruct prefetchAllocNTA( memory mem ) %{
 4874   predicate(AllocatePrefetchInstr==0);
 4875   match(PrefetchAllocation mem);
 4876   ins_cost(125);
 4877 
 4878   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 4879   ins_encode %{
 4880     __ prefetchnta($mem$$Address);
 4881   %}
 4882   ins_pipe(ialu_mem);
 4883 %}
 4884 
 4885 instruct prefetchAllocT0( memory mem ) %{
 4886   predicate(AllocatePrefetchInstr==1);
 4887   match(PrefetchAllocation mem);
 4888   ins_cost(125);
 4889 
 4890   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 4891   ins_encode %{
 4892     __ prefetcht0($mem$$Address);
 4893   %}
 4894   ins_pipe(ialu_mem);
 4895 %}
 4896 
 4897 instruct prefetchAllocT2( memory mem ) %{
 4898   predicate(AllocatePrefetchInstr==2);
 4899   match(PrefetchAllocation mem);
 4900   ins_cost(125);
 4901 
 4902   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 4903   ins_encode %{
 4904     __ prefetcht2($mem$$Address);
 4905   %}
 4906   ins_pipe(ialu_mem);
 4907 %}
 4908 
 4909 //----------Store Instructions-------------------------------------------------
 4910 
 4911 // Store Byte
 4912 instruct storeB(memory mem, rRegI src)
 4913 %{
 4914   match(Set mem (StoreB mem src));
 4915 
 4916   ins_cost(125); // XXX
 4917   format %{ "movb    $mem, $src\t# byte" %}
 4918   ins_encode %{
 4919     __ movb($mem$$Address, $src$$Register);
 4920   %}
 4921   ins_pipe(ialu_mem_reg);
 4922 %}
 4923 
 4924 // Store Char/Short
 4925 instruct storeC(memory mem, rRegI src)
 4926 %{
 4927   match(Set mem (StoreC mem src));
 4928 
 4929   ins_cost(125); // XXX
 4930   format %{ "movw    $mem, $src\t# char/short" %}
 4931   ins_encode %{
 4932     __ movw($mem$$Address, $src$$Register);
 4933   %}
 4934   ins_pipe(ialu_mem_reg);
 4935 %}
 4936 
 4937 // Store Integer
 4938 instruct storeI(memory mem, rRegI src)
 4939 %{
 4940   match(Set mem (StoreI mem src));
 4941 
 4942   ins_cost(125); // XXX
 4943   format %{ "movl    $mem, $src\t# int" %}
 4944   ins_encode %{
 4945     __ movl($mem$$Address, $src$$Register);
 4946   %}
 4947   ins_pipe(ialu_mem_reg);
 4948 %}
 4949 
 4950 // Store Long
 4951 instruct storeL(memory mem, rRegL src)
 4952 %{
 4953   match(Set mem (StoreL mem src));
 4954 
 4955   ins_cost(125); // XXX
 4956   format %{ "movq    $mem, $src\t# long" %}
 4957   ins_encode %{
 4958     __ movq($mem$$Address, $src$$Register);
 4959   %}
 4960   ins_pipe(ialu_mem_reg); // XXX
 4961 %}
 4962 
 4963 // Store Pointer
 4964 instruct storeP(memory mem, any_RegP src)
 4965 %{
 4966   predicate(n->as_Store()->barrier_data() == 0);
 4967   match(Set mem (StoreP mem src));
 4968 
 4969   ins_cost(125); // XXX
 4970   format %{ "movq    $mem, $src\t# ptr" %}
 4971   ins_encode %{
 4972     __ movq($mem$$Address, $src$$Register);
 4973   %}
 4974   ins_pipe(ialu_mem_reg);
 4975 %}
 4976 
 4977 instruct storeImmP0(memory mem, immP0 zero)
 4978 %{
 4979   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 4980   match(Set mem (StoreP mem zero));
 4981 
 4982   ins_cost(125); // XXX
 4983   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 4984   ins_encode %{
 4985     __ movq($mem$$Address, r12);
 4986   %}
 4987   ins_pipe(ialu_mem_reg);
 4988 %}
 4989 
 4990 // Store Null Pointer, mark word, or other simple pointer constant.
 4991 instruct storeImmP(memory mem, immP31 src)
 4992 %{
 4993   predicate(n->as_Store()->barrier_data() == 0);
 4994   match(Set mem (StoreP mem src));
 4995 
 4996   ins_cost(150); // XXX
 4997   format %{ "movq    $mem, $src\t# ptr" %}
 4998   ins_encode %{
 4999     __ movq($mem$$Address, $src$$constant);
 5000   %}
 5001   ins_pipe(ialu_mem_imm);
 5002 %}
 5003 
 5004 // Store Compressed Pointer
 5005 instruct storeN(memory mem, rRegN src)
 5006 %{
 5007   match(Set mem (StoreN mem src));
 5008 
 5009   ins_cost(125); // XXX
 5010   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5011   ins_encode %{
 5012     __ movl($mem$$Address, $src$$Register);
 5013   %}
 5014   ins_pipe(ialu_mem_reg);
 5015 %}
 5016 
 5017 instruct storeNKlass(memory mem, rRegN src)
 5018 %{
 5019   match(Set mem (StoreNKlass mem src));
 5020 
 5021   ins_cost(125); // XXX
 5022   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5023   ins_encode %{
 5024     __ movl($mem$$Address, $src$$Register);
 5025   %}
 5026   ins_pipe(ialu_mem_reg);
 5027 %}
 5028 
 5029 instruct storeImmN0(memory mem, immN0 zero)
 5030 %{
 5031   predicate(CompressedOops::base() == nullptr);
 5032   match(Set mem (StoreN mem zero));
 5033 
 5034   ins_cost(125); // XXX
 5035   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5036   ins_encode %{
 5037     __ movl($mem$$Address, r12);
 5038   %}
 5039   ins_pipe(ialu_mem_reg);
 5040 %}
 5041 
 5042 instruct storeImmN(memory mem, immN src)
 5043 %{
 5044   match(Set mem (StoreN mem src));
 5045 
 5046   ins_cost(150); // XXX
 5047   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5048   ins_encode %{
 5049     address con = (address)$src$$constant;
 5050     if (con == nullptr) {
 5051       __ movl($mem$$Address, 0);
 5052     } else {
 5053       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5054     }
 5055   %}
 5056   ins_pipe(ialu_mem_imm);
 5057 %}
 5058 
 5059 instruct storeImmNKlass(memory mem, immNKlass src)
 5060 %{
 5061   match(Set mem (StoreNKlass mem src));
 5062 
 5063   ins_cost(150); // XXX
 5064   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5065   ins_encode %{
 5066     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5067   %}
 5068   ins_pipe(ialu_mem_imm);
 5069 %}
 5070 
 5071 // Store Integer Immediate
 5072 instruct storeImmI0(memory mem, immI_0 zero)
 5073 %{
 5074   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5075   match(Set mem (StoreI mem zero));
 5076 
 5077   ins_cost(125); // XXX
 5078   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5079   ins_encode %{
 5080     __ movl($mem$$Address, r12);
 5081   %}
 5082   ins_pipe(ialu_mem_reg);
 5083 %}
 5084 
 5085 instruct storeImmI(memory mem, immI src)
 5086 %{
 5087   match(Set mem (StoreI mem src));
 5088 
 5089   ins_cost(150);
 5090   format %{ "movl    $mem, $src\t# int" %}
 5091   ins_encode %{
 5092     __ movl($mem$$Address, $src$$constant);
 5093   %}
 5094   ins_pipe(ialu_mem_imm);
 5095 %}
 5096 
 5097 // Store Long Immediate
 5098 instruct storeImmL0(memory mem, immL0 zero)
 5099 %{
 5100   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5101   match(Set mem (StoreL mem zero));
 5102 
 5103   ins_cost(125); // XXX
 5104   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5105   ins_encode %{
 5106     __ movq($mem$$Address, r12);
 5107   %}
 5108   ins_pipe(ialu_mem_reg);
 5109 %}
 5110 
 5111 instruct storeImmL(memory mem, immL32 src)
 5112 %{
 5113   match(Set mem (StoreL mem src));
 5114 
 5115   ins_cost(150);
 5116   format %{ "movq    $mem, $src\t# long" %}
 5117   ins_encode %{
 5118     __ movq($mem$$Address, $src$$constant);
 5119   %}
 5120   ins_pipe(ialu_mem_imm);
 5121 %}
 5122 
 5123 // Store Short/Char Immediate
 5124 instruct storeImmC0(memory mem, immI_0 zero)
 5125 %{
 5126   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5127   match(Set mem (StoreC mem zero));
 5128 
 5129   ins_cost(125); // XXX
 5130   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5131   ins_encode %{
 5132     __ movw($mem$$Address, r12);
 5133   %}
 5134   ins_pipe(ialu_mem_reg);
 5135 %}
 5136 
 5137 instruct storeImmI16(memory mem, immI16 src)
 5138 %{
 5139   predicate(UseStoreImmI16);
 5140   match(Set mem (StoreC mem src));
 5141 
 5142   ins_cost(150);
 5143   format %{ "movw    $mem, $src\t# short/char" %}
 5144   ins_encode %{
 5145     __ movw($mem$$Address, $src$$constant);
 5146   %}
 5147   ins_pipe(ialu_mem_imm);
 5148 %}
 5149 
 5150 // Store Byte Immediate
 5151 instruct storeImmB0(memory mem, immI_0 zero)
 5152 %{
 5153   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5154   match(Set mem (StoreB mem zero));
 5155 
 5156   ins_cost(125); // XXX
 5157   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5158   ins_encode %{
 5159     __ movb($mem$$Address, r12);
 5160   %}
 5161   ins_pipe(ialu_mem_reg);
 5162 %}
 5163 
 5164 instruct storeImmB(memory mem, immI8 src)
 5165 %{
 5166   match(Set mem (StoreB mem src));
 5167 
 5168   ins_cost(150); // XXX
 5169   format %{ "movb    $mem, $src\t# byte" %}
 5170   ins_encode %{
 5171     __ movb($mem$$Address, $src$$constant);
 5172   %}
 5173   ins_pipe(ialu_mem_imm);
 5174 %}
 5175 
 5176 // Store CMS card-mark Immediate
 5177 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 5178 %{
 5179   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5180   match(Set mem (StoreCM mem zero));
 5181 
 5182   ins_cost(125); // XXX
 5183   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 5184   ins_encode %{
 5185     __ movb($mem$$Address, r12);
 5186   %}
 5187   ins_pipe(ialu_mem_reg);
 5188 %}
 5189 
 5190 instruct storeImmCM0(memory mem, immI_0 src)
 5191 %{
 5192   match(Set mem (StoreCM mem src));
 5193 
 5194   ins_cost(150); // XXX
 5195   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 5196   ins_encode %{
 5197     __ movb($mem$$Address, $src$$constant);
 5198   %}
 5199   ins_pipe(ialu_mem_imm);
 5200 %}
 5201 
 5202 // Store Float
 5203 instruct storeF(memory mem, regF src)
 5204 %{
 5205   match(Set mem (StoreF mem src));
 5206 
 5207   ins_cost(95); // XXX
 5208   format %{ "movss   $mem, $src\t# float" %}
 5209   ins_encode %{
 5210     __ movflt($mem$$Address, $src$$XMMRegister);
 5211   %}
 5212   ins_pipe(pipe_slow); // XXX
 5213 %}
 5214 
 5215 // Store immediate Float value (it is faster than store from XMM register)
 5216 instruct storeF0(memory mem, immF0 zero)
 5217 %{
 5218   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5219   match(Set mem (StoreF mem zero));
 5220 
 5221   ins_cost(25); // XXX
 5222   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5223   ins_encode %{
 5224     __ movl($mem$$Address, r12);
 5225   %}
 5226   ins_pipe(ialu_mem_reg);
 5227 %}
 5228 
 5229 instruct storeF_imm(memory mem, immF src)
 5230 %{
 5231   match(Set mem (StoreF mem src));
 5232 
 5233   ins_cost(50);
 5234   format %{ "movl    $mem, $src\t# float" %}
 5235   ins_encode %{
 5236     __ movl($mem$$Address, jint_cast($src$$constant));
 5237   %}
 5238   ins_pipe(ialu_mem_imm);
 5239 %}
 5240 
 5241 // Store Double
 5242 instruct storeD(memory mem, regD src)
 5243 %{
 5244   match(Set mem (StoreD mem src));
 5245 
 5246   ins_cost(95); // XXX
 5247   format %{ "movsd   $mem, $src\t# double" %}
 5248   ins_encode %{
 5249     __ movdbl($mem$$Address, $src$$XMMRegister);
 5250   %}
 5251   ins_pipe(pipe_slow); // XXX
 5252 %}
 5253 
 5254 // Store immediate double 0.0 (it is faster than store from XMM register)
 5255 instruct storeD0_imm(memory mem, immD0 src)
 5256 %{
 5257   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5258   match(Set mem (StoreD mem src));
 5259 
 5260   ins_cost(50);
 5261   format %{ "movq    $mem, $src\t# double 0." %}
 5262   ins_encode %{
 5263     __ movq($mem$$Address, $src$$constant);
 5264   %}
 5265   ins_pipe(ialu_mem_imm);
 5266 %}
 5267 
 5268 instruct storeD0(memory mem, immD0 zero)
 5269 %{
 5270   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5271   match(Set mem (StoreD mem zero));
 5272 
 5273   ins_cost(25); // XXX
 5274   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5275   ins_encode %{
 5276     __ movq($mem$$Address, r12);
 5277   %}
 5278   ins_pipe(ialu_mem_reg);
 5279 %}
 5280 
 5281 instruct storeSSI(stackSlotI dst, rRegI src)
 5282 %{
 5283   match(Set dst src);
 5284 
 5285   ins_cost(100);
 5286   format %{ "movl    $dst, $src\t# int stk" %}
 5287   ins_encode %{
 5288     __ movl($dst$$Address, $src$$Register);
 5289   %}
 5290   ins_pipe( ialu_mem_reg );
 5291 %}
 5292 
 5293 instruct storeSSL(stackSlotL dst, rRegL src)
 5294 %{
 5295   match(Set dst src);
 5296 
 5297   ins_cost(100);
 5298   format %{ "movq    $dst, $src\t# long stk" %}
 5299   ins_encode %{
 5300     __ movq($dst$$Address, $src$$Register);
 5301   %}
 5302   ins_pipe(ialu_mem_reg);
 5303 %}
 5304 
 5305 instruct storeSSP(stackSlotP dst, rRegP src)
 5306 %{
 5307   match(Set dst src);
 5308 
 5309   ins_cost(100);
 5310   format %{ "movq    $dst, $src\t# ptr stk" %}
 5311   ins_encode %{
 5312     __ movq($dst$$Address, $src$$Register);
 5313   %}
 5314   ins_pipe(ialu_mem_reg);
 5315 %}
 5316 
 5317 instruct storeSSF(stackSlotF dst, regF src)
 5318 %{
 5319   match(Set dst src);
 5320 
 5321   ins_cost(95); // XXX
 5322   format %{ "movss   $dst, $src\t# float stk" %}
 5323   ins_encode %{
 5324     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5325   %}
 5326   ins_pipe(pipe_slow); // XXX
 5327 %}
 5328 
 5329 instruct storeSSD(stackSlotD dst, regD src)
 5330 %{
 5331   match(Set dst src);
 5332 
 5333   ins_cost(95); // XXX
 5334   format %{ "movsd   $dst, $src\t# double stk" %}
 5335   ins_encode %{
 5336     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5337   %}
 5338   ins_pipe(pipe_slow); // XXX
 5339 %}
 5340 
 5341 instruct cacheWB(indirect addr)
 5342 %{
 5343   predicate(VM_Version::supports_data_cache_line_flush());
 5344   match(CacheWB addr);
 5345 
 5346   ins_cost(100);
 5347   format %{"cache wb $addr" %}
 5348   ins_encode %{
 5349     assert($addr->index_position() < 0, "should be");
 5350     assert($addr$$disp == 0, "should be");
 5351     __ cache_wb(Address($addr$$base$$Register, 0));
 5352   %}
 5353   ins_pipe(pipe_slow); // XXX
 5354 %}
 5355 
 5356 instruct cacheWBPreSync()
 5357 %{
 5358   predicate(VM_Version::supports_data_cache_line_flush());
 5359   match(CacheWBPreSync);
 5360 
 5361   ins_cost(100);
 5362   format %{"cache wb presync" %}
 5363   ins_encode %{
 5364     __ cache_wbsync(true);
 5365   %}
 5366   ins_pipe(pipe_slow); // XXX
 5367 %}
 5368 
 5369 instruct cacheWBPostSync()
 5370 %{
 5371   predicate(VM_Version::supports_data_cache_line_flush());
 5372   match(CacheWBPostSync);
 5373 
 5374   ins_cost(100);
 5375   format %{"cache wb postsync" %}
 5376   ins_encode %{
 5377     __ cache_wbsync(false);
 5378   %}
 5379   ins_pipe(pipe_slow); // XXX
 5380 %}
 5381 
 5382 //----------BSWAP Instructions-------------------------------------------------
 5383 instruct bytes_reverse_int(rRegI dst) %{
 5384   match(Set dst (ReverseBytesI dst));
 5385 
 5386   format %{ "bswapl  $dst" %}
 5387   ins_encode %{
 5388     __ bswapl($dst$$Register);
 5389   %}
 5390   ins_pipe( ialu_reg );
 5391 %}
 5392 
 5393 instruct bytes_reverse_long(rRegL dst) %{
 5394   match(Set dst (ReverseBytesL dst));
 5395 
 5396   format %{ "bswapq  $dst" %}
 5397   ins_encode %{
 5398     __ bswapq($dst$$Register);
 5399   %}
 5400   ins_pipe( ialu_reg);
 5401 %}
 5402 
 5403 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5404   match(Set dst (ReverseBytesUS dst));
 5405   effect(KILL cr);
 5406 
 5407   format %{ "bswapl  $dst\n\t"
 5408             "shrl    $dst,16\n\t" %}
 5409   ins_encode %{
 5410     __ bswapl($dst$$Register);
 5411     __ shrl($dst$$Register, 16);
 5412   %}
 5413   ins_pipe( ialu_reg );
 5414 %}
 5415 
 5416 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5417   match(Set dst (ReverseBytesS dst));
 5418   effect(KILL cr);
 5419 
 5420   format %{ "bswapl  $dst\n\t"
 5421             "sar     $dst,16\n\t" %}
 5422   ins_encode %{
 5423     __ bswapl($dst$$Register);
 5424     __ sarl($dst$$Register, 16);
 5425   %}
 5426   ins_pipe( ialu_reg );
 5427 %}
 5428 
 5429 //---------- Zeros Count Instructions ------------------------------------------
 5430 
 5431 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5432   predicate(UseCountLeadingZerosInstruction);
 5433   match(Set dst (CountLeadingZerosI src));
 5434   effect(KILL cr);
 5435 
 5436   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5437   ins_encode %{
 5438     __ lzcntl($dst$$Register, $src$$Register);
 5439   %}
 5440   ins_pipe(ialu_reg);
 5441 %}
 5442 
 5443 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5444   predicate(UseCountLeadingZerosInstruction);
 5445   match(Set dst (CountLeadingZerosI (LoadI src)));
 5446   effect(KILL cr);
 5447   ins_cost(175);
 5448   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5449   ins_encode %{
 5450     __ lzcntl($dst$$Register, $src$$Address);
 5451   %}
 5452   ins_pipe(ialu_reg_mem);
 5453 %}
 5454 
 5455 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5456   predicate(!UseCountLeadingZerosInstruction);
 5457   match(Set dst (CountLeadingZerosI src));
 5458   effect(KILL cr);
 5459 
 5460   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5461             "jnz     skip\n\t"
 5462             "movl    $dst, -1\n"
 5463       "skip:\n\t"
 5464             "negl    $dst\n\t"
 5465             "addl    $dst, 31" %}
 5466   ins_encode %{
 5467     Register Rdst = $dst$$Register;
 5468     Register Rsrc = $src$$Register;
 5469     Label skip;
 5470     __ bsrl(Rdst, Rsrc);
 5471     __ jccb(Assembler::notZero, skip);
 5472     __ movl(Rdst, -1);
 5473     __ bind(skip);
 5474     __ negl(Rdst);
 5475     __ addl(Rdst, BitsPerInt - 1);
 5476   %}
 5477   ins_pipe(ialu_reg);
 5478 %}
 5479 
 5480 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5481   predicate(UseCountLeadingZerosInstruction);
 5482   match(Set dst (CountLeadingZerosL src));
 5483   effect(KILL cr);
 5484 
 5485   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5486   ins_encode %{
 5487     __ lzcntq($dst$$Register, $src$$Register);
 5488   %}
 5489   ins_pipe(ialu_reg);
 5490 %}
 5491 
 5492 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5493   predicate(UseCountLeadingZerosInstruction);
 5494   match(Set dst (CountLeadingZerosL (LoadL src)));
 5495   effect(KILL cr);
 5496   ins_cost(175);
 5497   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5498   ins_encode %{
 5499     __ lzcntq($dst$$Register, $src$$Address);
 5500   %}
 5501   ins_pipe(ialu_reg_mem);
 5502 %}
 5503 
 5504 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5505   predicate(!UseCountLeadingZerosInstruction);
 5506   match(Set dst (CountLeadingZerosL src));
 5507   effect(KILL cr);
 5508 
 5509   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5510             "jnz     skip\n\t"
 5511             "movl    $dst, -1\n"
 5512       "skip:\n\t"
 5513             "negl    $dst\n\t"
 5514             "addl    $dst, 63" %}
 5515   ins_encode %{
 5516     Register Rdst = $dst$$Register;
 5517     Register Rsrc = $src$$Register;
 5518     Label skip;
 5519     __ bsrq(Rdst, Rsrc);
 5520     __ jccb(Assembler::notZero, skip);
 5521     __ movl(Rdst, -1);
 5522     __ bind(skip);
 5523     __ negl(Rdst);
 5524     __ addl(Rdst, BitsPerLong - 1);
 5525   %}
 5526   ins_pipe(ialu_reg);
 5527 %}
 5528 
 5529 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5530   predicate(UseCountTrailingZerosInstruction);
 5531   match(Set dst (CountTrailingZerosI src));
 5532   effect(KILL cr);
 5533 
 5534   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5535   ins_encode %{
 5536     __ tzcntl($dst$$Register, $src$$Register);
 5537   %}
 5538   ins_pipe(ialu_reg);
 5539 %}
 5540 
 5541 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5542   predicate(UseCountTrailingZerosInstruction);
 5543   match(Set dst (CountTrailingZerosI (LoadI src)));
 5544   effect(KILL cr);
 5545   ins_cost(175);
 5546   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5547   ins_encode %{
 5548     __ tzcntl($dst$$Register, $src$$Address);
 5549   %}
 5550   ins_pipe(ialu_reg_mem);
 5551 %}
 5552 
 5553 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5554   predicate(!UseCountTrailingZerosInstruction);
 5555   match(Set dst (CountTrailingZerosI src));
 5556   effect(KILL cr);
 5557 
 5558   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5559             "jnz     done\n\t"
 5560             "movl    $dst, 32\n"
 5561       "done:" %}
 5562   ins_encode %{
 5563     Register Rdst = $dst$$Register;
 5564     Label done;
 5565     __ bsfl(Rdst, $src$$Register);
 5566     __ jccb(Assembler::notZero, done);
 5567     __ movl(Rdst, BitsPerInt);
 5568     __ bind(done);
 5569   %}
 5570   ins_pipe(ialu_reg);
 5571 %}
 5572 
 5573 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5574   predicate(UseCountTrailingZerosInstruction);
 5575   match(Set dst (CountTrailingZerosL src));
 5576   effect(KILL cr);
 5577 
 5578   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5579   ins_encode %{
 5580     __ tzcntq($dst$$Register, $src$$Register);
 5581   %}
 5582   ins_pipe(ialu_reg);
 5583 %}
 5584 
 5585 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5586   predicate(UseCountTrailingZerosInstruction);
 5587   match(Set dst (CountTrailingZerosL (LoadL src)));
 5588   effect(KILL cr);
 5589   ins_cost(175);
 5590   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5591   ins_encode %{
 5592     __ tzcntq($dst$$Register, $src$$Address);
 5593   %}
 5594   ins_pipe(ialu_reg_mem);
 5595 %}
 5596 
 5597 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5598   predicate(!UseCountTrailingZerosInstruction);
 5599   match(Set dst (CountTrailingZerosL src));
 5600   effect(KILL cr);
 5601 
 5602   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5603             "jnz     done\n\t"
 5604             "movl    $dst, 64\n"
 5605       "done:" %}
 5606   ins_encode %{
 5607     Register Rdst = $dst$$Register;
 5608     Label done;
 5609     __ bsfq(Rdst, $src$$Register);
 5610     __ jccb(Assembler::notZero, done);
 5611     __ movl(Rdst, BitsPerLong);
 5612     __ bind(done);
 5613   %}
 5614   ins_pipe(ialu_reg);
 5615 %}
 5616 
 5617 //--------------- Reverse Operation Instructions ----------------
 5618 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5619   predicate(!VM_Version::supports_gfni());
 5620   match(Set dst (ReverseI src));
 5621   effect(TEMP dst, TEMP rtmp, KILL cr);
 5622   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5623   ins_encode %{
 5624     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5625   %}
 5626   ins_pipe( ialu_reg );
 5627 %}
 5628 
 5629 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5630   predicate(VM_Version::supports_gfni());
 5631   match(Set dst (ReverseI src));
 5632   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5633   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5634   ins_encode %{
 5635     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5636   %}
 5637   ins_pipe( ialu_reg );
 5638 %}
 5639 
 5640 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5641   predicate(!VM_Version::supports_gfni());
 5642   match(Set dst (ReverseL src));
 5643   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5644   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5645   ins_encode %{
 5646     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5647   %}
 5648   ins_pipe( ialu_reg );
 5649 %}
 5650 
 5651 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5652   predicate(VM_Version::supports_gfni());
 5653   match(Set dst (ReverseL src));
 5654   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5655   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5656   ins_encode %{
 5657     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5658   %}
 5659   ins_pipe( ialu_reg );
 5660 %}
 5661 
 5662 //---------- Population Count Instructions -------------------------------------
 5663 
 5664 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5665   predicate(UsePopCountInstruction);
 5666   match(Set dst (PopCountI src));
 5667   effect(KILL cr);
 5668 
 5669   format %{ "popcnt  $dst, $src" %}
 5670   ins_encode %{
 5671     __ popcntl($dst$$Register, $src$$Register);
 5672   %}
 5673   ins_pipe(ialu_reg);
 5674 %}
 5675 
 5676 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5677   predicate(UsePopCountInstruction);
 5678   match(Set dst (PopCountI (LoadI mem)));
 5679   effect(KILL cr);
 5680 
 5681   format %{ "popcnt  $dst, $mem" %}
 5682   ins_encode %{
 5683     __ popcntl($dst$$Register, $mem$$Address);
 5684   %}
 5685   ins_pipe(ialu_reg);
 5686 %}
 5687 
 5688 // Note: Long.bitCount(long) returns an int.
 5689 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5690   predicate(UsePopCountInstruction);
 5691   match(Set dst (PopCountL src));
 5692   effect(KILL cr);
 5693 
 5694   format %{ "popcnt  $dst, $src" %}
 5695   ins_encode %{
 5696     __ popcntq($dst$$Register, $src$$Register);
 5697   %}
 5698   ins_pipe(ialu_reg);
 5699 %}
 5700 
 5701 // Note: Long.bitCount(long) returns an int.
 5702 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5703   predicate(UsePopCountInstruction);
 5704   match(Set dst (PopCountL (LoadL mem)));
 5705   effect(KILL cr);
 5706 
 5707   format %{ "popcnt  $dst, $mem" %}
 5708   ins_encode %{
 5709     __ popcntq($dst$$Register, $mem$$Address);
 5710   %}
 5711   ins_pipe(ialu_reg);
 5712 %}
 5713 
 5714 
 5715 //----------MemBar Instructions-----------------------------------------------
 5716 // Memory barrier flavors
 5717 
 5718 instruct membar_acquire()
 5719 %{
 5720   match(MemBarAcquire);
 5721   match(LoadFence);
 5722   ins_cost(0);
 5723 
 5724   size(0);
 5725   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5726   ins_encode();
 5727   ins_pipe(empty);
 5728 %}
 5729 
 5730 instruct membar_acquire_lock()
 5731 %{
 5732   match(MemBarAcquireLock);
 5733   ins_cost(0);
 5734 
 5735   size(0);
 5736   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5737   ins_encode();
 5738   ins_pipe(empty);
 5739 %}
 5740 
 5741 instruct membar_release()
 5742 %{
 5743   match(MemBarRelease);
 5744   match(StoreFence);
 5745   ins_cost(0);
 5746 
 5747   size(0);
 5748   format %{ "MEMBAR-release ! (empty encoding)" %}
 5749   ins_encode();
 5750   ins_pipe(empty);
 5751 %}
 5752 
 5753 instruct membar_release_lock()
 5754 %{
 5755   match(MemBarReleaseLock);
 5756   ins_cost(0);
 5757 
 5758   size(0);
 5759   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5760   ins_encode();
 5761   ins_pipe(empty);
 5762 %}
 5763 
 5764 instruct membar_volatile(rFlagsReg cr) %{
 5765   match(MemBarVolatile);
 5766   effect(KILL cr);
 5767   ins_cost(400);
 5768 
 5769   format %{
 5770     $$template
 5771     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5772   %}
 5773   ins_encode %{
 5774     __ membar(Assembler::StoreLoad);
 5775   %}
 5776   ins_pipe(pipe_slow);
 5777 %}
 5778 
 5779 instruct unnecessary_membar_volatile()
 5780 %{
 5781   match(MemBarVolatile);
 5782   predicate(Matcher::post_store_load_barrier(n));
 5783   ins_cost(0);
 5784 
 5785   size(0);
 5786   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5787   ins_encode();
 5788   ins_pipe(empty);
 5789 %}
 5790 
 5791 instruct membar_storestore() %{
 5792   match(MemBarStoreStore);
 5793   match(StoreStoreFence);
 5794   ins_cost(0);
 5795 
 5796   size(0);
 5797   format %{ "MEMBAR-storestore (empty encoding)" %}
 5798   ins_encode( );
 5799   ins_pipe(empty);
 5800 %}
 5801 
 5802 //----------Move Instructions--------------------------------------------------
 5803 
 5804 instruct castX2P(rRegP dst, rRegL src)
 5805 %{
 5806   match(Set dst (CastX2P src));
 5807 
 5808   format %{ "movq    $dst, $src\t# long->ptr" %}
 5809   ins_encode %{
 5810     if ($dst$$reg != $src$$reg) {
 5811       __ movptr($dst$$Register, $src$$Register);
 5812     }
 5813   %}
 5814   ins_pipe(ialu_reg_reg); // XXX
 5815 %}
 5816 
 5817 instruct castP2X(rRegL dst, rRegP src)
 5818 %{
 5819   match(Set dst (CastP2X src));
 5820 
 5821   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5822   ins_encode %{
 5823     if ($dst$$reg != $src$$reg) {
 5824       __ movptr($dst$$Register, $src$$Register);
 5825     }
 5826   %}
 5827   ins_pipe(ialu_reg_reg); // XXX
 5828 %}
 5829 
 5830 // Convert oop into int for vectors alignment masking
 5831 instruct convP2I(rRegI dst, rRegP src)
 5832 %{
 5833   match(Set dst (ConvL2I (CastP2X src)));
 5834 
 5835   format %{ "movl    $dst, $src\t# ptr -> int" %}
 5836   ins_encode %{
 5837     __ movl($dst$$Register, $src$$Register);
 5838   %}
 5839   ins_pipe(ialu_reg_reg); // XXX
 5840 %}
 5841 
 5842 // Convert compressed oop into int for vectors alignment masking
 5843 // in case of 32bit oops (heap < 4Gb).
 5844 instruct convN2I(rRegI dst, rRegN src)
 5845 %{
 5846   predicate(CompressedOops::shift() == 0);
 5847   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 5848 
 5849   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 5850   ins_encode %{
 5851     __ movl($dst$$Register, $src$$Register);
 5852   %}
 5853   ins_pipe(ialu_reg_reg); // XXX
 5854 %}
 5855 
 5856 // Convert oop pointer into compressed form
 5857 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 5858   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 5859   match(Set dst (EncodeP src));
 5860   effect(KILL cr);
 5861   format %{ "encode_heap_oop $dst,$src" %}
 5862   ins_encode %{
 5863     Register s = $src$$Register;
 5864     Register d = $dst$$Register;
 5865     if (s != d) {
 5866       __ movq(d, s);
 5867     }
 5868     __ encode_heap_oop(d);
 5869   %}
 5870   ins_pipe(ialu_reg_long);
 5871 %}
 5872 
 5873 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 5874   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 5875   match(Set dst (EncodeP src));
 5876   effect(KILL cr);
 5877   format %{ "encode_heap_oop_not_null $dst,$src" %}
 5878   ins_encode %{
 5879     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 5880   %}
 5881   ins_pipe(ialu_reg_long);
 5882 %}
 5883 
 5884 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 5885   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 5886             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 5887   match(Set dst (DecodeN src));
 5888   effect(KILL cr);
 5889   format %{ "decode_heap_oop $dst,$src" %}
 5890   ins_encode %{
 5891     Register s = $src$$Register;
 5892     Register d = $dst$$Register;
 5893     if (s != d) {
 5894       __ movq(d, s);
 5895     }
 5896     __ decode_heap_oop(d);
 5897   %}
 5898   ins_pipe(ialu_reg_long);
 5899 %}
 5900 
 5901 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 5902   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 5903             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 5904   match(Set dst (DecodeN src));
 5905   effect(KILL cr);
 5906   format %{ "decode_heap_oop_not_null $dst,$src" %}
 5907   ins_encode %{
 5908     Register s = $src$$Register;
 5909     Register d = $dst$$Register;
 5910     if (s != d) {
 5911       __ decode_heap_oop_not_null(d, s);
 5912     } else {
 5913       __ decode_heap_oop_not_null(d);
 5914     }
 5915   %}
 5916   ins_pipe(ialu_reg_long);
 5917 %}
 5918 
 5919 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 5920   match(Set dst (EncodePKlass src));
 5921   effect(TEMP dst, KILL cr);
 5922   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 5923   ins_encode %{
 5924     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 5925   %}
 5926   ins_pipe(ialu_reg_long);
 5927 %}
 5928 
 5929 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 5930   match(Set dst (DecodeNKlass src));
 5931   effect(TEMP dst, KILL cr);
 5932   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 5933   ins_encode %{
 5934     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 5935   %}
 5936   ins_pipe(ialu_reg_long);
 5937 %}
 5938 
 5939 //----------Conditional Move---------------------------------------------------
 5940 // Jump
 5941 // dummy instruction for generating temp registers
 5942 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 5943   match(Jump (LShiftL switch_val shift));
 5944   ins_cost(350);
 5945   predicate(false);
 5946   effect(TEMP dest);
 5947 
 5948   format %{ "leaq    $dest, [$constantaddress]\n\t"
 5949             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 5950   ins_encode %{
 5951     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 5952     // to do that and the compiler is using that register as one it can allocate.
 5953     // So we build it all by hand.
 5954     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 5955     // ArrayAddress dispatch(table, index);
 5956     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 5957     __ lea($dest$$Register, $constantaddress);
 5958     __ jmp(dispatch);
 5959   %}
 5960   ins_pipe(pipe_jmp);
 5961 %}
 5962 
 5963 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 5964   match(Jump (AddL (LShiftL switch_val shift) offset));
 5965   ins_cost(350);
 5966   effect(TEMP dest);
 5967 
 5968   format %{ "leaq    $dest, [$constantaddress]\n\t"
 5969             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 5970   ins_encode %{
 5971     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 5972     // to do that and the compiler is using that register as one it can allocate.
 5973     // So we build it all by hand.
 5974     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 5975     // ArrayAddress dispatch(table, index);
 5976     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 5977     __ lea($dest$$Register, $constantaddress);
 5978     __ jmp(dispatch);
 5979   %}
 5980   ins_pipe(pipe_jmp);
 5981 %}
 5982 
 5983 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 5984   match(Jump switch_val);
 5985   ins_cost(350);
 5986   effect(TEMP dest);
 5987 
 5988   format %{ "leaq    $dest, [$constantaddress]\n\t"
 5989             "jmp     [$dest + $switch_val]\n\t" %}
 5990   ins_encode %{
 5991     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 5992     // to do that and the compiler is using that register as one it can allocate.
 5993     // So we build it all by hand.
 5994     // Address index(noreg, switch_reg, Address::times_1);
 5995     // ArrayAddress dispatch(table, index);
 5996     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 5997     __ lea($dest$$Register, $constantaddress);
 5998     __ jmp(dispatch);
 5999   %}
 6000   ins_pipe(pipe_jmp);
 6001 %}
 6002 
 6003 // Conditional move
 6004 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6005 %{
 6006   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6007   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6008 
 6009   ins_cost(100); // XXX
 6010   format %{ "setbn$cop $dst\t# signed, int" %}
 6011   ins_encode %{
 6012     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6013     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6014   %}
 6015   ins_pipe(ialu_reg);
 6016 %}
 6017 
 6018 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6019 %{
 6020   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6021 
 6022   ins_cost(200); // XXX
 6023   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6024   ins_encode %{
 6025     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6026   %}
 6027   ins_pipe(pipe_cmov_reg);
 6028 %}
 6029 
 6030 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6031 %{
 6032   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6033   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6034 
 6035   ins_cost(100); // XXX
 6036   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6037   ins_encode %{
 6038     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6039     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6040   %}
 6041   ins_pipe(ialu_reg);
 6042 %}
 6043 
 6044 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6045   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6046 
 6047   ins_cost(200); // XXX
 6048   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6049   ins_encode %{
 6050     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6051   %}
 6052   ins_pipe(pipe_cmov_reg);
 6053 %}
 6054 
 6055 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6056 %{
 6057   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6058   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6059 
 6060   ins_cost(100); // XXX
 6061   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6062   ins_encode %{
 6063     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6064     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6065   %}
 6066   ins_pipe(ialu_reg);
 6067 %}
 6068 
 6069 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6070   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6071   ins_cost(200);
 6072   expand %{
 6073     cmovI_regU(cop, cr, dst, src);
 6074   %}
 6075 %}
 6076 
 6077 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6078   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6079   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6080 
 6081   ins_cost(200); // XXX
 6082   format %{ "cmovpl  $dst, $src\n\t"
 6083             "cmovnel $dst, $src" %}
 6084   ins_encode %{
 6085     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6086     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6087   %}
 6088   ins_pipe(pipe_cmov_reg);
 6089 %}
 6090 
 6091 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6092 // inputs of the CMove
 6093 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6094   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6095   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6096 
 6097   ins_cost(200); // XXX
 6098   format %{ "cmovpl  $dst, $src\n\t"
 6099             "cmovnel $dst, $src" %}
 6100   ins_encode %{
 6101     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6102     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6103   %}
 6104   ins_pipe(pipe_cmov_reg);
 6105 %}
 6106 
 6107 // Conditional move
 6108 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6109   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6110 
 6111   ins_cost(250); // XXX
 6112   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6113   ins_encode %{
 6114     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6115   %}
 6116   ins_pipe(pipe_cmov_mem);
 6117 %}
 6118 
 6119 // Conditional move
 6120 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6121 %{
 6122   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6123 
 6124   ins_cost(250); // XXX
 6125   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6126   ins_encode %{
 6127     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6128   %}
 6129   ins_pipe(pipe_cmov_mem);
 6130 %}
 6131 
 6132 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6133   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6134   ins_cost(250);
 6135   expand %{
 6136     cmovI_memU(cop, cr, dst, src);
 6137   %}
 6138 %}
 6139 
 6140 // Conditional move
 6141 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6142 %{
 6143   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6144 
 6145   ins_cost(200); // XXX
 6146   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6147   ins_encode %{
 6148     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6149   %}
 6150   ins_pipe(pipe_cmov_reg);
 6151 %}
 6152 
 6153 // Conditional move
 6154 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6155 %{
 6156   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6157 
 6158   ins_cost(200); // XXX
 6159   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6160   ins_encode %{
 6161     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6162   %}
 6163   ins_pipe(pipe_cmov_reg);
 6164 %}
 6165 
 6166 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6167   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6168   ins_cost(200);
 6169   expand %{
 6170     cmovN_regU(cop, cr, dst, src);
 6171   %}
 6172 %}
 6173 
 6174 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6175   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6176   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6177 
 6178   ins_cost(200); // XXX
 6179   format %{ "cmovpl  $dst, $src\n\t"
 6180             "cmovnel $dst, $src" %}
 6181   ins_encode %{
 6182     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6183     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6184   %}
 6185   ins_pipe(pipe_cmov_reg);
 6186 %}
 6187 
 6188 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6189 // inputs of the CMove
 6190 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6191   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6192   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6193 
 6194   ins_cost(200); // XXX
 6195   format %{ "cmovpl  $dst, $src\n\t"
 6196             "cmovnel $dst, $src" %}
 6197   ins_encode %{
 6198     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6199     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6200   %}
 6201   ins_pipe(pipe_cmov_reg);
 6202 %}
 6203 
 6204 // Conditional move
 6205 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6206 %{
 6207   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6208 
 6209   ins_cost(200); // XXX
 6210   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6211   ins_encode %{
 6212     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6213   %}
 6214   ins_pipe(pipe_cmov_reg);  // XXX
 6215 %}
 6216 
 6217 // Conditional move
 6218 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6219 %{
 6220   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6221 
 6222   ins_cost(200); // XXX
 6223   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6224   ins_encode %{
 6225     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6226   %}
 6227   ins_pipe(pipe_cmov_reg); // XXX
 6228 %}
 6229 
 6230 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6231   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6232   ins_cost(200);
 6233   expand %{
 6234     cmovP_regU(cop, cr, dst, src);
 6235   %}
 6236 %}
 6237 
 6238 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6239   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6240   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6241 
 6242   ins_cost(200); // XXX
 6243   format %{ "cmovpq  $dst, $src\n\t"
 6244             "cmovneq $dst, $src" %}
 6245   ins_encode %{
 6246     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6247     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6248   %}
 6249   ins_pipe(pipe_cmov_reg);
 6250 %}
 6251 
 6252 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6253 // inputs of the CMove
 6254 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6255   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6256   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6257 
 6258   ins_cost(200); // XXX
 6259   format %{ "cmovpq  $dst, $src\n\t"
 6260             "cmovneq $dst, $src" %}
 6261   ins_encode %{
 6262     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6263     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6264   %}
 6265   ins_pipe(pipe_cmov_reg);
 6266 %}
 6267 
 6268 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6269 %{
 6270   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6271   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6272 
 6273   ins_cost(100); // XXX
 6274   format %{ "setbn$cop $dst\t# signed, long" %}
 6275   ins_encode %{
 6276     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6277     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6278   %}
 6279   ins_pipe(ialu_reg);
 6280 %}
 6281 
 6282 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6283 %{
 6284   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6285 
 6286   ins_cost(200); // XXX
 6287   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6288   ins_encode %{
 6289     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6290   %}
 6291   ins_pipe(pipe_cmov_reg);  // XXX
 6292 %}
 6293 
 6294 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6295 %{
 6296   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6297 
 6298   ins_cost(200); // XXX
 6299   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6300   ins_encode %{
 6301     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6302   %}
 6303   ins_pipe(pipe_cmov_mem);  // XXX
 6304 %}
 6305 
 6306 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6307 %{
 6308   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6309   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6310 
 6311   ins_cost(100); // XXX
 6312   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6313   ins_encode %{
 6314     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6315     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6316   %}
 6317   ins_pipe(ialu_reg);
 6318 %}
 6319 
 6320 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6321 %{
 6322   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6323 
 6324   ins_cost(200); // XXX
 6325   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6326   ins_encode %{
 6327     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6328   %}
 6329   ins_pipe(pipe_cmov_reg); // XXX
 6330 %}
 6331 
 6332 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6333 %{
 6334   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6335   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6336 
 6337   ins_cost(100); // XXX
 6338   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6339   ins_encode %{
 6340     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6341     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6342   %}
 6343   ins_pipe(ialu_reg);
 6344 %}
 6345 
 6346 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6347   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6348   ins_cost(200);
 6349   expand %{
 6350     cmovL_regU(cop, cr, dst, src);
 6351   %}
 6352 %}
 6353 
 6354 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6355   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6356   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6357 
 6358   ins_cost(200); // XXX
 6359   format %{ "cmovpq  $dst, $src\n\t"
 6360             "cmovneq $dst, $src" %}
 6361   ins_encode %{
 6362     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6363     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6364   %}
 6365   ins_pipe(pipe_cmov_reg);
 6366 %}
 6367 
 6368 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6369 // inputs of the CMove
 6370 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6371   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6372   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6373 
 6374   ins_cost(200); // XXX
 6375   format %{ "cmovpq  $dst, $src\n\t"
 6376             "cmovneq $dst, $src" %}
 6377   ins_encode %{
 6378     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6379     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6380   %}
 6381   ins_pipe(pipe_cmov_reg);
 6382 %}
 6383 
 6384 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6385 %{
 6386   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6387 
 6388   ins_cost(200); // XXX
 6389   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6390   ins_encode %{
 6391     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6392   %}
 6393   ins_pipe(pipe_cmov_mem); // XXX
 6394 %}
 6395 
 6396 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6397   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6398   ins_cost(200);
 6399   expand %{
 6400     cmovL_memU(cop, cr, dst, src);
 6401   %}
 6402 %}
 6403 
 6404 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6405 %{
 6406   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6407 
 6408   ins_cost(200); // XXX
 6409   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6410             "movss     $dst, $src\n"
 6411     "skip:" %}
 6412   ins_encode %{
 6413     Label Lskip;
 6414     // Invert sense of branch from sense of CMOV
 6415     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6416     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6417     __ bind(Lskip);
 6418   %}
 6419   ins_pipe(pipe_slow);
 6420 %}
 6421 
 6422 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6423 %{
 6424   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6425 
 6426   ins_cost(200); // XXX
 6427   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6428             "movss     $dst, $src\n"
 6429     "skip:" %}
 6430   ins_encode %{
 6431     Label Lskip;
 6432     // Invert sense of branch from sense of CMOV
 6433     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6434     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6435     __ bind(Lskip);
 6436   %}
 6437   ins_pipe(pipe_slow);
 6438 %}
 6439 
 6440 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6441   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6442   ins_cost(200);
 6443   expand %{
 6444     cmovF_regU(cop, cr, dst, src);
 6445   %}
 6446 %}
 6447 
 6448 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6449 %{
 6450   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6451 
 6452   ins_cost(200); // XXX
 6453   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6454             "movsd     $dst, $src\n"
 6455     "skip:" %}
 6456   ins_encode %{
 6457     Label Lskip;
 6458     // Invert sense of branch from sense of CMOV
 6459     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6460     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6461     __ bind(Lskip);
 6462   %}
 6463   ins_pipe(pipe_slow);
 6464 %}
 6465 
 6466 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6467 %{
 6468   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6469 
 6470   ins_cost(200); // XXX
 6471   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6472             "movsd     $dst, $src\n"
 6473     "skip:" %}
 6474   ins_encode %{
 6475     Label Lskip;
 6476     // Invert sense of branch from sense of CMOV
 6477     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6478     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6479     __ bind(Lskip);
 6480   %}
 6481   ins_pipe(pipe_slow);
 6482 %}
 6483 
 6484 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6485   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6486   ins_cost(200);
 6487   expand %{
 6488     cmovD_regU(cop, cr, dst, src);
 6489   %}
 6490 %}
 6491 
 6492 //----------Arithmetic Instructions--------------------------------------------
 6493 //----------Addition Instructions----------------------------------------------
 6494 
 6495 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6496 %{
 6497   match(Set dst (AddI dst src));
 6498   effect(KILL cr);
 6499   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6500   format %{ "addl    $dst, $src\t# int" %}
 6501   ins_encode %{
 6502     __ addl($dst$$Register, $src$$Register);
 6503   %}
 6504   ins_pipe(ialu_reg_reg);
 6505 %}
 6506 
 6507 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 6508 %{
 6509   match(Set dst (AddI dst src));
 6510   effect(KILL cr);
 6511   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6512 
 6513   format %{ "addl    $dst, $src\t# int" %}
 6514   ins_encode %{
 6515     __ addl($dst$$Register, $src$$constant);
 6516   %}
 6517   ins_pipe( ialu_reg );
 6518 %}
 6519 
 6520 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 6521 %{
 6522   match(Set dst (AddI dst (LoadI src)));
 6523   effect(KILL cr);
 6524   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6525 
 6526   ins_cost(150); // XXX
 6527   format %{ "addl    $dst, $src\t# int" %}
 6528   ins_encode %{
 6529     __ addl($dst$$Register, $src$$Address);
 6530   %}
 6531   ins_pipe(ialu_reg_mem);
 6532 %}
 6533 
 6534 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 6535 %{
 6536   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6537   effect(KILL cr);
 6538   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6539 
 6540   ins_cost(150); // XXX
 6541   format %{ "addl    $dst, $src\t# int" %}
 6542   ins_encode %{
 6543     __ addl($dst$$Address, $src$$Register);
 6544   %}
 6545   ins_pipe(ialu_mem_reg);
 6546 %}
 6547 
 6548 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 6549 %{
 6550   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6551   effect(KILL cr);
 6552   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6553 
 6554 
 6555   ins_cost(125); // XXX
 6556   format %{ "addl    $dst, $src\t# int" %}
 6557   ins_encode %{
 6558     __ addl($dst$$Address, $src$$constant);
 6559   %}
 6560   ins_pipe(ialu_mem_imm);
 6561 %}
 6562 
 6563 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 6564 %{
 6565   predicate(UseIncDec);
 6566   match(Set dst (AddI dst src));
 6567   effect(KILL cr);
 6568 
 6569   format %{ "incl    $dst\t# int" %}
 6570   ins_encode %{
 6571     __ incrementl($dst$$Register);
 6572   %}
 6573   ins_pipe(ialu_reg);
 6574 %}
 6575 
 6576 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 6577 %{
 6578   predicate(UseIncDec);
 6579   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6580   effect(KILL cr);
 6581 
 6582   ins_cost(125); // XXX
 6583   format %{ "incl    $dst\t# int" %}
 6584   ins_encode %{
 6585     __ incrementl($dst$$Address);
 6586   %}
 6587   ins_pipe(ialu_mem_imm);
 6588 %}
 6589 
 6590 // XXX why does that use AddI
 6591 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 6592 %{
 6593   predicate(UseIncDec);
 6594   match(Set dst (AddI dst src));
 6595   effect(KILL cr);
 6596 
 6597   format %{ "decl    $dst\t# int" %}
 6598   ins_encode %{
 6599     __ decrementl($dst$$Register);
 6600   %}
 6601   ins_pipe(ialu_reg);
 6602 %}
 6603 
 6604 // XXX why does that use AddI
 6605 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 6606 %{
 6607   predicate(UseIncDec);
 6608   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6609   effect(KILL cr);
 6610 
 6611   ins_cost(125); // XXX
 6612   format %{ "decl    $dst\t# int" %}
 6613   ins_encode %{
 6614     __ decrementl($dst$$Address);
 6615   %}
 6616   ins_pipe(ialu_mem_imm);
 6617 %}
 6618 
 6619 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 6620 %{
 6621   predicate(VM_Version::supports_fast_2op_lea());
 6622   match(Set dst (AddI (LShiftI index scale) disp));
 6623 
 6624   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 6625   ins_encode %{
 6626     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6627     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6628   %}
 6629   ins_pipe(ialu_reg_reg);
 6630 %}
 6631 
 6632 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 6633 %{
 6634   predicate(VM_Version::supports_fast_3op_lea());
 6635   match(Set dst (AddI (AddI base index) disp));
 6636 
 6637   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 6638   ins_encode %{
 6639     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6640   %}
 6641   ins_pipe(ialu_reg_reg);
 6642 %}
 6643 
 6644 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 6645 %{
 6646   predicate(VM_Version::supports_fast_2op_lea());
 6647   match(Set dst (AddI base (LShiftI index scale)));
 6648 
 6649   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 6650   ins_encode %{
 6651     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6652     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6653   %}
 6654   ins_pipe(ialu_reg_reg);
 6655 %}
 6656 
 6657 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 6658 %{
 6659   predicate(VM_Version::supports_fast_3op_lea());
 6660   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 6661 
 6662   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 6663   ins_encode %{
 6664     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6665     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6666   %}
 6667   ins_pipe(ialu_reg_reg);
 6668 %}
 6669 
 6670 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 6671 %{
 6672   match(Set dst (AddL dst src));
 6673   effect(KILL cr);
 6674   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6675 
 6676   format %{ "addq    $dst, $src\t# long" %}
 6677   ins_encode %{
 6678     __ addq($dst$$Register, $src$$Register);
 6679   %}
 6680   ins_pipe(ialu_reg_reg);
 6681 %}
 6682 
 6683 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 6684 %{
 6685   match(Set dst (AddL dst src));
 6686   effect(KILL cr);
 6687   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6688 
 6689   format %{ "addq    $dst, $src\t# long" %}
 6690   ins_encode %{
 6691     __ addq($dst$$Register, $src$$constant);
 6692   %}
 6693   ins_pipe( ialu_reg );
 6694 %}
 6695 
 6696 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 6697 %{
 6698   match(Set dst (AddL dst (LoadL src)));
 6699   effect(KILL cr);
 6700   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6701 
 6702   ins_cost(150); // XXX
 6703   format %{ "addq    $dst, $src\t# long" %}
 6704   ins_encode %{
 6705     __ addq($dst$$Register, $src$$Address);
 6706   %}
 6707   ins_pipe(ialu_reg_mem);
 6708 %}
 6709 
 6710 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 6711 %{
 6712   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6713   effect(KILL cr);
 6714   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6715 
 6716   ins_cost(150); // XXX
 6717   format %{ "addq    $dst, $src\t# long" %}
 6718   ins_encode %{
 6719     __ addq($dst$$Address, $src$$Register);
 6720   %}
 6721   ins_pipe(ialu_mem_reg);
 6722 %}
 6723 
 6724 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 6725 %{
 6726   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6727   effect(KILL cr);
 6728   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6729 
 6730   ins_cost(125); // XXX
 6731   format %{ "addq    $dst, $src\t# long" %}
 6732   ins_encode %{
 6733     __ addq($dst$$Address, $src$$constant);
 6734   %}
 6735   ins_pipe(ialu_mem_imm);
 6736 %}
 6737 
 6738 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 6739 %{
 6740   predicate(UseIncDec);
 6741   match(Set dst (AddL dst src));
 6742   effect(KILL cr);
 6743 
 6744   format %{ "incq    $dst\t# long" %}
 6745   ins_encode %{
 6746     __ incrementq($dst$$Register);
 6747   %}
 6748   ins_pipe(ialu_reg);
 6749 %}
 6750 
 6751 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 6752 %{
 6753   predicate(UseIncDec);
 6754   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6755   effect(KILL cr);
 6756 
 6757   ins_cost(125); // XXX
 6758   format %{ "incq    $dst\t# long" %}
 6759   ins_encode %{
 6760     __ incrementq($dst$$Address);
 6761   %}
 6762   ins_pipe(ialu_mem_imm);
 6763 %}
 6764 
 6765 // XXX why does that use AddL
 6766 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 6767 %{
 6768   predicate(UseIncDec);
 6769   match(Set dst (AddL dst src));
 6770   effect(KILL cr);
 6771 
 6772   format %{ "decq    $dst\t# long" %}
 6773   ins_encode %{
 6774     __ decrementq($dst$$Register);
 6775   %}
 6776   ins_pipe(ialu_reg);
 6777 %}
 6778 
 6779 // XXX why does that use AddL
 6780 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 6781 %{
 6782   predicate(UseIncDec);
 6783   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6784   effect(KILL cr);
 6785 
 6786   ins_cost(125); // XXX
 6787   format %{ "decq    $dst\t# long" %}
 6788   ins_encode %{
 6789     __ decrementq($dst$$Address);
 6790   %}
 6791   ins_pipe(ialu_mem_imm);
 6792 %}
 6793 
 6794 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 6795 %{
 6796   predicate(VM_Version::supports_fast_2op_lea());
 6797   match(Set dst (AddL (LShiftL index scale) disp));
 6798 
 6799   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 6800   ins_encode %{
 6801     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6802     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6803   %}
 6804   ins_pipe(ialu_reg_reg);
 6805 %}
 6806 
 6807 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 6808 %{
 6809   predicate(VM_Version::supports_fast_3op_lea());
 6810   match(Set dst (AddL (AddL base index) disp));
 6811 
 6812   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 6813   ins_encode %{
 6814     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6815   %}
 6816   ins_pipe(ialu_reg_reg);
 6817 %}
 6818 
 6819 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 6820 %{
 6821   predicate(VM_Version::supports_fast_2op_lea());
 6822   match(Set dst (AddL base (LShiftL index scale)));
 6823 
 6824   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 6825   ins_encode %{
 6826     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6827     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6828   %}
 6829   ins_pipe(ialu_reg_reg);
 6830 %}
 6831 
 6832 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 6833 %{
 6834   predicate(VM_Version::supports_fast_3op_lea());
 6835   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 6836 
 6837   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 6838   ins_encode %{
 6839     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6840     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6841   %}
 6842   ins_pipe(ialu_reg_reg);
 6843 %}
 6844 
 6845 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 6846 %{
 6847   match(Set dst (AddP dst src));
 6848   effect(KILL cr);
 6849   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6850 
 6851   format %{ "addq    $dst, $src\t# ptr" %}
 6852   ins_encode %{
 6853     __ addq($dst$$Register, $src$$Register);
 6854   %}
 6855   ins_pipe(ialu_reg_reg);
 6856 %}
 6857 
 6858 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 6859 %{
 6860   match(Set dst (AddP dst src));
 6861   effect(KILL cr);
 6862   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6863 
 6864   format %{ "addq    $dst, $src\t# ptr" %}
 6865   ins_encode %{
 6866     __ addq($dst$$Register, $src$$constant);
 6867   %}
 6868   ins_pipe( ialu_reg );
 6869 %}
 6870 
 6871 // XXX addP mem ops ????
 6872 
 6873 instruct checkCastPP(rRegP dst)
 6874 %{
 6875   match(Set dst (CheckCastPP dst));
 6876 
 6877   size(0);
 6878   format %{ "# checkcastPP of $dst" %}
 6879   ins_encode(/* empty encoding */);
 6880   ins_pipe(empty);
 6881 %}
 6882 
 6883 instruct castPP(rRegP dst)
 6884 %{
 6885   match(Set dst (CastPP dst));
 6886 
 6887   size(0);
 6888   format %{ "# castPP of $dst" %}
 6889   ins_encode(/* empty encoding */);
 6890   ins_pipe(empty);
 6891 %}
 6892 
 6893 instruct castII(rRegI dst)
 6894 %{
 6895   match(Set dst (CastII dst));
 6896 
 6897   size(0);
 6898   format %{ "# castII of $dst" %}
 6899   ins_encode(/* empty encoding */);
 6900   ins_cost(0);
 6901   ins_pipe(empty);
 6902 %}
 6903 
 6904 instruct castLL(rRegL dst)
 6905 %{
 6906   match(Set dst (CastLL dst));
 6907 
 6908   size(0);
 6909   format %{ "# castLL of $dst" %}
 6910   ins_encode(/* empty encoding */);
 6911   ins_cost(0);
 6912   ins_pipe(empty);
 6913 %}
 6914 
 6915 instruct castFF(regF dst)
 6916 %{
 6917   match(Set dst (CastFF dst));
 6918 
 6919   size(0);
 6920   format %{ "# castFF of $dst" %}
 6921   ins_encode(/* empty encoding */);
 6922   ins_cost(0);
 6923   ins_pipe(empty);
 6924 %}
 6925 
 6926 instruct castDD(regD dst)
 6927 %{
 6928   match(Set dst (CastDD dst));
 6929 
 6930   size(0);
 6931   format %{ "# castDD of $dst" %}
 6932   ins_encode(/* empty encoding */);
 6933   ins_cost(0);
 6934   ins_pipe(empty);
 6935 %}
 6936 
 6937 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 6938 instruct compareAndSwapP(rRegI res,
 6939                          memory mem_ptr,
 6940                          rax_RegP oldval, rRegP newval,
 6941                          rFlagsReg cr)
 6942 %{
 6943   predicate(n->as_LoadStore()->barrier_data() == 0);
 6944   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 6945   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 6946   effect(KILL cr, KILL oldval);
 6947 
 6948   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 6949             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 6950             "sete    $res\n\t"
 6951             "movzbl  $res, $res" %}
 6952   ins_encode %{
 6953     __ lock();
 6954     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 6955     __ setb(Assembler::equal, $res$$Register);
 6956     __ movzbl($res$$Register, $res$$Register);
 6957   %}
 6958   ins_pipe( pipe_cmpxchg );
 6959 %}
 6960 
 6961 instruct compareAndSwapL(rRegI res,
 6962                          memory mem_ptr,
 6963                          rax_RegL oldval, rRegL newval,
 6964                          rFlagsReg cr)
 6965 %{
 6966   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 6967   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 6968   effect(KILL cr, KILL oldval);
 6969 
 6970   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 6971             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 6972             "sete    $res\n\t"
 6973             "movzbl  $res, $res" %}
 6974   ins_encode %{
 6975     __ lock();
 6976     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 6977     __ setb(Assembler::equal, $res$$Register);
 6978     __ movzbl($res$$Register, $res$$Register);
 6979   %}
 6980   ins_pipe( pipe_cmpxchg );
 6981 %}
 6982 
 6983 instruct compareAndSwapI(rRegI res,
 6984                          memory mem_ptr,
 6985                          rax_RegI oldval, rRegI newval,
 6986                          rFlagsReg cr)
 6987 %{
 6988   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 6989   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 6990   effect(KILL cr, KILL oldval);
 6991 
 6992   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 6993             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 6994             "sete    $res\n\t"
 6995             "movzbl  $res, $res" %}
 6996   ins_encode %{
 6997     __ lock();
 6998     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 6999     __ setb(Assembler::equal, $res$$Register);
 7000     __ movzbl($res$$Register, $res$$Register);
 7001   %}
 7002   ins_pipe( pipe_cmpxchg );
 7003 %}
 7004 
 7005 instruct compareAndSwapB(rRegI res,
 7006                          memory mem_ptr,
 7007                          rax_RegI oldval, rRegI newval,
 7008                          rFlagsReg cr)
 7009 %{
 7010   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7011   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7012   effect(KILL cr, KILL oldval);
 7013 
 7014   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7015             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7016             "sete    $res\n\t"
 7017             "movzbl  $res, $res" %}
 7018   ins_encode %{
 7019     __ lock();
 7020     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7021     __ setb(Assembler::equal, $res$$Register);
 7022     __ movzbl($res$$Register, $res$$Register);
 7023   %}
 7024   ins_pipe( pipe_cmpxchg );
 7025 %}
 7026 
 7027 instruct compareAndSwapS(rRegI res,
 7028                          memory mem_ptr,
 7029                          rax_RegI oldval, rRegI newval,
 7030                          rFlagsReg cr)
 7031 %{
 7032   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7033   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7034   effect(KILL cr, KILL oldval);
 7035 
 7036   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7037             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7038             "sete    $res\n\t"
 7039             "movzbl  $res, $res" %}
 7040   ins_encode %{
 7041     __ lock();
 7042     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7043     __ setb(Assembler::equal, $res$$Register);
 7044     __ movzbl($res$$Register, $res$$Register);
 7045   %}
 7046   ins_pipe( pipe_cmpxchg );
 7047 %}
 7048 
 7049 instruct compareAndSwapN(rRegI res,
 7050                           memory mem_ptr,
 7051                           rax_RegN oldval, rRegN newval,
 7052                           rFlagsReg cr) %{
 7053   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7054   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7055   effect(KILL cr, KILL oldval);
 7056 
 7057   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7058             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7059             "sete    $res\n\t"
 7060             "movzbl  $res, $res" %}
 7061   ins_encode %{
 7062     __ lock();
 7063     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7064     __ setb(Assembler::equal, $res$$Register);
 7065     __ movzbl($res$$Register, $res$$Register);
 7066   %}
 7067   ins_pipe( pipe_cmpxchg );
 7068 %}
 7069 
 7070 instruct compareAndExchangeB(
 7071                          memory mem_ptr,
 7072                          rax_RegI oldval, rRegI newval,
 7073                          rFlagsReg cr)
 7074 %{
 7075   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7076   effect(KILL cr);
 7077 
 7078   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7079             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7080   ins_encode %{
 7081     __ lock();
 7082     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7083   %}
 7084   ins_pipe( pipe_cmpxchg );
 7085 %}
 7086 
 7087 instruct compareAndExchangeS(
 7088                          memory mem_ptr,
 7089                          rax_RegI oldval, rRegI newval,
 7090                          rFlagsReg cr)
 7091 %{
 7092   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7093   effect(KILL cr);
 7094 
 7095   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7096             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7097   ins_encode %{
 7098     __ lock();
 7099     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7100   %}
 7101   ins_pipe( pipe_cmpxchg );
 7102 %}
 7103 
 7104 instruct compareAndExchangeI(
 7105                          memory mem_ptr,
 7106                          rax_RegI oldval, rRegI newval,
 7107                          rFlagsReg cr)
 7108 %{
 7109   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7110   effect(KILL cr);
 7111 
 7112   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7113             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7114   ins_encode %{
 7115     __ lock();
 7116     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7117   %}
 7118   ins_pipe( pipe_cmpxchg );
 7119 %}
 7120 
 7121 instruct compareAndExchangeL(
 7122                          memory mem_ptr,
 7123                          rax_RegL oldval, rRegL newval,
 7124                          rFlagsReg cr)
 7125 %{
 7126   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7127   effect(KILL cr);
 7128 
 7129   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7130             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7131   ins_encode %{
 7132     __ lock();
 7133     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7134   %}
 7135   ins_pipe( pipe_cmpxchg );
 7136 %}
 7137 
 7138 instruct compareAndExchangeN(
 7139                           memory mem_ptr,
 7140                           rax_RegN oldval, rRegN newval,
 7141                           rFlagsReg cr) %{
 7142   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7143   effect(KILL cr);
 7144 
 7145   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7146             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7147   ins_encode %{
 7148     __ lock();
 7149     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7150   %}
 7151   ins_pipe( pipe_cmpxchg );
 7152 %}
 7153 
 7154 instruct compareAndExchangeP(
 7155                          memory mem_ptr,
 7156                          rax_RegP oldval, rRegP newval,
 7157                          rFlagsReg cr)
 7158 %{
 7159   predicate(n->as_LoadStore()->barrier_data() == 0);
 7160   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7161   effect(KILL cr);
 7162 
 7163   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7164             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7165   ins_encode %{
 7166     __ lock();
 7167     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7168   %}
 7169   ins_pipe( pipe_cmpxchg );
 7170 %}
 7171 
 7172 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7173   predicate(n->as_LoadStore()->result_not_used());
 7174   match(Set dummy (GetAndAddB mem add));
 7175   effect(KILL cr);
 7176   format %{ "addb_lock   $mem, $add" %}
 7177   ins_encode %{
 7178     __ lock();
 7179     __ addb($mem$$Address, $add$$Register);
 7180   %}
 7181   ins_pipe(pipe_cmpxchg);
 7182 %}
 7183 
 7184 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7185   predicate(n->as_LoadStore()->result_not_used());
 7186   match(Set dummy (GetAndAddB mem add));
 7187   effect(KILL cr);
 7188   format %{ "addb_lock   $mem, $add" %}
 7189   ins_encode %{
 7190     __ lock();
 7191     __ addb($mem$$Address, $add$$constant);
 7192   %}
 7193   ins_pipe(pipe_cmpxchg);
 7194 %}
 7195 
 7196 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7197   predicate(!n->as_LoadStore()->result_not_used());
 7198   match(Set newval (GetAndAddB mem newval));
 7199   effect(KILL cr);
 7200   format %{ "xaddb_lock  $mem, $newval" %}
 7201   ins_encode %{
 7202     __ lock();
 7203     __ xaddb($mem$$Address, $newval$$Register);
 7204   %}
 7205   ins_pipe(pipe_cmpxchg);
 7206 %}
 7207 
 7208 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7209   predicate(n->as_LoadStore()->result_not_used());
 7210   match(Set dummy (GetAndAddS mem add));
 7211   effect(KILL cr);
 7212   format %{ "addw_lock   $mem, $add" %}
 7213   ins_encode %{
 7214     __ lock();
 7215     __ addw($mem$$Address, $add$$Register);
 7216   %}
 7217   ins_pipe(pipe_cmpxchg);
 7218 %}
 7219 
 7220 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7221   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7222   match(Set dummy (GetAndAddS mem add));
 7223   effect(KILL cr);
 7224   format %{ "addw_lock   $mem, $add" %}
 7225   ins_encode %{
 7226     __ lock();
 7227     __ addw($mem$$Address, $add$$constant);
 7228   %}
 7229   ins_pipe(pipe_cmpxchg);
 7230 %}
 7231 
 7232 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7233   predicate(!n->as_LoadStore()->result_not_used());
 7234   match(Set newval (GetAndAddS mem newval));
 7235   effect(KILL cr);
 7236   format %{ "xaddw_lock  $mem, $newval" %}
 7237   ins_encode %{
 7238     __ lock();
 7239     __ xaddw($mem$$Address, $newval$$Register);
 7240   %}
 7241   ins_pipe(pipe_cmpxchg);
 7242 %}
 7243 
 7244 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7245   predicate(n->as_LoadStore()->result_not_used());
 7246   match(Set dummy (GetAndAddI mem add));
 7247   effect(KILL cr);
 7248   format %{ "addl_lock   $mem, $add" %}
 7249   ins_encode %{
 7250     __ lock();
 7251     __ addl($mem$$Address, $add$$Register);
 7252   %}
 7253   ins_pipe(pipe_cmpxchg);
 7254 %}
 7255 
 7256 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7257   predicate(n->as_LoadStore()->result_not_used());
 7258   match(Set dummy (GetAndAddI mem add));
 7259   effect(KILL cr);
 7260   format %{ "addl_lock   $mem, $add" %}
 7261   ins_encode %{
 7262     __ lock();
 7263     __ addl($mem$$Address, $add$$constant);
 7264   %}
 7265   ins_pipe(pipe_cmpxchg);
 7266 %}
 7267 
 7268 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 7269   predicate(!n->as_LoadStore()->result_not_used());
 7270   match(Set newval (GetAndAddI mem newval));
 7271   effect(KILL cr);
 7272   format %{ "xaddl_lock  $mem, $newval" %}
 7273   ins_encode %{
 7274     __ lock();
 7275     __ xaddl($mem$$Address, $newval$$Register);
 7276   %}
 7277   ins_pipe(pipe_cmpxchg);
 7278 %}
 7279 
 7280 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 7281   predicate(n->as_LoadStore()->result_not_used());
 7282   match(Set dummy (GetAndAddL mem add));
 7283   effect(KILL cr);
 7284   format %{ "addq_lock   $mem, $add" %}
 7285   ins_encode %{
 7286     __ lock();
 7287     __ addq($mem$$Address, $add$$Register);
 7288   %}
 7289   ins_pipe(pipe_cmpxchg);
 7290 %}
 7291 
 7292 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 7293   predicate(n->as_LoadStore()->result_not_used());
 7294   match(Set dummy (GetAndAddL mem add));
 7295   effect(KILL cr);
 7296   format %{ "addq_lock   $mem, $add" %}
 7297   ins_encode %{
 7298     __ lock();
 7299     __ addq($mem$$Address, $add$$constant);
 7300   %}
 7301   ins_pipe(pipe_cmpxchg);
 7302 %}
 7303 
 7304 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 7305   predicate(!n->as_LoadStore()->result_not_used());
 7306   match(Set newval (GetAndAddL mem newval));
 7307   effect(KILL cr);
 7308   format %{ "xaddq_lock  $mem, $newval" %}
 7309   ins_encode %{
 7310     __ lock();
 7311     __ xaddq($mem$$Address, $newval$$Register);
 7312   %}
 7313   ins_pipe(pipe_cmpxchg);
 7314 %}
 7315 
 7316 instruct xchgB( memory mem, rRegI newval) %{
 7317   match(Set newval (GetAndSetB mem newval));
 7318   format %{ "XCHGB  $newval,[$mem]" %}
 7319   ins_encode %{
 7320     __ xchgb($newval$$Register, $mem$$Address);
 7321   %}
 7322   ins_pipe( pipe_cmpxchg );
 7323 %}
 7324 
 7325 instruct xchgS( memory mem, rRegI newval) %{
 7326   match(Set newval (GetAndSetS mem newval));
 7327   format %{ "XCHGW  $newval,[$mem]" %}
 7328   ins_encode %{
 7329     __ xchgw($newval$$Register, $mem$$Address);
 7330   %}
 7331   ins_pipe( pipe_cmpxchg );
 7332 %}
 7333 
 7334 instruct xchgI( memory mem, rRegI newval) %{
 7335   match(Set newval (GetAndSetI mem newval));
 7336   format %{ "XCHGL  $newval,[$mem]" %}
 7337   ins_encode %{
 7338     __ xchgl($newval$$Register, $mem$$Address);
 7339   %}
 7340   ins_pipe( pipe_cmpxchg );
 7341 %}
 7342 
 7343 instruct xchgL( memory mem, rRegL newval) %{
 7344   match(Set newval (GetAndSetL mem newval));
 7345   format %{ "XCHGL  $newval,[$mem]" %}
 7346   ins_encode %{
 7347     __ xchgq($newval$$Register, $mem$$Address);
 7348   %}
 7349   ins_pipe( pipe_cmpxchg );
 7350 %}
 7351 
 7352 instruct xchgP( memory mem, rRegP newval) %{
 7353   match(Set newval (GetAndSetP mem newval));
 7354   predicate(n->as_LoadStore()->barrier_data() == 0);
 7355   format %{ "XCHGQ  $newval,[$mem]" %}
 7356   ins_encode %{
 7357     __ xchgq($newval$$Register, $mem$$Address);
 7358   %}
 7359   ins_pipe( pipe_cmpxchg );
 7360 %}
 7361 
 7362 instruct xchgN( memory mem, rRegN newval) %{
 7363   match(Set newval (GetAndSetN mem newval));
 7364   format %{ "XCHGL  $newval,$mem]" %}
 7365   ins_encode %{
 7366     __ xchgl($newval$$Register, $mem$$Address);
 7367   %}
 7368   ins_pipe( pipe_cmpxchg );
 7369 %}
 7370 
 7371 //----------Abs Instructions-------------------------------------------
 7372 
 7373 // Integer Absolute Instructions
 7374 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7375 %{
 7376   match(Set dst (AbsI src));
 7377   effect(TEMP dst, KILL cr);
 7378   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 7379             "subl    $dst, $src\n\t"
 7380             "cmovll  $dst, $src" %}
 7381   ins_encode %{
 7382     __ xorl($dst$$Register, $dst$$Register);
 7383     __ subl($dst$$Register, $src$$Register);
 7384     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 7385   %}
 7386 
 7387   ins_pipe(ialu_reg_reg);
 7388 %}
 7389 
 7390 // Long Absolute Instructions
 7391 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7392 %{
 7393   match(Set dst (AbsL src));
 7394   effect(TEMP dst, KILL cr);
 7395   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 7396             "subq    $dst, $src\n\t"
 7397             "cmovlq  $dst, $src" %}
 7398   ins_encode %{
 7399     __ xorl($dst$$Register, $dst$$Register);
 7400     __ subq($dst$$Register, $src$$Register);
 7401     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 7402   %}
 7403 
 7404   ins_pipe(ialu_reg_reg);
 7405 %}
 7406 
 7407 //----------Subtraction Instructions-------------------------------------------
 7408 
 7409 // Integer Subtraction Instructions
 7410 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7411 %{
 7412   match(Set dst (SubI dst src));
 7413   effect(KILL cr);
 7414   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7415 
 7416   format %{ "subl    $dst, $src\t# int" %}
 7417   ins_encode %{
 7418     __ subl($dst$$Register, $src$$Register);
 7419   %}
 7420   ins_pipe(ialu_reg_reg);
 7421 %}
 7422 
 7423 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7424 %{
 7425   match(Set dst (SubI dst (LoadI src)));
 7426   effect(KILL cr);
 7427   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7428 
 7429   ins_cost(150);
 7430   format %{ "subl    $dst, $src\t# int" %}
 7431   ins_encode %{
 7432     __ subl($dst$$Register, $src$$Address);
 7433   %}
 7434   ins_pipe(ialu_reg_mem);
 7435 %}
 7436 
 7437 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7438 %{
 7439   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 7440   effect(KILL cr);
 7441   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7442 
 7443   ins_cost(150);
 7444   format %{ "subl    $dst, $src\t# int" %}
 7445   ins_encode %{
 7446     __ subl($dst$$Address, $src$$Register);
 7447   %}
 7448   ins_pipe(ialu_mem_reg);
 7449 %}
 7450 
 7451 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7452 %{
 7453   match(Set dst (SubL dst src));
 7454   effect(KILL cr);
 7455   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7456 
 7457   format %{ "subq    $dst, $src\t# long" %}
 7458   ins_encode %{
 7459     __ subq($dst$$Register, $src$$Register);
 7460   %}
 7461   ins_pipe(ialu_reg_reg);
 7462 %}
 7463 
 7464 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7465 %{
 7466   match(Set dst (SubL dst (LoadL src)));
 7467   effect(KILL cr);
 7468   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7469 
 7470   ins_cost(150);
 7471   format %{ "subq    $dst, $src\t# long" %}
 7472   ins_encode %{
 7473     __ subq($dst$$Register, $src$$Address);
 7474   %}
 7475   ins_pipe(ialu_reg_mem);
 7476 %}
 7477 
 7478 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7479 %{
 7480   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 7481   effect(KILL cr);
 7482   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7483 
 7484   ins_cost(150);
 7485   format %{ "subq    $dst, $src\t# long" %}
 7486   ins_encode %{
 7487     __ subq($dst$$Address, $src$$Register);
 7488   %}
 7489   ins_pipe(ialu_mem_reg);
 7490 %}
 7491 
 7492 // Subtract from a pointer
 7493 // XXX hmpf???
 7494 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 7495 %{
 7496   match(Set dst (AddP dst (SubI zero src)));
 7497   effect(KILL cr);
 7498 
 7499   format %{ "subq    $dst, $src\t# ptr - int" %}
 7500   ins_encode %{
 7501     __ subq($dst$$Register, $src$$Register);
 7502   %}
 7503   ins_pipe(ialu_reg_reg);
 7504 %}
 7505 
 7506 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 7507 %{
 7508   match(Set dst (SubI zero dst));
 7509   effect(KILL cr);
 7510   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7511 
 7512   format %{ "negl    $dst\t# int" %}
 7513   ins_encode %{
 7514     __ negl($dst$$Register);
 7515   %}
 7516   ins_pipe(ialu_reg);
 7517 %}
 7518 
 7519 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 7520 %{
 7521   match(Set dst (NegI dst));
 7522   effect(KILL cr);
 7523   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7524 
 7525   format %{ "negl    $dst\t# int" %}
 7526   ins_encode %{
 7527     __ negl($dst$$Register);
 7528   %}
 7529   ins_pipe(ialu_reg);
 7530 %}
 7531 
 7532 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 7533 %{
 7534   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 7535   effect(KILL cr);
 7536   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7537 
 7538   format %{ "negl    $dst\t# int" %}
 7539   ins_encode %{
 7540     __ negl($dst$$Address);
 7541   %}
 7542   ins_pipe(ialu_reg);
 7543 %}
 7544 
 7545 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 7546 %{
 7547   match(Set dst (SubL zero dst));
 7548   effect(KILL cr);
 7549   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7550 
 7551   format %{ "negq    $dst\t# long" %}
 7552   ins_encode %{
 7553     __ negq($dst$$Register);
 7554   %}
 7555   ins_pipe(ialu_reg);
 7556 %}
 7557 
 7558 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 7559 %{
 7560   match(Set dst (NegL dst));
 7561   effect(KILL cr);
 7562   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7563 
 7564   format %{ "negq    $dst\t# int" %}
 7565   ins_encode %{
 7566     __ negq($dst$$Register);
 7567   %}
 7568   ins_pipe(ialu_reg);
 7569 %}
 7570 
 7571 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 7572 %{
 7573   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 7574   effect(KILL cr);
 7575   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7576 
 7577   format %{ "negq    $dst\t# long" %}
 7578   ins_encode %{
 7579     __ negq($dst$$Address);
 7580   %}
 7581   ins_pipe(ialu_reg);
 7582 %}
 7583 
 7584 //----------Multiplication/Division Instructions-------------------------------
 7585 // Integer Multiplication Instructions
 7586 // Multiply Register
 7587 
 7588 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7589 %{
 7590   match(Set dst (MulI dst src));
 7591   effect(KILL cr);
 7592 
 7593   ins_cost(300);
 7594   format %{ "imull   $dst, $src\t# int" %}
 7595   ins_encode %{
 7596     __ imull($dst$$Register, $src$$Register);
 7597   %}
 7598   ins_pipe(ialu_reg_reg_alu0);
 7599 %}
 7600 
 7601 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 7602 %{
 7603   match(Set dst (MulI src imm));
 7604   effect(KILL cr);
 7605 
 7606   ins_cost(300);
 7607   format %{ "imull   $dst, $src, $imm\t# int" %}
 7608   ins_encode %{
 7609     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 7610   %}
 7611   ins_pipe(ialu_reg_reg_alu0);
 7612 %}
 7613 
 7614 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 7615 %{
 7616   match(Set dst (MulI dst (LoadI src)));
 7617   effect(KILL cr);
 7618 
 7619   ins_cost(350);
 7620   format %{ "imull   $dst, $src\t# int" %}
 7621   ins_encode %{
 7622     __ imull($dst$$Register, $src$$Address);
 7623   %}
 7624   ins_pipe(ialu_reg_mem_alu0);
 7625 %}
 7626 
 7627 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 7628 %{
 7629   match(Set dst (MulI (LoadI src) imm));
 7630   effect(KILL cr);
 7631 
 7632   ins_cost(300);
 7633   format %{ "imull   $dst, $src, $imm\t# int" %}
 7634   ins_encode %{
 7635     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 7636   %}
 7637   ins_pipe(ialu_reg_mem_alu0);
 7638 %}
 7639 
 7640 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 7641 %{
 7642   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 7643   effect(KILL cr, KILL src2);
 7644 
 7645   expand %{ mulI_rReg(dst, src1, cr);
 7646            mulI_rReg(src2, src3, cr);
 7647            addI_rReg(dst, src2, cr); %}
 7648 %}
 7649 
 7650 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7651 %{
 7652   match(Set dst (MulL dst src));
 7653   effect(KILL cr);
 7654 
 7655   ins_cost(300);
 7656   format %{ "imulq   $dst, $src\t# long" %}
 7657   ins_encode %{
 7658     __ imulq($dst$$Register, $src$$Register);
 7659   %}
 7660   ins_pipe(ialu_reg_reg_alu0);
 7661 %}
 7662 
 7663 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 7664 %{
 7665   match(Set dst (MulL src imm));
 7666   effect(KILL cr);
 7667 
 7668   ins_cost(300);
 7669   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7670   ins_encode %{
 7671     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 7672   %}
 7673   ins_pipe(ialu_reg_reg_alu0);
 7674 %}
 7675 
 7676 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 7677 %{
 7678   match(Set dst (MulL dst (LoadL src)));
 7679   effect(KILL cr);
 7680 
 7681   ins_cost(350);
 7682   format %{ "imulq   $dst, $src\t# long" %}
 7683   ins_encode %{
 7684     __ imulq($dst$$Register, $src$$Address);
 7685   %}
 7686   ins_pipe(ialu_reg_mem_alu0);
 7687 %}
 7688 
 7689 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 7690 %{
 7691   match(Set dst (MulL (LoadL src) imm));
 7692   effect(KILL cr);
 7693 
 7694   ins_cost(300);
 7695   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7696   ins_encode %{
 7697     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 7698   %}
 7699   ins_pipe(ialu_reg_mem_alu0);
 7700 %}
 7701 
 7702 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7703 %{
 7704   match(Set dst (MulHiL src rax));
 7705   effect(USE_KILL rax, KILL cr);
 7706 
 7707   ins_cost(300);
 7708   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 7709   ins_encode %{
 7710     __ imulq($src$$Register);
 7711   %}
 7712   ins_pipe(ialu_reg_reg_alu0);
 7713 %}
 7714 
 7715 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7716 %{
 7717   match(Set dst (UMulHiL src rax));
 7718   effect(USE_KILL rax, KILL cr);
 7719 
 7720   ins_cost(300);
 7721   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 7722   ins_encode %{
 7723     __ mulq($src$$Register);
 7724   %}
 7725   ins_pipe(ialu_reg_reg_alu0);
 7726 %}
 7727 
 7728 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7729                    rFlagsReg cr)
 7730 %{
 7731   match(Set rax (DivI rax div));
 7732   effect(KILL rdx, KILL cr);
 7733 
 7734   ins_cost(30*100+10*100); // XXX
 7735   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7736             "jne,s   normal\n\t"
 7737             "xorl    rdx, rdx\n\t"
 7738             "cmpl    $div, -1\n\t"
 7739             "je,s    done\n"
 7740     "normal: cdql\n\t"
 7741             "idivl   $div\n"
 7742     "done:"        %}
 7743   ins_encode(cdql_enc(div));
 7744   ins_pipe(ialu_reg_reg_alu0);
 7745 %}
 7746 
 7747 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7748                    rFlagsReg cr)
 7749 %{
 7750   match(Set rax (DivL rax div));
 7751   effect(KILL rdx, KILL cr);
 7752 
 7753   ins_cost(30*100+10*100); // XXX
 7754   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7755             "cmpq    rax, rdx\n\t"
 7756             "jne,s   normal\n\t"
 7757             "xorl    rdx, rdx\n\t"
 7758             "cmpq    $div, -1\n\t"
 7759             "je,s    done\n"
 7760     "normal: cdqq\n\t"
 7761             "idivq   $div\n"
 7762     "done:"        %}
 7763   ins_encode(cdqq_enc(div));
 7764   ins_pipe(ialu_reg_reg_alu0);
 7765 %}
 7766 
 7767 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 7768 %{
 7769   match(Set rax (UDivI rax div));
 7770   effect(KILL rdx, KILL cr);
 7771 
 7772   ins_cost(300);
 7773   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 7774   ins_encode %{
 7775     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 7776   %}
 7777   ins_pipe(ialu_reg_reg_alu0);
 7778 %}
 7779 
 7780 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 7781 %{
 7782   match(Set rax (UDivL rax div));
 7783   effect(KILL rdx, KILL cr);
 7784 
 7785   ins_cost(300);
 7786   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 7787   ins_encode %{
 7788      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 7789   %}
 7790   ins_pipe(ialu_reg_reg_alu0);
 7791 %}
 7792 
 7793 // Integer DIVMOD with Register, both quotient and mod results
 7794 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7795                              rFlagsReg cr)
 7796 %{
 7797   match(DivModI rax div);
 7798   effect(KILL cr);
 7799 
 7800   ins_cost(30*100+10*100); // XXX
 7801   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7802             "jne,s   normal\n\t"
 7803             "xorl    rdx, rdx\n\t"
 7804             "cmpl    $div, -1\n\t"
 7805             "je,s    done\n"
 7806     "normal: cdql\n\t"
 7807             "idivl   $div\n"
 7808     "done:"        %}
 7809   ins_encode(cdql_enc(div));
 7810   ins_pipe(pipe_slow);
 7811 %}
 7812 
 7813 // Long DIVMOD with Register, both quotient and mod results
 7814 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7815                              rFlagsReg cr)
 7816 %{
 7817   match(DivModL rax div);
 7818   effect(KILL cr);
 7819 
 7820   ins_cost(30*100+10*100); // XXX
 7821   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7822             "cmpq    rax, rdx\n\t"
 7823             "jne,s   normal\n\t"
 7824             "xorl    rdx, rdx\n\t"
 7825             "cmpq    $div, -1\n\t"
 7826             "je,s    done\n"
 7827     "normal: cdqq\n\t"
 7828             "idivq   $div\n"
 7829     "done:"        %}
 7830   ins_encode(cdqq_enc(div));
 7831   ins_pipe(pipe_slow);
 7832 %}
 7833 
 7834 // Unsigned integer DIVMOD with Register, both quotient and mod results
 7835 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 7836                               no_rax_rdx_RegI div, rFlagsReg cr)
 7837 %{
 7838   match(UDivModI rax div);
 7839   effect(TEMP tmp, KILL cr);
 7840 
 7841   ins_cost(300);
 7842   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 7843             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 7844           %}
 7845   ins_encode %{
 7846     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 7847   %}
 7848   ins_pipe(pipe_slow);
 7849 %}
 7850 
 7851 // Unsigned long DIVMOD with Register, both quotient and mod results
 7852 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 7853                               no_rax_rdx_RegL div, rFlagsReg cr)
 7854 %{
 7855   match(UDivModL rax div);
 7856   effect(TEMP tmp, KILL cr);
 7857 
 7858   ins_cost(300);
 7859   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 7860             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 7861           %}
 7862   ins_encode %{
 7863     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 7864   %}
 7865   ins_pipe(pipe_slow);
 7866 %}
 7867 
 7868 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 7869                    rFlagsReg cr)
 7870 %{
 7871   match(Set rdx (ModI rax div));
 7872   effect(KILL rax, KILL cr);
 7873 
 7874   ins_cost(300); // XXX
 7875   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 7876             "jne,s   normal\n\t"
 7877             "xorl    rdx, rdx\n\t"
 7878             "cmpl    $div, -1\n\t"
 7879             "je,s    done\n"
 7880     "normal: cdql\n\t"
 7881             "idivl   $div\n"
 7882     "done:"        %}
 7883   ins_encode(cdql_enc(div));
 7884   ins_pipe(ialu_reg_reg_alu0);
 7885 %}
 7886 
 7887 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 7888                    rFlagsReg cr)
 7889 %{
 7890   match(Set rdx (ModL rax div));
 7891   effect(KILL rax, KILL cr);
 7892 
 7893   ins_cost(300); // XXX
 7894   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 7895             "cmpq    rax, rdx\n\t"
 7896             "jne,s   normal\n\t"
 7897             "xorl    rdx, rdx\n\t"
 7898             "cmpq    $div, -1\n\t"
 7899             "je,s    done\n"
 7900     "normal: cdqq\n\t"
 7901             "idivq   $div\n"
 7902     "done:"        %}
 7903   ins_encode(cdqq_enc(div));
 7904   ins_pipe(ialu_reg_reg_alu0);
 7905 %}
 7906 
 7907 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 7908 %{
 7909   match(Set rdx (UModI rax div));
 7910   effect(KILL rax, KILL cr);
 7911 
 7912   ins_cost(300);
 7913   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 7914   ins_encode %{
 7915     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 7916   %}
 7917   ins_pipe(ialu_reg_reg_alu0);
 7918 %}
 7919 
 7920 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 7921 %{
 7922   match(Set rdx (UModL rax div));
 7923   effect(KILL rax, KILL cr);
 7924 
 7925   ins_cost(300);
 7926   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 7927   ins_encode %{
 7928     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 7929   %}
 7930   ins_pipe(ialu_reg_reg_alu0);
 7931 %}
 7932 
 7933 // Integer Shift Instructions
 7934 // Shift Left by one, two, three
 7935 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 7936 %{
 7937   match(Set dst (LShiftI dst shift));
 7938   effect(KILL cr);
 7939 
 7940   format %{ "sall    $dst, $shift" %}
 7941   ins_encode %{
 7942     __ sall($dst$$Register, $shift$$constant);
 7943   %}
 7944   ins_pipe(ialu_reg);
 7945 %}
 7946 
 7947 // Shift Left by 8-bit immediate
 7948 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 7949 %{
 7950   match(Set dst (LShiftI dst shift));
 7951   effect(KILL cr);
 7952 
 7953   format %{ "sall    $dst, $shift" %}
 7954   ins_encode %{
 7955     __ sall($dst$$Register, $shift$$constant);
 7956   %}
 7957   ins_pipe(ialu_reg);
 7958 %}
 7959 
 7960 // Shift Left by 8-bit immediate
 7961 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 7962 %{
 7963   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 7964   effect(KILL cr);
 7965 
 7966   format %{ "sall    $dst, $shift" %}
 7967   ins_encode %{
 7968     __ sall($dst$$Address, $shift$$constant);
 7969   %}
 7970   ins_pipe(ialu_mem_imm);
 7971 %}
 7972 
 7973 // Shift Left by variable
 7974 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 7975 %{
 7976   predicate(!VM_Version::supports_bmi2());
 7977   match(Set dst (LShiftI dst shift));
 7978   effect(KILL cr);
 7979 
 7980   format %{ "sall    $dst, $shift" %}
 7981   ins_encode %{
 7982     __ sall($dst$$Register);
 7983   %}
 7984   ins_pipe(ialu_reg_reg);
 7985 %}
 7986 
 7987 // Shift Left by variable
 7988 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 7989 %{
 7990   predicate(!VM_Version::supports_bmi2());
 7991   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 7992   effect(KILL cr);
 7993 
 7994   format %{ "sall    $dst, $shift" %}
 7995   ins_encode %{
 7996     __ sall($dst$$Address);
 7997   %}
 7998   ins_pipe(ialu_mem_reg);
 7999 %}
 8000 
 8001 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8002 %{
 8003   predicate(VM_Version::supports_bmi2());
 8004   match(Set dst (LShiftI src shift));
 8005 
 8006   format %{ "shlxl   $dst, $src, $shift" %}
 8007   ins_encode %{
 8008     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 8009   %}
 8010   ins_pipe(ialu_reg_reg);
 8011 %}
 8012 
 8013 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8014 %{
 8015   predicate(VM_Version::supports_bmi2());
 8016   match(Set dst (LShiftI (LoadI src) shift));
 8017   ins_cost(175);
 8018   format %{ "shlxl   $dst, $src, $shift" %}
 8019   ins_encode %{
 8020     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 8021   %}
 8022   ins_pipe(ialu_reg_mem);
 8023 %}
 8024 
 8025 // Arithmetic Shift Right by 8-bit immediate
 8026 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8027 %{
 8028   match(Set dst (RShiftI dst shift));
 8029   effect(KILL cr);
 8030 
 8031   format %{ "sarl    $dst, $shift" %}
 8032   ins_encode %{
 8033     __ sarl($dst$$Register, $shift$$constant);
 8034   %}
 8035   ins_pipe(ialu_mem_imm);
 8036 %}
 8037 
 8038 // Arithmetic Shift Right by 8-bit immediate
 8039 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8040 %{
 8041   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8042   effect(KILL cr);
 8043 
 8044   format %{ "sarl    $dst, $shift" %}
 8045   ins_encode %{
 8046     __ sarl($dst$$Address, $shift$$constant);
 8047   %}
 8048   ins_pipe(ialu_mem_imm);
 8049 %}
 8050 
 8051 // Arithmetic Shift Right by variable
 8052 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8053 %{
 8054   predicate(!VM_Version::supports_bmi2());
 8055   match(Set dst (RShiftI dst shift));
 8056   effect(KILL cr);
 8057 
 8058   format %{ "sarl    $dst, $shift" %}
 8059   ins_encode %{
 8060     __ sarl($dst$$Register);
 8061   %}
 8062   ins_pipe(ialu_reg_reg);
 8063 %}
 8064 
 8065 // Arithmetic Shift Right by variable
 8066 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8067 %{
 8068   predicate(!VM_Version::supports_bmi2());
 8069   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8070   effect(KILL cr);
 8071 
 8072   format %{ "sarl    $dst, $shift" %}
 8073   ins_encode %{
 8074     __ sarl($dst$$Address);
 8075   %}
 8076   ins_pipe(ialu_mem_reg);
 8077 %}
 8078 
 8079 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8080 %{
 8081   predicate(VM_Version::supports_bmi2());
 8082   match(Set dst (RShiftI src shift));
 8083 
 8084   format %{ "sarxl   $dst, $src, $shift" %}
 8085   ins_encode %{
 8086     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 8087   %}
 8088   ins_pipe(ialu_reg_reg);
 8089 %}
 8090 
 8091 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8092 %{
 8093   predicate(VM_Version::supports_bmi2());
 8094   match(Set dst (RShiftI (LoadI src) shift));
 8095   ins_cost(175);
 8096   format %{ "sarxl   $dst, $src, $shift" %}
 8097   ins_encode %{
 8098     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 8099   %}
 8100   ins_pipe(ialu_reg_mem);
 8101 %}
 8102 
 8103 // Logical Shift Right by 8-bit immediate
 8104 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8105 %{
 8106   match(Set dst (URShiftI dst shift));
 8107   effect(KILL cr);
 8108 
 8109   format %{ "shrl    $dst, $shift" %}
 8110   ins_encode %{
 8111     __ shrl($dst$$Register, $shift$$constant);
 8112   %}
 8113   ins_pipe(ialu_reg);
 8114 %}
 8115 
 8116 // Logical Shift Right by 8-bit immediate
 8117 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8118 %{
 8119   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8120   effect(KILL cr);
 8121 
 8122   format %{ "shrl    $dst, $shift" %}
 8123   ins_encode %{
 8124     __ shrl($dst$$Address, $shift$$constant);
 8125   %}
 8126   ins_pipe(ialu_mem_imm);
 8127 %}
 8128 
 8129 // Logical Shift Right by variable
 8130 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8131 %{
 8132   predicate(!VM_Version::supports_bmi2());
 8133   match(Set dst (URShiftI dst shift));
 8134   effect(KILL cr);
 8135 
 8136   format %{ "shrl    $dst, $shift" %}
 8137   ins_encode %{
 8138     __ shrl($dst$$Register);
 8139   %}
 8140   ins_pipe(ialu_reg_reg);
 8141 %}
 8142 
 8143 // Logical Shift Right by variable
 8144 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8145 %{
 8146   predicate(!VM_Version::supports_bmi2());
 8147   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8148   effect(KILL cr);
 8149 
 8150   format %{ "shrl    $dst, $shift" %}
 8151   ins_encode %{
 8152     __ shrl($dst$$Address);
 8153   %}
 8154   ins_pipe(ialu_mem_reg);
 8155 %}
 8156 
 8157 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8158 %{
 8159   predicate(VM_Version::supports_bmi2());
 8160   match(Set dst (URShiftI src shift));
 8161 
 8162   format %{ "shrxl   $dst, $src, $shift" %}
 8163   ins_encode %{
 8164     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 8165   %}
 8166   ins_pipe(ialu_reg_reg);
 8167 %}
 8168 
 8169 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8170 %{
 8171   predicate(VM_Version::supports_bmi2());
 8172   match(Set dst (URShiftI (LoadI src) shift));
 8173   ins_cost(175);
 8174   format %{ "shrxl   $dst, $src, $shift" %}
 8175   ins_encode %{
 8176     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 8177   %}
 8178   ins_pipe(ialu_reg_mem);
 8179 %}
 8180 
 8181 // Long Shift Instructions
 8182 // Shift Left by one, two, three
 8183 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 8184 %{
 8185   match(Set dst (LShiftL dst shift));
 8186   effect(KILL cr);
 8187 
 8188   format %{ "salq    $dst, $shift" %}
 8189   ins_encode %{
 8190     __ salq($dst$$Register, $shift$$constant);
 8191   %}
 8192   ins_pipe(ialu_reg);
 8193 %}
 8194 
 8195 // Shift Left by 8-bit immediate
 8196 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8197 %{
 8198   match(Set dst (LShiftL dst shift));
 8199   effect(KILL cr);
 8200 
 8201   format %{ "salq    $dst, $shift" %}
 8202   ins_encode %{
 8203     __ salq($dst$$Register, $shift$$constant);
 8204   %}
 8205   ins_pipe(ialu_reg);
 8206 %}
 8207 
 8208 // Shift Left by 8-bit immediate
 8209 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8210 %{
 8211   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8212   effect(KILL cr);
 8213 
 8214   format %{ "salq    $dst, $shift" %}
 8215   ins_encode %{
 8216     __ salq($dst$$Address, $shift$$constant);
 8217   %}
 8218   ins_pipe(ialu_mem_imm);
 8219 %}
 8220 
 8221 // Shift Left by variable
 8222 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8223 %{
 8224   predicate(!VM_Version::supports_bmi2());
 8225   match(Set dst (LShiftL dst shift));
 8226   effect(KILL cr);
 8227 
 8228   format %{ "salq    $dst, $shift" %}
 8229   ins_encode %{
 8230     __ salq($dst$$Register);
 8231   %}
 8232   ins_pipe(ialu_reg_reg);
 8233 %}
 8234 
 8235 // Shift Left by variable
 8236 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8237 %{
 8238   predicate(!VM_Version::supports_bmi2());
 8239   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8240   effect(KILL cr);
 8241 
 8242   format %{ "salq    $dst, $shift" %}
 8243   ins_encode %{
 8244     __ salq($dst$$Address);
 8245   %}
 8246   ins_pipe(ialu_mem_reg);
 8247 %}
 8248 
 8249 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8250 %{
 8251   predicate(VM_Version::supports_bmi2());
 8252   match(Set dst (LShiftL src shift));
 8253 
 8254   format %{ "shlxq   $dst, $src, $shift" %}
 8255   ins_encode %{
 8256     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 8257   %}
 8258   ins_pipe(ialu_reg_reg);
 8259 %}
 8260 
 8261 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8262 %{
 8263   predicate(VM_Version::supports_bmi2());
 8264   match(Set dst (LShiftL (LoadL src) shift));
 8265   ins_cost(175);
 8266   format %{ "shlxq   $dst, $src, $shift" %}
 8267   ins_encode %{
 8268     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 8269   %}
 8270   ins_pipe(ialu_reg_mem);
 8271 %}
 8272 
 8273 // Arithmetic Shift Right by 8-bit immediate
 8274 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 8275 %{
 8276   match(Set dst (RShiftL dst shift));
 8277   effect(KILL cr);
 8278 
 8279   format %{ "sarq    $dst, $shift" %}
 8280   ins_encode %{
 8281     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 8282   %}
 8283   ins_pipe(ialu_mem_imm);
 8284 %}
 8285 
 8286 // Arithmetic Shift Right by 8-bit immediate
 8287 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 8288 %{
 8289   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8290   effect(KILL cr);
 8291 
 8292   format %{ "sarq    $dst, $shift" %}
 8293   ins_encode %{
 8294     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 8295   %}
 8296   ins_pipe(ialu_mem_imm);
 8297 %}
 8298 
 8299 // Arithmetic Shift Right by variable
 8300 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8301 %{
 8302   predicate(!VM_Version::supports_bmi2());
 8303   match(Set dst (RShiftL dst shift));
 8304   effect(KILL cr);
 8305 
 8306   format %{ "sarq    $dst, $shift" %}
 8307   ins_encode %{
 8308     __ sarq($dst$$Register);
 8309   %}
 8310   ins_pipe(ialu_reg_reg);
 8311 %}
 8312 
 8313 // Arithmetic Shift Right by variable
 8314 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8315 %{
 8316   predicate(!VM_Version::supports_bmi2());
 8317   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8318   effect(KILL cr);
 8319 
 8320   format %{ "sarq    $dst, $shift" %}
 8321   ins_encode %{
 8322     __ sarq($dst$$Address);
 8323   %}
 8324   ins_pipe(ialu_mem_reg);
 8325 %}
 8326 
 8327 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8328 %{
 8329   predicate(VM_Version::supports_bmi2());
 8330   match(Set dst (RShiftL src shift));
 8331 
 8332   format %{ "sarxq   $dst, $src, $shift" %}
 8333   ins_encode %{
 8334     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 8335   %}
 8336   ins_pipe(ialu_reg_reg);
 8337 %}
 8338 
 8339 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8340 %{
 8341   predicate(VM_Version::supports_bmi2());
 8342   match(Set dst (RShiftL (LoadL src) shift));
 8343   ins_cost(175);
 8344   format %{ "sarxq   $dst, $src, $shift" %}
 8345   ins_encode %{
 8346     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 8347   %}
 8348   ins_pipe(ialu_reg_mem);
 8349 %}
 8350 
 8351 // Logical Shift Right by 8-bit immediate
 8352 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8353 %{
 8354   match(Set dst (URShiftL dst shift));
 8355   effect(KILL cr);
 8356 
 8357   format %{ "shrq    $dst, $shift" %}
 8358   ins_encode %{
 8359     __ shrq($dst$$Register, $shift$$constant);
 8360   %}
 8361   ins_pipe(ialu_reg);
 8362 %}
 8363 
 8364 // Logical Shift Right by 8-bit immediate
 8365 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8366 %{
 8367   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8368   effect(KILL cr);
 8369 
 8370   format %{ "shrq    $dst, $shift" %}
 8371   ins_encode %{
 8372     __ shrq($dst$$Address, $shift$$constant);
 8373   %}
 8374   ins_pipe(ialu_mem_imm);
 8375 %}
 8376 
 8377 // Logical Shift Right by variable
 8378 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8379 %{
 8380   predicate(!VM_Version::supports_bmi2());
 8381   match(Set dst (URShiftL dst shift));
 8382   effect(KILL cr);
 8383 
 8384   format %{ "shrq    $dst, $shift" %}
 8385   ins_encode %{
 8386     __ shrq($dst$$Register);
 8387   %}
 8388   ins_pipe(ialu_reg_reg);
 8389 %}
 8390 
 8391 // Logical Shift Right by variable
 8392 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8393 %{
 8394   predicate(!VM_Version::supports_bmi2());
 8395   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8396   effect(KILL cr);
 8397 
 8398   format %{ "shrq    $dst, $shift" %}
 8399   ins_encode %{
 8400     __ shrq($dst$$Address);
 8401   %}
 8402   ins_pipe(ialu_mem_reg);
 8403 %}
 8404 
 8405 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8406 %{
 8407   predicate(VM_Version::supports_bmi2());
 8408   match(Set dst (URShiftL src shift));
 8409 
 8410   format %{ "shrxq   $dst, $src, $shift" %}
 8411   ins_encode %{
 8412     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 8413   %}
 8414   ins_pipe(ialu_reg_reg);
 8415 %}
 8416 
 8417 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8418 %{
 8419   predicate(VM_Version::supports_bmi2());
 8420   match(Set dst (URShiftL (LoadL src) shift));
 8421   ins_cost(175);
 8422   format %{ "shrxq   $dst, $src, $shift" %}
 8423   ins_encode %{
 8424     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 8425   %}
 8426   ins_pipe(ialu_reg_mem);
 8427 %}
 8428 
 8429 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 8430 // This idiom is used by the compiler for the i2b bytecode.
 8431 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 8432 %{
 8433   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 8434 
 8435   format %{ "movsbl  $dst, $src\t# i2b" %}
 8436   ins_encode %{
 8437     __ movsbl($dst$$Register, $src$$Register);
 8438   %}
 8439   ins_pipe(ialu_reg_reg);
 8440 %}
 8441 
 8442 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 8443 // This idiom is used by the compiler the i2s bytecode.
 8444 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 8445 %{
 8446   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 8447 
 8448   format %{ "movswl  $dst, $src\t# i2s" %}
 8449   ins_encode %{
 8450     __ movswl($dst$$Register, $src$$Register);
 8451   %}
 8452   ins_pipe(ialu_reg_reg);
 8453 %}
 8454 
 8455 // ROL/ROR instructions
 8456 
 8457 // Rotate left by constant.
 8458 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8459 %{
 8460   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8461   match(Set dst (RotateLeft dst shift));
 8462   effect(KILL cr);
 8463   format %{ "roll    $dst, $shift" %}
 8464   ins_encode %{
 8465     __ roll($dst$$Register, $shift$$constant);
 8466   %}
 8467   ins_pipe(ialu_reg);
 8468 %}
 8469 
 8470 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 8471 %{
 8472   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8473   match(Set dst (RotateLeft src shift));
 8474   format %{ "rolxl   $dst, $src, $shift" %}
 8475   ins_encode %{
 8476     int shift = 32 - ($shift$$constant & 31);
 8477     __ rorxl($dst$$Register, $src$$Register, shift);
 8478   %}
 8479   ins_pipe(ialu_reg_reg);
 8480 %}
 8481 
 8482 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8483 %{
 8484   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8485   match(Set dst (RotateLeft (LoadI src) shift));
 8486   ins_cost(175);
 8487   format %{ "rolxl   $dst, $src, $shift" %}
 8488   ins_encode %{
 8489     int shift = 32 - ($shift$$constant & 31);
 8490     __ rorxl($dst$$Register, $src$$Address, shift);
 8491   %}
 8492   ins_pipe(ialu_reg_mem);
 8493 %}
 8494 
 8495 // Rotate Left by variable
 8496 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8497 %{
 8498   predicate(n->bottom_type()->basic_type() == T_INT);
 8499   match(Set dst (RotateLeft dst shift));
 8500   effect(KILL cr);
 8501   format %{ "roll    $dst, $shift" %}
 8502   ins_encode %{
 8503     __ roll($dst$$Register);
 8504   %}
 8505   ins_pipe(ialu_reg_reg);
 8506 %}
 8507 
 8508 // Rotate Right by constant.
 8509 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8510 %{
 8511   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8512   match(Set dst (RotateRight dst shift));
 8513   effect(KILL cr);
 8514   format %{ "rorl    $dst, $shift" %}
 8515   ins_encode %{
 8516     __ rorl($dst$$Register, $shift$$constant);
 8517   %}
 8518   ins_pipe(ialu_reg);
 8519 %}
 8520 
 8521 // Rotate Right by constant.
 8522 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 8523 %{
 8524   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8525   match(Set dst (RotateRight src shift));
 8526   format %{ "rorxl   $dst, $src, $shift" %}
 8527   ins_encode %{
 8528     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 8529   %}
 8530   ins_pipe(ialu_reg_reg);
 8531 %}
 8532 
 8533 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8534 %{
 8535   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8536   match(Set dst (RotateRight (LoadI src) shift));
 8537   ins_cost(175);
 8538   format %{ "rorxl   $dst, $src, $shift" %}
 8539   ins_encode %{
 8540     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 8541   %}
 8542   ins_pipe(ialu_reg_mem);
 8543 %}
 8544 
 8545 // Rotate Right by variable
 8546 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8547 %{
 8548   predicate(n->bottom_type()->basic_type() == T_INT);
 8549   match(Set dst (RotateRight dst shift));
 8550   effect(KILL cr);
 8551   format %{ "rorl    $dst, $shift" %}
 8552   ins_encode %{
 8553     __ rorl($dst$$Register);
 8554   %}
 8555   ins_pipe(ialu_reg_reg);
 8556 %}
 8557 
 8558 // Rotate Left by constant.
 8559 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8560 %{
 8561   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8562   match(Set dst (RotateLeft dst shift));
 8563   effect(KILL cr);
 8564   format %{ "rolq    $dst, $shift" %}
 8565   ins_encode %{
 8566     __ rolq($dst$$Register, $shift$$constant);
 8567   %}
 8568   ins_pipe(ialu_reg);
 8569 %}
 8570 
 8571 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 8572 %{
 8573   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8574   match(Set dst (RotateLeft src shift));
 8575   format %{ "rolxq   $dst, $src, $shift" %}
 8576   ins_encode %{
 8577     int shift = 64 - ($shift$$constant & 63);
 8578     __ rorxq($dst$$Register, $src$$Register, shift);
 8579   %}
 8580   ins_pipe(ialu_reg_reg);
 8581 %}
 8582 
 8583 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8584 %{
 8585   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8586   match(Set dst (RotateLeft (LoadL src) shift));
 8587   ins_cost(175);
 8588   format %{ "rolxq   $dst, $src, $shift" %}
 8589   ins_encode %{
 8590     int shift = 64 - ($shift$$constant & 63);
 8591     __ rorxq($dst$$Register, $src$$Address, shift);
 8592   %}
 8593   ins_pipe(ialu_reg_mem);
 8594 %}
 8595 
 8596 // Rotate Left by variable
 8597 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8598 %{
 8599   predicate(n->bottom_type()->basic_type() == T_LONG);
 8600   match(Set dst (RotateLeft dst shift));
 8601   effect(KILL cr);
 8602   format %{ "rolq    $dst, $shift" %}
 8603   ins_encode %{
 8604     __ rolq($dst$$Register);
 8605   %}
 8606   ins_pipe(ialu_reg_reg);
 8607 %}
 8608 
 8609 // Rotate Right by constant.
 8610 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8611 %{
 8612   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8613   match(Set dst (RotateRight dst shift));
 8614   effect(KILL cr);
 8615   format %{ "rorq    $dst, $shift" %}
 8616   ins_encode %{
 8617     __ rorq($dst$$Register, $shift$$constant);
 8618   %}
 8619   ins_pipe(ialu_reg);
 8620 %}
 8621 
 8622 // Rotate Right by constant
 8623 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 8624 %{
 8625   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8626   match(Set dst (RotateRight src shift));
 8627   format %{ "rorxq   $dst, $src, $shift" %}
 8628   ins_encode %{
 8629     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 8630   %}
 8631   ins_pipe(ialu_reg_reg);
 8632 %}
 8633 
 8634 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8635 %{
 8636   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8637   match(Set dst (RotateRight (LoadL src) shift));
 8638   ins_cost(175);
 8639   format %{ "rorxq   $dst, $src, $shift" %}
 8640   ins_encode %{
 8641     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 8642   %}
 8643   ins_pipe(ialu_reg_mem);
 8644 %}
 8645 
 8646 // Rotate Right by variable
 8647 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8648 %{
 8649   predicate(n->bottom_type()->basic_type() == T_LONG);
 8650   match(Set dst (RotateRight dst shift));
 8651   effect(KILL cr);
 8652   format %{ "rorq    $dst, $shift" %}
 8653   ins_encode %{
 8654     __ rorq($dst$$Register);
 8655   %}
 8656   ins_pipe(ialu_reg_reg);
 8657 %}
 8658 
 8659 //----------------------------- CompressBits/ExpandBits ------------------------
 8660 
 8661 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8662   predicate(n->bottom_type()->isa_long());
 8663   match(Set dst (CompressBits src mask));
 8664   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8665   ins_encode %{
 8666     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 8667   %}
 8668   ins_pipe( pipe_slow );
 8669 %}
 8670 
 8671 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8672   predicate(n->bottom_type()->isa_long());
 8673   match(Set dst (ExpandBits src mask));
 8674   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8675   ins_encode %{
 8676     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 8677   %}
 8678   ins_pipe( pipe_slow );
 8679 %}
 8680 
 8681 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8682   predicate(n->bottom_type()->isa_long());
 8683   match(Set dst (CompressBits src (LoadL mask)));
 8684   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8685   ins_encode %{
 8686     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 8687   %}
 8688   ins_pipe( pipe_slow );
 8689 %}
 8690 
 8691 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8692   predicate(n->bottom_type()->isa_long());
 8693   match(Set dst (ExpandBits src (LoadL mask)));
 8694   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8695   ins_encode %{
 8696     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 8697   %}
 8698   ins_pipe( pipe_slow );
 8699 %}
 8700 
 8701 
 8702 // Logical Instructions
 8703 
 8704 // Integer Logical Instructions
 8705 
 8706 // And Instructions
 8707 // And Register with Register
 8708 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8709 %{
 8710   match(Set dst (AndI dst src));
 8711   effect(KILL cr);
 8712   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8713 
 8714   format %{ "andl    $dst, $src\t# int" %}
 8715   ins_encode %{
 8716     __ andl($dst$$Register, $src$$Register);
 8717   %}
 8718   ins_pipe(ialu_reg_reg);
 8719 %}
 8720 
 8721 // And Register with Immediate 255
 8722 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 8723 %{
 8724   match(Set dst (AndI src mask));
 8725 
 8726   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 8727   ins_encode %{
 8728     __ movzbl($dst$$Register, $src$$Register);
 8729   %}
 8730   ins_pipe(ialu_reg);
 8731 %}
 8732 
 8733 // And Register with Immediate 255 and promote to long
 8734 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 8735 %{
 8736   match(Set dst (ConvI2L (AndI src mask)));
 8737 
 8738   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 8739   ins_encode %{
 8740     __ movzbl($dst$$Register, $src$$Register);
 8741   %}
 8742   ins_pipe(ialu_reg);
 8743 %}
 8744 
 8745 // And Register with Immediate 65535
 8746 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 8747 %{
 8748   match(Set dst (AndI src mask));
 8749 
 8750   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 8751   ins_encode %{
 8752     __ movzwl($dst$$Register, $src$$Register);
 8753   %}
 8754   ins_pipe(ialu_reg);
 8755 %}
 8756 
 8757 // And Register with Immediate 65535 and promote to long
 8758 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 8759 %{
 8760   match(Set dst (ConvI2L (AndI src mask)));
 8761 
 8762   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 8763   ins_encode %{
 8764     __ movzwl($dst$$Register, $src$$Register);
 8765   %}
 8766   ins_pipe(ialu_reg);
 8767 %}
 8768 
 8769 // Can skip int2long conversions after AND with small bitmask
 8770 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 8771 %{
 8772   predicate(VM_Version::supports_bmi2());
 8773   ins_cost(125);
 8774   effect(TEMP tmp, KILL cr);
 8775   match(Set dst (ConvI2L (AndI src mask)));
 8776   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 8777   ins_encode %{
 8778     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 8779     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 8780   %}
 8781   ins_pipe(ialu_reg_reg);
 8782 %}
 8783 
 8784 // And Register with Immediate
 8785 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8786 %{
 8787   match(Set dst (AndI dst src));
 8788   effect(KILL cr);
 8789   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8790 
 8791   format %{ "andl    $dst, $src\t# int" %}
 8792   ins_encode %{
 8793     __ andl($dst$$Register, $src$$constant);
 8794   %}
 8795   ins_pipe(ialu_reg);
 8796 %}
 8797 
 8798 // And Register with Memory
 8799 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8800 %{
 8801   match(Set dst (AndI dst (LoadI src)));
 8802   effect(KILL cr);
 8803   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8804 
 8805   ins_cost(150);
 8806   format %{ "andl    $dst, $src\t# int" %}
 8807   ins_encode %{
 8808     __ andl($dst$$Register, $src$$Address);
 8809   %}
 8810   ins_pipe(ialu_reg_mem);
 8811 %}
 8812 
 8813 // And Memory with Register
 8814 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8815 %{
 8816   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 8817   effect(KILL cr);
 8818   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8819 
 8820   ins_cost(150);
 8821   format %{ "andb    $dst, $src\t# byte" %}
 8822   ins_encode %{
 8823     __ andb($dst$$Address, $src$$Register);
 8824   %}
 8825   ins_pipe(ialu_mem_reg);
 8826 %}
 8827 
 8828 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8829 %{
 8830   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 8831   effect(KILL cr);
 8832   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8833 
 8834   ins_cost(150);
 8835   format %{ "andl    $dst, $src\t# int" %}
 8836   ins_encode %{
 8837     __ andl($dst$$Address, $src$$Register);
 8838   %}
 8839   ins_pipe(ialu_mem_reg);
 8840 %}
 8841 
 8842 // And Memory with Immediate
 8843 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8844 %{
 8845   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 8846   effect(KILL cr);
 8847   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8848 
 8849   ins_cost(125);
 8850   format %{ "andl    $dst, $src\t# int" %}
 8851   ins_encode %{
 8852     __ andl($dst$$Address, $src$$constant);
 8853   %}
 8854   ins_pipe(ialu_mem_imm);
 8855 %}
 8856 
 8857 // BMI1 instructions
 8858 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 8859   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 8860   predicate(UseBMI1Instructions);
 8861   effect(KILL cr);
 8862   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8863 
 8864   ins_cost(125);
 8865   format %{ "andnl  $dst, $src1, $src2" %}
 8866 
 8867   ins_encode %{
 8868     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 8869   %}
 8870   ins_pipe(ialu_reg_mem);
 8871 %}
 8872 
 8873 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 8874   match(Set dst (AndI (XorI src1 minus_1) src2));
 8875   predicate(UseBMI1Instructions);
 8876   effect(KILL cr);
 8877   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8878 
 8879   format %{ "andnl  $dst, $src1, $src2" %}
 8880 
 8881   ins_encode %{
 8882     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 8883   %}
 8884   ins_pipe(ialu_reg);
 8885 %}
 8886 
 8887 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 8888   match(Set dst (AndI (SubI imm_zero src) src));
 8889   predicate(UseBMI1Instructions);
 8890   effect(KILL cr);
 8891   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 8892 
 8893   format %{ "blsil  $dst, $src" %}
 8894 
 8895   ins_encode %{
 8896     __ blsil($dst$$Register, $src$$Register);
 8897   %}
 8898   ins_pipe(ialu_reg);
 8899 %}
 8900 
 8901 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 8902   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 8903   predicate(UseBMI1Instructions);
 8904   effect(KILL cr);
 8905   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 8906 
 8907   ins_cost(125);
 8908   format %{ "blsil  $dst, $src" %}
 8909 
 8910   ins_encode %{
 8911     __ blsil($dst$$Register, $src$$Address);
 8912   %}
 8913   ins_pipe(ialu_reg_mem);
 8914 %}
 8915 
 8916 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 8917 %{
 8918   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 8919   predicate(UseBMI1Instructions);
 8920   effect(KILL cr);
 8921   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 8922 
 8923   ins_cost(125);
 8924   format %{ "blsmskl $dst, $src" %}
 8925 
 8926   ins_encode %{
 8927     __ blsmskl($dst$$Register, $src$$Address);
 8928   %}
 8929   ins_pipe(ialu_reg_mem);
 8930 %}
 8931 
 8932 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 8933 %{
 8934   match(Set dst (XorI (AddI src minus_1) src));
 8935   predicate(UseBMI1Instructions);
 8936   effect(KILL cr);
 8937   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 8938 
 8939   format %{ "blsmskl $dst, $src" %}
 8940 
 8941   ins_encode %{
 8942     __ blsmskl($dst$$Register, $src$$Register);
 8943   %}
 8944 
 8945   ins_pipe(ialu_reg);
 8946 %}
 8947 
 8948 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 8949 %{
 8950   match(Set dst (AndI (AddI src minus_1) src) );
 8951   predicate(UseBMI1Instructions);
 8952   effect(KILL cr);
 8953   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 8954 
 8955   format %{ "blsrl  $dst, $src" %}
 8956 
 8957   ins_encode %{
 8958     __ blsrl($dst$$Register, $src$$Register);
 8959   %}
 8960 
 8961   ins_pipe(ialu_reg_mem);
 8962 %}
 8963 
 8964 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 8965 %{
 8966   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 8967   predicate(UseBMI1Instructions);
 8968   effect(KILL cr);
 8969   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 8970 
 8971   ins_cost(125);
 8972   format %{ "blsrl  $dst, $src" %}
 8973 
 8974   ins_encode %{
 8975     __ blsrl($dst$$Register, $src$$Address);
 8976   %}
 8977 
 8978   ins_pipe(ialu_reg);
 8979 %}
 8980 
 8981 // Or Instructions
 8982 // Or Register with Register
 8983 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8984 %{
 8985   match(Set dst (OrI dst src));
 8986   effect(KILL cr);
 8987   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8988 
 8989   format %{ "orl     $dst, $src\t# int" %}
 8990   ins_encode %{
 8991     __ orl($dst$$Register, $src$$Register);
 8992   %}
 8993   ins_pipe(ialu_reg_reg);
 8994 %}
 8995 
 8996 // Or Register with Immediate
 8997 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8998 %{
 8999   match(Set dst (OrI dst src));
 9000   effect(KILL cr);
 9001   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9002 
 9003   format %{ "orl     $dst, $src\t# int" %}
 9004   ins_encode %{
 9005     __ orl($dst$$Register, $src$$constant);
 9006   %}
 9007   ins_pipe(ialu_reg);
 9008 %}
 9009 
 9010 // Or Register with Memory
 9011 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9012 %{
 9013   match(Set dst (OrI dst (LoadI src)));
 9014   effect(KILL cr);
 9015   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9016 
 9017   ins_cost(150);
 9018   format %{ "orl     $dst, $src\t# int" %}
 9019   ins_encode %{
 9020     __ orl($dst$$Register, $src$$Address);
 9021   %}
 9022   ins_pipe(ialu_reg_mem);
 9023 %}
 9024 
 9025 // Or Memory with Register
 9026 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9027 %{
 9028   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9029   effect(KILL cr);
 9030   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9031 
 9032   ins_cost(150);
 9033   format %{ "orb    $dst, $src\t# byte" %}
 9034   ins_encode %{
 9035     __ orb($dst$$Address, $src$$Register);
 9036   %}
 9037   ins_pipe(ialu_mem_reg);
 9038 %}
 9039 
 9040 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9041 %{
 9042   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9043   effect(KILL cr);
 9044   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9045 
 9046   ins_cost(150);
 9047   format %{ "orl     $dst, $src\t# int" %}
 9048   ins_encode %{
 9049     __ orl($dst$$Address, $src$$Register);
 9050   %}
 9051   ins_pipe(ialu_mem_reg);
 9052 %}
 9053 
 9054 // Or Memory with Immediate
 9055 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9056 %{
 9057   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9058   effect(KILL cr);
 9059   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9060 
 9061   ins_cost(125);
 9062   format %{ "orl     $dst, $src\t# int" %}
 9063   ins_encode %{
 9064     __ orl($dst$$Address, $src$$constant);
 9065   %}
 9066   ins_pipe(ialu_mem_imm);
 9067 %}
 9068 
 9069 // Xor Instructions
 9070 // Xor Register with Register
 9071 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9072 %{
 9073   match(Set dst (XorI dst src));
 9074   effect(KILL cr);
 9075   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9076 
 9077   format %{ "xorl    $dst, $src\t# int" %}
 9078   ins_encode %{
 9079     __ xorl($dst$$Register, $src$$Register);
 9080   %}
 9081   ins_pipe(ialu_reg_reg);
 9082 %}
 9083 
 9084 // Xor Register with Immediate -1
 9085 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9086   match(Set dst (XorI dst imm));
 9087 
 9088   format %{ "not    $dst" %}
 9089   ins_encode %{
 9090      __ notl($dst$$Register);
 9091   %}
 9092   ins_pipe(ialu_reg);
 9093 %}
 9094 
 9095 // Xor Register with Immediate
 9096 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9097 %{
 9098   match(Set dst (XorI dst src));
 9099   effect(KILL cr);
 9100   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9101 
 9102   format %{ "xorl    $dst, $src\t# int" %}
 9103   ins_encode %{
 9104     __ xorl($dst$$Register, $src$$constant);
 9105   %}
 9106   ins_pipe(ialu_reg);
 9107 %}
 9108 
 9109 // Xor Register with Memory
 9110 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9111 %{
 9112   match(Set dst (XorI dst (LoadI src)));
 9113   effect(KILL cr);
 9114   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9115 
 9116   ins_cost(150);
 9117   format %{ "xorl    $dst, $src\t# int" %}
 9118   ins_encode %{
 9119     __ xorl($dst$$Register, $src$$Address);
 9120   %}
 9121   ins_pipe(ialu_reg_mem);
 9122 %}
 9123 
 9124 // Xor Memory with Register
 9125 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9126 %{
 9127   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9128   effect(KILL cr);
 9129   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9130 
 9131   ins_cost(150);
 9132   format %{ "xorb    $dst, $src\t# byte" %}
 9133   ins_encode %{
 9134     __ xorb($dst$$Address, $src$$Register);
 9135   %}
 9136   ins_pipe(ialu_mem_reg);
 9137 %}
 9138 
 9139 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9140 %{
 9141   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9142   effect(KILL cr);
 9143   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9144 
 9145   ins_cost(150);
 9146   format %{ "xorl    $dst, $src\t# int" %}
 9147   ins_encode %{
 9148     __ xorl($dst$$Address, $src$$Register);
 9149   %}
 9150   ins_pipe(ialu_mem_reg);
 9151 %}
 9152 
 9153 // Xor Memory with Immediate
 9154 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9155 %{
 9156   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9157   effect(KILL cr);
 9158   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9159 
 9160   ins_cost(125);
 9161   format %{ "xorl    $dst, $src\t# int" %}
 9162   ins_encode %{
 9163     __ xorl($dst$$Address, $src$$constant);
 9164   %}
 9165   ins_pipe(ialu_mem_imm);
 9166 %}
 9167 
 9168 
 9169 // Long Logical Instructions
 9170 
 9171 // And Instructions
 9172 // And Register with Register
 9173 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9174 %{
 9175   match(Set dst (AndL dst src));
 9176   effect(KILL cr);
 9177   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9178 
 9179   format %{ "andq    $dst, $src\t# long" %}
 9180   ins_encode %{
 9181     __ andq($dst$$Register, $src$$Register);
 9182   %}
 9183   ins_pipe(ialu_reg_reg);
 9184 %}
 9185 
 9186 // And Register with Immediate 255
 9187 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
 9188 %{
 9189   match(Set dst (AndL src mask));
 9190 
 9191   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
 9192   ins_encode %{
 9193     // movzbl zeroes out the upper 32-bit and does not need REX.W
 9194     __ movzbl($dst$$Register, $src$$Register);
 9195   %}
 9196   ins_pipe(ialu_reg);
 9197 %}
 9198 
 9199 // And Register with Immediate 65535
 9200 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
 9201 %{
 9202   match(Set dst (AndL src mask));
 9203 
 9204   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
 9205   ins_encode %{
 9206     // movzwl zeroes out the upper 32-bit and does not need REX.W
 9207     __ movzwl($dst$$Register, $src$$Register);
 9208   %}
 9209   ins_pipe(ialu_reg);
 9210 %}
 9211 
 9212 // And Register with Immediate
 9213 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9214 %{
 9215   match(Set dst (AndL dst src));
 9216   effect(KILL cr);
 9217   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9218 
 9219   format %{ "andq    $dst, $src\t# long" %}
 9220   ins_encode %{
 9221     __ andq($dst$$Register, $src$$constant);
 9222   %}
 9223   ins_pipe(ialu_reg);
 9224 %}
 9225 
 9226 // And Register with Memory
 9227 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9228 %{
 9229   match(Set dst (AndL dst (LoadL src)));
 9230   effect(KILL cr);
 9231   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9232 
 9233   ins_cost(150);
 9234   format %{ "andq    $dst, $src\t# long" %}
 9235   ins_encode %{
 9236     __ andq($dst$$Register, $src$$Address);
 9237   %}
 9238   ins_pipe(ialu_reg_mem);
 9239 %}
 9240 
 9241 // And Memory with Register
 9242 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9243 %{
 9244   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9245   effect(KILL cr);
 9246   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9247 
 9248   ins_cost(150);
 9249   format %{ "andq    $dst, $src\t# long" %}
 9250   ins_encode %{
 9251     __ andq($dst$$Address, $src$$Register);
 9252   %}
 9253   ins_pipe(ialu_mem_reg);
 9254 %}
 9255 
 9256 // And Memory with Immediate
 9257 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9258 %{
 9259   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9260   effect(KILL cr);
 9261   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9262 
 9263   ins_cost(125);
 9264   format %{ "andq    $dst, $src\t# long" %}
 9265   ins_encode %{
 9266     __ andq($dst$$Address, $src$$constant);
 9267   %}
 9268   ins_pipe(ialu_mem_imm);
 9269 %}
 9270 
 9271 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9272 %{
 9273   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9274   // because AND/OR works well enough for 8/32-bit values.
 9275   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9276 
 9277   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9278   effect(KILL cr);
 9279 
 9280   ins_cost(125);
 9281   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9282   ins_encode %{
 9283     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9284   %}
 9285   ins_pipe(ialu_mem_imm);
 9286 %}
 9287 
 9288 // BMI1 instructions
 9289 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9290   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9291   predicate(UseBMI1Instructions);
 9292   effect(KILL cr);
 9293   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9294 
 9295   ins_cost(125);
 9296   format %{ "andnq  $dst, $src1, $src2" %}
 9297 
 9298   ins_encode %{
 9299     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9300   %}
 9301   ins_pipe(ialu_reg_mem);
 9302 %}
 9303 
 9304 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9305   match(Set dst (AndL (XorL src1 minus_1) src2));
 9306   predicate(UseBMI1Instructions);
 9307   effect(KILL cr);
 9308   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9309 
 9310   format %{ "andnq  $dst, $src1, $src2" %}
 9311 
 9312   ins_encode %{
 9313   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9314   %}
 9315   ins_pipe(ialu_reg_mem);
 9316 %}
 9317 
 9318 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9319   match(Set dst (AndL (SubL imm_zero src) src));
 9320   predicate(UseBMI1Instructions);
 9321   effect(KILL cr);
 9322   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9323 
 9324   format %{ "blsiq  $dst, $src" %}
 9325 
 9326   ins_encode %{
 9327     __ blsiq($dst$$Register, $src$$Register);
 9328   %}
 9329   ins_pipe(ialu_reg);
 9330 %}
 9331 
 9332 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9333   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9334   predicate(UseBMI1Instructions);
 9335   effect(KILL cr);
 9336   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9337 
 9338   ins_cost(125);
 9339   format %{ "blsiq  $dst, $src" %}
 9340 
 9341   ins_encode %{
 9342     __ blsiq($dst$$Register, $src$$Address);
 9343   %}
 9344   ins_pipe(ialu_reg_mem);
 9345 %}
 9346 
 9347 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9348 %{
 9349   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9350   predicate(UseBMI1Instructions);
 9351   effect(KILL cr);
 9352   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9353 
 9354   ins_cost(125);
 9355   format %{ "blsmskq $dst, $src" %}
 9356 
 9357   ins_encode %{
 9358     __ blsmskq($dst$$Register, $src$$Address);
 9359   %}
 9360   ins_pipe(ialu_reg_mem);
 9361 %}
 9362 
 9363 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9364 %{
 9365   match(Set dst (XorL (AddL src minus_1) src));
 9366   predicate(UseBMI1Instructions);
 9367   effect(KILL cr);
 9368   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9369 
 9370   format %{ "blsmskq $dst, $src" %}
 9371 
 9372   ins_encode %{
 9373     __ blsmskq($dst$$Register, $src$$Register);
 9374   %}
 9375 
 9376   ins_pipe(ialu_reg);
 9377 %}
 9378 
 9379 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9380 %{
 9381   match(Set dst (AndL (AddL src minus_1) src) );
 9382   predicate(UseBMI1Instructions);
 9383   effect(KILL cr);
 9384   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9385 
 9386   format %{ "blsrq  $dst, $src" %}
 9387 
 9388   ins_encode %{
 9389     __ blsrq($dst$$Register, $src$$Register);
 9390   %}
 9391 
 9392   ins_pipe(ialu_reg);
 9393 %}
 9394 
 9395 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9396 %{
 9397   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9398   predicate(UseBMI1Instructions);
 9399   effect(KILL cr);
 9400   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9401 
 9402   ins_cost(125);
 9403   format %{ "blsrq  $dst, $src" %}
 9404 
 9405   ins_encode %{
 9406     __ blsrq($dst$$Register, $src$$Address);
 9407   %}
 9408 
 9409   ins_pipe(ialu_reg);
 9410 %}
 9411 
 9412 // Or Instructions
 9413 // Or Register with Register
 9414 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9415 %{
 9416   match(Set dst (OrL dst src));
 9417   effect(KILL cr);
 9418   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9419 
 9420   format %{ "orq     $dst, $src\t# long" %}
 9421   ins_encode %{
 9422     __ orq($dst$$Register, $src$$Register);
 9423   %}
 9424   ins_pipe(ialu_reg_reg);
 9425 %}
 9426 
 9427 // Use any_RegP to match R15 (TLS register) without spilling.
 9428 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 9429   match(Set dst (OrL dst (CastP2X src)));
 9430   effect(KILL cr);
 9431   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9432 
 9433   format %{ "orq     $dst, $src\t# long" %}
 9434   ins_encode %{
 9435     __ orq($dst$$Register, $src$$Register);
 9436   %}
 9437   ins_pipe(ialu_reg_reg);
 9438 %}
 9439 
 9440 
 9441 // Or Register with Immediate
 9442 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9443 %{
 9444   match(Set dst (OrL dst src));
 9445   effect(KILL cr);
 9446   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9447 
 9448   format %{ "orq     $dst, $src\t# long" %}
 9449   ins_encode %{
 9450     __ orq($dst$$Register, $src$$constant);
 9451   %}
 9452   ins_pipe(ialu_reg);
 9453 %}
 9454 
 9455 // Or Register with Memory
 9456 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9457 %{
 9458   match(Set dst (OrL dst (LoadL src)));
 9459   effect(KILL cr);
 9460   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9461 
 9462   ins_cost(150);
 9463   format %{ "orq     $dst, $src\t# long" %}
 9464   ins_encode %{
 9465     __ orq($dst$$Register, $src$$Address);
 9466   %}
 9467   ins_pipe(ialu_reg_mem);
 9468 %}
 9469 
 9470 // Or Memory with Register
 9471 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9472 %{
 9473   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9474   effect(KILL cr);
 9475   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9476 
 9477   ins_cost(150);
 9478   format %{ "orq     $dst, $src\t# long" %}
 9479   ins_encode %{
 9480     __ orq($dst$$Address, $src$$Register);
 9481   %}
 9482   ins_pipe(ialu_mem_reg);
 9483 %}
 9484 
 9485 // Or Memory with Immediate
 9486 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9487 %{
 9488   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9489   effect(KILL cr);
 9490   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9491 
 9492   ins_cost(125);
 9493   format %{ "orq     $dst, $src\t# long" %}
 9494   ins_encode %{
 9495     __ orq($dst$$Address, $src$$constant);
 9496   %}
 9497   ins_pipe(ialu_mem_imm);
 9498 %}
 9499 
 9500 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
 9501 %{
 9502   // con should be a pure 64-bit power of 2 immediate
 9503   // because AND/OR works well enough for 8/32-bit values.
 9504   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
 9505 
 9506   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
 9507   effect(KILL cr);
 9508 
 9509   ins_cost(125);
 9510   format %{ "btsq    $dst, log2($con)\t# long" %}
 9511   ins_encode %{
 9512     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
 9513   %}
 9514   ins_pipe(ialu_mem_imm);
 9515 %}
 9516 
 9517 // Xor Instructions
 9518 // Xor Register with Register
 9519 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9520 %{
 9521   match(Set dst (XorL dst src));
 9522   effect(KILL cr);
 9523   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9524 
 9525   format %{ "xorq    $dst, $src\t# long" %}
 9526   ins_encode %{
 9527     __ xorq($dst$$Register, $src$$Register);
 9528   %}
 9529   ins_pipe(ialu_reg_reg);
 9530 %}
 9531 
 9532 // Xor Register with Immediate -1
 9533 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 9534   match(Set dst (XorL dst imm));
 9535 
 9536   format %{ "notq   $dst" %}
 9537   ins_encode %{
 9538      __ notq($dst$$Register);
 9539   %}
 9540   ins_pipe(ialu_reg);
 9541 %}
 9542 
 9543 // Xor Register with Immediate
 9544 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9545 %{
 9546   match(Set dst (XorL dst src));
 9547   effect(KILL cr);
 9548   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9549 
 9550   format %{ "xorq    $dst, $src\t# long" %}
 9551   ins_encode %{
 9552     __ xorq($dst$$Register, $src$$constant);
 9553   %}
 9554   ins_pipe(ialu_reg);
 9555 %}
 9556 
 9557 // Xor Register with Memory
 9558 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9559 %{
 9560   match(Set dst (XorL dst (LoadL src)));
 9561   effect(KILL cr);
 9562   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9563 
 9564   ins_cost(150);
 9565   format %{ "xorq    $dst, $src\t# long" %}
 9566   ins_encode %{
 9567     __ xorq($dst$$Register, $src$$Address);
 9568   %}
 9569   ins_pipe(ialu_reg_mem);
 9570 %}
 9571 
 9572 // Xor Memory with Register
 9573 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9574 %{
 9575   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9576   effect(KILL cr);
 9577   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9578 
 9579   ins_cost(150);
 9580   format %{ "xorq    $dst, $src\t# long" %}
 9581   ins_encode %{
 9582     __ xorq($dst$$Address, $src$$Register);
 9583   %}
 9584   ins_pipe(ialu_mem_reg);
 9585 %}
 9586 
 9587 // Xor Memory with Immediate
 9588 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9589 %{
 9590   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9591   effect(KILL cr);
 9592   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9593 
 9594   ins_cost(125);
 9595   format %{ "xorq    $dst, $src\t# long" %}
 9596   ins_encode %{
 9597     __ xorq($dst$$Address, $src$$constant);
 9598   %}
 9599   ins_pipe(ialu_mem_imm);
 9600 %}
 9601 
 9602 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 9603 %{
 9604   match(Set dst (CmpLTMask p q));
 9605   effect(KILL cr);
 9606 
 9607   ins_cost(400);
 9608   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 9609             "setlt   $dst\n\t"
 9610             "movzbl  $dst, $dst\n\t"
 9611             "negl    $dst" %}
 9612   ins_encode %{
 9613     __ cmpl($p$$Register, $q$$Register);
 9614     __ setb(Assembler::less, $dst$$Register);
 9615     __ movzbl($dst$$Register, $dst$$Register);
 9616     __ negl($dst$$Register);
 9617   %}
 9618   ins_pipe(pipe_slow);
 9619 %}
 9620 
 9621 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
 9622 %{
 9623   match(Set dst (CmpLTMask dst zero));
 9624   effect(KILL cr);
 9625 
 9626   ins_cost(100);
 9627   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 9628   ins_encode %{
 9629     __ sarl($dst$$Register, 31);
 9630   %}
 9631   ins_pipe(ialu_reg);
 9632 %}
 9633 
 9634 /* Better to save a register than avoid a branch */
 9635 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9636 %{
 9637   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 9638   effect(KILL cr);
 9639   ins_cost(300);
 9640   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
 9641             "jge     done\n\t"
 9642             "addl    $p,$y\n"
 9643             "done:   " %}
 9644   ins_encode %{
 9645     Register Rp = $p$$Register;
 9646     Register Rq = $q$$Register;
 9647     Register Ry = $y$$Register;
 9648     Label done;
 9649     __ subl(Rp, Rq);
 9650     __ jccb(Assembler::greaterEqual, done);
 9651     __ addl(Rp, Ry);
 9652     __ bind(done);
 9653   %}
 9654   ins_pipe(pipe_cmplt);
 9655 %}
 9656 
 9657 /* Better to save a register than avoid a branch */
 9658 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9659 %{
 9660   match(Set y (AndI (CmpLTMask p q) y));
 9661   effect(KILL cr);
 9662 
 9663   ins_cost(300);
 9664 
 9665   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
 9666             "jlt     done\n\t"
 9667             "xorl    $y, $y\n"
 9668             "done:   " %}
 9669   ins_encode %{
 9670     Register Rp = $p$$Register;
 9671     Register Rq = $q$$Register;
 9672     Register Ry = $y$$Register;
 9673     Label done;
 9674     __ cmpl(Rp, Rq);
 9675     __ jccb(Assembler::less, done);
 9676     __ xorl(Ry, Ry);
 9677     __ bind(done);
 9678   %}
 9679   ins_pipe(pipe_cmplt);
 9680 %}
 9681 
 9682 
 9683 //---------- FP Instructions------------------------------------------------
 9684 
 9685 // Really expensive, avoid
 9686 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 9687 %{
 9688   match(Set cr (CmpF src1 src2));
 9689 
 9690   ins_cost(500);
 9691   format %{ "ucomiss $src1, $src2\n\t"
 9692             "jnp,s   exit\n\t"
 9693             "pushfq\t# saw NaN, set CF\n\t"
 9694             "andq    [rsp], #0xffffff2b\n\t"
 9695             "popfq\n"
 9696     "exit:" %}
 9697   ins_encode %{
 9698     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9699     emit_cmpfp_fixup(masm);
 9700   %}
 9701   ins_pipe(pipe_slow);
 9702 %}
 9703 
 9704 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 9705   match(Set cr (CmpF src1 src2));
 9706 
 9707   ins_cost(100);
 9708   format %{ "ucomiss $src1, $src2" %}
 9709   ins_encode %{
 9710     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9711   %}
 9712   ins_pipe(pipe_slow);
 9713 %}
 9714 
 9715 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 9716   match(Set cr (CmpF src1 (LoadF src2)));
 9717 
 9718   ins_cost(100);
 9719   format %{ "ucomiss $src1, $src2" %}
 9720   ins_encode %{
 9721     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9722   %}
 9723   ins_pipe(pipe_slow);
 9724 %}
 9725 
 9726 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
 9727   match(Set cr (CmpF src con));
 9728   ins_cost(100);
 9729   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
 9730   ins_encode %{
 9731     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9732   %}
 9733   ins_pipe(pipe_slow);
 9734 %}
 9735 
 9736 // Really expensive, avoid
 9737 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 9738 %{
 9739   match(Set cr (CmpD src1 src2));
 9740 
 9741   ins_cost(500);
 9742   format %{ "ucomisd $src1, $src2\n\t"
 9743             "jnp,s   exit\n\t"
 9744             "pushfq\t# saw NaN, set CF\n\t"
 9745             "andq    [rsp], #0xffffff2b\n\t"
 9746             "popfq\n"
 9747     "exit:" %}
 9748   ins_encode %{
 9749     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9750     emit_cmpfp_fixup(masm);
 9751   %}
 9752   ins_pipe(pipe_slow);
 9753 %}
 9754 
 9755 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 9756   match(Set cr (CmpD src1 src2));
 9757 
 9758   ins_cost(100);
 9759   format %{ "ucomisd $src1, $src2 test" %}
 9760   ins_encode %{
 9761     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9762   %}
 9763   ins_pipe(pipe_slow);
 9764 %}
 9765 
 9766 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 9767   match(Set cr (CmpD src1 (LoadD src2)));
 9768 
 9769   ins_cost(100);
 9770   format %{ "ucomisd $src1, $src2" %}
 9771   ins_encode %{
 9772     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9773   %}
 9774   ins_pipe(pipe_slow);
 9775 %}
 9776 
 9777 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
 9778   match(Set cr (CmpD src con));
 9779   ins_cost(100);
 9780   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
 9781   ins_encode %{
 9782     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9783   %}
 9784   ins_pipe(pipe_slow);
 9785 %}
 9786 
 9787 // Compare into -1,0,1
 9788 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
 9789 %{
 9790   match(Set dst (CmpF3 src1 src2));
 9791   effect(KILL cr);
 9792 
 9793   ins_cost(275);
 9794   format %{ "ucomiss $src1, $src2\n\t"
 9795             "movl    $dst, #-1\n\t"
 9796             "jp,s    done\n\t"
 9797             "jb,s    done\n\t"
 9798             "setne   $dst\n\t"
 9799             "movzbl  $dst, $dst\n"
 9800     "done:" %}
 9801   ins_encode %{
 9802     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9803     emit_cmpfp3(masm, $dst$$Register);
 9804   %}
 9805   ins_pipe(pipe_slow);
 9806 %}
 9807 
 9808 // Compare into -1,0,1
 9809 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 9810 %{
 9811   match(Set dst (CmpF3 src1 (LoadF src2)));
 9812   effect(KILL cr);
 9813 
 9814   ins_cost(275);
 9815   format %{ "ucomiss $src1, $src2\n\t"
 9816             "movl    $dst, #-1\n\t"
 9817             "jp,s    done\n\t"
 9818             "jb,s    done\n\t"
 9819             "setne   $dst\n\t"
 9820             "movzbl  $dst, $dst\n"
 9821     "done:" %}
 9822   ins_encode %{
 9823     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9824     emit_cmpfp3(masm, $dst$$Register);
 9825   %}
 9826   ins_pipe(pipe_slow);
 9827 %}
 9828 
 9829 // Compare into -1,0,1
 9830 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
 9831   match(Set dst (CmpF3 src con));
 9832   effect(KILL cr);
 9833 
 9834   ins_cost(275);
 9835   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
 9836             "movl    $dst, #-1\n\t"
 9837             "jp,s    done\n\t"
 9838             "jb,s    done\n\t"
 9839             "setne   $dst\n\t"
 9840             "movzbl  $dst, $dst\n"
 9841     "done:" %}
 9842   ins_encode %{
 9843     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9844     emit_cmpfp3(masm, $dst$$Register);
 9845   %}
 9846   ins_pipe(pipe_slow);
 9847 %}
 9848 
 9849 // Compare into -1,0,1
 9850 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
 9851 %{
 9852   match(Set dst (CmpD3 src1 src2));
 9853   effect(KILL cr);
 9854 
 9855   ins_cost(275);
 9856   format %{ "ucomisd $src1, $src2\n\t"
 9857             "movl    $dst, #-1\n\t"
 9858             "jp,s    done\n\t"
 9859             "jb,s    done\n\t"
 9860             "setne   $dst\n\t"
 9861             "movzbl  $dst, $dst\n"
 9862     "done:" %}
 9863   ins_encode %{
 9864     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9865     emit_cmpfp3(masm, $dst$$Register);
 9866   %}
 9867   ins_pipe(pipe_slow);
 9868 %}
 9869 
 9870 // Compare into -1,0,1
 9871 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
 9872 %{
 9873   match(Set dst (CmpD3 src1 (LoadD src2)));
 9874   effect(KILL cr);
 9875 
 9876   ins_cost(275);
 9877   format %{ "ucomisd $src1, $src2\n\t"
 9878             "movl    $dst, #-1\n\t"
 9879             "jp,s    done\n\t"
 9880             "jb,s    done\n\t"
 9881             "setne   $dst\n\t"
 9882             "movzbl  $dst, $dst\n"
 9883     "done:" %}
 9884   ins_encode %{
 9885     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9886     emit_cmpfp3(masm, $dst$$Register);
 9887   %}
 9888   ins_pipe(pipe_slow);
 9889 %}
 9890 
 9891 // Compare into -1,0,1
 9892 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
 9893   match(Set dst (CmpD3 src con));
 9894   effect(KILL cr);
 9895 
 9896   ins_cost(275);
 9897   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
 9898             "movl    $dst, #-1\n\t"
 9899             "jp,s    done\n\t"
 9900             "jb,s    done\n\t"
 9901             "setne   $dst\n\t"
 9902             "movzbl  $dst, $dst\n"
 9903     "done:" %}
 9904   ins_encode %{
 9905     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9906     emit_cmpfp3(masm, $dst$$Register);
 9907   %}
 9908   ins_pipe(pipe_slow);
 9909 %}
 9910 
 9911 //----------Arithmetic Conversion Instructions---------------------------------
 9912 
 9913 instruct convF2D_reg_reg(regD dst, regF src)
 9914 %{
 9915   match(Set dst (ConvF2D src));
 9916 
 9917   format %{ "cvtss2sd $dst, $src" %}
 9918   ins_encode %{
 9919     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
 9920   %}
 9921   ins_pipe(pipe_slow); // XXX
 9922 %}
 9923 
 9924 instruct convF2D_reg_mem(regD dst, memory src)
 9925 %{
 9926   predicate(UseAVX == 0);
 9927   match(Set dst (ConvF2D (LoadF src)));
 9928 
 9929   format %{ "cvtss2sd $dst, $src" %}
 9930   ins_encode %{
 9931     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
 9932   %}
 9933   ins_pipe(pipe_slow); // XXX
 9934 %}
 9935 
 9936 instruct convD2F_reg_reg(regF dst, regD src)
 9937 %{
 9938   match(Set dst (ConvD2F src));
 9939 
 9940   format %{ "cvtsd2ss $dst, $src" %}
 9941   ins_encode %{
 9942     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
 9943   %}
 9944   ins_pipe(pipe_slow); // XXX
 9945 %}
 9946 
 9947 instruct convD2F_reg_mem(regF dst, memory src)
 9948 %{
 9949   predicate(UseAVX == 0);
 9950   match(Set dst (ConvD2F (LoadD src)));
 9951 
 9952   format %{ "cvtsd2ss $dst, $src" %}
 9953   ins_encode %{
 9954     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
 9955   %}
 9956   ins_pipe(pipe_slow); // XXX
 9957 %}
 9958 
 9959 // XXX do mem variants
 9960 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
 9961 %{
 9962   match(Set dst (ConvF2I src));
 9963   effect(KILL cr);
 9964   format %{ "convert_f2i $dst, $src" %}
 9965   ins_encode %{
 9966     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
 9967   %}
 9968   ins_pipe(pipe_slow);
 9969 %}
 9970 
 9971 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
 9972 %{
 9973   match(Set dst (ConvF2L src));
 9974   effect(KILL cr);
 9975   format %{ "convert_f2l $dst, $src"%}
 9976   ins_encode %{
 9977     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
 9978   %}
 9979   ins_pipe(pipe_slow);
 9980 %}
 9981 
 9982 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
 9983 %{
 9984   match(Set dst (ConvD2I src));
 9985   effect(KILL cr);
 9986   format %{ "convert_d2i $dst, $src"%}
 9987   ins_encode %{
 9988     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
 9989   %}
 9990   ins_pipe(pipe_slow);
 9991 %}
 9992 
 9993 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
 9994 %{
 9995   match(Set dst (ConvD2L src));
 9996   effect(KILL cr);
 9997   format %{ "convert_d2l $dst, $src"%}
 9998   ins_encode %{
 9999     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10000   %}
10001   ins_pipe(pipe_slow);
10002 %}
10003 
10004 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10005 %{
10006   match(Set dst (RoundD src));
10007   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10008   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10009   ins_encode %{
10010     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10011   %}
10012   ins_pipe(pipe_slow);
10013 %}
10014 
10015 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10016 %{
10017   match(Set dst (RoundF src));
10018   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10019   format %{ "round_float $dst,$src" %}
10020   ins_encode %{
10021     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10022   %}
10023   ins_pipe(pipe_slow);
10024 %}
10025 
10026 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
10027 %{
10028   predicate(!UseXmmI2F);
10029   match(Set dst (ConvI2F src));
10030 
10031   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10032   ins_encode %{
10033     if (UseAVX > 0) {
10034       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10035     }
10036     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10037   %}
10038   ins_pipe(pipe_slow); // XXX
10039 %}
10040 
10041 instruct convI2F_reg_mem(regF dst, memory src)
10042 %{
10043   predicate(UseAVX == 0);
10044   match(Set dst (ConvI2F (LoadI src)));
10045 
10046   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10047   ins_encode %{
10048     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10049   %}
10050   ins_pipe(pipe_slow); // XXX
10051 %}
10052 
10053 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
10054 %{
10055   predicate(!UseXmmI2D);
10056   match(Set dst (ConvI2D src));
10057 
10058   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10059   ins_encode %{
10060     if (UseAVX > 0) {
10061       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10062     }
10063     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10064   %}
10065   ins_pipe(pipe_slow); // XXX
10066 %}
10067 
10068 instruct convI2D_reg_mem(regD dst, memory src)
10069 %{
10070   predicate(UseAVX == 0);
10071   match(Set dst (ConvI2D (LoadI src)));
10072 
10073   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10074   ins_encode %{
10075     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10076   %}
10077   ins_pipe(pipe_slow); // XXX
10078 %}
10079 
10080 instruct convXI2F_reg(regF dst, rRegI src)
10081 %{
10082   predicate(UseXmmI2F);
10083   match(Set dst (ConvI2F src));
10084 
10085   format %{ "movdl $dst, $src\n\t"
10086             "cvtdq2psl $dst, $dst\t# i2f" %}
10087   ins_encode %{
10088     __ movdl($dst$$XMMRegister, $src$$Register);
10089     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10090   %}
10091   ins_pipe(pipe_slow); // XXX
10092 %}
10093 
10094 instruct convXI2D_reg(regD dst, rRegI src)
10095 %{
10096   predicate(UseXmmI2D);
10097   match(Set dst (ConvI2D src));
10098 
10099   format %{ "movdl $dst, $src\n\t"
10100             "cvtdq2pdl $dst, $dst\t# i2d" %}
10101   ins_encode %{
10102     __ movdl($dst$$XMMRegister, $src$$Register);
10103     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10104   %}
10105   ins_pipe(pipe_slow); // XXX
10106 %}
10107 
10108 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
10109 %{
10110   match(Set dst (ConvL2F src));
10111 
10112   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10113   ins_encode %{
10114     if (UseAVX > 0) {
10115       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10116     }
10117     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10118   %}
10119   ins_pipe(pipe_slow); // XXX
10120 %}
10121 
10122 instruct convL2F_reg_mem(regF dst, memory src)
10123 %{
10124   predicate(UseAVX == 0);
10125   match(Set dst (ConvL2F (LoadL src)));
10126 
10127   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10128   ins_encode %{
10129     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10130   %}
10131   ins_pipe(pipe_slow); // XXX
10132 %}
10133 
10134 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
10135 %{
10136   match(Set dst (ConvL2D src));
10137 
10138   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10139   ins_encode %{
10140     if (UseAVX > 0) {
10141       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10142     }
10143     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10144   %}
10145   ins_pipe(pipe_slow); // XXX
10146 %}
10147 
10148 instruct convL2D_reg_mem(regD dst, memory src)
10149 %{
10150   predicate(UseAVX == 0);
10151   match(Set dst (ConvL2D (LoadL src)));
10152 
10153   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10154   ins_encode %{
10155     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10156   %}
10157   ins_pipe(pipe_slow); // XXX
10158 %}
10159 
10160 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10161 %{
10162   match(Set dst (ConvI2L src));
10163 
10164   ins_cost(125);
10165   format %{ "movslq  $dst, $src\t# i2l" %}
10166   ins_encode %{
10167     __ movslq($dst$$Register, $src$$Register);
10168   %}
10169   ins_pipe(ialu_reg_reg);
10170 %}
10171 
10172 // Zero-extend convert int to long
10173 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10174 %{
10175   match(Set dst (AndL (ConvI2L src) mask));
10176 
10177   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10178   ins_encode %{
10179     if ($dst$$reg != $src$$reg) {
10180       __ movl($dst$$Register, $src$$Register);
10181     }
10182   %}
10183   ins_pipe(ialu_reg_reg);
10184 %}
10185 
10186 // Zero-extend convert int to long
10187 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10188 %{
10189   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10190 
10191   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10192   ins_encode %{
10193     __ movl($dst$$Register, $src$$Address);
10194   %}
10195   ins_pipe(ialu_reg_mem);
10196 %}
10197 
10198 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10199 %{
10200   match(Set dst (AndL src mask));
10201 
10202   format %{ "movl    $dst, $src\t# zero-extend long" %}
10203   ins_encode %{
10204     __ movl($dst$$Register, $src$$Register);
10205   %}
10206   ins_pipe(ialu_reg_reg);
10207 %}
10208 
10209 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10210 %{
10211   match(Set dst (ConvL2I src));
10212 
10213   format %{ "movl    $dst, $src\t# l2i" %}
10214   ins_encode %{
10215     __ movl($dst$$Register, $src$$Register);
10216   %}
10217   ins_pipe(ialu_reg_reg);
10218 %}
10219 
10220 
10221 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10222   match(Set dst (MoveF2I src));
10223   effect(DEF dst, USE src);
10224 
10225   ins_cost(125);
10226   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10227   ins_encode %{
10228     __ movl($dst$$Register, Address(rsp, $src$$disp));
10229   %}
10230   ins_pipe(ialu_reg_mem);
10231 %}
10232 
10233 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10234   match(Set dst (MoveI2F src));
10235   effect(DEF dst, USE src);
10236 
10237   ins_cost(125);
10238   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10239   ins_encode %{
10240     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10241   %}
10242   ins_pipe(pipe_slow);
10243 %}
10244 
10245 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10246   match(Set dst (MoveD2L src));
10247   effect(DEF dst, USE src);
10248 
10249   ins_cost(125);
10250   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10251   ins_encode %{
10252     __ movq($dst$$Register, Address(rsp, $src$$disp));
10253   %}
10254   ins_pipe(ialu_reg_mem);
10255 %}
10256 
10257 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10258   predicate(!UseXmmLoadAndClearUpper);
10259   match(Set dst (MoveL2D src));
10260   effect(DEF dst, USE src);
10261 
10262   ins_cost(125);
10263   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10264   ins_encode %{
10265     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10266   %}
10267   ins_pipe(pipe_slow);
10268 %}
10269 
10270 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10271   predicate(UseXmmLoadAndClearUpper);
10272   match(Set dst (MoveL2D src));
10273   effect(DEF dst, USE src);
10274 
10275   ins_cost(125);
10276   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10277   ins_encode %{
10278     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10279   %}
10280   ins_pipe(pipe_slow);
10281 %}
10282 
10283 
10284 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10285   match(Set dst (MoveF2I src));
10286   effect(DEF dst, USE src);
10287 
10288   ins_cost(95); // XXX
10289   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10290   ins_encode %{
10291     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10292   %}
10293   ins_pipe(pipe_slow);
10294 %}
10295 
10296 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10297   match(Set dst (MoveI2F src));
10298   effect(DEF dst, USE src);
10299 
10300   ins_cost(100);
10301   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10302   ins_encode %{
10303     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10304   %}
10305   ins_pipe( ialu_mem_reg );
10306 %}
10307 
10308 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10309   match(Set dst (MoveD2L src));
10310   effect(DEF dst, USE src);
10311 
10312   ins_cost(95); // XXX
10313   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10314   ins_encode %{
10315     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10316   %}
10317   ins_pipe(pipe_slow);
10318 %}
10319 
10320 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10321   match(Set dst (MoveL2D src));
10322   effect(DEF dst, USE src);
10323 
10324   ins_cost(100);
10325   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10326   ins_encode %{
10327     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10328   %}
10329   ins_pipe(ialu_mem_reg);
10330 %}
10331 
10332 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10333   match(Set dst (MoveF2I src));
10334   effect(DEF dst, USE src);
10335   ins_cost(85);
10336   format %{ "movd    $dst,$src\t# MoveF2I" %}
10337   ins_encode %{
10338     __ movdl($dst$$Register, $src$$XMMRegister);
10339   %}
10340   ins_pipe( pipe_slow );
10341 %}
10342 
10343 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10344   match(Set dst (MoveD2L src));
10345   effect(DEF dst, USE src);
10346   ins_cost(85);
10347   format %{ "movd    $dst,$src\t# MoveD2L" %}
10348   ins_encode %{
10349     __ movdq($dst$$Register, $src$$XMMRegister);
10350   %}
10351   ins_pipe( pipe_slow );
10352 %}
10353 
10354 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10355   match(Set dst (MoveI2F src));
10356   effect(DEF dst, USE src);
10357   ins_cost(100);
10358   format %{ "movd    $dst,$src\t# MoveI2F" %}
10359   ins_encode %{
10360     __ movdl($dst$$XMMRegister, $src$$Register);
10361   %}
10362   ins_pipe( pipe_slow );
10363 %}
10364 
10365 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10366   match(Set dst (MoveL2D src));
10367   effect(DEF dst, USE src);
10368   ins_cost(100);
10369   format %{ "movd    $dst,$src\t# MoveL2D" %}
10370   ins_encode %{
10371      __ movdq($dst$$XMMRegister, $src$$Register);
10372   %}
10373   ins_pipe( pipe_slow );
10374 %}
10375 
10376 // Fast clearing of an array
10377 // Small non-constant lenght ClearArray for non-AVX512 targets.
10378 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10379                   Universe dummy, rFlagsReg cr)
10380 %{
10381   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
10382   match(Set dummy (ClearArray cnt base));
10383   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10384 
10385   format %{ $$template
10386     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10387     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10388     $$emit$$"jg      LARGE\n\t"
10389     $$emit$$"dec     rcx\n\t"
10390     $$emit$$"js      DONE\t# Zero length\n\t"
10391     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10392     $$emit$$"dec     rcx\n\t"
10393     $$emit$$"jge     LOOP\n\t"
10394     $$emit$$"jmp     DONE\n\t"
10395     $$emit$$"# LARGE:\n\t"
10396     if (UseFastStosb) {
10397        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10398        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10399     } else if (UseXMMForObjInit) {
10400        $$emit$$"mov     rdi,rax\n\t"
10401        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10402        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10403        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10404        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10405        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10406        $$emit$$"add     0x40,rax\n\t"
10407        $$emit$$"# L_zero_64_bytes:\n\t"
10408        $$emit$$"sub     0x8,rcx\n\t"
10409        $$emit$$"jge     L_loop\n\t"
10410        $$emit$$"add     0x4,rcx\n\t"
10411        $$emit$$"jl      L_tail\n\t"
10412        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10413        $$emit$$"add     0x20,rax\n\t"
10414        $$emit$$"sub     0x4,rcx\n\t"
10415        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10416        $$emit$$"add     0x4,rcx\n\t"
10417        $$emit$$"jle     L_end\n\t"
10418        $$emit$$"dec     rcx\n\t"
10419        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10420        $$emit$$"vmovq   xmm0,(rax)\n\t"
10421        $$emit$$"add     0x8,rax\n\t"
10422        $$emit$$"dec     rcx\n\t"
10423        $$emit$$"jge     L_sloop\n\t"
10424        $$emit$$"# L_end:\n\t"
10425     } else {
10426        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10427     }
10428     $$emit$$"# DONE"
10429   %}
10430   ins_encode %{
10431     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10432                  $tmp$$XMMRegister, false, knoreg);
10433   %}
10434   ins_pipe(pipe_slow);
10435 %}
10436 
10437 // Small non-constant length ClearArray for AVX512 targets.
10438 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10439                        Universe dummy, rFlagsReg cr)
10440 %{
10441   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
10442   match(Set dummy (ClearArray cnt base));
10443   ins_cost(125);
10444   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10445 
10446   format %{ $$template
10447     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10448     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10449     $$emit$$"jg      LARGE\n\t"
10450     $$emit$$"dec     rcx\n\t"
10451     $$emit$$"js      DONE\t# Zero length\n\t"
10452     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10453     $$emit$$"dec     rcx\n\t"
10454     $$emit$$"jge     LOOP\n\t"
10455     $$emit$$"jmp     DONE\n\t"
10456     $$emit$$"# LARGE:\n\t"
10457     if (UseFastStosb) {
10458        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10459        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10460     } else if (UseXMMForObjInit) {
10461        $$emit$$"mov     rdi,rax\n\t"
10462        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10463        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10464        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10465        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10466        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10467        $$emit$$"add     0x40,rax\n\t"
10468        $$emit$$"# L_zero_64_bytes:\n\t"
10469        $$emit$$"sub     0x8,rcx\n\t"
10470        $$emit$$"jge     L_loop\n\t"
10471        $$emit$$"add     0x4,rcx\n\t"
10472        $$emit$$"jl      L_tail\n\t"
10473        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10474        $$emit$$"add     0x20,rax\n\t"
10475        $$emit$$"sub     0x4,rcx\n\t"
10476        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10477        $$emit$$"add     0x4,rcx\n\t"
10478        $$emit$$"jle     L_end\n\t"
10479        $$emit$$"dec     rcx\n\t"
10480        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10481        $$emit$$"vmovq   xmm0,(rax)\n\t"
10482        $$emit$$"add     0x8,rax\n\t"
10483        $$emit$$"dec     rcx\n\t"
10484        $$emit$$"jge     L_sloop\n\t"
10485        $$emit$$"# L_end:\n\t"
10486     } else {
10487        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10488     }
10489     $$emit$$"# DONE"
10490   %}
10491   ins_encode %{
10492     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10493                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
10494   %}
10495   ins_pipe(pipe_slow);
10496 %}
10497 
10498 // Large non-constant length ClearArray for non-AVX512 targets.
10499 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10500                         Universe dummy, rFlagsReg cr)
10501 %{
10502   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
10503   match(Set dummy (ClearArray cnt base));
10504   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10505 
10506   format %{ $$template
10507     if (UseFastStosb) {
10508        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10509        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10510        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10511     } else if (UseXMMForObjInit) {
10512        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10513        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10514        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10515        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10516        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10517        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10518        $$emit$$"add     0x40,rax\n\t"
10519        $$emit$$"# L_zero_64_bytes:\n\t"
10520        $$emit$$"sub     0x8,rcx\n\t"
10521        $$emit$$"jge     L_loop\n\t"
10522        $$emit$$"add     0x4,rcx\n\t"
10523        $$emit$$"jl      L_tail\n\t"
10524        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10525        $$emit$$"add     0x20,rax\n\t"
10526        $$emit$$"sub     0x4,rcx\n\t"
10527        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10528        $$emit$$"add     0x4,rcx\n\t"
10529        $$emit$$"jle     L_end\n\t"
10530        $$emit$$"dec     rcx\n\t"
10531        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10532        $$emit$$"vmovq   xmm0,(rax)\n\t"
10533        $$emit$$"add     0x8,rax\n\t"
10534        $$emit$$"dec     rcx\n\t"
10535        $$emit$$"jge     L_sloop\n\t"
10536        $$emit$$"# L_end:\n\t"
10537     } else {
10538        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10539        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10540     }
10541   %}
10542   ins_encode %{
10543     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10544                  $tmp$$XMMRegister, true, knoreg);
10545   %}
10546   ins_pipe(pipe_slow);
10547 %}
10548 
10549 // Large non-constant length ClearArray for AVX512 targets.
10550 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10551                              Universe dummy, rFlagsReg cr)
10552 %{
10553   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
10554   match(Set dummy (ClearArray cnt base));
10555   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10556 
10557   format %{ $$template
10558     if (UseFastStosb) {
10559        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10560        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10561        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10562     } else if (UseXMMForObjInit) {
10563        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10564        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10565        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10566        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10567        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10568        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10569        $$emit$$"add     0x40,rax\n\t"
10570        $$emit$$"# L_zero_64_bytes:\n\t"
10571        $$emit$$"sub     0x8,rcx\n\t"
10572        $$emit$$"jge     L_loop\n\t"
10573        $$emit$$"add     0x4,rcx\n\t"
10574        $$emit$$"jl      L_tail\n\t"
10575        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10576        $$emit$$"add     0x20,rax\n\t"
10577        $$emit$$"sub     0x4,rcx\n\t"
10578        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10579        $$emit$$"add     0x4,rcx\n\t"
10580        $$emit$$"jle     L_end\n\t"
10581        $$emit$$"dec     rcx\n\t"
10582        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10583        $$emit$$"vmovq   xmm0,(rax)\n\t"
10584        $$emit$$"add     0x8,rax\n\t"
10585        $$emit$$"dec     rcx\n\t"
10586        $$emit$$"jge     L_sloop\n\t"
10587        $$emit$$"# L_end:\n\t"
10588     } else {
10589        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10590        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10591     }
10592   %}
10593   ins_encode %{
10594     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10595                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
10596   %}
10597   ins_pipe(pipe_slow);
10598 %}
10599 
10600 // Small constant length ClearArray for AVX512 targets.
10601 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
10602 %{
10603   predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
10604   match(Set dummy (ClearArray cnt base));
10605   ins_cost(100);
10606   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
10607   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
10608   ins_encode %{
10609    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10610   %}
10611   ins_pipe(pipe_slow);
10612 %}
10613 
10614 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10615                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10616 %{
10617   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10618   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10619   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10620 
10621   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10622   ins_encode %{
10623     __ string_compare($str1$$Register, $str2$$Register,
10624                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10625                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
10626   %}
10627   ins_pipe( pipe_slow );
10628 %}
10629 
10630 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10631                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10632 %{
10633   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10634   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10635   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10636 
10637   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10638   ins_encode %{
10639     __ string_compare($str1$$Register, $str2$$Register,
10640                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10641                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
10642   %}
10643   ins_pipe( pipe_slow );
10644 %}
10645 
10646 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10647                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10648 %{
10649   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10650   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10651   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10652 
10653   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10654   ins_encode %{
10655     __ string_compare($str1$$Register, $str2$$Register,
10656                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10657                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
10658   %}
10659   ins_pipe( pipe_slow );
10660 %}
10661 
10662 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10663                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10664 %{
10665   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10666   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10667   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10668 
10669   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10670   ins_encode %{
10671     __ string_compare($str1$$Register, $str2$$Register,
10672                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10673                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
10674   %}
10675   ins_pipe( pipe_slow );
10676 %}
10677 
10678 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10679                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10680 %{
10681   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10682   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10683   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10684 
10685   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10686   ins_encode %{
10687     __ string_compare($str1$$Register, $str2$$Register,
10688                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10689                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
10690   %}
10691   ins_pipe( pipe_slow );
10692 %}
10693 
10694 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10695                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10696 %{
10697   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10698   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10699   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10700 
10701   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10702   ins_encode %{
10703     __ string_compare($str1$$Register, $str2$$Register,
10704                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10705                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
10706   %}
10707   ins_pipe( pipe_slow );
10708 %}
10709 
10710 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10711                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10712 %{
10713   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10714   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10715   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10716 
10717   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10718   ins_encode %{
10719     __ string_compare($str2$$Register, $str1$$Register,
10720                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10721                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
10722   %}
10723   ins_pipe( pipe_slow );
10724 %}
10725 
10726 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10727                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10728 %{
10729   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10730   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10731   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10732 
10733   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10734   ins_encode %{
10735     __ string_compare($str2$$Register, $str1$$Register,
10736                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10737                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
10738   %}
10739   ins_pipe( pipe_slow );
10740 %}
10741 
10742 // fast search of substring with known size.
10743 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10744                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10745 %{
10746   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10747   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10748   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10749 
10750   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10751   ins_encode %{
10752     int icnt2 = (int)$int_cnt2$$constant;
10753     if (icnt2 >= 16) {
10754       // IndexOf for constant substrings with size >= 16 elements
10755       // which don't need to be loaded through stack.
10756       __ string_indexofC8($str1$$Register, $str2$$Register,
10757                           $cnt1$$Register, $cnt2$$Register,
10758                           icnt2, $result$$Register,
10759                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10760     } else {
10761       // Small strings are loaded through stack if they cross page boundary.
10762       __ string_indexof($str1$$Register, $str2$$Register,
10763                         $cnt1$$Register, $cnt2$$Register,
10764                         icnt2, $result$$Register,
10765                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10766     }
10767   %}
10768   ins_pipe( pipe_slow );
10769 %}
10770 
10771 // fast search of substring with known size.
10772 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10773                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10774 %{
10775   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10776   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10777   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10778 
10779   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10780   ins_encode %{
10781     int icnt2 = (int)$int_cnt2$$constant;
10782     if (icnt2 >= 8) {
10783       // IndexOf for constant substrings with size >= 8 elements
10784       // which don't need to be loaded through stack.
10785       __ string_indexofC8($str1$$Register, $str2$$Register,
10786                           $cnt1$$Register, $cnt2$$Register,
10787                           icnt2, $result$$Register,
10788                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10789     } else {
10790       // Small strings are loaded through stack if they cross page boundary.
10791       __ string_indexof($str1$$Register, $str2$$Register,
10792                         $cnt1$$Register, $cnt2$$Register,
10793                         icnt2, $result$$Register,
10794                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10795     }
10796   %}
10797   ins_pipe( pipe_slow );
10798 %}
10799 
10800 // fast search of substring with known size.
10801 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10802                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10803 %{
10804   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10805   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10806   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10807 
10808   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10809   ins_encode %{
10810     int icnt2 = (int)$int_cnt2$$constant;
10811     if (icnt2 >= 8) {
10812       // IndexOf for constant substrings with size >= 8 elements
10813       // which don't need to be loaded through stack.
10814       __ string_indexofC8($str1$$Register, $str2$$Register,
10815                           $cnt1$$Register, $cnt2$$Register,
10816                           icnt2, $result$$Register,
10817                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10818     } else {
10819       // Small strings are loaded through stack if they cross page boundary.
10820       __ string_indexof($str1$$Register, $str2$$Register,
10821                         $cnt1$$Register, $cnt2$$Register,
10822                         icnt2, $result$$Register,
10823                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10824     }
10825   %}
10826   ins_pipe( pipe_slow );
10827 %}
10828 
10829 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10830                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10831 %{
10832   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10833   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10834   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10835 
10836   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10837   ins_encode %{
10838     __ string_indexof($str1$$Register, $str2$$Register,
10839                       $cnt1$$Register, $cnt2$$Register,
10840                       (-1), $result$$Register,
10841                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10842   %}
10843   ins_pipe( pipe_slow );
10844 %}
10845 
10846 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10847                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10848 %{
10849   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10850   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10851   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10852 
10853   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10854   ins_encode %{
10855     __ string_indexof($str1$$Register, $str2$$Register,
10856                       $cnt1$$Register, $cnt2$$Register,
10857                       (-1), $result$$Register,
10858                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10859   %}
10860   ins_pipe( pipe_slow );
10861 %}
10862 
10863 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10864                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10865 %{
10866   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10867   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10868   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10869 
10870   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10871   ins_encode %{
10872     __ string_indexof($str1$$Register, $str2$$Register,
10873                       $cnt1$$Register, $cnt2$$Register,
10874                       (-1), $result$$Register,
10875                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10876   %}
10877   ins_pipe( pipe_slow );
10878 %}
10879 
10880 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10881                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
10882 %{
10883   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
10884   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10885   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10886   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10887   ins_encode %{
10888     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10889                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
10890   %}
10891   ins_pipe( pipe_slow );
10892 %}
10893 
10894 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10895                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
10896 %{
10897   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
10898   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10899   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10900   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10901   ins_encode %{
10902     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10903                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
10904   %}
10905   ins_pipe( pipe_slow );
10906 %}
10907 
10908 // fast string equals
10909 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10910                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10911 %{
10912   predicate(!VM_Version::supports_avx512vlbw());
10913   match(Set result (StrEquals (Binary str1 str2) cnt));
10914   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10915 
10916   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10917   ins_encode %{
10918     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10919                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10920                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
10921   %}
10922   ins_pipe( pipe_slow );
10923 %}
10924 
10925 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10926                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
10927 %{
10928   predicate(VM_Version::supports_avx512vlbw());
10929   match(Set result (StrEquals (Binary str1 str2) cnt));
10930   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10931 
10932   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10933   ins_encode %{
10934     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10935                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10936                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
10937   %}
10938   ins_pipe( pipe_slow );
10939 %}
10940 
10941 // fast array equals
10942 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10943                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10944 %{
10945   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10946   match(Set result (AryEq ary1 ary2));
10947   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10948 
10949   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10950   ins_encode %{
10951     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10952                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10953                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
10954   %}
10955   ins_pipe( pipe_slow );
10956 %}
10957 
10958 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10959                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10960 %{
10961   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10962   match(Set result (AryEq ary1 ary2));
10963   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10964 
10965   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10966   ins_encode %{
10967     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10968                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10969                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
10970   %}
10971   ins_pipe( pipe_slow );
10972 %}
10973 
10974 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10975                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10976 %{
10977   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10978   match(Set result (AryEq ary1 ary2));
10979   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10980 
10981   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10982   ins_encode %{
10983     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10984                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10985                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
10986   %}
10987   ins_pipe( pipe_slow );
10988 %}
10989 
10990 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10991                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10992 %{
10993   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10994   match(Set result (AryEq ary1 ary2));
10995   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10996 
10997   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10998   ins_encode %{
10999     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11000                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11001                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11002   %}
11003   ins_pipe( pipe_slow );
11004 %}
11005 
11006 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11007                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11008                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11009                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11010                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11011 %{
11012   predicate(UseAVX >= 2);
11013   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11014   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11015          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11016          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11017          USE basic_type, KILL cr);
11018 
11019   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
11020   ins_encode %{
11021     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11022                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11023                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11024                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11025                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11026                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11027                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11028   %}
11029   ins_pipe( pipe_slow );
11030 %}
11031 
11032 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11033                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11034 %{
11035   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11036   match(Set result (CountPositives ary1 len));
11037   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11038 
11039   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11040   ins_encode %{
11041     __ count_positives($ary1$$Register, $len$$Register,
11042                        $result$$Register, $tmp3$$Register,
11043                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11044   %}
11045   ins_pipe( pipe_slow );
11046 %}
11047 
11048 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11049                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11050 %{
11051   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11052   match(Set result (CountPositives ary1 len));
11053   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11054 
11055   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11056   ins_encode %{
11057     __ count_positives($ary1$$Register, $len$$Register,
11058                        $result$$Register, $tmp3$$Register,
11059                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11060   %}
11061   ins_pipe( pipe_slow );
11062 %}
11063 
11064 // fast char[] to byte[] compression
11065 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11066                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11067   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11068   match(Set result (StrCompressedCopy src (Binary dst len)));
11069   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11070          USE_KILL len, KILL tmp5, KILL cr);
11071 
11072   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11073   ins_encode %{
11074     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11075                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11076                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11077                            knoreg, knoreg);
11078   %}
11079   ins_pipe( pipe_slow );
11080 %}
11081 
11082 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11083                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11084   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11085   match(Set result (StrCompressedCopy src (Binary dst len)));
11086   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11087          USE_KILL len, KILL tmp5, KILL cr);
11088 
11089   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11090   ins_encode %{
11091     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11092                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11093                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11094                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11095   %}
11096   ins_pipe( pipe_slow );
11097 %}
11098 // fast byte[] to char[] inflation
11099 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11100                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11101   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11102   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11103   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11104 
11105   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11106   ins_encode %{
11107     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11108                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11109   %}
11110   ins_pipe( pipe_slow );
11111 %}
11112 
11113 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11114                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11115   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11116   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11117   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11118 
11119   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11120   ins_encode %{
11121     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11122                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11123   %}
11124   ins_pipe( pipe_slow );
11125 %}
11126 
11127 // encode char[] to byte[] in ISO_8859_1
11128 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11129                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11130                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11131   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11132   match(Set result (EncodeISOArray src (Binary dst len)));
11133   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11134 
11135   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11136   ins_encode %{
11137     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11138                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11139                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11140   %}
11141   ins_pipe( pipe_slow );
11142 %}
11143 
11144 // encode char[] to byte[] in ASCII
11145 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11146                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11147                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11148   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11149   match(Set result (EncodeISOArray src (Binary dst len)));
11150   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11151 
11152   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11153   ins_encode %{
11154     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11155                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11156                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11157   %}
11158   ins_pipe( pipe_slow );
11159 %}
11160 
11161 //----------Overflow Math Instructions-----------------------------------------
11162 
11163 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11164 %{
11165   match(Set cr (OverflowAddI op1 op2));
11166   effect(DEF cr, USE_KILL op1, USE op2);
11167 
11168   format %{ "addl    $op1, $op2\t# overflow check int" %}
11169 
11170   ins_encode %{
11171     __ addl($op1$$Register, $op2$$Register);
11172   %}
11173   ins_pipe(ialu_reg_reg);
11174 %}
11175 
11176 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11177 %{
11178   match(Set cr (OverflowAddI op1 op2));
11179   effect(DEF cr, USE_KILL op1, USE op2);
11180 
11181   format %{ "addl    $op1, $op2\t# overflow check int" %}
11182 
11183   ins_encode %{
11184     __ addl($op1$$Register, $op2$$constant);
11185   %}
11186   ins_pipe(ialu_reg_reg);
11187 %}
11188 
11189 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11190 %{
11191   match(Set cr (OverflowAddL op1 op2));
11192   effect(DEF cr, USE_KILL op1, USE op2);
11193 
11194   format %{ "addq    $op1, $op2\t# overflow check long" %}
11195   ins_encode %{
11196     __ addq($op1$$Register, $op2$$Register);
11197   %}
11198   ins_pipe(ialu_reg_reg);
11199 %}
11200 
11201 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11202 %{
11203   match(Set cr (OverflowAddL op1 op2));
11204   effect(DEF cr, USE_KILL op1, USE op2);
11205 
11206   format %{ "addq    $op1, $op2\t# overflow check long" %}
11207   ins_encode %{
11208     __ addq($op1$$Register, $op2$$constant);
11209   %}
11210   ins_pipe(ialu_reg_reg);
11211 %}
11212 
11213 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11214 %{
11215   match(Set cr (OverflowSubI op1 op2));
11216 
11217   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11218   ins_encode %{
11219     __ cmpl($op1$$Register, $op2$$Register);
11220   %}
11221   ins_pipe(ialu_reg_reg);
11222 %}
11223 
11224 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11225 %{
11226   match(Set cr (OverflowSubI op1 op2));
11227 
11228   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11229   ins_encode %{
11230     __ cmpl($op1$$Register, $op2$$constant);
11231   %}
11232   ins_pipe(ialu_reg_reg);
11233 %}
11234 
11235 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11236 %{
11237   match(Set cr (OverflowSubL op1 op2));
11238 
11239   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11240   ins_encode %{
11241     __ cmpq($op1$$Register, $op2$$Register);
11242   %}
11243   ins_pipe(ialu_reg_reg);
11244 %}
11245 
11246 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11247 %{
11248   match(Set cr (OverflowSubL op1 op2));
11249 
11250   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11251   ins_encode %{
11252     __ cmpq($op1$$Register, $op2$$constant);
11253   %}
11254   ins_pipe(ialu_reg_reg);
11255 %}
11256 
11257 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11258 %{
11259   match(Set cr (OverflowSubI zero op2));
11260   effect(DEF cr, USE_KILL op2);
11261 
11262   format %{ "negl    $op2\t# overflow check int" %}
11263   ins_encode %{
11264     __ negl($op2$$Register);
11265   %}
11266   ins_pipe(ialu_reg_reg);
11267 %}
11268 
11269 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11270 %{
11271   match(Set cr (OverflowSubL zero op2));
11272   effect(DEF cr, USE_KILL op2);
11273 
11274   format %{ "negq    $op2\t# overflow check long" %}
11275   ins_encode %{
11276     __ negq($op2$$Register);
11277   %}
11278   ins_pipe(ialu_reg_reg);
11279 %}
11280 
11281 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11282 %{
11283   match(Set cr (OverflowMulI op1 op2));
11284   effect(DEF cr, USE_KILL op1, USE op2);
11285 
11286   format %{ "imull    $op1, $op2\t# overflow check int" %}
11287   ins_encode %{
11288     __ imull($op1$$Register, $op2$$Register);
11289   %}
11290   ins_pipe(ialu_reg_reg_alu0);
11291 %}
11292 
11293 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11294 %{
11295   match(Set cr (OverflowMulI op1 op2));
11296   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11297 
11298   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11299   ins_encode %{
11300     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11301   %}
11302   ins_pipe(ialu_reg_reg_alu0);
11303 %}
11304 
11305 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11306 %{
11307   match(Set cr (OverflowMulL op1 op2));
11308   effect(DEF cr, USE_KILL op1, USE op2);
11309 
11310   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11311   ins_encode %{
11312     __ imulq($op1$$Register, $op2$$Register);
11313   %}
11314   ins_pipe(ialu_reg_reg_alu0);
11315 %}
11316 
11317 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11318 %{
11319   match(Set cr (OverflowMulL op1 op2));
11320   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11321 
11322   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11323   ins_encode %{
11324     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11325   %}
11326   ins_pipe(ialu_reg_reg_alu0);
11327 %}
11328 
11329 
11330 //----------Control Flow Instructions------------------------------------------
11331 // Signed compare Instructions
11332 
11333 // XXX more variants!!
11334 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11335 %{
11336   match(Set cr (CmpI op1 op2));
11337   effect(DEF cr, USE op1, USE op2);
11338 
11339   format %{ "cmpl    $op1, $op2" %}
11340   ins_encode %{
11341     __ cmpl($op1$$Register, $op2$$Register);
11342   %}
11343   ins_pipe(ialu_cr_reg_reg);
11344 %}
11345 
11346 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11347 %{
11348   match(Set cr (CmpI op1 op2));
11349 
11350   format %{ "cmpl    $op1, $op2" %}
11351   ins_encode %{
11352     __ cmpl($op1$$Register, $op2$$constant);
11353   %}
11354   ins_pipe(ialu_cr_reg_imm);
11355 %}
11356 
11357 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11358 %{
11359   match(Set cr (CmpI op1 (LoadI op2)));
11360 
11361   ins_cost(500); // XXX
11362   format %{ "cmpl    $op1, $op2" %}
11363   ins_encode %{
11364     __ cmpl($op1$$Register, $op2$$Address);
11365   %}
11366   ins_pipe(ialu_cr_reg_mem);
11367 %}
11368 
11369 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11370 %{
11371   match(Set cr (CmpI src zero));
11372 
11373   format %{ "testl   $src, $src" %}
11374   ins_encode %{
11375     __ testl($src$$Register, $src$$Register);
11376   %}
11377   ins_pipe(ialu_cr_reg_imm);
11378 %}
11379 
11380 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11381 %{
11382   match(Set cr (CmpI (AndI src con) zero));
11383 
11384   format %{ "testl   $src, $con" %}
11385   ins_encode %{
11386     __ testl($src$$Register, $con$$constant);
11387   %}
11388   ins_pipe(ialu_cr_reg_imm);
11389 %}
11390 
11391 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11392 %{
11393   match(Set cr (CmpI (AndI src1 src2) zero));
11394 
11395   format %{ "testl   $src1, $src2" %}
11396   ins_encode %{
11397     __ testl($src1$$Register, $src2$$Register);
11398   %}
11399   ins_pipe(ialu_cr_reg_imm);
11400 %}
11401 
11402 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11403 %{
11404   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11405 
11406   format %{ "testl   $src, $mem" %}
11407   ins_encode %{
11408     __ testl($src$$Register, $mem$$Address);
11409   %}
11410   ins_pipe(ialu_cr_reg_mem);
11411 %}
11412 
11413 // Unsigned compare Instructions; really, same as signed except they
11414 // produce an rFlagsRegU instead of rFlagsReg.
11415 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11416 %{
11417   match(Set cr (CmpU op1 op2));
11418 
11419   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11420   ins_encode %{
11421     __ cmpl($op1$$Register, $op2$$Register);
11422   %}
11423   ins_pipe(ialu_cr_reg_reg);
11424 %}
11425 
11426 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11427 %{
11428   match(Set cr (CmpU op1 op2));
11429 
11430   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11431   ins_encode %{
11432     __ cmpl($op1$$Register, $op2$$constant);
11433   %}
11434   ins_pipe(ialu_cr_reg_imm);
11435 %}
11436 
11437 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11438 %{
11439   match(Set cr (CmpU op1 (LoadI op2)));
11440 
11441   ins_cost(500); // XXX
11442   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11443   ins_encode %{
11444     __ cmpl($op1$$Register, $op2$$Address);
11445   %}
11446   ins_pipe(ialu_cr_reg_mem);
11447 %}
11448 
11449 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11450 %{
11451   match(Set cr (CmpU src zero));
11452 
11453   format %{ "testl   $src, $src\t# unsigned" %}
11454   ins_encode %{
11455     __ testl($src$$Register, $src$$Register);
11456   %}
11457   ins_pipe(ialu_cr_reg_imm);
11458 %}
11459 
11460 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11461 %{
11462   match(Set cr (CmpP op1 op2));
11463 
11464   format %{ "cmpq    $op1, $op2\t# ptr" %}
11465   ins_encode %{
11466     __ cmpq($op1$$Register, $op2$$Register);
11467   %}
11468   ins_pipe(ialu_cr_reg_reg);
11469 %}
11470 
11471 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11472 %{
11473   match(Set cr (CmpP op1 (LoadP op2)));
11474   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11475 
11476   ins_cost(500); // XXX
11477   format %{ "cmpq    $op1, $op2\t# ptr" %}
11478   ins_encode %{
11479     __ cmpq($op1$$Register, $op2$$Address);
11480   %}
11481   ins_pipe(ialu_cr_reg_mem);
11482 %}
11483 
11484 // XXX this is generalized by compP_rReg_mem???
11485 // Compare raw pointer (used in out-of-heap check).
11486 // Only works because non-oop pointers must be raw pointers
11487 // and raw pointers have no anti-dependencies.
11488 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11489 %{
11490   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11491             n->in(2)->as_Load()->barrier_data() == 0);
11492   match(Set cr (CmpP op1 (LoadP op2)));
11493 
11494   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11495   ins_encode %{
11496     __ cmpq($op1$$Register, $op2$$Address);
11497   %}
11498   ins_pipe(ialu_cr_reg_mem);
11499 %}
11500 
11501 // This will generate a signed flags result. This should be OK since
11502 // any compare to a zero should be eq/neq.
11503 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11504 %{
11505   match(Set cr (CmpP src zero));
11506 
11507   format %{ "testq   $src, $src\t# ptr" %}
11508   ins_encode %{
11509     __ testq($src$$Register, $src$$Register);
11510   %}
11511   ins_pipe(ialu_cr_reg_imm);
11512 %}
11513 
11514 // This will generate a signed flags result. This should be OK since
11515 // any compare to a zero should be eq/neq.
11516 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11517 %{
11518   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
11519             n->in(1)->as_Load()->barrier_data() == 0);
11520   match(Set cr (CmpP (LoadP op) zero));
11521 
11522   ins_cost(500); // XXX
11523   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11524   ins_encode %{
11525     __ testq($op$$Address, 0xFFFFFFFF);
11526   %}
11527   ins_pipe(ialu_cr_reg_imm);
11528 %}
11529 
11530 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11531 %{
11532   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
11533             n->in(1)->as_Load()->barrier_data() == 0);
11534   match(Set cr (CmpP (LoadP mem) zero));
11535 
11536   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11537   ins_encode %{
11538     __ cmpq(r12, $mem$$Address);
11539   %}
11540   ins_pipe(ialu_cr_reg_mem);
11541 %}
11542 
11543 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11544 %{
11545   match(Set cr (CmpN op1 op2));
11546 
11547   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11548   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11549   ins_pipe(ialu_cr_reg_reg);
11550 %}
11551 
11552 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11553 %{
11554   match(Set cr (CmpN src (LoadN mem)));
11555 
11556   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11557   ins_encode %{
11558     __ cmpl($src$$Register, $mem$$Address);
11559   %}
11560   ins_pipe(ialu_cr_reg_mem);
11561 %}
11562 
11563 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11564   match(Set cr (CmpN op1 op2));
11565 
11566   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11567   ins_encode %{
11568     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11569   %}
11570   ins_pipe(ialu_cr_reg_imm);
11571 %}
11572 
11573 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11574 %{
11575   match(Set cr (CmpN src (LoadN mem)));
11576 
11577   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11578   ins_encode %{
11579     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11580   %}
11581   ins_pipe(ialu_cr_reg_mem);
11582 %}
11583 
11584 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11585   match(Set cr (CmpN op1 op2));
11586 
11587   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11588   ins_encode %{
11589     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11590   %}
11591   ins_pipe(ialu_cr_reg_imm);
11592 %}
11593 
11594 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11595 %{
11596   match(Set cr (CmpN src (LoadNKlass mem)));
11597 
11598   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11599   ins_encode %{
11600     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11601   %}
11602   ins_pipe(ialu_cr_reg_mem);
11603 %}
11604 
11605 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11606   match(Set cr (CmpN src zero));
11607 
11608   format %{ "testl   $src, $src\t# compressed ptr" %}
11609   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11610   ins_pipe(ialu_cr_reg_imm);
11611 %}
11612 
11613 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11614 %{
11615   predicate(CompressedOops::base() != nullptr);
11616   match(Set cr (CmpN (LoadN mem) zero));
11617 
11618   ins_cost(500); // XXX
11619   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11620   ins_encode %{
11621     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11622   %}
11623   ins_pipe(ialu_cr_reg_mem);
11624 %}
11625 
11626 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11627 %{
11628   predicate(CompressedOops::base() == nullptr);
11629   match(Set cr (CmpN (LoadN mem) zero));
11630 
11631   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11632   ins_encode %{
11633     __ cmpl(r12, $mem$$Address);
11634   %}
11635   ins_pipe(ialu_cr_reg_mem);
11636 %}
11637 
11638 // Yanked all unsigned pointer compare operations.
11639 // Pointer compares are done with CmpP which is already unsigned.
11640 
11641 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11642 %{
11643   match(Set cr (CmpL op1 op2));
11644 
11645   format %{ "cmpq    $op1, $op2" %}
11646   ins_encode %{
11647     __ cmpq($op1$$Register, $op2$$Register);
11648   %}
11649   ins_pipe(ialu_cr_reg_reg);
11650 %}
11651 
11652 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11653 %{
11654   match(Set cr (CmpL op1 op2));
11655 
11656   format %{ "cmpq    $op1, $op2" %}
11657   ins_encode %{
11658     __ cmpq($op1$$Register, $op2$$constant);
11659   %}
11660   ins_pipe(ialu_cr_reg_imm);
11661 %}
11662 
11663 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11664 %{
11665   match(Set cr (CmpL op1 (LoadL op2)));
11666 
11667   format %{ "cmpq    $op1, $op2" %}
11668   ins_encode %{
11669     __ cmpq($op1$$Register, $op2$$Address);
11670   %}
11671   ins_pipe(ialu_cr_reg_mem);
11672 %}
11673 
11674 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11675 %{
11676   match(Set cr (CmpL src zero));
11677 
11678   format %{ "testq   $src, $src" %}
11679   ins_encode %{
11680     __ testq($src$$Register, $src$$Register);
11681   %}
11682   ins_pipe(ialu_cr_reg_imm);
11683 %}
11684 
11685 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11686 %{
11687   match(Set cr (CmpL (AndL src con) zero));
11688 
11689   format %{ "testq   $src, $con\t# long" %}
11690   ins_encode %{
11691     __ testq($src$$Register, $con$$constant);
11692   %}
11693   ins_pipe(ialu_cr_reg_imm);
11694 %}
11695 
11696 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
11697 %{
11698   match(Set cr (CmpL (AndL src1 src2) zero));
11699 
11700   format %{ "testq   $src1, $src2\t# long" %}
11701   ins_encode %{
11702     __ testq($src1$$Register, $src2$$Register);
11703   %}
11704   ins_pipe(ialu_cr_reg_imm);
11705 %}
11706 
11707 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11708 %{
11709   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11710 
11711   format %{ "testq   $src, $mem" %}
11712   ins_encode %{
11713     __ testq($src$$Register, $mem$$Address);
11714   %}
11715   ins_pipe(ialu_cr_reg_mem);
11716 %}
11717 
11718 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11719 %{
11720   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11721 
11722   format %{ "testq   $src, $mem" %}
11723   ins_encode %{
11724     __ testq($src$$Register, $mem$$Address);
11725   %}
11726   ins_pipe(ialu_cr_reg_mem);
11727 %}
11728 
11729 // Manifest a CmpU result in an integer register.  Very painful.
11730 // This is the test to avoid.
11731 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
11732 %{
11733   match(Set dst (CmpU3 src1 src2));
11734   effect(KILL flags);
11735 
11736   ins_cost(275); // XXX
11737   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
11738             "movl    $dst, -1\n\t"
11739             "jb,u    done\n\t"
11740             "setne   $dst\n\t"
11741             "movzbl  $dst, $dst\n\t"
11742     "done:" %}
11743   ins_encode %{
11744     Label done;
11745     __ cmpl($src1$$Register, $src2$$Register);
11746     __ movl($dst$$Register, -1);
11747     __ jccb(Assembler::below, done);
11748     __ setb(Assembler::notZero, $dst$$Register);
11749     __ movzbl($dst$$Register, $dst$$Register);
11750     __ bind(done);
11751   %}
11752   ins_pipe(pipe_slow);
11753 %}
11754 
11755 // Manifest a CmpL result in an integer register.  Very painful.
11756 // This is the test to avoid.
11757 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11758 %{
11759   match(Set dst (CmpL3 src1 src2));
11760   effect(KILL flags);
11761 
11762   ins_cost(275); // XXX
11763   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11764             "movl    $dst, -1\n\t"
11765             "jl,s    done\n\t"
11766             "setne   $dst\n\t"
11767             "movzbl  $dst, $dst\n\t"
11768     "done:" %}
11769   ins_encode %{
11770     Label done;
11771     __ cmpq($src1$$Register, $src2$$Register);
11772     __ movl($dst$$Register, -1);
11773     __ jccb(Assembler::less, done);
11774     __ setb(Assembler::notZero, $dst$$Register);
11775     __ movzbl($dst$$Register, $dst$$Register);
11776     __ bind(done);
11777   %}
11778   ins_pipe(pipe_slow);
11779 %}
11780 
11781 // Manifest a CmpUL result in an integer register.  Very painful.
11782 // This is the test to avoid.
11783 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11784 %{
11785   match(Set dst (CmpUL3 src1 src2));
11786   effect(KILL flags);
11787 
11788   ins_cost(275); // XXX
11789   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11790             "movl    $dst, -1\n\t"
11791             "jb,u    done\n\t"
11792             "setne   $dst\n\t"
11793             "movzbl  $dst, $dst\n\t"
11794     "done:" %}
11795   ins_encode %{
11796     Label done;
11797     __ cmpq($src1$$Register, $src2$$Register);
11798     __ movl($dst$$Register, -1);
11799     __ jccb(Assembler::below, done);
11800     __ setb(Assembler::notZero, $dst$$Register);
11801     __ movzbl($dst$$Register, $dst$$Register);
11802     __ bind(done);
11803   %}
11804   ins_pipe(pipe_slow);
11805 %}
11806 
11807 // Unsigned long compare Instructions; really, same as signed long except they
11808 // produce an rFlagsRegU instead of rFlagsReg.
11809 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11810 %{
11811   match(Set cr (CmpUL op1 op2));
11812 
11813   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11814   ins_encode %{
11815     __ cmpq($op1$$Register, $op2$$Register);
11816   %}
11817   ins_pipe(ialu_cr_reg_reg);
11818 %}
11819 
11820 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11821 %{
11822   match(Set cr (CmpUL op1 op2));
11823 
11824   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11825   ins_encode %{
11826     __ cmpq($op1$$Register, $op2$$constant);
11827   %}
11828   ins_pipe(ialu_cr_reg_imm);
11829 %}
11830 
11831 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11832 %{
11833   match(Set cr (CmpUL op1 (LoadL op2)));
11834 
11835   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11836   ins_encode %{
11837     __ cmpq($op1$$Register, $op2$$Address);
11838   %}
11839   ins_pipe(ialu_cr_reg_mem);
11840 %}
11841 
11842 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11843 %{
11844   match(Set cr (CmpUL src zero));
11845 
11846   format %{ "testq   $src, $src\t# unsigned" %}
11847   ins_encode %{
11848     __ testq($src$$Register, $src$$Register);
11849   %}
11850   ins_pipe(ialu_cr_reg_imm);
11851 %}
11852 
11853 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11854 %{
11855   match(Set cr (CmpI (LoadB mem) imm));
11856 
11857   ins_cost(125);
11858   format %{ "cmpb    $mem, $imm" %}
11859   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
11860   ins_pipe(ialu_cr_reg_mem);
11861 %}
11862 
11863 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
11864 %{
11865   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
11866 
11867   ins_cost(125);
11868   format %{ "testb   $mem, $imm\t# ubyte" %}
11869   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11870   ins_pipe(ialu_cr_reg_mem);
11871 %}
11872 
11873 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
11874 %{
11875   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
11876 
11877   ins_cost(125);
11878   format %{ "testb   $mem, $imm\t# byte" %}
11879   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11880   ins_pipe(ialu_cr_reg_mem);
11881 %}
11882 
11883 //----------Max and Min--------------------------------------------------------
11884 // Min Instructions
11885 
11886 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11887 %{
11888   effect(USE_DEF dst, USE src, USE cr);
11889 
11890   format %{ "cmovlgt $dst, $src\t# min" %}
11891   ins_encode %{
11892     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
11893   %}
11894   ins_pipe(pipe_cmov_reg);
11895 %}
11896 
11897 
11898 instruct minI_rReg(rRegI dst, rRegI src)
11899 %{
11900   match(Set dst (MinI dst src));
11901 
11902   ins_cost(200);
11903   expand %{
11904     rFlagsReg cr;
11905     compI_rReg(cr, dst, src);
11906     cmovI_reg_g(dst, src, cr);
11907   %}
11908 %}
11909 
11910 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11911 %{
11912   effect(USE_DEF dst, USE src, USE cr);
11913 
11914   format %{ "cmovllt $dst, $src\t# max" %}
11915   ins_encode %{
11916     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
11917   %}
11918   ins_pipe(pipe_cmov_reg);
11919 %}
11920 
11921 
11922 instruct maxI_rReg(rRegI dst, rRegI src)
11923 %{
11924   match(Set dst (MaxI dst src));
11925 
11926   ins_cost(200);
11927   expand %{
11928     rFlagsReg cr;
11929     compI_rReg(cr, dst, src);
11930     cmovI_reg_l(dst, src, cr);
11931   %}
11932 %}
11933 
11934 // ============================================================================
11935 // Branch Instructions
11936 
11937 // Jump Direct - Label defines a relative address from JMP+1
11938 instruct jmpDir(label labl)
11939 %{
11940   match(Goto);
11941   effect(USE labl);
11942 
11943   ins_cost(300);
11944   format %{ "jmp     $labl" %}
11945   size(5);
11946   ins_encode %{
11947     Label* L = $labl$$label;
11948     __ jmp(*L, false); // Always long jump
11949   %}
11950   ins_pipe(pipe_jmp);
11951 %}
11952 
11953 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11954 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11955 %{
11956   match(If cop cr);
11957   effect(USE labl);
11958 
11959   ins_cost(300);
11960   format %{ "j$cop     $labl" %}
11961   size(6);
11962   ins_encode %{
11963     Label* L = $labl$$label;
11964     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11965   %}
11966   ins_pipe(pipe_jcc);
11967 %}
11968 
11969 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11970 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11971 %{
11972   match(CountedLoopEnd cop cr);
11973   effect(USE labl);
11974 
11975   ins_cost(300);
11976   format %{ "j$cop     $labl\t# loop end" %}
11977   size(6);
11978   ins_encode %{
11979     Label* L = $labl$$label;
11980     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11981   %}
11982   ins_pipe(pipe_jcc);
11983 %}
11984 
11985 // Jump Direct Conditional - using unsigned comparison
11986 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11987   match(If cop cmp);
11988   effect(USE labl);
11989 
11990   ins_cost(300);
11991   format %{ "j$cop,u   $labl" %}
11992   size(6);
11993   ins_encode %{
11994     Label* L = $labl$$label;
11995     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11996   %}
11997   ins_pipe(pipe_jcc);
11998 %}
11999 
12000 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12001   match(If cop cmp);
12002   effect(USE labl);
12003 
12004   ins_cost(200);
12005   format %{ "j$cop,u   $labl" %}
12006   size(6);
12007   ins_encode %{
12008     Label* L = $labl$$label;
12009     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12010   %}
12011   ins_pipe(pipe_jcc);
12012 %}
12013 
12014 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12015   match(If cop cmp);
12016   effect(USE labl);
12017 
12018   ins_cost(200);
12019   format %{ $$template
12020     if ($cop$$cmpcode == Assembler::notEqual) {
12021       $$emit$$"jp,u    $labl\n\t"
12022       $$emit$$"j$cop,u   $labl"
12023     } else {
12024       $$emit$$"jp,u    done\n\t"
12025       $$emit$$"j$cop,u   $labl\n\t"
12026       $$emit$$"done:"
12027     }
12028   %}
12029   ins_encode %{
12030     Label* l = $labl$$label;
12031     if ($cop$$cmpcode == Assembler::notEqual) {
12032       __ jcc(Assembler::parity, *l, false);
12033       __ jcc(Assembler::notEqual, *l, false);
12034     } else if ($cop$$cmpcode == Assembler::equal) {
12035       Label done;
12036       __ jccb(Assembler::parity, done);
12037       __ jcc(Assembler::equal, *l, false);
12038       __ bind(done);
12039     } else {
12040        ShouldNotReachHere();
12041     }
12042   %}
12043   ins_pipe(pipe_jcc);
12044 %}
12045 
12046 // ============================================================================
12047 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12048 // superklass array for an instance of the superklass.  Set a hidden
12049 // internal cache on a hit (cache is checked with exposed code in
12050 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12051 // encoding ALSO sets flags.
12052 
12053 instruct partialSubtypeCheck(rdi_RegP result,
12054                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12055                              rFlagsReg cr)
12056 %{
12057   match(Set result (PartialSubtypeCheck sub super));
12058   effect(KILL rcx, KILL cr);
12059 
12060   ins_cost(1100);  // slightly larger than the next version
12061   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12062             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12063             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12064             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12065             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12066             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12067             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12068     "miss:\t" %}
12069 
12070   opcode(0x1); // Force a XOR of RDI
12071   ins_encode(enc_PartialSubtypeCheck());
12072   ins_pipe(pipe_slow);
12073 %}
12074 
12075 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12076                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12077                                      immP0 zero,
12078                                      rdi_RegP result)
12079 %{
12080   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12081   effect(KILL rcx, KILL result);
12082 
12083   ins_cost(1000);
12084   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12085             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12086             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12087             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12088             "jne,s   miss\t\t# Missed: flags nz\n\t"
12089             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12090     "miss:\t" %}
12091 
12092   opcode(0x0); // No need to XOR RDI
12093   ins_encode(enc_PartialSubtypeCheck());
12094   ins_pipe(pipe_slow);
12095 %}
12096 
12097 // ============================================================================
12098 // Branch Instructions -- short offset versions
12099 //
12100 // These instructions are used to replace jumps of a long offset (the default
12101 // match) with jumps of a shorter offset.  These instructions are all tagged
12102 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12103 // match rules in general matching.  Instead, the ADLC generates a conversion
12104 // method in the MachNode which can be used to do in-place replacement of the
12105 // long variant with the shorter variant.  The compiler will determine if a
12106 // branch can be taken by the is_short_branch_offset() predicate in the machine
12107 // specific code section of the file.
12108 
12109 // Jump Direct - Label defines a relative address from JMP+1
12110 instruct jmpDir_short(label labl) %{
12111   match(Goto);
12112   effect(USE labl);
12113 
12114   ins_cost(300);
12115   format %{ "jmp,s   $labl" %}
12116   size(2);
12117   ins_encode %{
12118     Label* L = $labl$$label;
12119     __ jmpb(*L);
12120   %}
12121   ins_pipe(pipe_jmp);
12122   ins_short_branch(1);
12123 %}
12124 
12125 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12126 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12127   match(If cop cr);
12128   effect(USE labl);
12129 
12130   ins_cost(300);
12131   format %{ "j$cop,s   $labl" %}
12132   size(2);
12133   ins_encode %{
12134     Label* L = $labl$$label;
12135     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12136   %}
12137   ins_pipe(pipe_jcc);
12138   ins_short_branch(1);
12139 %}
12140 
12141 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12142 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12143   match(CountedLoopEnd cop cr);
12144   effect(USE labl);
12145 
12146   ins_cost(300);
12147   format %{ "j$cop,s   $labl\t# loop end" %}
12148   size(2);
12149   ins_encode %{
12150     Label* L = $labl$$label;
12151     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12152   %}
12153   ins_pipe(pipe_jcc);
12154   ins_short_branch(1);
12155 %}
12156 
12157 // Jump Direct Conditional - using unsigned comparison
12158 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12159   match(If cop cmp);
12160   effect(USE labl);
12161 
12162   ins_cost(300);
12163   format %{ "j$cop,us  $labl" %}
12164   size(2);
12165   ins_encode %{
12166     Label* L = $labl$$label;
12167     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12168   %}
12169   ins_pipe(pipe_jcc);
12170   ins_short_branch(1);
12171 %}
12172 
12173 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12174   match(If cop cmp);
12175   effect(USE labl);
12176 
12177   ins_cost(300);
12178   format %{ "j$cop,us  $labl" %}
12179   size(2);
12180   ins_encode %{
12181     Label* L = $labl$$label;
12182     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12183   %}
12184   ins_pipe(pipe_jcc);
12185   ins_short_branch(1);
12186 %}
12187 
12188 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12189   match(If cop cmp);
12190   effect(USE labl);
12191 
12192   ins_cost(300);
12193   format %{ $$template
12194     if ($cop$$cmpcode == Assembler::notEqual) {
12195       $$emit$$"jp,u,s  $labl\n\t"
12196       $$emit$$"j$cop,u,s  $labl"
12197     } else {
12198       $$emit$$"jp,u,s  done\n\t"
12199       $$emit$$"j$cop,u,s  $labl\n\t"
12200       $$emit$$"done:"
12201     }
12202   %}
12203   size(4);
12204   ins_encode %{
12205     Label* l = $labl$$label;
12206     if ($cop$$cmpcode == Assembler::notEqual) {
12207       __ jccb(Assembler::parity, *l);
12208       __ jccb(Assembler::notEqual, *l);
12209     } else if ($cop$$cmpcode == Assembler::equal) {
12210       Label done;
12211       __ jccb(Assembler::parity, done);
12212       __ jccb(Assembler::equal, *l);
12213       __ bind(done);
12214     } else {
12215        ShouldNotReachHere();
12216     }
12217   %}
12218   ins_pipe(pipe_jcc);
12219   ins_short_branch(1);
12220 %}
12221 
12222 // ============================================================================
12223 // inlined locking and unlocking
12224 
12225 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12226   predicate(Compile::current()->use_rtm());
12227   match(Set cr (FastLock object box));
12228   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12229   ins_cost(300);
12230   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12231   ins_encode %{
12232     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12233                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
12234                  _rtm_counters, _stack_rtm_counters,
12235                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12236                  true, ra_->C->profile_rtm());
12237   %}
12238   ins_pipe(pipe_slow);
12239 %}
12240 
12241 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12242   predicate(LockingMode != LM_LIGHTWEIGHT && !Compile::current()->use_rtm());
12243   match(Set cr (FastLock object box));
12244   effect(TEMP tmp, TEMP scr, USE_KILL box);
12245   ins_cost(300);
12246   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12247   ins_encode %{
12248     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12249                  $scr$$Register, noreg, noreg, r15_thread, nullptr, nullptr, nullptr, false, false);
12250   %}
12251   ins_pipe(pipe_slow);
12252 %}
12253 
12254 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12255   predicate(LockingMode != LM_LIGHTWEIGHT);
12256   match(Set cr (FastUnlock object box));
12257   effect(TEMP tmp, USE_KILL box);
12258   ins_cost(300);
12259   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12260   ins_encode %{
12261     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12262   %}
12263   ins_pipe(pipe_slow);
12264 %}
12265 
12266 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
12267   predicate(LockingMode == LM_LIGHTWEIGHT);
12268   match(Set cr (FastLock object box));
12269   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
12270   ins_cost(300);
12271   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
12272   ins_encode %{
12273     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12274   %}
12275   ins_pipe(pipe_slow);
12276 %}
12277 
12278 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
12279   predicate(LockingMode == LM_LIGHTWEIGHT);
12280   match(Set cr (FastUnlock object rax_reg));
12281   effect(TEMP tmp, USE_KILL rax_reg);
12282   ins_cost(300);
12283   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
12284   ins_encode %{
12285     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12286   %}
12287   ins_pipe(pipe_slow);
12288 %}
12289 
12290 
12291 // ============================================================================
12292 // Safepoint Instructions
12293 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12294 %{
12295   match(SafePoint poll);
12296   effect(KILL cr, USE poll);
12297 
12298   format %{ "testl   rax, [$poll]\t"
12299             "# Safepoint: poll for GC" %}
12300   ins_cost(125);
12301   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12302   ins_encode %{
12303     __ relocate(relocInfo::poll_type);
12304     address pre_pc = __ pc();
12305     __ testl(rax, Address($poll$$Register, 0));
12306     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12307   %}
12308   ins_pipe(ialu_reg_mem);
12309 %}
12310 
12311 instruct mask_all_evexL(kReg dst, rRegL src) %{
12312   match(Set dst (MaskAll src));
12313   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12314   ins_encode %{
12315     int mask_len = Matcher::vector_length(this);
12316     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12317   %}
12318   ins_pipe( pipe_slow );
12319 %}
12320 
12321 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12322   predicate(Matcher::vector_length(n) > 32);
12323   match(Set dst (MaskAll src));
12324   effect(TEMP tmp);
12325   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12326   ins_encode %{
12327     int mask_len = Matcher::vector_length(this);
12328     __ movslq($tmp$$Register, $src$$Register);
12329     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12330   %}
12331   ins_pipe( pipe_slow );
12332 %}
12333 
12334 // ============================================================================
12335 // Procedure Call/Return Instructions
12336 // Call Java Static Instruction
12337 // Note: If this code changes, the corresponding ret_addr_offset() and
12338 //       compute_padding() functions will have to be adjusted.
12339 instruct CallStaticJavaDirect(method meth) %{
12340   match(CallStaticJava);
12341   effect(USE meth);
12342 
12343   ins_cost(300);
12344   format %{ "call,static " %}
12345   opcode(0xE8); /* E8 cd */
12346   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12347   ins_pipe(pipe_slow);
12348   ins_alignment(4);
12349 %}
12350 
12351 // Call Java Dynamic Instruction
12352 // Note: If this code changes, the corresponding ret_addr_offset() and
12353 //       compute_padding() functions will have to be adjusted.
12354 instruct CallDynamicJavaDirect(method meth)
12355 %{
12356   match(CallDynamicJava);
12357   effect(USE meth);
12358 
12359   ins_cost(300);
12360   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12361             "call,dynamic " %}
12362   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12363   ins_pipe(pipe_slow);
12364   ins_alignment(4);
12365 %}
12366 
12367 // Call Runtime Instruction
12368 instruct CallRuntimeDirect(method meth)
12369 %{
12370   match(CallRuntime);
12371   effect(USE meth);
12372 
12373   ins_cost(300);
12374   format %{ "call,runtime " %}
12375   ins_encode(clear_avx, Java_To_Runtime(meth));
12376   ins_pipe(pipe_slow);
12377 %}
12378 
12379 // Call runtime without safepoint
12380 instruct CallLeafDirect(method meth)
12381 %{
12382   match(CallLeaf);
12383   effect(USE meth);
12384 
12385   ins_cost(300);
12386   format %{ "call_leaf,runtime " %}
12387   ins_encode(clear_avx, Java_To_Runtime(meth));
12388   ins_pipe(pipe_slow);
12389 %}
12390 
12391 // Call runtime without safepoint and with vector arguments
12392 instruct CallLeafDirectVector(method meth)
12393 %{
12394   match(CallLeafVector);
12395   effect(USE meth);
12396 
12397   ins_cost(300);
12398   format %{ "call_leaf,vector " %}
12399   ins_encode(Java_To_Runtime(meth));
12400   ins_pipe(pipe_slow);
12401 %}
12402 
12403 // Call runtime without safepoint
12404 instruct CallLeafNoFPDirect(method meth)
12405 %{
12406   match(CallLeafNoFP);
12407   effect(USE meth);
12408 
12409   ins_cost(300);
12410   format %{ "call_leaf_nofp,runtime " %}
12411   ins_encode(clear_avx, Java_To_Runtime(meth));
12412   ins_pipe(pipe_slow);
12413 %}
12414 
12415 // Return Instruction
12416 // Remove the return address & jump to it.
12417 // Notice: We always emit a nop after a ret to make sure there is room
12418 // for safepoint patching
12419 instruct Ret()
12420 %{
12421   match(Return);
12422 
12423   format %{ "ret" %}
12424   ins_encode %{
12425     __ ret(0);
12426   %}
12427   ins_pipe(pipe_jmp);
12428 %}
12429 
12430 // Tail Call; Jump from runtime stub to Java code.
12431 // Also known as an 'interprocedural jump'.
12432 // Target of jump will eventually return to caller.
12433 // TailJump below removes the return address.
12434 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12435 %{
12436   match(TailCall jump_target method_ptr);
12437 
12438   ins_cost(300);
12439   format %{ "jmp     $jump_target\t# rbx holds method" %}
12440   ins_encode %{
12441     __ jmp($jump_target$$Register);
12442   %}
12443   ins_pipe(pipe_jmp);
12444 %}
12445 
12446 // Tail Jump; remove the return address; jump to target.
12447 // TailCall above leaves the return address around.
12448 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12449 %{
12450   match(TailJump jump_target ex_oop);
12451 
12452   ins_cost(300);
12453   format %{ "popq    rdx\t# pop return address\n\t"
12454             "jmp     $jump_target" %}
12455   ins_encode %{
12456     __ popq(as_Register(RDX_enc));
12457     __ jmp($jump_target$$Register);
12458   %}
12459   ins_pipe(pipe_jmp);
12460 %}
12461 
12462 // Create exception oop: created by stack-crawling runtime code.
12463 // Created exception is now available to this handler, and is setup
12464 // just prior to jumping to this handler.  No code emitted.
12465 instruct CreateException(rax_RegP ex_oop)
12466 %{
12467   match(Set ex_oop (CreateEx));
12468 
12469   size(0);
12470   // use the following format syntax
12471   format %{ "# exception oop is in rax; no code emitted" %}
12472   ins_encode();
12473   ins_pipe(empty);
12474 %}
12475 
12476 // Rethrow exception:
12477 // The exception oop will come in the first argument position.
12478 // Then JUMP (not call) to the rethrow stub code.
12479 instruct RethrowException()
12480 %{
12481   match(Rethrow);
12482 
12483   // use the following format syntax
12484   format %{ "jmp     rethrow_stub" %}
12485   ins_encode %{
12486     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12487   %}
12488   ins_pipe(pipe_jmp);
12489 %}
12490 
12491 // ============================================================================
12492 // This name is KNOWN by the ADLC and cannot be changed.
12493 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12494 // for this guy.
12495 instruct tlsLoadP(r15_RegP dst) %{
12496   match(Set dst (ThreadLocal));
12497   effect(DEF dst);
12498 
12499   size(0);
12500   format %{ "# TLS is in R15" %}
12501   ins_encode( /*empty encoding*/ );
12502   ins_pipe(ialu_reg_reg);
12503 %}
12504 
12505 
12506 //----------PEEPHOLE RULES-----------------------------------------------------
12507 // These must follow all instruction definitions as they use the names
12508 // defined in the instructions definitions.
12509 //
12510 // peeppredicate ( rule_predicate );
12511 // // the predicate unless which the peephole rule will be ignored
12512 //
12513 // peepmatch ( root_instr_name [preceding_instruction]* );
12514 //
12515 // peepprocedure ( procedure_name );
12516 // // provide a procedure name to perform the optimization, the procedure should
12517 // // reside in the architecture dependent peephole file, the method has the
12518 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12519 // // with the arguments being the basic block, the current node index inside the
12520 // // block, the register allocator, the functions upon invoked return a new node
12521 // // defined in peepreplace, and the rules of the nodes appearing in the
12522 // // corresponding peepmatch, the function return true if successful, else
12523 // // return false
12524 //
12525 // peepconstraint %{
12526 // (instruction_number.operand_name relational_op instruction_number.operand_name
12527 //  [, ...] );
12528 // // instruction numbers are zero-based using left to right order in peepmatch
12529 //
12530 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12531 // // provide an instruction_number.operand_name for each operand that appears
12532 // // in the replacement instruction's match rule
12533 //
12534 // ---------VM FLAGS---------------------------------------------------------
12535 //
12536 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12537 //
12538 // Each peephole rule is given an identifying number starting with zero and
12539 // increasing by one in the order seen by the parser.  An individual peephole
12540 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12541 // on the command-line.
12542 //
12543 // ---------CURRENT LIMITATIONS----------------------------------------------
12544 //
12545 // Only transformations inside a basic block (do we need more for peephole)
12546 //
12547 // ---------EXAMPLE----------------------------------------------------------
12548 //
12549 // // pertinent parts of existing instructions in architecture description
12550 // instruct movI(rRegI dst, rRegI src)
12551 // %{
12552 //   match(Set dst (CopyI src));
12553 // %}
12554 //
12555 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12556 // %{
12557 //   match(Set dst (AddI dst src));
12558 //   effect(KILL cr);
12559 // %}
12560 //
12561 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12562 // %{
12563 //   match(Set dst (AddI dst src));
12564 // %}
12565 //
12566 // 1. Simple replacement
12567 // - Only match adjacent instructions in same basic block
12568 // - Only equality constraints
12569 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
12570 // - Only one replacement instruction
12571 //
12572 // // Change (inc mov) to lea
12573 // peephole %{
12574 //   // lea should only be emitted when beneficial
12575 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12576 //   // increment preceded by register-register move
12577 //   peepmatch ( incI_rReg movI );
12578 //   // require that the destination register of the increment
12579 //   // match the destination register of the move
12580 //   peepconstraint ( 0.dst == 1.dst );
12581 //   // construct a replacement instruction that sets
12582 //   // the destination to ( move's source register + one )
12583 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12584 // %}
12585 //
12586 // 2. Procedural replacement
12587 // - More flexible finding relevent nodes
12588 // - More flexible constraints
12589 // - More flexible transformations
12590 // - May utilise architecture-dependent API more effectively
12591 // - Currently only one replacement instruction due to adlc parsing capabilities
12592 //
12593 // // Change (inc mov) to lea
12594 // peephole %{
12595 //   // lea should only be emitted when beneficial
12596 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12597 //   // the rule numbers of these nodes inside are passed into the function below
12598 //   peepmatch ( incI_rReg movI );
12599 //   // the method that takes the responsibility of transformation
12600 //   peepprocedure ( inc_mov_to_lea );
12601 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
12602 //   // node is passed into the function above
12603 //   peepreplace ( leaI_rReg_immI() );
12604 // %}
12605 
12606 // These instructions is not matched by the matcher but used by the peephole
12607 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
12608 %{
12609   predicate(false);
12610   match(Set dst (AddI src1 src2));
12611   format %{ "leal    $dst, [$src1 + $src2]" %}
12612   ins_encode %{
12613     Register dst = $dst$$Register;
12614     Register src1 = $src1$$Register;
12615     Register src2 = $src2$$Register;
12616     if (src1 != rbp && src1 != r13) {
12617       __ leal(dst, Address(src1, src2, Address::times_1));
12618     } else {
12619       assert(src2 != rbp && src2 != r13, "");
12620       __ leal(dst, Address(src2, src1, Address::times_1));
12621     }
12622   %}
12623   ins_pipe(ialu_reg_reg);
12624 %}
12625 
12626 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
12627 %{
12628   predicate(false);
12629   match(Set dst (AddI src1 src2));
12630   format %{ "leal    $dst, [$src1 + $src2]" %}
12631   ins_encode %{
12632     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
12633   %}
12634   ins_pipe(ialu_reg_reg);
12635 %}
12636 
12637 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
12638 %{
12639   predicate(false);
12640   match(Set dst (LShiftI src shift));
12641   format %{ "leal    $dst, [$src << $shift]" %}
12642   ins_encode %{
12643     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12644     Register src = $src$$Register;
12645     if (scale == Address::times_2 && src != rbp && src != r13) {
12646       __ leal($dst$$Register, Address(src, src, Address::times_1));
12647     } else {
12648       __ leal($dst$$Register, Address(noreg, src, scale));
12649     }
12650   %}
12651   ins_pipe(ialu_reg_reg);
12652 %}
12653 
12654 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
12655 %{
12656   predicate(false);
12657   match(Set dst (AddL src1 src2));
12658   format %{ "leaq    $dst, [$src1 + $src2]" %}
12659   ins_encode %{
12660     Register dst = $dst$$Register;
12661     Register src1 = $src1$$Register;
12662     Register src2 = $src2$$Register;
12663     if (src1 != rbp && src1 != r13) {
12664       __ leaq(dst, Address(src1, src2, Address::times_1));
12665     } else {
12666       assert(src2 != rbp && src2 != r13, "");
12667       __ leaq(dst, Address(src2, src1, Address::times_1));
12668     }
12669   %}
12670   ins_pipe(ialu_reg_reg);
12671 %}
12672 
12673 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
12674 %{
12675   predicate(false);
12676   match(Set dst (AddL src1 src2));
12677   format %{ "leaq    $dst, [$src1 + $src2]" %}
12678   ins_encode %{
12679     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
12680   %}
12681   ins_pipe(ialu_reg_reg);
12682 %}
12683 
12684 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
12685 %{
12686   predicate(false);
12687   match(Set dst (LShiftL src shift));
12688   format %{ "leaq    $dst, [$src << $shift]" %}
12689   ins_encode %{
12690     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12691     Register src = $src$$Register;
12692     if (scale == Address::times_2 && src != rbp && src != r13) {
12693       __ leaq($dst$$Register, Address(src, src, Address::times_1));
12694     } else {
12695       __ leaq($dst$$Register, Address(noreg, src, scale));
12696     }
12697   %}
12698   ins_pipe(ialu_reg_reg);
12699 %}
12700 
12701 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
12702 // sal}) with lea instructions. The {add, sal} rules are beneficial in
12703 // processors with at least partial ALU support for lea
12704 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
12705 // beneficial for processors with full ALU support
12706 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
12707 
12708 peephole
12709 %{
12710   peeppredicate(VM_Version::supports_fast_2op_lea());
12711   peepmatch (addI_rReg);
12712   peepprocedure (lea_coalesce_reg);
12713   peepreplace (leaI_rReg_rReg_peep());
12714 %}
12715 
12716 peephole
12717 %{
12718   peeppredicate(VM_Version::supports_fast_2op_lea());
12719   peepmatch (addI_rReg_imm);
12720   peepprocedure (lea_coalesce_imm);
12721   peepreplace (leaI_rReg_immI_peep());
12722 %}
12723 
12724 peephole
12725 %{
12726   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12727                 VM_Version::is_intel_cascade_lake());
12728   peepmatch (incI_rReg);
12729   peepprocedure (lea_coalesce_imm);
12730   peepreplace (leaI_rReg_immI_peep());
12731 %}
12732 
12733 peephole
12734 %{
12735   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12736                 VM_Version::is_intel_cascade_lake());
12737   peepmatch (decI_rReg);
12738   peepprocedure (lea_coalesce_imm);
12739   peepreplace (leaI_rReg_immI_peep());
12740 %}
12741 
12742 peephole
12743 %{
12744   peeppredicate(VM_Version::supports_fast_2op_lea());
12745   peepmatch (salI_rReg_immI2);
12746   peepprocedure (lea_coalesce_imm);
12747   peepreplace (leaI_rReg_immI2_peep());
12748 %}
12749 
12750 peephole
12751 %{
12752   peeppredicate(VM_Version::supports_fast_2op_lea());
12753   peepmatch (addL_rReg);
12754   peepprocedure (lea_coalesce_reg);
12755   peepreplace (leaL_rReg_rReg_peep());
12756 %}
12757 
12758 peephole
12759 %{
12760   peeppredicate(VM_Version::supports_fast_2op_lea());
12761   peepmatch (addL_rReg_imm);
12762   peepprocedure (lea_coalesce_imm);
12763   peepreplace (leaL_rReg_immL32_peep());
12764 %}
12765 
12766 peephole
12767 %{
12768   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12769                 VM_Version::is_intel_cascade_lake());
12770   peepmatch (incL_rReg);
12771   peepprocedure (lea_coalesce_imm);
12772   peepreplace (leaL_rReg_immL32_peep());
12773 %}
12774 
12775 peephole
12776 %{
12777   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12778                 VM_Version::is_intel_cascade_lake());
12779   peepmatch (decL_rReg);
12780   peepprocedure (lea_coalesce_imm);
12781   peepreplace (leaL_rReg_immL32_peep());
12782 %}
12783 
12784 peephole
12785 %{
12786   peeppredicate(VM_Version::supports_fast_2op_lea());
12787   peepmatch (salL_rReg_immI2);
12788   peepprocedure (lea_coalesce_imm);
12789   peepreplace (leaL_rReg_immI2_peep());
12790 %}
12791 
12792 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
12793 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
12794 
12795 //int variant
12796 peephole
12797 %{
12798   peepmatch (testI_reg);
12799   peepprocedure (test_may_remove);
12800 %}
12801 
12802 //long variant
12803 peephole
12804 %{
12805   peepmatch (testL_reg);
12806   peepprocedure (test_may_remove);
12807 %}
12808 
12809 
12810 //----------SMARTSPILL RULES---------------------------------------------------
12811 // These must follow all instruction definitions as they use the names
12812 // defined in the instructions definitions.