1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for R11 long register
  300 reg_class long_r11_reg(R11, R11_H);
  301 
  302 // Singleton class for RAX int register
  303 reg_class int_rax_reg(RAX);
  304 
  305 // Singleton class for RBX int register
  306 reg_class int_rbx_reg(RBX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rcx_reg(RCX);
  310 
  311 // Singleton class for RDX int register
  312 reg_class int_rdx_reg(RDX);
  313 
  314 // Singleton class for RDI int register
  315 reg_class int_rdi_reg(RDI);
  316 
  317 // Singleton class for instruction pointer
  318 // reg_class ip_reg(RIP);
  319 
  320 %}
  321 
  322 //----------SOURCE BLOCK-------------------------------------------------------
  323 // This is a block of C++ code which provides values, functions, and
  324 // definitions necessary in the rest of the architecture description
  325 
  326 source_hpp %{
  327 
  328 #include "peephole_x86_64.hpp"
  329 
  330 %}
  331 
  332 // Register masks
  333 source_hpp %{
  334 
  335 extern RegMask _ANY_REG_mask;
  336 extern RegMask _PTR_REG_mask;
  337 extern RegMask _PTR_REG_NO_RBP_mask;
  338 extern RegMask _PTR_NO_RAX_REG_mask;
  339 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  340 extern RegMask _LONG_REG_mask;
  341 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  342 extern RegMask _LONG_NO_RCX_REG_mask;
  343 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  344 extern RegMask _INT_REG_mask;
  345 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  346 extern RegMask _INT_NO_RCX_REG_mask;
  347 extern RegMask _INT_NO_RBP_R13_REG_mask;
  348 extern RegMask _FLOAT_REG_mask;
  349 
  350 extern RegMask _STACK_OR_PTR_REG_mask;
  351 extern RegMask _STACK_OR_LONG_REG_mask;
  352 extern RegMask _STACK_OR_INT_REG_mask;
  353 
  354 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  355 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  356 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  357 
  358 %}
  359 
  360 source %{
  361 #define   RELOC_IMM64    Assembler::imm_operand
  362 #define   RELOC_DISP32   Assembler::disp32_operand
  363 
  364 #define __ masm->
  365 
  366 RegMask _ANY_REG_mask;
  367 RegMask _PTR_REG_mask;
  368 RegMask _PTR_REG_NO_RBP_mask;
  369 RegMask _PTR_NO_RAX_REG_mask;
  370 RegMask _PTR_NO_RAX_RBX_REG_mask;
  371 RegMask _LONG_REG_mask;
  372 RegMask _LONG_NO_RAX_RDX_REG_mask;
  373 RegMask _LONG_NO_RCX_REG_mask;
  374 RegMask _LONG_NO_RBP_R13_REG_mask;
  375 RegMask _INT_REG_mask;
  376 RegMask _INT_NO_RAX_RDX_REG_mask;
  377 RegMask _INT_NO_RCX_REG_mask;
  378 RegMask _INT_NO_RBP_R13_REG_mask;
  379 RegMask _FLOAT_REG_mask;
  380 RegMask _STACK_OR_PTR_REG_mask;
  381 RegMask _STACK_OR_LONG_REG_mask;
  382 RegMask _STACK_OR_INT_REG_mask;
  383 
  384 static bool need_r12_heapbase() {
  385   return UseCompressedOops;
  386 }
  387 
  388 void reg_mask_init() {
  389   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  390   // We derive a number of subsets from it.
  391   _ANY_REG_mask = _ALL_REG_mask;
  392 
  393   if (PreserveFramePointer) {
  394     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  396   }
  397   if (need_r12_heapbase()) {
  398     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  399     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  400   }
  401 
  402   _PTR_REG_mask = _ANY_REG_mask;
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  404   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  405   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  406   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  407 
  408   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  409   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  410 
  411   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  412   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  413   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  414 
  415   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  416   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  417   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  418 
  419   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  420   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  421   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  422 
  423   _LONG_REG_mask = _PTR_REG_mask;
  424   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  425   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  426 
  427   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  429   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  430   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  431   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  432 
  433   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  434   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  435   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  436 
  437   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  439   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  440   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  441   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  442 
  443   _INT_REG_mask = _ALL_INT_REG_mask;
  444   if (PreserveFramePointer) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  446   }
  447   if (need_r12_heapbase()) {
  448     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  449   }
  450 
  451   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  452   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  453 
  454   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  455   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  456   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  457 
  458   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  459   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  460 
  461   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  462   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  463   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  464 
  465   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  466   // from the float_reg_legacy/float_reg_evex register class.
  467   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  468 }
  469 
  470 static bool generate_vzeroupper(Compile* C) {
  471   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  472 }
  473 
  474 static int clear_avx_size() {
  475   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  476 }
  477 
  478 // !!!!! Special hack to get all types of calls to specify the byte offset
  479 //       from the start of the call to the point where the return address
  480 //       will point.
  481 int MachCallStaticJavaNode::ret_addr_offset()
  482 {
  483   int offset = 5; // 5 bytes from start of call to where return address points
  484   offset += clear_avx_size();
  485   return offset;
  486 }
  487 
  488 int MachCallDynamicJavaNode::ret_addr_offset()
  489 {
  490   int offset = 15; // 15 bytes from start of call to where return address points
  491   offset += clear_avx_size();
  492   return offset;
  493 }
  494 
  495 int MachCallRuntimeNode::ret_addr_offset() {
  496   int offset = 13; // movq r10,#addr; callq (r10)
  497   if (this->ideal_Opcode() != Op_CallLeafVector) {
  498     offset += clear_avx_size();
  499   }
  500   return offset;
  501 }
  502 //
  503 // Compute padding required for nodes which need alignment
  504 //
  505 
  506 // The address of the call instruction needs to be 4-byte aligned to
  507 // ensure that it does not span a cache line so that it can be patched.
  508 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  509 {
  510   current_offset += clear_avx_size(); // skip vzeroupper
  511   current_offset += 1; // skip call opcode byte
  512   return align_up(current_offset, alignment_required()) - current_offset;
  513 }
  514 
  515 // The address of the call instruction needs to be 4-byte aligned to
  516 // ensure that it does not span a cache line so that it can be patched.
  517 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  518 {
  519   current_offset += clear_avx_size(); // skip vzeroupper
  520   current_offset += 11; // skip movq instruction + call opcode byte
  521   return align_up(current_offset, alignment_required()) - current_offset;
  522 }
  523 
  524 // This could be in MacroAssembler but it's fairly C2 specific
  525 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  526   Label exit;
  527   __ jccb(Assembler::noParity, exit);
  528   __ pushf();
  529   //
  530   // comiss/ucomiss instructions set ZF,PF,CF flags and
  531   // zero OF,AF,SF for NaN values.
  532   // Fixup flags by zeroing ZF,PF so that compare of NaN
  533   // values returns 'less than' result (CF is set).
  534   // Leave the rest of flags unchanged.
  535   //
  536   //    7 6 5 4 3 2 1 0
  537   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  538   //    0 0 1 0 1 0 1 1   (0x2B)
  539   //
  540   __ andq(Address(rsp, 0), 0xffffff2b);
  541   __ popf();
  542   __ bind(exit);
  543 }
  544 
  545 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  546   Label done;
  547   __ movl(dst, -1);
  548   __ jcc(Assembler::parity, done);
  549   __ jcc(Assembler::below, done);
  550   __ setb(Assembler::notEqual, dst);
  551   __ movzbl(dst, dst);
  552   __ bind(done);
  553 }
  554 
  555 // Math.min()    # Math.max()
  556 // --------------------------
  557 // ucomis[s/d]   #
  558 // ja   -> b     # a
  559 // jp   -> NaN   # NaN
  560 // jb   -> a     # b
  561 // je            #
  562 // |-jz -> a | b # a & b
  563 // |    -> a     #
  564 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  565                             XMMRegister a, XMMRegister b,
  566                             XMMRegister xmmt, Register rt,
  567                             bool min, bool single) {
  568 
  569   Label nan, zero, below, above, done;
  570 
  571   if (single)
  572     __ ucomiss(a, b);
  573   else
  574     __ ucomisd(a, b);
  575 
  576   if (dst->encoding() != (min ? b : a)->encoding())
  577     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  578   else
  579     __ jccb(Assembler::above, done);
  580 
  581   __ jccb(Assembler::parity, nan);  // PF=1
  582   __ jccb(Assembler::below, below); // CF=1
  583 
  584   // equal
  585   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  586   if (single) {
  587     __ ucomiss(a, xmmt);
  588     __ jccb(Assembler::equal, zero);
  589 
  590     __ movflt(dst, a);
  591     __ jmp(done);
  592   }
  593   else {
  594     __ ucomisd(a, xmmt);
  595     __ jccb(Assembler::equal, zero);
  596 
  597     __ movdbl(dst, a);
  598     __ jmp(done);
  599   }
  600 
  601   __ bind(zero);
  602   if (min)
  603     __ vpor(dst, a, b, Assembler::AVX_128bit);
  604   else
  605     __ vpand(dst, a, b, Assembler::AVX_128bit);
  606 
  607   __ jmp(done);
  608 
  609   __ bind(above);
  610   if (single)
  611     __ movflt(dst, min ? b : a);
  612   else
  613     __ movdbl(dst, min ? b : a);
  614 
  615   __ jmp(done);
  616 
  617   __ bind(nan);
  618   if (single) {
  619     __ movl(rt, 0x7fc00000); // Float.NaN
  620     __ movdl(dst, rt);
  621   }
  622   else {
  623     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  624     __ movdq(dst, rt);
  625   }
  626   __ jmp(done);
  627 
  628   __ bind(below);
  629   if (single)
  630     __ movflt(dst, min ? a : b);
  631   else
  632     __ movdbl(dst, min ? a : b);
  633 
  634   __ bind(done);
  635 }
  636 
  637 //=============================================================================
  638 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  639 
  640 int ConstantTable::calculate_table_base_offset() const {
  641   return 0;  // absolute addressing, no offset
  642 }
  643 
  644 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  645 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  646   ShouldNotReachHere();
  647 }
  648 
  649 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  650   // Empty encoding
  651 }
  652 
  653 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  654   return 0;
  655 }
  656 
  657 #ifndef PRODUCT
  658 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  659   st->print("# MachConstantBaseNode (empty encoding)");
  660 }
  661 #endif
  662 
  663 
  664 //=============================================================================
  665 #ifndef PRODUCT
  666 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  667   Compile* C = ra_->C;
  668 
  669   int framesize = C->output()->frame_size_in_bytes();
  670   int bangsize = C->output()->bang_size_in_bytes();
  671   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  672   // Remove wordSize for return addr which is already pushed.
  673   framesize -= wordSize;
  674 
  675   if (C->output()->need_stack_bang(bangsize)) {
  676     framesize -= wordSize;
  677     st->print("# stack bang (%d bytes)", bangsize);
  678     st->print("\n\t");
  679     st->print("pushq   rbp\t# Save rbp");
  680     if (PreserveFramePointer) {
  681         st->print("\n\t");
  682         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  683     }
  684     if (framesize) {
  685       st->print("\n\t");
  686       st->print("subq    rsp, #%d\t# Create frame",framesize);
  687     }
  688   } else {
  689     st->print("subq    rsp, #%d\t# Create frame",framesize);
  690     st->print("\n\t");
  691     framesize -= wordSize;
  692     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  693     if (PreserveFramePointer) {
  694       st->print("\n\t");
  695       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  696       if (framesize > 0) {
  697         st->print("\n\t");
  698         st->print("addq    rbp, #%d", framesize);
  699       }
  700     }
  701   }
  702 
  703   if (VerifyStackAtCalls) {
  704     st->print("\n\t");
  705     framesize -= wordSize;
  706     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  707 #ifdef ASSERT
  708     st->print("\n\t");
  709     st->print("# stack alignment check");
  710 #endif
  711   }
  712   if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
  713     st->print("\n\t");
  714     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  715     st->print("\n\t");
  716     st->print("je      fast_entry\t");
  717     st->print("\n\t");
  718     st->print("call    #nmethod_entry_barrier_stub\t");
  719     st->print("\n\tfast_entry:");
  720   }
  721   st->cr();
  722 }
  723 #endif
  724 
  725 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  726   Compile* C = ra_->C;
  727 
  728   int framesize = C->output()->frame_size_in_bytes();
  729   int bangsize = C->output()->bang_size_in_bytes();
  730 
  731   if (C->clinit_barrier_on_entry()) {
  732     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  733     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  734 
  735     Label L_skip_barrier;
  736     Register klass = rscratch1;
  737 
  738     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  739     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  740 
  741     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  742 
  743     __ bind(L_skip_barrier);
  744   }
  745 
  746   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  747 
  748   C->output()->set_frame_complete(__ offset());
  749 
  750   if (C->has_mach_constant_base_node()) {
  751     // NOTE: We set the table base offset here because users might be
  752     // emitted before MachConstantBaseNode.
  753     ConstantTable& constant_table = C->output()->constant_table();
  754     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  755   }
  756 }
  757 
  758 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  759 {
  760   return MachNode::size(ra_); // too many variables; just compute it
  761                               // the hard way
  762 }
  763 
  764 int MachPrologNode::reloc() const
  765 {
  766   return 0; // a large enough number
  767 }
  768 
  769 //=============================================================================
  770 #ifndef PRODUCT
  771 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  772 {
  773   Compile* C = ra_->C;
  774   if (generate_vzeroupper(C)) {
  775     st->print("vzeroupper");
  776     st->cr(); st->print("\t");
  777   }
  778 
  779   int framesize = C->output()->frame_size_in_bytes();
  780   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  781   // Remove word for return adr already pushed
  782   // and RBP
  783   framesize -= 2*wordSize;
  784 
  785   if (framesize) {
  786     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  787     st->print("\t");
  788   }
  789 
  790   st->print_cr("popq    rbp");
  791   if (do_polling() && C->is_method_compilation()) {
  792     st->print("\t");
  793     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  794                  "ja      #safepoint_stub\t"
  795                  "# Safepoint: poll for GC");
  796   }
  797 }
  798 #endif
  799 
  800 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  801 {
  802   Compile* C = ra_->C;
  803 
  804   if (generate_vzeroupper(C)) {
  805     // Clear upper bits of YMM registers when current compiled code uses
  806     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  807     __ vzeroupper();
  808   }
  809 
  810   int framesize = C->output()->frame_size_in_bytes();
  811   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  812   // Remove word for return adr already pushed
  813   // and RBP
  814   framesize -= 2*wordSize;
  815 
  816   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  817 
  818   if (framesize) {
  819     __ addq(rsp, framesize);
  820   }
  821 
  822   __ popq(rbp);
  823 
  824   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  825     __ reserved_stack_check();
  826   }
  827 
  828   if (do_polling() && C->is_method_compilation()) {
  829     Label dummy_label;
  830     Label* code_stub = &dummy_label;
  831     if (!C->output()->in_scratch_emit_size()) {
  832       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  833       C->output()->add_stub(stub);
  834       code_stub = &stub->entry();
  835     }
  836     __ relocate(relocInfo::poll_return_type);
  837     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  838   }
  839 }
  840 
  841 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  842 {
  843   return MachNode::size(ra_); // too many variables; just compute it
  844                               // the hard way
  845 }
  846 
  847 int MachEpilogNode::reloc() const
  848 {
  849   return 2; // a large enough number
  850 }
  851 
  852 const Pipeline* MachEpilogNode::pipeline() const
  853 {
  854   return MachNode::pipeline_class();
  855 }
  856 
  857 //=============================================================================
  858 
  859 enum RC {
  860   rc_bad,
  861   rc_int,
  862   rc_kreg,
  863   rc_float,
  864   rc_stack
  865 };
  866 
  867 static enum RC rc_class(OptoReg::Name reg)
  868 {
  869   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  870 
  871   if (OptoReg::is_stack(reg)) return rc_stack;
  872 
  873   VMReg r = OptoReg::as_VMReg(reg);
  874 
  875   if (r->is_Register()) return rc_int;
  876 
  877   if (r->is_KRegister()) return rc_kreg;
  878 
  879   assert(r->is_XMMRegister(), "must be");
  880   return rc_float;
  881 }
  882 
  883 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  884 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
  885                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  886 
  887 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
  888                      int stack_offset, int reg, uint ireg, outputStream* st);
  889 
  890 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
  891                                       int dst_offset, uint ireg, outputStream* st) {
  892   if (masm) {
  893     switch (ireg) {
  894     case Op_VecS:
  895       __ movq(Address(rsp, -8), rax);
  896       __ movl(rax, Address(rsp, src_offset));
  897       __ movl(Address(rsp, dst_offset), rax);
  898       __ movq(rax, Address(rsp, -8));
  899       break;
  900     case Op_VecD:
  901       __ pushq(Address(rsp, src_offset));
  902       __ popq (Address(rsp, dst_offset));
  903       break;
  904     case Op_VecX:
  905       __ pushq(Address(rsp, src_offset));
  906       __ popq (Address(rsp, dst_offset));
  907       __ pushq(Address(rsp, src_offset+8));
  908       __ popq (Address(rsp, dst_offset+8));
  909       break;
  910     case Op_VecY:
  911       __ vmovdqu(Address(rsp, -32), xmm0);
  912       __ vmovdqu(xmm0, Address(rsp, src_offset));
  913       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  914       __ vmovdqu(xmm0, Address(rsp, -32));
  915       break;
  916     case Op_VecZ:
  917       __ evmovdquq(Address(rsp, -64), xmm0, 2);
  918       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
  919       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
  920       __ evmovdquq(xmm0, Address(rsp, -64), 2);
  921       break;
  922     default:
  923       ShouldNotReachHere();
  924     }
  925 #ifndef PRODUCT
  926   } else {
  927     switch (ireg) {
  928     case Op_VecS:
  929       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
  930                 "movl    rax, [rsp + #%d]\n\t"
  931                 "movl    [rsp + #%d], rax\n\t"
  932                 "movq    rax, [rsp - #8]",
  933                 src_offset, dst_offset);
  934       break;
  935     case Op_VecD:
  936       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
  937                 "popq    [rsp + #%d]",
  938                 src_offset, dst_offset);
  939       break;
  940      case Op_VecX:
  941       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
  942                 "popq    [rsp + #%d]\n\t"
  943                 "pushq   [rsp + #%d]\n\t"
  944                 "popq    [rsp + #%d]",
  945                 src_offset, dst_offset, src_offset+8, dst_offset+8);
  946       break;
  947     case Op_VecY:
  948       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
  949                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  950                 "vmovdqu [rsp + #%d], xmm0\n\t"
  951                 "vmovdqu xmm0, [rsp - #32]",
  952                 src_offset, dst_offset);
  953       break;
  954     case Op_VecZ:
  955       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
  956                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  957                 "vmovdqu [rsp + #%d], xmm0\n\t"
  958                 "vmovdqu xmm0, [rsp - #64]",
  959                 src_offset, dst_offset);
  960       break;
  961     default:
  962       ShouldNotReachHere();
  963     }
  964 #endif
  965   }
  966 }
  967 
  968 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
  969                                        PhaseRegAlloc* ra_,
  970                                        bool do_size,
  971                                        outputStream* st) const {
  972   assert(masm != nullptr || st  != nullptr, "sanity");
  973   // Get registers to move
  974   OptoReg::Name src_second = ra_->get_reg_second(in(1));
  975   OptoReg::Name src_first = ra_->get_reg_first(in(1));
  976   OptoReg::Name dst_second = ra_->get_reg_second(this);
  977   OptoReg::Name dst_first = ra_->get_reg_first(this);
  978 
  979   enum RC src_second_rc = rc_class(src_second);
  980   enum RC src_first_rc = rc_class(src_first);
  981   enum RC dst_second_rc = rc_class(dst_second);
  982   enum RC dst_first_rc = rc_class(dst_first);
  983 
  984   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
  985          "must move at least 1 register" );
  986 
  987   if (src_first == dst_first && src_second == dst_second) {
  988     // Self copy, no move
  989     return 0;
  990   }
  991   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
  992     uint ireg = ideal_reg();
  993     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
  994     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
  995     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
  996       // mem -> mem
  997       int src_offset = ra_->reg2offset(src_first);
  998       int dst_offset = ra_->reg2offset(dst_first);
  999       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1000     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1001       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1002     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1003       int stack_offset = ra_->reg2offset(dst_first);
 1004       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1005     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1006       int stack_offset = ra_->reg2offset(src_first);
 1007       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1008     } else {
 1009       ShouldNotReachHere();
 1010     }
 1011     return 0;
 1012   }
 1013   if (src_first_rc == rc_stack) {
 1014     // mem ->
 1015     if (dst_first_rc == rc_stack) {
 1016       // mem -> mem
 1017       assert(src_second != dst_first, "overlap");
 1018       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1019           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1020         // 64-bit
 1021         int src_offset = ra_->reg2offset(src_first);
 1022         int dst_offset = ra_->reg2offset(dst_first);
 1023         if (masm) {
 1024           __ pushq(Address(rsp, src_offset));
 1025           __ popq (Address(rsp, dst_offset));
 1026 #ifndef PRODUCT
 1027         } else {
 1028           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1029                     "popq    [rsp + #%d]",
 1030                      src_offset, dst_offset);
 1031 #endif
 1032         }
 1033       } else {
 1034         // 32-bit
 1035         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1036         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1037         // No pushl/popl, so:
 1038         int src_offset = ra_->reg2offset(src_first);
 1039         int dst_offset = ra_->reg2offset(dst_first);
 1040         if (masm) {
 1041           __ movq(Address(rsp, -8), rax);
 1042           __ movl(rax, Address(rsp, src_offset));
 1043           __ movl(Address(rsp, dst_offset), rax);
 1044           __ movq(rax, Address(rsp, -8));
 1045 #ifndef PRODUCT
 1046         } else {
 1047           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1048                     "movl    rax, [rsp + #%d]\n\t"
 1049                     "movl    [rsp + #%d], rax\n\t"
 1050                     "movq    rax, [rsp - #8]",
 1051                      src_offset, dst_offset);
 1052 #endif
 1053         }
 1054       }
 1055       return 0;
 1056     } else if (dst_first_rc == rc_int) {
 1057       // mem -> gpr
 1058       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1059           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1060         // 64-bit
 1061         int offset = ra_->reg2offset(src_first);
 1062         if (masm) {
 1063           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1064 #ifndef PRODUCT
 1065         } else {
 1066           st->print("movq    %s, [rsp + #%d]\t# spill",
 1067                      Matcher::regName[dst_first],
 1068                      offset);
 1069 #endif
 1070         }
 1071       } else {
 1072         // 32-bit
 1073         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1074         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1075         int offset = ra_->reg2offset(src_first);
 1076         if (masm) {
 1077           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1078 #ifndef PRODUCT
 1079         } else {
 1080           st->print("movl    %s, [rsp + #%d]\t# spill",
 1081                      Matcher::regName[dst_first],
 1082                      offset);
 1083 #endif
 1084         }
 1085       }
 1086       return 0;
 1087     } else if (dst_first_rc == rc_float) {
 1088       // mem-> xmm
 1089       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1090           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1091         // 64-bit
 1092         int offset = ra_->reg2offset(src_first);
 1093         if (masm) {
 1094           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1095 #ifndef PRODUCT
 1096         } else {
 1097           st->print("%s  %s, [rsp + #%d]\t# spill",
 1098                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1099                      Matcher::regName[dst_first],
 1100                      offset);
 1101 #endif
 1102         }
 1103       } else {
 1104         // 32-bit
 1105         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1106         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1107         int offset = ra_->reg2offset(src_first);
 1108         if (masm) {
 1109           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1110 #ifndef PRODUCT
 1111         } else {
 1112           st->print("movss   %s, [rsp + #%d]\t# spill",
 1113                      Matcher::regName[dst_first],
 1114                      offset);
 1115 #endif
 1116         }
 1117       }
 1118       return 0;
 1119     } else if (dst_first_rc == rc_kreg) {
 1120       // mem -> kreg
 1121       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1122           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1123         // 64-bit
 1124         int offset = ra_->reg2offset(src_first);
 1125         if (masm) {
 1126           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1127 #ifndef PRODUCT
 1128         } else {
 1129           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1130                      Matcher::regName[dst_first],
 1131                      offset);
 1132 #endif
 1133         }
 1134       }
 1135       return 0;
 1136     }
 1137   } else if (src_first_rc == rc_int) {
 1138     // gpr ->
 1139     if (dst_first_rc == rc_stack) {
 1140       // gpr -> mem
 1141       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1142           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1143         // 64-bit
 1144         int offset = ra_->reg2offset(dst_first);
 1145         if (masm) {
 1146           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1147 #ifndef PRODUCT
 1148         } else {
 1149           st->print("movq    [rsp + #%d], %s\t# spill",
 1150                      offset,
 1151                      Matcher::regName[src_first]);
 1152 #endif
 1153         }
 1154       } else {
 1155         // 32-bit
 1156         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1157         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1158         int offset = ra_->reg2offset(dst_first);
 1159         if (masm) {
 1160           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1161 #ifndef PRODUCT
 1162         } else {
 1163           st->print("movl    [rsp + #%d], %s\t# spill",
 1164                      offset,
 1165                      Matcher::regName[src_first]);
 1166 #endif
 1167         }
 1168       }
 1169       return 0;
 1170     } else if (dst_first_rc == rc_int) {
 1171       // gpr -> gpr
 1172       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1173           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1174         // 64-bit
 1175         if (masm) {
 1176           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1177                   as_Register(Matcher::_regEncode[src_first]));
 1178 #ifndef PRODUCT
 1179         } else {
 1180           st->print("movq    %s, %s\t# spill",
 1181                      Matcher::regName[dst_first],
 1182                      Matcher::regName[src_first]);
 1183 #endif
 1184         }
 1185         return 0;
 1186       } else {
 1187         // 32-bit
 1188         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1189         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1190         if (masm) {
 1191           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1192                   as_Register(Matcher::_regEncode[src_first]));
 1193 #ifndef PRODUCT
 1194         } else {
 1195           st->print("movl    %s, %s\t# spill",
 1196                      Matcher::regName[dst_first],
 1197                      Matcher::regName[src_first]);
 1198 #endif
 1199         }
 1200         return 0;
 1201       }
 1202     } else if (dst_first_rc == rc_float) {
 1203       // gpr -> xmm
 1204       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1205           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1206         // 64-bit
 1207         if (masm) {
 1208           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1209 #ifndef PRODUCT
 1210         } else {
 1211           st->print("movdq   %s, %s\t# spill",
 1212                      Matcher::regName[dst_first],
 1213                      Matcher::regName[src_first]);
 1214 #endif
 1215         }
 1216       } else {
 1217         // 32-bit
 1218         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1219         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1220         if (masm) {
 1221           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1222 #ifndef PRODUCT
 1223         } else {
 1224           st->print("movdl   %s, %s\t# spill",
 1225                      Matcher::regName[dst_first],
 1226                      Matcher::regName[src_first]);
 1227 #endif
 1228         }
 1229       }
 1230       return 0;
 1231     } else if (dst_first_rc == rc_kreg) {
 1232       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1233           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1234         // 64-bit
 1235         if (masm) {
 1236           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1237   #ifndef PRODUCT
 1238         } else {
 1239            st->print("kmovq   %s, %s\t# spill",
 1240                        Matcher::regName[dst_first],
 1241                        Matcher::regName[src_first]);
 1242   #endif
 1243         }
 1244       }
 1245       Unimplemented();
 1246       return 0;
 1247     }
 1248   } else if (src_first_rc == rc_float) {
 1249     // xmm ->
 1250     if (dst_first_rc == rc_stack) {
 1251       // xmm -> mem
 1252       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1253           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1254         // 64-bit
 1255         int offset = ra_->reg2offset(dst_first);
 1256         if (masm) {
 1257           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1258 #ifndef PRODUCT
 1259         } else {
 1260           st->print("movsd   [rsp + #%d], %s\t# spill",
 1261                      offset,
 1262                      Matcher::regName[src_first]);
 1263 #endif
 1264         }
 1265       } else {
 1266         // 32-bit
 1267         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1268         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1269         int offset = ra_->reg2offset(dst_first);
 1270         if (masm) {
 1271           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1272 #ifndef PRODUCT
 1273         } else {
 1274           st->print("movss   [rsp + #%d], %s\t# spill",
 1275                      offset,
 1276                      Matcher::regName[src_first]);
 1277 #endif
 1278         }
 1279       }
 1280       return 0;
 1281     } else if (dst_first_rc == rc_int) {
 1282       // xmm -> gpr
 1283       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1284           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1285         // 64-bit
 1286         if (masm) {
 1287           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1288 #ifndef PRODUCT
 1289         } else {
 1290           st->print("movdq   %s, %s\t# spill",
 1291                      Matcher::regName[dst_first],
 1292                      Matcher::regName[src_first]);
 1293 #endif
 1294         }
 1295       } else {
 1296         // 32-bit
 1297         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1298         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1299         if (masm) {
 1300           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1301 #ifndef PRODUCT
 1302         } else {
 1303           st->print("movdl   %s, %s\t# spill",
 1304                      Matcher::regName[dst_first],
 1305                      Matcher::regName[src_first]);
 1306 #endif
 1307         }
 1308       }
 1309       return 0;
 1310     } else if (dst_first_rc == rc_float) {
 1311       // xmm -> xmm
 1312       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1313           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1314         // 64-bit
 1315         if (masm) {
 1316           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1317 #ifndef PRODUCT
 1318         } else {
 1319           st->print("%s  %s, %s\t# spill",
 1320                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1321                      Matcher::regName[dst_first],
 1322                      Matcher::regName[src_first]);
 1323 #endif
 1324         }
 1325       } else {
 1326         // 32-bit
 1327         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1328         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1329         if (masm) {
 1330           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1331 #ifndef PRODUCT
 1332         } else {
 1333           st->print("%s  %s, %s\t# spill",
 1334                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1335                      Matcher::regName[dst_first],
 1336                      Matcher::regName[src_first]);
 1337 #endif
 1338         }
 1339       }
 1340       return 0;
 1341     } else if (dst_first_rc == rc_kreg) {
 1342       assert(false, "Illegal spilling");
 1343       return 0;
 1344     }
 1345   } else if (src_first_rc == rc_kreg) {
 1346     if (dst_first_rc == rc_stack) {
 1347       // mem -> kreg
 1348       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1349           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1350         // 64-bit
 1351         int offset = ra_->reg2offset(dst_first);
 1352         if (masm) {
 1353           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1354 #ifndef PRODUCT
 1355         } else {
 1356           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1357                      offset,
 1358                      Matcher::regName[src_first]);
 1359 #endif
 1360         }
 1361       }
 1362       return 0;
 1363     } else if (dst_first_rc == rc_int) {
 1364       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1365           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1366         // 64-bit
 1367         if (masm) {
 1368           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1369 #ifndef PRODUCT
 1370         } else {
 1371          st->print("kmovq   %s, %s\t# spill",
 1372                      Matcher::regName[dst_first],
 1373                      Matcher::regName[src_first]);
 1374 #endif
 1375         }
 1376       }
 1377       Unimplemented();
 1378       return 0;
 1379     } else if (dst_first_rc == rc_kreg) {
 1380       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1381           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1382         // 64-bit
 1383         if (masm) {
 1384           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1385 #ifndef PRODUCT
 1386         } else {
 1387          st->print("kmovq   %s, %s\t# spill",
 1388                      Matcher::regName[dst_first],
 1389                      Matcher::regName[src_first]);
 1390 #endif
 1391         }
 1392       }
 1393       return 0;
 1394     } else if (dst_first_rc == rc_float) {
 1395       assert(false, "Illegal spill");
 1396       return 0;
 1397     }
 1398   }
 1399 
 1400   assert(0," foo ");
 1401   Unimplemented();
 1402   return 0;
 1403 }
 1404 
 1405 #ifndef PRODUCT
 1406 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1407   implementation(nullptr, ra_, false, st);
 1408 }
 1409 #endif
 1410 
 1411 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1412   implementation(masm, ra_, false, nullptr);
 1413 }
 1414 
 1415 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1416   return MachNode::size(ra_);
 1417 }
 1418 
 1419 //=============================================================================
 1420 #ifndef PRODUCT
 1421 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1422 {
 1423   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1424   int reg = ra_->get_reg_first(this);
 1425   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1426             Matcher::regName[reg], offset);
 1427 }
 1428 #endif
 1429 
 1430 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1431 {
 1432   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1433   int reg = ra_->get_encode(this);
 1434 
 1435   __ lea(as_Register(reg), Address(rsp, offset));
 1436 }
 1437 
 1438 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1439 {
 1440   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1441   return (offset < 0x80) ? 5 : 8; // REX
 1442 }
 1443 
 1444 //=============================================================================
 1445 #ifndef PRODUCT
 1446 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1447 {
 1448   if (UseCompressedClassPointers) {
 1449     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1450     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1451   } else {
 1452     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1453     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1454   }
 1455   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1456 }
 1457 #endif
 1458 
 1459 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1460 {
 1461   __ ic_check(InteriorEntryAlignment);
 1462 }
 1463 
 1464 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1465 {
 1466   return MachNode::size(ra_); // too many variables; just compute it
 1467                               // the hard way
 1468 }
 1469 
 1470 
 1471 //=============================================================================
 1472 
 1473 bool Matcher::supports_vector_calling_convention(void) {
 1474   if (EnableVectorSupport && UseVectorStubs) {
 1475     return true;
 1476   }
 1477   return false;
 1478 }
 1479 
 1480 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1481   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1482   int lo = XMM0_num;
 1483   int hi = XMM0b_num;
 1484   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1485   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1486   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1487   return OptoRegPair(hi, lo);
 1488 }
 1489 
 1490 // Is this branch offset short enough that a short branch can be used?
 1491 //
 1492 // NOTE: If the platform does not provide any short branch variants, then
 1493 //       this method should return false for offset 0.
 1494 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1495   // The passed offset is relative to address of the branch.
 1496   // On 86 a branch displacement is calculated relative to address
 1497   // of a next instruction.
 1498   offset -= br_size;
 1499 
 1500   // the short version of jmpConUCF2 contains multiple branches,
 1501   // making the reach slightly less
 1502   if (rule == jmpConUCF2_rule)
 1503     return (-126 <= offset && offset <= 125);
 1504   return (-128 <= offset && offset <= 127);
 1505 }
 1506 
 1507 // Return whether or not this register is ever used as an argument.
 1508 // This function is used on startup to build the trampoline stubs in
 1509 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1510 // call in the trampoline, and arguments in those registers not be
 1511 // available to the callee.
 1512 bool Matcher::can_be_java_arg(int reg)
 1513 {
 1514   return
 1515     reg ==  RDI_num || reg == RDI_H_num ||
 1516     reg ==  RSI_num || reg == RSI_H_num ||
 1517     reg ==  RDX_num || reg == RDX_H_num ||
 1518     reg ==  RCX_num || reg == RCX_H_num ||
 1519     reg ==   R8_num || reg ==  R8_H_num ||
 1520     reg ==   R9_num || reg ==  R9_H_num ||
 1521     reg ==  R12_num || reg == R12_H_num ||
 1522     reg == XMM0_num || reg == XMM0b_num ||
 1523     reg == XMM1_num || reg == XMM1b_num ||
 1524     reg == XMM2_num || reg == XMM2b_num ||
 1525     reg == XMM3_num || reg == XMM3b_num ||
 1526     reg == XMM4_num || reg == XMM4b_num ||
 1527     reg == XMM5_num || reg == XMM5b_num ||
 1528     reg == XMM6_num || reg == XMM6b_num ||
 1529     reg == XMM7_num || reg == XMM7b_num;
 1530 }
 1531 
 1532 bool Matcher::is_spillable_arg(int reg)
 1533 {
 1534   return can_be_java_arg(reg);
 1535 }
 1536 
 1537 uint Matcher::int_pressure_limit()
 1538 {
 1539   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1540 }
 1541 
 1542 uint Matcher::float_pressure_limit()
 1543 {
 1544   // After experiment around with different values, the following default threshold
 1545   // works best for LCM's register pressure scheduling on x64.
 1546   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1547   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1548   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1549 }
 1550 
 1551 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1552   // In 64 bit mode a code which use multiply when
 1553   // devisor is constant is faster than hardware
 1554   // DIV instruction (it uses MulHiL).
 1555   return false;
 1556 }
 1557 
 1558 // Register for DIVI projection of divmodI
 1559 RegMask Matcher::divI_proj_mask() {
 1560   return INT_RAX_REG_mask();
 1561 }
 1562 
 1563 // Register for MODI projection of divmodI
 1564 RegMask Matcher::modI_proj_mask() {
 1565   return INT_RDX_REG_mask();
 1566 }
 1567 
 1568 // Register for DIVL projection of divmodL
 1569 RegMask Matcher::divL_proj_mask() {
 1570   return LONG_RAX_REG_mask();
 1571 }
 1572 
 1573 // Register for MODL projection of divmodL
 1574 RegMask Matcher::modL_proj_mask() {
 1575   return LONG_RDX_REG_mask();
 1576 }
 1577 
 1578 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1579 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1580     return NO_REG_mask();
 1581 }
 1582 
 1583 %}
 1584 
 1585 //----------ENCODING BLOCK-----------------------------------------------------
 1586 // This block specifies the encoding classes used by the compiler to
 1587 // output byte streams.  Encoding classes are parameterized macros
 1588 // used by Machine Instruction Nodes in order to generate the bit
 1589 // encoding of the instruction.  Operands specify their base encoding
 1590 // interface with the interface keyword.  There are currently
 1591 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1592 // COND_INTER.  REG_INTER causes an operand to generate a function
 1593 // which returns its register number when queried.  CONST_INTER causes
 1594 // an operand to generate a function which returns the value of the
 1595 // constant when queried.  MEMORY_INTER causes an operand to generate
 1596 // four functions which return the Base Register, the Index Register,
 1597 // the Scale Value, and the Offset Value of the operand when queried.
 1598 // COND_INTER causes an operand to generate six functions which return
 1599 // the encoding code (ie - encoding bits for the instruction)
 1600 // associated with each basic boolean condition for a conditional
 1601 // instruction.
 1602 //
 1603 // Instructions specify two basic values for encoding.  Again, a
 1604 // function is available to check if the constant displacement is an
 1605 // oop. They use the ins_encode keyword to specify their encoding
 1606 // classes (which must be a sequence of enc_class names, and their
 1607 // parameters, specified in the encoding block), and they use the
 1608 // opcode keyword to specify, in order, their primary, secondary, and
 1609 // tertiary opcode.  Only the opcode sections which a particular
 1610 // instruction needs for encoding need to be specified.
 1611 encode %{
 1612   enc_class cdql_enc(no_rax_rdx_RegI div)
 1613   %{
 1614     // Full implementation of Java idiv and irem; checks for
 1615     // special case as described in JVM spec., p.243 & p.271.
 1616     //
 1617     //         normal case                           special case
 1618     //
 1619     // input : rax: dividend                         min_int
 1620     //         reg: divisor                          -1
 1621     //
 1622     // output: rax: quotient  (= rax idiv reg)       min_int
 1623     //         rdx: remainder (= rax irem reg)       0
 1624     //
 1625     //  Code sequnce:
 1626     //
 1627     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1628     //    5:   75 07/08                jne    e <normal>
 1629     //    7:   33 d2                   xor    %edx,%edx
 1630     //  [div >= 8 -> offset + 1]
 1631     //  [REX_B]
 1632     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1633     //    c:   74 03/04                je     11 <done>
 1634     // 000000000000000e <normal>:
 1635     //    e:   99                      cltd
 1636     //  [div >= 8 -> offset + 1]
 1637     //  [REX_B]
 1638     //    f:   f7 f9                   idiv   $div
 1639     // 0000000000000011 <done>:
 1640     Label normal;
 1641     Label done;
 1642 
 1643     // cmp    $0x80000000,%eax
 1644     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1645 
 1646     // jne    e <normal>
 1647     __ jccb(Assembler::notEqual, normal);
 1648 
 1649     // xor    %edx,%edx
 1650     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1651 
 1652     // cmp    $0xffffffffffffffff,%ecx
 1653     __ cmpl($div$$Register, -1);
 1654 
 1655     // je     11 <done>
 1656     __ jccb(Assembler::equal, done);
 1657 
 1658     // <normal>
 1659     // cltd
 1660     __ bind(normal);
 1661     __ cdql();
 1662 
 1663     // idivl
 1664     // <done>
 1665     __ idivl($div$$Register);
 1666     __ bind(done);
 1667   %}
 1668 
 1669   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1670   %{
 1671     // Full implementation of Java ldiv and lrem; checks for
 1672     // special case as described in JVM spec., p.243 & p.271.
 1673     //
 1674     //         normal case                           special case
 1675     //
 1676     // input : rax: dividend                         min_long
 1677     //         reg: divisor                          -1
 1678     //
 1679     // output: rax: quotient  (= rax idiv reg)       min_long
 1680     //         rdx: remainder (= rax irem reg)       0
 1681     //
 1682     //  Code sequnce:
 1683     //
 1684     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1685     //    7:   00 00 80
 1686     //    a:   48 39 d0                cmp    %rdx,%rax
 1687     //    d:   75 08                   jne    17 <normal>
 1688     //    f:   33 d2                   xor    %edx,%edx
 1689     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1690     //   15:   74 05                   je     1c <done>
 1691     // 0000000000000017 <normal>:
 1692     //   17:   48 99                   cqto
 1693     //   19:   48 f7 f9                idiv   $div
 1694     // 000000000000001c <done>:
 1695     Label normal;
 1696     Label done;
 1697 
 1698     // mov    $0x8000000000000000,%rdx
 1699     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1700 
 1701     // cmp    %rdx,%rax
 1702     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1703 
 1704     // jne    17 <normal>
 1705     __ jccb(Assembler::notEqual, normal);
 1706 
 1707     // xor    %edx,%edx
 1708     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1709 
 1710     // cmp    $0xffffffffffffffff,$div
 1711     __ cmpq($div$$Register, -1);
 1712 
 1713     // je     1e <done>
 1714     __ jccb(Assembler::equal, done);
 1715 
 1716     // <normal>
 1717     // cqto
 1718     __ bind(normal);
 1719     __ cdqq();
 1720 
 1721     // idivq (note: must be emitted by the user of this rule)
 1722     // <done>
 1723     __ idivq($div$$Register);
 1724     __ bind(done);
 1725   %}
 1726 
 1727   enc_class enc_PartialSubtypeCheck()
 1728   %{
 1729     Register Rrdi = as_Register(RDI_enc); // result register
 1730     Register Rrax = as_Register(RAX_enc); // super class
 1731     Register Rrcx = as_Register(RCX_enc); // killed
 1732     Register Rrsi = as_Register(RSI_enc); // sub class
 1733     Label miss;
 1734     const bool set_cond_codes = true;
 1735 
 1736     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 1737                                      nullptr, &miss,
 1738                                      /*set_cond_codes:*/ true);
 1739     if ($primary) {
 1740       __ xorptr(Rrdi, Rrdi);
 1741     }
 1742     __ bind(miss);
 1743   %}
 1744 
 1745   enc_class clear_avx %{
 1746     debug_only(int off0 = __ offset());
 1747     if (generate_vzeroupper(Compile::current())) {
 1748       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1749       // Clear upper bits of YMM registers when current compiled code uses
 1750       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1751       __ vzeroupper();
 1752     }
 1753     debug_only(int off1 = __ offset());
 1754     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1755   %}
 1756 
 1757   enc_class Java_To_Runtime(method meth) %{
 1758     // No relocation needed
 1759     __ mov64(r10, (int64_t) $meth$$method);
 1760     __ call(r10);
 1761     __ post_call_nop();
 1762   %}
 1763 
 1764   enc_class Java_Static_Call(method meth)
 1765   %{
 1766     // JAVA STATIC CALL
 1767     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1768     // determine who we intended to call.
 1769     if (!_method) {
 1770       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1771     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1772       // The NOP here is purely to ensure that eliding a call to
 1773       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1774       __ addr_nop_5();
 1775       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1776     } else {
 1777       int method_index = resolved_method_index(masm);
 1778       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1779                                                   : static_call_Relocation::spec(method_index);
 1780       address mark = __ pc();
 1781       int call_offset = __ offset();
 1782       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1783       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1784         // Calls of the same statically bound method can share
 1785         // a stub to the interpreter.
 1786         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1787       } else {
 1788         // Emit stubs for static call.
 1789         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1790         __ clear_inst_mark();
 1791         if (stub == nullptr) {
 1792           ciEnv::current()->record_failure("CodeCache is full");
 1793           return;
 1794         }
 1795       }
 1796     }
 1797     __ post_call_nop();
 1798   %}
 1799 
 1800   enc_class Java_Dynamic_Call(method meth) %{
 1801     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1802     __ post_call_nop();
 1803   %}
 1804 
 1805 %}
 1806 
 1807 
 1808 
 1809 //----------FRAME--------------------------------------------------------------
 1810 // Definition of frame structure and management information.
 1811 //
 1812 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1813 //                             |   (to get allocators register number
 1814 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1815 //  r   CALLER     |        |
 1816 //  o     |        +--------+      pad to even-align allocators stack-slot
 1817 //  w     V        |  pad0  |        numbers; owned by CALLER
 1818 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1819 //  h     ^        |   in   |  5
 1820 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1821 //  |     |        |        |  3
 1822 //  |     |        +--------+
 1823 //  V     |        | old out|      Empty on Intel, window on Sparc
 1824 //        |    old |preserve|      Must be even aligned.
 1825 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1826 //        |        |   in   |  3   area for Intel ret address
 1827 //     Owned by    |preserve|      Empty on Sparc.
 1828 //       SELF      +--------+
 1829 //        |        |  pad2  |  2   pad to align old SP
 1830 //        |        +--------+  1
 1831 //        |        | locks  |  0
 1832 //        |        +--------+----> OptoReg::stack0(), even aligned
 1833 //        |        |  pad1  | 11   pad to align new SP
 1834 //        |        +--------+
 1835 //        |        |        | 10
 1836 //        |        | spills |  9   spills
 1837 //        V        |        |  8   (pad0 slot for callee)
 1838 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1839 //        ^        |  out   |  7
 1840 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1841 //     Owned by    +--------+
 1842 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1843 //        |    new |preserve|      Must be even-aligned.
 1844 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1845 //        |        |        |
 1846 //
 1847 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1848 //         known from SELF's arguments and the Java calling convention.
 1849 //         Region 6-7 is determined per call site.
 1850 // Note 2: If the calling convention leaves holes in the incoming argument
 1851 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1852 //         are owned by the CALLEE.  Holes should not be necessary in the
 1853 //         incoming area, as the Java calling convention is completely under
 1854 //         the control of the AD file.  Doubles can be sorted and packed to
 1855 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1856 //         varargs C calling conventions.
 1857 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1858 //         even aligned with pad0 as needed.
 1859 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1860 //         region 6-11 is even aligned; it may be padded out more so that
 1861 //         the region from SP to FP meets the minimum stack alignment.
 1862 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1863 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1864 //         SP meets the minimum alignment.
 1865 
 1866 frame
 1867 %{
 1868   // These three registers define part of the calling convention
 1869   // between compiled code and the interpreter.
 1870   inline_cache_reg(RAX);                // Inline Cache Register
 1871 
 1872   // Optional: name the operand used by cisc-spilling to access
 1873   // [stack_pointer + offset]
 1874   cisc_spilling_operand_name(indOffset32);
 1875 
 1876   // Number of stack slots consumed by locking an object
 1877   sync_stack_slots(2);
 1878 
 1879   // Compiled code's Frame Pointer
 1880   frame_pointer(RSP);
 1881 
 1882   // Interpreter stores its frame pointer in a register which is
 1883   // stored to the stack by I2CAdaptors.
 1884   // I2CAdaptors convert from interpreted java to compiled java.
 1885   interpreter_frame_pointer(RBP);
 1886 
 1887   // Stack alignment requirement
 1888   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1889 
 1890   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1891   // for calls to C.  Supports the var-args backing area for register parms.
 1892   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1893 
 1894   // The after-PROLOG location of the return address.  Location of
 1895   // return address specifies a type (REG or STACK) and a number
 1896   // representing the register number (i.e. - use a register name) or
 1897   // stack slot.
 1898   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1899   // Otherwise, it is above the locks and verification slot and alignment word
 1900   return_addr(STACK - 2 +
 1901               align_up((Compile::current()->in_preserve_stack_slots() +
 1902                         Compile::current()->fixed_slots()),
 1903                        stack_alignment_in_slots()));
 1904 
 1905   // Location of compiled Java return values.  Same as C for now.
 1906   return_value
 1907   %{
 1908     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 1909            "only return normal values");
 1910 
 1911     static const int lo[Op_RegL + 1] = {
 1912       0,
 1913       0,
 1914       RAX_num,  // Op_RegN
 1915       RAX_num,  // Op_RegI
 1916       RAX_num,  // Op_RegP
 1917       XMM0_num, // Op_RegF
 1918       XMM0_num, // Op_RegD
 1919       RAX_num   // Op_RegL
 1920     };
 1921     static const int hi[Op_RegL + 1] = {
 1922       0,
 1923       0,
 1924       OptoReg::Bad, // Op_RegN
 1925       OptoReg::Bad, // Op_RegI
 1926       RAX_H_num,    // Op_RegP
 1927       OptoReg::Bad, // Op_RegF
 1928       XMM0b_num,    // Op_RegD
 1929       RAX_H_num     // Op_RegL
 1930     };
 1931     // Excluded flags and vector registers.
 1932     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 1933     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 1934   %}
 1935 %}
 1936 
 1937 //----------ATTRIBUTES---------------------------------------------------------
 1938 //----------Operand Attributes-------------------------------------------------
 1939 op_attrib op_cost(0);        // Required cost attribute
 1940 
 1941 //----------Instruction Attributes---------------------------------------------
 1942 ins_attrib ins_cost(100);       // Required cost attribute
 1943 ins_attrib ins_size(8);         // Required size attribute (in bits)
 1944 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 1945                                 // a non-matching short branch variant
 1946                                 // of some long branch?
 1947 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 1948                                 // be a power of 2) specifies the
 1949                                 // alignment that some part of the
 1950                                 // instruction (not necessarily the
 1951                                 // start) requires.  If > 1, a
 1952                                 // compute_padding() function must be
 1953                                 // provided for the instruction
 1954 
 1955 //----------OPERANDS-----------------------------------------------------------
 1956 // Operand definitions must precede instruction definitions for correct parsing
 1957 // in the ADLC because operands constitute user defined types which are used in
 1958 // instruction definitions.
 1959 
 1960 //----------Simple Operands----------------------------------------------------
 1961 // Immediate Operands
 1962 // Integer Immediate
 1963 operand immI()
 1964 %{
 1965   match(ConI);
 1966 
 1967   op_cost(10);
 1968   format %{ %}
 1969   interface(CONST_INTER);
 1970 %}
 1971 
 1972 // Constant for test vs zero
 1973 operand immI_0()
 1974 %{
 1975   predicate(n->get_int() == 0);
 1976   match(ConI);
 1977 
 1978   op_cost(0);
 1979   format %{ %}
 1980   interface(CONST_INTER);
 1981 %}
 1982 
 1983 // Constant for increment
 1984 operand immI_1()
 1985 %{
 1986   predicate(n->get_int() == 1);
 1987   match(ConI);
 1988 
 1989   op_cost(0);
 1990   format %{ %}
 1991   interface(CONST_INTER);
 1992 %}
 1993 
 1994 // Constant for decrement
 1995 operand immI_M1()
 1996 %{
 1997   predicate(n->get_int() == -1);
 1998   match(ConI);
 1999 
 2000   op_cost(0);
 2001   format %{ %}
 2002   interface(CONST_INTER);
 2003 %}
 2004 
 2005 operand immI_2()
 2006 %{
 2007   predicate(n->get_int() == 2);
 2008   match(ConI);
 2009 
 2010   op_cost(0);
 2011   format %{ %}
 2012   interface(CONST_INTER);
 2013 %}
 2014 
 2015 operand immI_4()
 2016 %{
 2017   predicate(n->get_int() == 4);
 2018   match(ConI);
 2019 
 2020   op_cost(0);
 2021   format %{ %}
 2022   interface(CONST_INTER);
 2023 %}
 2024 
 2025 operand immI_8()
 2026 %{
 2027   predicate(n->get_int() == 8);
 2028   match(ConI);
 2029 
 2030   op_cost(0);
 2031   format %{ %}
 2032   interface(CONST_INTER);
 2033 %}
 2034 
 2035 // Valid scale values for addressing modes
 2036 operand immI2()
 2037 %{
 2038   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2039   match(ConI);
 2040 
 2041   format %{ %}
 2042   interface(CONST_INTER);
 2043 %}
 2044 
 2045 operand immU7()
 2046 %{
 2047   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2048   match(ConI);
 2049 
 2050   op_cost(5);
 2051   format %{ %}
 2052   interface(CONST_INTER);
 2053 %}
 2054 
 2055 operand immI8()
 2056 %{
 2057   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2058   match(ConI);
 2059 
 2060   op_cost(5);
 2061   format %{ %}
 2062   interface(CONST_INTER);
 2063 %}
 2064 
 2065 operand immU8()
 2066 %{
 2067   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2068   match(ConI);
 2069 
 2070   op_cost(5);
 2071   format %{ %}
 2072   interface(CONST_INTER);
 2073 %}
 2074 
 2075 operand immI16()
 2076 %{
 2077   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2078   match(ConI);
 2079 
 2080   op_cost(10);
 2081   format %{ %}
 2082   interface(CONST_INTER);
 2083 %}
 2084 
 2085 // Int Immediate non-negative
 2086 operand immU31()
 2087 %{
 2088   predicate(n->get_int() >= 0);
 2089   match(ConI);
 2090 
 2091   op_cost(0);
 2092   format %{ %}
 2093   interface(CONST_INTER);
 2094 %}
 2095 
 2096 // Pointer Immediate
 2097 operand immP()
 2098 %{
 2099   match(ConP);
 2100 
 2101   op_cost(10);
 2102   format %{ %}
 2103   interface(CONST_INTER);
 2104 %}
 2105 
 2106 // Null Pointer Immediate
 2107 operand immP0()
 2108 %{
 2109   predicate(n->get_ptr() == 0);
 2110   match(ConP);
 2111 
 2112   op_cost(5);
 2113   format %{ %}
 2114   interface(CONST_INTER);
 2115 %}
 2116 
 2117 // Pointer Immediate
 2118 operand immN() %{
 2119   match(ConN);
 2120 
 2121   op_cost(10);
 2122   format %{ %}
 2123   interface(CONST_INTER);
 2124 %}
 2125 
 2126 operand immNKlass() %{
 2127   match(ConNKlass);
 2128 
 2129   op_cost(10);
 2130   format %{ %}
 2131   interface(CONST_INTER);
 2132 %}
 2133 
 2134 // Null Pointer Immediate
 2135 operand immN0() %{
 2136   predicate(n->get_narrowcon() == 0);
 2137   match(ConN);
 2138 
 2139   op_cost(5);
 2140   format %{ %}
 2141   interface(CONST_INTER);
 2142 %}
 2143 
 2144 operand immP31()
 2145 %{
 2146   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2147             && (n->get_ptr() >> 31) == 0);
 2148   match(ConP);
 2149 
 2150   op_cost(5);
 2151   format %{ %}
 2152   interface(CONST_INTER);
 2153 %}
 2154 
 2155 
 2156 // Long Immediate
 2157 operand immL()
 2158 %{
 2159   match(ConL);
 2160 
 2161   op_cost(20);
 2162   format %{ %}
 2163   interface(CONST_INTER);
 2164 %}
 2165 
 2166 // Long Immediate 8-bit
 2167 operand immL8()
 2168 %{
 2169   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2170   match(ConL);
 2171 
 2172   op_cost(5);
 2173   format %{ %}
 2174   interface(CONST_INTER);
 2175 %}
 2176 
 2177 // Long Immediate 32-bit unsigned
 2178 operand immUL32()
 2179 %{
 2180   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2181   match(ConL);
 2182 
 2183   op_cost(10);
 2184   format %{ %}
 2185   interface(CONST_INTER);
 2186 %}
 2187 
 2188 // Long Immediate 32-bit signed
 2189 operand immL32()
 2190 %{
 2191   predicate(n->get_long() == (int) (n->get_long()));
 2192   match(ConL);
 2193 
 2194   op_cost(15);
 2195   format %{ %}
 2196   interface(CONST_INTER);
 2197 %}
 2198 
 2199 operand immL_Pow2()
 2200 %{
 2201   predicate(is_power_of_2((julong)n->get_long()));
 2202   match(ConL);
 2203 
 2204   op_cost(15);
 2205   format %{ %}
 2206   interface(CONST_INTER);
 2207 %}
 2208 
 2209 operand immL_NotPow2()
 2210 %{
 2211   predicate(is_power_of_2((julong)~n->get_long()));
 2212   match(ConL);
 2213 
 2214   op_cost(15);
 2215   format %{ %}
 2216   interface(CONST_INTER);
 2217 %}
 2218 
 2219 // Long Immediate zero
 2220 operand immL0()
 2221 %{
 2222   predicate(n->get_long() == 0L);
 2223   match(ConL);
 2224 
 2225   op_cost(10);
 2226   format %{ %}
 2227   interface(CONST_INTER);
 2228 %}
 2229 
 2230 // Constant for increment
 2231 operand immL1()
 2232 %{
 2233   predicate(n->get_long() == 1);
 2234   match(ConL);
 2235 
 2236   format %{ %}
 2237   interface(CONST_INTER);
 2238 %}
 2239 
 2240 // Constant for decrement
 2241 operand immL_M1()
 2242 %{
 2243   predicate(n->get_long() == -1);
 2244   match(ConL);
 2245 
 2246   format %{ %}
 2247   interface(CONST_INTER);
 2248 %}
 2249 
 2250 // Long Immediate: low 32-bit mask
 2251 operand immL_32bits()
 2252 %{
 2253   predicate(n->get_long() == 0xFFFFFFFFL);
 2254   match(ConL);
 2255   op_cost(20);
 2256 
 2257   format %{ %}
 2258   interface(CONST_INTER);
 2259 %}
 2260 
 2261 // Int Immediate: 2^n-1, positive
 2262 operand immI_Pow2M1()
 2263 %{
 2264   predicate((n->get_int() > 0)
 2265             && is_power_of_2((juint)n->get_int() + 1));
 2266   match(ConI);
 2267 
 2268   op_cost(20);
 2269   format %{ %}
 2270   interface(CONST_INTER);
 2271 %}
 2272 
 2273 // Float Immediate zero
 2274 operand immF0()
 2275 %{
 2276   predicate(jint_cast(n->getf()) == 0);
 2277   match(ConF);
 2278 
 2279   op_cost(5);
 2280   format %{ %}
 2281   interface(CONST_INTER);
 2282 %}
 2283 
 2284 // Float Immediate
 2285 operand immF()
 2286 %{
 2287   match(ConF);
 2288 
 2289   op_cost(15);
 2290   format %{ %}
 2291   interface(CONST_INTER);
 2292 %}
 2293 
 2294 // Double Immediate zero
 2295 operand immD0()
 2296 %{
 2297   predicate(jlong_cast(n->getd()) == 0);
 2298   match(ConD);
 2299 
 2300   op_cost(5);
 2301   format %{ %}
 2302   interface(CONST_INTER);
 2303 %}
 2304 
 2305 // Double Immediate
 2306 operand immD()
 2307 %{
 2308   match(ConD);
 2309 
 2310   op_cost(15);
 2311   format %{ %}
 2312   interface(CONST_INTER);
 2313 %}
 2314 
 2315 // Immediates for special shifts (sign extend)
 2316 
 2317 // Constants for increment
 2318 operand immI_16()
 2319 %{
 2320   predicate(n->get_int() == 16);
 2321   match(ConI);
 2322 
 2323   format %{ %}
 2324   interface(CONST_INTER);
 2325 %}
 2326 
 2327 operand immI_24()
 2328 %{
 2329   predicate(n->get_int() == 24);
 2330   match(ConI);
 2331 
 2332   format %{ %}
 2333   interface(CONST_INTER);
 2334 %}
 2335 
 2336 // Constant for byte-wide masking
 2337 operand immI_255()
 2338 %{
 2339   predicate(n->get_int() == 255);
 2340   match(ConI);
 2341 
 2342   format %{ %}
 2343   interface(CONST_INTER);
 2344 %}
 2345 
 2346 // Constant for short-wide masking
 2347 operand immI_65535()
 2348 %{
 2349   predicate(n->get_int() == 65535);
 2350   match(ConI);
 2351 
 2352   format %{ %}
 2353   interface(CONST_INTER);
 2354 %}
 2355 
 2356 // Constant for byte-wide masking
 2357 operand immL_255()
 2358 %{
 2359   predicate(n->get_long() == 255);
 2360   match(ConL);
 2361 
 2362   format %{ %}
 2363   interface(CONST_INTER);
 2364 %}
 2365 
 2366 // Constant for short-wide masking
 2367 operand immL_65535()
 2368 %{
 2369   predicate(n->get_long() == 65535);
 2370   match(ConL);
 2371 
 2372   format %{ %}
 2373   interface(CONST_INTER);
 2374 %}
 2375 
 2376 operand kReg()
 2377 %{
 2378   constraint(ALLOC_IN_RC(vectmask_reg));
 2379   match(RegVectMask);
 2380   format %{%}
 2381   interface(REG_INTER);
 2382 %}
 2383 
 2384 // Register Operands
 2385 // Integer Register
 2386 operand rRegI()
 2387 %{
 2388   constraint(ALLOC_IN_RC(int_reg));
 2389   match(RegI);
 2390 
 2391   match(rax_RegI);
 2392   match(rbx_RegI);
 2393   match(rcx_RegI);
 2394   match(rdx_RegI);
 2395   match(rdi_RegI);
 2396 
 2397   format %{ %}
 2398   interface(REG_INTER);
 2399 %}
 2400 
 2401 // Special Registers
 2402 operand rax_RegI()
 2403 %{
 2404   constraint(ALLOC_IN_RC(int_rax_reg));
 2405   match(RegI);
 2406   match(rRegI);
 2407 
 2408   format %{ "RAX" %}
 2409   interface(REG_INTER);
 2410 %}
 2411 
 2412 // Special Registers
 2413 operand rbx_RegI()
 2414 %{
 2415   constraint(ALLOC_IN_RC(int_rbx_reg));
 2416   match(RegI);
 2417   match(rRegI);
 2418 
 2419   format %{ "RBX" %}
 2420   interface(REG_INTER);
 2421 %}
 2422 
 2423 operand rcx_RegI()
 2424 %{
 2425   constraint(ALLOC_IN_RC(int_rcx_reg));
 2426   match(RegI);
 2427   match(rRegI);
 2428 
 2429   format %{ "RCX" %}
 2430   interface(REG_INTER);
 2431 %}
 2432 
 2433 operand rdx_RegI()
 2434 %{
 2435   constraint(ALLOC_IN_RC(int_rdx_reg));
 2436   match(RegI);
 2437   match(rRegI);
 2438 
 2439   format %{ "RDX" %}
 2440   interface(REG_INTER);
 2441 %}
 2442 
 2443 operand rdi_RegI()
 2444 %{
 2445   constraint(ALLOC_IN_RC(int_rdi_reg));
 2446   match(RegI);
 2447   match(rRegI);
 2448 
 2449   format %{ "RDI" %}
 2450   interface(REG_INTER);
 2451 %}
 2452 
 2453 operand no_rax_rdx_RegI()
 2454 %{
 2455   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2456   match(RegI);
 2457   match(rbx_RegI);
 2458   match(rcx_RegI);
 2459   match(rdi_RegI);
 2460 
 2461   format %{ %}
 2462   interface(REG_INTER);
 2463 %}
 2464 
 2465 operand no_rbp_r13_RegI()
 2466 %{
 2467   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2468   match(RegI);
 2469   match(rRegI);
 2470   match(rax_RegI);
 2471   match(rbx_RegI);
 2472   match(rcx_RegI);
 2473   match(rdx_RegI);
 2474   match(rdi_RegI);
 2475 
 2476   format %{ %}
 2477   interface(REG_INTER);
 2478 %}
 2479 
 2480 // Pointer Register
 2481 operand any_RegP()
 2482 %{
 2483   constraint(ALLOC_IN_RC(any_reg));
 2484   match(RegP);
 2485   match(rax_RegP);
 2486   match(rbx_RegP);
 2487   match(rdi_RegP);
 2488   match(rsi_RegP);
 2489   match(rbp_RegP);
 2490   match(r15_RegP);
 2491   match(rRegP);
 2492 
 2493   format %{ %}
 2494   interface(REG_INTER);
 2495 %}
 2496 
 2497 operand rRegP()
 2498 %{
 2499   constraint(ALLOC_IN_RC(ptr_reg));
 2500   match(RegP);
 2501   match(rax_RegP);
 2502   match(rbx_RegP);
 2503   match(rdi_RegP);
 2504   match(rsi_RegP);
 2505   match(rbp_RegP);  // See Q&A below about
 2506   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2507 
 2508   format %{ %}
 2509   interface(REG_INTER);
 2510 %}
 2511 
 2512 operand rRegN() %{
 2513   constraint(ALLOC_IN_RC(int_reg));
 2514   match(RegN);
 2515 
 2516   format %{ %}
 2517   interface(REG_INTER);
 2518 %}
 2519 
 2520 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2521 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2522 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2523 // The output of an instruction is controlled by the allocator, which respects
 2524 // register class masks, not match rules.  Unless an instruction mentions
 2525 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2526 // by the allocator as an input.
 2527 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2528 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2529 // result, RBP is not included in the output of the instruction either.
 2530 
 2531 // This operand is not allowed to use RBP even if
 2532 // RBP is not used to hold the frame pointer.
 2533 operand no_rbp_RegP()
 2534 %{
 2535   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2536   match(RegP);
 2537   match(rbx_RegP);
 2538   match(rsi_RegP);
 2539   match(rdi_RegP);
 2540 
 2541   format %{ %}
 2542   interface(REG_INTER);
 2543 %}
 2544 
 2545 // Special Registers
 2546 // Return a pointer value
 2547 operand rax_RegP()
 2548 %{
 2549   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2550   match(RegP);
 2551   match(rRegP);
 2552 
 2553   format %{ %}
 2554   interface(REG_INTER);
 2555 %}
 2556 
 2557 // Special Registers
 2558 // Return a compressed pointer value
 2559 operand rax_RegN()
 2560 %{
 2561   constraint(ALLOC_IN_RC(int_rax_reg));
 2562   match(RegN);
 2563   match(rRegN);
 2564 
 2565   format %{ %}
 2566   interface(REG_INTER);
 2567 %}
 2568 
 2569 // Used in AtomicAdd
 2570 operand rbx_RegP()
 2571 %{
 2572   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2573   match(RegP);
 2574   match(rRegP);
 2575 
 2576   format %{ %}
 2577   interface(REG_INTER);
 2578 %}
 2579 
 2580 operand rsi_RegP()
 2581 %{
 2582   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2583   match(RegP);
 2584   match(rRegP);
 2585 
 2586   format %{ %}
 2587   interface(REG_INTER);
 2588 %}
 2589 
 2590 operand rbp_RegP()
 2591 %{
 2592   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2593   match(RegP);
 2594   match(rRegP);
 2595 
 2596   format %{ %}
 2597   interface(REG_INTER);
 2598 %}
 2599 
 2600 // Used in rep stosq
 2601 operand rdi_RegP()
 2602 %{
 2603   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2604   match(RegP);
 2605   match(rRegP);
 2606 
 2607   format %{ %}
 2608   interface(REG_INTER);
 2609 %}
 2610 
 2611 operand r15_RegP()
 2612 %{
 2613   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2614   match(RegP);
 2615   match(rRegP);
 2616 
 2617   format %{ %}
 2618   interface(REG_INTER);
 2619 %}
 2620 
 2621 operand rRegL()
 2622 %{
 2623   constraint(ALLOC_IN_RC(long_reg));
 2624   match(RegL);
 2625   match(rax_RegL);
 2626   match(rdx_RegL);
 2627 
 2628   format %{ %}
 2629   interface(REG_INTER);
 2630 %}
 2631 
 2632 // Special Registers
 2633 operand no_rax_rdx_RegL()
 2634 %{
 2635   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2636   match(RegL);
 2637   match(rRegL);
 2638 
 2639   format %{ %}
 2640   interface(REG_INTER);
 2641 %}
 2642 
 2643 operand rax_RegL()
 2644 %{
 2645   constraint(ALLOC_IN_RC(long_rax_reg));
 2646   match(RegL);
 2647   match(rRegL);
 2648 
 2649   format %{ "RAX" %}
 2650   interface(REG_INTER);
 2651 %}
 2652 
 2653 operand rcx_RegL()
 2654 %{
 2655   constraint(ALLOC_IN_RC(long_rcx_reg));
 2656   match(RegL);
 2657   match(rRegL);
 2658 
 2659   format %{ %}
 2660   interface(REG_INTER);
 2661 %}
 2662 
 2663 operand rdx_RegL()
 2664 %{
 2665   constraint(ALLOC_IN_RC(long_rdx_reg));
 2666   match(RegL);
 2667   match(rRegL);
 2668 
 2669   format %{ %}
 2670   interface(REG_INTER);
 2671 %}
 2672 
 2673 operand r11_RegL()
 2674 %{
 2675   constraint(ALLOC_IN_RC(long_r11_reg));
 2676   match(RegL);
 2677   match(rRegL);
 2678 
 2679   format %{ %}
 2680   interface(REG_INTER);
 2681 %}
 2682 
 2683 operand no_rbp_r13_RegL()
 2684 %{
 2685   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2686   match(RegL);
 2687   match(rRegL);
 2688   match(rax_RegL);
 2689   match(rcx_RegL);
 2690   match(rdx_RegL);
 2691 
 2692   format %{ %}
 2693   interface(REG_INTER);
 2694 %}
 2695 
 2696 // Flags register, used as output of compare instructions
 2697 operand rFlagsReg()
 2698 %{
 2699   constraint(ALLOC_IN_RC(int_flags));
 2700   match(RegFlags);
 2701 
 2702   format %{ "RFLAGS" %}
 2703   interface(REG_INTER);
 2704 %}
 2705 
 2706 // Flags register, used as output of FLOATING POINT compare instructions
 2707 operand rFlagsRegU()
 2708 %{
 2709   constraint(ALLOC_IN_RC(int_flags));
 2710   match(RegFlags);
 2711 
 2712   format %{ "RFLAGS_U" %}
 2713   interface(REG_INTER);
 2714 %}
 2715 
 2716 operand rFlagsRegUCF() %{
 2717   constraint(ALLOC_IN_RC(int_flags));
 2718   match(RegFlags);
 2719   predicate(false);
 2720 
 2721   format %{ "RFLAGS_U_CF" %}
 2722   interface(REG_INTER);
 2723 %}
 2724 
 2725 // Float register operands
 2726 operand regF() %{
 2727    constraint(ALLOC_IN_RC(float_reg));
 2728    match(RegF);
 2729 
 2730    format %{ %}
 2731    interface(REG_INTER);
 2732 %}
 2733 
 2734 // Float register operands
 2735 operand legRegF() %{
 2736    constraint(ALLOC_IN_RC(float_reg_legacy));
 2737    match(RegF);
 2738 
 2739    format %{ %}
 2740    interface(REG_INTER);
 2741 %}
 2742 
 2743 // Float register operands
 2744 operand vlRegF() %{
 2745    constraint(ALLOC_IN_RC(float_reg_vl));
 2746    match(RegF);
 2747 
 2748    format %{ %}
 2749    interface(REG_INTER);
 2750 %}
 2751 
 2752 // Double register operands
 2753 operand regD() %{
 2754    constraint(ALLOC_IN_RC(double_reg));
 2755    match(RegD);
 2756 
 2757    format %{ %}
 2758    interface(REG_INTER);
 2759 %}
 2760 
 2761 // Double register operands
 2762 operand legRegD() %{
 2763    constraint(ALLOC_IN_RC(double_reg_legacy));
 2764    match(RegD);
 2765 
 2766    format %{ %}
 2767    interface(REG_INTER);
 2768 %}
 2769 
 2770 // Double register operands
 2771 operand vlRegD() %{
 2772    constraint(ALLOC_IN_RC(double_reg_vl));
 2773    match(RegD);
 2774 
 2775    format %{ %}
 2776    interface(REG_INTER);
 2777 %}
 2778 
 2779 //----------Memory Operands----------------------------------------------------
 2780 // Direct Memory Operand
 2781 // operand direct(immP addr)
 2782 // %{
 2783 //   match(addr);
 2784 
 2785 //   format %{ "[$addr]" %}
 2786 //   interface(MEMORY_INTER) %{
 2787 //     base(0xFFFFFFFF);
 2788 //     index(0x4);
 2789 //     scale(0x0);
 2790 //     disp($addr);
 2791 //   %}
 2792 // %}
 2793 
 2794 // Indirect Memory Operand
 2795 operand indirect(any_RegP reg)
 2796 %{
 2797   constraint(ALLOC_IN_RC(ptr_reg));
 2798   match(reg);
 2799 
 2800   format %{ "[$reg]" %}
 2801   interface(MEMORY_INTER) %{
 2802     base($reg);
 2803     index(0x4);
 2804     scale(0x0);
 2805     disp(0x0);
 2806   %}
 2807 %}
 2808 
 2809 // Indirect Memory Plus Short Offset Operand
 2810 operand indOffset8(any_RegP reg, immL8 off)
 2811 %{
 2812   constraint(ALLOC_IN_RC(ptr_reg));
 2813   match(AddP reg off);
 2814 
 2815   format %{ "[$reg + $off (8-bit)]" %}
 2816   interface(MEMORY_INTER) %{
 2817     base($reg);
 2818     index(0x4);
 2819     scale(0x0);
 2820     disp($off);
 2821   %}
 2822 %}
 2823 
 2824 // Indirect Memory Plus Long Offset Operand
 2825 operand indOffset32(any_RegP reg, immL32 off)
 2826 %{
 2827   constraint(ALLOC_IN_RC(ptr_reg));
 2828   match(AddP reg off);
 2829 
 2830   format %{ "[$reg + $off (32-bit)]" %}
 2831   interface(MEMORY_INTER) %{
 2832     base($reg);
 2833     index(0x4);
 2834     scale(0x0);
 2835     disp($off);
 2836   %}
 2837 %}
 2838 
 2839 // Indirect Memory Plus Index Register Plus Offset Operand
 2840 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2841 %{
 2842   constraint(ALLOC_IN_RC(ptr_reg));
 2843   match(AddP (AddP reg lreg) off);
 2844 
 2845   op_cost(10);
 2846   format %{"[$reg + $off + $lreg]" %}
 2847   interface(MEMORY_INTER) %{
 2848     base($reg);
 2849     index($lreg);
 2850     scale(0x0);
 2851     disp($off);
 2852   %}
 2853 %}
 2854 
 2855 // Indirect Memory Plus Index Register Plus Offset Operand
 2856 operand indIndex(any_RegP reg, rRegL lreg)
 2857 %{
 2858   constraint(ALLOC_IN_RC(ptr_reg));
 2859   match(AddP reg lreg);
 2860 
 2861   op_cost(10);
 2862   format %{"[$reg + $lreg]" %}
 2863   interface(MEMORY_INTER) %{
 2864     base($reg);
 2865     index($lreg);
 2866     scale(0x0);
 2867     disp(0x0);
 2868   %}
 2869 %}
 2870 
 2871 // Indirect Memory Times Scale Plus Index Register
 2872 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2873 %{
 2874   constraint(ALLOC_IN_RC(ptr_reg));
 2875   match(AddP reg (LShiftL lreg scale));
 2876 
 2877   op_cost(10);
 2878   format %{"[$reg + $lreg << $scale]" %}
 2879   interface(MEMORY_INTER) %{
 2880     base($reg);
 2881     index($lreg);
 2882     scale($scale);
 2883     disp(0x0);
 2884   %}
 2885 %}
 2886 
 2887 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 2888 %{
 2889   constraint(ALLOC_IN_RC(ptr_reg));
 2890   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 2891   match(AddP reg (LShiftL (ConvI2L idx) scale));
 2892 
 2893   op_cost(10);
 2894   format %{"[$reg + pos $idx << $scale]" %}
 2895   interface(MEMORY_INTER) %{
 2896     base($reg);
 2897     index($idx);
 2898     scale($scale);
 2899     disp(0x0);
 2900   %}
 2901 %}
 2902 
 2903 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 2904 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 2905 %{
 2906   constraint(ALLOC_IN_RC(ptr_reg));
 2907   match(AddP (AddP reg (LShiftL lreg scale)) off);
 2908 
 2909   op_cost(10);
 2910   format %{"[$reg + $off + $lreg << $scale]" %}
 2911   interface(MEMORY_INTER) %{
 2912     base($reg);
 2913     index($lreg);
 2914     scale($scale);
 2915     disp($off);
 2916   %}
 2917 %}
 2918 
 2919 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 2920 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 2921 %{
 2922   constraint(ALLOC_IN_RC(ptr_reg));
 2923   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 2924   match(AddP (AddP reg (ConvI2L idx)) off);
 2925 
 2926   op_cost(10);
 2927   format %{"[$reg + $off + $idx]" %}
 2928   interface(MEMORY_INTER) %{
 2929     base($reg);
 2930     index($idx);
 2931     scale(0x0);
 2932     disp($off);
 2933   %}
 2934 %}
 2935 
 2936 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 2937 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 2938 %{
 2939   constraint(ALLOC_IN_RC(ptr_reg));
 2940   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 2941   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 2942 
 2943   op_cost(10);
 2944   format %{"[$reg + $off + $idx << $scale]" %}
 2945   interface(MEMORY_INTER) %{
 2946     base($reg);
 2947     index($idx);
 2948     scale($scale);
 2949     disp($off);
 2950   %}
 2951 %}
 2952 
 2953 // Indirect Narrow Oop Plus Offset Operand
 2954 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 2955 // we can't free r12 even with CompressedOops::base() == nullptr.
 2956 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 2957   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 2958   constraint(ALLOC_IN_RC(ptr_reg));
 2959   match(AddP (DecodeN reg) off);
 2960 
 2961   op_cost(10);
 2962   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 2963   interface(MEMORY_INTER) %{
 2964     base(0xc); // R12
 2965     index($reg);
 2966     scale(0x3);
 2967     disp($off);
 2968   %}
 2969 %}
 2970 
 2971 // Indirect Memory Operand
 2972 operand indirectNarrow(rRegN reg)
 2973 %{
 2974   predicate(CompressedOops::shift() == 0);
 2975   constraint(ALLOC_IN_RC(ptr_reg));
 2976   match(DecodeN reg);
 2977 
 2978   format %{ "[$reg]" %}
 2979   interface(MEMORY_INTER) %{
 2980     base($reg);
 2981     index(0x4);
 2982     scale(0x0);
 2983     disp(0x0);
 2984   %}
 2985 %}
 2986 
 2987 // Indirect Memory Plus Short Offset Operand
 2988 operand indOffset8Narrow(rRegN reg, immL8 off)
 2989 %{
 2990   predicate(CompressedOops::shift() == 0);
 2991   constraint(ALLOC_IN_RC(ptr_reg));
 2992   match(AddP (DecodeN reg) off);
 2993 
 2994   format %{ "[$reg + $off (8-bit)]" %}
 2995   interface(MEMORY_INTER) %{
 2996     base($reg);
 2997     index(0x4);
 2998     scale(0x0);
 2999     disp($off);
 3000   %}
 3001 %}
 3002 
 3003 // Indirect Memory Plus Long Offset Operand
 3004 operand indOffset32Narrow(rRegN reg, immL32 off)
 3005 %{
 3006   predicate(CompressedOops::shift() == 0);
 3007   constraint(ALLOC_IN_RC(ptr_reg));
 3008   match(AddP (DecodeN reg) off);
 3009 
 3010   format %{ "[$reg + $off (32-bit)]" %}
 3011   interface(MEMORY_INTER) %{
 3012     base($reg);
 3013     index(0x4);
 3014     scale(0x0);
 3015     disp($off);
 3016   %}
 3017 %}
 3018 
 3019 // Indirect Memory Plus Index Register Plus Offset Operand
 3020 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3021 %{
 3022   predicate(CompressedOops::shift() == 0);
 3023   constraint(ALLOC_IN_RC(ptr_reg));
 3024   match(AddP (AddP (DecodeN reg) lreg) off);
 3025 
 3026   op_cost(10);
 3027   format %{"[$reg + $off + $lreg]" %}
 3028   interface(MEMORY_INTER) %{
 3029     base($reg);
 3030     index($lreg);
 3031     scale(0x0);
 3032     disp($off);
 3033   %}
 3034 %}
 3035 
 3036 // Indirect Memory Plus Index Register Plus Offset Operand
 3037 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3038 %{
 3039   predicate(CompressedOops::shift() == 0);
 3040   constraint(ALLOC_IN_RC(ptr_reg));
 3041   match(AddP (DecodeN reg) lreg);
 3042 
 3043   op_cost(10);
 3044   format %{"[$reg + $lreg]" %}
 3045   interface(MEMORY_INTER) %{
 3046     base($reg);
 3047     index($lreg);
 3048     scale(0x0);
 3049     disp(0x0);
 3050   %}
 3051 %}
 3052 
 3053 // Indirect Memory Times Scale Plus Index Register
 3054 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3055 %{
 3056   predicate(CompressedOops::shift() == 0);
 3057   constraint(ALLOC_IN_RC(ptr_reg));
 3058   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3059 
 3060   op_cost(10);
 3061   format %{"[$reg + $lreg << $scale]" %}
 3062   interface(MEMORY_INTER) %{
 3063     base($reg);
 3064     index($lreg);
 3065     scale($scale);
 3066     disp(0x0);
 3067   %}
 3068 %}
 3069 
 3070 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3071 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3072 %{
 3073   predicate(CompressedOops::shift() == 0);
 3074   constraint(ALLOC_IN_RC(ptr_reg));
 3075   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3076 
 3077   op_cost(10);
 3078   format %{"[$reg + $off + $lreg << $scale]" %}
 3079   interface(MEMORY_INTER) %{
 3080     base($reg);
 3081     index($lreg);
 3082     scale($scale);
 3083     disp($off);
 3084   %}
 3085 %}
 3086 
 3087 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3088 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3089 %{
 3090   constraint(ALLOC_IN_RC(ptr_reg));
 3091   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3092   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3093 
 3094   op_cost(10);
 3095   format %{"[$reg + $off + $idx]" %}
 3096   interface(MEMORY_INTER) %{
 3097     base($reg);
 3098     index($idx);
 3099     scale(0x0);
 3100     disp($off);
 3101   %}
 3102 %}
 3103 
 3104 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3105 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3106 %{
 3107   constraint(ALLOC_IN_RC(ptr_reg));
 3108   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3109   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3110 
 3111   op_cost(10);
 3112   format %{"[$reg + $off + $idx << $scale]" %}
 3113   interface(MEMORY_INTER) %{
 3114     base($reg);
 3115     index($idx);
 3116     scale($scale);
 3117     disp($off);
 3118   %}
 3119 %}
 3120 
 3121 //----------Special Memory Operands--------------------------------------------
 3122 // Stack Slot Operand - This operand is used for loading and storing temporary
 3123 //                      values on the stack where a match requires a value to
 3124 //                      flow through memory.
 3125 operand stackSlotP(sRegP reg)
 3126 %{
 3127   constraint(ALLOC_IN_RC(stack_slots));
 3128   // No match rule because this operand is only generated in matching
 3129 
 3130   format %{ "[$reg]" %}
 3131   interface(MEMORY_INTER) %{
 3132     base(0x4);   // RSP
 3133     index(0x4);  // No Index
 3134     scale(0x0);  // No Scale
 3135     disp($reg);  // Stack Offset
 3136   %}
 3137 %}
 3138 
 3139 operand stackSlotI(sRegI reg)
 3140 %{
 3141   constraint(ALLOC_IN_RC(stack_slots));
 3142   // No match rule because this operand is only generated in matching
 3143 
 3144   format %{ "[$reg]" %}
 3145   interface(MEMORY_INTER) %{
 3146     base(0x4);   // RSP
 3147     index(0x4);  // No Index
 3148     scale(0x0);  // No Scale
 3149     disp($reg);  // Stack Offset
 3150   %}
 3151 %}
 3152 
 3153 operand stackSlotF(sRegF reg)
 3154 %{
 3155   constraint(ALLOC_IN_RC(stack_slots));
 3156   // No match rule because this operand is only generated in matching
 3157 
 3158   format %{ "[$reg]" %}
 3159   interface(MEMORY_INTER) %{
 3160     base(0x4);   // RSP
 3161     index(0x4);  // No Index
 3162     scale(0x0);  // No Scale
 3163     disp($reg);  // Stack Offset
 3164   %}
 3165 %}
 3166 
 3167 operand stackSlotD(sRegD reg)
 3168 %{
 3169   constraint(ALLOC_IN_RC(stack_slots));
 3170   // No match rule because this operand is only generated in matching
 3171 
 3172   format %{ "[$reg]" %}
 3173   interface(MEMORY_INTER) %{
 3174     base(0x4);   // RSP
 3175     index(0x4);  // No Index
 3176     scale(0x0);  // No Scale
 3177     disp($reg);  // Stack Offset
 3178   %}
 3179 %}
 3180 operand stackSlotL(sRegL reg)
 3181 %{
 3182   constraint(ALLOC_IN_RC(stack_slots));
 3183   // No match rule because this operand is only generated in matching
 3184 
 3185   format %{ "[$reg]" %}
 3186   interface(MEMORY_INTER) %{
 3187     base(0x4);   // RSP
 3188     index(0x4);  // No Index
 3189     scale(0x0);  // No Scale
 3190     disp($reg);  // Stack Offset
 3191   %}
 3192 %}
 3193 
 3194 //----------Conditional Branch Operands----------------------------------------
 3195 // Comparison Op  - This is the operation of the comparison, and is limited to
 3196 //                  the following set of codes:
 3197 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3198 //
 3199 // Other attributes of the comparison, such as unsignedness, are specified
 3200 // by the comparison instruction that sets a condition code flags register.
 3201 // That result is represented by a flags operand whose subtype is appropriate
 3202 // to the unsignedness (etc.) of the comparison.
 3203 //
 3204 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3205 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3206 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3207 
 3208 // Comparison Code
 3209 operand cmpOp()
 3210 %{
 3211   match(Bool);
 3212 
 3213   format %{ "" %}
 3214   interface(COND_INTER) %{
 3215     equal(0x4, "e");
 3216     not_equal(0x5, "ne");
 3217     less(0xC, "l");
 3218     greater_equal(0xD, "ge");
 3219     less_equal(0xE, "le");
 3220     greater(0xF, "g");
 3221     overflow(0x0, "o");
 3222     no_overflow(0x1, "no");
 3223   %}
 3224 %}
 3225 
 3226 // Comparison Code, unsigned compare.  Used by FP also, with
 3227 // C2 (unordered) turned into GT or LT already.  The other bits
 3228 // C0 and C3 are turned into Carry & Zero flags.
 3229 operand cmpOpU()
 3230 %{
 3231   match(Bool);
 3232 
 3233   format %{ "" %}
 3234   interface(COND_INTER) %{
 3235     equal(0x4, "e");
 3236     not_equal(0x5, "ne");
 3237     less(0x2, "b");
 3238     greater_equal(0x3, "ae");
 3239     less_equal(0x6, "be");
 3240     greater(0x7, "a");
 3241     overflow(0x0, "o");
 3242     no_overflow(0x1, "no");
 3243   %}
 3244 %}
 3245 
 3246 
 3247 // Floating comparisons that don't require any fixup for the unordered case,
 3248 // If both inputs of the comparison are the same, ZF is always set so we
 3249 // don't need to use cmpOpUCF2 for eq/ne
 3250 operand cmpOpUCF() %{
 3251   match(Bool);
 3252   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3253             n->as_Bool()->_test._test == BoolTest::ge ||
 3254             n->as_Bool()->_test._test == BoolTest::le ||
 3255             n->as_Bool()->_test._test == BoolTest::gt ||
 3256             n->in(1)->in(1) == n->in(1)->in(2));
 3257   format %{ "" %}
 3258   interface(COND_INTER) %{
 3259     equal(0xb, "np");
 3260     not_equal(0xa, "p");
 3261     less(0x2, "b");
 3262     greater_equal(0x3, "ae");
 3263     less_equal(0x6, "be");
 3264     greater(0x7, "a");
 3265     overflow(0x0, "o");
 3266     no_overflow(0x1, "no");
 3267   %}
 3268 %}
 3269 
 3270 
 3271 // Floating comparisons that can be fixed up with extra conditional jumps
 3272 operand cmpOpUCF2() %{
 3273   match(Bool);
 3274   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3275              n->as_Bool()->_test._test == BoolTest::eq) &&
 3276             n->in(1)->in(1) != n->in(1)->in(2));
 3277   format %{ "" %}
 3278   interface(COND_INTER) %{
 3279     equal(0x4, "e");
 3280     not_equal(0x5, "ne");
 3281     less(0x2, "b");
 3282     greater_equal(0x3, "ae");
 3283     less_equal(0x6, "be");
 3284     greater(0x7, "a");
 3285     overflow(0x0, "o");
 3286     no_overflow(0x1, "no");
 3287   %}
 3288 %}
 3289 
 3290 //----------OPERAND CLASSES----------------------------------------------------
 3291 // Operand Classes are groups of operands that are used as to simplify
 3292 // instruction definitions by not requiring the AD writer to specify separate
 3293 // instructions for every form of operand when the instruction accepts
 3294 // multiple operand types with the same basic encoding and format.  The classic
 3295 // case of this is memory operands.
 3296 
 3297 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3298                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3299                indCompressedOopOffset,
 3300                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3301                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3302                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3303 
 3304 //----------PIPELINE-----------------------------------------------------------
 3305 // Rules which define the behavior of the target architectures pipeline.
 3306 pipeline %{
 3307 
 3308 //----------ATTRIBUTES---------------------------------------------------------
 3309 attributes %{
 3310   variable_size_instructions;        // Fixed size instructions
 3311   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3312   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3313   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3314   instruction_fetch_units = 1;       // of 16 bytes
 3315 
 3316   // List of nop instructions
 3317   nops( MachNop );
 3318 %}
 3319 
 3320 //----------RESOURCES----------------------------------------------------------
 3321 // Resources are the functional units available to the machine
 3322 
 3323 // Generic P2/P3 pipeline
 3324 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3325 // 3 instructions decoded per cycle.
 3326 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3327 // 3 ALU op, only ALU0 handles mul instructions.
 3328 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3329            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3330            BR, FPU,
 3331            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3332 
 3333 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3334 // Pipeline Description specifies the stages in the machine's pipeline
 3335 
 3336 // Generic P2/P3 pipeline
 3337 pipe_desc(S0, S1, S2, S3, S4, S5);
 3338 
 3339 //----------PIPELINE CLASSES---------------------------------------------------
 3340 // Pipeline Classes describe the stages in which input and output are
 3341 // referenced by the hardware pipeline.
 3342 
 3343 // Naming convention: ialu or fpu
 3344 // Then: _reg
 3345 // Then: _reg if there is a 2nd register
 3346 // Then: _long if it's a pair of instructions implementing a long
 3347 // Then: _fat if it requires the big decoder
 3348 //   Or: _mem if it requires the big decoder and a memory unit.
 3349 
 3350 // Integer ALU reg operation
 3351 pipe_class ialu_reg(rRegI dst)
 3352 %{
 3353     single_instruction;
 3354     dst    : S4(write);
 3355     dst    : S3(read);
 3356     DECODE : S0;        // any decoder
 3357     ALU    : S3;        // any alu
 3358 %}
 3359 
 3360 // Long ALU reg operation
 3361 pipe_class ialu_reg_long(rRegL dst)
 3362 %{
 3363     instruction_count(2);
 3364     dst    : S4(write);
 3365     dst    : S3(read);
 3366     DECODE : S0(2);     // any 2 decoders
 3367     ALU    : S3(2);     // both alus
 3368 %}
 3369 
 3370 // Integer ALU reg operation using big decoder
 3371 pipe_class ialu_reg_fat(rRegI dst)
 3372 %{
 3373     single_instruction;
 3374     dst    : S4(write);
 3375     dst    : S3(read);
 3376     D0     : S0;        // big decoder only
 3377     ALU    : S3;        // any alu
 3378 %}
 3379 
 3380 // Integer ALU reg-reg operation
 3381 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3382 %{
 3383     single_instruction;
 3384     dst    : S4(write);
 3385     src    : S3(read);
 3386     DECODE : S0;        // any decoder
 3387     ALU    : S3;        // any alu
 3388 %}
 3389 
 3390 // Integer ALU reg-reg operation
 3391 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3392 %{
 3393     single_instruction;
 3394     dst    : S4(write);
 3395     src    : S3(read);
 3396     D0     : S0;        // big decoder only
 3397     ALU    : S3;        // any alu
 3398 %}
 3399 
 3400 // Integer ALU reg-mem operation
 3401 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3402 %{
 3403     single_instruction;
 3404     dst    : S5(write);
 3405     mem    : S3(read);
 3406     D0     : S0;        // big decoder only
 3407     ALU    : S4;        // any alu
 3408     MEM    : S3;        // any mem
 3409 %}
 3410 
 3411 // Integer mem operation (prefetch)
 3412 pipe_class ialu_mem(memory mem)
 3413 %{
 3414     single_instruction;
 3415     mem    : S3(read);
 3416     D0     : S0;        // big decoder only
 3417     MEM    : S3;        // any mem
 3418 %}
 3419 
 3420 // Integer Store to Memory
 3421 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3422 %{
 3423     single_instruction;
 3424     mem    : S3(read);
 3425     src    : S5(read);
 3426     D0     : S0;        // big decoder only
 3427     ALU    : S4;        // any alu
 3428     MEM    : S3;
 3429 %}
 3430 
 3431 // // Long Store to Memory
 3432 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3433 // %{
 3434 //     instruction_count(2);
 3435 //     mem    : S3(read);
 3436 //     src    : S5(read);
 3437 //     D0     : S0(2);          // big decoder only; twice
 3438 //     ALU    : S4(2);     // any 2 alus
 3439 //     MEM    : S3(2);  // Both mems
 3440 // %}
 3441 
 3442 // Integer Store to Memory
 3443 pipe_class ialu_mem_imm(memory mem)
 3444 %{
 3445     single_instruction;
 3446     mem    : S3(read);
 3447     D0     : S0;        // big decoder only
 3448     ALU    : S4;        // any alu
 3449     MEM    : S3;
 3450 %}
 3451 
 3452 // Integer ALU0 reg-reg operation
 3453 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3454 %{
 3455     single_instruction;
 3456     dst    : S4(write);
 3457     src    : S3(read);
 3458     D0     : S0;        // Big decoder only
 3459     ALU0   : S3;        // only alu0
 3460 %}
 3461 
 3462 // Integer ALU0 reg-mem operation
 3463 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3464 %{
 3465     single_instruction;
 3466     dst    : S5(write);
 3467     mem    : S3(read);
 3468     D0     : S0;        // big decoder only
 3469     ALU0   : S4;        // ALU0 only
 3470     MEM    : S3;        // any mem
 3471 %}
 3472 
 3473 // Integer ALU reg-reg operation
 3474 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3475 %{
 3476     single_instruction;
 3477     cr     : S4(write);
 3478     src1   : S3(read);
 3479     src2   : S3(read);
 3480     DECODE : S0;        // any decoder
 3481     ALU    : S3;        // any alu
 3482 %}
 3483 
 3484 // Integer ALU reg-imm operation
 3485 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3486 %{
 3487     single_instruction;
 3488     cr     : S4(write);
 3489     src1   : S3(read);
 3490     DECODE : S0;        // any decoder
 3491     ALU    : S3;        // any alu
 3492 %}
 3493 
 3494 // Integer ALU reg-mem operation
 3495 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3496 %{
 3497     single_instruction;
 3498     cr     : S4(write);
 3499     src1   : S3(read);
 3500     src2   : S3(read);
 3501     D0     : S0;        // big decoder only
 3502     ALU    : S4;        // any alu
 3503     MEM    : S3;
 3504 %}
 3505 
 3506 // Conditional move reg-reg
 3507 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3508 %{
 3509     instruction_count(4);
 3510     y      : S4(read);
 3511     q      : S3(read);
 3512     p      : S3(read);
 3513     DECODE : S0(4);     // any decoder
 3514 %}
 3515 
 3516 // Conditional move reg-reg
 3517 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3518 %{
 3519     single_instruction;
 3520     dst    : S4(write);
 3521     src    : S3(read);
 3522     cr     : S3(read);
 3523     DECODE : S0;        // any decoder
 3524 %}
 3525 
 3526 // Conditional move reg-mem
 3527 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3528 %{
 3529     single_instruction;
 3530     dst    : S4(write);
 3531     src    : S3(read);
 3532     cr     : S3(read);
 3533     DECODE : S0;        // any decoder
 3534     MEM    : S3;
 3535 %}
 3536 
 3537 // Conditional move reg-reg long
 3538 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3539 %{
 3540     single_instruction;
 3541     dst    : S4(write);
 3542     src    : S3(read);
 3543     cr     : S3(read);
 3544     DECODE : S0(2);     // any 2 decoders
 3545 %}
 3546 
 3547 // Float reg-reg operation
 3548 pipe_class fpu_reg(regD dst)
 3549 %{
 3550     instruction_count(2);
 3551     dst    : S3(read);
 3552     DECODE : S0(2);     // any 2 decoders
 3553     FPU    : S3;
 3554 %}
 3555 
 3556 // Float reg-reg operation
 3557 pipe_class fpu_reg_reg(regD dst, regD src)
 3558 %{
 3559     instruction_count(2);
 3560     dst    : S4(write);
 3561     src    : S3(read);
 3562     DECODE : S0(2);     // any 2 decoders
 3563     FPU    : S3;
 3564 %}
 3565 
 3566 // Float reg-reg operation
 3567 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3568 %{
 3569     instruction_count(3);
 3570     dst    : S4(write);
 3571     src1   : S3(read);
 3572     src2   : S3(read);
 3573     DECODE : S0(3);     // any 3 decoders
 3574     FPU    : S3(2);
 3575 %}
 3576 
 3577 // Float reg-reg operation
 3578 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3579 %{
 3580     instruction_count(4);
 3581     dst    : S4(write);
 3582     src1   : S3(read);
 3583     src2   : S3(read);
 3584     src3   : S3(read);
 3585     DECODE : S0(4);     // any 3 decoders
 3586     FPU    : S3(2);
 3587 %}
 3588 
 3589 // Float reg-reg operation
 3590 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3591 %{
 3592     instruction_count(4);
 3593     dst    : S4(write);
 3594     src1   : S3(read);
 3595     src2   : S3(read);
 3596     src3   : S3(read);
 3597     DECODE : S1(3);     // any 3 decoders
 3598     D0     : S0;        // Big decoder only
 3599     FPU    : S3(2);
 3600     MEM    : S3;
 3601 %}
 3602 
 3603 // Float reg-mem operation
 3604 pipe_class fpu_reg_mem(regD dst, memory mem)
 3605 %{
 3606     instruction_count(2);
 3607     dst    : S5(write);
 3608     mem    : S3(read);
 3609     D0     : S0;        // big decoder only
 3610     DECODE : S1;        // any decoder for FPU POP
 3611     FPU    : S4;
 3612     MEM    : S3;        // any mem
 3613 %}
 3614 
 3615 // Float reg-mem operation
 3616 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3617 %{
 3618     instruction_count(3);
 3619     dst    : S5(write);
 3620     src1   : S3(read);
 3621     mem    : S3(read);
 3622     D0     : S0;        // big decoder only
 3623     DECODE : S1(2);     // any decoder for FPU POP
 3624     FPU    : S4;
 3625     MEM    : S3;        // any mem
 3626 %}
 3627 
 3628 // Float mem-reg operation
 3629 pipe_class fpu_mem_reg(memory mem, regD src)
 3630 %{
 3631     instruction_count(2);
 3632     src    : S5(read);
 3633     mem    : S3(read);
 3634     DECODE : S0;        // any decoder for FPU PUSH
 3635     D0     : S1;        // big decoder only
 3636     FPU    : S4;
 3637     MEM    : S3;        // any mem
 3638 %}
 3639 
 3640 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3641 %{
 3642     instruction_count(3);
 3643     src1   : S3(read);
 3644     src2   : S3(read);
 3645     mem    : S3(read);
 3646     DECODE : S0(2);     // any decoder for FPU PUSH
 3647     D0     : S1;        // big decoder only
 3648     FPU    : S4;
 3649     MEM    : S3;        // any mem
 3650 %}
 3651 
 3652 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3653 %{
 3654     instruction_count(3);
 3655     src1   : S3(read);
 3656     src2   : S3(read);
 3657     mem    : S4(read);
 3658     DECODE : S0;        // any decoder for FPU PUSH
 3659     D0     : S0(2);     // big decoder only
 3660     FPU    : S4;
 3661     MEM    : S3(2);     // any mem
 3662 %}
 3663 
 3664 pipe_class fpu_mem_mem(memory dst, memory src1)
 3665 %{
 3666     instruction_count(2);
 3667     src1   : S3(read);
 3668     dst    : S4(read);
 3669     D0     : S0(2);     // big decoder only
 3670     MEM    : S3(2);     // any mem
 3671 %}
 3672 
 3673 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3674 %{
 3675     instruction_count(3);
 3676     src1   : S3(read);
 3677     src2   : S3(read);
 3678     dst    : S4(read);
 3679     D0     : S0(3);     // big decoder only
 3680     FPU    : S4;
 3681     MEM    : S3(3);     // any mem
 3682 %}
 3683 
 3684 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3685 %{
 3686     instruction_count(3);
 3687     src1   : S4(read);
 3688     mem    : S4(read);
 3689     DECODE : S0;        // any decoder for FPU PUSH
 3690     D0     : S0(2);     // big decoder only
 3691     FPU    : S4;
 3692     MEM    : S3(2);     // any mem
 3693 %}
 3694 
 3695 // Float load constant
 3696 pipe_class fpu_reg_con(regD dst)
 3697 %{
 3698     instruction_count(2);
 3699     dst    : S5(write);
 3700     D0     : S0;        // big decoder only for the load
 3701     DECODE : S1;        // any decoder for FPU POP
 3702     FPU    : S4;
 3703     MEM    : S3;        // any mem
 3704 %}
 3705 
 3706 // Float load constant
 3707 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3708 %{
 3709     instruction_count(3);
 3710     dst    : S5(write);
 3711     src    : S3(read);
 3712     D0     : S0;        // big decoder only for the load
 3713     DECODE : S1(2);     // any decoder for FPU POP
 3714     FPU    : S4;
 3715     MEM    : S3;        // any mem
 3716 %}
 3717 
 3718 // UnConditional branch
 3719 pipe_class pipe_jmp(label labl)
 3720 %{
 3721     single_instruction;
 3722     BR   : S3;
 3723 %}
 3724 
 3725 // Conditional branch
 3726 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3727 %{
 3728     single_instruction;
 3729     cr    : S1(read);
 3730     BR    : S3;
 3731 %}
 3732 
 3733 // Allocation idiom
 3734 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3735 %{
 3736     instruction_count(1); force_serialization;
 3737     fixed_latency(6);
 3738     heap_ptr : S3(read);
 3739     DECODE   : S0(3);
 3740     D0       : S2;
 3741     MEM      : S3;
 3742     ALU      : S3(2);
 3743     dst      : S5(write);
 3744     BR       : S5;
 3745 %}
 3746 
 3747 // Generic big/slow expanded idiom
 3748 pipe_class pipe_slow()
 3749 %{
 3750     instruction_count(10); multiple_bundles; force_serialization;
 3751     fixed_latency(100);
 3752     D0  : S0(2);
 3753     MEM : S3(2);
 3754 %}
 3755 
 3756 // The real do-nothing guy
 3757 pipe_class empty()
 3758 %{
 3759     instruction_count(0);
 3760 %}
 3761 
 3762 // Define the class for the Nop node
 3763 define
 3764 %{
 3765    MachNop = empty;
 3766 %}
 3767 
 3768 %}
 3769 
 3770 //----------INSTRUCTIONS-------------------------------------------------------
 3771 //
 3772 // match      -- States which machine-independent subtree may be replaced
 3773 //               by this instruction.
 3774 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3775 //               selection to identify a minimum cost tree of machine
 3776 //               instructions that matches a tree of machine-independent
 3777 //               instructions.
 3778 // format     -- A string providing the disassembly for this instruction.
 3779 //               The value of an instruction's operand may be inserted
 3780 //               by referring to it with a '$' prefix.
 3781 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3782 //               to within an encode class as $primary, $secondary, and $tertiary
 3783 //               rrspectively.  The primary opcode is commonly used to
 3784 //               indicate the type of machine instruction, while secondary
 3785 //               and tertiary are often used for prefix options or addressing
 3786 //               modes.
 3787 // ins_encode -- A list of encode classes with parameters. The encode class
 3788 //               name must have been defined in an 'enc_class' specification
 3789 //               in the encode section of the architecture description.
 3790 
 3791 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3792 // Load Float
 3793 instruct MoveF2VL(vlRegF dst, regF src) %{
 3794   match(Set dst src);
 3795   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3796   ins_encode %{
 3797     ShouldNotReachHere();
 3798   %}
 3799   ins_pipe( fpu_reg_reg );
 3800 %}
 3801 
 3802 // Load Float
 3803 instruct MoveF2LEG(legRegF dst, regF src) %{
 3804   match(Set dst src);
 3805   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3806   ins_encode %{
 3807     ShouldNotReachHere();
 3808   %}
 3809   ins_pipe( fpu_reg_reg );
 3810 %}
 3811 
 3812 // Load Float
 3813 instruct MoveVL2F(regF dst, vlRegF src) %{
 3814   match(Set dst src);
 3815   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3816   ins_encode %{
 3817     ShouldNotReachHere();
 3818   %}
 3819   ins_pipe( fpu_reg_reg );
 3820 %}
 3821 
 3822 // Load Float
 3823 instruct MoveLEG2F(regF dst, legRegF src) %{
 3824   match(Set dst src);
 3825   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3826   ins_encode %{
 3827     ShouldNotReachHere();
 3828   %}
 3829   ins_pipe( fpu_reg_reg );
 3830 %}
 3831 
 3832 // Load Double
 3833 instruct MoveD2VL(vlRegD dst, regD src) %{
 3834   match(Set dst src);
 3835   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3836   ins_encode %{
 3837     ShouldNotReachHere();
 3838   %}
 3839   ins_pipe( fpu_reg_reg );
 3840 %}
 3841 
 3842 // Load Double
 3843 instruct MoveD2LEG(legRegD dst, regD src) %{
 3844   match(Set dst src);
 3845   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3846   ins_encode %{
 3847     ShouldNotReachHere();
 3848   %}
 3849   ins_pipe( fpu_reg_reg );
 3850 %}
 3851 
 3852 // Load Double
 3853 instruct MoveVL2D(regD dst, vlRegD src) %{
 3854   match(Set dst src);
 3855   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3856   ins_encode %{
 3857     ShouldNotReachHere();
 3858   %}
 3859   ins_pipe( fpu_reg_reg );
 3860 %}
 3861 
 3862 // Load Double
 3863 instruct MoveLEG2D(regD dst, legRegD src) %{
 3864   match(Set dst src);
 3865   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3866   ins_encode %{
 3867     ShouldNotReachHere();
 3868   %}
 3869   ins_pipe( fpu_reg_reg );
 3870 %}
 3871 
 3872 //----------Load/Store/Move Instructions---------------------------------------
 3873 //----------Load Instructions--------------------------------------------------
 3874 
 3875 // Load Byte (8 bit signed)
 3876 instruct loadB(rRegI dst, memory mem)
 3877 %{
 3878   match(Set dst (LoadB mem));
 3879 
 3880   ins_cost(125);
 3881   format %{ "movsbl  $dst, $mem\t# byte" %}
 3882 
 3883   ins_encode %{
 3884     __ movsbl($dst$$Register, $mem$$Address);
 3885   %}
 3886 
 3887   ins_pipe(ialu_reg_mem);
 3888 %}
 3889 
 3890 // Load Byte (8 bit signed) into Long Register
 3891 instruct loadB2L(rRegL dst, memory mem)
 3892 %{
 3893   match(Set dst (ConvI2L (LoadB mem)));
 3894 
 3895   ins_cost(125);
 3896   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 3897 
 3898   ins_encode %{
 3899     __ movsbq($dst$$Register, $mem$$Address);
 3900   %}
 3901 
 3902   ins_pipe(ialu_reg_mem);
 3903 %}
 3904 
 3905 // Load Unsigned Byte (8 bit UNsigned)
 3906 instruct loadUB(rRegI dst, memory mem)
 3907 %{
 3908   match(Set dst (LoadUB mem));
 3909 
 3910   ins_cost(125);
 3911   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 3912 
 3913   ins_encode %{
 3914     __ movzbl($dst$$Register, $mem$$Address);
 3915   %}
 3916 
 3917   ins_pipe(ialu_reg_mem);
 3918 %}
 3919 
 3920 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 3921 instruct loadUB2L(rRegL dst, memory mem)
 3922 %{
 3923   match(Set dst (ConvI2L (LoadUB mem)));
 3924 
 3925   ins_cost(125);
 3926   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 3927 
 3928   ins_encode %{
 3929     __ movzbq($dst$$Register, $mem$$Address);
 3930   %}
 3931 
 3932   ins_pipe(ialu_reg_mem);
 3933 %}
 3934 
 3935 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 3936 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 3937   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 3938   effect(KILL cr);
 3939 
 3940   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 3941             "andl    $dst, right_n_bits($mask, 8)" %}
 3942   ins_encode %{
 3943     Register Rdst = $dst$$Register;
 3944     __ movzbq(Rdst, $mem$$Address);
 3945     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 3946   %}
 3947   ins_pipe(ialu_reg_mem);
 3948 %}
 3949 
 3950 // Load Short (16 bit signed)
 3951 instruct loadS(rRegI dst, memory mem)
 3952 %{
 3953   match(Set dst (LoadS mem));
 3954 
 3955   ins_cost(125);
 3956   format %{ "movswl $dst, $mem\t# short" %}
 3957 
 3958   ins_encode %{
 3959     __ movswl($dst$$Register, $mem$$Address);
 3960   %}
 3961 
 3962   ins_pipe(ialu_reg_mem);
 3963 %}
 3964 
 3965 // Load Short (16 bit signed) to Byte (8 bit signed)
 3966 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 3967   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 3968 
 3969   ins_cost(125);
 3970   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 3971   ins_encode %{
 3972     __ movsbl($dst$$Register, $mem$$Address);
 3973   %}
 3974   ins_pipe(ialu_reg_mem);
 3975 %}
 3976 
 3977 // Load Short (16 bit signed) into Long Register
 3978 instruct loadS2L(rRegL dst, memory mem)
 3979 %{
 3980   match(Set dst (ConvI2L (LoadS mem)));
 3981 
 3982   ins_cost(125);
 3983   format %{ "movswq $dst, $mem\t# short -> long" %}
 3984 
 3985   ins_encode %{
 3986     __ movswq($dst$$Register, $mem$$Address);
 3987   %}
 3988 
 3989   ins_pipe(ialu_reg_mem);
 3990 %}
 3991 
 3992 // Load Unsigned Short/Char (16 bit UNsigned)
 3993 instruct loadUS(rRegI dst, memory mem)
 3994 %{
 3995   match(Set dst (LoadUS mem));
 3996 
 3997   ins_cost(125);
 3998   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 3999 
 4000   ins_encode %{
 4001     __ movzwl($dst$$Register, $mem$$Address);
 4002   %}
 4003 
 4004   ins_pipe(ialu_reg_mem);
 4005 %}
 4006 
 4007 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4008 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4009   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4010 
 4011   ins_cost(125);
 4012   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4013   ins_encode %{
 4014     __ movsbl($dst$$Register, $mem$$Address);
 4015   %}
 4016   ins_pipe(ialu_reg_mem);
 4017 %}
 4018 
 4019 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4020 instruct loadUS2L(rRegL dst, memory mem)
 4021 %{
 4022   match(Set dst (ConvI2L (LoadUS mem)));
 4023 
 4024   ins_cost(125);
 4025   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4026 
 4027   ins_encode %{
 4028     __ movzwq($dst$$Register, $mem$$Address);
 4029   %}
 4030 
 4031   ins_pipe(ialu_reg_mem);
 4032 %}
 4033 
 4034 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4035 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4036   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4037 
 4038   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4039   ins_encode %{
 4040     __ movzbq($dst$$Register, $mem$$Address);
 4041   %}
 4042   ins_pipe(ialu_reg_mem);
 4043 %}
 4044 
 4045 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4046 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4047   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4048   effect(KILL cr);
 4049 
 4050   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4051             "andl    $dst, right_n_bits($mask, 16)" %}
 4052   ins_encode %{
 4053     Register Rdst = $dst$$Register;
 4054     __ movzwq(Rdst, $mem$$Address);
 4055     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4056   %}
 4057   ins_pipe(ialu_reg_mem);
 4058 %}
 4059 
 4060 // Load Integer
 4061 instruct loadI(rRegI dst, memory mem)
 4062 %{
 4063   match(Set dst (LoadI mem));
 4064 
 4065   ins_cost(125);
 4066   format %{ "movl    $dst, $mem\t# int" %}
 4067 
 4068   ins_encode %{
 4069     __ movl($dst$$Register, $mem$$Address);
 4070   %}
 4071 
 4072   ins_pipe(ialu_reg_mem);
 4073 %}
 4074 
 4075 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4076 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4077   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4078 
 4079   ins_cost(125);
 4080   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4081   ins_encode %{
 4082     __ movsbl($dst$$Register, $mem$$Address);
 4083   %}
 4084   ins_pipe(ialu_reg_mem);
 4085 %}
 4086 
 4087 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4088 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4089   match(Set dst (AndI (LoadI mem) mask));
 4090 
 4091   ins_cost(125);
 4092   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4093   ins_encode %{
 4094     __ movzbl($dst$$Register, $mem$$Address);
 4095   %}
 4096   ins_pipe(ialu_reg_mem);
 4097 %}
 4098 
 4099 // Load Integer (32 bit signed) to Short (16 bit signed)
 4100 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4101   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4102 
 4103   ins_cost(125);
 4104   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4105   ins_encode %{
 4106     __ movswl($dst$$Register, $mem$$Address);
 4107   %}
 4108   ins_pipe(ialu_reg_mem);
 4109 %}
 4110 
 4111 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4112 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4113   match(Set dst (AndI (LoadI mem) mask));
 4114 
 4115   ins_cost(125);
 4116   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4117   ins_encode %{
 4118     __ movzwl($dst$$Register, $mem$$Address);
 4119   %}
 4120   ins_pipe(ialu_reg_mem);
 4121 %}
 4122 
 4123 // Load Integer into Long Register
 4124 instruct loadI2L(rRegL dst, memory mem)
 4125 %{
 4126   match(Set dst (ConvI2L (LoadI mem)));
 4127 
 4128   ins_cost(125);
 4129   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4130 
 4131   ins_encode %{
 4132     __ movslq($dst$$Register, $mem$$Address);
 4133   %}
 4134 
 4135   ins_pipe(ialu_reg_mem);
 4136 %}
 4137 
 4138 // Load Integer with mask 0xFF into Long Register
 4139 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4140   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4141 
 4142   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4143   ins_encode %{
 4144     __ movzbq($dst$$Register, $mem$$Address);
 4145   %}
 4146   ins_pipe(ialu_reg_mem);
 4147 %}
 4148 
 4149 // Load Integer with mask 0xFFFF into Long Register
 4150 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4151   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4152 
 4153   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4154   ins_encode %{
 4155     __ movzwq($dst$$Register, $mem$$Address);
 4156   %}
 4157   ins_pipe(ialu_reg_mem);
 4158 %}
 4159 
 4160 // Load Integer with a 31-bit mask into Long Register
 4161 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4162   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4163   effect(KILL cr);
 4164 
 4165   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4166             "andl    $dst, $mask" %}
 4167   ins_encode %{
 4168     Register Rdst = $dst$$Register;
 4169     __ movl(Rdst, $mem$$Address);
 4170     __ andl(Rdst, $mask$$constant);
 4171   %}
 4172   ins_pipe(ialu_reg_mem);
 4173 %}
 4174 
 4175 // Load Unsigned Integer into Long Register
 4176 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4177 %{
 4178   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4179 
 4180   ins_cost(125);
 4181   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4182 
 4183   ins_encode %{
 4184     __ movl($dst$$Register, $mem$$Address);
 4185   %}
 4186 
 4187   ins_pipe(ialu_reg_mem);
 4188 %}
 4189 
 4190 // Load Long
 4191 instruct loadL(rRegL dst, memory mem)
 4192 %{
 4193   match(Set dst (LoadL mem));
 4194 
 4195   ins_cost(125);
 4196   format %{ "movq    $dst, $mem\t# long" %}
 4197 
 4198   ins_encode %{
 4199     __ movq($dst$$Register, $mem$$Address);
 4200   %}
 4201 
 4202   ins_pipe(ialu_reg_mem); // XXX
 4203 %}
 4204 
 4205 // Load Range
 4206 instruct loadRange(rRegI dst, memory mem)
 4207 %{
 4208   match(Set dst (LoadRange mem));
 4209 
 4210   ins_cost(125); // XXX
 4211   format %{ "movl    $dst, $mem\t# range" %}
 4212   ins_encode %{
 4213     __ movl($dst$$Register, $mem$$Address);
 4214   %}
 4215   ins_pipe(ialu_reg_mem);
 4216 %}
 4217 
 4218 // Load Pointer
 4219 instruct loadP(rRegP dst, memory mem)
 4220 %{
 4221   match(Set dst (LoadP mem));
 4222   predicate(n->as_Load()->barrier_data() == 0);
 4223 
 4224   ins_cost(125); // XXX
 4225   format %{ "movq    $dst, $mem\t# ptr" %}
 4226   ins_encode %{
 4227     __ movq($dst$$Register, $mem$$Address);
 4228   %}
 4229   ins_pipe(ialu_reg_mem); // XXX
 4230 %}
 4231 
 4232 // Load Compressed Pointer
 4233 instruct loadN(rRegN dst, memory mem)
 4234 %{
 4235    match(Set dst (LoadN mem));
 4236 
 4237    ins_cost(125); // XXX
 4238    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4239    ins_encode %{
 4240      __ movl($dst$$Register, $mem$$Address);
 4241    %}
 4242    ins_pipe(ialu_reg_mem); // XXX
 4243 %}
 4244 
 4245 
 4246 // Load Klass Pointer
 4247 instruct loadKlass(rRegP dst, memory mem)
 4248 %{
 4249   match(Set dst (LoadKlass mem));
 4250 
 4251   ins_cost(125); // XXX
 4252   format %{ "movq    $dst, $mem\t# class" %}
 4253   ins_encode %{
 4254     __ movq($dst$$Register, $mem$$Address);
 4255   %}
 4256   ins_pipe(ialu_reg_mem); // XXX
 4257 %}
 4258 
 4259 // Load narrow Klass Pointer
 4260 instruct loadNKlass(rRegN dst, memory mem)
 4261 %{
 4262   match(Set dst (LoadNKlass mem));
 4263 
 4264   ins_cost(125); // XXX
 4265   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4266   ins_encode %{
 4267     __ movl($dst$$Register, $mem$$Address);
 4268   %}
 4269   ins_pipe(ialu_reg_mem); // XXX
 4270 %}
 4271 
 4272 // Load Float
 4273 instruct loadF(regF dst, memory mem)
 4274 %{
 4275   match(Set dst (LoadF mem));
 4276 
 4277   ins_cost(145); // XXX
 4278   format %{ "movss   $dst, $mem\t# float" %}
 4279   ins_encode %{
 4280     __ movflt($dst$$XMMRegister, $mem$$Address);
 4281   %}
 4282   ins_pipe(pipe_slow); // XXX
 4283 %}
 4284 
 4285 // Load Double
 4286 instruct loadD_partial(regD dst, memory mem)
 4287 %{
 4288   predicate(!UseXmmLoadAndClearUpper);
 4289   match(Set dst (LoadD mem));
 4290 
 4291   ins_cost(145); // XXX
 4292   format %{ "movlpd  $dst, $mem\t# double" %}
 4293   ins_encode %{
 4294     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4295   %}
 4296   ins_pipe(pipe_slow); // XXX
 4297 %}
 4298 
 4299 instruct loadD(regD dst, memory mem)
 4300 %{
 4301   predicate(UseXmmLoadAndClearUpper);
 4302   match(Set dst (LoadD mem));
 4303 
 4304   ins_cost(145); // XXX
 4305   format %{ "movsd   $dst, $mem\t# double" %}
 4306   ins_encode %{
 4307     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4308   %}
 4309   ins_pipe(pipe_slow); // XXX
 4310 %}
 4311 
 4312 // max = java.lang.Math.max(float a, float b)
 4313 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4314   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4315   match(Set dst (MaxF a b));
 4316   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4317   format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4318   ins_encode %{
 4319     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4320   %}
 4321   ins_pipe( pipe_slow );
 4322 %}
 4323 
 4324 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4325   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4326   match(Set dst (MaxF a b));
 4327   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4328 
 4329   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 4330   ins_encode %{
 4331     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4332                     false /*min*/, true /*single*/);
 4333   %}
 4334   ins_pipe( pipe_slow );
 4335 %}
 4336 
 4337 // max = java.lang.Math.max(double a, double b)
 4338 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4339   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4340   match(Set dst (MaxD a b));
 4341   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4342   format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4343   ins_encode %{
 4344     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4345   %}
 4346   ins_pipe( pipe_slow );
 4347 %}
 4348 
 4349 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4350   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4351   match(Set dst (MaxD a b));
 4352   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4353 
 4354   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 4355   ins_encode %{
 4356     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4357                     false /*min*/, false /*single*/);
 4358   %}
 4359   ins_pipe( pipe_slow );
 4360 %}
 4361 
 4362 // min = java.lang.Math.min(float a, float b)
 4363 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4364   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4365   match(Set dst (MinF a b));
 4366   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4367   format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4368   ins_encode %{
 4369     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4370   %}
 4371   ins_pipe( pipe_slow );
 4372 %}
 4373 
 4374 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4375   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4376   match(Set dst (MinF a b));
 4377   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4378 
 4379   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 4380   ins_encode %{
 4381     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4382                     true /*min*/, true /*single*/);
 4383   %}
 4384   ins_pipe( pipe_slow );
 4385 %}
 4386 
 4387 // min = java.lang.Math.min(double a, double b)
 4388 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4389   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4390   match(Set dst (MinD a b));
 4391   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4392     format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4393   ins_encode %{
 4394     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4395   %}
 4396   ins_pipe( pipe_slow );
 4397 %}
 4398 
 4399 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4400   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4401   match(Set dst (MinD a b));
 4402   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4403 
 4404   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 4405   ins_encode %{
 4406     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4407                     true /*min*/, false /*single*/);
 4408   %}
 4409   ins_pipe( pipe_slow );
 4410 %}
 4411 
 4412 // Load Effective Address
 4413 instruct leaP8(rRegP dst, indOffset8 mem)
 4414 %{
 4415   match(Set dst mem);
 4416 
 4417   ins_cost(110); // XXX
 4418   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4419   ins_encode %{
 4420     __ leaq($dst$$Register, $mem$$Address);
 4421   %}
 4422   ins_pipe(ialu_reg_reg_fat);
 4423 %}
 4424 
 4425 instruct leaP32(rRegP dst, indOffset32 mem)
 4426 %{
 4427   match(Set dst mem);
 4428 
 4429   ins_cost(110);
 4430   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4431   ins_encode %{
 4432     __ leaq($dst$$Register, $mem$$Address);
 4433   %}
 4434   ins_pipe(ialu_reg_reg_fat);
 4435 %}
 4436 
 4437 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4438 %{
 4439   match(Set dst mem);
 4440 
 4441   ins_cost(110);
 4442   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4443   ins_encode %{
 4444     __ leaq($dst$$Register, $mem$$Address);
 4445   %}
 4446   ins_pipe(ialu_reg_reg_fat);
 4447 %}
 4448 
 4449 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4450 %{
 4451   match(Set dst mem);
 4452 
 4453   ins_cost(110);
 4454   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4455   ins_encode %{
 4456     __ leaq($dst$$Register, $mem$$Address);
 4457   %}
 4458   ins_pipe(ialu_reg_reg_fat);
 4459 %}
 4460 
 4461 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4462 %{
 4463   match(Set dst mem);
 4464 
 4465   ins_cost(110);
 4466   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4467   ins_encode %{
 4468     __ leaq($dst$$Register, $mem$$Address);
 4469   %}
 4470   ins_pipe(ialu_reg_reg_fat);
 4471 %}
 4472 
 4473 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4474 %{
 4475   match(Set dst mem);
 4476 
 4477   ins_cost(110);
 4478   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4479   ins_encode %{
 4480     __ leaq($dst$$Register, $mem$$Address);
 4481   %}
 4482   ins_pipe(ialu_reg_reg_fat);
 4483 %}
 4484 
 4485 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4486 %{
 4487   match(Set dst mem);
 4488 
 4489   ins_cost(110);
 4490   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4491   ins_encode %{
 4492     __ leaq($dst$$Register, $mem$$Address);
 4493   %}
 4494   ins_pipe(ialu_reg_reg_fat);
 4495 %}
 4496 
 4497 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4498 %{
 4499   match(Set dst mem);
 4500 
 4501   ins_cost(110);
 4502   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4503   ins_encode %{
 4504     __ leaq($dst$$Register, $mem$$Address);
 4505   %}
 4506   ins_pipe(ialu_reg_reg_fat);
 4507 %}
 4508 
 4509 // Load Effective Address which uses Narrow (32-bits) oop
 4510 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4511 %{
 4512   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4513   match(Set dst mem);
 4514 
 4515   ins_cost(110);
 4516   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4517   ins_encode %{
 4518     __ leaq($dst$$Register, $mem$$Address);
 4519   %}
 4520   ins_pipe(ialu_reg_reg_fat);
 4521 %}
 4522 
 4523 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4524 %{
 4525   predicate(CompressedOops::shift() == 0);
 4526   match(Set dst mem);
 4527 
 4528   ins_cost(110); // XXX
 4529   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4530   ins_encode %{
 4531     __ leaq($dst$$Register, $mem$$Address);
 4532   %}
 4533   ins_pipe(ialu_reg_reg_fat);
 4534 %}
 4535 
 4536 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4537 %{
 4538   predicate(CompressedOops::shift() == 0);
 4539   match(Set dst mem);
 4540 
 4541   ins_cost(110);
 4542   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4543   ins_encode %{
 4544     __ leaq($dst$$Register, $mem$$Address);
 4545   %}
 4546   ins_pipe(ialu_reg_reg_fat);
 4547 %}
 4548 
 4549 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4550 %{
 4551   predicate(CompressedOops::shift() == 0);
 4552   match(Set dst mem);
 4553 
 4554   ins_cost(110);
 4555   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4556   ins_encode %{
 4557     __ leaq($dst$$Register, $mem$$Address);
 4558   %}
 4559   ins_pipe(ialu_reg_reg_fat);
 4560 %}
 4561 
 4562 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4563 %{
 4564   predicate(CompressedOops::shift() == 0);
 4565   match(Set dst mem);
 4566 
 4567   ins_cost(110);
 4568   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4569   ins_encode %{
 4570     __ leaq($dst$$Register, $mem$$Address);
 4571   %}
 4572   ins_pipe(ialu_reg_reg_fat);
 4573 %}
 4574 
 4575 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4576 %{
 4577   predicate(CompressedOops::shift() == 0);
 4578   match(Set dst mem);
 4579 
 4580   ins_cost(110);
 4581   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4582   ins_encode %{
 4583     __ leaq($dst$$Register, $mem$$Address);
 4584   %}
 4585   ins_pipe(ialu_reg_reg_fat);
 4586 %}
 4587 
 4588 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4589 %{
 4590   predicate(CompressedOops::shift() == 0);
 4591   match(Set dst mem);
 4592 
 4593   ins_cost(110);
 4594   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4595   ins_encode %{
 4596     __ leaq($dst$$Register, $mem$$Address);
 4597   %}
 4598   ins_pipe(ialu_reg_reg_fat);
 4599 %}
 4600 
 4601 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4602 %{
 4603   predicate(CompressedOops::shift() == 0);
 4604   match(Set dst mem);
 4605 
 4606   ins_cost(110);
 4607   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4608   ins_encode %{
 4609     __ leaq($dst$$Register, $mem$$Address);
 4610   %}
 4611   ins_pipe(ialu_reg_reg_fat);
 4612 %}
 4613 
 4614 instruct loadConI(rRegI dst, immI src)
 4615 %{
 4616   match(Set dst src);
 4617 
 4618   format %{ "movl    $dst, $src\t# int" %}
 4619   ins_encode %{
 4620     __ movl($dst$$Register, $src$$constant);
 4621   %}
 4622   ins_pipe(ialu_reg_fat); // XXX
 4623 %}
 4624 
 4625 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4626 %{
 4627   match(Set dst src);
 4628   effect(KILL cr);
 4629 
 4630   ins_cost(50);
 4631   format %{ "xorl    $dst, $dst\t# int" %}
 4632   ins_encode %{
 4633     __ xorl($dst$$Register, $dst$$Register);
 4634   %}
 4635   ins_pipe(ialu_reg);
 4636 %}
 4637 
 4638 instruct loadConL(rRegL dst, immL src)
 4639 %{
 4640   match(Set dst src);
 4641 
 4642   ins_cost(150);
 4643   format %{ "movq    $dst, $src\t# long" %}
 4644   ins_encode %{
 4645     __ mov64($dst$$Register, $src$$constant);
 4646   %}
 4647   ins_pipe(ialu_reg);
 4648 %}
 4649 
 4650 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4651 %{
 4652   match(Set dst src);
 4653   effect(KILL cr);
 4654 
 4655   ins_cost(50);
 4656   format %{ "xorl    $dst, $dst\t# long" %}
 4657   ins_encode %{
 4658     __ xorl($dst$$Register, $dst$$Register);
 4659   %}
 4660   ins_pipe(ialu_reg); // XXX
 4661 %}
 4662 
 4663 instruct loadConUL32(rRegL dst, immUL32 src)
 4664 %{
 4665   match(Set dst src);
 4666 
 4667   ins_cost(60);
 4668   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4669   ins_encode %{
 4670     __ movl($dst$$Register, $src$$constant);
 4671   %}
 4672   ins_pipe(ialu_reg);
 4673 %}
 4674 
 4675 instruct loadConL32(rRegL dst, immL32 src)
 4676 %{
 4677   match(Set dst src);
 4678 
 4679   ins_cost(70);
 4680   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4681   ins_encode %{
 4682     __ movq($dst$$Register, $src$$constant);
 4683   %}
 4684   ins_pipe(ialu_reg);
 4685 %}
 4686 
 4687 instruct loadConP(rRegP dst, immP con) %{
 4688   match(Set dst con);
 4689 
 4690   format %{ "movq    $dst, $con\t# ptr" %}
 4691   ins_encode %{
 4692     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4693   %}
 4694   ins_pipe(ialu_reg_fat); // XXX
 4695 %}
 4696 
 4697 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4698 %{
 4699   match(Set dst src);
 4700   effect(KILL cr);
 4701 
 4702   ins_cost(50);
 4703   format %{ "xorl    $dst, $dst\t# ptr" %}
 4704   ins_encode %{
 4705     __ xorl($dst$$Register, $dst$$Register);
 4706   %}
 4707   ins_pipe(ialu_reg);
 4708 %}
 4709 
 4710 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4711 %{
 4712   match(Set dst src);
 4713   effect(KILL cr);
 4714 
 4715   ins_cost(60);
 4716   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4717   ins_encode %{
 4718     __ movl($dst$$Register, $src$$constant);
 4719   %}
 4720   ins_pipe(ialu_reg);
 4721 %}
 4722 
 4723 instruct loadConF(regF dst, immF con) %{
 4724   match(Set dst con);
 4725   ins_cost(125);
 4726   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4727   ins_encode %{
 4728     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4729   %}
 4730   ins_pipe(pipe_slow);
 4731 %}
 4732 
 4733 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4734   match(Set dst src);
 4735   effect(KILL cr);
 4736   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4737   ins_encode %{
 4738     __ xorq($dst$$Register, $dst$$Register);
 4739   %}
 4740   ins_pipe(ialu_reg);
 4741 %}
 4742 
 4743 instruct loadConN(rRegN dst, immN src) %{
 4744   match(Set dst src);
 4745 
 4746   ins_cost(125);
 4747   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4748   ins_encode %{
 4749     address con = (address)$src$$constant;
 4750     if (con == nullptr) {
 4751       ShouldNotReachHere();
 4752     } else {
 4753       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4754     }
 4755   %}
 4756   ins_pipe(ialu_reg_fat); // XXX
 4757 %}
 4758 
 4759 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4760   match(Set dst src);
 4761 
 4762   ins_cost(125);
 4763   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4764   ins_encode %{
 4765     address con = (address)$src$$constant;
 4766     if (con == nullptr) {
 4767       ShouldNotReachHere();
 4768     } else {
 4769       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4770     }
 4771   %}
 4772   ins_pipe(ialu_reg_fat); // XXX
 4773 %}
 4774 
 4775 instruct loadConF0(regF dst, immF0 src)
 4776 %{
 4777   match(Set dst src);
 4778   ins_cost(100);
 4779 
 4780   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4781   ins_encode %{
 4782     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4783   %}
 4784   ins_pipe(pipe_slow);
 4785 %}
 4786 
 4787 // Use the same format since predicate() can not be used here.
 4788 instruct loadConD(regD dst, immD con) %{
 4789   match(Set dst con);
 4790   ins_cost(125);
 4791   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4792   ins_encode %{
 4793     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4794   %}
 4795   ins_pipe(pipe_slow);
 4796 %}
 4797 
 4798 instruct loadConD0(regD dst, immD0 src)
 4799 %{
 4800   match(Set dst src);
 4801   ins_cost(100);
 4802 
 4803   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4804   ins_encode %{
 4805     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4806   %}
 4807   ins_pipe(pipe_slow);
 4808 %}
 4809 
 4810 instruct loadSSI(rRegI dst, stackSlotI src)
 4811 %{
 4812   match(Set dst src);
 4813 
 4814   ins_cost(125);
 4815   format %{ "movl    $dst, $src\t# int stk" %}
 4816   ins_encode %{
 4817     __ movl($dst$$Register, $src$$Address);
 4818   %}
 4819   ins_pipe(ialu_reg_mem);
 4820 %}
 4821 
 4822 instruct loadSSL(rRegL dst, stackSlotL src)
 4823 %{
 4824   match(Set dst src);
 4825 
 4826   ins_cost(125);
 4827   format %{ "movq    $dst, $src\t# long stk" %}
 4828   ins_encode %{
 4829     __ movq($dst$$Register, $src$$Address);
 4830   %}
 4831   ins_pipe(ialu_reg_mem);
 4832 %}
 4833 
 4834 instruct loadSSP(rRegP dst, stackSlotP src)
 4835 %{
 4836   match(Set dst src);
 4837 
 4838   ins_cost(125);
 4839   format %{ "movq    $dst, $src\t# ptr stk" %}
 4840   ins_encode %{
 4841     __ movq($dst$$Register, $src$$Address);
 4842   %}
 4843   ins_pipe(ialu_reg_mem);
 4844 %}
 4845 
 4846 instruct loadSSF(regF dst, stackSlotF src)
 4847 %{
 4848   match(Set dst src);
 4849 
 4850   ins_cost(125);
 4851   format %{ "movss   $dst, $src\t# float stk" %}
 4852   ins_encode %{
 4853     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 4854   %}
 4855   ins_pipe(pipe_slow); // XXX
 4856 %}
 4857 
 4858 // Use the same format since predicate() can not be used here.
 4859 instruct loadSSD(regD dst, stackSlotD src)
 4860 %{
 4861   match(Set dst src);
 4862 
 4863   ins_cost(125);
 4864   format %{ "movsd   $dst, $src\t# double stk" %}
 4865   ins_encode  %{
 4866     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 4867   %}
 4868   ins_pipe(pipe_slow); // XXX
 4869 %}
 4870 
 4871 // Prefetch instructions for allocation.
 4872 // Must be safe to execute with invalid address (cannot fault).
 4873 
 4874 instruct prefetchAlloc( memory mem ) %{
 4875   predicate(AllocatePrefetchInstr==3);
 4876   match(PrefetchAllocation mem);
 4877   ins_cost(125);
 4878 
 4879   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 4880   ins_encode %{
 4881     __ prefetchw($mem$$Address);
 4882   %}
 4883   ins_pipe(ialu_mem);
 4884 %}
 4885 
 4886 instruct prefetchAllocNTA( memory mem ) %{
 4887   predicate(AllocatePrefetchInstr==0);
 4888   match(PrefetchAllocation mem);
 4889   ins_cost(125);
 4890 
 4891   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 4892   ins_encode %{
 4893     __ prefetchnta($mem$$Address);
 4894   %}
 4895   ins_pipe(ialu_mem);
 4896 %}
 4897 
 4898 instruct prefetchAllocT0( memory mem ) %{
 4899   predicate(AllocatePrefetchInstr==1);
 4900   match(PrefetchAllocation mem);
 4901   ins_cost(125);
 4902 
 4903   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 4904   ins_encode %{
 4905     __ prefetcht0($mem$$Address);
 4906   %}
 4907   ins_pipe(ialu_mem);
 4908 %}
 4909 
 4910 instruct prefetchAllocT2( memory mem ) %{
 4911   predicate(AllocatePrefetchInstr==2);
 4912   match(PrefetchAllocation mem);
 4913   ins_cost(125);
 4914 
 4915   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 4916   ins_encode %{
 4917     __ prefetcht2($mem$$Address);
 4918   %}
 4919   ins_pipe(ialu_mem);
 4920 %}
 4921 
 4922 //----------Store Instructions-------------------------------------------------
 4923 
 4924 // Store Byte
 4925 instruct storeB(memory mem, rRegI src)
 4926 %{
 4927   match(Set mem (StoreB mem src));
 4928 
 4929   ins_cost(125); // XXX
 4930   format %{ "movb    $mem, $src\t# byte" %}
 4931   ins_encode %{
 4932     __ movb($mem$$Address, $src$$Register);
 4933   %}
 4934   ins_pipe(ialu_mem_reg);
 4935 %}
 4936 
 4937 // Store Char/Short
 4938 instruct storeC(memory mem, rRegI src)
 4939 %{
 4940   match(Set mem (StoreC mem src));
 4941 
 4942   ins_cost(125); // XXX
 4943   format %{ "movw    $mem, $src\t# char/short" %}
 4944   ins_encode %{
 4945     __ movw($mem$$Address, $src$$Register);
 4946   %}
 4947   ins_pipe(ialu_mem_reg);
 4948 %}
 4949 
 4950 // Store Integer
 4951 instruct storeI(memory mem, rRegI src)
 4952 %{
 4953   match(Set mem (StoreI mem src));
 4954 
 4955   ins_cost(125); // XXX
 4956   format %{ "movl    $mem, $src\t# int" %}
 4957   ins_encode %{
 4958     __ movl($mem$$Address, $src$$Register);
 4959   %}
 4960   ins_pipe(ialu_mem_reg);
 4961 %}
 4962 
 4963 // Store Long
 4964 instruct storeL(memory mem, rRegL src)
 4965 %{
 4966   match(Set mem (StoreL mem src));
 4967 
 4968   ins_cost(125); // XXX
 4969   format %{ "movq    $mem, $src\t# long" %}
 4970   ins_encode %{
 4971     __ movq($mem$$Address, $src$$Register);
 4972   %}
 4973   ins_pipe(ialu_mem_reg); // XXX
 4974 %}
 4975 
 4976 // Store Pointer
 4977 instruct storeP(memory mem, any_RegP src)
 4978 %{
 4979   predicate(n->as_Store()->barrier_data() == 0);
 4980   match(Set mem (StoreP mem src));
 4981 
 4982   ins_cost(125); // XXX
 4983   format %{ "movq    $mem, $src\t# ptr" %}
 4984   ins_encode %{
 4985     __ movq($mem$$Address, $src$$Register);
 4986   %}
 4987   ins_pipe(ialu_mem_reg);
 4988 %}
 4989 
 4990 instruct storeImmP0(memory mem, immP0 zero)
 4991 %{
 4992   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 4993   match(Set mem (StoreP mem zero));
 4994 
 4995   ins_cost(125); // XXX
 4996   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 4997   ins_encode %{
 4998     __ movq($mem$$Address, r12);
 4999   %}
 5000   ins_pipe(ialu_mem_reg);
 5001 %}
 5002 
 5003 // Store Null Pointer, mark word, or other simple pointer constant.
 5004 instruct storeImmP(memory mem, immP31 src)
 5005 %{
 5006   predicate(n->as_Store()->barrier_data() == 0);
 5007   match(Set mem (StoreP mem src));
 5008 
 5009   ins_cost(150); // XXX
 5010   format %{ "movq    $mem, $src\t# ptr" %}
 5011   ins_encode %{
 5012     __ movq($mem$$Address, $src$$constant);
 5013   %}
 5014   ins_pipe(ialu_mem_imm);
 5015 %}
 5016 
 5017 // Store Compressed Pointer
 5018 instruct storeN(memory mem, rRegN src)
 5019 %{
 5020   match(Set mem (StoreN mem src));
 5021 
 5022   ins_cost(125); // XXX
 5023   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5024   ins_encode %{
 5025     __ movl($mem$$Address, $src$$Register);
 5026   %}
 5027   ins_pipe(ialu_mem_reg);
 5028 %}
 5029 
 5030 instruct storeNKlass(memory mem, rRegN src)
 5031 %{
 5032   match(Set mem (StoreNKlass mem src));
 5033 
 5034   ins_cost(125); // XXX
 5035   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5036   ins_encode %{
 5037     __ movl($mem$$Address, $src$$Register);
 5038   %}
 5039   ins_pipe(ialu_mem_reg);
 5040 %}
 5041 
 5042 instruct storeImmN0(memory mem, immN0 zero)
 5043 %{
 5044   predicate(CompressedOops::base() == nullptr);
 5045   match(Set mem (StoreN mem zero));
 5046 
 5047   ins_cost(125); // XXX
 5048   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5049   ins_encode %{
 5050     __ movl($mem$$Address, r12);
 5051   %}
 5052   ins_pipe(ialu_mem_reg);
 5053 %}
 5054 
 5055 instruct storeImmN(memory mem, immN src)
 5056 %{
 5057   match(Set mem (StoreN mem src));
 5058 
 5059   ins_cost(150); // XXX
 5060   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5061   ins_encode %{
 5062     address con = (address)$src$$constant;
 5063     if (con == nullptr) {
 5064       __ movl($mem$$Address, 0);
 5065     } else {
 5066       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5067     }
 5068   %}
 5069   ins_pipe(ialu_mem_imm);
 5070 %}
 5071 
 5072 instruct storeImmNKlass(memory mem, immNKlass src)
 5073 %{
 5074   match(Set mem (StoreNKlass mem src));
 5075 
 5076   ins_cost(150); // XXX
 5077   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5078   ins_encode %{
 5079     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5080   %}
 5081   ins_pipe(ialu_mem_imm);
 5082 %}
 5083 
 5084 // Store Integer Immediate
 5085 instruct storeImmI0(memory mem, immI_0 zero)
 5086 %{
 5087   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5088   match(Set mem (StoreI mem zero));
 5089 
 5090   ins_cost(125); // XXX
 5091   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5092   ins_encode %{
 5093     __ movl($mem$$Address, r12);
 5094   %}
 5095   ins_pipe(ialu_mem_reg);
 5096 %}
 5097 
 5098 instruct storeImmI(memory mem, immI src)
 5099 %{
 5100   match(Set mem (StoreI mem src));
 5101 
 5102   ins_cost(150);
 5103   format %{ "movl    $mem, $src\t# int" %}
 5104   ins_encode %{
 5105     __ movl($mem$$Address, $src$$constant);
 5106   %}
 5107   ins_pipe(ialu_mem_imm);
 5108 %}
 5109 
 5110 // Store Long Immediate
 5111 instruct storeImmL0(memory mem, immL0 zero)
 5112 %{
 5113   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5114   match(Set mem (StoreL mem zero));
 5115 
 5116   ins_cost(125); // XXX
 5117   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5118   ins_encode %{
 5119     __ movq($mem$$Address, r12);
 5120   %}
 5121   ins_pipe(ialu_mem_reg);
 5122 %}
 5123 
 5124 instruct storeImmL(memory mem, immL32 src)
 5125 %{
 5126   match(Set mem (StoreL mem src));
 5127 
 5128   ins_cost(150);
 5129   format %{ "movq    $mem, $src\t# long" %}
 5130   ins_encode %{
 5131     __ movq($mem$$Address, $src$$constant);
 5132   %}
 5133   ins_pipe(ialu_mem_imm);
 5134 %}
 5135 
 5136 // Store Short/Char Immediate
 5137 instruct storeImmC0(memory mem, immI_0 zero)
 5138 %{
 5139   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5140   match(Set mem (StoreC mem zero));
 5141 
 5142   ins_cost(125); // XXX
 5143   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5144   ins_encode %{
 5145     __ movw($mem$$Address, r12);
 5146   %}
 5147   ins_pipe(ialu_mem_reg);
 5148 %}
 5149 
 5150 instruct storeImmI16(memory mem, immI16 src)
 5151 %{
 5152   predicate(UseStoreImmI16);
 5153   match(Set mem (StoreC mem src));
 5154 
 5155   ins_cost(150);
 5156   format %{ "movw    $mem, $src\t# short/char" %}
 5157   ins_encode %{
 5158     __ movw($mem$$Address, $src$$constant);
 5159   %}
 5160   ins_pipe(ialu_mem_imm);
 5161 %}
 5162 
 5163 // Store Byte Immediate
 5164 instruct storeImmB0(memory mem, immI_0 zero)
 5165 %{
 5166   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5167   match(Set mem (StoreB mem zero));
 5168 
 5169   ins_cost(125); // XXX
 5170   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5171   ins_encode %{
 5172     __ movb($mem$$Address, r12);
 5173   %}
 5174   ins_pipe(ialu_mem_reg);
 5175 %}
 5176 
 5177 instruct storeImmB(memory mem, immI8 src)
 5178 %{
 5179   match(Set mem (StoreB mem src));
 5180 
 5181   ins_cost(150); // XXX
 5182   format %{ "movb    $mem, $src\t# byte" %}
 5183   ins_encode %{
 5184     __ movb($mem$$Address, $src$$constant);
 5185   %}
 5186   ins_pipe(ialu_mem_imm);
 5187 %}
 5188 
 5189 // Store CMS card-mark Immediate
 5190 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 5191 %{
 5192   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5193   match(Set mem (StoreCM mem zero));
 5194 
 5195   ins_cost(125); // XXX
 5196   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 5197   ins_encode %{
 5198     __ movb($mem$$Address, r12);
 5199   %}
 5200   ins_pipe(ialu_mem_reg);
 5201 %}
 5202 
 5203 instruct storeImmCM0(memory mem, immI_0 src)
 5204 %{
 5205   match(Set mem (StoreCM mem src));
 5206 
 5207   ins_cost(150); // XXX
 5208   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 5209   ins_encode %{
 5210     __ movb($mem$$Address, $src$$constant);
 5211   %}
 5212   ins_pipe(ialu_mem_imm);
 5213 %}
 5214 
 5215 // Store Float
 5216 instruct storeF(memory mem, regF src)
 5217 %{
 5218   match(Set mem (StoreF mem src));
 5219 
 5220   ins_cost(95); // XXX
 5221   format %{ "movss   $mem, $src\t# float" %}
 5222   ins_encode %{
 5223     __ movflt($mem$$Address, $src$$XMMRegister);
 5224   %}
 5225   ins_pipe(pipe_slow); // XXX
 5226 %}
 5227 
 5228 // Store immediate Float value (it is faster than store from XMM register)
 5229 instruct storeF0(memory mem, immF0 zero)
 5230 %{
 5231   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5232   match(Set mem (StoreF mem zero));
 5233 
 5234   ins_cost(25); // XXX
 5235   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5236   ins_encode %{
 5237     __ movl($mem$$Address, r12);
 5238   %}
 5239   ins_pipe(ialu_mem_reg);
 5240 %}
 5241 
 5242 instruct storeF_imm(memory mem, immF src)
 5243 %{
 5244   match(Set mem (StoreF mem src));
 5245 
 5246   ins_cost(50);
 5247   format %{ "movl    $mem, $src\t# float" %}
 5248   ins_encode %{
 5249     __ movl($mem$$Address, jint_cast($src$$constant));
 5250   %}
 5251   ins_pipe(ialu_mem_imm);
 5252 %}
 5253 
 5254 // Store Double
 5255 instruct storeD(memory mem, regD src)
 5256 %{
 5257   match(Set mem (StoreD mem src));
 5258 
 5259   ins_cost(95); // XXX
 5260   format %{ "movsd   $mem, $src\t# double" %}
 5261   ins_encode %{
 5262     __ movdbl($mem$$Address, $src$$XMMRegister);
 5263   %}
 5264   ins_pipe(pipe_slow); // XXX
 5265 %}
 5266 
 5267 // Store immediate double 0.0 (it is faster than store from XMM register)
 5268 instruct storeD0_imm(memory mem, immD0 src)
 5269 %{
 5270   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5271   match(Set mem (StoreD mem src));
 5272 
 5273   ins_cost(50);
 5274   format %{ "movq    $mem, $src\t# double 0." %}
 5275   ins_encode %{
 5276     __ movq($mem$$Address, $src$$constant);
 5277   %}
 5278   ins_pipe(ialu_mem_imm);
 5279 %}
 5280 
 5281 instruct storeD0(memory mem, immD0 zero)
 5282 %{
 5283   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5284   match(Set mem (StoreD mem zero));
 5285 
 5286   ins_cost(25); // XXX
 5287   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5288   ins_encode %{
 5289     __ movq($mem$$Address, r12);
 5290   %}
 5291   ins_pipe(ialu_mem_reg);
 5292 %}
 5293 
 5294 instruct storeSSI(stackSlotI dst, rRegI src)
 5295 %{
 5296   match(Set dst src);
 5297 
 5298   ins_cost(100);
 5299   format %{ "movl    $dst, $src\t# int stk" %}
 5300   ins_encode %{
 5301     __ movl($dst$$Address, $src$$Register);
 5302   %}
 5303   ins_pipe( ialu_mem_reg );
 5304 %}
 5305 
 5306 instruct storeSSL(stackSlotL dst, rRegL src)
 5307 %{
 5308   match(Set dst src);
 5309 
 5310   ins_cost(100);
 5311   format %{ "movq    $dst, $src\t# long stk" %}
 5312   ins_encode %{
 5313     __ movq($dst$$Address, $src$$Register);
 5314   %}
 5315   ins_pipe(ialu_mem_reg);
 5316 %}
 5317 
 5318 instruct storeSSP(stackSlotP dst, rRegP src)
 5319 %{
 5320   match(Set dst src);
 5321 
 5322   ins_cost(100);
 5323   format %{ "movq    $dst, $src\t# ptr stk" %}
 5324   ins_encode %{
 5325     __ movq($dst$$Address, $src$$Register);
 5326   %}
 5327   ins_pipe(ialu_mem_reg);
 5328 %}
 5329 
 5330 instruct storeSSF(stackSlotF dst, regF src)
 5331 %{
 5332   match(Set dst src);
 5333 
 5334   ins_cost(95); // XXX
 5335   format %{ "movss   $dst, $src\t# float stk" %}
 5336   ins_encode %{
 5337     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5338   %}
 5339   ins_pipe(pipe_slow); // XXX
 5340 %}
 5341 
 5342 instruct storeSSD(stackSlotD dst, regD src)
 5343 %{
 5344   match(Set dst src);
 5345 
 5346   ins_cost(95); // XXX
 5347   format %{ "movsd   $dst, $src\t# double stk" %}
 5348   ins_encode %{
 5349     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5350   %}
 5351   ins_pipe(pipe_slow); // XXX
 5352 %}
 5353 
 5354 instruct cacheWB(indirect addr)
 5355 %{
 5356   predicate(VM_Version::supports_data_cache_line_flush());
 5357   match(CacheWB addr);
 5358 
 5359   ins_cost(100);
 5360   format %{"cache wb $addr" %}
 5361   ins_encode %{
 5362     assert($addr->index_position() < 0, "should be");
 5363     assert($addr$$disp == 0, "should be");
 5364     __ cache_wb(Address($addr$$base$$Register, 0));
 5365   %}
 5366   ins_pipe(pipe_slow); // XXX
 5367 %}
 5368 
 5369 instruct cacheWBPreSync()
 5370 %{
 5371   predicate(VM_Version::supports_data_cache_line_flush());
 5372   match(CacheWBPreSync);
 5373 
 5374   ins_cost(100);
 5375   format %{"cache wb presync" %}
 5376   ins_encode %{
 5377     __ cache_wbsync(true);
 5378   %}
 5379   ins_pipe(pipe_slow); // XXX
 5380 %}
 5381 
 5382 instruct cacheWBPostSync()
 5383 %{
 5384   predicate(VM_Version::supports_data_cache_line_flush());
 5385   match(CacheWBPostSync);
 5386 
 5387   ins_cost(100);
 5388   format %{"cache wb postsync" %}
 5389   ins_encode %{
 5390     __ cache_wbsync(false);
 5391   %}
 5392   ins_pipe(pipe_slow); // XXX
 5393 %}
 5394 
 5395 //----------BSWAP Instructions-------------------------------------------------
 5396 instruct bytes_reverse_int(rRegI dst) %{
 5397   match(Set dst (ReverseBytesI dst));
 5398 
 5399   format %{ "bswapl  $dst" %}
 5400   ins_encode %{
 5401     __ bswapl($dst$$Register);
 5402   %}
 5403   ins_pipe( ialu_reg );
 5404 %}
 5405 
 5406 instruct bytes_reverse_long(rRegL dst) %{
 5407   match(Set dst (ReverseBytesL dst));
 5408 
 5409   format %{ "bswapq  $dst" %}
 5410   ins_encode %{
 5411     __ bswapq($dst$$Register);
 5412   %}
 5413   ins_pipe( ialu_reg);
 5414 %}
 5415 
 5416 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5417   match(Set dst (ReverseBytesUS dst));
 5418   effect(KILL cr);
 5419 
 5420   format %{ "bswapl  $dst\n\t"
 5421             "shrl    $dst,16\n\t" %}
 5422   ins_encode %{
 5423     __ bswapl($dst$$Register);
 5424     __ shrl($dst$$Register, 16);
 5425   %}
 5426   ins_pipe( ialu_reg );
 5427 %}
 5428 
 5429 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5430   match(Set dst (ReverseBytesS dst));
 5431   effect(KILL cr);
 5432 
 5433   format %{ "bswapl  $dst\n\t"
 5434             "sar     $dst,16\n\t" %}
 5435   ins_encode %{
 5436     __ bswapl($dst$$Register);
 5437     __ sarl($dst$$Register, 16);
 5438   %}
 5439   ins_pipe( ialu_reg );
 5440 %}
 5441 
 5442 //---------- Zeros Count Instructions ------------------------------------------
 5443 
 5444 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5445   predicate(UseCountLeadingZerosInstruction);
 5446   match(Set dst (CountLeadingZerosI src));
 5447   effect(KILL cr);
 5448 
 5449   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5450   ins_encode %{
 5451     __ lzcntl($dst$$Register, $src$$Register);
 5452   %}
 5453   ins_pipe(ialu_reg);
 5454 %}
 5455 
 5456 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5457   predicate(UseCountLeadingZerosInstruction);
 5458   match(Set dst (CountLeadingZerosI (LoadI src)));
 5459   effect(KILL cr);
 5460   ins_cost(175);
 5461   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5462   ins_encode %{
 5463     __ lzcntl($dst$$Register, $src$$Address);
 5464   %}
 5465   ins_pipe(ialu_reg_mem);
 5466 %}
 5467 
 5468 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5469   predicate(!UseCountLeadingZerosInstruction);
 5470   match(Set dst (CountLeadingZerosI src));
 5471   effect(KILL cr);
 5472 
 5473   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5474             "jnz     skip\n\t"
 5475             "movl    $dst, -1\n"
 5476       "skip:\n\t"
 5477             "negl    $dst\n\t"
 5478             "addl    $dst, 31" %}
 5479   ins_encode %{
 5480     Register Rdst = $dst$$Register;
 5481     Register Rsrc = $src$$Register;
 5482     Label skip;
 5483     __ bsrl(Rdst, Rsrc);
 5484     __ jccb(Assembler::notZero, skip);
 5485     __ movl(Rdst, -1);
 5486     __ bind(skip);
 5487     __ negl(Rdst);
 5488     __ addl(Rdst, BitsPerInt - 1);
 5489   %}
 5490   ins_pipe(ialu_reg);
 5491 %}
 5492 
 5493 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5494   predicate(UseCountLeadingZerosInstruction);
 5495   match(Set dst (CountLeadingZerosL src));
 5496   effect(KILL cr);
 5497 
 5498   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5499   ins_encode %{
 5500     __ lzcntq($dst$$Register, $src$$Register);
 5501   %}
 5502   ins_pipe(ialu_reg);
 5503 %}
 5504 
 5505 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5506   predicate(UseCountLeadingZerosInstruction);
 5507   match(Set dst (CountLeadingZerosL (LoadL src)));
 5508   effect(KILL cr);
 5509   ins_cost(175);
 5510   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5511   ins_encode %{
 5512     __ lzcntq($dst$$Register, $src$$Address);
 5513   %}
 5514   ins_pipe(ialu_reg_mem);
 5515 %}
 5516 
 5517 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5518   predicate(!UseCountLeadingZerosInstruction);
 5519   match(Set dst (CountLeadingZerosL src));
 5520   effect(KILL cr);
 5521 
 5522   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5523             "jnz     skip\n\t"
 5524             "movl    $dst, -1\n"
 5525       "skip:\n\t"
 5526             "negl    $dst\n\t"
 5527             "addl    $dst, 63" %}
 5528   ins_encode %{
 5529     Register Rdst = $dst$$Register;
 5530     Register Rsrc = $src$$Register;
 5531     Label skip;
 5532     __ bsrq(Rdst, Rsrc);
 5533     __ jccb(Assembler::notZero, skip);
 5534     __ movl(Rdst, -1);
 5535     __ bind(skip);
 5536     __ negl(Rdst);
 5537     __ addl(Rdst, BitsPerLong - 1);
 5538   %}
 5539   ins_pipe(ialu_reg);
 5540 %}
 5541 
 5542 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5543   predicate(UseCountTrailingZerosInstruction);
 5544   match(Set dst (CountTrailingZerosI src));
 5545   effect(KILL cr);
 5546 
 5547   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5548   ins_encode %{
 5549     __ tzcntl($dst$$Register, $src$$Register);
 5550   %}
 5551   ins_pipe(ialu_reg);
 5552 %}
 5553 
 5554 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5555   predicate(UseCountTrailingZerosInstruction);
 5556   match(Set dst (CountTrailingZerosI (LoadI src)));
 5557   effect(KILL cr);
 5558   ins_cost(175);
 5559   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5560   ins_encode %{
 5561     __ tzcntl($dst$$Register, $src$$Address);
 5562   %}
 5563   ins_pipe(ialu_reg_mem);
 5564 %}
 5565 
 5566 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5567   predicate(!UseCountTrailingZerosInstruction);
 5568   match(Set dst (CountTrailingZerosI src));
 5569   effect(KILL cr);
 5570 
 5571   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5572             "jnz     done\n\t"
 5573             "movl    $dst, 32\n"
 5574       "done:" %}
 5575   ins_encode %{
 5576     Register Rdst = $dst$$Register;
 5577     Label done;
 5578     __ bsfl(Rdst, $src$$Register);
 5579     __ jccb(Assembler::notZero, done);
 5580     __ movl(Rdst, BitsPerInt);
 5581     __ bind(done);
 5582   %}
 5583   ins_pipe(ialu_reg);
 5584 %}
 5585 
 5586 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5587   predicate(UseCountTrailingZerosInstruction);
 5588   match(Set dst (CountTrailingZerosL src));
 5589   effect(KILL cr);
 5590 
 5591   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5592   ins_encode %{
 5593     __ tzcntq($dst$$Register, $src$$Register);
 5594   %}
 5595   ins_pipe(ialu_reg);
 5596 %}
 5597 
 5598 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5599   predicate(UseCountTrailingZerosInstruction);
 5600   match(Set dst (CountTrailingZerosL (LoadL src)));
 5601   effect(KILL cr);
 5602   ins_cost(175);
 5603   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5604   ins_encode %{
 5605     __ tzcntq($dst$$Register, $src$$Address);
 5606   %}
 5607   ins_pipe(ialu_reg_mem);
 5608 %}
 5609 
 5610 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5611   predicate(!UseCountTrailingZerosInstruction);
 5612   match(Set dst (CountTrailingZerosL src));
 5613   effect(KILL cr);
 5614 
 5615   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5616             "jnz     done\n\t"
 5617             "movl    $dst, 64\n"
 5618       "done:" %}
 5619   ins_encode %{
 5620     Register Rdst = $dst$$Register;
 5621     Label done;
 5622     __ bsfq(Rdst, $src$$Register);
 5623     __ jccb(Assembler::notZero, done);
 5624     __ movl(Rdst, BitsPerLong);
 5625     __ bind(done);
 5626   %}
 5627   ins_pipe(ialu_reg);
 5628 %}
 5629 
 5630 //--------------- Reverse Operation Instructions ----------------
 5631 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5632   predicate(!VM_Version::supports_gfni());
 5633   match(Set dst (ReverseI src));
 5634   effect(TEMP dst, TEMP rtmp, KILL cr);
 5635   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5636   ins_encode %{
 5637     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5638   %}
 5639   ins_pipe( ialu_reg );
 5640 %}
 5641 
 5642 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5643   predicate(VM_Version::supports_gfni());
 5644   match(Set dst (ReverseI src));
 5645   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5646   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5647   ins_encode %{
 5648     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5649   %}
 5650   ins_pipe( ialu_reg );
 5651 %}
 5652 
 5653 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5654   predicate(!VM_Version::supports_gfni());
 5655   match(Set dst (ReverseL src));
 5656   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5657   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5658   ins_encode %{
 5659     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5660   %}
 5661   ins_pipe( ialu_reg );
 5662 %}
 5663 
 5664 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5665   predicate(VM_Version::supports_gfni());
 5666   match(Set dst (ReverseL src));
 5667   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5668   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5669   ins_encode %{
 5670     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5671   %}
 5672   ins_pipe( ialu_reg );
 5673 %}
 5674 
 5675 //---------- Population Count Instructions -------------------------------------
 5676 
 5677 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5678   predicate(UsePopCountInstruction);
 5679   match(Set dst (PopCountI src));
 5680   effect(KILL cr);
 5681 
 5682   format %{ "popcnt  $dst, $src" %}
 5683   ins_encode %{
 5684     __ popcntl($dst$$Register, $src$$Register);
 5685   %}
 5686   ins_pipe(ialu_reg);
 5687 %}
 5688 
 5689 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5690   predicate(UsePopCountInstruction);
 5691   match(Set dst (PopCountI (LoadI mem)));
 5692   effect(KILL cr);
 5693 
 5694   format %{ "popcnt  $dst, $mem" %}
 5695   ins_encode %{
 5696     __ popcntl($dst$$Register, $mem$$Address);
 5697   %}
 5698   ins_pipe(ialu_reg);
 5699 %}
 5700 
 5701 // Note: Long.bitCount(long) returns an int.
 5702 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5703   predicate(UsePopCountInstruction);
 5704   match(Set dst (PopCountL src));
 5705   effect(KILL cr);
 5706 
 5707   format %{ "popcnt  $dst, $src" %}
 5708   ins_encode %{
 5709     __ popcntq($dst$$Register, $src$$Register);
 5710   %}
 5711   ins_pipe(ialu_reg);
 5712 %}
 5713 
 5714 // Note: Long.bitCount(long) returns an int.
 5715 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5716   predicate(UsePopCountInstruction);
 5717   match(Set dst (PopCountL (LoadL mem)));
 5718   effect(KILL cr);
 5719 
 5720   format %{ "popcnt  $dst, $mem" %}
 5721   ins_encode %{
 5722     __ popcntq($dst$$Register, $mem$$Address);
 5723   %}
 5724   ins_pipe(ialu_reg);
 5725 %}
 5726 
 5727 
 5728 //----------MemBar Instructions-----------------------------------------------
 5729 // Memory barrier flavors
 5730 
 5731 instruct membar_acquire()
 5732 %{
 5733   match(MemBarAcquire);
 5734   match(LoadFence);
 5735   ins_cost(0);
 5736 
 5737   size(0);
 5738   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5739   ins_encode();
 5740   ins_pipe(empty);
 5741 %}
 5742 
 5743 instruct membar_acquire_lock()
 5744 %{
 5745   match(MemBarAcquireLock);
 5746   ins_cost(0);
 5747 
 5748   size(0);
 5749   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5750   ins_encode();
 5751   ins_pipe(empty);
 5752 %}
 5753 
 5754 instruct membar_release()
 5755 %{
 5756   match(MemBarRelease);
 5757   match(StoreFence);
 5758   ins_cost(0);
 5759 
 5760   size(0);
 5761   format %{ "MEMBAR-release ! (empty encoding)" %}
 5762   ins_encode();
 5763   ins_pipe(empty);
 5764 %}
 5765 
 5766 instruct membar_release_lock()
 5767 %{
 5768   match(MemBarReleaseLock);
 5769   ins_cost(0);
 5770 
 5771   size(0);
 5772   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5773   ins_encode();
 5774   ins_pipe(empty);
 5775 %}
 5776 
 5777 instruct membar_volatile(rFlagsReg cr) %{
 5778   match(MemBarVolatile);
 5779   effect(KILL cr);
 5780   ins_cost(400);
 5781 
 5782   format %{
 5783     $$template
 5784     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5785   %}
 5786   ins_encode %{
 5787     __ membar(Assembler::StoreLoad);
 5788   %}
 5789   ins_pipe(pipe_slow);
 5790 %}
 5791 
 5792 instruct unnecessary_membar_volatile()
 5793 %{
 5794   match(MemBarVolatile);
 5795   predicate(Matcher::post_store_load_barrier(n));
 5796   ins_cost(0);
 5797 
 5798   size(0);
 5799   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5800   ins_encode();
 5801   ins_pipe(empty);
 5802 %}
 5803 
 5804 instruct membar_storestore() %{
 5805   match(MemBarStoreStore);
 5806   match(StoreStoreFence);
 5807   ins_cost(0);
 5808 
 5809   size(0);
 5810   format %{ "MEMBAR-storestore (empty encoding)" %}
 5811   ins_encode( );
 5812   ins_pipe(empty);
 5813 %}
 5814 
 5815 //----------Move Instructions--------------------------------------------------
 5816 
 5817 instruct castX2P(rRegP dst, rRegL src)
 5818 %{
 5819   match(Set dst (CastX2P src));
 5820 
 5821   format %{ "movq    $dst, $src\t# long->ptr" %}
 5822   ins_encode %{
 5823     if ($dst$$reg != $src$$reg) {
 5824       __ movptr($dst$$Register, $src$$Register);
 5825     }
 5826   %}
 5827   ins_pipe(ialu_reg_reg); // XXX
 5828 %}
 5829 
 5830 instruct castP2X(rRegL dst, rRegP src)
 5831 %{
 5832   match(Set dst (CastP2X src));
 5833 
 5834   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5835   ins_encode %{
 5836     if ($dst$$reg != $src$$reg) {
 5837       __ movptr($dst$$Register, $src$$Register);
 5838     }
 5839   %}
 5840   ins_pipe(ialu_reg_reg); // XXX
 5841 %}
 5842 
 5843 // Convert oop into int for vectors alignment masking
 5844 instruct convP2I(rRegI dst, rRegP src)
 5845 %{
 5846   match(Set dst (ConvL2I (CastP2X src)));
 5847 
 5848   format %{ "movl    $dst, $src\t# ptr -> int" %}
 5849   ins_encode %{
 5850     __ movl($dst$$Register, $src$$Register);
 5851   %}
 5852   ins_pipe(ialu_reg_reg); // XXX
 5853 %}
 5854 
 5855 // Convert compressed oop into int for vectors alignment masking
 5856 // in case of 32bit oops (heap < 4Gb).
 5857 instruct convN2I(rRegI dst, rRegN src)
 5858 %{
 5859   predicate(CompressedOops::shift() == 0);
 5860   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 5861 
 5862   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 5863   ins_encode %{
 5864     __ movl($dst$$Register, $src$$Register);
 5865   %}
 5866   ins_pipe(ialu_reg_reg); // XXX
 5867 %}
 5868 
 5869 // Convert oop pointer into compressed form
 5870 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 5871   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 5872   match(Set dst (EncodeP src));
 5873   effect(KILL cr);
 5874   format %{ "encode_heap_oop $dst,$src" %}
 5875   ins_encode %{
 5876     Register s = $src$$Register;
 5877     Register d = $dst$$Register;
 5878     if (s != d) {
 5879       __ movq(d, s);
 5880     }
 5881     __ encode_heap_oop(d);
 5882   %}
 5883   ins_pipe(ialu_reg_long);
 5884 %}
 5885 
 5886 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 5887   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 5888   match(Set dst (EncodeP src));
 5889   effect(KILL cr);
 5890   format %{ "encode_heap_oop_not_null $dst,$src" %}
 5891   ins_encode %{
 5892     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 5893   %}
 5894   ins_pipe(ialu_reg_long);
 5895 %}
 5896 
 5897 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 5898   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 5899             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 5900   match(Set dst (DecodeN src));
 5901   effect(KILL cr);
 5902   format %{ "decode_heap_oop $dst,$src" %}
 5903   ins_encode %{
 5904     Register s = $src$$Register;
 5905     Register d = $dst$$Register;
 5906     if (s != d) {
 5907       __ movq(d, s);
 5908     }
 5909     __ decode_heap_oop(d);
 5910   %}
 5911   ins_pipe(ialu_reg_long);
 5912 %}
 5913 
 5914 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 5915   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 5916             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 5917   match(Set dst (DecodeN src));
 5918   effect(KILL cr);
 5919   format %{ "decode_heap_oop_not_null $dst,$src" %}
 5920   ins_encode %{
 5921     Register s = $src$$Register;
 5922     Register d = $dst$$Register;
 5923     if (s != d) {
 5924       __ decode_heap_oop_not_null(d, s);
 5925     } else {
 5926       __ decode_heap_oop_not_null(d);
 5927     }
 5928   %}
 5929   ins_pipe(ialu_reg_long);
 5930 %}
 5931 
 5932 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 5933   match(Set dst (EncodePKlass src));
 5934   effect(TEMP dst, KILL cr);
 5935   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 5936   ins_encode %{
 5937     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 5938   %}
 5939   ins_pipe(ialu_reg_long);
 5940 %}
 5941 
 5942 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 5943   match(Set dst (DecodeNKlass src));
 5944   effect(TEMP dst, KILL cr);
 5945   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 5946   ins_encode %{
 5947     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 5948   %}
 5949   ins_pipe(ialu_reg_long);
 5950 %}
 5951 
 5952 //----------Conditional Move---------------------------------------------------
 5953 // Jump
 5954 // dummy instruction for generating temp registers
 5955 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 5956   match(Jump (LShiftL switch_val shift));
 5957   ins_cost(350);
 5958   predicate(false);
 5959   effect(TEMP dest);
 5960 
 5961   format %{ "leaq    $dest, [$constantaddress]\n\t"
 5962             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 5963   ins_encode %{
 5964     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 5965     // to do that and the compiler is using that register as one it can allocate.
 5966     // So we build it all by hand.
 5967     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 5968     // ArrayAddress dispatch(table, index);
 5969     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 5970     __ lea($dest$$Register, $constantaddress);
 5971     __ jmp(dispatch);
 5972   %}
 5973   ins_pipe(pipe_jmp);
 5974 %}
 5975 
 5976 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 5977   match(Jump (AddL (LShiftL switch_val shift) offset));
 5978   ins_cost(350);
 5979   effect(TEMP dest);
 5980 
 5981   format %{ "leaq    $dest, [$constantaddress]\n\t"
 5982             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 5983   ins_encode %{
 5984     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 5985     // to do that and the compiler is using that register as one it can allocate.
 5986     // So we build it all by hand.
 5987     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 5988     // ArrayAddress dispatch(table, index);
 5989     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 5990     __ lea($dest$$Register, $constantaddress);
 5991     __ jmp(dispatch);
 5992   %}
 5993   ins_pipe(pipe_jmp);
 5994 %}
 5995 
 5996 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 5997   match(Jump switch_val);
 5998   ins_cost(350);
 5999   effect(TEMP dest);
 6000 
 6001   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6002             "jmp     [$dest + $switch_val]\n\t" %}
 6003   ins_encode %{
 6004     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6005     // to do that and the compiler is using that register as one it can allocate.
 6006     // So we build it all by hand.
 6007     // Address index(noreg, switch_reg, Address::times_1);
 6008     // ArrayAddress dispatch(table, index);
 6009     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6010     __ lea($dest$$Register, $constantaddress);
 6011     __ jmp(dispatch);
 6012   %}
 6013   ins_pipe(pipe_jmp);
 6014 %}
 6015 
 6016 // Conditional move
 6017 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6018 %{
 6019   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6020   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6021 
 6022   ins_cost(100); // XXX
 6023   format %{ "setbn$cop $dst\t# signed, int" %}
 6024   ins_encode %{
 6025     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6026     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6027   %}
 6028   ins_pipe(ialu_reg);
 6029 %}
 6030 
 6031 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6032 %{
 6033   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6034 
 6035   ins_cost(200); // XXX
 6036   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6037   ins_encode %{
 6038     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6039   %}
 6040   ins_pipe(pipe_cmov_reg);
 6041 %}
 6042 
 6043 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6044 %{
 6045   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6046   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6047 
 6048   ins_cost(100); // XXX
 6049   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6050   ins_encode %{
 6051     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6052     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6053   %}
 6054   ins_pipe(ialu_reg);
 6055 %}
 6056 
 6057 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6058   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6059 
 6060   ins_cost(200); // XXX
 6061   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6062   ins_encode %{
 6063     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6064   %}
 6065   ins_pipe(pipe_cmov_reg);
 6066 %}
 6067 
 6068 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6069 %{
 6070   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6071   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6072 
 6073   ins_cost(100); // XXX
 6074   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6075   ins_encode %{
 6076     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6077     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6078   %}
 6079   ins_pipe(ialu_reg);
 6080 %}
 6081 
 6082 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6083   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6084   ins_cost(200);
 6085   expand %{
 6086     cmovI_regU(cop, cr, dst, src);
 6087   %}
 6088 %}
 6089 
 6090 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6091   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6092   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6093 
 6094   ins_cost(200); // XXX
 6095   format %{ "cmovpl  $dst, $src\n\t"
 6096             "cmovnel $dst, $src" %}
 6097   ins_encode %{
 6098     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6099     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6100   %}
 6101   ins_pipe(pipe_cmov_reg);
 6102 %}
 6103 
 6104 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6105 // inputs of the CMove
 6106 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6107   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6108   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6109 
 6110   ins_cost(200); // XXX
 6111   format %{ "cmovpl  $dst, $src\n\t"
 6112             "cmovnel $dst, $src" %}
 6113   ins_encode %{
 6114     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6115     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6116   %}
 6117   ins_pipe(pipe_cmov_reg);
 6118 %}
 6119 
 6120 // Conditional move
 6121 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6122   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6123 
 6124   ins_cost(250); // XXX
 6125   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6126   ins_encode %{
 6127     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6128   %}
 6129   ins_pipe(pipe_cmov_mem);
 6130 %}
 6131 
 6132 // Conditional move
 6133 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6134 %{
 6135   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6136 
 6137   ins_cost(250); // XXX
 6138   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6139   ins_encode %{
 6140     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6141   %}
 6142   ins_pipe(pipe_cmov_mem);
 6143 %}
 6144 
 6145 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6146   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6147   ins_cost(250);
 6148   expand %{
 6149     cmovI_memU(cop, cr, dst, src);
 6150   %}
 6151 %}
 6152 
 6153 // Conditional move
 6154 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6155 %{
 6156   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6157 
 6158   ins_cost(200); // XXX
 6159   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6160   ins_encode %{
 6161     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6162   %}
 6163   ins_pipe(pipe_cmov_reg);
 6164 %}
 6165 
 6166 // Conditional move
 6167 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6168 %{
 6169   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6170 
 6171   ins_cost(200); // XXX
 6172   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6173   ins_encode %{
 6174     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6175   %}
 6176   ins_pipe(pipe_cmov_reg);
 6177 %}
 6178 
 6179 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6180   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6181   ins_cost(200);
 6182   expand %{
 6183     cmovN_regU(cop, cr, dst, src);
 6184   %}
 6185 %}
 6186 
 6187 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6188   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6189   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6190 
 6191   ins_cost(200); // XXX
 6192   format %{ "cmovpl  $dst, $src\n\t"
 6193             "cmovnel $dst, $src" %}
 6194   ins_encode %{
 6195     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6196     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6197   %}
 6198   ins_pipe(pipe_cmov_reg);
 6199 %}
 6200 
 6201 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6202 // inputs of the CMove
 6203 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6204   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6205   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6206 
 6207   ins_cost(200); // XXX
 6208   format %{ "cmovpl  $dst, $src\n\t"
 6209             "cmovnel $dst, $src" %}
 6210   ins_encode %{
 6211     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6212     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6213   %}
 6214   ins_pipe(pipe_cmov_reg);
 6215 %}
 6216 
 6217 // Conditional move
 6218 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6219 %{
 6220   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6221 
 6222   ins_cost(200); // XXX
 6223   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6224   ins_encode %{
 6225     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6226   %}
 6227   ins_pipe(pipe_cmov_reg);  // XXX
 6228 %}
 6229 
 6230 // Conditional move
 6231 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6232 %{
 6233   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6234 
 6235   ins_cost(200); // XXX
 6236   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6237   ins_encode %{
 6238     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6239   %}
 6240   ins_pipe(pipe_cmov_reg); // XXX
 6241 %}
 6242 
 6243 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6244   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6245   ins_cost(200);
 6246   expand %{
 6247     cmovP_regU(cop, cr, dst, src);
 6248   %}
 6249 %}
 6250 
 6251 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6252   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6253   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6254 
 6255   ins_cost(200); // XXX
 6256   format %{ "cmovpq  $dst, $src\n\t"
 6257             "cmovneq $dst, $src" %}
 6258   ins_encode %{
 6259     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6260     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6261   %}
 6262   ins_pipe(pipe_cmov_reg);
 6263 %}
 6264 
 6265 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6266 // inputs of the CMove
 6267 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6268   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6269   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6270 
 6271   ins_cost(200); // XXX
 6272   format %{ "cmovpq  $dst, $src\n\t"
 6273             "cmovneq $dst, $src" %}
 6274   ins_encode %{
 6275     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6276     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6277   %}
 6278   ins_pipe(pipe_cmov_reg);
 6279 %}
 6280 
 6281 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6282 %{
 6283   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6284   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6285 
 6286   ins_cost(100); // XXX
 6287   format %{ "setbn$cop $dst\t# signed, long" %}
 6288   ins_encode %{
 6289     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6290     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6291   %}
 6292   ins_pipe(ialu_reg);
 6293 %}
 6294 
 6295 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6296 %{
 6297   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6298 
 6299   ins_cost(200); // XXX
 6300   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6301   ins_encode %{
 6302     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6303   %}
 6304   ins_pipe(pipe_cmov_reg);  // XXX
 6305 %}
 6306 
 6307 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6308 %{
 6309   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6310 
 6311   ins_cost(200); // XXX
 6312   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6313   ins_encode %{
 6314     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6315   %}
 6316   ins_pipe(pipe_cmov_mem);  // XXX
 6317 %}
 6318 
 6319 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6320 %{
 6321   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6322   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6323 
 6324   ins_cost(100); // XXX
 6325   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6326   ins_encode %{
 6327     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6328     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6329   %}
 6330   ins_pipe(ialu_reg);
 6331 %}
 6332 
 6333 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6334 %{
 6335   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6336 
 6337   ins_cost(200); // XXX
 6338   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6339   ins_encode %{
 6340     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6341   %}
 6342   ins_pipe(pipe_cmov_reg); // XXX
 6343 %}
 6344 
 6345 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6346 %{
 6347   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6348   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6349 
 6350   ins_cost(100); // XXX
 6351   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6352   ins_encode %{
 6353     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6354     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6355   %}
 6356   ins_pipe(ialu_reg);
 6357 %}
 6358 
 6359 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6360   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6361   ins_cost(200);
 6362   expand %{
 6363     cmovL_regU(cop, cr, dst, src);
 6364   %}
 6365 %}
 6366 
 6367 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6368   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6369   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6370 
 6371   ins_cost(200); // XXX
 6372   format %{ "cmovpq  $dst, $src\n\t"
 6373             "cmovneq $dst, $src" %}
 6374   ins_encode %{
 6375     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6376     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6377   %}
 6378   ins_pipe(pipe_cmov_reg);
 6379 %}
 6380 
 6381 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6382 // inputs of the CMove
 6383 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6384   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6385   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6386 
 6387   ins_cost(200); // XXX
 6388   format %{ "cmovpq  $dst, $src\n\t"
 6389             "cmovneq $dst, $src" %}
 6390   ins_encode %{
 6391     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6392     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6393   %}
 6394   ins_pipe(pipe_cmov_reg);
 6395 %}
 6396 
 6397 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6398 %{
 6399   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6400 
 6401   ins_cost(200); // XXX
 6402   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6403   ins_encode %{
 6404     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6405   %}
 6406   ins_pipe(pipe_cmov_mem); // XXX
 6407 %}
 6408 
 6409 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6410   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6411   ins_cost(200);
 6412   expand %{
 6413     cmovL_memU(cop, cr, dst, src);
 6414   %}
 6415 %}
 6416 
 6417 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6418 %{
 6419   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6420 
 6421   ins_cost(200); // XXX
 6422   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6423             "movss     $dst, $src\n"
 6424     "skip:" %}
 6425   ins_encode %{
 6426     Label Lskip;
 6427     // Invert sense of branch from sense of CMOV
 6428     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6429     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6430     __ bind(Lskip);
 6431   %}
 6432   ins_pipe(pipe_slow);
 6433 %}
 6434 
 6435 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6436 %{
 6437   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6438 
 6439   ins_cost(200); // XXX
 6440   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6441             "movss     $dst, $src\n"
 6442     "skip:" %}
 6443   ins_encode %{
 6444     Label Lskip;
 6445     // Invert sense of branch from sense of CMOV
 6446     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6447     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6448     __ bind(Lskip);
 6449   %}
 6450   ins_pipe(pipe_slow);
 6451 %}
 6452 
 6453 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6454   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6455   ins_cost(200);
 6456   expand %{
 6457     cmovF_regU(cop, cr, dst, src);
 6458   %}
 6459 %}
 6460 
 6461 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6462 %{
 6463   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6464 
 6465   ins_cost(200); // XXX
 6466   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6467             "movsd     $dst, $src\n"
 6468     "skip:" %}
 6469   ins_encode %{
 6470     Label Lskip;
 6471     // Invert sense of branch from sense of CMOV
 6472     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6473     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6474     __ bind(Lskip);
 6475   %}
 6476   ins_pipe(pipe_slow);
 6477 %}
 6478 
 6479 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6480 %{
 6481   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6482 
 6483   ins_cost(200); // XXX
 6484   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6485             "movsd     $dst, $src\n"
 6486     "skip:" %}
 6487   ins_encode %{
 6488     Label Lskip;
 6489     // Invert sense of branch from sense of CMOV
 6490     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6491     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6492     __ bind(Lskip);
 6493   %}
 6494   ins_pipe(pipe_slow);
 6495 %}
 6496 
 6497 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6498   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6499   ins_cost(200);
 6500   expand %{
 6501     cmovD_regU(cop, cr, dst, src);
 6502   %}
 6503 %}
 6504 
 6505 //----------Arithmetic Instructions--------------------------------------------
 6506 //----------Addition Instructions----------------------------------------------
 6507 
 6508 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6509 %{
 6510   match(Set dst (AddI dst src));
 6511   effect(KILL cr);
 6512   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6513   format %{ "addl    $dst, $src\t# int" %}
 6514   ins_encode %{
 6515     __ addl($dst$$Register, $src$$Register);
 6516   %}
 6517   ins_pipe(ialu_reg_reg);
 6518 %}
 6519 
 6520 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 6521 %{
 6522   match(Set dst (AddI dst src));
 6523   effect(KILL cr);
 6524   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6525 
 6526   format %{ "addl    $dst, $src\t# int" %}
 6527   ins_encode %{
 6528     __ addl($dst$$Register, $src$$constant);
 6529   %}
 6530   ins_pipe( ialu_reg );
 6531 %}
 6532 
 6533 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 6534 %{
 6535   match(Set dst (AddI dst (LoadI src)));
 6536   effect(KILL cr);
 6537   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6538 
 6539   ins_cost(150); // XXX
 6540   format %{ "addl    $dst, $src\t# int" %}
 6541   ins_encode %{
 6542     __ addl($dst$$Register, $src$$Address);
 6543   %}
 6544   ins_pipe(ialu_reg_mem);
 6545 %}
 6546 
 6547 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 6548 %{
 6549   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6550   effect(KILL cr);
 6551   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6552 
 6553   ins_cost(150); // XXX
 6554   format %{ "addl    $dst, $src\t# int" %}
 6555   ins_encode %{
 6556     __ addl($dst$$Address, $src$$Register);
 6557   %}
 6558   ins_pipe(ialu_mem_reg);
 6559 %}
 6560 
 6561 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 6562 %{
 6563   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6564   effect(KILL cr);
 6565   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6566 
 6567 
 6568   ins_cost(125); // XXX
 6569   format %{ "addl    $dst, $src\t# int" %}
 6570   ins_encode %{
 6571     __ addl($dst$$Address, $src$$constant);
 6572   %}
 6573   ins_pipe(ialu_mem_imm);
 6574 %}
 6575 
 6576 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 6577 %{
 6578   predicate(UseIncDec);
 6579   match(Set dst (AddI dst src));
 6580   effect(KILL cr);
 6581 
 6582   format %{ "incl    $dst\t# int" %}
 6583   ins_encode %{
 6584     __ incrementl($dst$$Register);
 6585   %}
 6586   ins_pipe(ialu_reg);
 6587 %}
 6588 
 6589 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 6590 %{
 6591   predicate(UseIncDec);
 6592   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6593   effect(KILL cr);
 6594 
 6595   ins_cost(125); // XXX
 6596   format %{ "incl    $dst\t# int" %}
 6597   ins_encode %{
 6598     __ incrementl($dst$$Address);
 6599   %}
 6600   ins_pipe(ialu_mem_imm);
 6601 %}
 6602 
 6603 // XXX why does that use AddI
 6604 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 6605 %{
 6606   predicate(UseIncDec);
 6607   match(Set dst (AddI dst src));
 6608   effect(KILL cr);
 6609 
 6610   format %{ "decl    $dst\t# int" %}
 6611   ins_encode %{
 6612     __ decrementl($dst$$Register);
 6613   %}
 6614   ins_pipe(ialu_reg);
 6615 %}
 6616 
 6617 // XXX why does that use AddI
 6618 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 6619 %{
 6620   predicate(UseIncDec);
 6621   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6622   effect(KILL cr);
 6623 
 6624   ins_cost(125); // XXX
 6625   format %{ "decl    $dst\t# int" %}
 6626   ins_encode %{
 6627     __ decrementl($dst$$Address);
 6628   %}
 6629   ins_pipe(ialu_mem_imm);
 6630 %}
 6631 
 6632 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 6633 %{
 6634   predicate(VM_Version::supports_fast_2op_lea());
 6635   match(Set dst (AddI (LShiftI index scale) disp));
 6636 
 6637   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 6638   ins_encode %{
 6639     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6640     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6641   %}
 6642   ins_pipe(ialu_reg_reg);
 6643 %}
 6644 
 6645 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 6646 %{
 6647   predicate(VM_Version::supports_fast_3op_lea());
 6648   match(Set dst (AddI (AddI base index) disp));
 6649 
 6650   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 6651   ins_encode %{
 6652     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6653   %}
 6654   ins_pipe(ialu_reg_reg);
 6655 %}
 6656 
 6657 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 6658 %{
 6659   predicate(VM_Version::supports_fast_2op_lea());
 6660   match(Set dst (AddI base (LShiftI index scale)));
 6661 
 6662   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 6663   ins_encode %{
 6664     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6665     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6666   %}
 6667   ins_pipe(ialu_reg_reg);
 6668 %}
 6669 
 6670 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 6671 %{
 6672   predicate(VM_Version::supports_fast_3op_lea());
 6673   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 6674 
 6675   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 6676   ins_encode %{
 6677     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6678     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6679   %}
 6680   ins_pipe(ialu_reg_reg);
 6681 %}
 6682 
 6683 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 6684 %{
 6685   match(Set dst (AddL dst src));
 6686   effect(KILL cr);
 6687   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6688 
 6689   format %{ "addq    $dst, $src\t# long" %}
 6690   ins_encode %{
 6691     __ addq($dst$$Register, $src$$Register);
 6692   %}
 6693   ins_pipe(ialu_reg_reg);
 6694 %}
 6695 
 6696 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 6697 %{
 6698   match(Set dst (AddL dst src));
 6699   effect(KILL cr);
 6700   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6701 
 6702   format %{ "addq    $dst, $src\t# long" %}
 6703   ins_encode %{
 6704     __ addq($dst$$Register, $src$$constant);
 6705   %}
 6706   ins_pipe( ialu_reg );
 6707 %}
 6708 
 6709 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 6710 %{
 6711   match(Set dst (AddL dst (LoadL src)));
 6712   effect(KILL cr);
 6713   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6714 
 6715   ins_cost(150); // XXX
 6716   format %{ "addq    $dst, $src\t# long" %}
 6717   ins_encode %{
 6718     __ addq($dst$$Register, $src$$Address);
 6719   %}
 6720   ins_pipe(ialu_reg_mem);
 6721 %}
 6722 
 6723 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 6724 %{
 6725   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6726   effect(KILL cr);
 6727   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6728 
 6729   ins_cost(150); // XXX
 6730   format %{ "addq    $dst, $src\t# long" %}
 6731   ins_encode %{
 6732     __ addq($dst$$Address, $src$$Register);
 6733   %}
 6734   ins_pipe(ialu_mem_reg);
 6735 %}
 6736 
 6737 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 6738 %{
 6739   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6740   effect(KILL cr);
 6741   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6742 
 6743   ins_cost(125); // XXX
 6744   format %{ "addq    $dst, $src\t# long" %}
 6745   ins_encode %{
 6746     __ addq($dst$$Address, $src$$constant);
 6747   %}
 6748   ins_pipe(ialu_mem_imm);
 6749 %}
 6750 
 6751 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 6752 %{
 6753   predicate(UseIncDec);
 6754   match(Set dst (AddL dst src));
 6755   effect(KILL cr);
 6756 
 6757   format %{ "incq    $dst\t# long" %}
 6758   ins_encode %{
 6759     __ incrementq($dst$$Register);
 6760   %}
 6761   ins_pipe(ialu_reg);
 6762 %}
 6763 
 6764 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 6765 %{
 6766   predicate(UseIncDec);
 6767   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6768   effect(KILL cr);
 6769 
 6770   ins_cost(125); // XXX
 6771   format %{ "incq    $dst\t# long" %}
 6772   ins_encode %{
 6773     __ incrementq($dst$$Address);
 6774   %}
 6775   ins_pipe(ialu_mem_imm);
 6776 %}
 6777 
 6778 // XXX why does that use AddL
 6779 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 6780 %{
 6781   predicate(UseIncDec);
 6782   match(Set dst (AddL dst src));
 6783   effect(KILL cr);
 6784 
 6785   format %{ "decq    $dst\t# long" %}
 6786   ins_encode %{
 6787     __ decrementq($dst$$Register);
 6788   %}
 6789   ins_pipe(ialu_reg);
 6790 %}
 6791 
 6792 // XXX why does that use AddL
 6793 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 6794 %{
 6795   predicate(UseIncDec);
 6796   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6797   effect(KILL cr);
 6798 
 6799   ins_cost(125); // XXX
 6800   format %{ "decq    $dst\t# long" %}
 6801   ins_encode %{
 6802     __ decrementq($dst$$Address);
 6803   %}
 6804   ins_pipe(ialu_mem_imm);
 6805 %}
 6806 
 6807 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 6808 %{
 6809   predicate(VM_Version::supports_fast_2op_lea());
 6810   match(Set dst (AddL (LShiftL index scale) disp));
 6811 
 6812   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 6813   ins_encode %{
 6814     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6815     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6816   %}
 6817   ins_pipe(ialu_reg_reg);
 6818 %}
 6819 
 6820 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 6821 %{
 6822   predicate(VM_Version::supports_fast_3op_lea());
 6823   match(Set dst (AddL (AddL base index) disp));
 6824 
 6825   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 6826   ins_encode %{
 6827     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6828   %}
 6829   ins_pipe(ialu_reg_reg);
 6830 %}
 6831 
 6832 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 6833 %{
 6834   predicate(VM_Version::supports_fast_2op_lea());
 6835   match(Set dst (AddL base (LShiftL index scale)));
 6836 
 6837   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 6838   ins_encode %{
 6839     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6840     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6841   %}
 6842   ins_pipe(ialu_reg_reg);
 6843 %}
 6844 
 6845 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 6846 %{
 6847   predicate(VM_Version::supports_fast_3op_lea());
 6848   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 6849 
 6850   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 6851   ins_encode %{
 6852     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6853     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6854   %}
 6855   ins_pipe(ialu_reg_reg);
 6856 %}
 6857 
 6858 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 6859 %{
 6860   match(Set dst (AddP dst src));
 6861   effect(KILL cr);
 6862   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6863 
 6864   format %{ "addq    $dst, $src\t# ptr" %}
 6865   ins_encode %{
 6866     __ addq($dst$$Register, $src$$Register);
 6867   %}
 6868   ins_pipe(ialu_reg_reg);
 6869 %}
 6870 
 6871 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 6872 %{
 6873   match(Set dst (AddP dst src));
 6874   effect(KILL cr);
 6875   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6876 
 6877   format %{ "addq    $dst, $src\t# ptr" %}
 6878   ins_encode %{
 6879     __ addq($dst$$Register, $src$$constant);
 6880   %}
 6881   ins_pipe( ialu_reg );
 6882 %}
 6883 
 6884 // XXX addP mem ops ????
 6885 
 6886 instruct checkCastPP(rRegP dst)
 6887 %{
 6888   match(Set dst (CheckCastPP dst));
 6889 
 6890   size(0);
 6891   format %{ "# checkcastPP of $dst" %}
 6892   ins_encode(/* empty encoding */);
 6893   ins_pipe(empty);
 6894 %}
 6895 
 6896 instruct castPP(rRegP dst)
 6897 %{
 6898   match(Set dst (CastPP dst));
 6899 
 6900   size(0);
 6901   format %{ "# castPP of $dst" %}
 6902   ins_encode(/* empty encoding */);
 6903   ins_pipe(empty);
 6904 %}
 6905 
 6906 instruct castII(rRegI dst)
 6907 %{
 6908   match(Set dst (CastII dst));
 6909 
 6910   size(0);
 6911   format %{ "# castII of $dst" %}
 6912   ins_encode(/* empty encoding */);
 6913   ins_cost(0);
 6914   ins_pipe(empty);
 6915 %}
 6916 
 6917 instruct castLL(rRegL dst)
 6918 %{
 6919   match(Set dst (CastLL dst));
 6920 
 6921   size(0);
 6922   format %{ "# castLL of $dst" %}
 6923   ins_encode(/* empty encoding */);
 6924   ins_cost(0);
 6925   ins_pipe(empty);
 6926 %}
 6927 
 6928 instruct castFF(regF dst)
 6929 %{
 6930   match(Set dst (CastFF dst));
 6931 
 6932   size(0);
 6933   format %{ "# castFF of $dst" %}
 6934   ins_encode(/* empty encoding */);
 6935   ins_cost(0);
 6936   ins_pipe(empty);
 6937 %}
 6938 
 6939 instruct castDD(regD dst)
 6940 %{
 6941   match(Set dst (CastDD dst));
 6942 
 6943   size(0);
 6944   format %{ "# castDD of $dst" %}
 6945   ins_encode(/* empty encoding */);
 6946   ins_cost(0);
 6947   ins_pipe(empty);
 6948 %}
 6949 
 6950 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 6951 instruct compareAndSwapP(rRegI res,
 6952                          memory mem_ptr,
 6953                          rax_RegP oldval, rRegP newval,
 6954                          rFlagsReg cr)
 6955 %{
 6956   predicate(n->as_LoadStore()->barrier_data() == 0);
 6957   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 6958   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 6959   effect(KILL cr, KILL oldval);
 6960 
 6961   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 6962             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 6963             "sete    $res\n\t"
 6964             "movzbl  $res, $res" %}
 6965   ins_encode %{
 6966     __ lock();
 6967     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 6968     __ setb(Assembler::equal, $res$$Register);
 6969     __ movzbl($res$$Register, $res$$Register);
 6970   %}
 6971   ins_pipe( pipe_cmpxchg );
 6972 %}
 6973 
 6974 instruct compareAndSwapL(rRegI res,
 6975                          memory mem_ptr,
 6976                          rax_RegL oldval, rRegL newval,
 6977                          rFlagsReg cr)
 6978 %{
 6979   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 6980   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 6981   effect(KILL cr, KILL oldval);
 6982 
 6983   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 6984             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 6985             "sete    $res\n\t"
 6986             "movzbl  $res, $res" %}
 6987   ins_encode %{
 6988     __ lock();
 6989     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 6990     __ setb(Assembler::equal, $res$$Register);
 6991     __ movzbl($res$$Register, $res$$Register);
 6992   %}
 6993   ins_pipe( pipe_cmpxchg );
 6994 %}
 6995 
 6996 instruct compareAndSwapI(rRegI res,
 6997                          memory mem_ptr,
 6998                          rax_RegI oldval, rRegI newval,
 6999                          rFlagsReg cr)
 7000 %{
 7001   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7002   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7003   effect(KILL cr, KILL oldval);
 7004 
 7005   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7006             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7007             "sete    $res\n\t"
 7008             "movzbl  $res, $res" %}
 7009   ins_encode %{
 7010     __ lock();
 7011     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7012     __ setb(Assembler::equal, $res$$Register);
 7013     __ movzbl($res$$Register, $res$$Register);
 7014   %}
 7015   ins_pipe( pipe_cmpxchg );
 7016 %}
 7017 
 7018 instruct compareAndSwapB(rRegI res,
 7019                          memory mem_ptr,
 7020                          rax_RegI oldval, rRegI newval,
 7021                          rFlagsReg cr)
 7022 %{
 7023   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7024   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7025   effect(KILL cr, KILL oldval);
 7026 
 7027   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7028             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7029             "sete    $res\n\t"
 7030             "movzbl  $res, $res" %}
 7031   ins_encode %{
 7032     __ lock();
 7033     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7034     __ setb(Assembler::equal, $res$$Register);
 7035     __ movzbl($res$$Register, $res$$Register);
 7036   %}
 7037   ins_pipe( pipe_cmpxchg );
 7038 %}
 7039 
 7040 instruct compareAndSwapS(rRegI res,
 7041                          memory mem_ptr,
 7042                          rax_RegI oldval, rRegI newval,
 7043                          rFlagsReg cr)
 7044 %{
 7045   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7046   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7047   effect(KILL cr, KILL oldval);
 7048 
 7049   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7050             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7051             "sete    $res\n\t"
 7052             "movzbl  $res, $res" %}
 7053   ins_encode %{
 7054     __ lock();
 7055     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7056     __ setb(Assembler::equal, $res$$Register);
 7057     __ movzbl($res$$Register, $res$$Register);
 7058   %}
 7059   ins_pipe( pipe_cmpxchg );
 7060 %}
 7061 
 7062 instruct compareAndSwapN(rRegI res,
 7063                           memory mem_ptr,
 7064                           rax_RegN oldval, rRegN newval,
 7065                           rFlagsReg cr) %{
 7066   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7067   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7068   effect(KILL cr, KILL oldval);
 7069 
 7070   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7071             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7072             "sete    $res\n\t"
 7073             "movzbl  $res, $res" %}
 7074   ins_encode %{
 7075     __ lock();
 7076     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7077     __ setb(Assembler::equal, $res$$Register);
 7078     __ movzbl($res$$Register, $res$$Register);
 7079   %}
 7080   ins_pipe( pipe_cmpxchg );
 7081 %}
 7082 
 7083 instruct compareAndExchangeB(
 7084                          memory mem_ptr,
 7085                          rax_RegI oldval, rRegI newval,
 7086                          rFlagsReg cr)
 7087 %{
 7088   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7089   effect(KILL cr);
 7090 
 7091   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7092             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7093   ins_encode %{
 7094     __ lock();
 7095     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7096   %}
 7097   ins_pipe( pipe_cmpxchg );
 7098 %}
 7099 
 7100 instruct compareAndExchangeS(
 7101                          memory mem_ptr,
 7102                          rax_RegI oldval, rRegI newval,
 7103                          rFlagsReg cr)
 7104 %{
 7105   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7106   effect(KILL cr);
 7107 
 7108   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7109             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7110   ins_encode %{
 7111     __ lock();
 7112     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7113   %}
 7114   ins_pipe( pipe_cmpxchg );
 7115 %}
 7116 
 7117 instruct compareAndExchangeI(
 7118                          memory mem_ptr,
 7119                          rax_RegI oldval, rRegI newval,
 7120                          rFlagsReg cr)
 7121 %{
 7122   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7123   effect(KILL cr);
 7124 
 7125   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7126             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7127   ins_encode %{
 7128     __ lock();
 7129     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7130   %}
 7131   ins_pipe( pipe_cmpxchg );
 7132 %}
 7133 
 7134 instruct compareAndExchangeL(
 7135                          memory mem_ptr,
 7136                          rax_RegL oldval, rRegL newval,
 7137                          rFlagsReg cr)
 7138 %{
 7139   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7140   effect(KILL cr);
 7141 
 7142   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7143             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7144   ins_encode %{
 7145     __ lock();
 7146     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7147   %}
 7148   ins_pipe( pipe_cmpxchg );
 7149 %}
 7150 
 7151 instruct compareAndExchangeN(
 7152                           memory mem_ptr,
 7153                           rax_RegN oldval, rRegN newval,
 7154                           rFlagsReg cr) %{
 7155   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7156   effect(KILL cr);
 7157 
 7158   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7159             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7160   ins_encode %{
 7161     __ lock();
 7162     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7163   %}
 7164   ins_pipe( pipe_cmpxchg );
 7165 %}
 7166 
 7167 instruct compareAndExchangeP(
 7168                          memory mem_ptr,
 7169                          rax_RegP oldval, rRegP newval,
 7170                          rFlagsReg cr)
 7171 %{
 7172   predicate(n->as_LoadStore()->barrier_data() == 0);
 7173   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7174   effect(KILL cr);
 7175 
 7176   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7177             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7178   ins_encode %{
 7179     __ lock();
 7180     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7181   %}
 7182   ins_pipe( pipe_cmpxchg );
 7183 %}
 7184 
 7185 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7186   predicate(n->as_LoadStore()->result_not_used());
 7187   match(Set dummy (GetAndAddB mem add));
 7188   effect(KILL cr);
 7189   format %{ "addb_lock   $mem, $add" %}
 7190   ins_encode %{
 7191     __ lock();
 7192     __ addb($mem$$Address, $add$$Register);
 7193   %}
 7194   ins_pipe(pipe_cmpxchg);
 7195 %}
 7196 
 7197 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7198   predicate(n->as_LoadStore()->result_not_used());
 7199   match(Set dummy (GetAndAddB mem add));
 7200   effect(KILL cr);
 7201   format %{ "addb_lock   $mem, $add" %}
 7202   ins_encode %{
 7203     __ lock();
 7204     __ addb($mem$$Address, $add$$constant);
 7205   %}
 7206   ins_pipe(pipe_cmpxchg);
 7207 %}
 7208 
 7209 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7210   predicate(!n->as_LoadStore()->result_not_used());
 7211   match(Set newval (GetAndAddB mem newval));
 7212   effect(KILL cr);
 7213   format %{ "xaddb_lock  $mem, $newval" %}
 7214   ins_encode %{
 7215     __ lock();
 7216     __ xaddb($mem$$Address, $newval$$Register);
 7217   %}
 7218   ins_pipe(pipe_cmpxchg);
 7219 %}
 7220 
 7221 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7222   predicate(n->as_LoadStore()->result_not_used());
 7223   match(Set dummy (GetAndAddS mem add));
 7224   effect(KILL cr);
 7225   format %{ "addw_lock   $mem, $add" %}
 7226   ins_encode %{
 7227     __ lock();
 7228     __ addw($mem$$Address, $add$$Register);
 7229   %}
 7230   ins_pipe(pipe_cmpxchg);
 7231 %}
 7232 
 7233 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7234   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7235   match(Set dummy (GetAndAddS mem add));
 7236   effect(KILL cr);
 7237   format %{ "addw_lock   $mem, $add" %}
 7238   ins_encode %{
 7239     __ lock();
 7240     __ addw($mem$$Address, $add$$constant);
 7241   %}
 7242   ins_pipe(pipe_cmpxchg);
 7243 %}
 7244 
 7245 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7246   predicate(!n->as_LoadStore()->result_not_used());
 7247   match(Set newval (GetAndAddS mem newval));
 7248   effect(KILL cr);
 7249   format %{ "xaddw_lock  $mem, $newval" %}
 7250   ins_encode %{
 7251     __ lock();
 7252     __ xaddw($mem$$Address, $newval$$Register);
 7253   %}
 7254   ins_pipe(pipe_cmpxchg);
 7255 %}
 7256 
 7257 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7258   predicate(n->as_LoadStore()->result_not_used());
 7259   match(Set dummy (GetAndAddI mem add));
 7260   effect(KILL cr);
 7261   format %{ "addl_lock   $mem, $add" %}
 7262   ins_encode %{
 7263     __ lock();
 7264     __ addl($mem$$Address, $add$$Register);
 7265   %}
 7266   ins_pipe(pipe_cmpxchg);
 7267 %}
 7268 
 7269 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7270   predicate(n->as_LoadStore()->result_not_used());
 7271   match(Set dummy (GetAndAddI mem add));
 7272   effect(KILL cr);
 7273   format %{ "addl_lock   $mem, $add" %}
 7274   ins_encode %{
 7275     __ lock();
 7276     __ addl($mem$$Address, $add$$constant);
 7277   %}
 7278   ins_pipe(pipe_cmpxchg);
 7279 %}
 7280 
 7281 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 7282   predicate(!n->as_LoadStore()->result_not_used());
 7283   match(Set newval (GetAndAddI mem newval));
 7284   effect(KILL cr);
 7285   format %{ "xaddl_lock  $mem, $newval" %}
 7286   ins_encode %{
 7287     __ lock();
 7288     __ xaddl($mem$$Address, $newval$$Register);
 7289   %}
 7290   ins_pipe(pipe_cmpxchg);
 7291 %}
 7292 
 7293 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 7294   predicate(n->as_LoadStore()->result_not_used());
 7295   match(Set dummy (GetAndAddL mem add));
 7296   effect(KILL cr);
 7297   format %{ "addq_lock   $mem, $add" %}
 7298   ins_encode %{
 7299     __ lock();
 7300     __ addq($mem$$Address, $add$$Register);
 7301   %}
 7302   ins_pipe(pipe_cmpxchg);
 7303 %}
 7304 
 7305 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 7306   predicate(n->as_LoadStore()->result_not_used());
 7307   match(Set dummy (GetAndAddL mem add));
 7308   effect(KILL cr);
 7309   format %{ "addq_lock   $mem, $add" %}
 7310   ins_encode %{
 7311     __ lock();
 7312     __ addq($mem$$Address, $add$$constant);
 7313   %}
 7314   ins_pipe(pipe_cmpxchg);
 7315 %}
 7316 
 7317 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 7318   predicate(!n->as_LoadStore()->result_not_used());
 7319   match(Set newval (GetAndAddL mem newval));
 7320   effect(KILL cr);
 7321   format %{ "xaddq_lock  $mem, $newval" %}
 7322   ins_encode %{
 7323     __ lock();
 7324     __ xaddq($mem$$Address, $newval$$Register);
 7325   %}
 7326   ins_pipe(pipe_cmpxchg);
 7327 %}
 7328 
 7329 instruct xchgB( memory mem, rRegI newval) %{
 7330   match(Set newval (GetAndSetB mem newval));
 7331   format %{ "XCHGB  $newval,[$mem]" %}
 7332   ins_encode %{
 7333     __ xchgb($newval$$Register, $mem$$Address);
 7334   %}
 7335   ins_pipe( pipe_cmpxchg );
 7336 %}
 7337 
 7338 instruct xchgS( memory mem, rRegI newval) %{
 7339   match(Set newval (GetAndSetS mem newval));
 7340   format %{ "XCHGW  $newval,[$mem]" %}
 7341   ins_encode %{
 7342     __ xchgw($newval$$Register, $mem$$Address);
 7343   %}
 7344   ins_pipe( pipe_cmpxchg );
 7345 %}
 7346 
 7347 instruct xchgI( memory mem, rRegI newval) %{
 7348   match(Set newval (GetAndSetI mem newval));
 7349   format %{ "XCHGL  $newval,[$mem]" %}
 7350   ins_encode %{
 7351     __ xchgl($newval$$Register, $mem$$Address);
 7352   %}
 7353   ins_pipe( pipe_cmpxchg );
 7354 %}
 7355 
 7356 instruct xchgL( memory mem, rRegL newval) %{
 7357   match(Set newval (GetAndSetL mem newval));
 7358   format %{ "XCHGL  $newval,[$mem]" %}
 7359   ins_encode %{
 7360     __ xchgq($newval$$Register, $mem$$Address);
 7361   %}
 7362   ins_pipe( pipe_cmpxchg );
 7363 %}
 7364 
 7365 instruct xchgP( memory mem, rRegP newval) %{
 7366   match(Set newval (GetAndSetP mem newval));
 7367   predicate(n->as_LoadStore()->barrier_data() == 0);
 7368   format %{ "XCHGQ  $newval,[$mem]" %}
 7369   ins_encode %{
 7370     __ xchgq($newval$$Register, $mem$$Address);
 7371   %}
 7372   ins_pipe( pipe_cmpxchg );
 7373 %}
 7374 
 7375 instruct xchgN( memory mem, rRegN newval) %{
 7376   match(Set newval (GetAndSetN mem newval));
 7377   format %{ "XCHGL  $newval,$mem]" %}
 7378   ins_encode %{
 7379     __ xchgl($newval$$Register, $mem$$Address);
 7380   %}
 7381   ins_pipe( pipe_cmpxchg );
 7382 %}
 7383 
 7384 //----------Abs Instructions-------------------------------------------
 7385 
 7386 // Integer Absolute Instructions
 7387 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7388 %{
 7389   match(Set dst (AbsI src));
 7390   effect(TEMP dst, KILL cr);
 7391   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 7392             "subl    $dst, $src\n\t"
 7393             "cmovll  $dst, $src" %}
 7394   ins_encode %{
 7395     __ xorl($dst$$Register, $dst$$Register);
 7396     __ subl($dst$$Register, $src$$Register);
 7397     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 7398   %}
 7399 
 7400   ins_pipe(ialu_reg_reg);
 7401 %}
 7402 
 7403 // Long Absolute Instructions
 7404 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7405 %{
 7406   match(Set dst (AbsL src));
 7407   effect(TEMP dst, KILL cr);
 7408   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 7409             "subq    $dst, $src\n\t"
 7410             "cmovlq  $dst, $src" %}
 7411   ins_encode %{
 7412     __ xorl($dst$$Register, $dst$$Register);
 7413     __ subq($dst$$Register, $src$$Register);
 7414     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 7415   %}
 7416 
 7417   ins_pipe(ialu_reg_reg);
 7418 %}
 7419 
 7420 //----------Subtraction Instructions-------------------------------------------
 7421 
 7422 // Integer Subtraction Instructions
 7423 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7424 %{
 7425   match(Set dst (SubI dst src));
 7426   effect(KILL cr);
 7427   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7428 
 7429   format %{ "subl    $dst, $src\t# int" %}
 7430   ins_encode %{
 7431     __ subl($dst$$Register, $src$$Register);
 7432   %}
 7433   ins_pipe(ialu_reg_reg);
 7434 %}
 7435 
 7436 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7437 %{
 7438   match(Set dst (SubI dst (LoadI src)));
 7439   effect(KILL cr);
 7440   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7441 
 7442   ins_cost(150);
 7443   format %{ "subl    $dst, $src\t# int" %}
 7444   ins_encode %{
 7445     __ subl($dst$$Register, $src$$Address);
 7446   %}
 7447   ins_pipe(ialu_reg_mem);
 7448 %}
 7449 
 7450 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7451 %{
 7452   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 7453   effect(KILL cr);
 7454   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7455 
 7456   ins_cost(150);
 7457   format %{ "subl    $dst, $src\t# int" %}
 7458   ins_encode %{
 7459     __ subl($dst$$Address, $src$$Register);
 7460   %}
 7461   ins_pipe(ialu_mem_reg);
 7462 %}
 7463 
 7464 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7465 %{
 7466   match(Set dst (SubL dst src));
 7467   effect(KILL cr);
 7468   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7469 
 7470   format %{ "subq    $dst, $src\t# long" %}
 7471   ins_encode %{
 7472     __ subq($dst$$Register, $src$$Register);
 7473   %}
 7474   ins_pipe(ialu_reg_reg);
 7475 %}
 7476 
 7477 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7478 %{
 7479   match(Set dst (SubL dst (LoadL src)));
 7480   effect(KILL cr);
 7481   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7482 
 7483   ins_cost(150);
 7484   format %{ "subq    $dst, $src\t# long" %}
 7485   ins_encode %{
 7486     __ subq($dst$$Register, $src$$Address);
 7487   %}
 7488   ins_pipe(ialu_reg_mem);
 7489 %}
 7490 
 7491 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7492 %{
 7493   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 7494   effect(KILL cr);
 7495   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7496 
 7497   ins_cost(150);
 7498   format %{ "subq    $dst, $src\t# long" %}
 7499   ins_encode %{
 7500     __ subq($dst$$Address, $src$$Register);
 7501   %}
 7502   ins_pipe(ialu_mem_reg);
 7503 %}
 7504 
 7505 // Subtract from a pointer
 7506 // XXX hmpf???
 7507 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 7508 %{
 7509   match(Set dst (AddP dst (SubI zero src)));
 7510   effect(KILL cr);
 7511 
 7512   format %{ "subq    $dst, $src\t# ptr - int" %}
 7513   ins_encode %{
 7514     __ subq($dst$$Register, $src$$Register);
 7515   %}
 7516   ins_pipe(ialu_reg_reg);
 7517 %}
 7518 
 7519 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 7520 %{
 7521   match(Set dst (SubI zero dst));
 7522   effect(KILL cr);
 7523   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7524 
 7525   format %{ "negl    $dst\t# int" %}
 7526   ins_encode %{
 7527     __ negl($dst$$Register);
 7528   %}
 7529   ins_pipe(ialu_reg);
 7530 %}
 7531 
 7532 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 7533 %{
 7534   match(Set dst (NegI dst));
 7535   effect(KILL cr);
 7536   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7537 
 7538   format %{ "negl    $dst\t# int" %}
 7539   ins_encode %{
 7540     __ negl($dst$$Register);
 7541   %}
 7542   ins_pipe(ialu_reg);
 7543 %}
 7544 
 7545 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 7546 %{
 7547   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 7548   effect(KILL cr);
 7549   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7550 
 7551   format %{ "negl    $dst\t# int" %}
 7552   ins_encode %{
 7553     __ negl($dst$$Address);
 7554   %}
 7555   ins_pipe(ialu_reg);
 7556 %}
 7557 
 7558 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 7559 %{
 7560   match(Set dst (SubL zero dst));
 7561   effect(KILL cr);
 7562   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7563 
 7564   format %{ "negq    $dst\t# long" %}
 7565   ins_encode %{
 7566     __ negq($dst$$Register);
 7567   %}
 7568   ins_pipe(ialu_reg);
 7569 %}
 7570 
 7571 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 7572 %{
 7573   match(Set dst (NegL dst));
 7574   effect(KILL cr);
 7575   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7576 
 7577   format %{ "negq    $dst\t# int" %}
 7578   ins_encode %{
 7579     __ negq($dst$$Register);
 7580   %}
 7581   ins_pipe(ialu_reg);
 7582 %}
 7583 
 7584 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 7585 %{
 7586   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 7587   effect(KILL cr);
 7588   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7589 
 7590   format %{ "negq    $dst\t# long" %}
 7591   ins_encode %{
 7592     __ negq($dst$$Address);
 7593   %}
 7594   ins_pipe(ialu_reg);
 7595 %}
 7596 
 7597 //----------Multiplication/Division Instructions-------------------------------
 7598 // Integer Multiplication Instructions
 7599 // Multiply Register
 7600 
 7601 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7602 %{
 7603   match(Set dst (MulI dst src));
 7604   effect(KILL cr);
 7605 
 7606   ins_cost(300);
 7607   format %{ "imull   $dst, $src\t# int" %}
 7608   ins_encode %{
 7609     __ imull($dst$$Register, $src$$Register);
 7610   %}
 7611   ins_pipe(ialu_reg_reg_alu0);
 7612 %}
 7613 
 7614 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 7615 %{
 7616   match(Set dst (MulI src imm));
 7617   effect(KILL cr);
 7618 
 7619   ins_cost(300);
 7620   format %{ "imull   $dst, $src, $imm\t# int" %}
 7621   ins_encode %{
 7622     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 7623   %}
 7624   ins_pipe(ialu_reg_reg_alu0);
 7625 %}
 7626 
 7627 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 7628 %{
 7629   match(Set dst (MulI dst (LoadI src)));
 7630   effect(KILL cr);
 7631 
 7632   ins_cost(350);
 7633   format %{ "imull   $dst, $src\t# int" %}
 7634   ins_encode %{
 7635     __ imull($dst$$Register, $src$$Address);
 7636   %}
 7637   ins_pipe(ialu_reg_mem_alu0);
 7638 %}
 7639 
 7640 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 7641 %{
 7642   match(Set dst (MulI (LoadI src) imm));
 7643   effect(KILL cr);
 7644 
 7645   ins_cost(300);
 7646   format %{ "imull   $dst, $src, $imm\t# int" %}
 7647   ins_encode %{
 7648     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 7649   %}
 7650   ins_pipe(ialu_reg_mem_alu0);
 7651 %}
 7652 
 7653 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 7654 %{
 7655   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 7656   effect(KILL cr, KILL src2);
 7657 
 7658   expand %{ mulI_rReg(dst, src1, cr);
 7659            mulI_rReg(src2, src3, cr);
 7660            addI_rReg(dst, src2, cr); %}
 7661 %}
 7662 
 7663 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7664 %{
 7665   match(Set dst (MulL dst src));
 7666   effect(KILL cr);
 7667 
 7668   ins_cost(300);
 7669   format %{ "imulq   $dst, $src\t# long" %}
 7670   ins_encode %{
 7671     __ imulq($dst$$Register, $src$$Register);
 7672   %}
 7673   ins_pipe(ialu_reg_reg_alu0);
 7674 %}
 7675 
 7676 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 7677 %{
 7678   match(Set dst (MulL src imm));
 7679   effect(KILL cr);
 7680 
 7681   ins_cost(300);
 7682   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7683   ins_encode %{
 7684     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 7685   %}
 7686   ins_pipe(ialu_reg_reg_alu0);
 7687 %}
 7688 
 7689 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 7690 %{
 7691   match(Set dst (MulL dst (LoadL src)));
 7692   effect(KILL cr);
 7693 
 7694   ins_cost(350);
 7695   format %{ "imulq   $dst, $src\t# long" %}
 7696   ins_encode %{
 7697     __ imulq($dst$$Register, $src$$Address);
 7698   %}
 7699   ins_pipe(ialu_reg_mem_alu0);
 7700 %}
 7701 
 7702 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 7703 %{
 7704   match(Set dst (MulL (LoadL src) imm));
 7705   effect(KILL cr);
 7706 
 7707   ins_cost(300);
 7708   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7709   ins_encode %{
 7710     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 7711   %}
 7712   ins_pipe(ialu_reg_mem_alu0);
 7713 %}
 7714 
 7715 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7716 %{
 7717   match(Set dst (MulHiL src rax));
 7718   effect(USE_KILL rax, KILL cr);
 7719 
 7720   ins_cost(300);
 7721   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 7722   ins_encode %{
 7723     __ imulq($src$$Register);
 7724   %}
 7725   ins_pipe(ialu_reg_reg_alu0);
 7726 %}
 7727 
 7728 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7729 %{
 7730   match(Set dst (UMulHiL src rax));
 7731   effect(USE_KILL rax, KILL cr);
 7732 
 7733   ins_cost(300);
 7734   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 7735   ins_encode %{
 7736     __ mulq($src$$Register);
 7737   %}
 7738   ins_pipe(ialu_reg_reg_alu0);
 7739 %}
 7740 
 7741 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7742                    rFlagsReg cr)
 7743 %{
 7744   match(Set rax (DivI rax div));
 7745   effect(KILL rdx, KILL cr);
 7746 
 7747   ins_cost(30*100+10*100); // XXX
 7748   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7749             "jne,s   normal\n\t"
 7750             "xorl    rdx, rdx\n\t"
 7751             "cmpl    $div, -1\n\t"
 7752             "je,s    done\n"
 7753     "normal: cdql\n\t"
 7754             "idivl   $div\n"
 7755     "done:"        %}
 7756   ins_encode(cdql_enc(div));
 7757   ins_pipe(ialu_reg_reg_alu0);
 7758 %}
 7759 
 7760 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7761                    rFlagsReg cr)
 7762 %{
 7763   match(Set rax (DivL rax div));
 7764   effect(KILL rdx, KILL cr);
 7765 
 7766   ins_cost(30*100+10*100); // XXX
 7767   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7768             "cmpq    rax, rdx\n\t"
 7769             "jne,s   normal\n\t"
 7770             "xorl    rdx, rdx\n\t"
 7771             "cmpq    $div, -1\n\t"
 7772             "je,s    done\n"
 7773     "normal: cdqq\n\t"
 7774             "idivq   $div\n"
 7775     "done:"        %}
 7776   ins_encode(cdqq_enc(div));
 7777   ins_pipe(ialu_reg_reg_alu0);
 7778 %}
 7779 
 7780 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 7781 %{
 7782   match(Set rax (UDivI rax div));
 7783   effect(KILL rdx, KILL cr);
 7784 
 7785   ins_cost(300);
 7786   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 7787   ins_encode %{
 7788     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 7789   %}
 7790   ins_pipe(ialu_reg_reg_alu0);
 7791 %}
 7792 
 7793 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 7794 %{
 7795   match(Set rax (UDivL rax div));
 7796   effect(KILL rdx, KILL cr);
 7797 
 7798   ins_cost(300);
 7799   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 7800   ins_encode %{
 7801      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 7802   %}
 7803   ins_pipe(ialu_reg_reg_alu0);
 7804 %}
 7805 
 7806 // Integer DIVMOD with Register, both quotient and mod results
 7807 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7808                              rFlagsReg cr)
 7809 %{
 7810   match(DivModI rax div);
 7811   effect(KILL cr);
 7812 
 7813   ins_cost(30*100+10*100); // XXX
 7814   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7815             "jne,s   normal\n\t"
 7816             "xorl    rdx, rdx\n\t"
 7817             "cmpl    $div, -1\n\t"
 7818             "je,s    done\n"
 7819     "normal: cdql\n\t"
 7820             "idivl   $div\n"
 7821     "done:"        %}
 7822   ins_encode(cdql_enc(div));
 7823   ins_pipe(pipe_slow);
 7824 %}
 7825 
 7826 // Long DIVMOD with Register, both quotient and mod results
 7827 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7828                              rFlagsReg cr)
 7829 %{
 7830   match(DivModL rax div);
 7831   effect(KILL cr);
 7832 
 7833   ins_cost(30*100+10*100); // XXX
 7834   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7835             "cmpq    rax, rdx\n\t"
 7836             "jne,s   normal\n\t"
 7837             "xorl    rdx, rdx\n\t"
 7838             "cmpq    $div, -1\n\t"
 7839             "je,s    done\n"
 7840     "normal: cdqq\n\t"
 7841             "idivq   $div\n"
 7842     "done:"        %}
 7843   ins_encode(cdqq_enc(div));
 7844   ins_pipe(pipe_slow);
 7845 %}
 7846 
 7847 // Unsigned integer DIVMOD with Register, both quotient and mod results
 7848 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 7849                               no_rax_rdx_RegI div, rFlagsReg cr)
 7850 %{
 7851   match(UDivModI rax div);
 7852   effect(TEMP tmp, KILL cr);
 7853 
 7854   ins_cost(300);
 7855   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 7856             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 7857           %}
 7858   ins_encode %{
 7859     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 7860   %}
 7861   ins_pipe(pipe_slow);
 7862 %}
 7863 
 7864 // Unsigned long DIVMOD with Register, both quotient and mod results
 7865 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 7866                               no_rax_rdx_RegL div, rFlagsReg cr)
 7867 %{
 7868   match(UDivModL rax div);
 7869   effect(TEMP tmp, KILL cr);
 7870 
 7871   ins_cost(300);
 7872   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 7873             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 7874           %}
 7875   ins_encode %{
 7876     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 7877   %}
 7878   ins_pipe(pipe_slow);
 7879 %}
 7880 
 7881 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 7882                    rFlagsReg cr)
 7883 %{
 7884   match(Set rdx (ModI rax div));
 7885   effect(KILL rax, KILL cr);
 7886 
 7887   ins_cost(300); // XXX
 7888   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 7889             "jne,s   normal\n\t"
 7890             "xorl    rdx, rdx\n\t"
 7891             "cmpl    $div, -1\n\t"
 7892             "je,s    done\n"
 7893     "normal: cdql\n\t"
 7894             "idivl   $div\n"
 7895     "done:"        %}
 7896   ins_encode(cdql_enc(div));
 7897   ins_pipe(ialu_reg_reg_alu0);
 7898 %}
 7899 
 7900 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 7901                    rFlagsReg cr)
 7902 %{
 7903   match(Set rdx (ModL rax div));
 7904   effect(KILL rax, KILL cr);
 7905 
 7906   ins_cost(300); // XXX
 7907   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 7908             "cmpq    rax, rdx\n\t"
 7909             "jne,s   normal\n\t"
 7910             "xorl    rdx, rdx\n\t"
 7911             "cmpq    $div, -1\n\t"
 7912             "je,s    done\n"
 7913     "normal: cdqq\n\t"
 7914             "idivq   $div\n"
 7915     "done:"        %}
 7916   ins_encode(cdqq_enc(div));
 7917   ins_pipe(ialu_reg_reg_alu0);
 7918 %}
 7919 
 7920 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 7921 %{
 7922   match(Set rdx (UModI rax div));
 7923   effect(KILL rax, KILL cr);
 7924 
 7925   ins_cost(300);
 7926   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 7927   ins_encode %{
 7928     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 7929   %}
 7930   ins_pipe(ialu_reg_reg_alu0);
 7931 %}
 7932 
 7933 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 7934 %{
 7935   match(Set rdx (UModL rax div));
 7936   effect(KILL rax, KILL cr);
 7937 
 7938   ins_cost(300);
 7939   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 7940   ins_encode %{
 7941     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 7942   %}
 7943   ins_pipe(ialu_reg_reg_alu0);
 7944 %}
 7945 
 7946 // Integer Shift Instructions
 7947 // Shift Left by one, two, three
 7948 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 7949 %{
 7950   match(Set dst (LShiftI dst shift));
 7951   effect(KILL cr);
 7952 
 7953   format %{ "sall    $dst, $shift" %}
 7954   ins_encode %{
 7955     __ sall($dst$$Register, $shift$$constant);
 7956   %}
 7957   ins_pipe(ialu_reg);
 7958 %}
 7959 
 7960 // Shift Left by 8-bit immediate
 7961 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 7962 %{
 7963   match(Set dst (LShiftI dst shift));
 7964   effect(KILL cr);
 7965 
 7966   format %{ "sall    $dst, $shift" %}
 7967   ins_encode %{
 7968     __ sall($dst$$Register, $shift$$constant);
 7969   %}
 7970   ins_pipe(ialu_reg);
 7971 %}
 7972 
 7973 // Shift Left by 8-bit immediate
 7974 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 7975 %{
 7976   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 7977   effect(KILL cr);
 7978 
 7979   format %{ "sall    $dst, $shift" %}
 7980   ins_encode %{
 7981     __ sall($dst$$Address, $shift$$constant);
 7982   %}
 7983   ins_pipe(ialu_mem_imm);
 7984 %}
 7985 
 7986 // Shift Left by variable
 7987 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 7988 %{
 7989   predicate(!VM_Version::supports_bmi2());
 7990   match(Set dst (LShiftI dst shift));
 7991   effect(KILL cr);
 7992 
 7993   format %{ "sall    $dst, $shift" %}
 7994   ins_encode %{
 7995     __ sall($dst$$Register);
 7996   %}
 7997   ins_pipe(ialu_reg_reg);
 7998 %}
 7999 
 8000 // Shift Left by variable
 8001 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8002 %{
 8003   predicate(!VM_Version::supports_bmi2());
 8004   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8005   effect(KILL cr);
 8006 
 8007   format %{ "sall    $dst, $shift" %}
 8008   ins_encode %{
 8009     __ sall($dst$$Address);
 8010   %}
 8011   ins_pipe(ialu_mem_reg);
 8012 %}
 8013 
 8014 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8015 %{
 8016   predicate(VM_Version::supports_bmi2());
 8017   match(Set dst (LShiftI src shift));
 8018 
 8019   format %{ "shlxl   $dst, $src, $shift" %}
 8020   ins_encode %{
 8021     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 8022   %}
 8023   ins_pipe(ialu_reg_reg);
 8024 %}
 8025 
 8026 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8027 %{
 8028   predicate(VM_Version::supports_bmi2());
 8029   match(Set dst (LShiftI (LoadI src) shift));
 8030   ins_cost(175);
 8031   format %{ "shlxl   $dst, $src, $shift" %}
 8032   ins_encode %{
 8033     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 8034   %}
 8035   ins_pipe(ialu_reg_mem);
 8036 %}
 8037 
 8038 // Arithmetic Shift Right by 8-bit immediate
 8039 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8040 %{
 8041   match(Set dst (RShiftI dst shift));
 8042   effect(KILL cr);
 8043 
 8044   format %{ "sarl    $dst, $shift" %}
 8045   ins_encode %{
 8046     __ sarl($dst$$Register, $shift$$constant);
 8047   %}
 8048   ins_pipe(ialu_mem_imm);
 8049 %}
 8050 
 8051 // Arithmetic Shift Right by 8-bit immediate
 8052 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8053 %{
 8054   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8055   effect(KILL cr);
 8056 
 8057   format %{ "sarl    $dst, $shift" %}
 8058   ins_encode %{
 8059     __ sarl($dst$$Address, $shift$$constant);
 8060   %}
 8061   ins_pipe(ialu_mem_imm);
 8062 %}
 8063 
 8064 // Arithmetic Shift Right by variable
 8065 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8066 %{
 8067   predicate(!VM_Version::supports_bmi2());
 8068   match(Set dst (RShiftI dst shift));
 8069   effect(KILL cr);
 8070 
 8071   format %{ "sarl    $dst, $shift" %}
 8072   ins_encode %{
 8073     __ sarl($dst$$Register);
 8074   %}
 8075   ins_pipe(ialu_reg_reg);
 8076 %}
 8077 
 8078 // Arithmetic Shift Right by variable
 8079 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8080 %{
 8081   predicate(!VM_Version::supports_bmi2());
 8082   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8083   effect(KILL cr);
 8084 
 8085   format %{ "sarl    $dst, $shift" %}
 8086   ins_encode %{
 8087     __ sarl($dst$$Address);
 8088   %}
 8089   ins_pipe(ialu_mem_reg);
 8090 %}
 8091 
 8092 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8093 %{
 8094   predicate(VM_Version::supports_bmi2());
 8095   match(Set dst (RShiftI src shift));
 8096 
 8097   format %{ "sarxl   $dst, $src, $shift" %}
 8098   ins_encode %{
 8099     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 8100   %}
 8101   ins_pipe(ialu_reg_reg);
 8102 %}
 8103 
 8104 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8105 %{
 8106   predicate(VM_Version::supports_bmi2());
 8107   match(Set dst (RShiftI (LoadI src) shift));
 8108   ins_cost(175);
 8109   format %{ "sarxl   $dst, $src, $shift" %}
 8110   ins_encode %{
 8111     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 8112   %}
 8113   ins_pipe(ialu_reg_mem);
 8114 %}
 8115 
 8116 // Logical Shift Right by 8-bit immediate
 8117 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8118 %{
 8119   match(Set dst (URShiftI dst shift));
 8120   effect(KILL cr);
 8121 
 8122   format %{ "shrl    $dst, $shift" %}
 8123   ins_encode %{
 8124     __ shrl($dst$$Register, $shift$$constant);
 8125   %}
 8126   ins_pipe(ialu_reg);
 8127 %}
 8128 
 8129 // Logical Shift Right by 8-bit immediate
 8130 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8131 %{
 8132   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8133   effect(KILL cr);
 8134 
 8135   format %{ "shrl    $dst, $shift" %}
 8136   ins_encode %{
 8137     __ shrl($dst$$Address, $shift$$constant);
 8138   %}
 8139   ins_pipe(ialu_mem_imm);
 8140 %}
 8141 
 8142 // Logical Shift Right by variable
 8143 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8144 %{
 8145   predicate(!VM_Version::supports_bmi2());
 8146   match(Set dst (URShiftI dst shift));
 8147   effect(KILL cr);
 8148 
 8149   format %{ "shrl    $dst, $shift" %}
 8150   ins_encode %{
 8151     __ shrl($dst$$Register);
 8152   %}
 8153   ins_pipe(ialu_reg_reg);
 8154 %}
 8155 
 8156 // Logical Shift Right by variable
 8157 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8158 %{
 8159   predicate(!VM_Version::supports_bmi2());
 8160   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8161   effect(KILL cr);
 8162 
 8163   format %{ "shrl    $dst, $shift" %}
 8164   ins_encode %{
 8165     __ shrl($dst$$Address);
 8166   %}
 8167   ins_pipe(ialu_mem_reg);
 8168 %}
 8169 
 8170 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8171 %{
 8172   predicate(VM_Version::supports_bmi2());
 8173   match(Set dst (URShiftI src shift));
 8174 
 8175   format %{ "shrxl   $dst, $src, $shift" %}
 8176   ins_encode %{
 8177     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 8178   %}
 8179   ins_pipe(ialu_reg_reg);
 8180 %}
 8181 
 8182 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8183 %{
 8184   predicate(VM_Version::supports_bmi2());
 8185   match(Set dst (URShiftI (LoadI src) shift));
 8186   ins_cost(175);
 8187   format %{ "shrxl   $dst, $src, $shift" %}
 8188   ins_encode %{
 8189     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 8190   %}
 8191   ins_pipe(ialu_reg_mem);
 8192 %}
 8193 
 8194 // Long Shift Instructions
 8195 // Shift Left by one, two, three
 8196 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 8197 %{
 8198   match(Set dst (LShiftL dst shift));
 8199   effect(KILL cr);
 8200 
 8201   format %{ "salq    $dst, $shift" %}
 8202   ins_encode %{
 8203     __ salq($dst$$Register, $shift$$constant);
 8204   %}
 8205   ins_pipe(ialu_reg);
 8206 %}
 8207 
 8208 // Shift Left by 8-bit immediate
 8209 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8210 %{
 8211   match(Set dst (LShiftL dst shift));
 8212   effect(KILL cr);
 8213 
 8214   format %{ "salq    $dst, $shift" %}
 8215   ins_encode %{
 8216     __ salq($dst$$Register, $shift$$constant);
 8217   %}
 8218   ins_pipe(ialu_reg);
 8219 %}
 8220 
 8221 // Shift Left by 8-bit immediate
 8222 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8223 %{
 8224   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8225   effect(KILL cr);
 8226 
 8227   format %{ "salq    $dst, $shift" %}
 8228   ins_encode %{
 8229     __ salq($dst$$Address, $shift$$constant);
 8230   %}
 8231   ins_pipe(ialu_mem_imm);
 8232 %}
 8233 
 8234 // Shift Left by variable
 8235 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8236 %{
 8237   predicate(!VM_Version::supports_bmi2());
 8238   match(Set dst (LShiftL dst shift));
 8239   effect(KILL cr);
 8240 
 8241   format %{ "salq    $dst, $shift" %}
 8242   ins_encode %{
 8243     __ salq($dst$$Register);
 8244   %}
 8245   ins_pipe(ialu_reg_reg);
 8246 %}
 8247 
 8248 // Shift Left by variable
 8249 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8250 %{
 8251   predicate(!VM_Version::supports_bmi2());
 8252   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8253   effect(KILL cr);
 8254 
 8255   format %{ "salq    $dst, $shift" %}
 8256   ins_encode %{
 8257     __ salq($dst$$Address);
 8258   %}
 8259   ins_pipe(ialu_mem_reg);
 8260 %}
 8261 
 8262 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8263 %{
 8264   predicate(VM_Version::supports_bmi2());
 8265   match(Set dst (LShiftL src shift));
 8266 
 8267   format %{ "shlxq   $dst, $src, $shift" %}
 8268   ins_encode %{
 8269     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 8270   %}
 8271   ins_pipe(ialu_reg_reg);
 8272 %}
 8273 
 8274 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8275 %{
 8276   predicate(VM_Version::supports_bmi2());
 8277   match(Set dst (LShiftL (LoadL src) shift));
 8278   ins_cost(175);
 8279   format %{ "shlxq   $dst, $src, $shift" %}
 8280   ins_encode %{
 8281     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 8282   %}
 8283   ins_pipe(ialu_reg_mem);
 8284 %}
 8285 
 8286 // Arithmetic Shift Right by 8-bit immediate
 8287 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 8288 %{
 8289   match(Set dst (RShiftL dst shift));
 8290   effect(KILL cr);
 8291 
 8292   format %{ "sarq    $dst, $shift" %}
 8293   ins_encode %{
 8294     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 8295   %}
 8296   ins_pipe(ialu_mem_imm);
 8297 %}
 8298 
 8299 // Arithmetic Shift Right by 8-bit immediate
 8300 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 8301 %{
 8302   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8303   effect(KILL cr);
 8304 
 8305   format %{ "sarq    $dst, $shift" %}
 8306   ins_encode %{
 8307     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 8308   %}
 8309   ins_pipe(ialu_mem_imm);
 8310 %}
 8311 
 8312 // Arithmetic Shift Right by variable
 8313 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8314 %{
 8315   predicate(!VM_Version::supports_bmi2());
 8316   match(Set dst (RShiftL dst shift));
 8317   effect(KILL cr);
 8318 
 8319   format %{ "sarq    $dst, $shift" %}
 8320   ins_encode %{
 8321     __ sarq($dst$$Register);
 8322   %}
 8323   ins_pipe(ialu_reg_reg);
 8324 %}
 8325 
 8326 // Arithmetic Shift Right by variable
 8327 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8328 %{
 8329   predicate(!VM_Version::supports_bmi2());
 8330   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8331   effect(KILL cr);
 8332 
 8333   format %{ "sarq    $dst, $shift" %}
 8334   ins_encode %{
 8335     __ sarq($dst$$Address);
 8336   %}
 8337   ins_pipe(ialu_mem_reg);
 8338 %}
 8339 
 8340 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8341 %{
 8342   predicate(VM_Version::supports_bmi2());
 8343   match(Set dst (RShiftL src shift));
 8344 
 8345   format %{ "sarxq   $dst, $src, $shift" %}
 8346   ins_encode %{
 8347     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 8348   %}
 8349   ins_pipe(ialu_reg_reg);
 8350 %}
 8351 
 8352 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8353 %{
 8354   predicate(VM_Version::supports_bmi2());
 8355   match(Set dst (RShiftL (LoadL src) shift));
 8356   ins_cost(175);
 8357   format %{ "sarxq   $dst, $src, $shift" %}
 8358   ins_encode %{
 8359     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 8360   %}
 8361   ins_pipe(ialu_reg_mem);
 8362 %}
 8363 
 8364 // Logical Shift Right by 8-bit immediate
 8365 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8366 %{
 8367   match(Set dst (URShiftL dst shift));
 8368   effect(KILL cr);
 8369 
 8370   format %{ "shrq    $dst, $shift" %}
 8371   ins_encode %{
 8372     __ shrq($dst$$Register, $shift$$constant);
 8373   %}
 8374   ins_pipe(ialu_reg);
 8375 %}
 8376 
 8377 // Logical Shift Right by 8-bit immediate
 8378 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8379 %{
 8380   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8381   effect(KILL cr);
 8382 
 8383   format %{ "shrq    $dst, $shift" %}
 8384   ins_encode %{
 8385     __ shrq($dst$$Address, $shift$$constant);
 8386   %}
 8387   ins_pipe(ialu_mem_imm);
 8388 %}
 8389 
 8390 // Logical Shift Right by variable
 8391 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8392 %{
 8393   predicate(!VM_Version::supports_bmi2());
 8394   match(Set dst (URShiftL dst shift));
 8395   effect(KILL cr);
 8396 
 8397   format %{ "shrq    $dst, $shift" %}
 8398   ins_encode %{
 8399     __ shrq($dst$$Register);
 8400   %}
 8401   ins_pipe(ialu_reg_reg);
 8402 %}
 8403 
 8404 // Logical Shift Right by variable
 8405 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8406 %{
 8407   predicate(!VM_Version::supports_bmi2());
 8408   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8409   effect(KILL cr);
 8410 
 8411   format %{ "shrq    $dst, $shift" %}
 8412   ins_encode %{
 8413     __ shrq($dst$$Address);
 8414   %}
 8415   ins_pipe(ialu_mem_reg);
 8416 %}
 8417 
 8418 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8419 %{
 8420   predicate(VM_Version::supports_bmi2());
 8421   match(Set dst (URShiftL src shift));
 8422 
 8423   format %{ "shrxq   $dst, $src, $shift" %}
 8424   ins_encode %{
 8425     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 8426   %}
 8427   ins_pipe(ialu_reg_reg);
 8428 %}
 8429 
 8430 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8431 %{
 8432   predicate(VM_Version::supports_bmi2());
 8433   match(Set dst (URShiftL (LoadL src) shift));
 8434   ins_cost(175);
 8435   format %{ "shrxq   $dst, $src, $shift" %}
 8436   ins_encode %{
 8437     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 8438   %}
 8439   ins_pipe(ialu_reg_mem);
 8440 %}
 8441 
 8442 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 8443 // This idiom is used by the compiler for the i2b bytecode.
 8444 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 8445 %{
 8446   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 8447 
 8448   format %{ "movsbl  $dst, $src\t# i2b" %}
 8449   ins_encode %{
 8450     __ movsbl($dst$$Register, $src$$Register);
 8451   %}
 8452   ins_pipe(ialu_reg_reg);
 8453 %}
 8454 
 8455 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 8456 // This idiom is used by the compiler the i2s bytecode.
 8457 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 8458 %{
 8459   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 8460 
 8461   format %{ "movswl  $dst, $src\t# i2s" %}
 8462   ins_encode %{
 8463     __ movswl($dst$$Register, $src$$Register);
 8464   %}
 8465   ins_pipe(ialu_reg_reg);
 8466 %}
 8467 
 8468 // ROL/ROR instructions
 8469 
 8470 // Rotate left by constant.
 8471 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8472 %{
 8473   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8474   match(Set dst (RotateLeft dst shift));
 8475   effect(KILL cr);
 8476   format %{ "roll    $dst, $shift" %}
 8477   ins_encode %{
 8478     __ roll($dst$$Register, $shift$$constant);
 8479   %}
 8480   ins_pipe(ialu_reg);
 8481 %}
 8482 
 8483 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 8484 %{
 8485   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8486   match(Set dst (RotateLeft src shift));
 8487   format %{ "rolxl   $dst, $src, $shift" %}
 8488   ins_encode %{
 8489     int shift = 32 - ($shift$$constant & 31);
 8490     __ rorxl($dst$$Register, $src$$Register, shift);
 8491   %}
 8492   ins_pipe(ialu_reg_reg);
 8493 %}
 8494 
 8495 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8496 %{
 8497   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8498   match(Set dst (RotateLeft (LoadI src) shift));
 8499   ins_cost(175);
 8500   format %{ "rolxl   $dst, $src, $shift" %}
 8501   ins_encode %{
 8502     int shift = 32 - ($shift$$constant & 31);
 8503     __ rorxl($dst$$Register, $src$$Address, shift);
 8504   %}
 8505   ins_pipe(ialu_reg_mem);
 8506 %}
 8507 
 8508 // Rotate Left by variable
 8509 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8510 %{
 8511   predicate(n->bottom_type()->basic_type() == T_INT);
 8512   match(Set dst (RotateLeft dst shift));
 8513   effect(KILL cr);
 8514   format %{ "roll    $dst, $shift" %}
 8515   ins_encode %{
 8516     __ roll($dst$$Register);
 8517   %}
 8518   ins_pipe(ialu_reg_reg);
 8519 %}
 8520 
 8521 // Rotate Right by constant.
 8522 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8523 %{
 8524   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8525   match(Set dst (RotateRight dst shift));
 8526   effect(KILL cr);
 8527   format %{ "rorl    $dst, $shift" %}
 8528   ins_encode %{
 8529     __ rorl($dst$$Register, $shift$$constant);
 8530   %}
 8531   ins_pipe(ialu_reg);
 8532 %}
 8533 
 8534 // Rotate Right by constant.
 8535 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 8536 %{
 8537   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8538   match(Set dst (RotateRight src shift));
 8539   format %{ "rorxl   $dst, $src, $shift" %}
 8540   ins_encode %{
 8541     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 8542   %}
 8543   ins_pipe(ialu_reg_reg);
 8544 %}
 8545 
 8546 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8547 %{
 8548   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8549   match(Set dst (RotateRight (LoadI src) shift));
 8550   ins_cost(175);
 8551   format %{ "rorxl   $dst, $src, $shift" %}
 8552   ins_encode %{
 8553     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 8554   %}
 8555   ins_pipe(ialu_reg_mem);
 8556 %}
 8557 
 8558 // Rotate Right by variable
 8559 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8560 %{
 8561   predicate(n->bottom_type()->basic_type() == T_INT);
 8562   match(Set dst (RotateRight dst shift));
 8563   effect(KILL cr);
 8564   format %{ "rorl    $dst, $shift" %}
 8565   ins_encode %{
 8566     __ rorl($dst$$Register);
 8567   %}
 8568   ins_pipe(ialu_reg_reg);
 8569 %}
 8570 
 8571 // Rotate Left by constant.
 8572 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8573 %{
 8574   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8575   match(Set dst (RotateLeft dst shift));
 8576   effect(KILL cr);
 8577   format %{ "rolq    $dst, $shift" %}
 8578   ins_encode %{
 8579     __ rolq($dst$$Register, $shift$$constant);
 8580   %}
 8581   ins_pipe(ialu_reg);
 8582 %}
 8583 
 8584 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 8585 %{
 8586   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8587   match(Set dst (RotateLeft src shift));
 8588   format %{ "rolxq   $dst, $src, $shift" %}
 8589   ins_encode %{
 8590     int shift = 64 - ($shift$$constant & 63);
 8591     __ rorxq($dst$$Register, $src$$Register, shift);
 8592   %}
 8593   ins_pipe(ialu_reg_reg);
 8594 %}
 8595 
 8596 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8597 %{
 8598   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8599   match(Set dst (RotateLeft (LoadL src) shift));
 8600   ins_cost(175);
 8601   format %{ "rolxq   $dst, $src, $shift" %}
 8602   ins_encode %{
 8603     int shift = 64 - ($shift$$constant & 63);
 8604     __ rorxq($dst$$Register, $src$$Address, shift);
 8605   %}
 8606   ins_pipe(ialu_reg_mem);
 8607 %}
 8608 
 8609 // Rotate Left by variable
 8610 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8611 %{
 8612   predicate(n->bottom_type()->basic_type() == T_LONG);
 8613   match(Set dst (RotateLeft dst shift));
 8614   effect(KILL cr);
 8615   format %{ "rolq    $dst, $shift" %}
 8616   ins_encode %{
 8617     __ rolq($dst$$Register);
 8618   %}
 8619   ins_pipe(ialu_reg_reg);
 8620 %}
 8621 
 8622 // Rotate Right by constant.
 8623 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8624 %{
 8625   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8626   match(Set dst (RotateRight dst shift));
 8627   effect(KILL cr);
 8628   format %{ "rorq    $dst, $shift" %}
 8629   ins_encode %{
 8630     __ rorq($dst$$Register, $shift$$constant);
 8631   %}
 8632   ins_pipe(ialu_reg);
 8633 %}
 8634 
 8635 // Rotate Right by constant
 8636 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 8637 %{
 8638   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8639   match(Set dst (RotateRight src shift));
 8640   format %{ "rorxq   $dst, $src, $shift" %}
 8641   ins_encode %{
 8642     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 8643   %}
 8644   ins_pipe(ialu_reg_reg);
 8645 %}
 8646 
 8647 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8648 %{
 8649   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8650   match(Set dst (RotateRight (LoadL src) shift));
 8651   ins_cost(175);
 8652   format %{ "rorxq   $dst, $src, $shift" %}
 8653   ins_encode %{
 8654     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 8655   %}
 8656   ins_pipe(ialu_reg_mem);
 8657 %}
 8658 
 8659 // Rotate Right by variable
 8660 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8661 %{
 8662   predicate(n->bottom_type()->basic_type() == T_LONG);
 8663   match(Set dst (RotateRight dst shift));
 8664   effect(KILL cr);
 8665   format %{ "rorq    $dst, $shift" %}
 8666   ins_encode %{
 8667     __ rorq($dst$$Register);
 8668   %}
 8669   ins_pipe(ialu_reg_reg);
 8670 %}
 8671 
 8672 //----------------------------- CompressBits/ExpandBits ------------------------
 8673 
 8674 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8675   predicate(n->bottom_type()->isa_long());
 8676   match(Set dst (CompressBits src mask));
 8677   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8678   ins_encode %{
 8679     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 8680   %}
 8681   ins_pipe( pipe_slow );
 8682 %}
 8683 
 8684 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8685   predicate(n->bottom_type()->isa_long());
 8686   match(Set dst (ExpandBits src mask));
 8687   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8688   ins_encode %{
 8689     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 8690   %}
 8691   ins_pipe( pipe_slow );
 8692 %}
 8693 
 8694 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8695   predicate(n->bottom_type()->isa_long());
 8696   match(Set dst (CompressBits src (LoadL mask)));
 8697   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8698   ins_encode %{
 8699     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 8700   %}
 8701   ins_pipe( pipe_slow );
 8702 %}
 8703 
 8704 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8705   predicate(n->bottom_type()->isa_long());
 8706   match(Set dst (ExpandBits src (LoadL mask)));
 8707   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8708   ins_encode %{
 8709     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 8710   %}
 8711   ins_pipe( pipe_slow );
 8712 %}
 8713 
 8714 
 8715 // Logical Instructions
 8716 
 8717 // Integer Logical Instructions
 8718 
 8719 // And Instructions
 8720 // And Register with Register
 8721 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8722 %{
 8723   match(Set dst (AndI dst src));
 8724   effect(KILL cr);
 8725   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8726 
 8727   format %{ "andl    $dst, $src\t# int" %}
 8728   ins_encode %{
 8729     __ andl($dst$$Register, $src$$Register);
 8730   %}
 8731   ins_pipe(ialu_reg_reg);
 8732 %}
 8733 
 8734 // And Register with Immediate 255
 8735 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 8736 %{
 8737   match(Set dst (AndI src mask));
 8738 
 8739   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 8740   ins_encode %{
 8741     __ movzbl($dst$$Register, $src$$Register);
 8742   %}
 8743   ins_pipe(ialu_reg);
 8744 %}
 8745 
 8746 // And Register with Immediate 255 and promote to long
 8747 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 8748 %{
 8749   match(Set dst (ConvI2L (AndI src mask)));
 8750 
 8751   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 8752   ins_encode %{
 8753     __ movzbl($dst$$Register, $src$$Register);
 8754   %}
 8755   ins_pipe(ialu_reg);
 8756 %}
 8757 
 8758 // And Register with Immediate 65535
 8759 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 8760 %{
 8761   match(Set dst (AndI src mask));
 8762 
 8763   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 8764   ins_encode %{
 8765     __ movzwl($dst$$Register, $src$$Register);
 8766   %}
 8767   ins_pipe(ialu_reg);
 8768 %}
 8769 
 8770 // And Register with Immediate 65535 and promote to long
 8771 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 8772 %{
 8773   match(Set dst (ConvI2L (AndI src mask)));
 8774 
 8775   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 8776   ins_encode %{
 8777     __ movzwl($dst$$Register, $src$$Register);
 8778   %}
 8779   ins_pipe(ialu_reg);
 8780 %}
 8781 
 8782 // Can skip int2long conversions after AND with small bitmask
 8783 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 8784 %{
 8785   predicate(VM_Version::supports_bmi2());
 8786   ins_cost(125);
 8787   effect(TEMP tmp, KILL cr);
 8788   match(Set dst (ConvI2L (AndI src mask)));
 8789   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 8790   ins_encode %{
 8791     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 8792     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 8793   %}
 8794   ins_pipe(ialu_reg_reg);
 8795 %}
 8796 
 8797 // And Register with Immediate
 8798 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8799 %{
 8800   match(Set dst (AndI dst src));
 8801   effect(KILL cr);
 8802   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8803 
 8804   format %{ "andl    $dst, $src\t# int" %}
 8805   ins_encode %{
 8806     __ andl($dst$$Register, $src$$constant);
 8807   %}
 8808   ins_pipe(ialu_reg);
 8809 %}
 8810 
 8811 // And Register with Memory
 8812 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8813 %{
 8814   match(Set dst (AndI dst (LoadI src)));
 8815   effect(KILL cr);
 8816   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8817 
 8818   ins_cost(150);
 8819   format %{ "andl    $dst, $src\t# int" %}
 8820   ins_encode %{
 8821     __ andl($dst$$Register, $src$$Address);
 8822   %}
 8823   ins_pipe(ialu_reg_mem);
 8824 %}
 8825 
 8826 // And Memory with Register
 8827 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8828 %{
 8829   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 8830   effect(KILL cr);
 8831   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8832 
 8833   ins_cost(150);
 8834   format %{ "andb    $dst, $src\t# byte" %}
 8835   ins_encode %{
 8836     __ andb($dst$$Address, $src$$Register);
 8837   %}
 8838   ins_pipe(ialu_mem_reg);
 8839 %}
 8840 
 8841 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8842 %{
 8843   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 8844   effect(KILL cr);
 8845   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8846 
 8847   ins_cost(150);
 8848   format %{ "andl    $dst, $src\t# int" %}
 8849   ins_encode %{
 8850     __ andl($dst$$Address, $src$$Register);
 8851   %}
 8852   ins_pipe(ialu_mem_reg);
 8853 %}
 8854 
 8855 // And Memory with Immediate
 8856 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8857 %{
 8858   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 8859   effect(KILL cr);
 8860   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8861 
 8862   ins_cost(125);
 8863   format %{ "andl    $dst, $src\t# int" %}
 8864   ins_encode %{
 8865     __ andl($dst$$Address, $src$$constant);
 8866   %}
 8867   ins_pipe(ialu_mem_imm);
 8868 %}
 8869 
 8870 // BMI1 instructions
 8871 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 8872   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 8873   predicate(UseBMI1Instructions);
 8874   effect(KILL cr);
 8875   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8876 
 8877   ins_cost(125);
 8878   format %{ "andnl  $dst, $src1, $src2" %}
 8879 
 8880   ins_encode %{
 8881     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 8882   %}
 8883   ins_pipe(ialu_reg_mem);
 8884 %}
 8885 
 8886 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 8887   match(Set dst (AndI (XorI src1 minus_1) src2));
 8888   predicate(UseBMI1Instructions);
 8889   effect(KILL cr);
 8890   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8891 
 8892   format %{ "andnl  $dst, $src1, $src2" %}
 8893 
 8894   ins_encode %{
 8895     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 8896   %}
 8897   ins_pipe(ialu_reg);
 8898 %}
 8899 
 8900 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 8901   match(Set dst (AndI (SubI imm_zero src) src));
 8902   predicate(UseBMI1Instructions);
 8903   effect(KILL cr);
 8904   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 8905 
 8906   format %{ "blsil  $dst, $src" %}
 8907 
 8908   ins_encode %{
 8909     __ blsil($dst$$Register, $src$$Register);
 8910   %}
 8911   ins_pipe(ialu_reg);
 8912 %}
 8913 
 8914 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 8915   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 8916   predicate(UseBMI1Instructions);
 8917   effect(KILL cr);
 8918   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 8919 
 8920   ins_cost(125);
 8921   format %{ "blsil  $dst, $src" %}
 8922 
 8923   ins_encode %{
 8924     __ blsil($dst$$Register, $src$$Address);
 8925   %}
 8926   ins_pipe(ialu_reg_mem);
 8927 %}
 8928 
 8929 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 8930 %{
 8931   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 8932   predicate(UseBMI1Instructions);
 8933   effect(KILL cr);
 8934   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 8935 
 8936   ins_cost(125);
 8937   format %{ "blsmskl $dst, $src" %}
 8938 
 8939   ins_encode %{
 8940     __ blsmskl($dst$$Register, $src$$Address);
 8941   %}
 8942   ins_pipe(ialu_reg_mem);
 8943 %}
 8944 
 8945 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 8946 %{
 8947   match(Set dst (XorI (AddI src minus_1) src));
 8948   predicate(UseBMI1Instructions);
 8949   effect(KILL cr);
 8950   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 8951 
 8952   format %{ "blsmskl $dst, $src" %}
 8953 
 8954   ins_encode %{
 8955     __ blsmskl($dst$$Register, $src$$Register);
 8956   %}
 8957 
 8958   ins_pipe(ialu_reg);
 8959 %}
 8960 
 8961 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 8962 %{
 8963   match(Set dst (AndI (AddI src minus_1) src) );
 8964   predicate(UseBMI1Instructions);
 8965   effect(KILL cr);
 8966   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 8967 
 8968   format %{ "blsrl  $dst, $src" %}
 8969 
 8970   ins_encode %{
 8971     __ blsrl($dst$$Register, $src$$Register);
 8972   %}
 8973 
 8974   ins_pipe(ialu_reg_mem);
 8975 %}
 8976 
 8977 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 8978 %{
 8979   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 8980   predicate(UseBMI1Instructions);
 8981   effect(KILL cr);
 8982   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 8983 
 8984   ins_cost(125);
 8985   format %{ "blsrl  $dst, $src" %}
 8986 
 8987   ins_encode %{
 8988     __ blsrl($dst$$Register, $src$$Address);
 8989   %}
 8990 
 8991   ins_pipe(ialu_reg);
 8992 %}
 8993 
 8994 // Or Instructions
 8995 // Or Register with Register
 8996 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8997 %{
 8998   match(Set dst (OrI dst src));
 8999   effect(KILL cr);
 9000   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9001 
 9002   format %{ "orl     $dst, $src\t# int" %}
 9003   ins_encode %{
 9004     __ orl($dst$$Register, $src$$Register);
 9005   %}
 9006   ins_pipe(ialu_reg_reg);
 9007 %}
 9008 
 9009 // Or Register with Immediate
 9010 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9011 %{
 9012   match(Set dst (OrI dst src));
 9013   effect(KILL cr);
 9014   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9015 
 9016   format %{ "orl     $dst, $src\t# int" %}
 9017   ins_encode %{
 9018     __ orl($dst$$Register, $src$$constant);
 9019   %}
 9020   ins_pipe(ialu_reg);
 9021 %}
 9022 
 9023 // Or Register with Memory
 9024 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9025 %{
 9026   match(Set dst (OrI dst (LoadI src)));
 9027   effect(KILL cr);
 9028   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9029 
 9030   ins_cost(150);
 9031   format %{ "orl     $dst, $src\t# int" %}
 9032   ins_encode %{
 9033     __ orl($dst$$Register, $src$$Address);
 9034   %}
 9035   ins_pipe(ialu_reg_mem);
 9036 %}
 9037 
 9038 // Or Memory with Register
 9039 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9040 %{
 9041   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9042   effect(KILL cr);
 9043   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9044 
 9045   ins_cost(150);
 9046   format %{ "orb    $dst, $src\t# byte" %}
 9047   ins_encode %{
 9048     __ orb($dst$$Address, $src$$Register);
 9049   %}
 9050   ins_pipe(ialu_mem_reg);
 9051 %}
 9052 
 9053 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9054 %{
 9055   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9056   effect(KILL cr);
 9057   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9058 
 9059   ins_cost(150);
 9060   format %{ "orl     $dst, $src\t# int" %}
 9061   ins_encode %{
 9062     __ orl($dst$$Address, $src$$Register);
 9063   %}
 9064   ins_pipe(ialu_mem_reg);
 9065 %}
 9066 
 9067 // Or Memory with Immediate
 9068 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9069 %{
 9070   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9071   effect(KILL cr);
 9072   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9073 
 9074   ins_cost(125);
 9075   format %{ "orl     $dst, $src\t# int" %}
 9076   ins_encode %{
 9077     __ orl($dst$$Address, $src$$constant);
 9078   %}
 9079   ins_pipe(ialu_mem_imm);
 9080 %}
 9081 
 9082 // Xor Instructions
 9083 // Xor Register with Register
 9084 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9085 %{
 9086   match(Set dst (XorI dst src));
 9087   effect(KILL cr);
 9088   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9089 
 9090   format %{ "xorl    $dst, $src\t# int" %}
 9091   ins_encode %{
 9092     __ xorl($dst$$Register, $src$$Register);
 9093   %}
 9094   ins_pipe(ialu_reg_reg);
 9095 %}
 9096 
 9097 // Xor Register with Immediate -1
 9098 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9099   match(Set dst (XorI dst imm));
 9100 
 9101   format %{ "not    $dst" %}
 9102   ins_encode %{
 9103      __ notl($dst$$Register);
 9104   %}
 9105   ins_pipe(ialu_reg);
 9106 %}
 9107 
 9108 // Xor Register with Immediate
 9109 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9110 %{
 9111   match(Set dst (XorI dst src));
 9112   effect(KILL cr);
 9113   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9114 
 9115   format %{ "xorl    $dst, $src\t# int" %}
 9116   ins_encode %{
 9117     __ xorl($dst$$Register, $src$$constant);
 9118   %}
 9119   ins_pipe(ialu_reg);
 9120 %}
 9121 
 9122 // Xor Register with Memory
 9123 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9124 %{
 9125   match(Set dst (XorI dst (LoadI src)));
 9126   effect(KILL cr);
 9127   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9128 
 9129   ins_cost(150);
 9130   format %{ "xorl    $dst, $src\t# int" %}
 9131   ins_encode %{
 9132     __ xorl($dst$$Register, $src$$Address);
 9133   %}
 9134   ins_pipe(ialu_reg_mem);
 9135 %}
 9136 
 9137 // Xor Memory with Register
 9138 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9139 %{
 9140   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9141   effect(KILL cr);
 9142   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9143 
 9144   ins_cost(150);
 9145   format %{ "xorb    $dst, $src\t# byte" %}
 9146   ins_encode %{
 9147     __ xorb($dst$$Address, $src$$Register);
 9148   %}
 9149   ins_pipe(ialu_mem_reg);
 9150 %}
 9151 
 9152 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9153 %{
 9154   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9155   effect(KILL cr);
 9156   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9157 
 9158   ins_cost(150);
 9159   format %{ "xorl    $dst, $src\t# int" %}
 9160   ins_encode %{
 9161     __ xorl($dst$$Address, $src$$Register);
 9162   %}
 9163   ins_pipe(ialu_mem_reg);
 9164 %}
 9165 
 9166 // Xor Memory with Immediate
 9167 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9168 %{
 9169   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9170   effect(KILL cr);
 9171   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9172 
 9173   ins_cost(125);
 9174   format %{ "xorl    $dst, $src\t# int" %}
 9175   ins_encode %{
 9176     __ xorl($dst$$Address, $src$$constant);
 9177   %}
 9178   ins_pipe(ialu_mem_imm);
 9179 %}
 9180 
 9181 
 9182 // Long Logical Instructions
 9183 
 9184 // And Instructions
 9185 // And Register with Register
 9186 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9187 %{
 9188   match(Set dst (AndL dst src));
 9189   effect(KILL cr);
 9190   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9191 
 9192   format %{ "andq    $dst, $src\t# long" %}
 9193   ins_encode %{
 9194     __ andq($dst$$Register, $src$$Register);
 9195   %}
 9196   ins_pipe(ialu_reg_reg);
 9197 %}
 9198 
 9199 // And Register with Immediate 255
 9200 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
 9201 %{
 9202   match(Set dst (AndL src mask));
 9203 
 9204   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
 9205   ins_encode %{
 9206     // movzbl zeroes out the upper 32-bit and does not need REX.W
 9207     __ movzbl($dst$$Register, $src$$Register);
 9208   %}
 9209   ins_pipe(ialu_reg);
 9210 %}
 9211 
 9212 // And Register with Immediate 65535
 9213 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
 9214 %{
 9215   match(Set dst (AndL src mask));
 9216 
 9217   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
 9218   ins_encode %{
 9219     // movzwl zeroes out the upper 32-bit and does not need REX.W
 9220     __ movzwl($dst$$Register, $src$$Register);
 9221   %}
 9222   ins_pipe(ialu_reg);
 9223 %}
 9224 
 9225 // And Register with Immediate
 9226 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9227 %{
 9228   match(Set dst (AndL dst src));
 9229   effect(KILL cr);
 9230   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9231 
 9232   format %{ "andq    $dst, $src\t# long" %}
 9233   ins_encode %{
 9234     __ andq($dst$$Register, $src$$constant);
 9235   %}
 9236   ins_pipe(ialu_reg);
 9237 %}
 9238 
 9239 // And Register with Memory
 9240 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9241 %{
 9242   match(Set dst (AndL dst (LoadL src)));
 9243   effect(KILL cr);
 9244   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9245 
 9246   ins_cost(150);
 9247   format %{ "andq    $dst, $src\t# long" %}
 9248   ins_encode %{
 9249     __ andq($dst$$Register, $src$$Address);
 9250   %}
 9251   ins_pipe(ialu_reg_mem);
 9252 %}
 9253 
 9254 // And Memory with Register
 9255 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9256 %{
 9257   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9258   effect(KILL cr);
 9259   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9260 
 9261   ins_cost(150);
 9262   format %{ "andq    $dst, $src\t# long" %}
 9263   ins_encode %{
 9264     __ andq($dst$$Address, $src$$Register);
 9265   %}
 9266   ins_pipe(ialu_mem_reg);
 9267 %}
 9268 
 9269 // And Memory with Immediate
 9270 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9271 %{
 9272   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9273   effect(KILL cr);
 9274   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9275 
 9276   ins_cost(125);
 9277   format %{ "andq    $dst, $src\t# long" %}
 9278   ins_encode %{
 9279     __ andq($dst$$Address, $src$$constant);
 9280   %}
 9281   ins_pipe(ialu_mem_imm);
 9282 %}
 9283 
 9284 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9285 %{
 9286   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9287   // because AND/OR works well enough for 8/32-bit values.
 9288   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9289 
 9290   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9291   effect(KILL cr);
 9292 
 9293   ins_cost(125);
 9294   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9295   ins_encode %{
 9296     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9297   %}
 9298   ins_pipe(ialu_mem_imm);
 9299 %}
 9300 
 9301 // BMI1 instructions
 9302 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9303   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9304   predicate(UseBMI1Instructions);
 9305   effect(KILL cr);
 9306   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9307 
 9308   ins_cost(125);
 9309   format %{ "andnq  $dst, $src1, $src2" %}
 9310 
 9311   ins_encode %{
 9312     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9313   %}
 9314   ins_pipe(ialu_reg_mem);
 9315 %}
 9316 
 9317 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9318   match(Set dst (AndL (XorL src1 minus_1) src2));
 9319   predicate(UseBMI1Instructions);
 9320   effect(KILL cr);
 9321   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9322 
 9323   format %{ "andnq  $dst, $src1, $src2" %}
 9324 
 9325   ins_encode %{
 9326   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9327   %}
 9328   ins_pipe(ialu_reg_mem);
 9329 %}
 9330 
 9331 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9332   match(Set dst (AndL (SubL imm_zero src) src));
 9333   predicate(UseBMI1Instructions);
 9334   effect(KILL cr);
 9335   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9336 
 9337   format %{ "blsiq  $dst, $src" %}
 9338 
 9339   ins_encode %{
 9340     __ blsiq($dst$$Register, $src$$Register);
 9341   %}
 9342   ins_pipe(ialu_reg);
 9343 %}
 9344 
 9345 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9346   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9347   predicate(UseBMI1Instructions);
 9348   effect(KILL cr);
 9349   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9350 
 9351   ins_cost(125);
 9352   format %{ "blsiq  $dst, $src" %}
 9353 
 9354   ins_encode %{
 9355     __ blsiq($dst$$Register, $src$$Address);
 9356   %}
 9357   ins_pipe(ialu_reg_mem);
 9358 %}
 9359 
 9360 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9361 %{
 9362   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9363   predicate(UseBMI1Instructions);
 9364   effect(KILL cr);
 9365   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9366 
 9367   ins_cost(125);
 9368   format %{ "blsmskq $dst, $src" %}
 9369 
 9370   ins_encode %{
 9371     __ blsmskq($dst$$Register, $src$$Address);
 9372   %}
 9373   ins_pipe(ialu_reg_mem);
 9374 %}
 9375 
 9376 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9377 %{
 9378   match(Set dst (XorL (AddL src minus_1) src));
 9379   predicate(UseBMI1Instructions);
 9380   effect(KILL cr);
 9381   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9382 
 9383   format %{ "blsmskq $dst, $src" %}
 9384 
 9385   ins_encode %{
 9386     __ blsmskq($dst$$Register, $src$$Register);
 9387   %}
 9388 
 9389   ins_pipe(ialu_reg);
 9390 %}
 9391 
 9392 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9393 %{
 9394   match(Set dst (AndL (AddL src minus_1) src) );
 9395   predicate(UseBMI1Instructions);
 9396   effect(KILL cr);
 9397   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9398 
 9399   format %{ "blsrq  $dst, $src" %}
 9400 
 9401   ins_encode %{
 9402     __ blsrq($dst$$Register, $src$$Register);
 9403   %}
 9404 
 9405   ins_pipe(ialu_reg);
 9406 %}
 9407 
 9408 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9409 %{
 9410   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9411   predicate(UseBMI1Instructions);
 9412   effect(KILL cr);
 9413   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9414 
 9415   ins_cost(125);
 9416   format %{ "blsrq  $dst, $src" %}
 9417 
 9418   ins_encode %{
 9419     __ blsrq($dst$$Register, $src$$Address);
 9420   %}
 9421 
 9422   ins_pipe(ialu_reg);
 9423 %}
 9424 
 9425 // Or Instructions
 9426 // Or Register with Register
 9427 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9428 %{
 9429   match(Set dst (OrL dst src));
 9430   effect(KILL cr);
 9431   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9432 
 9433   format %{ "orq     $dst, $src\t# long" %}
 9434   ins_encode %{
 9435     __ orq($dst$$Register, $src$$Register);
 9436   %}
 9437   ins_pipe(ialu_reg_reg);
 9438 %}
 9439 
 9440 // Use any_RegP to match R15 (TLS register) without spilling.
 9441 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 9442   match(Set dst (OrL dst (CastP2X src)));
 9443   effect(KILL cr);
 9444   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9445 
 9446   format %{ "orq     $dst, $src\t# long" %}
 9447   ins_encode %{
 9448     __ orq($dst$$Register, $src$$Register);
 9449   %}
 9450   ins_pipe(ialu_reg_reg);
 9451 %}
 9452 
 9453 
 9454 // Or Register with Immediate
 9455 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9456 %{
 9457   match(Set dst (OrL dst src));
 9458   effect(KILL cr);
 9459   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9460 
 9461   format %{ "orq     $dst, $src\t# long" %}
 9462   ins_encode %{
 9463     __ orq($dst$$Register, $src$$constant);
 9464   %}
 9465   ins_pipe(ialu_reg);
 9466 %}
 9467 
 9468 // Or Register with Memory
 9469 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9470 %{
 9471   match(Set dst (OrL dst (LoadL src)));
 9472   effect(KILL cr);
 9473   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9474 
 9475   ins_cost(150);
 9476   format %{ "orq     $dst, $src\t# long" %}
 9477   ins_encode %{
 9478     __ orq($dst$$Register, $src$$Address);
 9479   %}
 9480   ins_pipe(ialu_reg_mem);
 9481 %}
 9482 
 9483 // Or Memory with Register
 9484 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9485 %{
 9486   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9487   effect(KILL cr);
 9488   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9489 
 9490   ins_cost(150);
 9491   format %{ "orq     $dst, $src\t# long" %}
 9492   ins_encode %{
 9493     __ orq($dst$$Address, $src$$Register);
 9494   %}
 9495   ins_pipe(ialu_mem_reg);
 9496 %}
 9497 
 9498 // Or Memory with Immediate
 9499 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9500 %{
 9501   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9502   effect(KILL cr);
 9503   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9504 
 9505   ins_cost(125);
 9506   format %{ "orq     $dst, $src\t# long" %}
 9507   ins_encode %{
 9508     __ orq($dst$$Address, $src$$constant);
 9509   %}
 9510   ins_pipe(ialu_mem_imm);
 9511 %}
 9512 
 9513 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
 9514 %{
 9515   // con should be a pure 64-bit power of 2 immediate
 9516   // because AND/OR works well enough for 8/32-bit values.
 9517   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
 9518 
 9519   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
 9520   effect(KILL cr);
 9521 
 9522   ins_cost(125);
 9523   format %{ "btsq    $dst, log2($con)\t# long" %}
 9524   ins_encode %{
 9525     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
 9526   %}
 9527   ins_pipe(ialu_mem_imm);
 9528 %}
 9529 
 9530 // Xor Instructions
 9531 // Xor Register with Register
 9532 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9533 %{
 9534   match(Set dst (XorL dst src));
 9535   effect(KILL cr);
 9536   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9537 
 9538   format %{ "xorq    $dst, $src\t# long" %}
 9539   ins_encode %{
 9540     __ xorq($dst$$Register, $src$$Register);
 9541   %}
 9542   ins_pipe(ialu_reg_reg);
 9543 %}
 9544 
 9545 // Xor Register with Immediate -1
 9546 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 9547   match(Set dst (XorL dst imm));
 9548 
 9549   format %{ "notq   $dst" %}
 9550   ins_encode %{
 9551      __ notq($dst$$Register);
 9552   %}
 9553   ins_pipe(ialu_reg);
 9554 %}
 9555 
 9556 // Xor Register with Immediate
 9557 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9558 %{
 9559   match(Set dst (XorL dst src));
 9560   effect(KILL cr);
 9561   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9562 
 9563   format %{ "xorq    $dst, $src\t# long" %}
 9564   ins_encode %{
 9565     __ xorq($dst$$Register, $src$$constant);
 9566   %}
 9567   ins_pipe(ialu_reg);
 9568 %}
 9569 
 9570 // Xor Register with Memory
 9571 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9572 %{
 9573   match(Set dst (XorL dst (LoadL src)));
 9574   effect(KILL cr);
 9575   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9576 
 9577   ins_cost(150);
 9578   format %{ "xorq    $dst, $src\t# long" %}
 9579   ins_encode %{
 9580     __ xorq($dst$$Register, $src$$Address);
 9581   %}
 9582   ins_pipe(ialu_reg_mem);
 9583 %}
 9584 
 9585 // Xor Memory with Register
 9586 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9587 %{
 9588   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9589   effect(KILL cr);
 9590   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9591 
 9592   ins_cost(150);
 9593   format %{ "xorq    $dst, $src\t# long" %}
 9594   ins_encode %{
 9595     __ xorq($dst$$Address, $src$$Register);
 9596   %}
 9597   ins_pipe(ialu_mem_reg);
 9598 %}
 9599 
 9600 // Xor Memory with Immediate
 9601 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9602 %{
 9603   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9604   effect(KILL cr);
 9605   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9606 
 9607   ins_cost(125);
 9608   format %{ "xorq    $dst, $src\t# long" %}
 9609   ins_encode %{
 9610     __ xorq($dst$$Address, $src$$constant);
 9611   %}
 9612   ins_pipe(ialu_mem_imm);
 9613 %}
 9614 
 9615 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 9616 %{
 9617   match(Set dst (CmpLTMask p q));
 9618   effect(KILL cr);
 9619 
 9620   ins_cost(400);
 9621   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 9622             "setlt   $dst\n\t"
 9623             "movzbl  $dst, $dst\n\t"
 9624             "negl    $dst" %}
 9625   ins_encode %{
 9626     __ cmpl($p$$Register, $q$$Register);
 9627     __ setb(Assembler::less, $dst$$Register);
 9628     __ movzbl($dst$$Register, $dst$$Register);
 9629     __ negl($dst$$Register);
 9630   %}
 9631   ins_pipe(pipe_slow);
 9632 %}
 9633 
 9634 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
 9635 %{
 9636   match(Set dst (CmpLTMask dst zero));
 9637   effect(KILL cr);
 9638 
 9639   ins_cost(100);
 9640   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 9641   ins_encode %{
 9642     __ sarl($dst$$Register, 31);
 9643   %}
 9644   ins_pipe(ialu_reg);
 9645 %}
 9646 
 9647 /* Better to save a register than avoid a branch */
 9648 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9649 %{
 9650   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 9651   effect(KILL cr);
 9652   ins_cost(300);
 9653   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
 9654             "jge     done\n\t"
 9655             "addl    $p,$y\n"
 9656             "done:   " %}
 9657   ins_encode %{
 9658     Register Rp = $p$$Register;
 9659     Register Rq = $q$$Register;
 9660     Register Ry = $y$$Register;
 9661     Label done;
 9662     __ subl(Rp, Rq);
 9663     __ jccb(Assembler::greaterEqual, done);
 9664     __ addl(Rp, Ry);
 9665     __ bind(done);
 9666   %}
 9667   ins_pipe(pipe_cmplt);
 9668 %}
 9669 
 9670 /* Better to save a register than avoid a branch */
 9671 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9672 %{
 9673   match(Set y (AndI (CmpLTMask p q) y));
 9674   effect(KILL cr);
 9675 
 9676   ins_cost(300);
 9677 
 9678   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
 9679             "jlt     done\n\t"
 9680             "xorl    $y, $y\n"
 9681             "done:   " %}
 9682   ins_encode %{
 9683     Register Rp = $p$$Register;
 9684     Register Rq = $q$$Register;
 9685     Register Ry = $y$$Register;
 9686     Label done;
 9687     __ cmpl(Rp, Rq);
 9688     __ jccb(Assembler::less, done);
 9689     __ xorl(Ry, Ry);
 9690     __ bind(done);
 9691   %}
 9692   ins_pipe(pipe_cmplt);
 9693 %}
 9694 
 9695 
 9696 //---------- FP Instructions------------------------------------------------
 9697 
 9698 // Really expensive, avoid
 9699 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 9700 %{
 9701   match(Set cr (CmpF src1 src2));
 9702 
 9703   ins_cost(500);
 9704   format %{ "ucomiss $src1, $src2\n\t"
 9705             "jnp,s   exit\n\t"
 9706             "pushfq\t# saw NaN, set CF\n\t"
 9707             "andq    [rsp], #0xffffff2b\n\t"
 9708             "popfq\n"
 9709     "exit:" %}
 9710   ins_encode %{
 9711     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9712     emit_cmpfp_fixup(masm);
 9713   %}
 9714   ins_pipe(pipe_slow);
 9715 %}
 9716 
 9717 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 9718   match(Set cr (CmpF src1 src2));
 9719 
 9720   ins_cost(100);
 9721   format %{ "ucomiss $src1, $src2" %}
 9722   ins_encode %{
 9723     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9724   %}
 9725   ins_pipe(pipe_slow);
 9726 %}
 9727 
 9728 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 9729   match(Set cr (CmpF src1 (LoadF src2)));
 9730 
 9731   ins_cost(100);
 9732   format %{ "ucomiss $src1, $src2" %}
 9733   ins_encode %{
 9734     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9735   %}
 9736   ins_pipe(pipe_slow);
 9737 %}
 9738 
 9739 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
 9740   match(Set cr (CmpF src con));
 9741   ins_cost(100);
 9742   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
 9743   ins_encode %{
 9744     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9745   %}
 9746   ins_pipe(pipe_slow);
 9747 %}
 9748 
 9749 // Really expensive, avoid
 9750 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 9751 %{
 9752   match(Set cr (CmpD src1 src2));
 9753 
 9754   ins_cost(500);
 9755   format %{ "ucomisd $src1, $src2\n\t"
 9756             "jnp,s   exit\n\t"
 9757             "pushfq\t# saw NaN, set CF\n\t"
 9758             "andq    [rsp], #0xffffff2b\n\t"
 9759             "popfq\n"
 9760     "exit:" %}
 9761   ins_encode %{
 9762     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9763     emit_cmpfp_fixup(masm);
 9764   %}
 9765   ins_pipe(pipe_slow);
 9766 %}
 9767 
 9768 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 9769   match(Set cr (CmpD src1 src2));
 9770 
 9771   ins_cost(100);
 9772   format %{ "ucomisd $src1, $src2 test" %}
 9773   ins_encode %{
 9774     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9775   %}
 9776   ins_pipe(pipe_slow);
 9777 %}
 9778 
 9779 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 9780   match(Set cr (CmpD src1 (LoadD src2)));
 9781 
 9782   ins_cost(100);
 9783   format %{ "ucomisd $src1, $src2" %}
 9784   ins_encode %{
 9785     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9786   %}
 9787   ins_pipe(pipe_slow);
 9788 %}
 9789 
 9790 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
 9791   match(Set cr (CmpD src con));
 9792   ins_cost(100);
 9793   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
 9794   ins_encode %{
 9795     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9796   %}
 9797   ins_pipe(pipe_slow);
 9798 %}
 9799 
 9800 // Compare into -1,0,1
 9801 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
 9802 %{
 9803   match(Set dst (CmpF3 src1 src2));
 9804   effect(KILL cr);
 9805 
 9806   ins_cost(275);
 9807   format %{ "ucomiss $src1, $src2\n\t"
 9808             "movl    $dst, #-1\n\t"
 9809             "jp,s    done\n\t"
 9810             "jb,s    done\n\t"
 9811             "setne   $dst\n\t"
 9812             "movzbl  $dst, $dst\n"
 9813     "done:" %}
 9814   ins_encode %{
 9815     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9816     emit_cmpfp3(masm, $dst$$Register);
 9817   %}
 9818   ins_pipe(pipe_slow);
 9819 %}
 9820 
 9821 // Compare into -1,0,1
 9822 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 9823 %{
 9824   match(Set dst (CmpF3 src1 (LoadF src2)));
 9825   effect(KILL cr);
 9826 
 9827   ins_cost(275);
 9828   format %{ "ucomiss $src1, $src2\n\t"
 9829             "movl    $dst, #-1\n\t"
 9830             "jp,s    done\n\t"
 9831             "jb,s    done\n\t"
 9832             "setne   $dst\n\t"
 9833             "movzbl  $dst, $dst\n"
 9834     "done:" %}
 9835   ins_encode %{
 9836     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9837     emit_cmpfp3(masm, $dst$$Register);
 9838   %}
 9839   ins_pipe(pipe_slow);
 9840 %}
 9841 
 9842 // Compare into -1,0,1
 9843 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
 9844   match(Set dst (CmpF3 src con));
 9845   effect(KILL cr);
 9846 
 9847   ins_cost(275);
 9848   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
 9849             "movl    $dst, #-1\n\t"
 9850             "jp,s    done\n\t"
 9851             "jb,s    done\n\t"
 9852             "setne   $dst\n\t"
 9853             "movzbl  $dst, $dst\n"
 9854     "done:" %}
 9855   ins_encode %{
 9856     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9857     emit_cmpfp3(masm, $dst$$Register);
 9858   %}
 9859   ins_pipe(pipe_slow);
 9860 %}
 9861 
 9862 // Compare into -1,0,1
 9863 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
 9864 %{
 9865   match(Set dst (CmpD3 src1 src2));
 9866   effect(KILL cr);
 9867 
 9868   ins_cost(275);
 9869   format %{ "ucomisd $src1, $src2\n\t"
 9870             "movl    $dst, #-1\n\t"
 9871             "jp,s    done\n\t"
 9872             "jb,s    done\n\t"
 9873             "setne   $dst\n\t"
 9874             "movzbl  $dst, $dst\n"
 9875     "done:" %}
 9876   ins_encode %{
 9877     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9878     emit_cmpfp3(masm, $dst$$Register);
 9879   %}
 9880   ins_pipe(pipe_slow);
 9881 %}
 9882 
 9883 // Compare into -1,0,1
 9884 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
 9885 %{
 9886   match(Set dst (CmpD3 src1 (LoadD src2)));
 9887   effect(KILL cr);
 9888 
 9889   ins_cost(275);
 9890   format %{ "ucomisd $src1, $src2\n\t"
 9891             "movl    $dst, #-1\n\t"
 9892             "jp,s    done\n\t"
 9893             "jb,s    done\n\t"
 9894             "setne   $dst\n\t"
 9895             "movzbl  $dst, $dst\n"
 9896     "done:" %}
 9897   ins_encode %{
 9898     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9899     emit_cmpfp3(masm, $dst$$Register);
 9900   %}
 9901   ins_pipe(pipe_slow);
 9902 %}
 9903 
 9904 // Compare into -1,0,1
 9905 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
 9906   match(Set dst (CmpD3 src con));
 9907   effect(KILL cr);
 9908 
 9909   ins_cost(275);
 9910   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
 9911             "movl    $dst, #-1\n\t"
 9912             "jp,s    done\n\t"
 9913             "jb,s    done\n\t"
 9914             "setne   $dst\n\t"
 9915             "movzbl  $dst, $dst\n"
 9916     "done:" %}
 9917   ins_encode %{
 9918     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9919     emit_cmpfp3(masm, $dst$$Register);
 9920   %}
 9921   ins_pipe(pipe_slow);
 9922 %}
 9923 
 9924 //----------Arithmetic Conversion Instructions---------------------------------
 9925 
 9926 instruct convF2D_reg_reg(regD dst, regF src)
 9927 %{
 9928   match(Set dst (ConvF2D src));
 9929 
 9930   format %{ "cvtss2sd $dst, $src" %}
 9931   ins_encode %{
 9932     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
 9933   %}
 9934   ins_pipe(pipe_slow); // XXX
 9935 %}
 9936 
 9937 instruct convF2D_reg_mem(regD dst, memory src)
 9938 %{
 9939   predicate(UseAVX == 0);
 9940   match(Set dst (ConvF2D (LoadF src)));
 9941 
 9942   format %{ "cvtss2sd $dst, $src" %}
 9943   ins_encode %{
 9944     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
 9945   %}
 9946   ins_pipe(pipe_slow); // XXX
 9947 %}
 9948 
 9949 instruct convD2F_reg_reg(regF dst, regD src)
 9950 %{
 9951   match(Set dst (ConvD2F src));
 9952 
 9953   format %{ "cvtsd2ss $dst, $src" %}
 9954   ins_encode %{
 9955     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
 9956   %}
 9957   ins_pipe(pipe_slow); // XXX
 9958 %}
 9959 
 9960 instruct convD2F_reg_mem(regF dst, memory src)
 9961 %{
 9962   predicate(UseAVX == 0);
 9963   match(Set dst (ConvD2F (LoadD src)));
 9964 
 9965   format %{ "cvtsd2ss $dst, $src" %}
 9966   ins_encode %{
 9967     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
 9968   %}
 9969   ins_pipe(pipe_slow); // XXX
 9970 %}
 9971 
 9972 // XXX do mem variants
 9973 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
 9974 %{
 9975   match(Set dst (ConvF2I src));
 9976   effect(KILL cr);
 9977   format %{ "convert_f2i $dst, $src" %}
 9978   ins_encode %{
 9979     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
 9980   %}
 9981   ins_pipe(pipe_slow);
 9982 %}
 9983 
 9984 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
 9985 %{
 9986   match(Set dst (ConvF2L src));
 9987   effect(KILL cr);
 9988   format %{ "convert_f2l $dst, $src"%}
 9989   ins_encode %{
 9990     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
 9991   %}
 9992   ins_pipe(pipe_slow);
 9993 %}
 9994 
 9995 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
 9996 %{
 9997   match(Set dst (ConvD2I src));
 9998   effect(KILL cr);
 9999   format %{ "convert_d2i $dst, $src"%}
10000   ins_encode %{
10001     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10002   %}
10003   ins_pipe(pipe_slow);
10004 %}
10005 
10006 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10007 %{
10008   match(Set dst (ConvD2L src));
10009   effect(KILL cr);
10010   format %{ "convert_d2l $dst, $src"%}
10011   ins_encode %{
10012     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10013   %}
10014   ins_pipe(pipe_slow);
10015 %}
10016 
10017 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10018 %{
10019   match(Set dst (RoundD src));
10020   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10021   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10022   ins_encode %{
10023     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10024   %}
10025   ins_pipe(pipe_slow);
10026 %}
10027 
10028 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10029 %{
10030   match(Set dst (RoundF src));
10031   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10032   format %{ "round_float $dst,$src" %}
10033   ins_encode %{
10034     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10035   %}
10036   ins_pipe(pipe_slow);
10037 %}
10038 
10039 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
10040 %{
10041   predicate(!UseXmmI2F);
10042   match(Set dst (ConvI2F src));
10043 
10044   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10045   ins_encode %{
10046     if (UseAVX > 0) {
10047       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10048     }
10049     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10050   %}
10051   ins_pipe(pipe_slow); // XXX
10052 %}
10053 
10054 instruct convI2F_reg_mem(regF dst, memory src)
10055 %{
10056   predicate(UseAVX == 0);
10057   match(Set dst (ConvI2F (LoadI src)));
10058 
10059   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10060   ins_encode %{
10061     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10062   %}
10063   ins_pipe(pipe_slow); // XXX
10064 %}
10065 
10066 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
10067 %{
10068   predicate(!UseXmmI2D);
10069   match(Set dst (ConvI2D src));
10070 
10071   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10072   ins_encode %{
10073     if (UseAVX > 0) {
10074       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10075     }
10076     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10077   %}
10078   ins_pipe(pipe_slow); // XXX
10079 %}
10080 
10081 instruct convI2D_reg_mem(regD dst, memory src)
10082 %{
10083   predicate(UseAVX == 0);
10084   match(Set dst (ConvI2D (LoadI src)));
10085 
10086   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10087   ins_encode %{
10088     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10089   %}
10090   ins_pipe(pipe_slow); // XXX
10091 %}
10092 
10093 instruct convXI2F_reg(regF dst, rRegI src)
10094 %{
10095   predicate(UseXmmI2F);
10096   match(Set dst (ConvI2F src));
10097 
10098   format %{ "movdl $dst, $src\n\t"
10099             "cvtdq2psl $dst, $dst\t# i2f" %}
10100   ins_encode %{
10101     __ movdl($dst$$XMMRegister, $src$$Register);
10102     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10103   %}
10104   ins_pipe(pipe_slow); // XXX
10105 %}
10106 
10107 instruct convXI2D_reg(regD dst, rRegI src)
10108 %{
10109   predicate(UseXmmI2D);
10110   match(Set dst (ConvI2D src));
10111 
10112   format %{ "movdl $dst, $src\n\t"
10113             "cvtdq2pdl $dst, $dst\t# i2d" %}
10114   ins_encode %{
10115     __ movdl($dst$$XMMRegister, $src$$Register);
10116     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10117   %}
10118   ins_pipe(pipe_slow); // XXX
10119 %}
10120 
10121 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
10122 %{
10123   match(Set dst (ConvL2F src));
10124 
10125   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10126   ins_encode %{
10127     if (UseAVX > 0) {
10128       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10129     }
10130     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10131   %}
10132   ins_pipe(pipe_slow); // XXX
10133 %}
10134 
10135 instruct convL2F_reg_mem(regF dst, memory src)
10136 %{
10137   predicate(UseAVX == 0);
10138   match(Set dst (ConvL2F (LoadL src)));
10139 
10140   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10141   ins_encode %{
10142     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10143   %}
10144   ins_pipe(pipe_slow); // XXX
10145 %}
10146 
10147 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
10148 %{
10149   match(Set dst (ConvL2D src));
10150 
10151   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10152   ins_encode %{
10153     if (UseAVX > 0) {
10154       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10155     }
10156     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10157   %}
10158   ins_pipe(pipe_slow); // XXX
10159 %}
10160 
10161 instruct convL2D_reg_mem(regD dst, memory src)
10162 %{
10163   predicate(UseAVX == 0);
10164   match(Set dst (ConvL2D (LoadL src)));
10165 
10166   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10167   ins_encode %{
10168     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10169   %}
10170   ins_pipe(pipe_slow); // XXX
10171 %}
10172 
10173 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10174 %{
10175   match(Set dst (ConvI2L src));
10176 
10177   ins_cost(125);
10178   format %{ "movslq  $dst, $src\t# i2l" %}
10179   ins_encode %{
10180     __ movslq($dst$$Register, $src$$Register);
10181   %}
10182   ins_pipe(ialu_reg_reg);
10183 %}
10184 
10185 // Zero-extend convert int to long
10186 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10187 %{
10188   match(Set dst (AndL (ConvI2L src) mask));
10189 
10190   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10191   ins_encode %{
10192     if ($dst$$reg != $src$$reg) {
10193       __ movl($dst$$Register, $src$$Register);
10194     }
10195   %}
10196   ins_pipe(ialu_reg_reg);
10197 %}
10198 
10199 // Zero-extend convert int to long
10200 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10201 %{
10202   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10203 
10204   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10205   ins_encode %{
10206     __ movl($dst$$Register, $src$$Address);
10207   %}
10208   ins_pipe(ialu_reg_mem);
10209 %}
10210 
10211 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10212 %{
10213   match(Set dst (AndL src mask));
10214 
10215   format %{ "movl    $dst, $src\t# zero-extend long" %}
10216   ins_encode %{
10217     __ movl($dst$$Register, $src$$Register);
10218   %}
10219   ins_pipe(ialu_reg_reg);
10220 %}
10221 
10222 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10223 %{
10224   match(Set dst (ConvL2I src));
10225 
10226   format %{ "movl    $dst, $src\t# l2i" %}
10227   ins_encode %{
10228     __ movl($dst$$Register, $src$$Register);
10229   %}
10230   ins_pipe(ialu_reg_reg);
10231 %}
10232 
10233 
10234 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10235   match(Set dst (MoveF2I src));
10236   effect(DEF dst, USE src);
10237 
10238   ins_cost(125);
10239   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10240   ins_encode %{
10241     __ movl($dst$$Register, Address(rsp, $src$$disp));
10242   %}
10243   ins_pipe(ialu_reg_mem);
10244 %}
10245 
10246 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10247   match(Set dst (MoveI2F src));
10248   effect(DEF dst, USE src);
10249 
10250   ins_cost(125);
10251   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10252   ins_encode %{
10253     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10254   %}
10255   ins_pipe(pipe_slow);
10256 %}
10257 
10258 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10259   match(Set dst (MoveD2L src));
10260   effect(DEF dst, USE src);
10261 
10262   ins_cost(125);
10263   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10264   ins_encode %{
10265     __ movq($dst$$Register, Address(rsp, $src$$disp));
10266   %}
10267   ins_pipe(ialu_reg_mem);
10268 %}
10269 
10270 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10271   predicate(!UseXmmLoadAndClearUpper);
10272   match(Set dst (MoveL2D src));
10273   effect(DEF dst, USE src);
10274 
10275   ins_cost(125);
10276   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10277   ins_encode %{
10278     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10279   %}
10280   ins_pipe(pipe_slow);
10281 %}
10282 
10283 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10284   predicate(UseXmmLoadAndClearUpper);
10285   match(Set dst (MoveL2D src));
10286   effect(DEF dst, USE src);
10287 
10288   ins_cost(125);
10289   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10290   ins_encode %{
10291     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10292   %}
10293   ins_pipe(pipe_slow);
10294 %}
10295 
10296 
10297 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10298   match(Set dst (MoveF2I src));
10299   effect(DEF dst, USE src);
10300 
10301   ins_cost(95); // XXX
10302   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10303   ins_encode %{
10304     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10305   %}
10306   ins_pipe(pipe_slow);
10307 %}
10308 
10309 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10310   match(Set dst (MoveI2F src));
10311   effect(DEF dst, USE src);
10312 
10313   ins_cost(100);
10314   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10315   ins_encode %{
10316     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10317   %}
10318   ins_pipe( ialu_mem_reg );
10319 %}
10320 
10321 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10322   match(Set dst (MoveD2L src));
10323   effect(DEF dst, USE src);
10324 
10325   ins_cost(95); // XXX
10326   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10327   ins_encode %{
10328     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10329   %}
10330   ins_pipe(pipe_slow);
10331 %}
10332 
10333 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10334   match(Set dst (MoveL2D src));
10335   effect(DEF dst, USE src);
10336 
10337   ins_cost(100);
10338   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10339   ins_encode %{
10340     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10341   %}
10342   ins_pipe(ialu_mem_reg);
10343 %}
10344 
10345 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10346   match(Set dst (MoveF2I src));
10347   effect(DEF dst, USE src);
10348   ins_cost(85);
10349   format %{ "movd    $dst,$src\t# MoveF2I" %}
10350   ins_encode %{
10351     __ movdl($dst$$Register, $src$$XMMRegister);
10352   %}
10353   ins_pipe( pipe_slow );
10354 %}
10355 
10356 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10357   match(Set dst (MoveD2L src));
10358   effect(DEF dst, USE src);
10359   ins_cost(85);
10360   format %{ "movd    $dst,$src\t# MoveD2L" %}
10361   ins_encode %{
10362     __ movdq($dst$$Register, $src$$XMMRegister);
10363   %}
10364   ins_pipe( pipe_slow );
10365 %}
10366 
10367 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10368   match(Set dst (MoveI2F src));
10369   effect(DEF dst, USE src);
10370   ins_cost(100);
10371   format %{ "movd    $dst,$src\t# MoveI2F" %}
10372   ins_encode %{
10373     __ movdl($dst$$XMMRegister, $src$$Register);
10374   %}
10375   ins_pipe( pipe_slow );
10376 %}
10377 
10378 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10379   match(Set dst (MoveL2D src));
10380   effect(DEF dst, USE src);
10381   ins_cost(100);
10382   format %{ "movd    $dst,$src\t# MoveL2D" %}
10383   ins_encode %{
10384      __ movdq($dst$$XMMRegister, $src$$Register);
10385   %}
10386   ins_pipe( pipe_slow );
10387 %}
10388 
10389 // Fast clearing of an array
10390 // Small non-constant lenght ClearArray for non-AVX512 targets.
10391 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10392                   Universe dummy, rFlagsReg cr)
10393 %{
10394   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
10395   match(Set dummy (ClearArray cnt base));
10396   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10397 
10398   format %{ $$template
10399     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10400     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10401     $$emit$$"jg      LARGE\n\t"
10402     $$emit$$"dec     rcx\n\t"
10403     $$emit$$"js      DONE\t# Zero length\n\t"
10404     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10405     $$emit$$"dec     rcx\n\t"
10406     $$emit$$"jge     LOOP\n\t"
10407     $$emit$$"jmp     DONE\n\t"
10408     $$emit$$"# LARGE:\n\t"
10409     if (UseFastStosb) {
10410        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10411        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10412     } else if (UseXMMForObjInit) {
10413        $$emit$$"mov     rdi,rax\n\t"
10414        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10415        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10416        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10417        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10418        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10419        $$emit$$"add     0x40,rax\n\t"
10420        $$emit$$"# L_zero_64_bytes:\n\t"
10421        $$emit$$"sub     0x8,rcx\n\t"
10422        $$emit$$"jge     L_loop\n\t"
10423        $$emit$$"add     0x4,rcx\n\t"
10424        $$emit$$"jl      L_tail\n\t"
10425        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10426        $$emit$$"add     0x20,rax\n\t"
10427        $$emit$$"sub     0x4,rcx\n\t"
10428        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10429        $$emit$$"add     0x4,rcx\n\t"
10430        $$emit$$"jle     L_end\n\t"
10431        $$emit$$"dec     rcx\n\t"
10432        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10433        $$emit$$"vmovq   xmm0,(rax)\n\t"
10434        $$emit$$"add     0x8,rax\n\t"
10435        $$emit$$"dec     rcx\n\t"
10436        $$emit$$"jge     L_sloop\n\t"
10437        $$emit$$"# L_end:\n\t"
10438     } else {
10439        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10440     }
10441     $$emit$$"# DONE"
10442   %}
10443   ins_encode %{
10444     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10445                  $tmp$$XMMRegister, false, knoreg);
10446   %}
10447   ins_pipe(pipe_slow);
10448 %}
10449 
10450 // Small non-constant length ClearArray for AVX512 targets.
10451 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10452                        Universe dummy, rFlagsReg cr)
10453 %{
10454   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
10455   match(Set dummy (ClearArray cnt base));
10456   ins_cost(125);
10457   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10458 
10459   format %{ $$template
10460     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10461     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10462     $$emit$$"jg      LARGE\n\t"
10463     $$emit$$"dec     rcx\n\t"
10464     $$emit$$"js      DONE\t# Zero length\n\t"
10465     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10466     $$emit$$"dec     rcx\n\t"
10467     $$emit$$"jge     LOOP\n\t"
10468     $$emit$$"jmp     DONE\n\t"
10469     $$emit$$"# LARGE:\n\t"
10470     if (UseFastStosb) {
10471        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10472        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10473     } else if (UseXMMForObjInit) {
10474        $$emit$$"mov     rdi,rax\n\t"
10475        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10476        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10477        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10478        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10479        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10480        $$emit$$"add     0x40,rax\n\t"
10481        $$emit$$"# L_zero_64_bytes:\n\t"
10482        $$emit$$"sub     0x8,rcx\n\t"
10483        $$emit$$"jge     L_loop\n\t"
10484        $$emit$$"add     0x4,rcx\n\t"
10485        $$emit$$"jl      L_tail\n\t"
10486        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10487        $$emit$$"add     0x20,rax\n\t"
10488        $$emit$$"sub     0x4,rcx\n\t"
10489        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10490        $$emit$$"add     0x4,rcx\n\t"
10491        $$emit$$"jle     L_end\n\t"
10492        $$emit$$"dec     rcx\n\t"
10493        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10494        $$emit$$"vmovq   xmm0,(rax)\n\t"
10495        $$emit$$"add     0x8,rax\n\t"
10496        $$emit$$"dec     rcx\n\t"
10497        $$emit$$"jge     L_sloop\n\t"
10498        $$emit$$"# L_end:\n\t"
10499     } else {
10500        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10501     }
10502     $$emit$$"# DONE"
10503   %}
10504   ins_encode %{
10505     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10506                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
10507   %}
10508   ins_pipe(pipe_slow);
10509 %}
10510 
10511 // Large non-constant length ClearArray for non-AVX512 targets.
10512 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10513                         Universe dummy, rFlagsReg cr)
10514 %{
10515   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
10516   match(Set dummy (ClearArray cnt base));
10517   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10518 
10519   format %{ $$template
10520     if (UseFastStosb) {
10521        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10522        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10523        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10524     } else if (UseXMMForObjInit) {
10525        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10526        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10527        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10528        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10529        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10530        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10531        $$emit$$"add     0x40,rax\n\t"
10532        $$emit$$"# L_zero_64_bytes:\n\t"
10533        $$emit$$"sub     0x8,rcx\n\t"
10534        $$emit$$"jge     L_loop\n\t"
10535        $$emit$$"add     0x4,rcx\n\t"
10536        $$emit$$"jl      L_tail\n\t"
10537        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10538        $$emit$$"add     0x20,rax\n\t"
10539        $$emit$$"sub     0x4,rcx\n\t"
10540        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10541        $$emit$$"add     0x4,rcx\n\t"
10542        $$emit$$"jle     L_end\n\t"
10543        $$emit$$"dec     rcx\n\t"
10544        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10545        $$emit$$"vmovq   xmm0,(rax)\n\t"
10546        $$emit$$"add     0x8,rax\n\t"
10547        $$emit$$"dec     rcx\n\t"
10548        $$emit$$"jge     L_sloop\n\t"
10549        $$emit$$"# L_end:\n\t"
10550     } else {
10551        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10552        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10553     }
10554   %}
10555   ins_encode %{
10556     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10557                  $tmp$$XMMRegister, true, knoreg);
10558   %}
10559   ins_pipe(pipe_slow);
10560 %}
10561 
10562 // Large non-constant length ClearArray for AVX512 targets.
10563 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10564                              Universe dummy, rFlagsReg cr)
10565 %{
10566   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
10567   match(Set dummy (ClearArray cnt base));
10568   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10569 
10570   format %{ $$template
10571     if (UseFastStosb) {
10572        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10573        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10574        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10575     } else if (UseXMMForObjInit) {
10576        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10577        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10578        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10579        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10580        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10581        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10582        $$emit$$"add     0x40,rax\n\t"
10583        $$emit$$"# L_zero_64_bytes:\n\t"
10584        $$emit$$"sub     0x8,rcx\n\t"
10585        $$emit$$"jge     L_loop\n\t"
10586        $$emit$$"add     0x4,rcx\n\t"
10587        $$emit$$"jl      L_tail\n\t"
10588        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10589        $$emit$$"add     0x20,rax\n\t"
10590        $$emit$$"sub     0x4,rcx\n\t"
10591        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10592        $$emit$$"add     0x4,rcx\n\t"
10593        $$emit$$"jle     L_end\n\t"
10594        $$emit$$"dec     rcx\n\t"
10595        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10596        $$emit$$"vmovq   xmm0,(rax)\n\t"
10597        $$emit$$"add     0x8,rax\n\t"
10598        $$emit$$"dec     rcx\n\t"
10599        $$emit$$"jge     L_sloop\n\t"
10600        $$emit$$"# L_end:\n\t"
10601     } else {
10602        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10603        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10604     }
10605   %}
10606   ins_encode %{
10607     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10608                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
10609   %}
10610   ins_pipe(pipe_slow);
10611 %}
10612 
10613 // Small constant length ClearArray for AVX512 targets.
10614 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
10615 %{
10616   predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
10617   match(Set dummy (ClearArray cnt base));
10618   ins_cost(100);
10619   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
10620   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
10621   ins_encode %{
10622    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10623   %}
10624   ins_pipe(pipe_slow);
10625 %}
10626 
10627 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10628                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10629 %{
10630   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10631   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10632   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10633 
10634   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10635   ins_encode %{
10636     __ string_compare($str1$$Register, $str2$$Register,
10637                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10638                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
10639   %}
10640   ins_pipe( pipe_slow );
10641 %}
10642 
10643 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10644                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10645 %{
10646   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10647   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10648   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10649 
10650   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10651   ins_encode %{
10652     __ string_compare($str1$$Register, $str2$$Register,
10653                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10654                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
10655   %}
10656   ins_pipe( pipe_slow );
10657 %}
10658 
10659 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10660                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10661 %{
10662   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10663   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10664   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10665 
10666   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10667   ins_encode %{
10668     __ string_compare($str1$$Register, $str2$$Register,
10669                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10670                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
10671   %}
10672   ins_pipe( pipe_slow );
10673 %}
10674 
10675 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10676                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10677 %{
10678   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10679   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10680   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10681 
10682   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10683   ins_encode %{
10684     __ string_compare($str1$$Register, $str2$$Register,
10685                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10686                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
10687   %}
10688   ins_pipe( pipe_slow );
10689 %}
10690 
10691 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10692                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10693 %{
10694   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10695   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10696   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10697 
10698   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10699   ins_encode %{
10700     __ string_compare($str1$$Register, $str2$$Register,
10701                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10702                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
10703   %}
10704   ins_pipe( pipe_slow );
10705 %}
10706 
10707 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10708                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10709 %{
10710   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10711   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10712   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10713 
10714   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10715   ins_encode %{
10716     __ string_compare($str1$$Register, $str2$$Register,
10717                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10718                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
10719   %}
10720   ins_pipe( pipe_slow );
10721 %}
10722 
10723 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10724                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10725 %{
10726   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10727   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10728   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10729 
10730   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10731   ins_encode %{
10732     __ string_compare($str2$$Register, $str1$$Register,
10733                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10734                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
10735   %}
10736   ins_pipe( pipe_slow );
10737 %}
10738 
10739 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10740                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10741 %{
10742   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10743   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10744   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10745 
10746   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10747   ins_encode %{
10748     __ string_compare($str2$$Register, $str1$$Register,
10749                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10750                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
10751   %}
10752   ins_pipe( pipe_slow );
10753 %}
10754 
10755 // fast search of substring with known size.
10756 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10757                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10758 %{
10759   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10760   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10761   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10762 
10763   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10764   ins_encode %{
10765     int icnt2 = (int)$int_cnt2$$constant;
10766     if (icnt2 >= 16) {
10767       // IndexOf for constant substrings with size >= 16 elements
10768       // which don't need to be loaded through stack.
10769       __ string_indexofC8($str1$$Register, $str2$$Register,
10770                           $cnt1$$Register, $cnt2$$Register,
10771                           icnt2, $result$$Register,
10772                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10773     } else {
10774       // Small strings are loaded through stack if they cross page boundary.
10775       __ string_indexof($str1$$Register, $str2$$Register,
10776                         $cnt1$$Register, $cnt2$$Register,
10777                         icnt2, $result$$Register,
10778                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10779     }
10780   %}
10781   ins_pipe( pipe_slow );
10782 %}
10783 
10784 // fast search of substring with known size.
10785 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10786                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10787 %{
10788   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10789   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10790   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10791 
10792   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10793   ins_encode %{
10794     int icnt2 = (int)$int_cnt2$$constant;
10795     if (icnt2 >= 8) {
10796       // IndexOf for constant substrings with size >= 8 elements
10797       // which don't need to be loaded through stack.
10798       __ string_indexofC8($str1$$Register, $str2$$Register,
10799                           $cnt1$$Register, $cnt2$$Register,
10800                           icnt2, $result$$Register,
10801                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10802     } else {
10803       // Small strings are loaded through stack if they cross page boundary.
10804       __ string_indexof($str1$$Register, $str2$$Register,
10805                         $cnt1$$Register, $cnt2$$Register,
10806                         icnt2, $result$$Register,
10807                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10808     }
10809   %}
10810   ins_pipe( pipe_slow );
10811 %}
10812 
10813 // fast search of substring with known size.
10814 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10815                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10816 %{
10817   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10818   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10819   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10820 
10821   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10822   ins_encode %{
10823     int icnt2 = (int)$int_cnt2$$constant;
10824     if (icnt2 >= 8) {
10825       // IndexOf for constant substrings with size >= 8 elements
10826       // which don't need to be loaded through stack.
10827       __ string_indexofC8($str1$$Register, $str2$$Register,
10828                           $cnt1$$Register, $cnt2$$Register,
10829                           icnt2, $result$$Register,
10830                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10831     } else {
10832       // Small strings are loaded through stack if they cross page boundary.
10833       __ string_indexof($str1$$Register, $str2$$Register,
10834                         $cnt1$$Register, $cnt2$$Register,
10835                         icnt2, $result$$Register,
10836                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10837     }
10838   %}
10839   ins_pipe( pipe_slow );
10840 %}
10841 
10842 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10843                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10844 %{
10845   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10846   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10847   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10848 
10849   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10850   ins_encode %{
10851     __ string_indexof($str1$$Register, $str2$$Register,
10852                       $cnt1$$Register, $cnt2$$Register,
10853                       (-1), $result$$Register,
10854                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10855   %}
10856   ins_pipe( pipe_slow );
10857 %}
10858 
10859 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10860                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10861 %{
10862   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10863   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10864   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10865 
10866   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10867   ins_encode %{
10868     __ string_indexof($str1$$Register, $str2$$Register,
10869                       $cnt1$$Register, $cnt2$$Register,
10870                       (-1), $result$$Register,
10871                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10872   %}
10873   ins_pipe( pipe_slow );
10874 %}
10875 
10876 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10877                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10878 %{
10879   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10880   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10881   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10882 
10883   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10884   ins_encode %{
10885     __ string_indexof($str1$$Register, $str2$$Register,
10886                       $cnt1$$Register, $cnt2$$Register,
10887                       (-1), $result$$Register,
10888                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10889   %}
10890   ins_pipe( pipe_slow );
10891 %}
10892 
10893 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10894                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
10895 %{
10896   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
10897   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10898   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10899   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10900   ins_encode %{
10901     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10902                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
10903   %}
10904   ins_pipe( pipe_slow );
10905 %}
10906 
10907 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10908                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
10909 %{
10910   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
10911   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10912   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10913   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10914   ins_encode %{
10915     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10916                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
10917   %}
10918   ins_pipe( pipe_slow );
10919 %}
10920 
10921 // fast string equals
10922 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10923                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10924 %{
10925   predicate(!VM_Version::supports_avx512vlbw());
10926   match(Set result (StrEquals (Binary str1 str2) cnt));
10927   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10928 
10929   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10930   ins_encode %{
10931     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10932                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10933                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
10934   %}
10935   ins_pipe( pipe_slow );
10936 %}
10937 
10938 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10939                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
10940 %{
10941   predicate(VM_Version::supports_avx512vlbw());
10942   match(Set result (StrEquals (Binary str1 str2) cnt));
10943   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10944 
10945   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10946   ins_encode %{
10947     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10948                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10949                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
10950   %}
10951   ins_pipe( pipe_slow );
10952 %}
10953 
10954 // fast array equals
10955 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10956                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10957 %{
10958   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10959   match(Set result (AryEq ary1 ary2));
10960   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10961 
10962   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10963   ins_encode %{
10964     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10965                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10966                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
10967   %}
10968   ins_pipe( pipe_slow );
10969 %}
10970 
10971 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10972                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10973 %{
10974   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10975   match(Set result (AryEq ary1 ary2));
10976   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10977 
10978   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10979   ins_encode %{
10980     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10981                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10982                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
10983   %}
10984   ins_pipe( pipe_slow );
10985 %}
10986 
10987 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10988                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10989 %{
10990   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10991   match(Set result (AryEq ary1 ary2));
10992   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10993 
10994   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10995   ins_encode %{
10996     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10997                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10998                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
10999   %}
11000   ins_pipe( pipe_slow );
11001 %}
11002 
11003 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11004                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11005 %{
11006   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11007   match(Set result (AryEq ary1 ary2));
11008   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11009 
11010   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11011   ins_encode %{
11012     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11013                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11014                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11015   %}
11016   ins_pipe( pipe_slow );
11017 %}
11018 
11019 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11020                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11021                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11022                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11023                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11024 %{
11025   predicate(UseAVX >= 2);
11026   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11027   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11028          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11029          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11030          USE basic_type, KILL cr);
11031 
11032   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
11033   ins_encode %{
11034     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11035                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11036                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11037                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11038                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11039                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11040                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11041   %}
11042   ins_pipe( pipe_slow );
11043 %}
11044 
11045 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11046                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11047 %{
11048   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11049   match(Set result (CountPositives ary1 len));
11050   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11051 
11052   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11053   ins_encode %{
11054     __ count_positives($ary1$$Register, $len$$Register,
11055                        $result$$Register, $tmp3$$Register,
11056                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11057   %}
11058   ins_pipe( pipe_slow );
11059 %}
11060 
11061 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11062                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11063 %{
11064   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11065   match(Set result (CountPositives ary1 len));
11066   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11067 
11068   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11069   ins_encode %{
11070     __ count_positives($ary1$$Register, $len$$Register,
11071                        $result$$Register, $tmp3$$Register,
11072                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11073   %}
11074   ins_pipe( pipe_slow );
11075 %}
11076 
11077 // fast char[] to byte[] compression
11078 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11079                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11080   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11081   match(Set result (StrCompressedCopy src (Binary dst len)));
11082   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11083          USE_KILL len, KILL tmp5, KILL cr);
11084 
11085   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11086   ins_encode %{
11087     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11088                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11089                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11090                            knoreg, knoreg);
11091   %}
11092   ins_pipe( pipe_slow );
11093 %}
11094 
11095 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11096                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11097   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11098   match(Set result (StrCompressedCopy src (Binary dst len)));
11099   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11100          USE_KILL len, KILL tmp5, KILL cr);
11101 
11102   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11103   ins_encode %{
11104     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11105                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11106                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11107                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11108   %}
11109   ins_pipe( pipe_slow );
11110 %}
11111 // fast byte[] to char[] inflation
11112 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11113                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11114   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11115   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11116   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11117 
11118   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11119   ins_encode %{
11120     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11121                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11122   %}
11123   ins_pipe( pipe_slow );
11124 %}
11125 
11126 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11127                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11128   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11129   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11130   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11131 
11132   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11133   ins_encode %{
11134     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11135                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11136   %}
11137   ins_pipe( pipe_slow );
11138 %}
11139 
11140 // encode char[] to byte[] in ISO_8859_1
11141 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11142                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11143                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11144   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11145   match(Set result (EncodeISOArray src (Binary dst len)));
11146   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11147 
11148   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11149   ins_encode %{
11150     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11151                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11152                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11153   %}
11154   ins_pipe( pipe_slow );
11155 %}
11156 
11157 // encode char[] to byte[] in ASCII
11158 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11159                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11160                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11161   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11162   match(Set result (EncodeISOArray src (Binary dst len)));
11163   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11164 
11165   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11166   ins_encode %{
11167     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11168                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11169                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11170   %}
11171   ins_pipe( pipe_slow );
11172 %}
11173 
11174 //----------Overflow Math Instructions-----------------------------------------
11175 
11176 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11177 %{
11178   match(Set cr (OverflowAddI op1 op2));
11179   effect(DEF cr, USE_KILL op1, USE op2);
11180 
11181   format %{ "addl    $op1, $op2\t# overflow check int" %}
11182 
11183   ins_encode %{
11184     __ addl($op1$$Register, $op2$$Register);
11185   %}
11186   ins_pipe(ialu_reg_reg);
11187 %}
11188 
11189 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11190 %{
11191   match(Set cr (OverflowAddI op1 op2));
11192   effect(DEF cr, USE_KILL op1, USE op2);
11193 
11194   format %{ "addl    $op1, $op2\t# overflow check int" %}
11195 
11196   ins_encode %{
11197     __ addl($op1$$Register, $op2$$constant);
11198   %}
11199   ins_pipe(ialu_reg_reg);
11200 %}
11201 
11202 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11203 %{
11204   match(Set cr (OverflowAddL op1 op2));
11205   effect(DEF cr, USE_KILL op1, USE op2);
11206 
11207   format %{ "addq    $op1, $op2\t# overflow check long" %}
11208   ins_encode %{
11209     __ addq($op1$$Register, $op2$$Register);
11210   %}
11211   ins_pipe(ialu_reg_reg);
11212 %}
11213 
11214 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11215 %{
11216   match(Set cr (OverflowAddL op1 op2));
11217   effect(DEF cr, USE_KILL op1, USE op2);
11218 
11219   format %{ "addq    $op1, $op2\t# overflow check long" %}
11220   ins_encode %{
11221     __ addq($op1$$Register, $op2$$constant);
11222   %}
11223   ins_pipe(ialu_reg_reg);
11224 %}
11225 
11226 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11227 %{
11228   match(Set cr (OverflowSubI op1 op2));
11229 
11230   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11231   ins_encode %{
11232     __ cmpl($op1$$Register, $op2$$Register);
11233   %}
11234   ins_pipe(ialu_reg_reg);
11235 %}
11236 
11237 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11238 %{
11239   match(Set cr (OverflowSubI op1 op2));
11240 
11241   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11242   ins_encode %{
11243     __ cmpl($op1$$Register, $op2$$constant);
11244   %}
11245   ins_pipe(ialu_reg_reg);
11246 %}
11247 
11248 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11249 %{
11250   match(Set cr (OverflowSubL op1 op2));
11251 
11252   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11253   ins_encode %{
11254     __ cmpq($op1$$Register, $op2$$Register);
11255   %}
11256   ins_pipe(ialu_reg_reg);
11257 %}
11258 
11259 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11260 %{
11261   match(Set cr (OverflowSubL op1 op2));
11262 
11263   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11264   ins_encode %{
11265     __ cmpq($op1$$Register, $op2$$constant);
11266   %}
11267   ins_pipe(ialu_reg_reg);
11268 %}
11269 
11270 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11271 %{
11272   match(Set cr (OverflowSubI zero op2));
11273   effect(DEF cr, USE_KILL op2);
11274 
11275   format %{ "negl    $op2\t# overflow check int" %}
11276   ins_encode %{
11277     __ negl($op2$$Register);
11278   %}
11279   ins_pipe(ialu_reg_reg);
11280 %}
11281 
11282 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11283 %{
11284   match(Set cr (OverflowSubL zero op2));
11285   effect(DEF cr, USE_KILL op2);
11286 
11287   format %{ "negq    $op2\t# overflow check long" %}
11288   ins_encode %{
11289     __ negq($op2$$Register);
11290   %}
11291   ins_pipe(ialu_reg_reg);
11292 %}
11293 
11294 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11295 %{
11296   match(Set cr (OverflowMulI op1 op2));
11297   effect(DEF cr, USE_KILL op1, USE op2);
11298 
11299   format %{ "imull    $op1, $op2\t# overflow check int" %}
11300   ins_encode %{
11301     __ imull($op1$$Register, $op2$$Register);
11302   %}
11303   ins_pipe(ialu_reg_reg_alu0);
11304 %}
11305 
11306 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11307 %{
11308   match(Set cr (OverflowMulI op1 op2));
11309   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11310 
11311   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11312   ins_encode %{
11313     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11314   %}
11315   ins_pipe(ialu_reg_reg_alu0);
11316 %}
11317 
11318 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11319 %{
11320   match(Set cr (OverflowMulL op1 op2));
11321   effect(DEF cr, USE_KILL op1, USE op2);
11322 
11323   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11324   ins_encode %{
11325     __ imulq($op1$$Register, $op2$$Register);
11326   %}
11327   ins_pipe(ialu_reg_reg_alu0);
11328 %}
11329 
11330 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11331 %{
11332   match(Set cr (OverflowMulL op1 op2));
11333   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11334 
11335   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11336   ins_encode %{
11337     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11338   %}
11339   ins_pipe(ialu_reg_reg_alu0);
11340 %}
11341 
11342 
11343 //----------Control Flow Instructions------------------------------------------
11344 // Signed compare Instructions
11345 
11346 // XXX more variants!!
11347 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11348 %{
11349   match(Set cr (CmpI op1 op2));
11350   effect(DEF cr, USE op1, USE op2);
11351 
11352   format %{ "cmpl    $op1, $op2" %}
11353   ins_encode %{
11354     __ cmpl($op1$$Register, $op2$$Register);
11355   %}
11356   ins_pipe(ialu_cr_reg_reg);
11357 %}
11358 
11359 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11360 %{
11361   match(Set cr (CmpI op1 op2));
11362 
11363   format %{ "cmpl    $op1, $op2" %}
11364   ins_encode %{
11365     __ cmpl($op1$$Register, $op2$$constant);
11366   %}
11367   ins_pipe(ialu_cr_reg_imm);
11368 %}
11369 
11370 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11371 %{
11372   match(Set cr (CmpI op1 (LoadI op2)));
11373 
11374   ins_cost(500); // XXX
11375   format %{ "cmpl    $op1, $op2" %}
11376   ins_encode %{
11377     __ cmpl($op1$$Register, $op2$$Address);
11378   %}
11379   ins_pipe(ialu_cr_reg_mem);
11380 %}
11381 
11382 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11383 %{
11384   match(Set cr (CmpI src zero));
11385 
11386   format %{ "testl   $src, $src" %}
11387   ins_encode %{
11388     __ testl($src$$Register, $src$$Register);
11389   %}
11390   ins_pipe(ialu_cr_reg_imm);
11391 %}
11392 
11393 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11394 %{
11395   match(Set cr (CmpI (AndI src con) zero));
11396 
11397   format %{ "testl   $src, $con" %}
11398   ins_encode %{
11399     __ testl($src$$Register, $con$$constant);
11400   %}
11401   ins_pipe(ialu_cr_reg_imm);
11402 %}
11403 
11404 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11405 %{
11406   match(Set cr (CmpI (AndI src1 src2) zero));
11407 
11408   format %{ "testl   $src1, $src2" %}
11409   ins_encode %{
11410     __ testl($src1$$Register, $src2$$Register);
11411   %}
11412   ins_pipe(ialu_cr_reg_imm);
11413 %}
11414 
11415 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11416 %{
11417   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11418 
11419   format %{ "testl   $src, $mem" %}
11420   ins_encode %{
11421     __ testl($src$$Register, $mem$$Address);
11422   %}
11423   ins_pipe(ialu_cr_reg_mem);
11424 %}
11425 
11426 // Unsigned compare Instructions; really, same as signed except they
11427 // produce an rFlagsRegU instead of rFlagsReg.
11428 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11429 %{
11430   match(Set cr (CmpU op1 op2));
11431 
11432   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11433   ins_encode %{
11434     __ cmpl($op1$$Register, $op2$$Register);
11435   %}
11436   ins_pipe(ialu_cr_reg_reg);
11437 %}
11438 
11439 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11440 %{
11441   match(Set cr (CmpU op1 op2));
11442 
11443   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11444   ins_encode %{
11445     __ cmpl($op1$$Register, $op2$$constant);
11446   %}
11447   ins_pipe(ialu_cr_reg_imm);
11448 %}
11449 
11450 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11451 %{
11452   match(Set cr (CmpU op1 (LoadI op2)));
11453 
11454   ins_cost(500); // XXX
11455   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11456   ins_encode %{
11457     __ cmpl($op1$$Register, $op2$$Address);
11458   %}
11459   ins_pipe(ialu_cr_reg_mem);
11460 %}
11461 
11462 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11463 %{
11464   match(Set cr (CmpU src zero));
11465 
11466   format %{ "testl   $src, $src\t# unsigned" %}
11467   ins_encode %{
11468     __ testl($src$$Register, $src$$Register);
11469   %}
11470   ins_pipe(ialu_cr_reg_imm);
11471 %}
11472 
11473 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11474 %{
11475   match(Set cr (CmpP op1 op2));
11476 
11477   format %{ "cmpq    $op1, $op2\t# ptr" %}
11478   ins_encode %{
11479     __ cmpq($op1$$Register, $op2$$Register);
11480   %}
11481   ins_pipe(ialu_cr_reg_reg);
11482 %}
11483 
11484 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11485 %{
11486   match(Set cr (CmpP op1 (LoadP op2)));
11487   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11488 
11489   ins_cost(500); // XXX
11490   format %{ "cmpq    $op1, $op2\t# ptr" %}
11491   ins_encode %{
11492     __ cmpq($op1$$Register, $op2$$Address);
11493   %}
11494   ins_pipe(ialu_cr_reg_mem);
11495 %}
11496 
11497 // XXX this is generalized by compP_rReg_mem???
11498 // Compare raw pointer (used in out-of-heap check).
11499 // Only works because non-oop pointers must be raw pointers
11500 // and raw pointers have no anti-dependencies.
11501 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11502 %{
11503   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11504             n->in(2)->as_Load()->barrier_data() == 0);
11505   match(Set cr (CmpP op1 (LoadP op2)));
11506 
11507   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11508   ins_encode %{
11509     __ cmpq($op1$$Register, $op2$$Address);
11510   %}
11511   ins_pipe(ialu_cr_reg_mem);
11512 %}
11513 
11514 // This will generate a signed flags result. This should be OK since
11515 // any compare to a zero should be eq/neq.
11516 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11517 %{
11518   match(Set cr (CmpP src zero));
11519 
11520   format %{ "testq   $src, $src\t# ptr" %}
11521   ins_encode %{
11522     __ testq($src$$Register, $src$$Register);
11523   %}
11524   ins_pipe(ialu_cr_reg_imm);
11525 %}
11526 
11527 // This will generate a signed flags result. This should be OK since
11528 // any compare to a zero should be eq/neq.
11529 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11530 %{
11531   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
11532             n->in(1)->as_Load()->barrier_data() == 0);
11533   match(Set cr (CmpP (LoadP op) zero));
11534 
11535   ins_cost(500); // XXX
11536   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11537   ins_encode %{
11538     __ testq($op$$Address, 0xFFFFFFFF);
11539   %}
11540   ins_pipe(ialu_cr_reg_imm);
11541 %}
11542 
11543 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11544 %{
11545   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
11546             n->in(1)->as_Load()->barrier_data() == 0);
11547   match(Set cr (CmpP (LoadP mem) zero));
11548 
11549   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11550   ins_encode %{
11551     __ cmpq(r12, $mem$$Address);
11552   %}
11553   ins_pipe(ialu_cr_reg_mem);
11554 %}
11555 
11556 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11557 %{
11558   match(Set cr (CmpN op1 op2));
11559 
11560   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11561   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11562   ins_pipe(ialu_cr_reg_reg);
11563 %}
11564 
11565 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11566 %{
11567   match(Set cr (CmpN src (LoadN mem)));
11568 
11569   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11570   ins_encode %{
11571     __ cmpl($src$$Register, $mem$$Address);
11572   %}
11573   ins_pipe(ialu_cr_reg_mem);
11574 %}
11575 
11576 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11577   match(Set cr (CmpN op1 op2));
11578 
11579   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11580   ins_encode %{
11581     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11582   %}
11583   ins_pipe(ialu_cr_reg_imm);
11584 %}
11585 
11586 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11587 %{
11588   match(Set cr (CmpN src (LoadN mem)));
11589 
11590   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11591   ins_encode %{
11592     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11593   %}
11594   ins_pipe(ialu_cr_reg_mem);
11595 %}
11596 
11597 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11598   match(Set cr (CmpN op1 op2));
11599 
11600   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11601   ins_encode %{
11602     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11603   %}
11604   ins_pipe(ialu_cr_reg_imm);
11605 %}
11606 
11607 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11608 %{
11609   match(Set cr (CmpN src (LoadNKlass mem)));
11610 
11611   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11612   ins_encode %{
11613     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11614   %}
11615   ins_pipe(ialu_cr_reg_mem);
11616 %}
11617 
11618 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11619   match(Set cr (CmpN src zero));
11620 
11621   format %{ "testl   $src, $src\t# compressed ptr" %}
11622   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11623   ins_pipe(ialu_cr_reg_imm);
11624 %}
11625 
11626 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11627 %{
11628   predicate(CompressedOops::base() != nullptr);
11629   match(Set cr (CmpN (LoadN mem) zero));
11630 
11631   ins_cost(500); // XXX
11632   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11633   ins_encode %{
11634     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11635   %}
11636   ins_pipe(ialu_cr_reg_mem);
11637 %}
11638 
11639 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11640 %{
11641   predicate(CompressedOops::base() == nullptr);
11642   match(Set cr (CmpN (LoadN mem) zero));
11643 
11644   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11645   ins_encode %{
11646     __ cmpl(r12, $mem$$Address);
11647   %}
11648   ins_pipe(ialu_cr_reg_mem);
11649 %}
11650 
11651 // Yanked all unsigned pointer compare operations.
11652 // Pointer compares are done with CmpP which is already unsigned.
11653 
11654 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11655 %{
11656   match(Set cr (CmpL op1 op2));
11657 
11658   format %{ "cmpq    $op1, $op2" %}
11659   ins_encode %{
11660     __ cmpq($op1$$Register, $op2$$Register);
11661   %}
11662   ins_pipe(ialu_cr_reg_reg);
11663 %}
11664 
11665 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11666 %{
11667   match(Set cr (CmpL op1 op2));
11668 
11669   format %{ "cmpq    $op1, $op2" %}
11670   ins_encode %{
11671     __ cmpq($op1$$Register, $op2$$constant);
11672   %}
11673   ins_pipe(ialu_cr_reg_imm);
11674 %}
11675 
11676 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11677 %{
11678   match(Set cr (CmpL op1 (LoadL op2)));
11679 
11680   format %{ "cmpq    $op1, $op2" %}
11681   ins_encode %{
11682     __ cmpq($op1$$Register, $op2$$Address);
11683   %}
11684   ins_pipe(ialu_cr_reg_mem);
11685 %}
11686 
11687 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11688 %{
11689   match(Set cr (CmpL src zero));
11690 
11691   format %{ "testq   $src, $src" %}
11692   ins_encode %{
11693     __ testq($src$$Register, $src$$Register);
11694   %}
11695   ins_pipe(ialu_cr_reg_imm);
11696 %}
11697 
11698 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11699 %{
11700   match(Set cr (CmpL (AndL src con) zero));
11701 
11702   format %{ "testq   $src, $con\t# long" %}
11703   ins_encode %{
11704     __ testq($src$$Register, $con$$constant);
11705   %}
11706   ins_pipe(ialu_cr_reg_imm);
11707 %}
11708 
11709 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
11710 %{
11711   match(Set cr (CmpL (AndL src1 src2) zero));
11712 
11713   format %{ "testq   $src1, $src2\t# long" %}
11714   ins_encode %{
11715     __ testq($src1$$Register, $src2$$Register);
11716   %}
11717   ins_pipe(ialu_cr_reg_imm);
11718 %}
11719 
11720 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11721 %{
11722   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11723 
11724   format %{ "testq   $src, $mem" %}
11725   ins_encode %{
11726     __ testq($src$$Register, $mem$$Address);
11727   %}
11728   ins_pipe(ialu_cr_reg_mem);
11729 %}
11730 
11731 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11732 %{
11733   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11734 
11735   format %{ "testq   $src, $mem" %}
11736   ins_encode %{
11737     __ testq($src$$Register, $mem$$Address);
11738   %}
11739   ins_pipe(ialu_cr_reg_mem);
11740 %}
11741 
11742 // Manifest a CmpU result in an integer register.  Very painful.
11743 // This is the test to avoid.
11744 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
11745 %{
11746   match(Set dst (CmpU3 src1 src2));
11747   effect(KILL flags);
11748 
11749   ins_cost(275); // XXX
11750   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
11751             "movl    $dst, -1\n\t"
11752             "jb,u    done\n\t"
11753             "setne   $dst\n\t"
11754             "movzbl  $dst, $dst\n\t"
11755     "done:" %}
11756   ins_encode %{
11757     Label done;
11758     __ cmpl($src1$$Register, $src2$$Register);
11759     __ movl($dst$$Register, -1);
11760     __ jccb(Assembler::below, done);
11761     __ setb(Assembler::notZero, $dst$$Register);
11762     __ movzbl($dst$$Register, $dst$$Register);
11763     __ bind(done);
11764   %}
11765   ins_pipe(pipe_slow);
11766 %}
11767 
11768 // Manifest a CmpL result in an integer register.  Very painful.
11769 // This is the test to avoid.
11770 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11771 %{
11772   match(Set dst (CmpL3 src1 src2));
11773   effect(KILL flags);
11774 
11775   ins_cost(275); // XXX
11776   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11777             "movl    $dst, -1\n\t"
11778             "jl,s    done\n\t"
11779             "setne   $dst\n\t"
11780             "movzbl  $dst, $dst\n\t"
11781     "done:" %}
11782   ins_encode %{
11783     Label done;
11784     __ cmpq($src1$$Register, $src2$$Register);
11785     __ movl($dst$$Register, -1);
11786     __ jccb(Assembler::less, done);
11787     __ setb(Assembler::notZero, $dst$$Register);
11788     __ movzbl($dst$$Register, $dst$$Register);
11789     __ bind(done);
11790   %}
11791   ins_pipe(pipe_slow);
11792 %}
11793 
11794 // Manifest a CmpUL result in an integer register.  Very painful.
11795 // This is the test to avoid.
11796 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11797 %{
11798   match(Set dst (CmpUL3 src1 src2));
11799   effect(KILL flags);
11800 
11801   ins_cost(275); // XXX
11802   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11803             "movl    $dst, -1\n\t"
11804             "jb,u    done\n\t"
11805             "setne   $dst\n\t"
11806             "movzbl  $dst, $dst\n\t"
11807     "done:" %}
11808   ins_encode %{
11809     Label done;
11810     __ cmpq($src1$$Register, $src2$$Register);
11811     __ movl($dst$$Register, -1);
11812     __ jccb(Assembler::below, done);
11813     __ setb(Assembler::notZero, $dst$$Register);
11814     __ movzbl($dst$$Register, $dst$$Register);
11815     __ bind(done);
11816   %}
11817   ins_pipe(pipe_slow);
11818 %}
11819 
11820 // Unsigned long compare Instructions; really, same as signed long except they
11821 // produce an rFlagsRegU instead of rFlagsReg.
11822 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11823 %{
11824   match(Set cr (CmpUL op1 op2));
11825 
11826   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11827   ins_encode %{
11828     __ cmpq($op1$$Register, $op2$$Register);
11829   %}
11830   ins_pipe(ialu_cr_reg_reg);
11831 %}
11832 
11833 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11834 %{
11835   match(Set cr (CmpUL op1 op2));
11836 
11837   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11838   ins_encode %{
11839     __ cmpq($op1$$Register, $op2$$constant);
11840   %}
11841   ins_pipe(ialu_cr_reg_imm);
11842 %}
11843 
11844 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11845 %{
11846   match(Set cr (CmpUL op1 (LoadL op2)));
11847 
11848   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11849   ins_encode %{
11850     __ cmpq($op1$$Register, $op2$$Address);
11851   %}
11852   ins_pipe(ialu_cr_reg_mem);
11853 %}
11854 
11855 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11856 %{
11857   match(Set cr (CmpUL src zero));
11858 
11859   format %{ "testq   $src, $src\t# unsigned" %}
11860   ins_encode %{
11861     __ testq($src$$Register, $src$$Register);
11862   %}
11863   ins_pipe(ialu_cr_reg_imm);
11864 %}
11865 
11866 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11867 %{
11868   match(Set cr (CmpI (LoadB mem) imm));
11869 
11870   ins_cost(125);
11871   format %{ "cmpb    $mem, $imm" %}
11872   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
11873   ins_pipe(ialu_cr_reg_mem);
11874 %}
11875 
11876 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
11877 %{
11878   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
11879 
11880   ins_cost(125);
11881   format %{ "testb   $mem, $imm\t# ubyte" %}
11882   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11883   ins_pipe(ialu_cr_reg_mem);
11884 %}
11885 
11886 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
11887 %{
11888   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
11889 
11890   ins_cost(125);
11891   format %{ "testb   $mem, $imm\t# byte" %}
11892   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11893   ins_pipe(ialu_cr_reg_mem);
11894 %}
11895 
11896 //----------Max and Min--------------------------------------------------------
11897 // Min Instructions
11898 
11899 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11900 %{
11901   effect(USE_DEF dst, USE src, USE cr);
11902 
11903   format %{ "cmovlgt $dst, $src\t# min" %}
11904   ins_encode %{
11905     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
11906   %}
11907   ins_pipe(pipe_cmov_reg);
11908 %}
11909 
11910 
11911 instruct minI_rReg(rRegI dst, rRegI src)
11912 %{
11913   match(Set dst (MinI dst src));
11914 
11915   ins_cost(200);
11916   expand %{
11917     rFlagsReg cr;
11918     compI_rReg(cr, dst, src);
11919     cmovI_reg_g(dst, src, cr);
11920   %}
11921 %}
11922 
11923 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11924 %{
11925   effect(USE_DEF dst, USE src, USE cr);
11926 
11927   format %{ "cmovllt $dst, $src\t# max" %}
11928   ins_encode %{
11929     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
11930   %}
11931   ins_pipe(pipe_cmov_reg);
11932 %}
11933 
11934 
11935 instruct maxI_rReg(rRegI dst, rRegI src)
11936 %{
11937   match(Set dst (MaxI dst src));
11938 
11939   ins_cost(200);
11940   expand %{
11941     rFlagsReg cr;
11942     compI_rReg(cr, dst, src);
11943     cmovI_reg_l(dst, src, cr);
11944   %}
11945 %}
11946 
11947 // ============================================================================
11948 // Branch Instructions
11949 
11950 // Jump Direct - Label defines a relative address from JMP+1
11951 instruct jmpDir(label labl)
11952 %{
11953   match(Goto);
11954   effect(USE labl);
11955 
11956   ins_cost(300);
11957   format %{ "jmp     $labl" %}
11958   size(5);
11959   ins_encode %{
11960     Label* L = $labl$$label;
11961     __ jmp(*L, false); // Always long jump
11962   %}
11963   ins_pipe(pipe_jmp);
11964 %}
11965 
11966 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11967 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11968 %{
11969   match(If cop cr);
11970   effect(USE labl);
11971 
11972   ins_cost(300);
11973   format %{ "j$cop     $labl" %}
11974   size(6);
11975   ins_encode %{
11976     Label* L = $labl$$label;
11977     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11978   %}
11979   ins_pipe(pipe_jcc);
11980 %}
11981 
11982 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11983 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11984 %{
11985   match(CountedLoopEnd cop cr);
11986   effect(USE labl);
11987 
11988   ins_cost(300);
11989   format %{ "j$cop     $labl\t# loop end" %}
11990   size(6);
11991   ins_encode %{
11992     Label* L = $labl$$label;
11993     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11994   %}
11995   ins_pipe(pipe_jcc);
11996 %}
11997 
11998 // Jump Direct Conditional - using unsigned comparison
11999 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12000   match(If cop cmp);
12001   effect(USE labl);
12002 
12003   ins_cost(300);
12004   format %{ "j$cop,u   $labl" %}
12005   size(6);
12006   ins_encode %{
12007     Label* L = $labl$$label;
12008     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12009   %}
12010   ins_pipe(pipe_jcc);
12011 %}
12012 
12013 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12014   match(If cop cmp);
12015   effect(USE labl);
12016 
12017   ins_cost(200);
12018   format %{ "j$cop,u   $labl" %}
12019   size(6);
12020   ins_encode %{
12021     Label* L = $labl$$label;
12022     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12023   %}
12024   ins_pipe(pipe_jcc);
12025 %}
12026 
12027 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12028   match(If cop cmp);
12029   effect(USE labl);
12030 
12031   ins_cost(200);
12032   format %{ $$template
12033     if ($cop$$cmpcode == Assembler::notEqual) {
12034       $$emit$$"jp,u    $labl\n\t"
12035       $$emit$$"j$cop,u   $labl"
12036     } else {
12037       $$emit$$"jp,u    done\n\t"
12038       $$emit$$"j$cop,u   $labl\n\t"
12039       $$emit$$"done:"
12040     }
12041   %}
12042   ins_encode %{
12043     Label* l = $labl$$label;
12044     if ($cop$$cmpcode == Assembler::notEqual) {
12045       __ jcc(Assembler::parity, *l, false);
12046       __ jcc(Assembler::notEqual, *l, false);
12047     } else if ($cop$$cmpcode == Assembler::equal) {
12048       Label done;
12049       __ jccb(Assembler::parity, done);
12050       __ jcc(Assembler::equal, *l, false);
12051       __ bind(done);
12052     } else {
12053        ShouldNotReachHere();
12054     }
12055   %}
12056   ins_pipe(pipe_jcc);
12057 %}
12058 
12059 // ============================================================================
12060 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12061 // superklass array for an instance of the superklass.  Set a hidden
12062 // internal cache on a hit (cache is checked with exposed code in
12063 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12064 // encoding ALSO sets flags.
12065 
12066 instruct partialSubtypeCheck(rdi_RegP result,
12067                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12068                              rFlagsReg cr)
12069 %{
12070   match(Set result (PartialSubtypeCheck sub super));
12071   effect(KILL rcx, KILL cr);
12072 
12073   ins_cost(1100);  // slightly larger than the next version
12074   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12075             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12076             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12077             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12078             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12079             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12080             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12081     "miss:\t" %}
12082 
12083   opcode(0x1); // Force a XOR of RDI
12084   ins_encode(enc_PartialSubtypeCheck());
12085   ins_pipe(pipe_slow);
12086 %}
12087 
12088 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
12089                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
12090                                        rFlagsReg cr)
12091 %{
12092   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
12093   predicate(UseSecondarySupersTable);
12094   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
12095 
12096   ins_cost(700);  // smaller than the next version
12097   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
12098 
12099   ins_encode %{
12100     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
12101     if (InlineSecondarySupersTest) {
12102       __ lookup_secondary_supers_table($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
12103                                        $temp3$$Register, $temp4$$Register, $result$$Register,
12104                                        super_klass_slot);
12105     } else {
12106       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
12107     }
12108   %}
12109 
12110   ins_pipe(pipe_slow);
12111 %}
12112 
12113 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12114                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12115                                      immP0 zero,
12116                                      rdi_RegP result)
12117 %{
12118   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12119   effect(KILL rcx, KILL result);
12120 
12121   ins_cost(1000);
12122   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12123             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12124             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12125             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12126             "jne,s   miss\t\t# Missed: flags nz\n\t"
12127             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12128     "miss:\t" %}
12129 
12130   opcode(0x0); // No need to XOR RDI
12131   ins_encode(enc_PartialSubtypeCheck());
12132   ins_pipe(pipe_slow);
12133 %}
12134 
12135 // ============================================================================
12136 // Branch Instructions -- short offset versions
12137 //
12138 // These instructions are used to replace jumps of a long offset (the default
12139 // match) with jumps of a shorter offset.  These instructions are all tagged
12140 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12141 // match rules in general matching.  Instead, the ADLC generates a conversion
12142 // method in the MachNode which can be used to do in-place replacement of the
12143 // long variant with the shorter variant.  The compiler will determine if a
12144 // branch can be taken by the is_short_branch_offset() predicate in the machine
12145 // specific code section of the file.
12146 
12147 // Jump Direct - Label defines a relative address from JMP+1
12148 instruct jmpDir_short(label labl) %{
12149   match(Goto);
12150   effect(USE labl);
12151 
12152   ins_cost(300);
12153   format %{ "jmp,s   $labl" %}
12154   size(2);
12155   ins_encode %{
12156     Label* L = $labl$$label;
12157     __ jmpb(*L);
12158   %}
12159   ins_pipe(pipe_jmp);
12160   ins_short_branch(1);
12161 %}
12162 
12163 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12164 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12165   match(If cop cr);
12166   effect(USE labl);
12167 
12168   ins_cost(300);
12169   format %{ "j$cop,s   $labl" %}
12170   size(2);
12171   ins_encode %{
12172     Label* L = $labl$$label;
12173     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12174   %}
12175   ins_pipe(pipe_jcc);
12176   ins_short_branch(1);
12177 %}
12178 
12179 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12180 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12181   match(CountedLoopEnd cop cr);
12182   effect(USE labl);
12183 
12184   ins_cost(300);
12185   format %{ "j$cop,s   $labl\t# loop end" %}
12186   size(2);
12187   ins_encode %{
12188     Label* L = $labl$$label;
12189     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12190   %}
12191   ins_pipe(pipe_jcc);
12192   ins_short_branch(1);
12193 %}
12194 
12195 // Jump Direct Conditional - using unsigned comparison
12196 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12197   match(If cop cmp);
12198   effect(USE labl);
12199 
12200   ins_cost(300);
12201   format %{ "j$cop,us  $labl" %}
12202   size(2);
12203   ins_encode %{
12204     Label* L = $labl$$label;
12205     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12206   %}
12207   ins_pipe(pipe_jcc);
12208   ins_short_branch(1);
12209 %}
12210 
12211 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12212   match(If cop cmp);
12213   effect(USE labl);
12214 
12215   ins_cost(300);
12216   format %{ "j$cop,us  $labl" %}
12217   size(2);
12218   ins_encode %{
12219     Label* L = $labl$$label;
12220     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12221   %}
12222   ins_pipe(pipe_jcc);
12223   ins_short_branch(1);
12224 %}
12225 
12226 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12227   match(If cop cmp);
12228   effect(USE labl);
12229 
12230   ins_cost(300);
12231   format %{ $$template
12232     if ($cop$$cmpcode == Assembler::notEqual) {
12233       $$emit$$"jp,u,s  $labl\n\t"
12234       $$emit$$"j$cop,u,s  $labl"
12235     } else {
12236       $$emit$$"jp,u,s  done\n\t"
12237       $$emit$$"j$cop,u,s  $labl\n\t"
12238       $$emit$$"done:"
12239     }
12240   %}
12241   size(4);
12242   ins_encode %{
12243     Label* l = $labl$$label;
12244     if ($cop$$cmpcode == Assembler::notEqual) {
12245       __ jccb(Assembler::parity, *l);
12246       __ jccb(Assembler::notEqual, *l);
12247     } else if ($cop$$cmpcode == Assembler::equal) {
12248       Label done;
12249       __ jccb(Assembler::parity, done);
12250       __ jccb(Assembler::equal, *l);
12251       __ bind(done);
12252     } else {
12253        ShouldNotReachHere();
12254     }
12255   %}
12256   ins_pipe(pipe_jcc);
12257   ins_short_branch(1);
12258 %}
12259 
12260 // ============================================================================
12261 // inlined locking and unlocking
12262 
12263 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12264   predicate(LockingMode != LM_LIGHTWEIGHT);
12265   match(Set cr (FastLock object box));
12266   effect(TEMP tmp, TEMP scr, USE_KILL box);
12267   ins_cost(300);
12268   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12269   ins_encode %{
12270     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12271                  $scr$$Register, noreg, noreg, r15_thread, nullptr);
12272   %}
12273   ins_pipe(pipe_slow);
12274 %}
12275 
12276 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12277   predicate(LockingMode != LM_LIGHTWEIGHT);
12278   match(Set cr (FastUnlock object box));
12279   effect(TEMP tmp, USE_KILL box);
12280   ins_cost(300);
12281   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12282   ins_encode %{
12283     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
12284   %}
12285   ins_pipe(pipe_slow);
12286 %}
12287 
12288 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
12289   predicate(LockingMode == LM_LIGHTWEIGHT);
12290   match(Set cr (FastLock object box));
12291   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
12292   ins_cost(300);
12293   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
12294   ins_encode %{
12295     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12296   %}
12297   ins_pipe(pipe_slow);
12298 %}
12299 
12300 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
12301   predicate(LockingMode == LM_LIGHTWEIGHT);
12302   match(Set cr (FastUnlock object rax_reg));
12303   effect(TEMP tmp, USE_KILL rax_reg);
12304   ins_cost(300);
12305   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
12306   ins_encode %{
12307     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12308   %}
12309   ins_pipe(pipe_slow);
12310 %}
12311 
12312 
12313 // ============================================================================
12314 // Safepoint Instructions
12315 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12316 %{
12317   match(SafePoint poll);
12318   effect(KILL cr, USE poll);
12319 
12320   format %{ "testl   rax, [$poll]\t"
12321             "# Safepoint: poll for GC" %}
12322   ins_cost(125);
12323   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12324   ins_encode %{
12325     __ relocate(relocInfo::poll_type);
12326     address pre_pc = __ pc();
12327     __ testl(rax, Address($poll$$Register, 0));
12328     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12329   %}
12330   ins_pipe(ialu_reg_mem);
12331 %}
12332 
12333 instruct mask_all_evexL(kReg dst, rRegL src) %{
12334   match(Set dst (MaskAll src));
12335   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12336   ins_encode %{
12337     int mask_len = Matcher::vector_length(this);
12338     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12339   %}
12340   ins_pipe( pipe_slow );
12341 %}
12342 
12343 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12344   predicate(Matcher::vector_length(n) > 32);
12345   match(Set dst (MaskAll src));
12346   effect(TEMP tmp);
12347   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12348   ins_encode %{
12349     int mask_len = Matcher::vector_length(this);
12350     __ movslq($tmp$$Register, $src$$Register);
12351     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12352   %}
12353   ins_pipe( pipe_slow );
12354 %}
12355 
12356 // ============================================================================
12357 // Procedure Call/Return Instructions
12358 // Call Java Static Instruction
12359 // Note: If this code changes, the corresponding ret_addr_offset() and
12360 //       compute_padding() functions will have to be adjusted.
12361 instruct CallStaticJavaDirect(method meth) %{
12362   match(CallStaticJava);
12363   effect(USE meth);
12364 
12365   ins_cost(300);
12366   format %{ "call,static " %}
12367   opcode(0xE8); /* E8 cd */
12368   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12369   ins_pipe(pipe_slow);
12370   ins_alignment(4);
12371 %}
12372 
12373 // Call Java Dynamic Instruction
12374 // Note: If this code changes, the corresponding ret_addr_offset() and
12375 //       compute_padding() functions will have to be adjusted.
12376 instruct CallDynamicJavaDirect(method meth)
12377 %{
12378   match(CallDynamicJava);
12379   effect(USE meth);
12380 
12381   ins_cost(300);
12382   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12383             "call,dynamic " %}
12384   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12385   ins_pipe(pipe_slow);
12386   ins_alignment(4);
12387 %}
12388 
12389 // Call Runtime Instruction
12390 instruct CallRuntimeDirect(method meth)
12391 %{
12392   match(CallRuntime);
12393   effect(USE meth);
12394 
12395   ins_cost(300);
12396   format %{ "call,runtime " %}
12397   ins_encode(clear_avx, Java_To_Runtime(meth));
12398   ins_pipe(pipe_slow);
12399 %}
12400 
12401 // Call runtime without safepoint
12402 instruct CallLeafDirect(method meth)
12403 %{
12404   match(CallLeaf);
12405   effect(USE meth);
12406 
12407   ins_cost(300);
12408   format %{ "call_leaf,runtime " %}
12409   ins_encode(clear_avx, Java_To_Runtime(meth));
12410   ins_pipe(pipe_slow);
12411 %}
12412 
12413 // Call runtime without safepoint and with vector arguments
12414 instruct CallLeafDirectVector(method meth)
12415 %{
12416   match(CallLeafVector);
12417   effect(USE meth);
12418 
12419   ins_cost(300);
12420   format %{ "call_leaf,vector " %}
12421   ins_encode(Java_To_Runtime(meth));
12422   ins_pipe(pipe_slow);
12423 %}
12424 
12425 // Call runtime without safepoint
12426 instruct CallLeafNoFPDirect(method meth)
12427 %{
12428   match(CallLeafNoFP);
12429   effect(USE meth);
12430 
12431   ins_cost(300);
12432   format %{ "call_leaf_nofp,runtime " %}
12433   ins_encode(clear_avx, Java_To_Runtime(meth));
12434   ins_pipe(pipe_slow);
12435 %}
12436 
12437 // Return Instruction
12438 // Remove the return address & jump to it.
12439 // Notice: We always emit a nop after a ret to make sure there is room
12440 // for safepoint patching
12441 instruct Ret()
12442 %{
12443   match(Return);
12444 
12445   format %{ "ret" %}
12446   ins_encode %{
12447     __ ret(0);
12448   %}
12449   ins_pipe(pipe_jmp);
12450 %}
12451 
12452 // Tail Call; Jump from runtime stub to Java code.
12453 // Also known as an 'interprocedural jump'.
12454 // Target of jump will eventually return to caller.
12455 // TailJump below removes the return address.
12456 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
12457 // emitted just above the TailCall which has reset rbp to the caller state.
12458 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12459 %{
12460   match(TailCall jump_target method_ptr);
12461 
12462   ins_cost(300);
12463   format %{ "jmp     $jump_target\t# rbx holds method" %}
12464   ins_encode %{
12465     __ jmp($jump_target$$Register);
12466   %}
12467   ins_pipe(pipe_jmp);
12468 %}
12469 
12470 // Tail Jump; remove the return address; jump to target.
12471 // TailCall above leaves the return address around.
12472 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12473 %{
12474   match(TailJump jump_target ex_oop);
12475 
12476   ins_cost(300);
12477   format %{ "popq    rdx\t# pop return address\n\t"
12478             "jmp     $jump_target" %}
12479   ins_encode %{
12480     __ popq(as_Register(RDX_enc));
12481     __ jmp($jump_target$$Register);
12482   %}
12483   ins_pipe(pipe_jmp);
12484 %}
12485 
12486 // Create exception oop: created by stack-crawling runtime code.
12487 // Created exception is now available to this handler, and is setup
12488 // just prior to jumping to this handler.  No code emitted.
12489 instruct CreateException(rax_RegP ex_oop)
12490 %{
12491   match(Set ex_oop (CreateEx));
12492 
12493   size(0);
12494   // use the following format syntax
12495   format %{ "# exception oop is in rax; no code emitted" %}
12496   ins_encode();
12497   ins_pipe(empty);
12498 %}
12499 
12500 // Rethrow exception:
12501 // The exception oop will come in the first argument position.
12502 // Then JUMP (not call) to the rethrow stub code.
12503 instruct RethrowException()
12504 %{
12505   match(Rethrow);
12506 
12507   // use the following format syntax
12508   format %{ "jmp     rethrow_stub" %}
12509   ins_encode %{
12510     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12511   %}
12512   ins_pipe(pipe_jmp);
12513 %}
12514 
12515 // ============================================================================
12516 // This name is KNOWN by the ADLC and cannot be changed.
12517 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12518 // for this guy.
12519 instruct tlsLoadP(r15_RegP dst) %{
12520   match(Set dst (ThreadLocal));
12521   effect(DEF dst);
12522 
12523   size(0);
12524   format %{ "# TLS is in R15" %}
12525   ins_encode( /*empty encoding*/ );
12526   ins_pipe(ialu_reg_reg);
12527 %}
12528 
12529 
12530 //----------PEEPHOLE RULES-----------------------------------------------------
12531 // These must follow all instruction definitions as they use the names
12532 // defined in the instructions definitions.
12533 //
12534 // peeppredicate ( rule_predicate );
12535 // // the predicate unless which the peephole rule will be ignored
12536 //
12537 // peepmatch ( root_instr_name [preceding_instruction]* );
12538 //
12539 // peepprocedure ( procedure_name );
12540 // // provide a procedure name to perform the optimization, the procedure should
12541 // // reside in the architecture dependent peephole file, the method has the
12542 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12543 // // with the arguments being the basic block, the current node index inside the
12544 // // block, the register allocator, the functions upon invoked return a new node
12545 // // defined in peepreplace, and the rules of the nodes appearing in the
12546 // // corresponding peepmatch, the function return true if successful, else
12547 // // return false
12548 //
12549 // peepconstraint %{
12550 // (instruction_number.operand_name relational_op instruction_number.operand_name
12551 //  [, ...] );
12552 // // instruction numbers are zero-based using left to right order in peepmatch
12553 //
12554 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12555 // // provide an instruction_number.operand_name for each operand that appears
12556 // // in the replacement instruction's match rule
12557 //
12558 // ---------VM FLAGS---------------------------------------------------------
12559 //
12560 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12561 //
12562 // Each peephole rule is given an identifying number starting with zero and
12563 // increasing by one in the order seen by the parser.  An individual peephole
12564 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12565 // on the command-line.
12566 //
12567 // ---------CURRENT LIMITATIONS----------------------------------------------
12568 //
12569 // Only transformations inside a basic block (do we need more for peephole)
12570 //
12571 // ---------EXAMPLE----------------------------------------------------------
12572 //
12573 // // pertinent parts of existing instructions in architecture description
12574 // instruct movI(rRegI dst, rRegI src)
12575 // %{
12576 //   match(Set dst (CopyI src));
12577 // %}
12578 //
12579 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12580 // %{
12581 //   match(Set dst (AddI dst src));
12582 //   effect(KILL cr);
12583 // %}
12584 //
12585 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12586 // %{
12587 //   match(Set dst (AddI dst src));
12588 // %}
12589 //
12590 // 1. Simple replacement
12591 // - Only match adjacent instructions in same basic block
12592 // - Only equality constraints
12593 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
12594 // - Only one replacement instruction
12595 //
12596 // // Change (inc mov) to lea
12597 // peephole %{
12598 //   // lea should only be emitted when beneficial
12599 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12600 //   // increment preceded by register-register move
12601 //   peepmatch ( incI_rReg movI );
12602 //   // require that the destination register of the increment
12603 //   // match the destination register of the move
12604 //   peepconstraint ( 0.dst == 1.dst );
12605 //   // construct a replacement instruction that sets
12606 //   // the destination to ( move's source register + one )
12607 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12608 // %}
12609 //
12610 // 2. Procedural replacement
12611 // - More flexible finding relevent nodes
12612 // - More flexible constraints
12613 // - More flexible transformations
12614 // - May utilise architecture-dependent API more effectively
12615 // - Currently only one replacement instruction due to adlc parsing capabilities
12616 //
12617 // // Change (inc mov) to lea
12618 // peephole %{
12619 //   // lea should only be emitted when beneficial
12620 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12621 //   // the rule numbers of these nodes inside are passed into the function below
12622 //   peepmatch ( incI_rReg movI );
12623 //   // the method that takes the responsibility of transformation
12624 //   peepprocedure ( inc_mov_to_lea );
12625 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
12626 //   // node is passed into the function above
12627 //   peepreplace ( leaI_rReg_immI() );
12628 // %}
12629 
12630 // These instructions is not matched by the matcher but used by the peephole
12631 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
12632 %{
12633   predicate(false);
12634   match(Set dst (AddI src1 src2));
12635   format %{ "leal    $dst, [$src1 + $src2]" %}
12636   ins_encode %{
12637     Register dst = $dst$$Register;
12638     Register src1 = $src1$$Register;
12639     Register src2 = $src2$$Register;
12640     if (src1 != rbp && src1 != r13) {
12641       __ leal(dst, Address(src1, src2, Address::times_1));
12642     } else {
12643       assert(src2 != rbp && src2 != r13, "");
12644       __ leal(dst, Address(src2, src1, Address::times_1));
12645     }
12646   %}
12647   ins_pipe(ialu_reg_reg);
12648 %}
12649 
12650 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
12651 %{
12652   predicate(false);
12653   match(Set dst (AddI src1 src2));
12654   format %{ "leal    $dst, [$src1 + $src2]" %}
12655   ins_encode %{
12656     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
12657   %}
12658   ins_pipe(ialu_reg_reg);
12659 %}
12660 
12661 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
12662 %{
12663   predicate(false);
12664   match(Set dst (LShiftI src shift));
12665   format %{ "leal    $dst, [$src << $shift]" %}
12666   ins_encode %{
12667     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12668     Register src = $src$$Register;
12669     if (scale == Address::times_2 && src != rbp && src != r13) {
12670       __ leal($dst$$Register, Address(src, src, Address::times_1));
12671     } else {
12672       __ leal($dst$$Register, Address(noreg, src, scale));
12673     }
12674   %}
12675   ins_pipe(ialu_reg_reg);
12676 %}
12677 
12678 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
12679 %{
12680   predicate(false);
12681   match(Set dst (AddL src1 src2));
12682   format %{ "leaq    $dst, [$src1 + $src2]" %}
12683   ins_encode %{
12684     Register dst = $dst$$Register;
12685     Register src1 = $src1$$Register;
12686     Register src2 = $src2$$Register;
12687     if (src1 != rbp && src1 != r13) {
12688       __ leaq(dst, Address(src1, src2, Address::times_1));
12689     } else {
12690       assert(src2 != rbp && src2 != r13, "");
12691       __ leaq(dst, Address(src2, src1, Address::times_1));
12692     }
12693   %}
12694   ins_pipe(ialu_reg_reg);
12695 %}
12696 
12697 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
12698 %{
12699   predicate(false);
12700   match(Set dst (AddL src1 src2));
12701   format %{ "leaq    $dst, [$src1 + $src2]" %}
12702   ins_encode %{
12703     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
12704   %}
12705   ins_pipe(ialu_reg_reg);
12706 %}
12707 
12708 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
12709 %{
12710   predicate(false);
12711   match(Set dst (LShiftL src shift));
12712   format %{ "leaq    $dst, [$src << $shift]" %}
12713   ins_encode %{
12714     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12715     Register src = $src$$Register;
12716     if (scale == Address::times_2 && src != rbp && src != r13) {
12717       __ leaq($dst$$Register, Address(src, src, Address::times_1));
12718     } else {
12719       __ leaq($dst$$Register, Address(noreg, src, scale));
12720     }
12721   %}
12722   ins_pipe(ialu_reg_reg);
12723 %}
12724 
12725 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
12726 // sal}) with lea instructions. The {add, sal} rules are beneficial in
12727 // processors with at least partial ALU support for lea
12728 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
12729 // beneficial for processors with full ALU support
12730 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
12731 
12732 peephole
12733 %{
12734   peeppredicate(VM_Version::supports_fast_2op_lea());
12735   peepmatch (addI_rReg);
12736   peepprocedure (lea_coalesce_reg);
12737   peepreplace (leaI_rReg_rReg_peep());
12738 %}
12739 
12740 peephole
12741 %{
12742   peeppredicate(VM_Version::supports_fast_2op_lea());
12743   peepmatch (addI_rReg_imm);
12744   peepprocedure (lea_coalesce_imm);
12745   peepreplace (leaI_rReg_immI_peep());
12746 %}
12747 
12748 peephole
12749 %{
12750   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12751                 VM_Version::is_intel_cascade_lake());
12752   peepmatch (incI_rReg);
12753   peepprocedure (lea_coalesce_imm);
12754   peepreplace (leaI_rReg_immI_peep());
12755 %}
12756 
12757 peephole
12758 %{
12759   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12760                 VM_Version::is_intel_cascade_lake());
12761   peepmatch (decI_rReg);
12762   peepprocedure (lea_coalesce_imm);
12763   peepreplace (leaI_rReg_immI_peep());
12764 %}
12765 
12766 peephole
12767 %{
12768   peeppredicate(VM_Version::supports_fast_2op_lea());
12769   peepmatch (salI_rReg_immI2);
12770   peepprocedure (lea_coalesce_imm);
12771   peepreplace (leaI_rReg_immI2_peep());
12772 %}
12773 
12774 peephole
12775 %{
12776   peeppredicate(VM_Version::supports_fast_2op_lea());
12777   peepmatch (addL_rReg);
12778   peepprocedure (lea_coalesce_reg);
12779   peepreplace (leaL_rReg_rReg_peep());
12780 %}
12781 
12782 peephole
12783 %{
12784   peeppredicate(VM_Version::supports_fast_2op_lea());
12785   peepmatch (addL_rReg_imm);
12786   peepprocedure (lea_coalesce_imm);
12787   peepreplace (leaL_rReg_immL32_peep());
12788 %}
12789 
12790 peephole
12791 %{
12792   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12793                 VM_Version::is_intel_cascade_lake());
12794   peepmatch (incL_rReg);
12795   peepprocedure (lea_coalesce_imm);
12796   peepreplace (leaL_rReg_immL32_peep());
12797 %}
12798 
12799 peephole
12800 %{
12801   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12802                 VM_Version::is_intel_cascade_lake());
12803   peepmatch (decL_rReg);
12804   peepprocedure (lea_coalesce_imm);
12805   peepreplace (leaL_rReg_immL32_peep());
12806 %}
12807 
12808 peephole
12809 %{
12810   peeppredicate(VM_Version::supports_fast_2op_lea());
12811   peepmatch (salL_rReg_immI2);
12812   peepprocedure (lea_coalesce_imm);
12813   peepreplace (leaL_rReg_immI2_peep());
12814 %}
12815 
12816 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
12817 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
12818 
12819 //int variant
12820 peephole
12821 %{
12822   peepmatch (testI_reg);
12823   peepprocedure (test_may_remove);
12824 %}
12825 
12826 //long variant
12827 peephole
12828 %{
12829   peepmatch (testL_reg);
12830   peepprocedure (test_may_remove);
12831 %}
12832 
12833 
12834 //----------SMARTSPILL RULES---------------------------------------------------
12835 // These must follow all instruction definitions as they use the names
12836 // defined in the instructions definitions.