1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RDX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RDI int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 
  323 source_hpp %{
  324 
  325 #include "peephole_x86_64.hpp"
  326 
  327 %}
  328 
  329 // Register masks
  330 source_hpp %{
  331 
  332 extern RegMask _ANY_REG_mask;
  333 extern RegMask _PTR_REG_mask;
  334 extern RegMask _PTR_REG_NO_RBP_mask;
  335 extern RegMask _PTR_NO_RAX_REG_mask;
  336 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  337 extern RegMask _LONG_REG_mask;
  338 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  339 extern RegMask _LONG_NO_RCX_REG_mask;
  340 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  341 extern RegMask _INT_REG_mask;
  342 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  343 extern RegMask _INT_NO_RCX_REG_mask;
  344 extern RegMask _INT_NO_RBP_R13_REG_mask;
  345 extern RegMask _FLOAT_REG_mask;
  346 
  347 extern RegMask _STACK_OR_PTR_REG_mask;
  348 extern RegMask _STACK_OR_LONG_REG_mask;
  349 extern RegMask _STACK_OR_INT_REG_mask;
  350 
  351 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  352 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  353 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_INT_REG_mask;
  380 
  381 static bool need_r12_heapbase() {
  382   return UseCompressedOops;
  383 }
  384 
  385 void reg_mask_init() {
  386   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  387   // We derive a number of subsets from it.
  388   _ANY_REG_mask = _ALL_REG_mask;
  389 
  390   if (PreserveFramePointer) {
  391     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  392     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  393   }
  394   if (need_r12_heapbase()) {
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  396     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  397   }
  398 
  399   _PTR_REG_mask = _ANY_REG_mask;
  400   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  401   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  404 
  405   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  406   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  407 
  408   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  409   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  410   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  411 
  412   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  413   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  414   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  415 
  416   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  417   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  418   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  419 
  420   _LONG_REG_mask = _PTR_REG_mask;
  421   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  422   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  423 
  424   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  425   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  426   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  429 
  430   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  431   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  432   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  433 
  434   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  435   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  436   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  439 
  440   _INT_REG_mask = _ALL_INT_REG_mask;
  441   if (PreserveFramePointer) {
  442     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  443   }
  444   if (need_r12_heapbase()) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  446   }
  447 
  448   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  449   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  450 
  451   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  452   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  453   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  454 
  455   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  456   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  457 
  458   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  459   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  460   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  461 
  462   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  463   // from the float_reg_legacy/float_reg_evex register class.
  464   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  465 }
  466 
  467 static bool generate_vzeroupper(Compile* C) {
  468   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  469 }
  470 
  471 static int clear_avx_size() {
  472   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  473 }
  474 
  475 // !!!!! Special hack to get all types of calls to specify the byte offset
  476 //       from the start of the call to the point where the return address
  477 //       will point.
  478 int MachCallStaticJavaNode::ret_addr_offset()
  479 {
  480   int offset = 5; // 5 bytes from start of call to where return address points
  481   offset += clear_avx_size();
  482   return offset;
  483 }
  484 
  485 int MachCallDynamicJavaNode::ret_addr_offset()
  486 {
  487   int offset = 15; // 15 bytes from start of call to where return address points
  488   offset += clear_avx_size();
  489   return offset;
  490 }
  491 
  492 int MachCallRuntimeNode::ret_addr_offset() {
  493   int offset = 13; // movq r10,#addr; callq (r10)
  494   if (this->ideal_Opcode() != Op_CallLeafVector) {
  495     offset += clear_avx_size();
  496   }
  497   return offset;
  498 }
  499 //
  500 // Compute padding required for nodes which need alignment
  501 //
  502 
  503 // The address of the call instruction needs to be 4-byte aligned to
  504 // ensure that it does not span a cache line so that it can be patched.
  505 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  506 {
  507   current_offset += clear_avx_size(); // skip vzeroupper
  508   current_offset += 1; // skip call opcode byte
  509   return align_up(current_offset, alignment_required()) - current_offset;
  510 }
  511 
  512 // The address of the call instruction needs to be 4-byte aligned to
  513 // ensure that it does not span a cache line so that it can be patched.
  514 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  515 {
  516   current_offset += clear_avx_size(); // skip vzeroupper
  517   current_offset += 11; // skip movq instruction + call opcode byte
  518   return align_up(current_offset, alignment_required()) - current_offset;
  519 }
  520 
  521 // This could be in MacroAssembler but it's fairly C2 specific
  522 static void emit_cmpfp_fixup(MacroAssembler& _masm) {
  523   Label exit;
  524   __ jccb(Assembler::noParity, exit);
  525   __ pushf();
  526   //
  527   // comiss/ucomiss instructions set ZF,PF,CF flags and
  528   // zero OF,AF,SF for NaN values.
  529   // Fixup flags by zeroing ZF,PF so that compare of NaN
  530   // values returns 'less than' result (CF is set).
  531   // Leave the rest of flags unchanged.
  532   //
  533   //    7 6 5 4 3 2 1 0
  534   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  535   //    0 0 1 0 1 0 1 1   (0x2B)
  536   //
  537   __ andq(Address(rsp, 0), 0xffffff2b);
  538   __ popf();
  539   __ bind(exit);
  540 }
  541 
  542 static void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  543   Label done;
  544   __ movl(dst, -1);
  545   __ jcc(Assembler::parity, done);
  546   __ jcc(Assembler::below, done);
  547   __ setb(Assembler::notEqual, dst);
  548   __ movzbl(dst, dst);
  549   __ bind(done);
  550 }
  551 
  552 // Math.min()    # Math.max()
  553 // --------------------------
  554 // ucomis[s/d]   #
  555 // ja   -> b     # a
  556 // jp   -> NaN   # NaN
  557 // jb   -> a     # b
  558 // je            #
  559 // |-jz -> a | b # a & b
  560 // |    -> a     #
  561 static void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  562                             XMMRegister a, XMMRegister b,
  563                             XMMRegister xmmt, Register rt,
  564                             bool min, bool single) {
  565 
  566   Label nan, zero, below, above, done;
  567 
  568   if (single)
  569     __ ucomiss(a, b);
  570   else
  571     __ ucomisd(a, b);
  572 
  573   if (dst->encoding() != (min ? b : a)->encoding())
  574     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  575   else
  576     __ jccb(Assembler::above, done);
  577 
  578   __ jccb(Assembler::parity, nan);  // PF=1
  579   __ jccb(Assembler::below, below); // CF=1
  580 
  581   // equal
  582   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  583   if (single) {
  584     __ ucomiss(a, xmmt);
  585     __ jccb(Assembler::equal, zero);
  586 
  587     __ movflt(dst, a);
  588     __ jmp(done);
  589   }
  590   else {
  591     __ ucomisd(a, xmmt);
  592     __ jccb(Assembler::equal, zero);
  593 
  594     __ movdbl(dst, a);
  595     __ jmp(done);
  596   }
  597 
  598   __ bind(zero);
  599   if (min)
  600     __ vpor(dst, a, b, Assembler::AVX_128bit);
  601   else
  602     __ vpand(dst, a, b, Assembler::AVX_128bit);
  603 
  604   __ jmp(done);
  605 
  606   __ bind(above);
  607   if (single)
  608     __ movflt(dst, min ? b : a);
  609   else
  610     __ movdbl(dst, min ? b : a);
  611 
  612   __ jmp(done);
  613 
  614   __ bind(nan);
  615   if (single) {
  616     __ movl(rt, 0x7fc00000); // Float.NaN
  617     __ movdl(dst, rt);
  618   }
  619   else {
  620     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  621     __ movdq(dst, rt);
  622   }
  623   __ jmp(done);
  624 
  625   __ bind(below);
  626   if (single)
  627     __ movflt(dst, min ? a : b);
  628   else
  629     __ movdbl(dst, min ? a : b);
  630 
  631   __ bind(done);
  632 }
  633 
  634 //=============================================================================
  635 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  636 
  637 int ConstantTable::calculate_table_base_offset() const {
  638   return 0;  // absolute addressing, no offset
  639 }
  640 
  641 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  642 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  643   ShouldNotReachHere();
  644 }
  645 
  646 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  647   // Empty encoding
  648 }
  649 
  650 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  651   return 0;
  652 }
  653 
  654 #ifndef PRODUCT
  655 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  656   st->print("# MachConstantBaseNode (empty encoding)");
  657 }
  658 #endif
  659 
  660 
  661 //=============================================================================
  662 #ifndef PRODUCT
  663 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  664   Compile* C = ra_->C;
  665 
  666   int framesize = C->output()->frame_size_in_bytes();
  667   int bangsize = C->output()->bang_size_in_bytes();
  668   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  669   // Remove wordSize for return addr which is already pushed.
  670   framesize -= wordSize;
  671 
  672   if (C->output()->need_stack_bang(bangsize)) {
  673     framesize -= wordSize;
  674     st->print("# stack bang (%d bytes)", bangsize);
  675     st->print("\n\t");
  676     st->print("pushq   rbp\t# Save rbp");
  677     if (PreserveFramePointer) {
  678         st->print("\n\t");
  679         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  680     }
  681     if (framesize) {
  682       st->print("\n\t");
  683       st->print("subq    rsp, #%d\t# Create frame",framesize);
  684     }
  685   } else {
  686     st->print("subq    rsp, #%d\t# Create frame",framesize);
  687     st->print("\n\t");
  688     framesize -= wordSize;
  689     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  690     if (PreserveFramePointer) {
  691       st->print("\n\t");
  692       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  693       if (framesize > 0) {
  694         st->print("\n\t");
  695         st->print("addq    rbp, #%d", framesize);
  696       }
  697     }
  698   }
  699 
  700   if (VerifyStackAtCalls) {
  701     st->print("\n\t");
  702     framesize -= wordSize;
  703     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  704 #ifdef ASSERT
  705     st->print("\n\t");
  706     st->print("# stack alignment check");
  707 #endif
  708   }
  709   if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
  710     st->print("\n\t");
  711     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  712     st->print("\n\t");
  713     st->print("je      fast_entry\t");
  714     st->print("\n\t");
  715     st->print("call    #nmethod_entry_barrier_stub\t");
  716     st->print("\n\tfast_entry:");
  717   }
  718   st->cr();
  719 }
  720 #endif
  721 
  722 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  723   Compile* C = ra_->C;
  724   C2_MacroAssembler _masm(&cbuf);
  725 
  726   int framesize = C->output()->frame_size_in_bytes();
  727   int bangsize = C->output()->bang_size_in_bytes();
  728 
  729   if (C->clinit_barrier_on_entry()) {
  730     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  731     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  732 
  733     Label L_skip_barrier;
  734     Register klass = rscratch1;
  735 
  736     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  737     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  738 
  739     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  740 
  741     __ bind(L_skip_barrier);
  742   }
  743 
  744   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  745 
  746   C->output()->set_frame_complete(cbuf.insts_size());
  747 
  748   if (C->has_mach_constant_base_node()) {
  749     // NOTE: We set the table base offset here because users might be
  750     // emitted before MachConstantBaseNode.
  751     ConstantTable& constant_table = C->output()->constant_table();
  752     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  753   }
  754 }
  755 
  756 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  757 {
  758   return MachNode::size(ra_); // too many variables; just compute it
  759                               // the hard way
  760 }
  761 
  762 int MachPrologNode::reloc() const
  763 {
  764   return 0; // a large enough number
  765 }
  766 
  767 //=============================================================================
  768 #ifndef PRODUCT
  769 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  770 {
  771   Compile* C = ra_->C;
  772   if (generate_vzeroupper(C)) {
  773     st->print("vzeroupper");
  774     st->cr(); st->print("\t");
  775   }
  776 
  777   int framesize = C->output()->frame_size_in_bytes();
  778   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  779   // Remove word for return adr already pushed
  780   // and RBP
  781   framesize -= 2*wordSize;
  782 
  783   if (framesize) {
  784     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  785     st->print("\t");
  786   }
  787 
  788   st->print_cr("popq    rbp");
  789   if (do_polling() && C->is_method_compilation()) {
  790     st->print("\t");
  791     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  792                  "ja      #safepoint_stub\t"
  793                  "# Safepoint: poll for GC");
  794   }
  795 }
  796 #endif
  797 
  798 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  799 {
  800   Compile* C = ra_->C;
  801   MacroAssembler _masm(&cbuf);
  802 
  803   if (generate_vzeroupper(C)) {
  804     // Clear upper bits of YMM registers when current compiled code uses
  805     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  806     __ vzeroupper();
  807   }
  808 
  809   int framesize = C->output()->frame_size_in_bytes();
  810   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  811   // Remove word for return adr already pushed
  812   // and RBP
  813   framesize -= 2*wordSize;
  814 
  815   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  816 
  817   if (framesize) {
  818     __ addq(rsp, framesize);
  819   }
  820 
  821   __ popq(rbp);
  822 
  823   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  824     __ reserved_stack_check();
  825   }
  826 
  827   if (do_polling() && C->is_method_compilation()) {
  828     MacroAssembler _masm(&cbuf);
  829     Label dummy_label;
  830     Label* code_stub = &dummy_label;
  831     if (!C->output()->in_scratch_emit_size()) {
  832       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  833       C->output()->add_stub(stub);
  834       code_stub = &stub->entry();
  835     }
  836     __ relocate(relocInfo::poll_return_type);
  837     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  838   }
  839 }
  840 
  841 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  842 {
  843   return MachNode::size(ra_); // too many variables; just compute it
  844                               // the hard way
  845 }
  846 
  847 int MachEpilogNode::reloc() const
  848 {
  849   return 2; // a large enough number
  850 }
  851 
  852 const Pipeline* MachEpilogNode::pipeline() const
  853 {
  854   return MachNode::pipeline_class();
  855 }
  856 
  857 //=============================================================================
  858 
  859 enum RC {
  860   rc_bad,
  861   rc_int,
  862   rc_kreg,
  863   rc_float,
  864   rc_stack
  865 };
  866 
  867 static enum RC rc_class(OptoReg::Name reg)
  868 {
  869   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  870 
  871   if (OptoReg::is_stack(reg)) return rc_stack;
  872 
  873   VMReg r = OptoReg::as_VMReg(reg);
  874 
  875   if (r->is_Register()) return rc_int;
  876 
  877   if (r->is_KRegister()) return rc_kreg;
  878 
  879   assert(r->is_XMMRegister(), "must be");
  880   return rc_float;
  881 }
  882 
  883 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  884 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
  885                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  886 
  887 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
  888                      int stack_offset, int reg, uint ireg, outputStream* st);
  889 
  890 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
  891                                       int dst_offset, uint ireg, outputStream* st) {
  892   if (cbuf) {
  893     MacroAssembler _masm(cbuf);
  894     switch (ireg) {
  895     case Op_VecS:
  896       __ movq(Address(rsp, -8), rax);
  897       __ movl(rax, Address(rsp, src_offset));
  898       __ movl(Address(rsp, dst_offset), rax);
  899       __ movq(rax, Address(rsp, -8));
  900       break;
  901     case Op_VecD:
  902       __ pushq(Address(rsp, src_offset));
  903       __ popq (Address(rsp, dst_offset));
  904       break;
  905     case Op_VecX:
  906       __ pushq(Address(rsp, src_offset));
  907       __ popq (Address(rsp, dst_offset));
  908       __ pushq(Address(rsp, src_offset+8));
  909       __ popq (Address(rsp, dst_offset+8));
  910       break;
  911     case Op_VecY:
  912       __ vmovdqu(Address(rsp, -32), xmm0);
  913       __ vmovdqu(xmm0, Address(rsp, src_offset));
  914       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  915       __ vmovdqu(xmm0, Address(rsp, -32));
  916       break;
  917     case Op_VecZ:
  918       __ evmovdquq(Address(rsp, -64), xmm0, 2);
  919       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
  920       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
  921       __ evmovdquq(xmm0, Address(rsp, -64), 2);
  922       break;
  923     default:
  924       ShouldNotReachHere();
  925     }
  926 #ifndef PRODUCT
  927   } else {
  928     switch (ireg) {
  929     case Op_VecS:
  930       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
  931                 "movl    rax, [rsp + #%d]\n\t"
  932                 "movl    [rsp + #%d], rax\n\t"
  933                 "movq    rax, [rsp - #8]",
  934                 src_offset, dst_offset);
  935       break;
  936     case Op_VecD:
  937       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
  938                 "popq    [rsp + #%d]",
  939                 src_offset, dst_offset);
  940       break;
  941      case Op_VecX:
  942       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
  943                 "popq    [rsp + #%d]\n\t"
  944                 "pushq   [rsp + #%d]\n\t"
  945                 "popq    [rsp + #%d]",
  946                 src_offset, dst_offset, src_offset+8, dst_offset+8);
  947       break;
  948     case Op_VecY:
  949       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
  950                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  951                 "vmovdqu [rsp + #%d], xmm0\n\t"
  952                 "vmovdqu xmm0, [rsp - #32]",
  953                 src_offset, dst_offset);
  954       break;
  955     case Op_VecZ:
  956       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
  957                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  958                 "vmovdqu [rsp + #%d], xmm0\n\t"
  959                 "vmovdqu xmm0, [rsp - #64]",
  960                 src_offset, dst_offset);
  961       break;
  962     default:
  963       ShouldNotReachHere();
  964     }
  965 #endif
  966   }
  967 }
  968 
  969 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
  970                                        PhaseRegAlloc* ra_,
  971                                        bool do_size,
  972                                        outputStream* st) const {
  973   assert(cbuf != nullptr || st  != nullptr, "sanity");
  974   // Get registers to move
  975   OptoReg::Name src_second = ra_->get_reg_second(in(1));
  976   OptoReg::Name src_first = ra_->get_reg_first(in(1));
  977   OptoReg::Name dst_second = ra_->get_reg_second(this);
  978   OptoReg::Name dst_first = ra_->get_reg_first(this);
  979 
  980   enum RC src_second_rc = rc_class(src_second);
  981   enum RC src_first_rc = rc_class(src_first);
  982   enum RC dst_second_rc = rc_class(dst_second);
  983   enum RC dst_first_rc = rc_class(dst_first);
  984 
  985   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
  986          "must move at least 1 register" );
  987 
  988   if (src_first == dst_first && src_second == dst_second) {
  989     // Self copy, no move
  990     return 0;
  991   }
  992   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
  993     uint ireg = ideal_reg();
  994     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
  995     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
  996     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
  997       // mem -> mem
  998       int src_offset = ra_->reg2offset(src_first);
  999       int dst_offset = ra_->reg2offset(dst_first);
 1000       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1001     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1002       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1003     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1004       int stack_offset = ra_->reg2offset(dst_first);
 1005       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1006     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1007       int stack_offset = ra_->reg2offset(src_first);
 1008       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1009     } else {
 1010       ShouldNotReachHere();
 1011     }
 1012     return 0;
 1013   }
 1014   if (src_first_rc == rc_stack) {
 1015     // mem ->
 1016     if (dst_first_rc == rc_stack) {
 1017       // mem -> mem
 1018       assert(src_second != dst_first, "overlap");
 1019       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1020           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1021         // 64-bit
 1022         int src_offset = ra_->reg2offset(src_first);
 1023         int dst_offset = ra_->reg2offset(dst_first);
 1024         if (cbuf) {
 1025           MacroAssembler _masm(cbuf);
 1026           __ pushq(Address(rsp, src_offset));
 1027           __ popq (Address(rsp, dst_offset));
 1028 #ifndef PRODUCT
 1029         } else {
 1030           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1031                     "popq    [rsp + #%d]",
 1032                      src_offset, dst_offset);
 1033 #endif
 1034         }
 1035       } else {
 1036         // 32-bit
 1037         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1038         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1039         // No pushl/popl, so:
 1040         int src_offset = ra_->reg2offset(src_first);
 1041         int dst_offset = ra_->reg2offset(dst_first);
 1042         if (cbuf) {
 1043           MacroAssembler _masm(cbuf);
 1044           __ movq(Address(rsp, -8), rax);
 1045           __ movl(rax, Address(rsp, src_offset));
 1046           __ movl(Address(rsp, dst_offset), rax);
 1047           __ movq(rax, Address(rsp, -8));
 1048 #ifndef PRODUCT
 1049         } else {
 1050           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                     "movl    rax, [rsp + #%d]\n\t"
 1052                     "movl    [rsp + #%d], rax\n\t"
 1053                     "movq    rax, [rsp - #8]",
 1054                      src_offset, dst_offset);
 1055 #endif
 1056         }
 1057       }
 1058       return 0;
 1059     } else if (dst_first_rc == rc_int) {
 1060       // mem -> gpr
 1061       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1062           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1063         // 64-bit
 1064         int offset = ra_->reg2offset(src_first);
 1065         if (cbuf) {
 1066           MacroAssembler _masm(cbuf);
 1067           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1068 #ifndef PRODUCT
 1069         } else {
 1070           st->print("movq    %s, [rsp + #%d]\t# spill",
 1071                      Matcher::regName[dst_first],
 1072                      offset);
 1073 #endif
 1074         }
 1075       } else {
 1076         // 32-bit
 1077         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1078         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1079         int offset = ra_->reg2offset(src_first);
 1080         if (cbuf) {
 1081           MacroAssembler _masm(cbuf);
 1082           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1083 #ifndef PRODUCT
 1084         } else {
 1085           st->print("movl    %s, [rsp + #%d]\t# spill",
 1086                      Matcher::regName[dst_first],
 1087                      offset);
 1088 #endif
 1089         }
 1090       }
 1091       return 0;
 1092     } else if (dst_first_rc == rc_float) {
 1093       // mem-> xmm
 1094       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1095           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1096         // 64-bit
 1097         int offset = ra_->reg2offset(src_first);
 1098         if (cbuf) {
 1099           MacroAssembler _masm(cbuf);
 1100           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1101 #ifndef PRODUCT
 1102         } else {
 1103           st->print("%s  %s, [rsp + #%d]\t# spill",
 1104                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1105                      Matcher::regName[dst_first],
 1106                      offset);
 1107 #endif
 1108         }
 1109       } else {
 1110         // 32-bit
 1111         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1112         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1113         int offset = ra_->reg2offset(src_first);
 1114         if (cbuf) {
 1115           MacroAssembler _masm(cbuf);
 1116           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1117 #ifndef PRODUCT
 1118         } else {
 1119           st->print("movss   %s, [rsp + #%d]\t# spill",
 1120                      Matcher::regName[dst_first],
 1121                      offset);
 1122 #endif
 1123         }
 1124       }
 1125       return 0;
 1126     } else if (dst_first_rc == rc_kreg) {
 1127       // mem -> kreg
 1128       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1129           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1130         // 64-bit
 1131         int offset = ra_->reg2offset(src_first);
 1132         if (cbuf) {
 1133           MacroAssembler _masm(cbuf);
 1134           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1135 #ifndef PRODUCT
 1136         } else {
 1137           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1138                      Matcher::regName[dst_first],
 1139                      offset);
 1140 #endif
 1141         }
 1142       }
 1143       return 0;
 1144     }
 1145   } else if (src_first_rc == rc_int) {
 1146     // gpr ->
 1147     if (dst_first_rc == rc_stack) {
 1148       // gpr -> mem
 1149       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1150           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1151         // 64-bit
 1152         int offset = ra_->reg2offset(dst_first);
 1153         if (cbuf) {
 1154           MacroAssembler _masm(cbuf);
 1155           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1156 #ifndef PRODUCT
 1157         } else {
 1158           st->print("movq    [rsp + #%d], %s\t# spill",
 1159                      offset,
 1160                      Matcher::regName[src_first]);
 1161 #endif
 1162         }
 1163       } else {
 1164         // 32-bit
 1165         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1166         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1167         int offset = ra_->reg2offset(dst_first);
 1168         if (cbuf) {
 1169           MacroAssembler _masm(cbuf);
 1170           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1171 #ifndef PRODUCT
 1172         } else {
 1173           st->print("movl    [rsp + #%d], %s\t# spill",
 1174                      offset,
 1175                      Matcher::regName[src_first]);
 1176 #endif
 1177         }
 1178       }
 1179       return 0;
 1180     } else if (dst_first_rc == rc_int) {
 1181       // gpr -> gpr
 1182       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1183           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1184         // 64-bit
 1185         if (cbuf) {
 1186           MacroAssembler _masm(cbuf);
 1187           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1188                   as_Register(Matcher::_regEncode[src_first]));
 1189 #ifndef PRODUCT
 1190         } else {
 1191           st->print("movq    %s, %s\t# spill",
 1192                      Matcher::regName[dst_first],
 1193                      Matcher::regName[src_first]);
 1194 #endif
 1195         }
 1196         return 0;
 1197       } else {
 1198         // 32-bit
 1199         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1200         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1201         if (cbuf) {
 1202           MacroAssembler _masm(cbuf);
 1203           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1204                   as_Register(Matcher::_regEncode[src_first]));
 1205 #ifndef PRODUCT
 1206         } else {
 1207           st->print("movl    %s, %s\t# spill",
 1208                      Matcher::regName[dst_first],
 1209                      Matcher::regName[src_first]);
 1210 #endif
 1211         }
 1212         return 0;
 1213       }
 1214     } else if (dst_first_rc == rc_float) {
 1215       // gpr -> xmm
 1216       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1217           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1218         // 64-bit
 1219         if (cbuf) {
 1220           MacroAssembler _masm(cbuf);
 1221           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1222 #ifndef PRODUCT
 1223         } else {
 1224           st->print("movdq   %s, %s\t# spill",
 1225                      Matcher::regName[dst_first],
 1226                      Matcher::regName[src_first]);
 1227 #endif
 1228         }
 1229       } else {
 1230         // 32-bit
 1231         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1232         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1233         if (cbuf) {
 1234           MacroAssembler _masm(cbuf);
 1235           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1236 #ifndef PRODUCT
 1237         } else {
 1238           st->print("movdl   %s, %s\t# spill",
 1239                      Matcher::regName[dst_first],
 1240                      Matcher::regName[src_first]);
 1241 #endif
 1242         }
 1243       }
 1244       return 0;
 1245     } else if (dst_first_rc == rc_kreg) {
 1246       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1247           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1248         // 64-bit
 1249         if (cbuf) {
 1250           MacroAssembler _masm(cbuf);
 1251           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1252   #ifndef PRODUCT
 1253         } else {
 1254            st->print("kmovq   %s, %s\t# spill",
 1255                        Matcher::regName[dst_first],
 1256                        Matcher::regName[src_first]);
 1257   #endif
 1258         }
 1259       }
 1260       Unimplemented();
 1261       return 0;
 1262     }
 1263   } else if (src_first_rc == rc_float) {
 1264     // xmm ->
 1265     if (dst_first_rc == rc_stack) {
 1266       // xmm -> mem
 1267       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1268           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1269         // 64-bit
 1270         int offset = ra_->reg2offset(dst_first);
 1271         if (cbuf) {
 1272           MacroAssembler _masm(cbuf);
 1273           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1274 #ifndef PRODUCT
 1275         } else {
 1276           st->print("movsd   [rsp + #%d], %s\t# spill",
 1277                      offset,
 1278                      Matcher::regName[src_first]);
 1279 #endif
 1280         }
 1281       } else {
 1282         // 32-bit
 1283         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1284         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1285         int offset = ra_->reg2offset(dst_first);
 1286         if (cbuf) {
 1287           MacroAssembler _masm(cbuf);
 1288           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1289 #ifndef PRODUCT
 1290         } else {
 1291           st->print("movss   [rsp + #%d], %s\t# spill",
 1292                      offset,
 1293                      Matcher::regName[src_first]);
 1294 #endif
 1295         }
 1296       }
 1297       return 0;
 1298     } else if (dst_first_rc == rc_int) {
 1299       // xmm -> gpr
 1300       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1301           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1302         // 64-bit
 1303         if (cbuf) {
 1304           MacroAssembler _masm(cbuf);
 1305           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1306 #ifndef PRODUCT
 1307         } else {
 1308           st->print("movdq   %s, %s\t# spill",
 1309                      Matcher::regName[dst_first],
 1310                      Matcher::regName[src_first]);
 1311 #endif
 1312         }
 1313       } else {
 1314         // 32-bit
 1315         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1316         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1317         if (cbuf) {
 1318           MacroAssembler _masm(cbuf);
 1319           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1320 #ifndef PRODUCT
 1321         } else {
 1322           st->print("movdl   %s, %s\t# spill",
 1323                      Matcher::regName[dst_first],
 1324                      Matcher::regName[src_first]);
 1325 #endif
 1326         }
 1327       }
 1328       return 0;
 1329     } else if (dst_first_rc == rc_float) {
 1330       // xmm -> xmm
 1331       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1332           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1333         // 64-bit
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("%s  %s, %s\t# spill",
 1340                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1341                      Matcher::regName[dst_first],
 1342                      Matcher::regName[src_first]);
 1343 #endif
 1344         }
 1345       } else {
 1346         // 32-bit
 1347         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1348         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("%s  %s, %s\t# spill",
 1355                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1356                      Matcher::regName[dst_first],
 1357                      Matcher::regName[src_first]);
 1358 #endif
 1359         }
 1360       }
 1361       return 0;
 1362     } else if (dst_first_rc == rc_kreg) {
 1363       assert(false, "Illegal spilling");
 1364       return 0;
 1365     }
 1366   } else if (src_first_rc == rc_kreg) {
 1367     if (dst_first_rc == rc_stack) {
 1368       // mem -> kreg
 1369       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1370           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1371         // 64-bit
 1372         int offset = ra_->reg2offset(dst_first);
 1373         if (cbuf) {
 1374           MacroAssembler _masm(cbuf);
 1375           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1376 #ifndef PRODUCT
 1377         } else {
 1378           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1379                      offset,
 1380                      Matcher::regName[src_first]);
 1381 #endif
 1382         }
 1383       }
 1384       return 0;
 1385     } else if (dst_first_rc == rc_int) {
 1386       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1387           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1388         // 64-bit
 1389         if (cbuf) {
 1390           MacroAssembler _masm(cbuf);
 1391           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1392 #ifndef PRODUCT
 1393         } else {
 1394          st->print("kmovq   %s, %s\t# spill",
 1395                      Matcher::regName[dst_first],
 1396                      Matcher::regName[src_first]);
 1397 #endif
 1398         }
 1399       }
 1400       Unimplemented();
 1401       return 0;
 1402     } else if (dst_first_rc == rc_kreg) {
 1403       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1404           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1405         // 64-bit
 1406         if (cbuf) {
 1407           MacroAssembler _masm(cbuf);
 1408           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411          st->print("kmovq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       }
 1417       return 0;
 1418     } else if (dst_first_rc == rc_float) {
 1419       assert(false, "Illegal spill");
 1420       return 0;
 1421     }
 1422   }
 1423 
 1424   assert(0," foo ");
 1425   Unimplemented();
 1426   return 0;
 1427 }
 1428 
 1429 #ifndef PRODUCT
 1430 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1431   implementation(nullptr, ra_, false, st);
 1432 }
 1433 #endif
 1434 
 1435 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1436   implementation(&cbuf, ra_, false, nullptr);
 1437 }
 1438 
 1439 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1440   return MachNode::size(ra_);
 1441 }
 1442 
 1443 //=============================================================================
 1444 #ifndef PRODUCT
 1445 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1446 {
 1447   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1448   int reg = ra_->get_reg_first(this);
 1449   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1450             Matcher::regName[reg], offset);
 1451 }
 1452 #endif
 1453 
 1454 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1455 {
 1456   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1457   int reg = ra_->get_encode(this);
 1458 
 1459   MacroAssembler masm(&cbuf);
 1460   masm.lea(as_Register(reg), Address(rsp, offset));
 1461 }
 1462 
 1463 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1464 {
 1465   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1466   return (offset < 0x80) ? 5 : 8; // REX
 1467 }
 1468 
 1469 //=============================================================================
 1470 #ifndef PRODUCT
 1471 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1472 {
 1473   if (UseCompressedClassPointers) {
 1474     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1475     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1476   } else {
 1477     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1478     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1479   }
 1480   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1481 }
 1482 #endif
 1483 
 1484 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1485 {
 1486   MacroAssembler masm(&cbuf);
 1487   masm.ic_check(InteriorEntryAlignment);
 1488 }
 1489 
 1490 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1491 {
 1492   return MachNode::size(ra_); // too many variables; just compute it
 1493                               // the hard way
 1494 }
 1495 
 1496 
 1497 //=============================================================================
 1498 
 1499 bool Matcher::supports_vector_calling_convention(void) {
 1500   if (EnableVectorSupport && UseVectorStubs) {
 1501     return true;
 1502   }
 1503   return false;
 1504 }
 1505 
 1506 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1507   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1508   int lo = XMM0_num;
 1509   int hi = XMM0b_num;
 1510   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1511   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1512   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1513   return OptoRegPair(hi, lo);
 1514 }
 1515 
 1516 // Is this branch offset short enough that a short branch can be used?
 1517 //
 1518 // NOTE: If the platform does not provide any short branch variants, then
 1519 //       this method should return false for offset 0.
 1520 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1521   // The passed offset is relative to address of the branch.
 1522   // On 86 a branch displacement is calculated relative to address
 1523   // of a next instruction.
 1524   offset -= br_size;
 1525 
 1526   // the short version of jmpConUCF2 contains multiple branches,
 1527   // making the reach slightly less
 1528   if (rule == jmpConUCF2_rule)
 1529     return (-126 <= offset && offset <= 125);
 1530   return (-128 <= offset && offset <= 127);
 1531 }
 1532 
 1533 // Return whether or not this register is ever used as an argument.
 1534 // This function is used on startup to build the trampoline stubs in
 1535 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1536 // call in the trampoline, and arguments in those registers not be
 1537 // available to the callee.
 1538 bool Matcher::can_be_java_arg(int reg)
 1539 {
 1540   return
 1541     reg ==  RDI_num || reg == RDI_H_num ||
 1542     reg ==  RSI_num || reg == RSI_H_num ||
 1543     reg ==  RDX_num || reg == RDX_H_num ||
 1544     reg ==  RCX_num || reg == RCX_H_num ||
 1545     reg ==   R8_num || reg ==  R8_H_num ||
 1546     reg ==   R9_num || reg ==  R9_H_num ||
 1547     reg ==  R12_num || reg == R12_H_num ||
 1548     reg == XMM0_num || reg == XMM0b_num ||
 1549     reg == XMM1_num || reg == XMM1b_num ||
 1550     reg == XMM2_num || reg == XMM2b_num ||
 1551     reg == XMM3_num || reg == XMM3b_num ||
 1552     reg == XMM4_num || reg == XMM4b_num ||
 1553     reg == XMM5_num || reg == XMM5b_num ||
 1554     reg == XMM6_num || reg == XMM6b_num ||
 1555     reg == XMM7_num || reg == XMM7b_num;
 1556 }
 1557 
 1558 bool Matcher::is_spillable_arg(int reg)
 1559 {
 1560   return can_be_java_arg(reg);
 1561 }
 1562 
 1563 uint Matcher::int_pressure_limit()
 1564 {
 1565   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1566 }
 1567 
 1568 uint Matcher::float_pressure_limit()
 1569 {
 1570   // After experiment around with different values, the following default threshold
 1571   // works best for LCM's register pressure scheduling on x64.
 1572   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1573   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1574   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1575 }
 1576 
 1577 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1578   // In 64 bit mode a code which use multiply when
 1579   // devisor is constant is faster than hardware
 1580   // DIV instruction (it uses MulHiL).
 1581   return false;
 1582 }
 1583 
 1584 // Register for DIVI projection of divmodI
 1585 RegMask Matcher::divI_proj_mask() {
 1586   return INT_RAX_REG_mask();
 1587 }
 1588 
 1589 // Register for MODI projection of divmodI
 1590 RegMask Matcher::modI_proj_mask() {
 1591   return INT_RDX_REG_mask();
 1592 }
 1593 
 1594 // Register for DIVL projection of divmodL
 1595 RegMask Matcher::divL_proj_mask() {
 1596   return LONG_RAX_REG_mask();
 1597 }
 1598 
 1599 // Register for MODL projection of divmodL
 1600 RegMask Matcher::modL_proj_mask() {
 1601   return LONG_RDX_REG_mask();
 1602 }
 1603 
 1604 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1605 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1606     return NO_REG_mask();
 1607 }
 1608 
 1609 %}
 1610 
 1611 //----------ENCODING BLOCK-----------------------------------------------------
 1612 // This block specifies the encoding classes used by the compiler to
 1613 // output byte streams.  Encoding classes are parameterized macros
 1614 // used by Machine Instruction Nodes in order to generate the bit
 1615 // encoding of the instruction.  Operands specify their base encoding
 1616 // interface with the interface keyword.  There are currently
 1617 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1618 // COND_INTER.  REG_INTER causes an operand to generate a function
 1619 // which returns its register number when queried.  CONST_INTER causes
 1620 // an operand to generate a function which returns the value of the
 1621 // constant when queried.  MEMORY_INTER causes an operand to generate
 1622 // four functions which return the Base Register, the Index Register,
 1623 // the Scale Value, and the Offset Value of the operand when queried.
 1624 // COND_INTER causes an operand to generate six functions which return
 1625 // the encoding code (ie - encoding bits for the instruction)
 1626 // associated with each basic boolean condition for a conditional
 1627 // instruction.
 1628 //
 1629 // Instructions specify two basic values for encoding.  Again, a
 1630 // function is available to check if the constant displacement is an
 1631 // oop. They use the ins_encode keyword to specify their encoding
 1632 // classes (which must be a sequence of enc_class names, and their
 1633 // parameters, specified in the encoding block), and they use the
 1634 // opcode keyword to specify, in order, their primary, secondary, and
 1635 // tertiary opcode.  Only the opcode sections which a particular
 1636 // instruction needs for encoding need to be specified.
 1637 encode %{
 1638   enc_class cdql_enc(no_rax_rdx_RegI div)
 1639   %{
 1640     // Full implementation of Java idiv and irem; checks for
 1641     // special case as described in JVM spec., p.243 & p.271.
 1642     //
 1643     //         normal case                           special case
 1644     //
 1645     // input : rax: dividend                         min_int
 1646     //         reg: divisor                          -1
 1647     //
 1648     // output: rax: quotient  (= rax idiv reg)       min_int
 1649     //         rdx: remainder (= rax irem reg)       0
 1650     //
 1651     //  Code sequnce:
 1652     //
 1653     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1654     //    5:   75 07/08                jne    e <normal>
 1655     //    7:   33 d2                   xor    %edx,%edx
 1656     //  [div >= 8 -> offset + 1]
 1657     //  [REX_B]
 1658     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1659     //    c:   74 03/04                je     11 <done>
 1660     // 000000000000000e <normal>:
 1661     //    e:   99                      cltd
 1662     //  [div >= 8 -> offset + 1]
 1663     //  [REX_B]
 1664     //    f:   f7 f9                   idiv   $div
 1665     // 0000000000000011 <done>:
 1666     MacroAssembler _masm(&cbuf);
 1667     Label normal;
 1668     Label done;
 1669 
 1670     // cmp    $0x80000000,%eax
 1671     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1672 
 1673     // jne    e <normal>
 1674     __ jccb(Assembler::notEqual, normal);
 1675 
 1676     // xor    %edx,%edx
 1677     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1678 
 1679     // cmp    $0xffffffffffffffff,%ecx
 1680     __ cmpl($div$$Register, -1);
 1681 
 1682     // je     11 <done>
 1683     __ jccb(Assembler::equal, done);
 1684 
 1685     // <normal>
 1686     // cltd
 1687     __ bind(normal);
 1688     __ cdql();
 1689 
 1690     // idivl
 1691     // <done>
 1692     __ idivl($div$$Register);
 1693     __ bind(done);
 1694   %}
 1695 
 1696   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1697   %{
 1698     // Full implementation of Java ldiv and lrem; checks for
 1699     // special case as described in JVM spec., p.243 & p.271.
 1700     //
 1701     //         normal case                           special case
 1702     //
 1703     // input : rax: dividend                         min_long
 1704     //         reg: divisor                          -1
 1705     //
 1706     // output: rax: quotient  (= rax idiv reg)       min_long
 1707     //         rdx: remainder (= rax irem reg)       0
 1708     //
 1709     //  Code sequnce:
 1710     //
 1711     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1712     //    7:   00 00 80
 1713     //    a:   48 39 d0                cmp    %rdx,%rax
 1714     //    d:   75 08                   jne    17 <normal>
 1715     //    f:   33 d2                   xor    %edx,%edx
 1716     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1717     //   15:   74 05                   je     1c <done>
 1718     // 0000000000000017 <normal>:
 1719     //   17:   48 99                   cqto
 1720     //   19:   48 f7 f9                idiv   $div
 1721     // 000000000000001c <done>:
 1722     MacroAssembler _masm(&cbuf);
 1723     Label normal;
 1724     Label done;
 1725 
 1726     // mov    $0x8000000000000000,%rdx
 1727     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1728 
 1729     // cmp    %rdx,%rax
 1730     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1731 
 1732     // jne    17 <normal>
 1733     __ jccb(Assembler::notEqual, normal);
 1734 
 1735     // xor    %edx,%edx
 1736     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1737 
 1738     // cmp    $0xffffffffffffffff,$div
 1739     __ cmpq($div$$Register, -1);
 1740 
 1741     // je     1e <done>
 1742     __ jccb(Assembler::equal, done);
 1743 
 1744     // <normal>
 1745     // cqto
 1746     __ bind(normal);
 1747     __ cdqq();
 1748 
 1749     // idivq (note: must be emitted by the user of this rule)
 1750     // <done>
 1751     __ idivq($div$$Register);
 1752     __ bind(done);
 1753   %}
 1754 
 1755   enc_class enc_PartialSubtypeCheck()
 1756   %{
 1757     Register Rrdi = as_Register(RDI_enc); // result register
 1758     Register Rrax = as_Register(RAX_enc); // super class
 1759     Register Rrcx = as_Register(RCX_enc); // killed
 1760     Register Rrsi = as_Register(RSI_enc); // sub class
 1761     Label miss;
 1762     const bool set_cond_codes = true;
 1763 
 1764     MacroAssembler _masm(&cbuf);
 1765     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 1766                                      nullptr, &miss,
 1767                                      /*set_cond_codes:*/ true);
 1768     if ($primary) {
 1769       __ xorptr(Rrdi, Rrdi);
 1770     }
 1771     __ bind(miss);
 1772   %}
 1773 
 1774   enc_class clear_avx %{
 1775     debug_only(int off0 = cbuf.insts_size());
 1776     if (generate_vzeroupper(Compile::current())) {
 1777       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1778       // Clear upper bits of YMM registers when current compiled code uses
 1779       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1780       MacroAssembler _masm(&cbuf);
 1781       __ vzeroupper();
 1782     }
 1783     debug_only(int off1 = cbuf.insts_size());
 1784     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1785   %}
 1786 
 1787   enc_class Java_To_Runtime(method meth) %{
 1788     // No relocation needed
 1789     MacroAssembler _masm(&cbuf);
 1790     __ mov64(r10, (int64_t) $meth$$method);
 1791     __ call(r10);
 1792     __ post_call_nop();
 1793   %}
 1794 
 1795   enc_class Java_Static_Call(method meth)
 1796   %{
 1797     // JAVA STATIC CALL
 1798     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1799     // determine who we intended to call.
 1800     MacroAssembler _masm(&cbuf);
 1801 
 1802     if (!_method) {
 1803       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1804     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1805       // The NOP here is purely to ensure that eliding a call to
 1806       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1807       __ addr_nop_5();
 1808       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1809     } else {
 1810       int method_index = resolved_method_index(cbuf);
 1811       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1812                                                   : static_call_Relocation::spec(method_index);
 1813       address mark = __ pc();
 1814       int call_offset = __ offset();
 1815       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1816       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1817         // Calls of the same statically bound method can share
 1818         // a stub to the interpreter.
 1819         cbuf.shared_stub_to_interp_for(_method, call_offset);
 1820       } else {
 1821         // Emit stubs for static call.
 1822         address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, mark);
 1823         if (stub == nullptr) {
 1824           ciEnv::current()->record_failure("CodeCache is full");
 1825           return;
 1826         }
 1827       }
 1828     }
 1829     __ post_call_nop();
 1830   %}
 1831 
 1832   enc_class Java_Dynamic_Call(method meth) %{
 1833     MacroAssembler _masm(&cbuf);
 1834     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 1835     __ post_call_nop();
 1836   %}
 1837 
 1838 %}
 1839 
 1840 
 1841 
 1842 //----------FRAME--------------------------------------------------------------
 1843 // Definition of frame structure and management information.
 1844 //
 1845 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1846 //                             |   (to get allocators register number
 1847 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1848 //  r   CALLER     |        |
 1849 //  o     |        +--------+      pad to even-align allocators stack-slot
 1850 //  w     V        |  pad0  |        numbers; owned by CALLER
 1851 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1852 //  h     ^        |   in   |  5
 1853 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1854 //  |     |        |        |  3
 1855 //  |     |        +--------+
 1856 //  V     |        | old out|      Empty on Intel, window on Sparc
 1857 //        |    old |preserve|      Must be even aligned.
 1858 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1859 //        |        |   in   |  3   area for Intel ret address
 1860 //     Owned by    |preserve|      Empty on Sparc.
 1861 //       SELF      +--------+
 1862 //        |        |  pad2  |  2   pad to align old SP
 1863 //        |        +--------+  1
 1864 //        |        | locks  |  0
 1865 //        |        +--------+----> OptoReg::stack0(), even aligned
 1866 //        |        |  pad1  | 11   pad to align new SP
 1867 //        |        +--------+
 1868 //        |        |        | 10
 1869 //        |        | spills |  9   spills
 1870 //        V        |        |  8   (pad0 slot for callee)
 1871 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1872 //        ^        |  out   |  7
 1873 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1874 //     Owned by    +--------+
 1875 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1876 //        |    new |preserve|      Must be even-aligned.
 1877 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1878 //        |        |        |
 1879 //
 1880 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1881 //         known from SELF's arguments and the Java calling convention.
 1882 //         Region 6-7 is determined per call site.
 1883 // Note 2: If the calling convention leaves holes in the incoming argument
 1884 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1885 //         are owned by the CALLEE.  Holes should not be necessary in the
 1886 //         incoming area, as the Java calling convention is completely under
 1887 //         the control of the AD file.  Doubles can be sorted and packed to
 1888 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1889 //         varargs C calling conventions.
 1890 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1891 //         even aligned with pad0 as needed.
 1892 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1893 //         region 6-11 is even aligned; it may be padded out more so that
 1894 //         the region from SP to FP meets the minimum stack alignment.
 1895 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1896 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1897 //         SP meets the minimum alignment.
 1898 
 1899 frame
 1900 %{
 1901   // These three registers define part of the calling convention
 1902   // between compiled code and the interpreter.
 1903   inline_cache_reg(RAX);                // Inline Cache Register
 1904 
 1905   // Optional: name the operand used by cisc-spilling to access
 1906   // [stack_pointer + offset]
 1907   cisc_spilling_operand_name(indOffset32);
 1908 
 1909   // Number of stack slots consumed by locking an object
 1910   sync_stack_slots(2);
 1911 
 1912   // Compiled code's Frame Pointer
 1913   frame_pointer(RSP);
 1914 
 1915   // Interpreter stores its frame pointer in a register which is
 1916   // stored to the stack by I2CAdaptors.
 1917   // I2CAdaptors convert from interpreted java to compiled java.
 1918   interpreter_frame_pointer(RBP);
 1919 
 1920   // Stack alignment requirement
 1921   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1922 
 1923   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1924   // for calls to C.  Supports the var-args backing area for register parms.
 1925   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1926 
 1927   // The after-PROLOG location of the return address.  Location of
 1928   // return address specifies a type (REG or STACK) and a number
 1929   // representing the register number (i.e. - use a register name) or
 1930   // stack slot.
 1931   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1932   // Otherwise, it is above the locks and verification slot and alignment word
 1933   return_addr(STACK - 2 +
 1934               align_up((Compile::current()->in_preserve_stack_slots() +
 1935                         Compile::current()->fixed_slots()),
 1936                        stack_alignment_in_slots()));
 1937 
 1938   // Location of compiled Java return values.  Same as C for now.
 1939   return_value
 1940   %{
 1941     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 1942            "only return normal values");
 1943 
 1944     static const int lo[Op_RegL + 1] = {
 1945       0,
 1946       0,
 1947       RAX_num,  // Op_RegN
 1948       RAX_num,  // Op_RegI
 1949       RAX_num,  // Op_RegP
 1950       XMM0_num, // Op_RegF
 1951       XMM0_num, // Op_RegD
 1952       RAX_num   // Op_RegL
 1953     };
 1954     static const int hi[Op_RegL + 1] = {
 1955       0,
 1956       0,
 1957       OptoReg::Bad, // Op_RegN
 1958       OptoReg::Bad, // Op_RegI
 1959       RAX_H_num,    // Op_RegP
 1960       OptoReg::Bad, // Op_RegF
 1961       XMM0b_num,    // Op_RegD
 1962       RAX_H_num     // Op_RegL
 1963     };
 1964     // Excluded flags and vector registers.
 1965     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 1966     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 1967   %}
 1968 %}
 1969 
 1970 //----------ATTRIBUTES---------------------------------------------------------
 1971 //----------Operand Attributes-------------------------------------------------
 1972 op_attrib op_cost(0);        // Required cost attribute
 1973 
 1974 //----------Instruction Attributes---------------------------------------------
 1975 ins_attrib ins_cost(100);       // Required cost attribute
 1976 ins_attrib ins_size(8);         // Required size attribute (in bits)
 1977 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 1978                                 // a non-matching short branch variant
 1979                                 // of some long branch?
 1980 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 1981                                 // be a power of 2) specifies the
 1982                                 // alignment that some part of the
 1983                                 // instruction (not necessarily the
 1984                                 // start) requires.  If > 1, a
 1985                                 // compute_padding() function must be
 1986                                 // provided for the instruction
 1987 
 1988 //----------OPERANDS-----------------------------------------------------------
 1989 // Operand definitions must precede instruction definitions for correct parsing
 1990 // in the ADLC because operands constitute user defined types which are used in
 1991 // instruction definitions.
 1992 
 1993 //----------Simple Operands----------------------------------------------------
 1994 // Immediate Operands
 1995 // Integer Immediate
 1996 operand immI()
 1997 %{
 1998   match(ConI);
 1999 
 2000   op_cost(10);
 2001   format %{ %}
 2002   interface(CONST_INTER);
 2003 %}
 2004 
 2005 // Constant for test vs zero
 2006 operand immI_0()
 2007 %{
 2008   predicate(n->get_int() == 0);
 2009   match(ConI);
 2010 
 2011   op_cost(0);
 2012   format %{ %}
 2013   interface(CONST_INTER);
 2014 %}
 2015 
 2016 // Constant for increment
 2017 operand immI_1()
 2018 %{
 2019   predicate(n->get_int() == 1);
 2020   match(ConI);
 2021 
 2022   op_cost(0);
 2023   format %{ %}
 2024   interface(CONST_INTER);
 2025 %}
 2026 
 2027 // Constant for decrement
 2028 operand immI_M1()
 2029 %{
 2030   predicate(n->get_int() == -1);
 2031   match(ConI);
 2032 
 2033   op_cost(0);
 2034   format %{ %}
 2035   interface(CONST_INTER);
 2036 %}
 2037 
 2038 operand immI_2()
 2039 %{
 2040   predicate(n->get_int() == 2);
 2041   match(ConI);
 2042 
 2043   op_cost(0);
 2044   format %{ %}
 2045   interface(CONST_INTER);
 2046 %}
 2047 
 2048 operand immI_4()
 2049 %{
 2050   predicate(n->get_int() == 4);
 2051   match(ConI);
 2052 
 2053   op_cost(0);
 2054   format %{ %}
 2055   interface(CONST_INTER);
 2056 %}
 2057 
 2058 operand immI_8()
 2059 %{
 2060   predicate(n->get_int() == 8);
 2061   match(ConI);
 2062 
 2063   op_cost(0);
 2064   format %{ %}
 2065   interface(CONST_INTER);
 2066 %}
 2067 
 2068 // Valid scale values for addressing modes
 2069 operand immI2()
 2070 %{
 2071   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2072   match(ConI);
 2073 
 2074   format %{ %}
 2075   interface(CONST_INTER);
 2076 %}
 2077 
 2078 operand immU7()
 2079 %{
 2080   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2081   match(ConI);
 2082 
 2083   op_cost(5);
 2084   format %{ %}
 2085   interface(CONST_INTER);
 2086 %}
 2087 
 2088 operand immI8()
 2089 %{
 2090   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2091   match(ConI);
 2092 
 2093   op_cost(5);
 2094   format %{ %}
 2095   interface(CONST_INTER);
 2096 %}
 2097 
 2098 operand immU8()
 2099 %{
 2100   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2101   match(ConI);
 2102 
 2103   op_cost(5);
 2104   format %{ %}
 2105   interface(CONST_INTER);
 2106 %}
 2107 
 2108 operand immI16()
 2109 %{
 2110   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2111   match(ConI);
 2112 
 2113   op_cost(10);
 2114   format %{ %}
 2115   interface(CONST_INTER);
 2116 %}
 2117 
 2118 // Int Immediate non-negative
 2119 operand immU31()
 2120 %{
 2121   predicate(n->get_int() >= 0);
 2122   match(ConI);
 2123 
 2124   op_cost(0);
 2125   format %{ %}
 2126   interface(CONST_INTER);
 2127 %}
 2128 
 2129 // Constant for long shifts
 2130 operand immI_32()
 2131 %{
 2132   predicate( n->get_int() == 32 );
 2133   match(ConI);
 2134 
 2135   op_cost(0);
 2136   format %{ %}
 2137   interface(CONST_INTER);
 2138 %}
 2139 
 2140 // Constant for long shifts
 2141 operand immI_64()
 2142 %{
 2143   predicate( n->get_int() == 64 );
 2144   match(ConI);
 2145 
 2146   op_cost(0);
 2147   format %{ %}
 2148   interface(CONST_INTER);
 2149 %}
 2150 
 2151 // Pointer Immediate
 2152 operand immP()
 2153 %{
 2154   match(ConP);
 2155 
 2156   op_cost(10);
 2157   format %{ %}
 2158   interface(CONST_INTER);
 2159 %}
 2160 
 2161 // Null Pointer Immediate
 2162 operand immP0()
 2163 %{
 2164   predicate(n->get_ptr() == 0);
 2165   match(ConP);
 2166 
 2167   op_cost(5);
 2168   format %{ %}
 2169   interface(CONST_INTER);
 2170 %}
 2171 
 2172 // Pointer Immediate
 2173 operand immN() %{
 2174   match(ConN);
 2175 
 2176   op_cost(10);
 2177   format %{ %}
 2178   interface(CONST_INTER);
 2179 %}
 2180 
 2181 operand immNKlass() %{
 2182   match(ConNKlass);
 2183 
 2184   op_cost(10);
 2185   format %{ %}
 2186   interface(CONST_INTER);
 2187 %}
 2188 
 2189 // Null Pointer Immediate
 2190 operand immN0() %{
 2191   predicate(n->get_narrowcon() == 0);
 2192   match(ConN);
 2193 
 2194   op_cost(5);
 2195   format %{ %}
 2196   interface(CONST_INTER);
 2197 %}
 2198 
 2199 operand immP31()
 2200 %{
 2201   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2202             && (n->get_ptr() >> 31) == 0);
 2203   match(ConP);
 2204 
 2205   op_cost(5);
 2206   format %{ %}
 2207   interface(CONST_INTER);
 2208 %}
 2209 
 2210 
 2211 // Long Immediate
 2212 operand immL()
 2213 %{
 2214   match(ConL);
 2215 
 2216   op_cost(20);
 2217   format %{ %}
 2218   interface(CONST_INTER);
 2219 %}
 2220 
 2221 // Long Immediate 8-bit
 2222 operand immL8()
 2223 %{
 2224   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2225   match(ConL);
 2226 
 2227   op_cost(5);
 2228   format %{ %}
 2229   interface(CONST_INTER);
 2230 %}
 2231 
 2232 // Long Immediate 32-bit unsigned
 2233 operand immUL32()
 2234 %{
 2235   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2236   match(ConL);
 2237 
 2238   op_cost(10);
 2239   format %{ %}
 2240   interface(CONST_INTER);
 2241 %}
 2242 
 2243 // Long Immediate 32-bit signed
 2244 operand immL32()
 2245 %{
 2246   predicate(n->get_long() == (int) (n->get_long()));
 2247   match(ConL);
 2248 
 2249   op_cost(15);
 2250   format %{ %}
 2251   interface(CONST_INTER);
 2252 %}
 2253 
 2254 operand immL_Pow2()
 2255 %{
 2256   predicate(is_power_of_2((julong)n->get_long()));
 2257   match(ConL);
 2258 
 2259   op_cost(15);
 2260   format %{ %}
 2261   interface(CONST_INTER);
 2262 %}
 2263 
 2264 operand immL_NotPow2()
 2265 %{
 2266   predicate(is_power_of_2((julong)~n->get_long()));
 2267   match(ConL);
 2268 
 2269   op_cost(15);
 2270   format %{ %}
 2271   interface(CONST_INTER);
 2272 %}
 2273 
 2274 // Long Immediate zero
 2275 operand immL0()
 2276 %{
 2277   predicate(n->get_long() == 0L);
 2278   match(ConL);
 2279 
 2280   op_cost(10);
 2281   format %{ %}
 2282   interface(CONST_INTER);
 2283 %}
 2284 
 2285 // Constant for increment
 2286 operand immL1()
 2287 %{
 2288   predicate(n->get_long() == 1);
 2289   match(ConL);
 2290 
 2291   format %{ %}
 2292   interface(CONST_INTER);
 2293 %}
 2294 
 2295 // Constant for decrement
 2296 operand immL_M1()
 2297 %{
 2298   predicate(n->get_long() == -1);
 2299   match(ConL);
 2300 
 2301   format %{ %}
 2302   interface(CONST_INTER);
 2303 %}
 2304 
 2305 // Long Immediate: the value 10
 2306 operand immL10()
 2307 %{
 2308   predicate(n->get_long() == 10);
 2309   match(ConL);
 2310 
 2311   format %{ %}
 2312   interface(CONST_INTER);
 2313 %}
 2314 
 2315 // Long immediate from 0 to 127.
 2316 // Used for a shorter form of long mul by 10.
 2317 operand immL_127()
 2318 %{
 2319   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 2320   match(ConL);
 2321 
 2322   op_cost(10);
 2323   format %{ %}
 2324   interface(CONST_INTER);
 2325 %}
 2326 
 2327 // Long Immediate: low 32-bit mask
 2328 operand immL_32bits()
 2329 %{
 2330   predicate(n->get_long() == 0xFFFFFFFFL);
 2331   match(ConL);
 2332   op_cost(20);
 2333 
 2334   format %{ %}
 2335   interface(CONST_INTER);
 2336 %}
 2337 
 2338 // Int Immediate: 2^n-1, positive
 2339 operand immI_Pow2M1()
 2340 %{
 2341   predicate((n->get_int() > 0)
 2342             && is_power_of_2((juint)n->get_int() + 1));
 2343   match(ConI);
 2344 
 2345   op_cost(20);
 2346   format %{ %}
 2347   interface(CONST_INTER);
 2348 %}
 2349 
 2350 // Float Immediate zero
 2351 operand immF0()
 2352 %{
 2353   predicate(jint_cast(n->getf()) == 0);
 2354   match(ConF);
 2355 
 2356   op_cost(5);
 2357   format %{ %}
 2358   interface(CONST_INTER);
 2359 %}
 2360 
 2361 // Float Immediate
 2362 operand immF()
 2363 %{
 2364   match(ConF);
 2365 
 2366   op_cost(15);
 2367   format %{ %}
 2368   interface(CONST_INTER);
 2369 %}
 2370 
 2371 // Double Immediate zero
 2372 operand immD0()
 2373 %{
 2374   predicate(jlong_cast(n->getd()) == 0);
 2375   match(ConD);
 2376 
 2377   op_cost(5);
 2378   format %{ %}
 2379   interface(CONST_INTER);
 2380 %}
 2381 
 2382 // Double Immediate
 2383 operand immD()
 2384 %{
 2385   match(ConD);
 2386 
 2387   op_cost(15);
 2388   format %{ %}
 2389   interface(CONST_INTER);
 2390 %}
 2391 
 2392 // Immediates for special shifts (sign extend)
 2393 
 2394 // Constants for increment
 2395 operand immI_16()
 2396 %{
 2397   predicate(n->get_int() == 16);
 2398   match(ConI);
 2399 
 2400   format %{ %}
 2401   interface(CONST_INTER);
 2402 %}
 2403 
 2404 operand immI_24()
 2405 %{
 2406   predicate(n->get_int() == 24);
 2407   match(ConI);
 2408 
 2409   format %{ %}
 2410   interface(CONST_INTER);
 2411 %}
 2412 
 2413 // Constant for byte-wide masking
 2414 operand immI_255()
 2415 %{
 2416   predicate(n->get_int() == 255);
 2417   match(ConI);
 2418 
 2419   format %{ %}
 2420   interface(CONST_INTER);
 2421 %}
 2422 
 2423 // Constant for short-wide masking
 2424 operand immI_65535()
 2425 %{
 2426   predicate(n->get_int() == 65535);
 2427   match(ConI);
 2428 
 2429   format %{ %}
 2430   interface(CONST_INTER);
 2431 %}
 2432 
 2433 // Constant for byte-wide masking
 2434 operand immL_255()
 2435 %{
 2436   predicate(n->get_long() == 255);
 2437   match(ConL);
 2438 
 2439   format %{ %}
 2440   interface(CONST_INTER);
 2441 %}
 2442 
 2443 // Constant for short-wide masking
 2444 operand immL_65535()
 2445 %{
 2446   predicate(n->get_long() == 65535);
 2447   match(ConL);
 2448 
 2449   format %{ %}
 2450   interface(CONST_INTER);
 2451 %}
 2452 
 2453 operand kReg()
 2454 %{
 2455   constraint(ALLOC_IN_RC(vectmask_reg));
 2456   match(RegVectMask);
 2457   format %{%}
 2458   interface(REG_INTER);
 2459 %}
 2460 
 2461 operand kReg_K1()
 2462 %{
 2463   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 2464   match(RegVectMask);
 2465   format %{%}
 2466   interface(REG_INTER);
 2467 %}
 2468 
 2469 operand kReg_K2()
 2470 %{
 2471   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 2472   match(RegVectMask);
 2473   format %{%}
 2474   interface(REG_INTER);
 2475 %}
 2476 
 2477 // Special Registers
 2478 operand kReg_K3()
 2479 %{
 2480   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 2481   match(RegVectMask);
 2482   format %{%}
 2483   interface(REG_INTER);
 2484 %}
 2485 
 2486 operand kReg_K4()
 2487 %{
 2488   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 2489   match(RegVectMask);
 2490   format %{%}
 2491   interface(REG_INTER);
 2492 %}
 2493 
 2494 operand kReg_K5()
 2495 %{
 2496   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 2497   match(RegVectMask);
 2498   format %{%}
 2499   interface(REG_INTER);
 2500 %}
 2501 
 2502 operand kReg_K6()
 2503 %{
 2504   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 2505   match(RegVectMask);
 2506   format %{%}
 2507   interface(REG_INTER);
 2508 %}
 2509 
 2510 // Special Registers
 2511 operand kReg_K7()
 2512 %{
 2513   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 2514   match(RegVectMask);
 2515   format %{%}
 2516   interface(REG_INTER);
 2517 %}
 2518 
 2519 // Register Operands
 2520 // Integer Register
 2521 operand rRegI()
 2522 %{
 2523   constraint(ALLOC_IN_RC(int_reg));
 2524   match(RegI);
 2525 
 2526   match(rax_RegI);
 2527   match(rbx_RegI);
 2528   match(rcx_RegI);
 2529   match(rdx_RegI);
 2530   match(rdi_RegI);
 2531 
 2532   format %{ %}
 2533   interface(REG_INTER);
 2534 %}
 2535 
 2536 // Special Registers
 2537 operand rax_RegI()
 2538 %{
 2539   constraint(ALLOC_IN_RC(int_rax_reg));
 2540   match(RegI);
 2541   match(rRegI);
 2542 
 2543   format %{ "RAX" %}
 2544   interface(REG_INTER);
 2545 %}
 2546 
 2547 // Special Registers
 2548 operand rbx_RegI()
 2549 %{
 2550   constraint(ALLOC_IN_RC(int_rbx_reg));
 2551   match(RegI);
 2552   match(rRegI);
 2553 
 2554   format %{ "RBX" %}
 2555   interface(REG_INTER);
 2556 %}
 2557 
 2558 operand rcx_RegI()
 2559 %{
 2560   constraint(ALLOC_IN_RC(int_rcx_reg));
 2561   match(RegI);
 2562   match(rRegI);
 2563 
 2564   format %{ "RCX" %}
 2565   interface(REG_INTER);
 2566 %}
 2567 
 2568 operand rdx_RegI()
 2569 %{
 2570   constraint(ALLOC_IN_RC(int_rdx_reg));
 2571   match(RegI);
 2572   match(rRegI);
 2573 
 2574   format %{ "RDX" %}
 2575   interface(REG_INTER);
 2576 %}
 2577 
 2578 operand rdi_RegI()
 2579 %{
 2580   constraint(ALLOC_IN_RC(int_rdi_reg));
 2581   match(RegI);
 2582   match(rRegI);
 2583 
 2584   format %{ "RDI" %}
 2585   interface(REG_INTER);
 2586 %}
 2587 
 2588 operand no_rax_rdx_RegI()
 2589 %{
 2590   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2591   match(RegI);
 2592   match(rbx_RegI);
 2593   match(rcx_RegI);
 2594   match(rdi_RegI);
 2595 
 2596   format %{ %}
 2597   interface(REG_INTER);
 2598 %}
 2599 
 2600 operand no_rbp_r13_RegI()
 2601 %{
 2602   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2603   match(RegI);
 2604   match(rRegI);
 2605   match(rax_RegI);
 2606   match(rbx_RegI);
 2607   match(rcx_RegI);
 2608   match(rdx_RegI);
 2609   match(rdi_RegI);
 2610 
 2611   format %{ %}
 2612   interface(REG_INTER);
 2613 %}
 2614 
 2615 // Pointer Register
 2616 operand any_RegP()
 2617 %{
 2618   constraint(ALLOC_IN_RC(any_reg));
 2619   match(RegP);
 2620   match(rax_RegP);
 2621   match(rbx_RegP);
 2622   match(rdi_RegP);
 2623   match(rsi_RegP);
 2624   match(rbp_RegP);
 2625   match(r15_RegP);
 2626   match(rRegP);
 2627 
 2628   format %{ %}
 2629   interface(REG_INTER);
 2630 %}
 2631 
 2632 operand rRegP()
 2633 %{
 2634   constraint(ALLOC_IN_RC(ptr_reg));
 2635   match(RegP);
 2636   match(rax_RegP);
 2637   match(rbx_RegP);
 2638   match(rdi_RegP);
 2639   match(rsi_RegP);
 2640   match(rbp_RegP);  // See Q&A below about
 2641   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2642 
 2643   format %{ %}
 2644   interface(REG_INTER);
 2645 %}
 2646 
 2647 operand rRegN() %{
 2648   constraint(ALLOC_IN_RC(int_reg));
 2649   match(RegN);
 2650 
 2651   format %{ %}
 2652   interface(REG_INTER);
 2653 %}
 2654 
 2655 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2656 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2657 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2658 // The output of an instruction is controlled by the allocator, which respects
 2659 // register class masks, not match rules.  Unless an instruction mentions
 2660 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2661 // by the allocator as an input.
 2662 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2663 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2664 // result, RBP is not included in the output of the instruction either.
 2665 
 2666 operand no_rax_RegP()
 2667 %{
 2668   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 2669   match(RegP);
 2670   match(rbx_RegP);
 2671   match(rsi_RegP);
 2672   match(rdi_RegP);
 2673 
 2674   format %{ %}
 2675   interface(REG_INTER);
 2676 %}
 2677 
 2678 // This operand is not allowed to use RBP even if
 2679 // RBP is not used to hold the frame pointer.
 2680 operand no_rbp_RegP()
 2681 %{
 2682   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2683   match(RegP);
 2684   match(rbx_RegP);
 2685   match(rsi_RegP);
 2686   match(rdi_RegP);
 2687 
 2688   format %{ %}
 2689   interface(REG_INTER);
 2690 %}
 2691 
 2692 operand no_rax_rbx_RegP()
 2693 %{
 2694   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 2695   match(RegP);
 2696   match(rsi_RegP);
 2697   match(rdi_RegP);
 2698 
 2699   format %{ %}
 2700   interface(REG_INTER);
 2701 %}
 2702 
 2703 // Special Registers
 2704 // Return a pointer value
 2705 operand rax_RegP()
 2706 %{
 2707   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2708   match(RegP);
 2709   match(rRegP);
 2710 
 2711   format %{ %}
 2712   interface(REG_INTER);
 2713 %}
 2714 
 2715 // Special Registers
 2716 // Return a compressed pointer value
 2717 operand rax_RegN()
 2718 %{
 2719   constraint(ALLOC_IN_RC(int_rax_reg));
 2720   match(RegN);
 2721   match(rRegN);
 2722 
 2723   format %{ %}
 2724   interface(REG_INTER);
 2725 %}
 2726 
 2727 // Used in AtomicAdd
 2728 operand rbx_RegP()
 2729 %{
 2730   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2731   match(RegP);
 2732   match(rRegP);
 2733 
 2734   format %{ %}
 2735   interface(REG_INTER);
 2736 %}
 2737 
 2738 operand rsi_RegP()
 2739 %{
 2740   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2741   match(RegP);
 2742   match(rRegP);
 2743 
 2744   format %{ %}
 2745   interface(REG_INTER);
 2746 %}
 2747 
 2748 operand rbp_RegP()
 2749 %{
 2750   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2751   match(RegP);
 2752   match(rRegP);
 2753 
 2754   format %{ %}
 2755   interface(REG_INTER);
 2756 %}
 2757 
 2758 // Used in rep stosq
 2759 operand rdi_RegP()
 2760 %{
 2761   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2762   match(RegP);
 2763   match(rRegP);
 2764 
 2765   format %{ %}
 2766   interface(REG_INTER);
 2767 %}
 2768 
 2769 operand r15_RegP()
 2770 %{
 2771   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2772   match(RegP);
 2773   match(rRegP);
 2774 
 2775   format %{ %}
 2776   interface(REG_INTER);
 2777 %}
 2778 
 2779 operand rRegL()
 2780 %{
 2781   constraint(ALLOC_IN_RC(long_reg));
 2782   match(RegL);
 2783   match(rax_RegL);
 2784   match(rdx_RegL);
 2785 
 2786   format %{ %}
 2787   interface(REG_INTER);
 2788 %}
 2789 
 2790 // Special Registers
 2791 operand no_rax_rdx_RegL()
 2792 %{
 2793   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2794   match(RegL);
 2795   match(rRegL);
 2796 
 2797   format %{ %}
 2798   interface(REG_INTER);
 2799 %}
 2800 
 2801 operand rax_RegL()
 2802 %{
 2803   constraint(ALLOC_IN_RC(long_rax_reg));
 2804   match(RegL);
 2805   match(rRegL);
 2806 
 2807   format %{ "RAX" %}
 2808   interface(REG_INTER);
 2809 %}
 2810 
 2811 operand rcx_RegL()
 2812 %{
 2813   constraint(ALLOC_IN_RC(long_rcx_reg));
 2814   match(RegL);
 2815   match(rRegL);
 2816 
 2817   format %{ %}
 2818   interface(REG_INTER);
 2819 %}
 2820 
 2821 operand rdx_RegL()
 2822 %{
 2823   constraint(ALLOC_IN_RC(long_rdx_reg));
 2824   match(RegL);
 2825   match(rRegL);
 2826 
 2827   format %{ %}
 2828   interface(REG_INTER);
 2829 %}
 2830 
 2831 operand no_rbp_r13_RegL()
 2832 %{
 2833   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2834   match(RegL);
 2835   match(rRegL);
 2836   match(rax_RegL);
 2837   match(rcx_RegL);
 2838   match(rdx_RegL);
 2839 
 2840   format %{ %}
 2841   interface(REG_INTER);
 2842 %}
 2843 
 2844 // Flags register, used as output of compare instructions
 2845 operand rFlagsReg()
 2846 %{
 2847   constraint(ALLOC_IN_RC(int_flags));
 2848   match(RegFlags);
 2849 
 2850   format %{ "RFLAGS" %}
 2851   interface(REG_INTER);
 2852 %}
 2853 
 2854 // Flags register, used as output of FLOATING POINT compare instructions
 2855 operand rFlagsRegU()
 2856 %{
 2857   constraint(ALLOC_IN_RC(int_flags));
 2858   match(RegFlags);
 2859 
 2860   format %{ "RFLAGS_U" %}
 2861   interface(REG_INTER);
 2862 %}
 2863 
 2864 operand rFlagsRegUCF() %{
 2865   constraint(ALLOC_IN_RC(int_flags));
 2866   match(RegFlags);
 2867   predicate(false);
 2868 
 2869   format %{ "RFLAGS_U_CF" %}
 2870   interface(REG_INTER);
 2871 %}
 2872 
 2873 // Float register operands
 2874 operand regF() %{
 2875    constraint(ALLOC_IN_RC(float_reg));
 2876    match(RegF);
 2877 
 2878    format %{ %}
 2879    interface(REG_INTER);
 2880 %}
 2881 
 2882 // Float register operands
 2883 operand legRegF() %{
 2884    constraint(ALLOC_IN_RC(float_reg_legacy));
 2885    match(RegF);
 2886 
 2887    format %{ %}
 2888    interface(REG_INTER);
 2889 %}
 2890 
 2891 // Float register operands
 2892 operand vlRegF() %{
 2893    constraint(ALLOC_IN_RC(float_reg_vl));
 2894    match(RegF);
 2895 
 2896    format %{ %}
 2897    interface(REG_INTER);
 2898 %}
 2899 
 2900 // Double register operands
 2901 operand regD() %{
 2902    constraint(ALLOC_IN_RC(double_reg));
 2903    match(RegD);
 2904 
 2905    format %{ %}
 2906    interface(REG_INTER);
 2907 %}
 2908 
 2909 // Double register operands
 2910 operand legRegD() %{
 2911    constraint(ALLOC_IN_RC(double_reg_legacy));
 2912    match(RegD);
 2913 
 2914    format %{ %}
 2915    interface(REG_INTER);
 2916 %}
 2917 
 2918 // Double register operands
 2919 operand vlRegD() %{
 2920    constraint(ALLOC_IN_RC(double_reg_vl));
 2921    match(RegD);
 2922 
 2923    format %{ %}
 2924    interface(REG_INTER);
 2925 %}
 2926 
 2927 //----------Memory Operands----------------------------------------------------
 2928 // Direct Memory Operand
 2929 // operand direct(immP addr)
 2930 // %{
 2931 //   match(addr);
 2932 
 2933 //   format %{ "[$addr]" %}
 2934 //   interface(MEMORY_INTER) %{
 2935 //     base(0xFFFFFFFF);
 2936 //     index(0x4);
 2937 //     scale(0x0);
 2938 //     disp($addr);
 2939 //   %}
 2940 // %}
 2941 
 2942 // Indirect Memory Operand
 2943 operand indirect(any_RegP reg)
 2944 %{
 2945   constraint(ALLOC_IN_RC(ptr_reg));
 2946   match(reg);
 2947 
 2948   format %{ "[$reg]" %}
 2949   interface(MEMORY_INTER) %{
 2950     base($reg);
 2951     index(0x4);
 2952     scale(0x0);
 2953     disp(0x0);
 2954   %}
 2955 %}
 2956 
 2957 // Indirect Memory Plus Short Offset Operand
 2958 operand indOffset8(any_RegP reg, immL8 off)
 2959 %{
 2960   constraint(ALLOC_IN_RC(ptr_reg));
 2961   match(AddP reg off);
 2962 
 2963   format %{ "[$reg + $off (8-bit)]" %}
 2964   interface(MEMORY_INTER) %{
 2965     base($reg);
 2966     index(0x4);
 2967     scale(0x0);
 2968     disp($off);
 2969   %}
 2970 %}
 2971 
 2972 // Indirect Memory Plus Long Offset Operand
 2973 operand indOffset32(any_RegP reg, immL32 off)
 2974 %{
 2975   constraint(ALLOC_IN_RC(ptr_reg));
 2976   match(AddP reg off);
 2977 
 2978   format %{ "[$reg + $off (32-bit)]" %}
 2979   interface(MEMORY_INTER) %{
 2980     base($reg);
 2981     index(0x4);
 2982     scale(0x0);
 2983     disp($off);
 2984   %}
 2985 %}
 2986 
 2987 // Indirect Memory Plus Index Register Plus Offset Operand
 2988 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2989 %{
 2990   constraint(ALLOC_IN_RC(ptr_reg));
 2991   match(AddP (AddP reg lreg) off);
 2992 
 2993   op_cost(10);
 2994   format %{"[$reg + $off + $lreg]" %}
 2995   interface(MEMORY_INTER) %{
 2996     base($reg);
 2997     index($lreg);
 2998     scale(0x0);
 2999     disp($off);
 3000   %}
 3001 %}
 3002 
 3003 // Indirect Memory Plus Index Register Plus Offset Operand
 3004 operand indIndex(any_RegP reg, rRegL lreg)
 3005 %{
 3006   constraint(ALLOC_IN_RC(ptr_reg));
 3007   match(AddP reg lreg);
 3008 
 3009   op_cost(10);
 3010   format %{"[$reg + $lreg]" %}
 3011   interface(MEMORY_INTER) %{
 3012     base($reg);
 3013     index($lreg);
 3014     scale(0x0);
 3015     disp(0x0);
 3016   %}
 3017 %}
 3018 
 3019 // Indirect Memory Times Scale Plus Index Register
 3020 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3021 %{
 3022   constraint(ALLOC_IN_RC(ptr_reg));
 3023   match(AddP reg (LShiftL lreg scale));
 3024 
 3025   op_cost(10);
 3026   format %{"[$reg + $lreg << $scale]" %}
 3027   interface(MEMORY_INTER) %{
 3028     base($reg);
 3029     index($lreg);
 3030     scale($scale);
 3031     disp(0x0);
 3032   %}
 3033 %}
 3034 
 3035 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3036 %{
 3037   constraint(ALLOC_IN_RC(ptr_reg));
 3038   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3039   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3040 
 3041   op_cost(10);
 3042   format %{"[$reg + pos $idx << $scale]" %}
 3043   interface(MEMORY_INTER) %{
 3044     base($reg);
 3045     index($idx);
 3046     scale($scale);
 3047     disp(0x0);
 3048   %}
 3049 %}
 3050 
 3051 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3052 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3053 %{
 3054   constraint(ALLOC_IN_RC(ptr_reg));
 3055   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3056 
 3057   op_cost(10);
 3058   format %{"[$reg + $off + $lreg << $scale]" %}
 3059   interface(MEMORY_INTER) %{
 3060     base($reg);
 3061     index($lreg);
 3062     scale($scale);
 3063     disp($off);
 3064   %}
 3065 %}
 3066 
 3067 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3068 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3069 %{
 3070   constraint(ALLOC_IN_RC(ptr_reg));
 3071   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3072   match(AddP (AddP reg (ConvI2L idx)) off);
 3073 
 3074   op_cost(10);
 3075   format %{"[$reg + $off + $idx]" %}
 3076   interface(MEMORY_INTER) %{
 3077     base($reg);
 3078     index($idx);
 3079     scale(0x0);
 3080     disp($off);
 3081   %}
 3082 %}
 3083 
 3084 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3085 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3086 %{
 3087   constraint(ALLOC_IN_RC(ptr_reg));
 3088   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3089   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3090 
 3091   op_cost(10);
 3092   format %{"[$reg + $off + $idx << $scale]" %}
 3093   interface(MEMORY_INTER) %{
 3094     base($reg);
 3095     index($idx);
 3096     scale($scale);
 3097     disp($off);
 3098   %}
 3099 %}
 3100 
 3101 // Indirect Narrow Oop Plus Offset Operand
 3102 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3103 // we can't free r12 even with CompressedOops::base() == nullptr.
 3104 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3105   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3106   constraint(ALLOC_IN_RC(ptr_reg));
 3107   match(AddP (DecodeN reg) off);
 3108 
 3109   op_cost(10);
 3110   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3111   interface(MEMORY_INTER) %{
 3112     base(0xc); // R12
 3113     index($reg);
 3114     scale(0x3);
 3115     disp($off);
 3116   %}
 3117 %}
 3118 
 3119 // Indirect Memory Operand
 3120 operand indirectNarrow(rRegN reg)
 3121 %{
 3122   predicate(CompressedOops::shift() == 0);
 3123   constraint(ALLOC_IN_RC(ptr_reg));
 3124   match(DecodeN reg);
 3125 
 3126   format %{ "[$reg]" %}
 3127   interface(MEMORY_INTER) %{
 3128     base($reg);
 3129     index(0x4);
 3130     scale(0x0);
 3131     disp(0x0);
 3132   %}
 3133 %}
 3134 
 3135 // Indirect Memory Plus Short Offset Operand
 3136 operand indOffset8Narrow(rRegN reg, immL8 off)
 3137 %{
 3138   predicate(CompressedOops::shift() == 0);
 3139   constraint(ALLOC_IN_RC(ptr_reg));
 3140   match(AddP (DecodeN reg) off);
 3141 
 3142   format %{ "[$reg + $off (8-bit)]" %}
 3143   interface(MEMORY_INTER) %{
 3144     base($reg);
 3145     index(0x4);
 3146     scale(0x0);
 3147     disp($off);
 3148   %}
 3149 %}
 3150 
 3151 // Indirect Memory Plus Long Offset Operand
 3152 operand indOffset32Narrow(rRegN reg, immL32 off)
 3153 %{
 3154   predicate(CompressedOops::shift() == 0);
 3155   constraint(ALLOC_IN_RC(ptr_reg));
 3156   match(AddP (DecodeN reg) off);
 3157 
 3158   format %{ "[$reg + $off (32-bit)]" %}
 3159   interface(MEMORY_INTER) %{
 3160     base($reg);
 3161     index(0x4);
 3162     scale(0x0);
 3163     disp($off);
 3164   %}
 3165 %}
 3166 
 3167 // Indirect Memory Plus Index Register Plus Offset Operand
 3168 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3169 %{
 3170   predicate(CompressedOops::shift() == 0);
 3171   constraint(ALLOC_IN_RC(ptr_reg));
 3172   match(AddP (AddP (DecodeN reg) lreg) off);
 3173 
 3174   op_cost(10);
 3175   format %{"[$reg + $off + $lreg]" %}
 3176   interface(MEMORY_INTER) %{
 3177     base($reg);
 3178     index($lreg);
 3179     scale(0x0);
 3180     disp($off);
 3181   %}
 3182 %}
 3183 
 3184 // Indirect Memory Plus Index Register Plus Offset Operand
 3185 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3186 %{
 3187   predicate(CompressedOops::shift() == 0);
 3188   constraint(ALLOC_IN_RC(ptr_reg));
 3189   match(AddP (DecodeN reg) lreg);
 3190 
 3191   op_cost(10);
 3192   format %{"[$reg + $lreg]" %}
 3193   interface(MEMORY_INTER) %{
 3194     base($reg);
 3195     index($lreg);
 3196     scale(0x0);
 3197     disp(0x0);
 3198   %}
 3199 %}
 3200 
 3201 // Indirect Memory Times Scale Plus Index Register
 3202 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3203 %{
 3204   predicate(CompressedOops::shift() == 0);
 3205   constraint(ALLOC_IN_RC(ptr_reg));
 3206   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3207 
 3208   op_cost(10);
 3209   format %{"[$reg + $lreg << $scale]" %}
 3210   interface(MEMORY_INTER) %{
 3211     base($reg);
 3212     index($lreg);
 3213     scale($scale);
 3214     disp(0x0);
 3215   %}
 3216 %}
 3217 
 3218 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3219 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3220 %{
 3221   predicate(CompressedOops::shift() == 0);
 3222   constraint(ALLOC_IN_RC(ptr_reg));
 3223   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3224 
 3225   op_cost(10);
 3226   format %{"[$reg + $off + $lreg << $scale]" %}
 3227   interface(MEMORY_INTER) %{
 3228     base($reg);
 3229     index($lreg);
 3230     scale($scale);
 3231     disp($off);
 3232   %}
 3233 %}
 3234 
 3235 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3236 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3237 %{
 3238   constraint(ALLOC_IN_RC(ptr_reg));
 3239   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3240   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3241 
 3242   op_cost(10);
 3243   format %{"[$reg + $off + $idx]" %}
 3244   interface(MEMORY_INTER) %{
 3245     base($reg);
 3246     index($idx);
 3247     scale(0x0);
 3248     disp($off);
 3249   %}
 3250 %}
 3251 
 3252 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3253 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3254 %{
 3255   constraint(ALLOC_IN_RC(ptr_reg));
 3256   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3257   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3258 
 3259   op_cost(10);
 3260   format %{"[$reg + $off + $idx << $scale]" %}
 3261   interface(MEMORY_INTER) %{
 3262     base($reg);
 3263     index($idx);
 3264     scale($scale);
 3265     disp($off);
 3266   %}
 3267 %}
 3268 
 3269 //----------Special Memory Operands--------------------------------------------
 3270 // Stack Slot Operand - This operand is used for loading and storing temporary
 3271 //                      values on the stack where a match requires a value to
 3272 //                      flow through memory.
 3273 operand stackSlotP(sRegP reg)
 3274 %{
 3275   constraint(ALLOC_IN_RC(stack_slots));
 3276   // No match rule because this operand is only generated in matching
 3277 
 3278   format %{ "[$reg]" %}
 3279   interface(MEMORY_INTER) %{
 3280     base(0x4);   // RSP
 3281     index(0x4);  // No Index
 3282     scale(0x0);  // No Scale
 3283     disp($reg);  // Stack Offset
 3284   %}
 3285 %}
 3286 
 3287 operand stackSlotI(sRegI reg)
 3288 %{
 3289   constraint(ALLOC_IN_RC(stack_slots));
 3290   // No match rule because this operand is only generated in matching
 3291 
 3292   format %{ "[$reg]" %}
 3293   interface(MEMORY_INTER) %{
 3294     base(0x4);   // RSP
 3295     index(0x4);  // No Index
 3296     scale(0x0);  // No Scale
 3297     disp($reg);  // Stack Offset
 3298   %}
 3299 %}
 3300 
 3301 operand stackSlotF(sRegF reg)
 3302 %{
 3303   constraint(ALLOC_IN_RC(stack_slots));
 3304   // No match rule because this operand is only generated in matching
 3305 
 3306   format %{ "[$reg]" %}
 3307   interface(MEMORY_INTER) %{
 3308     base(0x4);   // RSP
 3309     index(0x4);  // No Index
 3310     scale(0x0);  // No Scale
 3311     disp($reg);  // Stack Offset
 3312   %}
 3313 %}
 3314 
 3315 operand stackSlotD(sRegD reg)
 3316 %{
 3317   constraint(ALLOC_IN_RC(stack_slots));
 3318   // No match rule because this operand is only generated in matching
 3319 
 3320   format %{ "[$reg]" %}
 3321   interface(MEMORY_INTER) %{
 3322     base(0x4);   // RSP
 3323     index(0x4);  // No Index
 3324     scale(0x0);  // No Scale
 3325     disp($reg);  // Stack Offset
 3326   %}
 3327 %}
 3328 operand stackSlotL(sRegL reg)
 3329 %{
 3330   constraint(ALLOC_IN_RC(stack_slots));
 3331   // No match rule because this operand is only generated in matching
 3332 
 3333   format %{ "[$reg]" %}
 3334   interface(MEMORY_INTER) %{
 3335     base(0x4);   // RSP
 3336     index(0x4);  // No Index
 3337     scale(0x0);  // No Scale
 3338     disp($reg);  // Stack Offset
 3339   %}
 3340 %}
 3341 
 3342 //----------Conditional Branch Operands----------------------------------------
 3343 // Comparison Op  - This is the operation of the comparison, and is limited to
 3344 //                  the following set of codes:
 3345 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3346 //
 3347 // Other attributes of the comparison, such as unsignedness, are specified
 3348 // by the comparison instruction that sets a condition code flags register.
 3349 // That result is represented by a flags operand whose subtype is appropriate
 3350 // to the unsignedness (etc.) of the comparison.
 3351 //
 3352 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3353 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3354 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3355 
 3356 // Comparison Code
 3357 operand cmpOp()
 3358 %{
 3359   match(Bool);
 3360 
 3361   format %{ "" %}
 3362   interface(COND_INTER) %{
 3363     equal(0x4, "e");
 3364     not_equal(0x5, "ne");
 3365     less(0xC, "l");
 3366     greater_equal(0xD, "ge");
 3367     less_equal(0xE, "le");
 3368     greater(0xF, "g");
 3369     overflow(0x0, "o");
 3370     no_overflow(0x1, "no");
 3371   %}
 3372 %}
 3373 
 3374 // Comparison Code, unsigned compare.  Used by FP also, with
 3375 // C2 (unordered) turned into GT or LT already.  The other bits
 3376 // C0 and C3 are turned into Carry & Zero flags.
 3377 operand cmpOpU()
 3378 %{
 3379   match(Bool);
 3380 
 3381   format %{ "" %}
 3382   interface(COND_INTER) %{
 3383     equal(0x4, "e");
 3384     not_equal(0x5, "ne");
 3385     less(0x2, "b");
 3386     greater_equal(0x3, "ae");
 3387     less_equal(0x6, "be");
 3388     greater(0x7, "a");
 3389     overflow(0x0, "o");
 3390     no_overflow(0x1, "no");
 3391   %}
 3392 %}
 3393 
 3394 
 3395 // Floating comparisons that don't require any fixup for the unordered case,
 3396 // If both inputs of the comparison are the same, ZF is always set so we
 3397 // don't need to use cmpOpUCF2 for eq/ne
 3398 operand cmpOpUCF() %{
 3399   match(Bool);
 3400   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3401             n->as_Bool()->_test._test == BoolTest::ge ||
 3402             n->as_Bool()->_test._test == BoolTest::le ||
 3403             n->as_Bool()->_test._test == BoolTest::gt ||
 3404             n->in(1)->in(1) == n->in(1)->in(2));
 3405   format %{ "" %}
 3406   interface(COND_INTER) %{
 3407     equal(0xb, "np");
 3408     not_equal(0xa, "p");
 3409     less(0x2, "b");
 3410     greater_equal(0x3, "ae");
 3411     less_equal(0x6, "be");
 3412     greater(0x7, "a");
 3413     overflow(0x0, "o");
 3414     no_overflow(0x1, "no");
 3415   %}
 3416 %}
 3417 
 3418 
 3419 // Floating comparisons that can be fixed up with extra conditional jumps
 3420 operand cmpOpUCF2() %{
 3421   match(Bool);
 3422   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3423              n->as_Bool()->_test._test == BoolTest::eq) &&
 3424             n->in(1)->in(1) != n->in(1)->in(2));
 3425   format %{ "" %}
 3426   interface(COND_INTER) %{
 3427     equal(0x4, "e");
 3428     not_equal(0x5, "ne");
 3429     less(0x2, "b");
 3430     greater_equal(0x3, "ae");
 3431     less_equal(0x6, "be");
 3432     greater(0x7, "a");
 3433     overflow(0x0, "o");
 3434     no_overflow(0x1, "no");
 3435   %}
 3436 %}
 3437 
 3438 //----------OPERAND CLASSES----------------------------------------------------
 3439 // Operand Classes are groups of operands that are used as to simplify
 3440 // instruction definitions by not requiring the AD writer to specify separate
 3441 // instructions for every form of operand when the instruction accepts
 3442 // multiple operand types with the same basic encoding and format.  The classic
 3443 // case of this is memory operands.
 3444 
 3445 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3446                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3447                indCompressedOopOffset,
 3448                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3449                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3450                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3451 
 3452 //----------PIPELINE-----------------------------------------------------------
 3453 // Rules which define the behavior of the target architectures pipeline.
 3454 pipeline %{
 3455 
 3456 //----------ATTRIBUTES---------------------------------------------------------
 3457 attributes %{
 3458   variable_size_instructions;        // Fixed size instructions
 3459   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3460   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3461   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3462   instruction_fetch_units = 1;       // of 16 bytes
 3463 
 3464   // List of nop instructions
 3465   nops( MachNop );
 3466 %}
 3467 
 3468 //----------RESOURCES----------------------------------------------------------
 3469 // Resources are the functional units available to the machine
 3470 
 3471 // Generic P2/P3 pipeline
 3472 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3473 // 3 instructions decoded per cycle.
 3474 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3475 // 3 ALU op, only ALU0 handles mul instructions.
 3476 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3477            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3478            BR, FPU,
 3479            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3480 
 3481 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3482 // Pipeline Description specifies the stages in the machine's pipeline
 3483 
 3484 // Generic P2/P3 pipeline
 3485 pipe_desc(S0, S1, S2, S3, S4, S5);
 3486 
 3487 //----------PIPELINE CLASSES---------------------------------------------------
 3488 // Pipeline Classes describe the stages in which input and output are
 3489 // referenced by the hardware pipeline.
 3490 
 3491 // Naming convention: ialu or fpu
 3492 // Then: _reg
 3493 // Then: _reg if there is a 2nd register
 3494 // Then: _long if it's a pair of instructions implementing a long
 3495 // Then: _fat if it requires the big decoder
 3496 //   Or: _mem if it requires the big decoder and a memory unit.
 3497 
 3498 // Integer ALU reg operation
 3499 pipe_class ialu_reg(rRegI dst)
 3500 %{
 3501     single_instruction;
 3502     dst    : S4(write);
 3503     dst    : S3(read);
 3504     DECODE : S0;        // any decoder
 3505     ALU    : S3;        // any alu
 3506 %}
 3507 
 3508 // Long ALU reg operation
 3509 pipe_class ialu_reg_long(rRegL dst)
 3510 %{
 3511     instruction_count(2);
 3512     dst    : S4(write);
 3513     dst    : S3(read);
 3514     DECODE : S0(2);     // any 2 decoders
 3515     ALU    : S3(2);     // both alus
 3516 %}
 3517 
 3518 // Integer ALU reg operation using big decoder
 3519 pipe_class ialu_reg_fat(rRegI dst)
 3520 %{
 3521     single_instruction;
 3522     dst    : S4(write);
 3523     dst    : S3(read);
 3524     D0     : S0;        // big decoder only
 3525     ALU    : S3;        // any alu
 3526 %}
 3527 
 3528 // Integer ALU reg-reg operation
 3529 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3530 %{
 3531     single_instruction;
 3532     dst    : S4(write);
 3533     src    : S3(read);
 3534     DECODE : S0;        // any decoder
 3535     ALU    : S3;        // any alu
 3536 %}
 3537 
 3538 // Integer ALU reg-reg operation
 3539 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3540 %{
 3541     single_instruction;
 3542     dst    : S4(write);
 3543     src    : S3(read);
 3544     D0     : S0;        // big decoder only
 3545     ALU    : S3;        // any alu
 3546 %}
 3547 
 3548 // Integer ALU reg-mem operation
 3549 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3550 %{
 3551     single_instruction;
 3552     dst    : S5(write);
 3553     mem    : S3(read);
 3554     D0     : S0;        // big decoder only
 3555     ALU    : S4;        // any alu
 3556     MEM    : S3;        // any mem
 3557 %}
 3558 
 3559 // Integer mem operation (prefetch)
 3560 pipe_class ialu_mem(memory mem)
 3561 %{
 3562     single_instruction;
 3563     mem    : S3(read);
 3564     D0     : S0;        // big decoder only
 3565     MEM    : S3;        // any mem
 3566 %}
 3567 
 3568 // Integer Store to Memory
 3569 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3570 %{
 3571     single_instruction;
 3572     mem    : S3(read);
 3573     src    : S5(read);
 3574     D0     : S0;        // big decoder only
 3575     ALU    : S4;        // any alu
 3576     MEM    : S3;
 3577 %}
 3578 
 3579 // // Long Store to Memory
 3580 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3581 // %{
 3582 //     instruction_count(2);
 3583 //     mem    : S3(read);
 3584 //     src    : S5(read);
 3585 //     D0     : S0(2);          // big decoder only; twice
 3586 //     ALU    : S4(2);     // any 2 alus
 3587 //     MEM    : S3(2);  // Both mems
 3588 // %}
 3589 
 3590 // Integer Store to Memory
 3591 pipe_class ialu_mem_imm(memory mem)
 3592 %{
 3593     single_instruction;
 3594     mem    : S3(read);
 3595     D0     : S0;        // big decoder only
 3596     ALU    : S4;        // any alu
 3597     MEM    : S3;
 3598 %}
 3599 
 3600 // Integer ALU0 reg-reg operation
 3601 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3602 %{
 3603     single_instruction;
 3604     dst    : S4(write);
 3605     src    : S3(read);
 3606     D0     : S0;        // Big decoder only
 3607     ALU0   : S3;        // only alu0
 3608 %}
 3609 
 3610 // Integer ALU0 reg-mem operation
 3611 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3612 %{
 3613     single_instruction;
 3614     dst    : S5(write);
 3615     mem    : S3(read);
 3616     D0     : S0;        // big decoder only
 3617     ALU0   : S4;        // ALU0 only
 3618     MEM    : S3;        // any mem
 3619 %}
 3620 
 3621 // Integer ALU reg-reg operation
 3622 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3623 %{
 3624     single_instruction;
 3625     cr     : S4(write);
 3626     src1   : S3(read);
 3627     src2   : S3(read);
 3628     DECODE : S0;        // any decoder
 3629     ALU    : S3;        // any alu
 3630 %}
 3631 
 3632 // Integer ALU reg-imm operation
 3633 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3634 %{
 3635     single_instruction;
 3636     cr     : S4(write);
 3637     src1   : S3(read);
 3638     DECODE : S0;        // any decoder
 3639     ALU    : S3;        // any alu
 3640 %}
 3641 
 3642 // Integer ALU reg-mem operation
 3643 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3644 %{
 3645     single_instruction;
 3646     cr     : S4(write);
 3647     src1   : S3(read);
 3648     src2   : S3(read);
 3649     D0     : S0;        // big decoder only
 3650     ALU    : S4;        // any alu
 3651     MEM    : S3;
 3652 %}
 3653 
 3654 // Conditional move reg-reg
 3655 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3656 %{
 3657     instruction_count(4);
 3658     y      : S4(read);
 3659     q      : S3(read);
 3660     p      : S3(read);
 3661     DECODE : S0(4);     // any decoder
 3662 %}
 3663 
 3664 // Conditional move reg-reg
 3665 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3666 %{
 3667     single_instruction;
 3668     dst    : S4(write);
 3669     src    : S3(read);
 3670     cr     : S3(read);
 3671     DECODE : S0;        // any decoder
 3672 %}
 3673 
 3674 // Conditional move reg-mem
 3675 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3676 %{
 3677     single_instruction;
 3678     dst    : S4(write);
 3679     src    : S3(read);
 3680     cr     : S3(read);
 3681     DECODE : S0;        // any decoder
 3682     MEM    : S3;
 3683 %}
 3684 
 3685 // Conditional move reg-reg long
 3686 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3687 %{
 3688     single_instruction;
 3689     dst    : S4(write);
 3690     src    : S3(read);
 3691     cr     : S3(read);
 3692     DECODE : S0(2);     // any 2 decoders
 3693 %}
 3694 
 3695 // Float reg-reg operation
 3696 pipe_class fpu_reg(regD dst)
 3697 %{
 3698     instruction_count(2);
 3699     dst    : S3(read);
 3700     DECODE : S0(2);     // any 2 decoders
 3701     FPU    : S3;
 3702 %}
 3703 
 3704 // Float reg-reg operation
 3705 pipe_class fpu_reg_reg(regD dst, regD src)
 3706 %{
 3707     instruction_count(2);
 3708     dst    : S4(write);
 3709     src    : S3(read);
 3710     DECODE : S0(2);     // any 2 decoders
 3711     FPU    : S3;
 3712 %}
 3713 
 3714 // Float reg-reg operation
 3715 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3716 %{
 3717     instruction_count(3);
 3718     dst    : S4(write);
 3719     src1   : S3(read);
 3720     src2   : S3(read);
 3721     DECODE : S0(3);     // any 3 decoders
 3722     FPU    : S3(2);
 3723 %}
 3724 
 3725 // Float reg-reg operation
 3726 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3727 %{
 3728     instruction_count(4);
 3729     dst    : S4(write);
 3730     src1   : S3(read);
 3731     src2   : S3(read);
 3732     src3   : S3(read);
 3733     DECODE : S0(4);     // any 3 decoders
 3734     FPU    : S3(2);
 3735 %}
 3736 
 3737 // Float reg-reg operation
 3738 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3739 %{
 3740     instruction_count(4);
 3741     dst    : S4(write);
 3742     src1   : S3(read);
 3743     src2   : S3(read);
 3744     src3   : S3(read);
 3745     DECODE : S1(3);     // any 3 decoders
 3746     D0     : S0;        // Big decoder only
 3747     FPU    : S3(2);
 3748     MEM    : S3;
 3749 %}
 3750 
 3751 // Float reg-mem operation
 3752 pipe_class fpu_reg_mem(regD dst, memory mem)
 3753 %{
 3754     instruction_count(2);
 3755     dst    : S5(write);
 3756     mem    : S3(read);
 3757     D0     : S0;        // big decoder only
 3758     DECODE : S1;        // any decoder for FPU POP
 3759     FPU    : S4;
 3760     MEM    : S3;        // any mem
 3761 %}
 3762 
 3763 // Float reg-mem operation
 3764 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3765 %{
 3766     instruction_count(3);
 3767     dst    : S5(write);
 3768     src1   : S3(read);
 3769     mem    : S3(read);
 3770     D0     : S0;        // big decoder only
 3771     DECODE : S1(2);     // any decoder for FPU POP
 3772     FPU    : S4;
 3773     MEM    : S3;        // any mem
 3774 %}
 3775 
 3776 // Float mem-reg operation
 3777 pipe_class fpu_mem_reg(memory mem, regD src)
 3778 %{
 3779     instruction_count(2);
 3780     src    : S5(read);
 3781     mem    : S3(read);
 3782     DECODE : S0;        // any decoder for FPU PUSH
 3783     D0     : S1;        // big decoder only
 3784     FPU    : S4;
 3785     MEM    : S3;        // any mem
 3786 %}
 3787 
 3788 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3789 %{
 3790     instruction_count(3);
 3791     src1   : S3(read);
 3792     src2   : S3(read);
 3793     mem    : S3(read);
 3794     DECODE : S0(2);     // any decoder for FPU PUSH
 3795     D0     : S1;        // big decoder only
 3796     FPU    : S4;
 3797     MEM    : S3;        // any mem
 3798 %}
 3799 
 3800 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3801 %{
 3802     instruction_count(3);
 3803     src1   : S3(read);
 3804     src2   : S3(read);
 3805     mem    : S4(read);
 3806     DECODE : S0;        // any decoder for FPU PUSH
 3807     D0     : S0(2);     // big decoder only
 3808     FPU    : S4;
 3809     MEM    : S3(2);     // any mem
 3810 %}
 3811 
 3812 pipe_class fpu_mem_mem(memory dst, memory src1)
 3813 %{
 3814     instruction_count(2);
 3815     src1   : S3(read);
 3816     dst    : S4(read);
 3817     D0     : S0(2);     // big decoder only
 3818     MEM    : S3(2);     // any mem
 3819 %}
 3820 
 3821 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3822 %{
 3823     instruction_count(3);
 3824     src1   : S3(read);
 3825     src2   : S3(read);
 3826     dst    : S4(read);
 3827     D0     : S0(3);     // big decoder only
 3828     FPU    : S4;
 3829     MEM    : S3(3);     // any mem
 3830 %}
 3831 
 3832 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3833 %{
 3834     instruction_count(3);
 3835     src1   : S4(read);
 3836     mem    : S4(read);
 3837     DECODE : S0;        // any decoder for FPU PUSH
 3838     D0     : S0(2);     // big decoder only
 3839     FPU    : S4;
 3840     MEM    : S3(2);     // any mem
 3841 %}
 3842 
 3843 // Float load constant
 3844 pipe_class fpu_reg_con(regD dst)
 3845 %{
 3846     instruction_count(2);
 3847     dst    : S5(write);
 3848     D0     : S0;        // big decoder only for the load
 3849     DECODE : S1;        // any decoder for FPU POP
 3850     FPU    : S4;
 3851     MEM    : S3;        // any mem
 3852 %}
 3853 
 3854 // Float load constant
 3855 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3856 %{
 3857     instruction_count(3);
 3858     dst    : S5(write);
 3859     src    : S3(read);
 3860     D0     : S0;        // big decoder only for the load
 3861     DECODE : S1(2);     // any decoder for FPU POP
 3862     FPU    : S4;
 3863     MEM    : S3;        // any mem
 3864 %}
 3865 
 3866 // UnConditional branch
 3867 pipe_class pipe_jmp(label labl)
 3868 %{
 3869     single_instruction;
 3870     BR   : S3;
 3871 %}
 3872 
 3873 // Conditional branch
 3874 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3875 %{
 3876     single_instruction;
 3877     cr    : S1(read);
 3878     BR    : S3;
 3879 %}
 3880 
 3881 // Allocation idiom
 3882 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3883 %{
 3884     instruction_count(1); force_serialization;
 3885     fixed_latency(6);
 3886     heap_ptr : S3(read);
 3887     DECODE   : S0(3);
 3888     D0       : S2;
 3889     MEM      : S3;
 3890     ALU      : S3(2);
 3891     dst      : S5(write);
 3892     BR       : S5;
 3893 %}
 3894 
 3895 // Generic big/slow expanded idiom
 3896 pipe_class pipe_slow()
 3897 %{
 3898     instruction_count(10); multiple_bundles; force_serialization;
 3899     fixed_latency(100);
 3900     D0  : S0(2);
 3901     MEM : S3(2);
 3902 %}
 3903 
 3904 // The real do-nothing guy
 3905 pipe_class empty()
 3906 %{
 3907     instruction_count(0);
 3908 %}
 3909 
 3910 // Define the class for the Nop node
 3911 define
 3912 %{
 3913    MachNop = empty;
 3914 %}
 3915 
 3916 %}
 3917 
 3918 //----------INSTRUCTIONS-------------------------------------------------------
 3919 //
 3920 // match      -- States which machine-independent subtree may be replaced
 3921 //               by this instruction.
 3922 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3923 //               selection to identify a minimum cost tree of machine
 3924 //               instructions that matches a tree of machine-independent
 3925 //               instructions.
 3926 // format     -- A string providing the disassembly for this instruction.
 3927 //               The value of an instruction's operand may be inserted
 3928 //               by referring to it with a '$' prefix.
 3929 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3930 //               to within an encode class as $primary, $secondary, and $tertiary
 3931 //               rrspectively.  The primary opcode is commonly used to
 3932 //               indicate the type of machine instruction, while secondary
 3933 //               and tertiary are often used for prefix options or addressing
 3934 //               modes.
 3935 // ins_encode -- A list of encode classes with parameters. The encode class
 3936 //               name must have been defined in an 'enc_class' specification
 3937 //               in the encode section of the architecture description.
 3938 
 3939 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3940 // Load Float
 3941 instruct MoveF2VL(vlRegF dst, regF src) %{
 3942   match(Set dst src);
 3943   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3944   ins_encode %{
 3945     ShouldNotReachHere();
 3946   %}
 3947   ins_pipe( fpu_reg_reg );
 3948 %}
 3949 
 3950 // Load Float
 3951 instruct MoveF2LEG(legRegF dst, regF src) %{
 3952   match(Set dst src);
 3953   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3954   ins_encode %{
 3955     ShouldNotReachHere();
 3956   %}
 3957   ins_pipe( fpu_reg_reg );
 3958 %}
 3959 
 3960 // Load Float
 3961 instruct MoveVL2F(regF dst, vlRegF src) %{
 3962   match(Set dst src);
 3963   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3964   ins_encode %{
 3965     ShouldNotReachHere();
 3966   %}
 3967   ins_pipe( fpu_reg_reg );
 3968 %}
 3969 
 3970 // Load Float
 3971 instruct MoveLEG2F(regF dst, legRegF src) %{
 3972   match(Set dst src);
 3973   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3974   ins_encode %{
 3975     ShouldNotReachHere();
 3976   %}
 3977   ins_pipe( fpu_reg_reg );
 3978 %}
 3979 
 3980 // Load Double
 3981 instruct MoveD2VL(vlRegD dst, regD src) %{
 3982   match(Set dst src);
 3983   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3984   ins_encode %{
 3985     ShouldNotReachHere();
 3986   %}
 3987   ins_pipe( fpu_reg_reg );
 3988 %}
 3989 
 3990 // Load Double
 3991 instruct MoveD2LEG(legRegD dst, regD src) %{
 3992   match(Set dst src);
 3993   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3994   ins_encode %{
 3995     ShouldNotReachHere();
 3996   %}
 3997   ins_pipe( fpu_reg_reg );
 3998 %}
 3999 
 4000 // Load Double
 4001 instruct MoveVL2D(regD dst, vlRegD src) %{
 4002   match(Set dst src);
 4003   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4004   ins_encode %{
 4005     ShouldNotReachHere();
 4006   %}
 4007   ins_pipe( fpu_reg_reg );
 4008 %}
 4009 
 4010 // Load Double
 4011 instruct MoveLEG2D(regD dst, legRegD src) %{
 4012   match(Set dst src);
 4013   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4014   ins_encode %{
 4015     ShouldNotReachHere();
 4016   %}
 4017   ins_pipe( fpu_reg_reg );
 4018 %}
 4019 
 4020 //----------Load/Store/Move Instructions---------------------------------------
 4021 //----------Load Instructions--------------------------------------------------
 4022 
 4023 // Load Byte (8 bit signed)
 4024 instruct loadB(rRegI dst, memory mem)
 4025 %{
 4026   match(Set dst (LoadB mem));
 4027 
 4028   ins_cost(125);
 4029   format %{ "movsbl  $dst, $mem\t# byte" %}
 4030 
 4031   ins_encode %{
 4032     __ movsbl($dst$$Register, $mem$$Address);
 4033   %}
 4034 
 4035   ins_pipe(ialu_reg_mem);
 4036 %}
 4037 
 4038 // Load Byte (8 bit signed) into Long Register
 4039 instruct loadB2L(rRegL dst, memory mem)
 4040 %{
 4041   match(Set dst (ConvI2L (LoadB mem)));
 4042 
 4043   ins_cost(125);
 4044   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4045 
 4046   ins_encode %{
 4047     __ movsbq($dst$$Register, $mem$$Address);
 4048   %}
 4049 
 4050   ins_pipe(ialu_reg_mem);
 4051 %}
 4052 
 4053 // Load Unsigned Byte (8 bit UNsigned)
 4054 instruct loadUB(rRegI dst, memory mem)
 4055 %{
 4056   match(Set dst (LoadUB mem));
 4057 
 4058   ins_cost(125);
 4059   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4060 
 4061   ins_encode %{
 4062     __ movzbl($dst$$Register, $mem$$Address);
 4063   %}
 4064 
 4065   ins_pipe(ialu_reg_mem);
 4066 %}
 4067 
 4068 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4069 instruct loadUB2L(rRegL dst, memory mem)
 4070 %{
 4071   match(Set dst (ConvI2L (LoadUB mem)));
 4072 
 4073   ins_cost(125);
 4074   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4075 
 4076   ins_encode %{
 4077     __ movzbq($dst$$Register, $mem$$Address);
 4078   %}
 4079 
 4080   ins_pipe(ialu_reg_mem);
 4081 %}
 4082 
 4083 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4084 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4085   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4086   effect(KILL cr);
 4087 
 4088   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4089             "andl    $dst, right_n_bits($mask, 8)" %}
 4090   ins_encode %{
 4091     Register Rdst = $dst$$Register;
 4092     __ movzbq(Rdst, $mem$$Address);
 4093     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4094   %}
 4095   ins_pipe(ialu_reg_mem);
 4096 %}
 4097 
 4098 // Load Short (16 bit signed)
 4099 instruct loadS(rRegI dst, memory mem)
 4100 %{
 4101   match(Set dst (LoadS mem));
 4102 
 4103   ins_cost(125);
 4104   format %{ "movswl $dst, $mem\t# short" %}
 4105 
 4106   ins_encode %{
 4107     __ movswl($dst$$Register, $mem$$Address);
 4108   %}
 4109 
 4110   ins_pipe(ialu_reg_mem);
 4111 %}
 4112 
 4113 // Load Short (16 bit signed) to Byte (8 bit signed)
 4114 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4115   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4116 
 4117   ins_cost(125);
 4118   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4119   ins_encode %{
 4120     __ movsbl($dst$$Register, $mem$$Address);
 4121   %}
 4122   ins_pipe(ialu_reg_mem);
 4123 %}
 4124 
 4125 // Load Short (16 bit signed) into Long Register
 4126 instruct loadS2L(rRegL dst, memory mem)
 4127 %{
 4128   match(Set dst (ConvI2L (LoadS mem)));
 4129 
 4130   ins_cost(125);
 4131   format %{ "movswq $dst, $mem\t# short -> long" %}
 4132 
 4133   ins_encode %{
 4134     __ movswq($dst$$Register, $mem$$Address);
 4135   %}
 4136 
 4137   ins_pipe(ialu_reg_mem);
 4138 %}
 4139 
 4140 // Load Unsigned Short/Char (16 bit UNsigned)
 4141 instruct loadUS(rRegI dst, memory mem)
 4142 %{
 4143   match(Set dst (LoadUS mem));
 4144 
 4145   ins_cost(125);
 4146   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4147 
 4148   ins_encode %{
 4149     __ movzwl($dst$$Register, $mem$$Address);
 4150   %}
 4151 
 4152   ins_pipe(ialu_reg_mem);
 4153 %}
 4154 
 4155 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4156 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4157   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4158 
 4159   ins_cost(125);
 4160   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4161   ins_encode %{
 4162     __ movsbl($dst$$Register, $mem$$Address);
 4163   %}
 4164   ins_pipe(ialu_reg_mem);
 4165 %}
 4166 
 4167 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4168 instruct loadUS2L(rRegL dst, memory mem)
 4169 %{
 4170   match(Set dst (ConvI2L (LoadUS mem)));
 4171 
 4172   ins_cost(125);
 4173   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4174 
 4175   ins_encode %{
 4176     __ movzwq($dst$$Register, $mem$$Address);
 4177   %}
 4178 
 4179   ins_pipe(ialu_reg_mem);
 4180 %}
 4181 
 4182 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4183 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4184   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4185 
 4186   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4187   ins_encode %{
 4188     __ movzbq($dst$$Register, $mem$$Address);
 4189   %}
 4190   ins_pipe(ialu_reg_mem);
 4191 %}
 4192 
 4193 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4194 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4195   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4196   effect(KILL cr);
 4197 
 4198   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4199             "andl    $dst, right_n_bits($mask, 16)" %}
 4200   ins_encode %{
 4201     Register Rdst = $dst$$Register;
 4202     __ movzwq(Rdst, $mem$$Address);
 4203     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4204   %}
 4205   ins_pipe(ialu_reg_mem);
 4206 %}
 4207 
 4208 // Load Integer
 4209 instruct loadI(rRegI dst, memory mem)
 4210 %{
 4211   match(Set dst (LoadI mem));
 4212 
 4213   ins_cost(125);
 4214   format %{ "movl    $dst, $mem\t# int" %}
 4215 
 4216   ins_encode %{
 4217     __ movl($dst$$Register, $mem$$Address);
 4218   %}
 4219 
 4220   ins_pipe(ialu_reg_mem);
 4221 %}
 4222 
 4223 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4224 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4225   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4226 
 4227   ins_cost(125);
 4228   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4229   ins_encode %{
 4230     __ movsbl($dst$$Register, $mem$$Address);
 4231   %}
 4232   ins_pipe(ialu_reg_mem);
 4233 %}
 4234 
 4235 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4236 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4237   match(Set dst (AndI (LoadI mem) mask));
 4238 
 4239   ins_cost(125);
 4240   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4241   ins_encode %{
 4242     __ movzbl($dst$$Register, $mem$$Address);
 4243   %}
 4244   ins_pipe(ialu_reg_mem);
 4245 %}
 4246 
 4247 // Load Integer (32 bit signed) to Short (16 bit signed)
 4248 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4249   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4250 
 4251   ins_cost(125);
 4252   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4253   ins_encode %{
 4254     __ movswl($dst$$Register, $mem$$Address);
 4255   %}
 4256   ins_pipe(ialu_reg_mem);
 4257 %}
 4258 
 4259 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4260 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4261   match(Set dst (AndI (LoadI mem) mask));
 4262 
 4263   ins_cost(125);
 4264   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4265   ins_encode %{
 4266     __ movzwl($dst$$Register, $mem$$Address);
 4267   %}
 4268   ins_pipe(ialu_reg_mem);
 4269 %}
 4270 
 4271 // Load Integer into Long Register
 4272 instruct loadI2L(rRegL dst, memory mem)
 4273 %{
 4274   match(Set dst (ConvI2L (LoadI mem)));
 4275 
 4276   ins_cost(125);
 4277   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4278 
 4279   ins_encode %{
 4280     __ movslq($dst$$Register, $mem$$Address);
 4281   %}
 4282 
 4283   ins_pipe(ialu_reg_mem);
 4284 %}
 4285 
 4286 // Load Integer with mask 0xFF into Long Register
 4287 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4288   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4289 
 4290   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4291   ins_encode %{
 4292     __ movzbq($dst$$Register, $mem$$Address);
 4293   %}
 4294   ins_pipe(ialu_reg_mem);
 4295 %}
 4296 
 4297 // Load Integer with mask 0xFFFF into Long Register
 4298 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4299   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4300 
 4301   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4302   ins_encode %{
 4303     __ movzwq($dst$$Register, $mem$$Address);
 4304   %}
 4305   ins_pipe(ialu_reg_mem);
 4306 %}
 4307 
 4308 // Load Integer with a 31-bit mask into Long Register
 4309 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4310   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4311   effect(KILL cr);
 4312 
 4313   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4314             "andl    $dst, $mask" %}
 4315   ins_encode %{
 4316     Register Rdst = $dst$$Register;
 4317     __ movl(Rdst, $mem$$Address);
 4318     __ andl(Rdst, $mask$$constant);
 4319   %}
 4320   ins_pipe(ialu_reg_mem);
 4321 %}
 4322 
 4323 // Load Unsigned Integer into Long Register
 4324 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4325 %{
 4326   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4327 
 4328   ins_cost(125);
 4329   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4330 
 4331   ins_encode %{
 4332     __ movl($dst$$Register, $mem$$Address);
 4333   %}
 4334 
 4335   ins_pipe(ialu_reg_mem);
 4336 %}
 4337 
 4338 // Load Long
 4339 instruct loadL(rRegL dst, memory mem)
 4340 %{
 4341   match(Set dst (LoadL mem));
 4342 
 4343   ins_cost(125);
 4344   format %{ "movq    $dst, $mem\t# long" %}
 4345 
 4346   ins_encode %{
 4347     __ movq($dst$$Register, $mem$$Address);
 4348   %}
 4349 
 4350   ins_pipe(ialu_reg_mem); // XXX
 4351 %}
 4352 
 4353 // Load Range
 4354 instruct loadRange(rRegI dst, memory mem)
 4355 %{
 4356   match(Set dst (LoadRange mem));
 4357 
 4358   ins_cost(125); // XXX
 4359   format %{ "movl    $dst, $mem\t# range" %}
 4360   ins_encode %{
 4361     __ movl($dst$$Register, $mem$$Address);
 4362   %}
 4363   ins_pipe(ialu_reg_mem);
 4364 %}
 4365 
 4366 // Load Pointer
 4367 instruct loadP(rRegP dst, memory mem)
 4368 %{
 4369   match(Set dst (LoadP mem));
 4370   predicate(n->as_Load()->barrier_data() == 0);
 4371 
 4372   ins_cost(125); // XXX
 4373   format %{ "movq    $dst, $mem\t# ptr" %}
 4374   ins_encode %{
 4375     __ movq($dst$$Register, $mem$$Address);
 4376   %}
 4377   ins_pipe(ialu_reg_mem); // XXX
 4378 %}
 4379 
 4380 // Load Compressed Pointer
 4381 instruct loadN(rRegN dst, memory mem)
 4382 %{
 4383    match(Set dst (LoadN mem));
 4384 
 4385    ins_cost(125); // XXX
 4386    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4387    ins_encode %{
 4388      __ movl($dst$$Register, $mem$$Address);
 4389    %}
 4390    ins_pipe(ialu_reg_mem); // XXX
 4391 %}
 4392 
 4393 
 4394 // Load Klass Pointer
 4395 instruct loadKlass(rRegP dst, memory mem)
 4396 %{
 4397   match(Set dst (LoadKlass mem));
 4398 
 4399   ins_cost(125); // XXX
 4400   format %{ "movq    $dst, $mem\t# class" %}
 4401   ins_encode %{
 4402     __ movq($dst$$Register, $mem$$Address);
 4403   %}
 4404   ins_pipe(ialu_reg_mem); // XXX
 4405 %}
 4406 
 4407 // Load narrow Klass Pointer
 4408 instruct loadNKlass(rRegN dst, memory mem)
 4409 %{
 4410   match(Set dst (LoadNKlass mem));
 4411 
 4412   ins_cost(125); // XXX
 4413   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4414   ins_encode %{
 4415     __ movl($dst$$Register, $mem$$Address);
 4416   %}
 4417   ins_pipe(ialu_reg_mem); // XXX
 4418 %}
 4419 
 4420 // Load Float
 4421 instruct loadF(regF dst, memory mem)
 4422 %{
 4423   match(Set dst (LoadF mem));
 4424 
 4425   ins_cost(145); // XXX
 4426   format %{ "movss   $dst, $mem\t# float" %}
 4427   ins_encode %{
 4428     __ movflt($dst$$XMMRegister, $mem$$Address);
 4429   %}
 4430   ins_pipe(pipe_slow); // XXX
 4431 %}
 4432 
 4433 // Load Double
 4434 instruct loadD_partial(regD dst, memory mem)
 4435 %{
 4436   predicate(!UseXmmLoadAndClearUpper);
 4437   match(Set dst (LoadD mem));
 4438 
 4439   ins_cost(145); // XXX
 4440   format %{ "movlpd  $dst, $mem\t# double" %}
 4441   ins_encode %{
 4442     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4443   %}
 4444   ins_pipe(pipe_slow); // XXX
 4445 %}
 4446 
 4447 instruct loadD(regD dst, memory mem)
 4448 %{
 4449   predicate(UseXmmLoadAndClearUpper);
 4450   match(Set dst (LoadD mem));
 4451 
 4452   ins_cost(145); // XXX
 4453   format %{ "movsd   $dst, $mem\t# double" %}
 4454   ins_encode %{
 4455     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4456   %}
 4457   ins_pipe(pipe_slow); // XXX
 4458 %}
 4459 
 4460 // max = java.lang.Math.max(float a, float b)
 4461 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4462   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4463   match(Set dst (MaxF a b));
 4464   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4465   format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4466   ins_encode %{
 4467     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4468   %}
 4469   ins_pipe( pipe_slow );
 4470 %}
 4471 
 4472 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4473   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4474   match(Set dst (MaxF a b));
 4475   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4476 
 4477   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 4478   ins_encode %{
 4479     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4480                     false /*min*/, true /*single*/);
 4481   %}
 4482   ins_pipe( pipe_slow );
 4483 %}
 4484 
 4485 // max = java.lang.Math.max(double a, double b)
 4486 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4487   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4488   match(Set dst (MaxD a b));
 4489   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4490   format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4491   ins_encode %{
 4492     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4493   %}
 4494   ins_pipe( pipe_slow );
 4495 %}
 4496 
 4497 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4498   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4499   match(Set dst (MaxD a b));
 4500   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4501 
 4502   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 4503   ins_encode %{
 4504     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4505                     false /*min*/, false /*single*/);
 4506   %}
 4507   ins_pipe( pipe_slow );
 4508 %}
 4509 
 4510 // min = java.lang.Math.min(float a, float b)
 4511 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4512   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4513   match(Set dst (MinF a b));
 4514   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4515   format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4516   ins_encode %{
 4517     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4518   %}
 4519   ins_pipe( pipe_slow );
 4520 %}
 4521 
 4522 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4523   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4524   match(Set dst (MinF a b));
 4525   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4526 
 4527   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 4528   ins_encode %{
 4529     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4530                     true /*min*/, true /*single*/);
 4531   %}
 4532   ins_pipe( pipe_slow );
 4533 %}
 4534 
 4535 // min = java.lang.Math.min(double a, double b)
 4536 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4537   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4538   match(Set dst (MinD a b));
 4539   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4540     format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4541   ins_encode %{
 4542     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4543   %}
 4544   ins_pipe( pipe_slow );
 4545 %}
 4546 
 4547 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4548   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4549   match(Set dst (MinD a b));
 4550   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4551 
 4552   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 4553   ins_encode %{
 4554     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4555                     true /*min*/, false /*single*/);
 4556   %}
 4557   ins_pipe( pipe_slow );
 4558 %}
 4559 
 4560 // Load Effective Address
 4561 instruct leaP8(rRegP dst, indOffset8 mem)
 4562 %{
 4563   match(Set dst mem);
 4564 
 4565   ins_cost(110); // XXX
 4566   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4567   ins_encode %{
 4568     __ leaq($dst$$Register, $mem$$Address);
 4569   %}
 4570   ins_pipe(ialu_reg_reg_fat);
 4571 %}
 4572 
 4573 instruct leaP32(rRegP dst, indOffset32 mem)
 4574 %{
 4575   match(Set dst mem);
 4576 
 4577   ins_cost(110);
 4578   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4579   ins_encode %{
 4580     __ leaq($dst$$Register, $mem$$Address);
 4581   %}
 4582   ins_pipe(ialu_reg_reg_fat);
 4583 %}
 4584 
 4585 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4586 %{
 4587   match(Set dst mem);
 4588 
 4589   ins_cost(110);
 4590   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4591   ins_encode %{
 4592     __ leaq($dst$$Register, $mem$$Address);
 4593   %}
 4594   ins_pipe(ialu_reg_reg_fat);
 4595 %}
 4596 
 4597 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4598 %{
 4599   match(Set dst mem);
 4600 
 4601   ins_cost(110);
 4602   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4603   ins_encode %{
 4604     __ leaq($dst$$Register, $mem$$Address);
 4605   %}
 4606   ins_pipe(ialu_reg_reg_fat);
 4607 %}
 4608 
 4609 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4610 %{
 4611   match(Set dst mem);
 4612 
 4613   ins_cost(110);
 4614   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4615   ins_encode %{
 4616     __ leaq($dst$$Register, $mem$$Address);
 4617   %}
 4618   ins_pipe(ialu_reg_reg_fat);
 4619 %}
 4620 
 4621 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4622 %{
 4623   match(Set dst mem);
 4624 
 4625   ins_cost(110);
 4626   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4627   ins_encode %{
 4628     __ leaq($dst$$Register, $mem$$Address);
 4629   %}
 4630   ins_pipe(ialu_reg_reg_fat);
 4631 %}
 4632 
 4633 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4634 %{
 4635   match(Set dst mem);
 4636 
 4637   ins_cost(110);
 4638   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4639   ins_encode %{
 4640     __ leaq($dst$$Register, $mem$$Address);
 4641   %}
 4642   ins_pipe(ialu_reg_reg_fat);
 4643 %}
 4644 
 4645 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4646 %{
 4647   match(Set dst mem);
 4648 
 4649   ins_cost(110);
 4650   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4651   ins_encode %{
 4652     __ leaq($dst$$Register, $mem$$Address);
 4653   %}
 4654   ins_pipe(ialu_reg_reg_fat);
 4655 %}
 4656 
 4657 // Load Effective Address which uses Narrow (32-bits) oop
 4658 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4659 %{
 4660   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4661   match(Set dst mem);
 4662 
 4663   ins_cost(110);
 4664   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4665   ins_encode %{
 4666     __ leaq($dst$$Register, $mem$$Address);
 4667   %}
 4668   ins_pipe(ialu_reg_reg_fat);
 4669 %}
 4670 
 4671 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4672 %{
 4673   predicate(CompressedOops::shift() == 0);
 4674   match(Set dst mem);
 4675 
 4676   ins_cost(110); // XXX
 4677   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4678   ins_encode %{
 4679     __ leaq($dst$$Register, $mem$$Address);
 4680   %}
 4681   ins_pipe(ialu_reg_reg_fat);
 4682 %}
 4683 
 4684 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4685 %{
 4686   predicate(CompressedOops::shift() == 0);
 4687   match(Set dst mem);
 4688 
 4689   ins_cost(110);
 4690   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4691   ins_encode %{
 4692     __ leaq($dst$$Register, $mem$$Address);
 4693   %}
 4694   ins_pipe(ialu_reg_reg_fat);
 4695 %}
 4696 
 4697 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4698 %{
 4699   predicate(CompressedOops::shift() == 0);
 4700   match(Set dst mem);
 4701 
 4702   ins_cost(110);
 4703   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4704   ins_encode %{
 4705     __ leaq($dst$$Register, $mem$$Address);
 4706   %}
 4707   ins_pipe(ialu_reg_reg_fat);
 4708 %}
 4709 
 4710 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4711 %{
 4712   predicate(CompressedOops::shift() == 0);
 4713   match(Set dst mem);
 4714 
 4715   ins_cost(110);
 4716   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4717   ins_encode %{
 4718     __ leaq($dst$$Register, $mem$$Address);
 4719   %}
 4720   ins_pipe(ialu_reg_reg_fat);
 4721 %}
 4722 
 4723 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4724 %{
 4725   predicate(CompressedOops::shift() == 0);
 4726   match(Set dst mem);
 4727 
 4728   ins_cost(110);
 4729   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4730   ins_encode %{
 4731     __ leaq($dst$$Register, $mem$$Address);
 4732   %}
 4733   ins_pipe(ialu_reg_reg_fat);
 4734 %}
 4735 
 4736 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4737 %{
 4738   predicate(CompressedOops::shift() == 0);
 4739   match(Set dst mem);
 4740 
 4741   ins_cost(110);
 4742   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4743   ins_encode %{
 4744     __ leaq($dst$$Register, $mem$$Address);
 4745   %}
 4746   ins_pipe(ialu_reg_reg_fat);
 4747 %}
 4748 
 4749 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4750 %{
 4751   predicate(CompressedOops::shift() == 0);
 4752   match(Set dst mem);
 4753 
 4754   ins_cost(110);
 4755   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4756   ins_encode %{
 4757     __ leaq($dst$$Register, $mem$$Address);
 4758   %}
 4759   ins_pipe(ialu_reg_reg_fat);
 4760 %}
 4761 
 4762 instruct loadConI(rRegI dst, immI src)
 4763 %{
 4764   match(Set dst src);
 4765 
 4766   format %{ "movl    $dst, $src\t# int" %}
 4767   ins_encode %{
 4768     __ movl($dst$$Register, $src$$constant);
 4769   %}
 4770   ins_pipe(ialu_reg_fat); // XXX
 4771 %}
 4772 
 4773 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4774 %{
 4775   match(Set dst src);
 4776   effect(KILL cr);
 4777 
 4778   ins_cost(50);
 4779   format %{ "xorl    $dst, $dst\t# int" %}
 4780   ins_encode %{
 4781     __ xorl($dst$$Register, $dst$$Register);
 4782   %}
 4783   ins_pipe(ialu_reg);
 4784 %}
 4785 
 4786 instruct loadConL(rRegL dst, immL src)
 4787 %{
 4788   match(Set dst src);
 4789 
 4790   ins_cost(150);
 4791   format %{ "movq    $dst, $src\t# long" %}
 4792   ins_encode %{
 4793     __ mov64($dst$$Register, $src$$constant);
 4794   %}
 4795   ins_pipe(ialu_reg);
 4796 %}
 4797 
 4798 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4799 %{
 4800   match(Set dst src);
 4801   effect(KILL cr);
 4802 
 4803   ins_cost(50);
 4804   format %{ "xorl    $dst, $dst\t# long" %}
 4805   ins_encode %{
 4806     __ xorl($dst$$Register, $dst$$Register);
 4807   %}
 4808   ins_pipe(ialu_reg); // XXX
 4809 %}
 4810 
 4811 instruct loadConUL32(rRegL dst, immUL32 src)
 4812 %{
 4813   match(Set dst src);
 4814 
 4815   ins_cost(60);
 4816   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4817   ins_encode %{
 4818     __ movl($dst$$Register, $src$$constant);
 4819   %}
 4820   ins_pipe(ialu_reg);
 4821 %}
 4822 
 4823 instruct loadConL32(rRegL dst, immL32 src)
 4824 %{
 4825   match(Set dst src);
 4826 
 4827   ins_cost(70);
 4828   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4829   ins_encode %{
 4830     __ movq($dst$$Register, $src$$constant);
 4831   %}
 4832   ins_pipe(ialu_reg);
 4833 %}
 4834 
 4835 instruct loadConP(rRegP dst, immP con) %{
 4836   match(Set dst con);
 4837 
 4838   format %{ "movq    $dst, $con\t# ptr" %}
 4839   ins_encode %{
 4840     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4841   %}
 4842   ins_pipe(ialu_reg_fat); // XXX
 4843 %}
 4844 
 4845 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4846 %{
 4847   match(Set dst src);
 4848   effect(KILL cr);
 4849 
 4850   ins_cost(50);
 4851   format %{ "xorl    $dst, $dst\t# ptr" %}
 4852   ins_encode %{
 4853     __ xorl($dst$$Register, $dst$$Register);
 4854   %}
 4855   ins_pipe(ialu_reg);
 4856 %}
 4857 
 4858 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4859 %{
 4860   match(Set dst src);
 4861   effect(KILL cr);
 4862 
 4863   ins_cost(60);
 4864   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4865   ins_encode %{
 4866     __ movl($dst$$Register, $src$$constant);
 4867   %}
 4868   ins_pipe(ialu_reg);
 4869 %}
 4870 
 4871 instruct loadConF(regF dst, immF con) %{
 4872   match(Set dst con);
 4873   ins_cost(125);
 4874   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4875   ins_encode %{
 4876     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4877   %}
 4878   ins_pipe(pipe_slow);
 4879 %}
 4880 
 4881 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4882   match(Set dst src);
 4883   effect(KILL cr);
 4884   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4885   ins_encode %{
 4886     __ xorq($dst$$Register, $dst$$Register);
 4887   %}
 4888   ins_pipe(ialu_reg);
 4889 %}
 4890 
 4891 instruct loadConN(rRegN dst, immN src) %{
 4892   match(Set dst src);
 4893 
 4894   ins_cost(125);
 4895   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4896   ins_encode %{
 4897     address con = (address)$src$$constant;
 4898     if (con == nullptr) {
 4899       ShouldNotReachHere();
 4900     } else {
 4901       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4902     }
 4903   %}
 4904   ins_pipe(ialu_reg_fat); // XXX
 4905 %}
 4906 
 4907 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4908   match(Set dst src);
 4909 
 4910   ins_cost(125);
 4911   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4912   ins_encode %{
 4913     address con = (address)$src$$constant;
 4914     if (con == nullptr) {
 4915       ShouldNotReachHere();
 4916     } else {
 4917       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4918     }
 4919   %}
 4920   ins_pipe(ialu_reg_fat); // XXX
 4921 %}
 4922 
 4923 instruct loadConF0(regF dst, immF0 src)
 4924 %{
 4925   match(Set dst src);
 4926   ins_cost(100);
 4927 
 4928   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4929   ins_encode %{
 4930     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4931   %}
 4932   ins_pipe(pipe_slow);
 4933 %}
 4934 
 4935 // Use the same format since predicate() can not be used here.
 4936 instruct loadConD(regD dst, immD con) %{
 4937   match(Set dst con);
 4938   ins_cost(125);
 4939   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4940   ins_encode %{
 4941     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4942   %}
 4943   ins_pipe(pipe_slow);
 4944 %}
 4945 
 4946 instruct loadConD0(regD dst, immD0 src)
 4947 %{
 4948   match(Set dst src);
 4949   ins_cost(100);
 4950 
 4951   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4952   ins_encode %{
 4953     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4954   %}
 4955   ins_pipe(pipe_slow);
 4956 %}
 4957 
 4958 instruct loadSSI(rRegI dst, stackSlotI src)
 4959 %{
 4960   match(Set dst src);
 4961 
 4962   ins_cost(125);
 4963   format %{ "movl    $dst, $src\t# int stk" %}
 4964   ins_encode %{
 4965     __ movl($dst$$Register, $src$$Address);
 4966   %}
 4967   ins_pipe(ialu_reg_mem);
 4968 %}
 4969 
 4970 instruct loadSSL(rRegL dst, stackSlotL src)
 4971 %{
 4972   match(Set dst src);
 4973 
 4974   ins_cost(125);
 4975   format %{ "movq    $dst, $src\t# long stk" %}
 4976   ins_encode %{
 4977     __ movq($dst$$Register, $src$$Address);
 4978   %}
 4979   ins_pipe(ialu_reg_mem);
 4980 %}
 4981 
 4982 instruct loadSSP(rRegP dst, stackSlotP src)
 4983 %{
 4984   match(Set dst src);
 4985 
 4986   ins_cost(125);
 4987   format %{ "movq    $dst, $src\t# ptr stk" %}
 4988   ins_encode %{
 4989     __ movq($dst$$Register, $src$$Address);
 4990   %}
 4991   ins_pipe(ialu_reg_mem);
 4992 %}
 4993 
 4994 instruct loadSSF(regF dst, stackSlotF src)
 4995 %{
 4996   match(Set dst src);
 4997 
 4998   ins_cost(125);
 4999   format %{ "movss   $dst, $src\t# float stk" %}
 5000   ins_encode %{
 5001     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5002   %}
 5003   ins_pipe(pipe_slow); // XXX
 5004 %}
 5005 
 5006 // Use the same format since predicate() can not be used here.
 5007 instruct loadSSD(regD dst, stackSlotD src)
 5008 %{
 5009   match(Set dst src);
 5010 
 5011   ins_cost(125);
 5012   format %{ "movsd   $dst, $src\t# double stk" %}
 5013   ins_encode  %{
 5014     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5015   %}
 5016   ins_pipe(pipe_slow); // XXX
 5017 %}
 5018 
 5019 // Prefetch instructions for allocation.
 5020 // Must be safe to execute with invalid address (cannot fault).
 5021 
 5022 instruct prefetchAlloc( memory mem ) %{
 5023   predicate(AllocatePrefetchInstr==3);
 5024   match(PrefetchAllocation mem);
 5025   ins_cost(125);
 5026 
 5027   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5028   ins_encode %{
 5029     __ prefetchw($mem$$Address);
 5030   %}
 5031   ins_pipe(ialu_mem);
 5032 %}
 5033 
 5034 instruct prefetchAllocNTA( memory mem ) %{
 5035   predicate(AllocatePrefetchInstr==0);
 5036   match(PrefetchAllocation mem);
 5037   ins_cost(125);
 5038 
 5039   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5040   ins_encode %{
 5041     __ prefetchnta($mem$$Address);
 5042   %}
 5043   ins_pipe(ialu_mem);
 5044 %}
 5045 
 5046 instruct prefetchAllocT0( memory mem ) %{
 5047   predicate(AllocatePrefetchInstr==1);
 5048   match(PrefetchAllocation mem);
 5049   ins_cost(125);
 5050 
 5051   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5052   ins_encode %{
 5053     __ prefetcht0($mem$$Address);
 5054   %}
 5055   ins_pipe(ialu_mem);
 5056 %}
 5057 
 5058 instruct prefetchAllocT2( memory mem ) %{
 5059   predicate(AllocatePrefetchInstr==2);
 5060   match(PrefetchAllocation mem);
 5061   ins_cost(125);
 5062 
 5063   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5064   ins_encode %{
 5065     __ prefetcht2($mem$$Address);
 5066   %}
 5067   ins_pipe(ialu_mem);
 5068 %}
 5069 
 5070 //----------Store Instructions-------------------------------------------------
 5071 
 5072 // Store Byte
 5073 instruct storeB(memory mem, rRegI src)
 5074 %{
 5075   match(Set mem (StoreB mem src));
 5076 
 5077   ins_cost(125); // XXX
 5078   format %{ "movb    $mem, $src\t# byte" %}
 5079   ins_encode %{
 5080     __ movb($mem$$Address, $src$$Register);
 5081   %}
 5082   ins_pipe(ialu_mem_reg);
 5083 %}
 5084 
 5085 // Store Char/Short
 5086 instruct storeC(memory mem, rRegI src)
 5087 %{
 5088   match(Set mem (StoreC mem src));
 5089 
 5090   ins_cost(125); // XXX
 5091   format %{ "movw    $mem, $src\t# char/short" %}
 5092   ins_encode %{
 5093     __ movw($mem$$Address, $src$$Register);
 5094   %}
 5095   ins_pipe(ialu_mem_reg);
 5096 %}
 5097 
 5098 // Store Integer
 5099 instruct storeI(memory mem, rRegI src)
 5100 %{
 5101   match(Set mem (StoreI mem src));
 5102 
 5103   ins_cost(125); // XXX
 5104   format %{ "movl    $mem, $src\t# int" %}
 5105   ins_encode %{
 5106     __ movl($mem$$Address, $src$$Register);
 5107   %}
 5108   ins_pipe(ialu_mem_reg);
 5109 %}
 5110 
 5111 // Store Long
 5112 instruct storeL(memory mem, rRegL src)
 5113 %{
 5114   match(Set mem (StoreL mem src));
 5115 
 5116   ins_cost(125); // XXX
 5117   format %{ "movq    $mem, $src\t# long" %}
 5118   ins_encode %{
 5119     __ movq($mem$$Address, $src$$Register);
 5120   %}
 5121   ins_pipe(ialu_mem_reg); // XXX
 5122 %}
 5123 
 5124 // Store Pointer
 5125 instruct storeP(memory mem, any_RegP src)
 5126 %{
 5127   predicate(n->as_Store()->barrier_data() == 0);
 5128   match(Set mem (StoreP mem src));
 5129 
 5130   ins_cost(125); // XXX
 5131   format %{ "movq    $mem, $src\t# ptr" %}
 5132   ins_encode %{
 5133     __ movq($mem$$Address, $src$$Register);
 5134   %}
 5135   ins_pipe(ialu_mem_reg);
 5136 %}
 5137 
 5138 instruct storeImmP0(memory mem, immP0 zero)
 5139 %{
 5140   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5141   match(Set mem (StoreP mem zero));
 5142 
 5143   ins_cost(125); // XXX
 5144   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5145   ins_encode %{
 5146     __ movq($mem$$Address, r12);
 5147   %}
 5148   ins_pipe(ialu_mem_reg);
 5149 %}
 5150 
 5151 // Store Null Pointer, mark word, or other simple pointer constant.
 5152 instruct storeImmP(memory mem, immP31 src)
 5153 %{
 5154   predicate(n->as_Store()->barrier_data() == 0);
 5155   match(Set mem (StoreP mem src));
 5156 
 5157   ins_cost(150); // XXX
 5158   format %{ "movq    $mem, $src\t# ptr" %}
 5159   ins_encode %{
 5160     __ movq($mem$$Address, $src$$constant);
 5161   %}
 5162   ins_pipe(ialu_mem_imm);
 5163 %}
 5164 
 5165 // Store Compressed Pointer
 5166 instruct storeN(memory mem, rRegN src)
 5167 %{
 5168   match(Set mem (StoreN mem src));
 5169 
 5170   ins_cost(125); // XXX
 5171   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5172   ins_encode %{
 5173     __ movl($mem$$Address, $src$$Register);
 5174   %}
 5175   ins_pipe(ialu_mem_reg);
 5176 %}
 5177 
 5178 instruct storeNKlass(memory mem, rRegN src)
 5179 %{
 5180   match(Set mem (StoreNKlass mem src));
 5181 
 5182   ins_cost(125); // XXX
 5183   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5184   ins_encode %{
 5185     __ movl($mem$$Address, $src$$Register);
 5186   %}
 5187   ins_pipe(ialu_mem_reg);
 5188 %}
 5189 
 5190 instruct storeImmN0(memory mem, immN0 zero)
 5191 %{
 5192   predicate(CompressedOops::base() == nullptr);
 5193   match(Set mem (StoreN mem zero));
 5194 
 5195   ins_cost(125); // XXX
 5196   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5197   ins_encode %{
 5198     __ movl($mem$$Address, r12);
 5199   %}
 5200   ins_pipe(ialu_mem_reg);
 5201 %}
 5202 
 5203 instruct storeImmN(memory mem, immN src)
 5204 %{
 5205   match(Set mem (StoreN mem src));
 5206 
 5207   ins_cost(150); // XXX
 5208   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5209   ins_encode %{
 5210     address con = (address)$src$$constant;
 5211     if (con == nullptr) {
 5212       __ movl($mem$$Address, 0);
 5213     } else {
 5214       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5215     }
 5216   %}
 5217   ins_pipe(ialu_mem_imm);
 5218 %}
 5219 
 5220 instruct storeImmNKlass(memory mem, immNKlass src)
 5221 %{
 5222   match(Set mem (StoreNKlass mem src));
 5223 
 5224   ins_cost(150); // XXX
 5225   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5226   ins_encode %{
 5227     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5228   %}
 5229   ins_pipe(ialu_mem_imm);
 5230 %}
 5231 
 5232 // Store Integer Immediate
 5233 instruct storeImmI0(memory mem, immI_0 zero)
 5234 %{
 5235   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5236   match(Set mem (StoreI mem zero));
 5237 
 5238   ins_cost(125); // XXX
 5239   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5240   ins_encode %{
 5241     __ movl($mem$$Address, r12);
 5242   %}
 5243   ins_pipe(ialu_mem_reg);
 5244 %}
 5245 
 5246 instruct storeImmI(memory mem, immI src)
 5247 %{
 5248   match(Set mem (StoreI mem src));
 5249 
 5250   ins_cost(150);
 5251   format %{ "movl    $mem, $src\t# int" %}
 5252   ins_encode %{
 5253     __ movl($mem$$Address, $src$$constant);
 5254   %}
 5255   ins_pipe(ialu_mem_imm);
 5256 %}
 5257 
 5258 // Store Long Immediate
 5259 instruct storeImmL0(memory mem, immL0 zero)
 5260 %{
 5261   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5262   match(Set mem (StoreL mem zero));
 5263 
 5264   ins_cost(125); // XXX
 5265   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5266   ins_encode %{
 5267     __ movq($mem$$Address, r12);
 5268   %}
 5269   ins_pipe(ialu_mem_reg);
 5270 %}
 5271 
 5272 instruct storeImmL(memory mem, immL32 src)
 5273 %{
 5274   match(Set mem (StoreL mem src));
 5275 
 5276   ins_cost(150);
 5277   format %{ "movq    $mem, $src\t# long" %}
 5278   ins_encode %{
 5279     __ movq($mem$$Address, $src$$constant);
 5280   %}
 5281   ins_pipe(ialu_mem_imm);
 5282 %}
 5283 
 5284 // Store Short/Char Immediate
 5285 instruct storeImmC0(memory mem, immI_0 zero)
 5286 %{
 5287   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5288   match(Set mem (StoreC mem zero));
 5289 
 5290   ins_cost(125); // XXX
 5291   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5292   ins_encode %{
 5293     __ movw($mem$$Address, r12);
 5294   %}
 5295   ins_pipe(ialu_mem_reg);
 5296 %}
 5297 
 5298 instruct storeImmI16(memory mem, immI16 src)
 5299 %{
 5300   predicate(UseStoreImmI16);
 5301   match(Set mem (StoreC mem src));
 5302 
 5303   ins_cost(150);
 5304   format %{ "movw    $mem, $src\t# short/char" %}
 5305   ins_encode %{
 5306     __ movw($mem$$Address, $src$$constant);
 5307   %}
 5308   ins_pipe(ialu_mem_imm);
 5309 %}
 5310 
 5311 // Store Byte Immediate
 5312 instruct storeImmB0(memory mem, immI_0 zero)
 5313 %{
 5314   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5315   match(Set mem (StoreB mem zero));
 5316 
 5317   ins_cost(125); // XXX
 5318   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5319   ins_encode %{
 5320     __ movb($mem$$Address, r12);
 5321   %}
 5322   ins_pipe(ialu_mem_reg);
 5323 %}
 5324 
 5325 instruct storeImmB(memory mem, immI8 src)
 5326 %{
 5327   match(Set mem (StoreB mem src));
 5328 
 5329   ins_cost(150); // XXX
 5330   format %{ "movb    $mem, $src\t# byte" %}
 5331   ins_encode %{
 5332     __ movb($mem$$Address, $src$$constant);
 5333   %}
 5334   ins_pipe(ialu_mem_imm);
 5335 %}
 5336 
 5337 // Store CMS card-mark Immediate
 5338 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 5339 %{
 5340   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5341   match(Set mem (StoreCM mem zero));
 5342 
 5343   ins_cost(125); // XXX
 5344   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 5345   ins_encode %{
 5346     __ movb($mem$$Address, r12);
 5347   %}
 5348   ins_pipe(ialu_mem_reg);
 5349 %}
 5350 
 5351 instruct storeImmCM0(memory mem, immI_0 src)
 5352 %{
 5353   match(Set mem (StoreCM mem src));
 5354 
 5355   ins_cost(150); // XXX
 5356   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 5357   ins_encode %{
 5358     __ movb($mem$$Address, $src$$constant);
 5359   %}
 5360   ins_pipe(ialu_mem_imm);
 5361 %}
 5362 
 5363 // Store Float
 5364 instruct storeF(memory mem, regF src)
 5365 %{
 5366   match(Set mem (StoreF mem src));
 5367 
 5368   ins_cost(95); // XXX
 5369   format %{ "movss   $mem, $src\t# float" %}
 5370   ins_encode %{
 5371     __ movflt($mem$$Address, $src$$XMMRegister);
 5372   %}
 5373   ins_pipe(pipe_slow); // XXX
 5374 %}
 5375 
 5376 // Store immediate Float value (it is faster than store from XMM register)
 5377 instruct storeF0(memory mem, immF0 zero)
 5378 %{
 5379   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5380   match(Set mem (StoreF mem zero));
 5381 
 5382   ins_cost(25); // XXX
 5383   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5384   ins_encode %{
 5385     __ movl($mem$$Address, r12);
 5386   %}
 5387   ins_pipe(ialu_mem_reg);
 5388 %}
 5389 
 5390 instruct storeF_imm(memory mem, immF src)
 5391 %{
 5392   match(Set mem (StoreF mem src));
 5393 
 5394   ins_cost(50);
 5395   format %{ "movl    $mem, $src\t# float" %}
 5396   ins_encode %{
 5397     __ movl($mem$$Address, jint_cast($src$$constant));
 5398   %}
 5399   ins_pipe(ialu_mem_imm);
 5400 %}
 5401 
 5402 // Store Double
 5403 instruct storeD(memory mem, regD src)
 5404 %{
 5405   match(Set mem (StoreD mem src));
 5406 
 5407   ins_cost(95); // XXX
 5408   format %{ "movsd   $mem, $src\t# double" %}
 5409   ins_encode %{
 5410     __ movdbl($mem$$Address, $src$$XMMRegister);
 5411   %}
 5412   ins_pipe(pipe_slow); // XXX
 5413 %}
 5414 
 5415 // Store immediate double 0.0 (it is faster than store from XMM register)
 5416 instruct storeD0_imm(memory mem, immD0 src)
 5417 %{
 5418   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5419   match(Set mem (StoreD mem src));
 5420 
 5421   ins_cost(50);
 5422   format %{ "movq    $mem, $src\t# double 0." %}
 5423   ins_encode %{
 5424     __ movq($mem$$Address, $src$$constant);
 5425   %}
 5426   ins_pipe(ialu_mem_imm);
 5427 %}
 5428 
 5429 instruct storeD0(memory mem, immD0 zero)
 5430 %{
 5431   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5432   match(Set mem (StoreD mem zero));
 5433 
 5434   ins_cost(25); // XXX
 5435   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5436   ins_encode %{
 5437     __ movq($mem$$Address, r12);
 5438   %}
 5439   ins_pipe(ialu_mem_reg);
 5440 %}
 5441 
 5442 instruct storeSSI(stackSlotI dst, rRegI src)
 5443 %{
 5444   match(Set dst src);
 5445 
 5446   ins_cost(100);
 5447   format %{ "movl    $dst, $src\t# int stk" %}
 5448   ins_encode %{
 5449     __ movl($dst$$Address, $src$$Register);
 5450   %}
 5451   ins_pipe( ialu_mem_reg );
 5452 %}
 5453 
 5454 instruct storeSSL(stackSlotL dst, rRegL src)
 5455 %{
 5456   match(Set dst src);
 5457 
 5458   ins_cost(100);
 5459   format %{ "movq    $dst, $src\t# long stk" %}
 5460   ins_encode %{
 5461     __ movq($dst$$Address, $src$$Register);
 5462   %}
 5463   ins_pipe(ialu_mem_reg);
 5464 %}
 5465 
 5466 instruct storeSSP(stackSlotP dst, rRegP src)
 5467 %{
 5468   match(Set dst src);
 5469 
 5470   ins_cost(100);
 5471   format %{ "movq    $dst, $src\t# ptr stk" %}
 5472   ins_encode %{
 5473     __ movq($dst$$Address, $src$$Register);
 5474   %}
 5475   ins_pipe(ialu_mem_reg);
 5476 %}
 5477 
 5478 instruct storeSSF(stackSlotF dst, regF src)
 5479 %{
 5480   match(Set dst src);
 5481 
 5482   ins_cost(95); // XXX
 5483   format %{ "movss   $dst, $src\t# float stk" %}
 5484   ins_encode %{
 5485     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5486   %}
 5487   ins_pipe(pipe_slow); // XXX
 5488 %}
 5489 
 5490 instruct storeSSD(stackSlotD dst, regD src)
 5491 %{
 5492   match(Set dst src);
 5493 
 5494   ins_cost(95); // XXX
 5495   format %{ "movsd   $dst, $src\t# double stk" %}
 5496   ins_encode %{
 5497     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5498   %}
 5499   ins_pipe(pipe_slow); // XXX
 5500 %}
 5501 
 5502 instruct cacheWB(indirect addr)
 5503 %{
 5504   predicate(VM_Version::supports_data_cache_line_flush());
 5505   match(CacheWB addr);
 5506 
 5507   ins_cost(100);
 5508   format %{"cache wb $addr" %}
 5509   ins_encode %{
 5510     assert($addr->index_position() < 0, "should be");
 5511     assert($addr$$disp == 0, "should be");
 5512     __ cache_wb(Address($addr$$base$$Register, 0));
 5513   %}
 5514   ins_pipe(pipe_slow); // XXX
 5515 %}
 5516 
 5517 instruct cacheWBPreSync()
 5518 %{
 5519   predicate(VM_Version::supports_data_cache_line_flush());
 5520   match(CacheWBPreSync);
 5521 
 5522   ins_cost(100);
 5523   format %{"cache wb presync" %}
 5524   ins_encode %{
 5525     __ cache_wbsync(true);
 5526   %}
 5527   ins_pipe(pipe_slow); // XXX
 5528 %}
 5529 
 5530 instruct cacheWBPostSync()
 5531 %{
 5532   predicate(VM_Version::supports_data_cache_line_flush());
 5533   match(CacheWBPostSync);
 5534 
 5535   ins_cost(100);
 5536   format %{"cache wb postsync" %}
 5537   ins_encode %{
 5538     __ cache_wbsync(false);
 5539   %}
 5540   ins_pipe(pipe_slow); // XXX
 5541 %}
 5542 
 5543 //----------BSWAP Instructions-------------------------------------------------
 5544 instruct bytes_reverse_int(rRegI dst) %{
 5545   match(Set dst (ReverseBytesI dst));
 5546 
 5547   format %{ "bswapl  $dst" %}
 5548   ins_encode %{
 5549     __ bswapl($dst$$Register);
 5550   %}
 5551   ins_pipe( ialu_reg );
 5552 %}
 5553 
 5554 instruct bytes_reverse_long(rRegL dst) %{
 5555   match(Set dst (ReverseBytesL dst));
 5556 
 5557   format %{ "bswapq  $dst" %}
 5558   ins_encode %{
 5559     __ bswapq($dst$$Register);
 5560   %}
 5561   ins_pipe( ialu_reg);
 5562 %}
 5563 
 5564 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5565   match(Set dst (ReverseBytesUS dst));
 5566   effect(KILL cr);
 5567 
 5568   format %{ "bswapl  $dst\n\t"
 5569             "shrl    $dst,16\n\t" %}
 5570   ins_encode %{
 5571     __ bswapl($dst$$Register);
 5572     __ shrl($dst$$Register, 16);
 5573   %}
 5574   ins_pipe( ialu_reg );
 5575 %}
 5576 
 5577 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5578   match(Set dst (ReverseBytesS dst));
 5579   effect(KILL cr);
 5580 
 5581   format %{ "bswapl  $dst\n\t"
 5582             "sar     $dst,16\n\t" %}
 5583   ins_encode %{
 5584     __ bswapl($dst$$Register);
 5585     __ sarl($dst$$Register, 16);
 5586   %}
 5587   ins_pipe( ialu_reg );
 5588 %}
 5589 
 5590 //---------- Zeros Count Instructions ------------------------------------------
 5591 
 5592 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5593   predicate(UseCountLeadingZerosInstruction);
 5594   match(Set dst (CountLeadingZerosI src));
 5595   effect(KILL cr);
 5596 
 5597   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5598   ins_encode %{
 5599     __ lzcntl($dst$$Register, $src$$Register);
 5600   %}
 5601   ins_pipe(ialu_reg);
 5602 %}
 5603 
 5604 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5605   predicate(UseCountLeadingZerosInstruction);
 5606   match(Set dst (CountLeadingZerosI (LoadI src)));
 5607   effect(KILL cr);
 5608   ins_cost(175);
 5609   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5610   ins_encode %{
 5611     __ lzcntl($dst$$Register, $src$$Address);
 5612   %}
 5613   ins_pipe(ialu_reg_mem);
 5614 %}
 5615 
 5616 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5617   predicate(!UseCountLeadingZerosInstruction);
 5618   match(Set dst (CountLeadingZerosI src));
 5619   effect(KILL cr);
 5620 
 5621   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5622             "jnz     skip\n\t"
 5623             "movl    $dst, -1\n"
 5624       "skip:\n\t"
 5625             "negl    $dst\n\t"
 5626             "addl    $dst, 31" %}
 5627   ins_encode %{
 5628     Register Rdst = $dst$$Register;
 5629     Register Rsrc = $src$$Register;
 5630     Label skip;
 5631     __ bsrl(Rdst, Rsrc);
 5632     __ jccb(Assembler::notZero, skip);
 5633     __ movl(Rdst, -1);
 5634     __ bind(skip);
 5635     __ negl(Rdst);
 5636     __ addl(Rdst, BitsPerInt - 1);
 5637   %}
 5638   ins_pipe(ialu_reg);
 5639 %}
 5640 
 5641 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5642   predicate(UseCountLeadingZerosInstruction);
 5643   match(Set dst (CountLeadingZerosL src));
 5644   effect(KILL cr);
 5645 
 5646   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5647   ins_encode %{
 5648     __ lzcntq($dst$$Register, $src$$Register);
 5649   %}
 5650   ins_pipe(ialu_reg);
 5651 %}
 5652 
 5653 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5654   predicate(UseCountLeadingZerosInstruction);
 5655   match(Set dst (CountLeadingZerosL (LoadL src)));
 5656   effect(KILL cr);
 5657   ins_cost(175);
 5658   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5659   ins_encode %{
 5660     __ lzcntq($dst$$Register, $src$$Address);
 5661   %}
 5662   ins_pipe(ialu_reg_mem);
 5663 %}
 5664 
 5665 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5666   predicate(!UseCountLeadingZerosInstruction);
 5667   match(Set dst (CountLeadingZerosL src));
 5668   effect(KILL cr);
 5669 
 5670   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5671             "jnz     skip\n\t"
 5672             "movl    $dst, -1\n"
 5673       "skip:\n\t"
 5674             "negl    $dst\n\t"
 5675             "addl    $dst, 63" %}
 5676   ins_encode %{
 5677     Register Rdst = $dst$$Register;
 5678     Register Rsrc = $src$$Register;
 5679     Label skip;
 5680     __ bsrq(Rdst, Rsrc);
 5681     __ jccb(Assembler::notZero, skip);
 5682     __ movl(Rdst, -1);
 5683     __ bind(skip);
 5684     __ negl(Rdst);
 5685     __ addl(Rdst, BitsPerLong - 1);
 5686   %}
 5687   ins_pipe(ialu_reg);
 5688 %}
 5689 
 5690 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5691   predicate(UseCountTrailingZerosInstruction);
 5692   match(Set dst (CountTrailingZerosI src));
 5693   effect(KILL cr);
 5694 
 5695   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5696   ins_encode %{
 5697     __ tzcntl($dst$$Register, $src$$Register);
 5698   %}
 5699   ins_pipe(ialu_reg);
 5700 %}
 5701 
 5702 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5703   predicate(UseCountTrailingZerosInstruction);
 5704   match(Set dst (CountTrailingZerosI (LoadI src)));
 5705   effect(KILL cr);
 5706   ins_cost(175);
 5707   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5708   ins_encode %{
 5709     __ tzcntl($dst$$Register, $src$$Address);
 5710   %}
 5711   ins_pipe(ialu_reg_mem);
 5712 %}
 5713 
 5714 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5715   predicate(!UseCountTrailingZerosInstruction);
 5716   match(Set dst (CountTrailingZerosI src));
 5717   effect(KILL cr);
 5718 
 5719   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5720             "jnz     done\n\t"
 5721             "movl    $dst, 32\n"
 5722       "done:" %}
 5723   ins_encode %{
 5724     Register Rdst = $dst$$Register;
 5725     Label done;
 5726     __ bsfl(Rdst, $src$$Register);
 5727     __ jccb(Assembler::notZero, done);
 5728     __ movl(Rdst, BitsPerInt);
 5729     __ bind(done);
 5730   %}
 5731   ins_pipe(ialu_reg);
 5732 %}
 5733 
 5734 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5735   predicate(UseCountTrailingZerosInstruction);
 5736   match(Set dst (CountTrailingZerosL src));
 5737   effect(KILL cr);
 5738 
 5739   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5740   ins_encode %{
 5741     __ tzcntq($dst$$Register, $src$$Register);
 5742   %}
 5743   ins_pipe(ialu_reg);
 5744 %}
 5745 
 5746 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5747   predicate(UseCountTrailingZerosInstruction);
 5748   match(Set dst (CountTrailingZerosL (LoadL src)));
 5749   effect(KILL cr);
 5750   ins_cost(175);
 5751   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5752   ins_encode %{
 5753     __ tzcntq($dst$$Register, $src$$Address);
 5754   %}
 5755   ins_pipe(ialu_reg_mem);
 5756 %}
 5757 
 5758 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5759   predicate(!UseCountTrailingZerosInstruction);
 5760   match(Set dst (CountTrailingZerosL src));
 5761   effect(KILL cr);
 5762 
 5763   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5764             "jnz     done\n\t"
 5765             "movl    $dst, 64\n"
 5766       "done:" %}
 5767   ins_encode %{
 5768     Register Rdst = $dst$$Register;
 5769     Label done;
 5770     __ bsfq(Rdst, $src$$Register);
 5771     __ jccb(Assembler::notZero, done);
 5772     __ movl(Rdst, BitsPerLong);
 5773     __ bind(done);
 5774   %}
 5775   ins_pipe(ialu_reg);
 5776 %}
 5777 
 5778 //--------------- Reverse Operation Instructions ----------------
 5779 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5780   predicate(!VM_Version::supports_gfni());
 5781   match(Set dst (ReverseI src));
 5782   effect(TEMP dst, TEMP rtmp, KILL cr);
 5783   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5784   ins_encode %{
 5785     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5786   %}
 5787   ins_pipe( ialu_reg );
 5788 %}
 5789 
 5790 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5791   predicate(VM_Version::supports_gfni());
 5792   match(Set dst (ReverseI src));
 5793   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5794   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5795   ins_encode %{
 5796     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5797   %}
 5798   ins_pipe( ialu_reg );
 5799 %}
 5800 
 5801 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5802   predicate(!VM_Version::supports_gfni());
 5803   match(Set dst (ReverseL src));
 5804   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5805   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5806   ins_encode %{
 5807     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5808   %}
 5809   ins_pipe( ialu_reg );
 5810 %}
 5811 
 5812 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5813   predicate(VM_Version::supports_gfni());
 5814   match(Set dst (ReverseL src));
 5815   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5816   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5817   ins_encode %{
 5818     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5819   %}
 5820   ins_pipe( ialu_reg );
 5821 %}
 5822 
 5823 //---------- Population Count Instructions -------------------------------------
 5824 
 5825 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5826   predicate(UsePopCountInstruction);
 5827   match(Set dst (PopCountI src));
 5828   effect(KILL cr);
 5829 
 5830   format %{ "popcnt  $dst, $src" %}
 5831   ins_encode %{
 5832     __ popcntl($dst$$Register, $src$$Register);
 5833   %}
 5834   ins_pipe(ialu_reg);
 5835 %}
 5836 
 5837 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5838   predicate(UsePopCountInstruction);
 5839   match(Set dst (PopCountI (LoadI mem)));
 5840   effect(KILL cr);
 5841 
 5842   format %{ "popcnt  $dst, $mem" %}
 5843   ins_encode %{
 5844     __ popcntl($dst$$Register, $mem$$Address);
 5845   %}
 5846   ins_pipe(ialu_reg);
 5847 %}
 5848 
 5849 // Note: Long.bitCount(long) returns an int.
 5850 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5851   predicate(UsePopCountInstruction);
 5852   match(Set dst (PopCountL src));
 5853   effect(KILL cr);
 5854 
 5855   format %{ "popcnt  $dst, $src" %}
 5856   ins_encode %{
 5857     __ popcntq($dst$$Register, $src$$Register);
 5858   %}
 5859   ins_pipe(ialu_reg);
 5860 %}
 5861 
 5862 // Note: Long.bitCount(long) returns an int.
 5863 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5864   predicate(UsePopCountInstruction);
 5865   match(Set dst (PopCountL (LoadL mem)));
 5866   effect(KILL cr);
 5867 
 5868   format %{ "popcnt  $dst, $mem" %}
 5869   ins_encode %{
 5870     __ popcntq($dst$$Register, $mem$$Address);
 5871   %}
 5872   ins_pipe(ialu_reg);
 5873 %}
 5874 
 5875 
 5876 //----------MemBar Instructions-----------------------------------------------
 5877 // Memory barrier flavors
 5878 
 5879 instruct membar_acquire()
 5880 %{
 5881   match(MemBarAcquire);
 5882   match(LoadFence);
 5883   ins_cost(0);
 5884 
 5885   size(0);
 5886   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5887   ins_encode();
 5888   ins_pipe(empty);
 5889 %}
 5890 
 5891 instruct membar_acquire_lock()
 5892 %{
 5893   match(MemBarAcquireLock);
 5894   ins_cost(0);
 5895 
 5896   size(0);
 5897   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5898   ins_encode();
 5899   ins_pipe(empty);
 5900 %}
 5901 
 5902 instruct membar_release()
 5903 %{
 5904   match(MemBarRelease);
 5905   match(StoreFence);
 5906   ins_cost(0);
 5907 
 5908   size(0);
 5909   format %{ "MEMBAR-release ! (empty encoding)" %}
 5910   ins_encode();
 5911   ins_pipe(empty);
 5912 %}
 5913 
 5914 instruct membar_release_lock()
 5915 %{
 5916   match(MemBarReleaseLock);
 5917   ins_cost(0);
 5918 
 5919   size(0);
 5920   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5921   ins_encode();
 5922   ins_pipe(empty);
 5923 %}
 5924 
 5925 instruct membar_volatile(rFlagsReg cr) %{
 5926   match(MemBarVolatile);
 5927   effect(KILL cr);
 5928   ins_cost(400);
 5929 
 5930   format %{
 5931     $$template
 5932     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5933   %}
 5934   ins_encode %{
 5935     __ membar(Assembler::StoreLoad);
 5936   %}
 5937   ins_pipe(pipe_slow);
 5938 %}
 5939 
 5940 instruct unnecessary_membar_volatile()
 5941 %{
 5942   match(MemBarVolatile);
 5943   predicate(Matcher::post_store_load_barrier(n));
 5944   ins_cost(0);
 5945 
 5946   size(0);
 5947   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5948   ins_encode();
 5949   ins_pipe(empty);
 5950 %}
 5951 
 5952 instruct membar_storestore() %{
 5953   match(MemBarStoreStore);
 5954   match(StoreStoreFence);
 5955   ins_cost(0);
 5956 
 5957   size(0);
 5958   format %{ "MEMBAR-storestore (empty encoding)" %}
 5959   ins_encode( );
 5960   ins_pipe(empty);
 5961 %}
 5962 
 5963 //----------Move Instructions--------------------------------------------------
 5964 
 5965 instruct castX2P(rRegP dst, rRegL src)
 5966 %{
 5967   match(Set dst (CastX2P src));
 5968 
 5969   format %{ "movq    $dst, $src\t# long->ptr" %}
 5970   ins_encode %{
 5971     if ($dst$$reg != $src$$reg) {
 5972       __ movptr($dst$$Register, $src$$Register);
 5973     }
 5974   %}
 5975   ins_pipe(ialu_reg_reg); // XXX
 5976 %}
 5977 
 5978 instruct castP2X(rRegL dst, rRegP src)
 5979 %{
 5980   match(Set dst (CastP2X src));
 5981 
 5982   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5983   ins_encode %{
 5984     if ($dst$$reg != $src$$reg) {
 5985       __ movptr($dst$$Register, $src$$Register);
 5986     }
 5987   %}
 5988   ins_pipe(ialu_reg_reg); // XXX
 5989 %}
 5990 
 5991 // Convert oop into int for vectors alignment masking
 5992 instruct convP2I(rRegI dst, rRegP src)
 5993 %{
 5994   match(Set dst (ConvL2I (CastP2X src)));
 5995 
 5996   format %{ "movl    $dst, $src\t# ptr -> int" %}
 5997   ins_encode %{
 5998     __ movl($dst$$Register, $src$$Register);
 5999   %}
 6000   ins_pipe(ialu_reg_reg); // XXX
 6001 %}
 6002 
 6003 // Convert compressed oop into int for vectors alignment masking
 6004 // in case of 32bit oops (heap < 4Gb).
 6005 instruct convN2I(rRegI dst, rRegN src)
 6006 %{
 6007   predicate(CompressedOops::shift() == 0);
 6008   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6009 
 6010   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6011   ins_encode %{
 6012     __ movl($dst$$Register, $src$$Register);
 6013   %}
 6014   ins_pipe(ialu_reg_reg); // XXX
 6015 %}
 6016 
 6017 // Convert oop pointer into compressed form
 6018 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6019   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6020   match(Set dst (EncodeP src));
 6021   effect(KILL cr);
 6022   format %{ "encode_heap_oop $dst,$src" %}
 6023   ins_encode %{
 6024     Register s = $src$$Register;
 6025     Register d = $dst$$Register;
 6026     if (s != d) {
 6027       __ movq(d, s);
 6028     }
 6029     __ encode_heap_oop(d);
 6030   %}
 6031   ins_pipe(ialu_reg_long);
 6032 %}
 6033 
 6034 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6035   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6036   match(Set dst (EncodeP src));
 6037   effect(KILL cr);
 6038   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6039   ins_encode %{
 6040     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6041   %}
 6042   ins_pipe(ialu_reg_long);
 6043 %}
 6044 
 6045 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6046   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6047             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6048   match(Set dst (DecodeN src));
 6049   effect(KILL cr);
 6050   format %{ "decode_heap_oop $dst,$src" %}
 6051   ins_encode %{
 6052     Register s = $src$$Register;
 6053     Register d = $dst$$Register;
 6054     if (s != d) {
 6055       __ movq(d, s);
 6056     }
 6057     __ decode_heap_oop(d);
 6058   %}
 6059   ins_pipe(ialu_reg_long);
 6060 %}
 6061 
 6062 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6063   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6064             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6065   match(Set dst (DecodeN src));
 6066   effect(KILL cr);
 6067   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6068   ins_encode %{
 6069     Register s = $src$$Register;
 6070     Register d = $dst$$Register;
 6071     if (s != d) {
 6072       __ decode_heap_oop_not_null(d, s);
 6073     } else {
 6074       __ decode_heap_oop_not_null(d);
 6075     }
 6076   %}
 6077   ins_pipe(ialu_reg_long);
 6078 %}
 6079 
 6080 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6081   match(Set dst (EncodePKlass src));
 6082   effect(TEMP dst, KILL cr);
 6083   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6084   ins_encode %{
 6085     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6086   %}
 6087   ins_pipe(ialu_reg_long);
 6088 %}
 6089 
 6090 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6091   match(Set dst (DecodeNKlass src));
 6092   effect(TEMP dst, KILL cr);
 6093   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6094   ins_encode %{
 6095     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6096   %}
 6097   ins_pipe(ialu_reg_long);
 6098 %}
 6099 
 6100 //----------Conditional Move---------------------------------------------------
 6101 // Jump
 6102 // dummy instruction for generating temp registers
 6103 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6104   match(Jump (LShiftL switch_val shift));
 6105   ins_cost(350);
 6106   predicate(false);
 6107   effect(TEMP dest);
 6108 
 6109   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6110             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6111   ins_encode %{
 6112     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6113     // to do that and the compiler is using that register as one it can allocate.
 6114     // So we build it all by hand.
 6115     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6116     // ArrayAddress dispatch(table, index);
 6117     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6118     __ lea($dest$$Register, $constantaddress);
 6119     __ jmp(dispatch);
 6120   %}
 6121   ins_pipe(pipe_jmp);
 6122 %}
 6123 
 6124 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6125   match(Jump (AddL (LShiftL switch_val shift) offset));
 6126   ins_cost(350);
 6127   effect(TEMP dest);
 6128 
 6129   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6130             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6131   ins_encode %{
 6132     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6133     // to do that and the compiler is using that register as one it can allocate.
 6134     // So we build it all by hand.
 6135     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6136     // ArrayAddress dispatch(table, index);
 6137     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6138     __ lea($dest$$Register, $constantaddress);
 6139     __ jmp(dispatch);
 6140   %}
 6141   ins_pipe(pipe_jmp);
 6142 %}
 6143 
 6144 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6145   match(Jump switch_val);
 6146   ins_cost(350);
 6147   effect(TEMP dest);
 6148 
 6149   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6150             "jmp     [$dest + $switch_val]\n\t" %}
 6151   ins_encode %{
 6152     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6153     // to do that and the compiler is using that register as one it can allocate.
 6154     // So we build it all by hand.
 6155     // Address index(noreg, switch_reg, Address::times_1);
 6156     // ArrayAddress dispatch(table, index);
 6157     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6158     __ lea($dest$$Register, $constantaddress);
 6159     __ jmp(dispatch);
 6160   %}
 6161   ins_pipe(pipe_jmp);
 6162 %}
 6163 
 6164 // Conditional move
 6165 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6166 %{
 6167   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6168   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6169 
 6170   ins_cost(100); // XXX
 6171   format %{ "setbn$cop $dst\t# signed, int" %}
 6172   ins_encode %{
 6173     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6174     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6175   %}
 6176   ins_pipe(ialu_reg);
 6177 %}
 6178 
 6179 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6180 %{
 6181   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6182 
 6183   ins_cost(200); // XXX
 6184   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6185   ins_encode %{
 6186     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6187   %}
 6188   ins_pipe(pipe_cmov_reg);
 6189 %}
 6190 
 6191 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6192 %{
 6193   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6194   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6195 
 6196   ins_cost(100); // XXX
 6197   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6198   ins_encode %{
 6199     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6200     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6201   %}
 6202   ins_pipe(ialu_reg);
 6203 %}
 6204 
 6205 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6206   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6207 
 6208   ins_cost(200); // XXX
 6209   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6210   ins_encode %{
 6211     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6212   %}
 6213   ins_pipe(pipe_cmov_reg);
 6214 %}
 6215 
 6216 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6217 %{
 6218   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6219   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6220 
 6221   ins_cost(100); // XXX
 6222   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6223   ins_encode %{
 6224     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6225     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6226   %}
 6227   ins_pipe(ialu_reg);
 6228 %}
 6229 
 6230 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6231   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6232   ins_cost(200);
 6233   expand %{
 6234     cmovI_regU(cop, cr, dst, src);
 6235   %}
 6236 %}
 6237 
 6238 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6239   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6240   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6241 
 6242   ins_cost(200); // XXX
 6243   format %{ "cmovpl  $dst, $src\n\t"
 6244             "cmovnel $dst, $src" %}
 6245   ins_encode %{
 6246     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6247     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6248   %}
 6249   ins_pipe(pipe_cmov_reg);
 6250 %}
 6251 
 6252 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6253 // inputs of the CMove
 6254 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6255   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6256   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6257 
 6258   ins_cost(200); // XXX
 6259   format %{ "cmovpl  $dst, $src\n\t"
 6260             "cmovnel $dst, $src" %}
 6261   ins_encode %{
 6262     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6263     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6264   %}
 6265   ins_pipe(pipe_cmov_reg);
 6266 %}
 6267 
 6268 // Conditional move
 6269 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6270   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6271 
 6272   ins_cost(250); // XXX
 6273   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6274   ins_encode %{
 6275     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6276   %}
 6277   ins_pipe(pipe_cmov_mem);
 6278 %}
 6279 
 6280 // Conditional move
 6281 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6282 %{
 6283   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6284 
 6285   ins_cost(250); // XXX
 6286   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6287   ins_encode %{
 6288     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6289   %}
 6290   ins_pipe(pipe_cmov_mem);
 6291 %}
 6292 
 6293 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6294   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6295   ins_cost(250);
 6296   expand %{
 6297     cmovI_memU(cop, cr, dst, src);
 6298   %}
 6299 %}
 6300 
 6301 // Conditional move
 6302 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6303 %{
 6304   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6305 
 6306   ins_cost(200); // XXX
 6307   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6308   ins_encode %{
 6309     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6310   %}
 6311   ins_pipe(pipe_cmov_reg);
 6312 %}
 6313 
 6314 // Conditional move
 6315 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6316 %{
 6317   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6318 
 6319   ins_cost(200); // XXX
 6320   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6321   ins_encode %{
 6322     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6323   %}
 6324   ins_pipe(pipe_cmov_reg);
 6325 %}
 6326 
 6327 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6328   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6329   ins_cost(200);
 6330   expand %{
 6331     cmovN_regU(cop, cr, dst, src);
 6332   %}
 6333 %}
 6334 
 6335 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6336   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6337   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6338 
 6339   ins_cost(200); // XXX
 6340   format %{ "cmovpl  $dst, $src\n\t"
 6341             "cmovnel $dst, $src" %}
 6342   ins_encode %{
 6343     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6344     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6345   %}
 6346   ins_pipe(pipe_cmov_reg);
 6347 %}
 6348 
 6349 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6350 // inputs of the CMove
 6351 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6352   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6353   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6354 
 6355   ins_cost(200); // XXX
 6356   format %{ "cmovpl  $dst, $src\n\t"
 6357             "cmovnel $dst, $src" %}
 6358   ins_encode %{
 6359     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6360     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6361   %}
 6362   ins_pipe(pipe_cmov_reg);
 6363 %}
 6364 
 6365 // Conditional move
 6366 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6367 %{
 6368   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6369 
 6370   ins_cost(200); // XXX
 6371   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6372   ins_encode %{
 6373     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6374   %}
 6375   ins_pipe(pipe_cmov_reg);  // XXX
 6376 %}
 6377 
 6378 // Conditional move
 6379 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6380 %{
 6381   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6382 
 6383   ins_cost(200); // XXX
 6384   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6385   ins_encode %{
 6386     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6387   %}
 6388   ins_pipe(pipe_cmov_reg); // XXX
 6389 %}
 6390 
 6391 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6392   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6393   ins_cost(200);
 6394   expand %{
 6395     cmovP_regU(cop, cr, dst, src);
 6396   %}
 6397 %}
 6398 
 6399 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6400   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6401   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6402 
 6403   ins_cost(200); // XXX
 6404   format %{ "cmovpq  $dst, $src\n\t"
 6405             "cmovneq $dst, $src" %}
 6406   ins_encode %{
 6407     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6408     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6409   %}
 6410   ins_pipe(pipe_cmov_reg);
 6411 %}
 6412 
 6413 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6414 // inputs of the CMove
 6415 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6416   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6417   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6418 
 6419   ins_cost(200); // XXX
 6420   format %{ "cmovpq  $dst, $src\n\t"
 6421             "cmovneq $dst, $src" %}
 6422   ins_encode %{
 6423     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6424     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6425   %}
 6426   ins_pipe(pipe_cmov_reg);
 6427 %}
 6428 
 6429 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6430 %{
 6431   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6432   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6433 
 6434   ins_cost(100); // XXX
 6435   format %{ "setbn$cop $dst\t# signed, long" %}
 6436   ins_encode %{
 6437     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6438     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6439   %}
 6440   ins_pipe(ialu_reg);
 6441 %}
 6442 
 6443 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6444 %{
 6445   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6446 
 6447   ins_cost(200); // XXX
 6448   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6449   ins_encode %{
 6450     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6451   %}
 6452   ins_pipe(pipe_cmov_reg);  // XXX
 6453 %}
 6454 
 6455 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6456 %{
 6457   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6458 
 6459   ins_cost(200); // XXX
 6460   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6461   ins_encode %{
 6462     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6463   %}
 6464   ins_pipe(pipe_cmov_mem);  // XXX
 6465 %}
 6466 
 6467 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6468 %{
 6469   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6470   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6471 
 6472   ins_cost(100); // XXX
 6473   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6474   ins_encode %{
 6475     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6476     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6477   %}
 6478   ins_pipe(ialu_reg);
 6479 %}
 6480 
 6481 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6482 %{
 6483   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6484 
 6485   ins_cost(200); // XXX
 6486   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6487   ins_encode %{
 6488     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6489   %}
 6490   ins_pipe(pipe_cmov_reg); // XXX
 6491 %}
 6492 
 6493 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6494 %{
 6495   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6496   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6497 
 6498   ins_cost(100); // XXX
 6499   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6500   ins_encode %{
 6501     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6502     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6503   %}
 6504   ins_pipe(ialu_reg);
 6505 %}
 6506 
 6507 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6508   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6509   ins_cost(200);
 6510   expand %{
 6511     cmovL_regU(cop, cr, dst, src);
 6512   %}
 6513 %}
 6514 
 6515 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6516   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6517   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6518 
 6519   ins_cost(200); // XXX
 6520   format %{ "cmovpq  $dst, $src\n\t"
 6521             "cmovneq $dst, $src" %}
 6522   ins_encode %{
 6523     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6524     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6525   %}
 6526   ins_pipe(pipe_cmov_reg);
 6527 %}
 6528 
 6529 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6530 // inputs of the CMove
 6531 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6532   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6533   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6534 
 6535   ins_cost(200); // XXX
 6536   format %{ "cmovpq  $dst, $src\n\t"
 6537             "cmovneq $dst, $src" %}
 6538   ins_encode %{
 6539     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6540     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6541   %}
 6542   ins_pipe(pipe_cmov_reg);
 6543 %}
 6544 
 6545 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6546 %{
 6547   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6548 
 6549   ins_cost(200); // XXX
 6550   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6551   ins_encode %{
 6552     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6553   %}
 6554   ins_pipe(pipe_cmov_mem); // XXX
 6555 %}
 6556 
 6557 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6558   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6559   ins_cost(200);
 6560   expand %{
 6561     cmovL_memU(cop, cr, dst, src);
 6562   %}
 6563 %}
 6564 
 6565 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6566 %{
 6567   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6568 
 6569   ins_cost(200); // XXX
 6570   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6571             "movss     $dst, $src\n"
 6572     "skip:" %}
 6573   ins_encode %{
 6574     Label Lskip;
 6575     // Invert sense of branch from sense of CMOV
 6576     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6577     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6578     __ bind(Lskip);
 6579   %}
 6580   ins_pipe(pipe_slow);
 6581 %}
 6582 
 6583 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6584 %{
 6585   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6586 
 6587   ins_cost(200); // XXX
 6588   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6589             "movss     $dst, $src\n"
 6590     "skip:" %}
 6591   ins_encode %{
 6592     Label Lskip;
 6593     // Invert sense of branch from sense of CMOV
 6594     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6595     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6596     __ bind(Lskip);
 6597   %}
 6598   ins_pipe(pipe_slow);
 6599 %}
 6600 
 6601 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6602   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6603   ins_cost(200);
 6604   expand %{
 6605     cmovF_regU(cop, cr, dst, src);
 6606   %}
 6607 %}
 6608 
 6609 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6610 %{
 6611   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6612 
 6613   ins_cost(200); // XXX
 6614   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6615             "movsd     $dst, $src\n"
 6616     "skip:" %}
 6617   ins_encode %{
 6618     Label Lskip;
 6619     // Invert sense of branch from sense of CMOV
 6620     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6621     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6622     __ bind(Lskip);
 6623   %}
 6624   ins_pipe(pipe_slow);
 6625 %}
 6626 
 6627 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6628 %{
 6629   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6630 
 6631   ins_cost(200); // XXX
 6632   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6633             "movsd     $dst, $src\n"
 6634     "skip:" %}
 6635   ins_encode %{
 6636     Label Lskip;
 6637     // Invert sense of branch from sense of CMOV
 6638     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6639     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6640     __ bind(Lskip);
 6641   %}
 6642   ins_pipe(pipe_slow);
 6643 %}
 6644 
 6645 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6646   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6647   ins_cost(200);
 6648   expand %{
 6649     cmovD_regU(cop, cr, dst, src);
 6650   %}
 6651 %}
 6652 
 6653 //----------Arithmetic Instructions--------------------------------------------
 6654 //----------Addition Instructions----------------------------------------------
 6655 
 6656 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6657 %{
 6658   match(Set dst (AddI dst src));
 6659   effect(KILL cr);
 6660   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6661   format %{ "addl    $dst, $src\t# int" %}
 6662   ins_encode %{
 6663     __ addl($dst$$Register, $src$$Register);
 6664   %}
 6665   ins_pipe(ialu_reg_reg);
 6666 %}
 6667 
 6668 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 6669 %{
 6670   match(Set dst (AddI dst src));
 6671   effect(KILL cr);
 6672   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6673 
 6674   format %{ "addl    $dst, $src\t# int" %}
 6675   ins_encode %{
 6676     __ addl($dst$$Register, $src$$constant);
 6677   %}
 6678   ins_pipe( ialu_reg );
 6679 %}
 6680 
 6681 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 6682 %{
 6683   match(Set dst (AddI dst (LoadI src)));
 6684   effect(KILL cr);
 6685   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6686 
 6687   ins_cost(150); // XXX
 6688   format %{ "addl    $dst, $src\t# int" %}
 6689   ins_encode %{
 6690     __ addl($dst$$Register, $src$$Address);
 6691   %}
 6692   ins_pipe(ialu_reg_mem);
 6693 %}
 6694 
 6695 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 6696 %{
 6697   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6698   effect(KILL cr);
 6699   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6700 
 6701   ins_cost(150); // XXX
 6702   format %{ "addl    $dst, $src\t# int" %}
 6703   ins_encode %{
 6704     __ addl($dst$$Address, $src$$Register);
 6705   %}
 6706   ins_pipe(ialu_mem_reg);
 6707 %}
 6708 
 6709 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 6710 %{
 6711   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6712   effect(KILL cr);
 6713   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6714 
 6715 
 6716   ins_cost(125); // XXX
 6717   format %{ "addl    $dst, $src\t# int" %}
 6718   ins_encode %{
 6719     __ addl($dst$$Address, $src$$constant);
 6720   %}
 6721   ins_pipe(ialu_mem_imm);
 6722 %}
 6723 
 6724 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 6725 %{
 6726   predicate(UseIncDec);
 6727   match(Set dst (AddI dst src));
 6728   effect(KILL cr);
 6729 
 6730   format %{ "incl    $dst\t# int" %}
 6731   ins_encode %{
 6732     __ incrementl($dst$$Register);
 6733   %}
 6734   ins_pipe(ialu_reg);
 6735 %}
 6736 
 6737 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 6738 %{
 6739   predicate(UseIncDec);
 6740   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6741   effect(KILL cr);
 6742 
 6743   ins_cost(125); // XXX
 6744   format %{ "incl    $dst\t# int" %}
 6745   ins_encode %{
 6746     __ incrementl($dst$$Address);
 6747   %}
 6748   ins_pipe(ialu_mem_imm);
 6749 %}
 6750 
 6751 // XXX why does that use AddI
 6752 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 6753 %{
 6754   predicate(UseIncDec);
 6755   match(Set dst (AddI dst src));
 6756   effect(KILL cr);
 6757 
 6758   format %{ "decl    $dst\t# int" %}
 6759   ins_encode %{
 6760     __ decrementl($dst$$Register);
 6761   %}
 6762   ins_pipe(ialu_reg);
 6763 %}
 6764 
 6765 // XXX why does that use AddI
 6766 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 6767 %{
 6768   predicate(UseIncDec);
 6769   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6770   effect(KILL cr);
 6771 
 6772   ins_cost(125); // XXX
 6773   format %{ "decl    $dst\t# int" %}
 6774   ins_encode %{
 6775     __ decrementl($dst$$Address);
 6776   %}
 6777   ins_pipe(ialu_mem_imm);
 6778 %}
 6779 
 6780 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 6781 %{
 6782   predicate(VM_Version::supports_fast_2op_lea());
 6783   match(Set dst (AddI (LShiftI index scale) disp));
 6784 
 6785   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 6786   ins_encode %{
 6787     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6788     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6789   %}
 6790   ins_pipe(ialu_reg_reg);
 6791 %}
 6792 
 6793 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 6794 %{
 6795   predicate(VM_Version::supports_fast_3op_lea());
 6796   match(Set dst (AddI (AddI base index) disp));
 6797 
 6798   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 6799   ins_encode %{
 6800     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6801   %}
 6802   ins_pipe(ialu_reg_reg);
 6803 %}
 6804 
 6805 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 6806 %{
 6807   predicate(VM_Version::supports_fast_2op_lea());
 6808   match(Set dst (AddI base (LShiftI index scale)));
 6809 
 6810   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 6811   ins_encode %{
 6812     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6813     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6814   %}
 6815   ins_pipe(ialu_reg_reg);
 6816 %}
 6817 
 6818 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 6819 %{
 6820   predicate(VM_Version::supports_fast_3op_lea());
 6821   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 6822 
 6823   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 6824   ins_encode %{
 6825     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6826     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6827   %}
 6828   ins_pipe(ialu_reg_reg);
 6829 %}
 6830 
 6831 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 6832 %{
 6833   match(Set dst (AddL dst src));
 6834   effect(KILL cr);
 6835   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6836 
 6837   format %{ "addq    $dst, $src\t# long" %}
 6838   ins_encode %{
 6839     __ addq($dst$$Register, $src$$Register);
 6840   %}
 6841   ins_pipe(ialu_reg_reg);
 6842 %}
 6843 
 6844 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 6845 %{
 6846   match(Set dst (AddL dst src));
 6847   effect(KILL cr);
 6848   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6849 
 6850   format %{ "addq    $dst, $src\t# long" %}
 6851   ins_encode %{
 6852     __ addq($dst$$Register, $src$$constant);
 6853   %}
 6854   ins_pipe( ialu_reg );
 6855 %}
 6856 
 6857 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 6858 %{
 6859   match(Set dst (AddL dst (LoadL src)));
 6860   effect(KILL cr);
 6861   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6862 
 6863   ins_cost(150); // XXX
 6864   format %{ "addq    $dst, $src\t# long" %}
 6865   ins_encode %{
 6866     __ addq($dst$$Register, $src$$Address);
 6867   %}
 6868   ins_pipe(ialu_reg_mem);
 6869 %}
 6870 
 6871 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 6872 %{
 6873   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6874   effect(KILL cr);
 6875   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6876 
 6877   ins_cost(150); // XXX
 6878   format %{ "addq    $dst, $src\t# long" %}
 6879   ins_encode %{
 6880     __ addq($dst$$Address, $src$$Register);
 6881   %}
 6882   ins_pipe(ialu_mem_reg);
 6883 %}
 6884 
 6885 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 6886 %{
 6887   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6888   effect(KILL cr);
 6889   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6890 
 6891   ins_cost(125); // XXX
 6892   format %{ "addq    $dst, $src\t# long" %}
 6893   ins_encode %{
 6894     __ addq($dst$$Address, $src$$constant);
 6895   %}
 6896   ins_pipe(ialu_mem_imm);
 6897 %}
 6898 
 6899 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 6900 %{
 6901   predicate(UseIncDec);
 6902   match(Set dst (AddL dst src));
 6903   effect(KILL cr);
 6904 
 6905   format %{ "incq    $dst\t# long" %}
 6906   ins_encode %{
 6907     __ incrementq($dst$$Register);
 6908   %}
 6909   ins_pipe(ialu_reg);
 6910 %}
 6911 
 6912 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 6913 %{
 6914   predicate(UseIncDec);
 6915   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6916   effect(KILL cr);
 6917 
 6918   ins_cost(125); // XXX
 6919   format %{ "incq    $dst\t# long" %}
 6920   ins_encode %{
 6921     __ incrementq($dst$$Address);
 6922   %}
 6923   ins_pipe(ialu_mem_imm);
 6924 %}
 6925 
 6926 // XXX why does that use AddL
 6927 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 6928 %{
 6929   predicate(UseIncDec);
 6930   match(Set dst (AddL dst src));
 6931   effect(KILL cr);
 6932 
 6933   format %{ "decq    $dst\t# long" %}
 6934   ins_encode %{
 6935     __ decrementq($dst$$Register);
 6936   %}
 6937   ins_pipe(ialu_reg);
 6938 %}
 6939 
 6940 // XXX why does that use AddL
 6941 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 6942 %{
 6943   predicate(UseIncDec);
 6944   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6945   effect(KILL cr);
 6946 
 6947   ins_cost(125); // XXX
 6948   format %{ "decq    $dst\t# long" %}
 6949   ins_encode %{
 6950     __ decrementq($dst$$Address);
 6951   %}
 6952   ins_pipe(ialu_mem_imm);
 6953 %}
 6954 
 6955 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 6956 %{
 6957   predicate(VM_Version::supports_fast_2op_lea());
 6958   match(Set dst (AddL (LShiftL index scale) disp));
 6959 
 6960   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 6961   ins_encode %{
 6962     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6963     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6964   %}
 6965   ins_pipe(ialu_reg_reg);
 6966 %}
 6967 
 6968 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 6969 %{
 6970   predicate(VM_Version::supports_fast_3op_lea());
 6971   match(Set dst (AddL (AddL base index) disp));
 6972 
 6973   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 6974   ins_encode %{
 6975     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6976   %}
 6977   ins_pipe(ialu_reg_reg);
 6978 %}
 6979 
 6980 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 6981 %{
 6982   predicate(VM_Version::supports_fast_2op_lea());
 6983   match(Set dst (AddL base (LShiftL index scale)));
 6984 
 6985   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 6986   ins_encode %{
 6987     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6988     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6989   %}
 6990   ins_pipe(ialu_reg_reg);
 6991 %}
 6992 
 6993 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 6994 %{
 6995   predicate(VM_Version::supports_fast_3op_lea());
 6996   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 6997 
 6998   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 6999   ins_encode %{
 7000     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7001     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7002   %}
 7003   ins_pipe(ialu_reg_reg);
 7004 %}
 7005 
 7006 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7007 %{
 7008   match(Set dst (AddP dst src));
 7009   effect(KILL cr);
 7010   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7011 
 7012   format %{ "addq    $dst, $src\t# ptr" %}
 7013   ins_encode %{
 7014     __ addq($dst$$Register, $src$$Register);
 7015   %}
 7016   ins_pipe(ialu_reg_reg);
 7017 %}
 7018 
 7019 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7020 %{
 7021   match(Set dst (AddP dst src));
 7022   effect(KILL cr);
 7023   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7024 
 7025   format %{ "addq    $dst, $src\t# ptr" %}
 7026   ins_encode %{
 7027     __ addq($dst$$Register, $src$$constant);
 7028   %}
 7029   ins_pipe( ialu_reg );
 7030 %}
 7031 
 7032 // XXX addP mem ops ????
 7033 
 7034 instruct checkCastPP(rRegP dst)
 7035 %{
 7036   match(Set dst (CheckCastPP dst));
 7037 
 7038   size(0);
 7039   format %{ "# checkcastPP of $dst" %}
 7040   ins_encode(/* empty encoding */);
 7041   ins_pipe(empty);
 7042 %}
 7043 
 7044 instruct castPP(rRegP dst)
 7045 %{
 7046   match(Set dst (CastPP dst));
 7047 
 7048   size(0);
 7049   format %{ "# castPP of $dst" %}
 7050   ins_encode(/* empty encoding */);
 7051   ins_pipe(empty);
 7052 %}
 7053 
 7054 instruct castII(rRegI dst)
 7055 %{
 7056   match(Set dst (CastII dst));
 7057 
 7058   size(0);
 7059   format %{ "# castII of $dst" %}
 7060   ins_encode(/* empty encoding */);
 7061   ins_cost(0);
 7062   ins_pipe(empty);
 7063 %}
 7064 
 7065 instruct castLL(rRegL dst)
 7066 %{
 7067   match(Set dst (CastLL dst));
 7068 
 7069   size(0);
 7070   format %{ "# castLL of $dst" %}
 7071   ins_encode(/* empty encoding */);
 7072   ins_cost(0);
 7073   ins_pipe(empty);
 7074 %}
 7075 
 7076 instruct castFF(regF dst)
 7077 %{
 7078   match(Set dst (CastFF dst));
 7079 
 7080   size(0);
 7081   format %{ "# castFF of $dst" %}
 7082   ins_encode(/* empty encoding */);
 7083   ins_cost(0);
 7084   ins_pipe(empty);
 7085 %}
 7086 
 7087 instruct castDD(regD dst)
 7088 %{
 7089   match(Set dst (CastDD dst));
 7090 
 7091   size(0);
 7092   format %{ "# castDD of $dst" %}
 7093   ins_encode(/* empty encoding */);
 7094   ins_cost(0);
 7095   ins_pipe(empty);
 7096 %}
 7097 
 7098 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7099 instruct compareAndSwapP(rRegI res,
 7100                          memory mem_ptr,
 7101                          rax_RegP oldval, rRegP newval,
 7102                          rFlagsReg cr)
 7103 %{
 7104   predicate(n->as_LoadStore()->barrier_data() == 0);
 7105   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7106   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7107   effect(KILL cr, KILL oldval);
 7108 
 7109   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7110             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7111             "sete    $res\n\t"
 7112             "movzbl  $res, $res" %}
 7113   ins_encode %{
 7114     __ lock();
 7115     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7116     __ setb(Assembler::equal, $res$$Register);
 7117     __ movzbl($res$$Register, $res$$Register);
 7118   %}
 7119   ins_pipe( pipe_cmpxchg );
 7120 %}
 7121 
 7122 instruct compareAndSwapL(rRegI res,
 7123                          memory mem_ptr,
 7124                          rax_RegL oldval, rRegL newval,
 7125                          rFlagsReg cr)
 7126 %{
 7127   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7128   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7129   effect(KILL cr, KILL oldval);
 7130 
 7131   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7132             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7133             "sete    $res\n\t"
 7134             "movzbl  $res, $res" %}
 7135   ins_encode %{
 7136     __ lock();
 7137     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7138     __ setb(Assembler::equal, $res$$Register);
 7139     __ movzbl($res$$Register, $res$$Register);
 7140   %}
 7141   ins_pipe( pipe_cmpxchg );
 7142 %}
 7143 
 7144 instruct compareAndSwapI(rRegI res,
 7145                          memory mem_ptr,
 7146                          rax_RegI oldval, rRegI newval,
 7147                          rFlagsReg cr)
 7148 %{
 7149   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7150   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7151   effect(KILL cr, KILL oldval);
 7152 
 7153   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7154             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7155             "sete    $res\n\t"
 7156             "movzbl  $res, $res" %}
 7157   ins_encode %{
 7158     __ lock();
 7159     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7160     __ setb(Assembler::equal, $res$$Register);
 7161     __ movzbl($res$$Register, $res$$Register);
 7162   %}
 7163   ins_pipe( pipe_cmpxchg );
 7164 %}
 7165 
 7166 instruct compareAndSwapB(rRegI res,
 7167                          memory mem_ptr,
 7168                          rax_RegI oldval, rRegI newval,
 7169                          rFlagsReg cr)
 7170 %{
 7171   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7172   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7173   effect(KILL cr, KILL oldval);
 7174 
 7175   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7176             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7177             "sete    $res\n\t"
 7178             "movzbl  $res, $res" %}
 7179   ins_encode %{
 7180     __ lock();
 7181     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7182     __ setb(Assembler::equal, $res$$Register);
 7183     __ movzbl($res$$Register, $res$$Register);
 7184   %}
 7185   ins_pipe( pipe_cmpxchg );
 7186 %}
 7187 
 7188 instruct compareAndSwapS(rRegI res,
 7189                          memory mem_ptr,
 7190                          rax_RegI oldval, rRegI newval,
 7191                          rFlagsReg cr)
 7192 %{
 7193   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7194   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7195   effect(KILL cr, KILL oldval);
 7196 
 7197   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7198             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7199             "sete    $res\n\t"
 7200             "movzbl  $res, $res" %}
 7201   ins_encode %{
 7202     __ lock();
 7203     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7204     __ setb(Assembler::equal, $res$$Register);
 7205     __ movzbl($res$$Register, $res$$Register);
 7206   %}
 7207   ins_pipe( pipe_cmpxchg );
 7208 %}
 7209 
 7210 instruct compareAndSwapN(rRegI res,
 7211                           memory mem_ptr,
 7212                           rax_RegN oldval, rRegN newval,
 7213                           rFlagsReg cr) %{
 7214   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7215   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7216   effect(KILL cr, KILL oldval);
 7217 
 7218   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7219             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7220             "sete    $res\n\t"
 7221             "movzbl  $res, $res" %}
 7222   ins_encode %{
 7223     __ lock();
 7224     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7225     __ setb(Assembler::equal, $res$$Register);
 7226     __ movzbl($res$$Register, $res$$Register);
 7227   %}
 7228   ins_pipe( pipe_cmpxchg );
 7229 %}
 7230 
 7231 instruct compareAndExchangeB(
 7232                          memory mem_ptr,
 7233                          rax_RegI oldval, rRegI newval,
 7234                          rFlagsReg cr)
 7235 %{
 7236   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7237   effect(KILL cr);
 7238 
 7239   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7240             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7241   ins_encode %{
 7242     __ lock();
 7243     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7244   %}
 7245   ins_pipe( pipe_cmpxchg );
 7246 %}
 7247 
 7248 instruct compareAndExchangeS(
 7249                          memory mem_ptr,
 7250                          rax_RegI oldval, rRegI newval,
 7251                          rFlagsReg cr)
 7252 %{
 7253   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7254   effect(KILL cr);
 7255 
 7256   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7257             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7258   ins_encode %{
 7259     __ lock();
 7260     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7261   %}
 7262   ins_pipe( pipe_cmpxchg );
 7263 %}
 7264 
 7265 instruct compareAndExchangeI(
 7266                          memory mem_ptr,
 7267                          rax_RegI oldval, rRegI newval,
 7268                          rFlagsReg cr)
 7269 %{
 7270   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7271   effect(KILL cr);
 7272 
 7273   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7274             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7275   ins_encode %{
 7276     __ lock();
 7277     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7278   %}
 7279   ins_pipe( pipe_cmpxchg );
 7280 %}
 7281 
 7282 instruct compareAndExchangeL(
 7283                          memory mem_ptr,
 7284                          rax_RegL oldval, rRegL newval,
 7285                          rFlagsReg cr)
 7286 %{
 7287   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7288   effect(KILL cr);
 7289 
 7290   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7291             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7292   ins_encode %{
 7293     __ lock();
 7294     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7295   %}
 7296   ins_pipe( pipe_cmpxchg );
 7297 %}
 7298 
 7299 instruct compareAndExchangeN(
 7300                           memory mem_ptr,
 7301                           rax_RegN oldval, rRegN newval,
 7302                           rFlagsReg cr) %{
 7303   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7304   effect(KILL cr);
 7305 
 7306   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7307             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7308   ins_encode %{
 7309     __ lock();
 7310     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7311   %}
 7312   ins_pipe( pipe_cmpxchg );
 7313 %}
 7314 
 7315 instruct compareAndExchangeP(
 7316                          memory mem_ptr,
 7317                          rax_RegP oldval, rRegP newval,
 7318                          rFlagsReg cr)
 7319 %{
 7320   predicate(n->as_LoadStore()->barrier_data() == 0);
 7321   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7322   effect(KILL cr);
 7323 
 7324   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7325             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7326   ins_encode %{
 7327     __ lock();
 7328     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7329   %}
 7330   ins_pipe( pipe_cmpxchg );
 7331 %}
 7332 
 7333 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7334   predicate(n->as_LoadStore()->result_not_used());
 7335   match(Set dummy (GetAndAddB mem add));
 7336   effect(KILL cr);
 7337   format %{ "addb_lock   $mem, $add" %}
 7338   ins_encode %{
 7339     __ lock();
 7340     __ addb($mem$$Address, $add$$Register);
 7341   %}
 7342   ins_pipe(pipe_cmpxchg);
 7343 %}
 7344 
 7345 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7346   predicate(n->as_LoadStore()->result_not_used());
 7347   match(Set dummy (GetAndAddB mem add));
 7348   effect(KILL cr);
 7349   format %{ "addb_lock   $mem, $add" %}
 7350   ins_encode %{
 7351     __ lock();
 7352     __ addb($mem$$Address, $add$$constant);
 7353   %}
 7354   ins_pipe(pipe_cmpxchg);
 7355 %}
 7356 
 7357 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7358   predicate(!n->as_LoadStore()->result_not_used());
 7359   match(Set newval (GetAndAddB mem newval));
 7360   effect(KILL cr);
 7361   format %{ "xaddb_lock  $mem, $newval" %}
 7362   ins_encode %{
 7363     __ lock();
 7364     __ xaddb($mem$$Address, $newval$$Register);
 7365   %}
 7366   ins_pipe(pipe_cmpxchg);
 7367 %}
 7368 
 7369 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7370   predicate(n->as_LoadStore()->result_not_used());
 7371   match(Set dummy (GetAndAddS mem add));
 7372   effect(KILL cr);
 7373   format %{ "addw_lock   $mem, $add" %}
 7374   ins_encode %{
 7375     __ lock();
 7376     __ addw($mem$$Address, $add$$Register);
 7377   %}
 7378   ins_pipe(pipe_cmpxchg);
 7379 %}
 7380 
 7381 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7382   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7383   match(Set dummy (GetAndAddS mem add));
 7384   effect(KILL cr);
 7385   format %{ "addw_lock   $mem, $add" %}
 7386   ins_encode %{
 7387     __ lock();
 7388     __ addw($mem$$Address, $add$$constant);
 7389   %}
 7390   ins_pipe(pipe_cmpxchg);
 7391 %}
 7392 
 7393 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7394   predicate(!n->as_LoadStore()->result_not_used());
 7395   match(Set newval (GetAndAddS mem newval));
 7396   effect(KILL cr);
 7397   format %{ "xaddw_lock  $mem, $newval" %}
 7398   ins_encode %{
 7399     __ lock();
 7400     __ xaddw($mem$$Address, $newval$$Register);
 7401   %}
 7402   ins_pipe(pipe_cmpxchg);
 7403 %}
 7404 
 7405 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7406   predicate(n->as_LoadStore()->result_not_used());
 7407   match(Set dummy (GetAndAddI mem add));
 7408   effect(KILL cr);
 7409   format %{ "addl_lock   $mem, $add" %}
 7410   ins_encode %{
 7411     __ lock();
 7412     __ addl($mem$$Address, $add$$Register);
 7413   %}
 7414   ins_pipe(pipe_cmpxchg);
 7415 %}
 7416 
 7417 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7418   predicate(n->as_LoadStore()->result_not_used());
 7419   match(Set dummy (GetAndAddI mem add));
 7420   effect(KILL cr);
 7421   format %{ "addl_lock   $mem, $add" %}
 7422   ins_encode %{
 7423     __ lock();
 7424     __ addl($mem$$Address, $add$$constant);
 7425   %}
 7426   ins_pipe(pipe_cmpxchg);
 7427 %}
 7428 
 7429 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 7430   predicate(!n->as_LoadStore()->result_not_used());
 7431   match(Set newval (GetAndAddI mem newval));
 7432   effect(KILL cr);
 7433   format %{ "xaddl_lock  $mem, $newval" %}
 7434   ins_encode %{
 7435     __ lock();
 7436     __ xaddl($mem$$Address, $newval$$Register);
 7437   %}
 7438   ins_pipe(pipe_cmpxchg);
 7439 %}
 7440 
 7441 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 7442   predicate(n->as_LoadStore()->result_not_used());
 7443   match(Set dummy (GetAndAddL mem add));
 7444   effect(KILL cr);
 7445   format %{ "addq_lock   $mem, $add" %}
 7446   ins_encode %{
 7447     __ lock();
 7448     __ addq($mem$$Address, $add$$Register);
 7449   %}
 7450   ins_pipe(pipe_cmpxchg);
 7451 %}
 7452 
 7453 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 7454   predicate(n->as_LoadStore()->result_not_used());
 7455   match(Set dummy (GetAndAddL mem add));
 7456   effect(KILL cr);
 7457   format %{ "addq_lock   $mem, $add" %}
 7458   ins_encode %{
 7459     __ lock();
 7460     __ addq($mem$$Address, $add$$constant);
 7461   %}
 7462   ins_pipe(pipe_cmpxchg);
 7463 %}
 7464 
 7465 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 7466   predicate(!n->as_LoadStore()->result_not_used());
 7467   match(Set newval (GetAndAddL mem newval));
 7468   effect(KILL cr);
 7469   format %{ "xaddq_lock  $mem, $newval" %}
 7470   ins_encode %{
 7471     __ lock();
 7472     __ xaddq($mem$$Address, $newval$$Register);
 7473   %}
 7474   ins_pipe(pipe_cmpxchg);
 7475 %}
 7476 
 7477 instruct xchgB( memory mem, rRegI newval) %{
 7478   match(Set newval (GetAndSetB mem newval));
 7479   format %{ "XCHGB  $newval,[$mem]" %}
 7480   ins_encode %{
 7481     __ xchgb($newval$$Register, $mem$$Address);
 7482   %}
 7483   ins_pipe( pipe_cmpxchg );
 7484 %}
 7485 
 7486 instruct xchgS( memory mem, rRegI newval) %{
 7487   match(Set newval (GetAndSetS mem newval));
 7488   format %{ "XCHGW  $newval,[$mem]" %}
 7489   ins_encode %{
 7490     __ xchgw($newval$$Register, $mem$$Address);
 7491   %}
 7492   ins_pipe( pipe_cmpxchg );
 7493 %}
 7494 
 7495 instruct xchgI( memory mem, rRegI newval) %{
 7496   match(Set newval (GetAndSetI mem newval));
 7497   format %{ "XCHGL  $newval,[$mem]" %}
 7498   ins_encode %{
 7499     __ xchgl($newval$$Register, $mem$$Address);
 7500   %}
 7501   ins_pipe( pipe_cmpxchg );
 7502 %}
 7503 
 7504 instruct xchgL( memory mem, rRegL newval) %{
 7505   match(Set newval (GetAndSetL mem newval));
 7506   format %{ "XCHGL  $newval,[$mem]" %}
 7507   ins_encode %{
 7508     __ xchgq($newval$$Register, $mem$$Address);
 7509   %}
 7510   ins_pipe( pipe_cmpxchg );
 7511 %}
 7512 
 7513 instruct xchgP( memory mem, rRegP newval) %{
 7514   match(Set newval (GetAndSetP mem newval));
 7515   predicate(n->as_LoadStore()->barrier_data() == 0);
 7516   format %{ "XCHGQ  $newval,[$mem]" %}
 7517   ins_encode %{
 7518     __ xchgq($newval$$Register, $mem$$Address);
 7519   %}
 7520   ins_pipe( pipe_cmpxchg );
 7521 %}
 7522 
 7523 instruct xchgN( memory mem, rRegN newval) %{
 7524   match(Set newval (GetAndSetN mem newval));
 7525   format %{ "XCHGL  $newval,$mem]" %}
 7526   ins_encode %{
 7527     __ xchgl($newval$$Register, $mem$$Address);
 7528   %}
 7529   ins_pipe( pipe_cmpxchg );
 7530 %}
 7531 
 7532 //----------Abs Instructions-------------------------------------------
 7533 
 7534 // Integer Absolute Instructions
 7535 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7536 %{
 7537   match(Set dst (AbsI src));
 7538   effect(TEMP dst, KILL cr);
 7539   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 7540             "subl    $dst, $src\n\t"
 7541             "cmovll  $dst, $src" %}
 7542   ins_encode %{
 7543     __ xorl($dst$$Register, $dst$$Register);
 7544     __ subl($dst$$Register, $src$$Register);
 7545     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 7546   %}
 7547 
 7548   ins_pipe(ialu_reg_reg);
 7549 %}
 7550 
 7551 // Long Absolute Instructions
 7552 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7553 %{
 7554   match(Set dst (AbsL src));
 7555   effect(TEMP dst, KILL cr);
 7556   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 7557             "subq    $dst, $src\n\t"
 7558             "cmovlq  $dst, $src" %}
 7559   ins_encode %{
 7560     __ xorl($dst$$Register, $dst$$Register);
 7561     __ subq($dst$$Register, $src$$Register);
 7562     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 7563   %}
 7564 
 7565   ins_pipe(ialu_reg_reg);
 7566 %}
 7567 
 7568 //----------Subtraction Instructions-------------------------------------------
 7569 
 7570 // Integer Subtraction Instructions
 7571 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7572 %{
 7573   match(Set dst (SubI dst src));
 7574   effect(KILL cr);
 7575   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7576 
 7577   format %{ "subl    $dst, $src\t# int" %}
 7578   ins_encode %{
 7579     __ subl($dst$$Register, $src$$Register);
 7580   %}
 7581   ins_pipe(ialu_reg_reg);
 7582 %}
 7583 
 7584 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7585 %{
 7586   match(Set dst (SubI dst (LoadI src)));
 7587   effect(KILL cr);
 7588   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7589 
 7590   ins_cost(150);
 7591   format %{ "subl    $dst, $src\t# int" %}
 7592   ins_encode %{
 7593     __ subl($dst$$Register, $src$$Address);
 7594   %}
 7595   ins_pipe(ialu_reg_mem);
 7596 %}
 7597 
 7598 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7599 %{
 7600   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 7601   effect(KILL cr);
 7602   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7603 
 7604   ins_cost(150);
 7605   format %{ "subl    $dst, $src\t# int" %}
 7606   ins_encode %{
 7607     __ subl($dst$$Address, $src$$Register);
 7608   %}
 7609   ins_pipe(ialu_mem_reg);
 7610 %}
 7611 
 7612 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7613 %{
 7614   match(Set dst (SubL dst src));
 7615   effect(KILL cr);
 7616   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7617 
 7618   format %{ "subq    $dst, $src\t# long" %}
 7619   ins_encode %{
 7620     __ subq($dst$$Register, $src$$Register);
 7621   %}
 7622   ins_pipe(ialu_reg_reg);
 7623 %}
 7624 
 7625 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7626 %{
 7627   match(Set dst (SubL dst (LoadL src)));
 7628   effect(KILL cr);
 7629   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7630 
 7631   ins_cost(150);
 7632   format %{ "subq    $dst, $src\t# long" %}
 7633   ins_encode %{
 7634     __ subq($dst$$Register, $src$$Address);
 7635   %}
 7636   ins_pipe(ialu_reg_mem);
 7637 %}
 7638 
 7639 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7640 %{
 7641   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 7642   effect(KILL cr);
 7643   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7644 
 7645   ins_cost(150);
 7646   format %{ "subq    $dst, $src\t# long" %}
 7647   ins_encode %{
 7648     __ subq($dst$$Address, $src$$Register);
 7649   %}
 7650   ins_pipe(ialu_mem_reg);
 7651 %}
 7652 
 7653 // Subtract from a pointer
 7654 // XXX hmpf???
 7655 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 7656 %{
 7657   match(Set dst (AddP dst (SubI zero src)));
 7658   effect(KILL cr);
 7659 
 7660   format %{ "subq    $dst, $src\t# ptr - int" %}
 7661   ins_encode %{
 7662     __ subq($dst$$Register, $src$$Register);
 7663   %}
 7664   ins_pipe(ialu_reg_reg);
 7665 %}
 7666 
 7667 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 7668 %{
 7669   match(Set dst (SubI zero dst));
 7670   effect(KILL cr);
 7671   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7672 
 7673   format %{ "negl    $dst\t# int" %}
 7674   ins_encode %{
 7675     __ negl($dst$$Register);
 7676   %}
 7677   ins_pipe(ialu_reg);
 7678 %}
 7679 
 7680 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 7681 %{
 7682   match(Set dst (NegI dst));
 7683   effect(KILL cr);
 7684   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7685 
 7686   format %{ "negl    $dst\t# int" %}
 7687   ins_encode %{
 7688     __ negl($dst$$Register);
 7689   %}
 7690   ins_pipe(ialu_reg);
 7691 %}
 7692 
 7693 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 7694 %{
 7695   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 7696   effect(KILL cr);
 7697   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7698 
 7699   format %{ "negl    $dst\t# int" %}
 7700   ins_encode %{
 7701     __ negl($dst$$Address);
 7702   %}
 7703   ins_pipe(ialu_reg);
 7704 %}
 7705 
 7706 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 7707 %{
 7708   match(Set dst (SubL zero dst));
 7709   effect(KILL cr);
 7710   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7711 
 7712   format %{ "negq    $dst\t# long" %}
 7713   ins_encode %{
 7714     __ negq($dst$$Register);
 7715   %}
 7716   ins_pipe(ialu_reg);
 7717 %}
 7718 
 7719 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 7720 %{
 7721   match(Set dst (NegL dst));
 7722   effect(KILL cr);
 7723   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7724 
 7725   format %{ "negq    $dst\t# int" %}
 7726   ins_encode %{
 7727     __ negq($dst$$Register);
 7728   %}
 7729   ins_pipe(ialu_reg);
 7730 %}
 7731 
 7732 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 7733 %{
 7734   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 7735   effect(KILL cr);
 7736   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7737 
 7738   format %{ "negq    $dst\t# long" %}
 7739   ins_encode %{
 7740     __ negq($dst$$Address);
 7741   %}
 7742   ins_pipe(ialu_reg);
 7743 %}
 7744 
 7745 //----------Multiplication/Division Instructions-------------------------------
 7746 // Integer Multiplication Instructions
 7747 // Multiply Register
 7748 
 7749 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7750 %{
 7751   match(Set dst (MulI dst src));
 7752   effect(KILL cr);
 7753 
 7754   ins_cost(300);
 7755   format %{ "imull   $dst, $src\t# int" %}
 7756   ins_encode %{
 7757     __ imull($dst$$Register, $src$$Register);
 7758   %}
 7759   ins_pipe(ialu_reg_reg_alu0);
 7760 %}
 7761 
 7762 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 7763 %{
 7764   match(Set dst (MulI src imm));
 7765   effect(KILL cr);
 7766 
 7767   ins_cost(300);
 7768   format %{ "imull   $dst, $src, $imm\t# int" %}
 7769   ins_encode %{
 7770     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 7771   %}
 7772   ins_pipe(ialu_reg_reg_alu0);
 7773 %}
 7774 
 7775 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 7776 %{
 7777   match(Set dst (MulI dst (LoadI src)));
 7778   effect(KILL cr);
 7779 
 7780   ins_cost(350);
 7781   format %{ "imull   $dst, $src\t# int" %}
 7782   ins_encode %{
 7783     __ imull($dst$$Register, $src$$Address);
 7784   %}
 7785   ins_pipe(ialu_reg_mem_alu0);
 7786 %}
 7787 
 7788 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 7789 %{
 7790   match(Set dst (MulI (LoadI src) imm));
 7791   effect(KILL cr);
 7792 
 7793   ins_cost(300);
 7794   format %{ "imull   $dst, $src, $imm\t# int" %}
 7795   ins_encode %{
 7796     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 7797   %}
 7798   ins_pipe(ialu_reg_mem_alu0);
 7799 %}
 7800 
 7801 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 7802 %{
 7803   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 7804   effect(KILL cr, KILL src2);
 7805 
 7806   expand %{ mulI_rReg(dst, src1, cr);
 7807            mulI_rReg(src2, src3, cr);
 7808            addI_rReg(dst, src2, cr); %}
 7809 %}
 7810 
 7811 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7812 %{
 7813   match(Set dst (MulL dst src));
 7814   effect(KILL cr);
 7815 
 7816   ins_cost(300);
 7817   format %{ "imulq   $dst, $src\t# long" %}
 7818   ins_encode %{
 7819     __ imulq($dst$$Register, $src$$Register);
 7820   %}
 7821   ins_pipe(ialu_reg_reg_alu0);
 7822 %}
 7823 
 7824 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 7825 %{
 7826   match(Set dst (MulL src imm));
 7827   effect(KILL cr);
 7828 
 7829   ins_cost(300);
 7830   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7831   ins_encode %{
 7832     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 7833   %}
 7834   ins_pipe(ialu_reg_reg_alu0);
 7835 %}
 7836 
 7837 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 7838 %{
 7839   match(Set dst (MulL dst (LoadL src)));
 7840   effect(KILL cr);
 7841 
 7842   ins_cost(350);
 7843   format %{ "imulq   $dst, $src\t# long" %}
 7844   ins_encode %{
 7845     __ imulq($dst$$Register, $src$$Address);
 7846   %}
 7847   ins_pipe(ialu_reg_mem_alu0);
 7848 %}
 7849 
 7850 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 7851 %{
 7852   match(Set dst (MulL (LoadL src) imm));
 7853   effect(KILL cr);
 7854 
 7855   ins_cost(300);
 7856   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7857   ins_encode %{
 7858     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 7859   %}
 7860   ins_pipe(ialu_reg_mem_alu0);
 7861 %}
 7862 
 7863 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7864 %{
 7865   match(Set dst (MulHiL src rax));
 7866   effect(USE_KILL rax, KILL cr);
 7867 
 7868   ins_cost(300);
 7869   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 7870   ins_encode %{
 7871     __ imulq($src$$Register);
 7872   %}
 7873   ins_pipe(ialu_reg_reg_alu0);
 7874 %}
 7875 
 7876 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7877 %{
 7878   match(Set dst (UMulHiL src rax));
 7879   effect(USE_KILL rax, KILL cr);
 7880 
 7881   ins_cost(300);
 7882   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 7883   ins_encode %{
 7884     __ mulq($src$$Register);
 7885   %}
 7886   ins_pipe(ialu_reg_reg_alu0);
 7887 %}
 7888 
 7889 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7890                    rFlagsReg cr)
 7891 %{
 7892   match(Set rax (DivI rax div));
 7893   effect(KILL rdx, KILL cr);
 7894 
 7895   ins_cost(30*100+10*100); // XXX
 7896   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7897             "jne,s   normal\n\t"
 7898             "xorl    rdx, rdx\n\t"
 7899             "cmpl    $div, -1\n\t"
 7900             "je,s    done\n"
 7901     "normal: cdql\n\t"
 7902             "idivl   $div\n"
 7903     "done:"        %}
 7904   ins_encode(cdql_enc(div));
 7905   ins_pipe(ialu_reg_reg_alu0);
 7906 %}
 7907 
 7908 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7909                    rFlagsReg cr)
 7910 %{
 7911   match(Set rax (DivL rax div));
 7912   effect(KILL rdx, KILL cr);
 7913 
 7914   ins_cost(30*100+10*100); // XXX
 7915   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7916             "cmpq    rax, rdx\n\t"
 7917             "jne,s   normal\n\t"
 7918             "xorl    rdx, rdx\n\t"
 7919             "cmpq    $div, -1\n\t"
 7920             "je,s    done\n"
 7921     "normal: cdqq\n\t"
 7922             "idivq   $div\n"
 7923     "done:"        %}
 7924   ins_encode(cdqq_enc(div));
 7925   ins_pipe(ialu_reg_reg_alu0);
 7926 %}
 7927 
 7928 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 7929 %{
 7930   match(Set rax (UDivI rax div));
 7931   effect(KILL rdx, KILL cr);
 7932 
 7933   ins_cost(300);
 7934   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 7935   ins_encode %{
 7936     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 7937   %}
 7938   ins_pipe(ialu_reg_reg_alu0);
 7939 %}
 7940 
 7941 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 7942 %{
 7943   match(Set rax (UDivL rax div));
 7944   effect(KILL rdx, KILL cr);
 7945 
 7946   ins_cost(300);
 7947   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 7948   ins_encode %{
 7949      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 7950   %}
 7951   ins_pipe(ialu_reg_reg_alu0);
 7952 %}
 7953 
 7954 // Integer DIVMOD with Register, both quotient and mod results
 7955 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7956                              rFlagsReg cr)
 7957 %{
 7958   match(DivModI rax div);
 7959   effect(KILL cr);
 7960 
 7961   ins_cost(30*100+10*100); // XXX
 7962   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7963             "jne,s   normal\n\t"
 7964             "xorl    rdx, rdx\n\t"
 7965             "cmpl    $div, -1\n\t"
 7966             "je,s    done\n"
 7967     "normal: cdql\n\t"
 7968             "idivl   $div\n"
 7969     "done:"        %}
 7970   ins_encode(cdql_enc(div));
 7971   ins_pipe(pipe_slow);
 7972 %}
 7973 
 7974 // Long DIVMOD with Register, both quotient and mod results
 7975 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7976                              rFlagsReg cr)
 7977 %{
 7978   match(DivModL rax div);
 7979   effect(KILL cr);
 7980 
 7981   ins_cost(30*100+10*100); // XXX
 7982   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7983             "cmpq    rax, rdx\n\t"
 7984             "jne,s   normal\n\t"
 7985             "xorl    rdx, rdx\n\t"
 7986             "cmpq    $div, -1\n\t"
 7987             "je,s    done\n"
 7988     "normal: cdqq\n\t"
 7989             "idivq   $div\n"
 7990     "done:"        %}
 7991   ins_encode(cdqq_enc(div));
 7992   ins_pipe(pipe_slow);
 7993 %}
 7994 
 7995 // Unsigned integer DIVMOD with Register, both quotient and mod results
 7996 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 7997                               no_rax_rdx_RegI div, rFlagsReg cr)
 7998 %{
 7999   match(UDivModI rax div);
 8000   effect(TEMP tmp, KILL cr);
 8001 
 8002   ins_cost(300);
 8003   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8004             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8005           %}
 8006   ins_encode %{
 8007     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8008   %}
 8009   ins_pipe(pipe_slow);
 8010 %}
 8011 
 8012 // Unsigned long DIVMOD with Register, both quotient and mod results
 8013 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8014                               no_rax_rdx_RegL div, rFlagsReg cr)
 8015 %{
 8016   match(UDivModL rax div);
 8017   effect(TEMP tmp, KILL cr);
 8018 
 8019   ins_cost(300);
 8020   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8021             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8022           %}
 8023   ins_encode %{
 8024     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8025   %}
 8026   ins_pipe(pipe_slow);
 8027 %}
 8028 
 8029 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8030                    rFlagsReg cr)
 8031 %{
 8032   match(Set rdx (ModI rax div));
 8033   effect(KILL rax, KILL cr);
 8034 
 8035   ins_cost(300); // XXX
 8036   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8037             "jne,s   normal\n\t"
 8038             "xorl    rdx, rdx\n\t"
 8039             "cmpl    $div, -1\n\t"
 8040             "je,s    done\n"
 8041     "normal: cdql\n\t"
 8042             "idivl   $div\n"
 8043     "done:"        %}
 8044   ins_encode(cdql_enc(div));
 8045   ins_pipe(ialu_reg_reg_alu0);
 8046 %}
 8047 
 8048 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8049                    rFlagsReg cr)
 8050 %{
 8051   match(Set rdx (ModL rax div));
 8052   effect(KILL rax, KILL cr);
 8053 
 8054   ins_cost(300); // XXX
 8055   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8056             "cmpq    rax, rdx\n\t"
 8057             "jne,s   normal\n\t"
 8058             "xorl    rdx, rdx\n\t"
 8059             "cmpq    $div, -1\n\t"
 8060             "je,s    done\n"
 8061     "normal: cdqq\n\t"
 8062             "idivq   $div\n"
 8063     "done:"        %}
 8064   ins_encode(cdqq_enc(div));
 8065   ins_pipe(ialu_reg_reg_alu0);
 8066 %}
 8067 
 8068 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8069 %{
 8070   match(Set rdx (UModI rax div));
 8071   effect(KILL rax, KILL cr);
 8072 
 8073   ins_cost(300);
 8074   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8075   ins_encode %{
 8076     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8077   %}
 8078   ins_pipe(ialu_reg_reg_alu0);
 8079 %}
 8080 
 8081 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8082 %{
 8083   match(Set rdx (UModL rax div));
 8084   effect(KILL rax, KILL cr);
 8085 
 8086   ins_cost(300);
 8087   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8088   ins_encode %{
 8089     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8090   %}
 8091   ins_pipe(ialu_reg_reg_alu0);
 8092 %}
 8093 
 8094 // Integer Shift Instructions
 8095 // Shift Left by one, two, three
 8096 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8097 %{
 8098   match(Set dst (LShiftI dst shift));
 8099   effect(KILL cr);
 8100 
 8101   format %{ "sall    $dst, $shift" %}
 8102   ins_encode %{
 8103     __ sall($dst$$Register, $shift$$constant);
 8104   %}
 8105   ins_pipe(ialu_reg);
 8106 %}
 8107 
 8108 // Shift Left by 8-bit immediate
 8109 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8110 %{
 8111   match(Set dst (LShiftI dst shift));
 8112   effect(KILL cr);
 8113 
 8114   format %{ "sall    $dst, $shift" %}
 8115   ins_encode %{
 8116     __ sall($dst$$Register, $shift$$constant);
 8117   %}
 8118   ins_pipe(ialu_reg);
 8119 %}
 8120 
 8121 // Shift Left by 8-bit immediate
 8122 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8123 %{
 8124   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8125   effect(KILL cr);
 8126 
 8127   format %{ "sall    $dst, $shift" %}
 8128   ins_encode %{
 8129     __ sall($dst$$Address, $shift$$constant);
 8130   %}
 8131   ins_pipe(ialu_mem_imm);
 8132 %}
 8133 
 8134 // Shift Left by variable
 8135 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8136 %{
 8137   predicate(!VM_Version::supports_bmi2());
 8138   match(Set dst (LShiftI dst shift));
 8139   effect(KILL cr);
 8140 
 8141   format %{ "sall    $dst, $shift" %}
 8142   ins_encode %{
 8143     __ sall($dst$$Register);
 8144   %}
 8145   ins_pipe(ialu_reg_reg);
 8146 %}
 8147 
 8148 // Shift Left by variable
 8149 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8150 %{
 8151   predicate(!VM_Version::supports_bmi2());
 8152   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8153   effect(KILL cr);
 8154 
 8155   format %{ "sall    $dst, $shift" %}
 8156   ins_encode %{
 8157     __ sall($dst$$Address);
 8158   %}
 8159   ins_pipe(ialu_mem_reg);
 8160 %}
 8161 
 8162 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8163 %{
 8164   predicate(VM_Version::supports_bmi2());
 8165   match(Set dst (LShiftI src shift));
 8166 
 8167   format %{ "shlxl   $dst, $src, $shift" %}
 8168   ins_encode %{
 8169     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 8170   %}
 8171   ins_pipe(ialu_reg_reg);
 8172 %}
 8173 
 8174 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8175 %{
 8176   predicate(VM_Version::supports_bmi2());
 8177   match(Set dst (LShiftI (LoadI src) shift));
 8178   ins_cost(175);
 8179   format %{ "shlxl   $dst, $src, $shift" %}
 8180   ins_encode %{
 8181     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 8182   %}
 8183   ins_pipe(ialu_reg_mem);
 8184 %}
 8185 
 8186 // Arithmetic Shift Right by 8-bit immediate
 8187 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8188 %{
 8189   match(Set dst (RShiftI dst shift));
 8190   effect(KILL cr);
 8191 
 8192   format %{ "sarl    $dst, $shift" %}
 8193   ins_encode %{
 8194     __ sarl($dst$$Register, $shift$$constant);
 8195   %}
 8196   ins_pipe(ialu_mem_imm);
 8197 %}
 8198 
 8199 // Arithmetic Shift Right by 8-bit immediate
 8200 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8201 %{
 8202   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8203   effect(KILL cr);
 8204 
 8205   format %{ "sarl    $dst, $shift" %}
 8206   ins_encode %{
 8207     __ sarl($dst$$Address, $shift$$constant);
 8208   %}
 8209   ins_pipe(ialu_mem_imm);
 8210 %}
 8211 
 8212 // Arithmetic Shift Right by variable
 8213 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8214 %{
 8215   predicate(!VM_Version::supports_bmi2());
 8216   match(Set dst (RShiftI dst shift));
 8217   effect(KILL cr);
 8218 
 8219   format %{ "sarl    $dst, $shift" %}
 8220   ins_encode %{
 8221     __ sarl($dst$$Register);
 8222   %}
 8223   ins_pipe(ialu_reg_reg);
 8224 %}
 8225 
 8226 // Arithmetic Shift Right by variable
 8227 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8228 %{
 8229   predicate(!VM_Version::supports_bmi2());
 8230   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8231   effect(KILL cr);
 8232 
 8233   format %{ "sarl    $dst, $shift" %}
 8234   ins_encode %{
 8235     __ sarl($dst$$Address);
 8236   %}
 8237   ins_pipe(ialu_mem_reg);
 8238 %}
 8239 
 8240 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8241 %{
 8242   predicate(VM_Version::supports_bmi2());
 8243   match(Set dst (RShiftI src shift));
 8244 
 8245   format %{ "sarxl   $dst, $src, $shift" %}
 8246   ins_encode %{
 8247     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 8248   %}
 8249   ins_pipe(ialu_reg_reg);
 8250 %}
 8251 
 8252 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8253 %{
 8254   predicate(VM_Version::supports_bmi2());
 8255   match(Set dst (RShiftI (LoadI src) shift));
 8256   ins_cost(175);
 8257   format %{ "sarxl   $dst, $src, $shift" %}
 8258   ins_encode %{
 8259     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 8260   %}
 8261   ins_pipe(ialu_reg_mem);
 8262 %}
 8263 
 8264 // Logical Shift Right by 8-bit immediate
 8265 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8266 %{
 8267   match(Set dst (URShiftI dst shift));
 8268   effect(KILL cr);
 8269 
 8270   format %{ "shrl    $dst, $shift" %}
 8271   ins_encode %{
 8272     __ shrl($dst$$Register, $shift$$constant);
 8273   %}
 8274   ins_pipe(ialu_reg);
 8275 %}
 8276 
 8277 // Logical Shift Right by 8-bit immediate
 8278 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8279 %{
 8280   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8281   effect(KILL cr);
 8282 
 8283   format %{ "shrl    $dst, $shift" %}
 8284   ins_encode %{
 8285     __ shrl($dst$$Address, $shift$$constant);
 8286   %}
 8287   ins_pipe(ialu_mem_imm);
 8288 %}
 8289 
 8290 // Logical Shift Right by variable
 8291 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8292 %{
 8293   predicate(!VM_Version::supports_bmi2());
 8294   match(Set dst (URShiftI dst shift));
 8295   effect(KILL cr);
 8296 
 8297   format %{ "shrl    $dst, $shift" %}
 8298   ins_encode %{
 8299     __ shrl($dst$$Register);
 8300   %}
 8301   ins_pipe(ialu_reg_reg);
 8302 %}
 8303 
 8304 // Logical Shift Right by variable
 8305 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8306 %{
 8307   predicate(!VM_Version::supports_bmi2());
 8308   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8309   effect(KILL cr);
 8310 
 8311   format %{ "shrl    $dst, $shift" %}
 8312   ins_encode %{
 8313     __ shrl($dst$$Address);
 8314   %}
 8315   ins_pipe(ialu_mem_reg);
 8316 %}
 8317 
 8318 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8319 %{
 8320   predicate(VM_Version::supports_bmi2());
 8321   match(Set dst (URShiftI src shift));
 8322 
 8323   format %{ "shrxl   $dst, $src, $shift" %}
 8324   ins_encode %{
 8325     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 8326   %}
 8327   ins_pipe(ialu_reg_reg);
 8328 %}
 8329 
 8330 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8331 %{
 8332   predicate(VM_Version::supports_bmi2());
 8333   match(Set dst (URShiftI (LoadI src) shift));
 8334   ins_cost(175);
 8335   format %{ "shrxl   $dst, $src, $shift" %}
 8336   ins_encode %{
 8337     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 8338   %}
 8339   ins_pipe(ialu_reg_mem);
 8340 %}
 8341 
 8342 // Long Shift Instructions
 8343 // Shift Left by one, two, three
 8344 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 8345 %{
 8346   match(Set dst (LShiftL dst shift));
 8347   effect(KILL cr);
 8348 
 8349   format %{ "salq    $dst, $shift" %}
 8350   ins_encode %{
 8351     __ salq($dst$$Register, $shift$$constant);
 8352   %}
 8353   ins_pipe(ialu_reg);
 8354 %}
 8355 
 8356 // Shift Left by 8-bit immediate
 8357 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8358 %{
 8359   match(Set dst (LShiftL dst shift));
 8360   effect(KILL cr);
 8361 
 8362   format %{ "salq    $dst, $shift" %}
 8363   ins_encode %{
 8364     __ salq($dst$$Register, $shift$$constant);
 8365   %}
 8366   ins_pipe(ialu_reg);
 8367 %}
 8368 
 8369 // Shift Left by 8-bit immediate
 8370 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8371 %{
 8372   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8373   effect(KILL cr);
 8374 
 8375   format %{ "salq    $dst, $shift" %}
 8376   ins_encode %{
 8377     __ salq($dst$$Address, $shift$$constant);
 8378   %}
 8379   ins_pipe(ialu_mem_imm);
 8380 %}
 8381 
 8382 // Shift Left by variable
 8383 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8384 %{
 8385   predicate(!VM_Version::supports_bmi2());
 8386   match(Set dst (LShiftL dst shift));
 8387   effect(KILL cr);
 8388 
 8389   format %{ "salq    $dst, $shift" %}
 8390   ins_encode %{
 8391     __ salq($dst$$Register);
 8392   %}
 8393   ins_pipe(ialu_reg_reg);
 8394 %}
 8395 
 8396 // Shift Left by variable
 8397 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8398 %{
 8399   predicate(!VM_Version::supports_bmi2());
 8400   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8401   effect(KILL cr);
 8402 
 8403   format %{ "salq    $dst, $shift" %}
 8404   ins_encode %{
 8405     __ salq($dst$$Address);
 8406   %}
 8407   ins_pipe(ialu_mem_reg);
 8408 %}
 8409 
 8410 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8411 %{
 8412   predicate(VM_Version::supports_bmi2());
 8413   match(Set dst (LShiftL src shift));
 8414 
 8415   format %{ "shlxq   $dst, $src, $shift" %}
 8416   ins_encode %{
 8417     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 8418   %}
 8419   ins_pipe(ialu_reg_reg);
 8420 %}
 8421 
 8422 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8423 %{
 8424   predicate(VM_Version::supports_bmi2());
 8425   match(Set dst (LShiftL (LoadL src) shift));
 8426   ins_cost(175);
 8427   format %{ "shlxq   $dst, $src, $shift" %}
 8428   ins_encode %{
 8429     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 8430   %}
 8431   ins_pipe(ialu_reg_mem);
 8432 %}
 8433 
 8434 // Arithmetic Shift Right by 8-bit immediate
 8435 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 8436 %{
 8437   match(Set dst (RShiftL dst shift));
 8438   effect(KILL cr);
 8439 
 8440   format %{ "sarq    $dst, $shift" %}
 8441   ins_encode %{
 8442     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 8443   %}
 8444   ins_pipe(ialu_mem_imm);
 8445 %}
 8446 
 8447 // Arithmetic Shift Right by 8-bit immediate
 8448 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 8449 %{
 8450   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8451   effect(KILL cr);
 8452 
 8453   format %{ "sarq    $dst, $shift" %}
 8454   ins_encode %{
 8455     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 8456   %}
 8457   ins_pipe(ialu_mem_imm);
 8458 %}
 8459 
 8460 // Arithmetic Shift Right by variable
 8461 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8462 %{
 8463   predicate(!VM_Version::supports_bmi2());
 8464   match(Set dst (RShiftL dst shift));
 8465   effect(KILL cr);
 8466 
 8467   format %{ "sarq    $dst, $shift" %}
 8468   ins_encode %{
 8469     __ sarq($dst$$Register);
 8470   %}
 8471   ins_pipe(ialu_reg_reg);
 8472 %}
 8473 
 8474 // Arithmetic Shift Right by variable
 8475 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8476 %{
 8477   predicate(!VM_Version::supports_bmi2());
 8478   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8479   effect(KILL cr);
 8480 
 8481   format %{ "sarq    $dst, $shift" %}
 8482   ins_encode %{
 8483     __ sarq($dst$$Address);
 8484   %}
 8485   ins_pipe(ialu_mem_reg);
 8486 %}
 8487 
 8488 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8489 %{
 8490   predicate(VM_Version::supports_bmi2());
 8491   match(Set dst (RShiftL src shift));
 8492 
 8493   format %{ "sarxq   $dst, $src, $shift" %}
 8494   ins_encode %{
 8495     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 8496   %}
 8497   ins_pipe(ialu_reg_reg);
 8498 %}
 8499 
 8500 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8501 %{
 8502   predicate(VM_Version::supports_bmi2());
 8503   match(Set dst (RShiftL (LoadL src) shift));
 8504   ins_cost(175);
 8505   format %{ "sarxq   $dst, $src, $shift" %}
 8506   ins_encode %{
 8507     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 8508   %}
 8509   ins_pipe(ialu_reg_mem);
 8510 %}
 8511 
 8512 // Logical Shift Right by 8-bit immediate
 8513 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8514 %{
 8515   match(Set dst (URShiftL dst shift));
 8516   effect(KILL cr);
 8517 
 8518   format %{ "shrq    $dst, $shift" %}
 8519   ins_encode %{
 8520     __ shrq($dst$$Register, $shift$$constant);
 8521   %}
 8522   ins_pipe(ialu_reg);
 8523 %}
 8524 
 8525 // Logical Shift Right by 8-bit immediate
 8526 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8527 %{
 8528   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8529   effect(KILL cr);
 8530 
 8531   format %{ "shrq    $dst, $shift" %}
 8532   ins_encode %{
 8533     __ shrq($dst$$Address, $shift$$constant);
 8534   %}
 8535   ins_pipe(ialu_mem_imm);
 8536 %}
 8537 
 8538 // Logical Shift Right by variable
 8539 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8540 %{
 8541   predicate(!VM_Version::supports_bmi2());
 8542   match(Set dst (URShiftL dst shift));
 8543   effect(KILL cr);
 8544 
 8545   format %{ "shrq    $dst, $shift" %}
 8546   ins_encode %{
 8547     __ shrq($dst$$Register);
 8548   %}
 8549   ins_pipe(ialu_reg_reg);
 8550 %}
 8551 
 8552 // Logical Shift Right by variable
 8553 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8554 %{
 8555   predicate(!VM_Version::supports_bmi2());
 8556   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8557   effect(KILL cr);
 8558 
 8559   format %{ "shrq    $dst, $shift" %}
 8560   ins_encode %{
 8561     __ shrq($dst$$Address);
 8562   %}
 8563   ins_pipe(ialu_mem_reg);
 8564 %}
 8565 
 8566 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8567 %{
 8568   predicate(VM_Version::supports_bmi2());
 8569   match(Set dst (URShiftL src shift));
 8570 
 8571   format %{ "shrxq   $dst, $src, $shift" %}
 8572   ins_encode %{
 8573     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 8574   %}
 8575   ins_pipe(ialu_reg_reg);
 8576 %}
 8577 
 8578 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8579 %{
 8580   predicate(VM_Version::supports_bmi2());
 8581   match(Set dst (URShiftL (LoadL src) shift));
 8582   ins_cost(175);
 8583   format %{ "shrxq   $dst, $src, $shift" %}
 8584   ins_encode %{
 8585     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 8586   %}
 8587   ins_pipe(ialu_reg_mem);
 8588 %}
 8589 
 8590 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 8591 // This idiom is used by the compiler for the i2b bytecode.
 8592 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 8593 %{
 8594   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 8595 
 8596   format %{ "movsbl  $dst, $src\t# i2b" %}
 8597   ins_encode %{
 8598     __ movsbl($dst$$Register, $src$$Register);
 8599   %}
 8600   ins_pipe(ialu_reg_reg);
 8601 %}
 8602 
 8603 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 8604 // This idiom is used by the compiler the i2s bytecode.
 8605 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 8606 %{
 8607   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 8608 
 8609   format %{ "movswl  $dst, $src\t# i2s" %}
 8610   ins_encode %{
 8611     __ movswl($dst$$Register, $src$$Register);
 8612   %}
 8613   ins_pipe(ialu_reg_reg);
 8614 %}
 8615 
 8616 // ROL/ROR instructions
 8617 
 8618 // Rotate left by constant.
 8619 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8620 %{
 8621   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8622   match(Set dst (RotateLeft dst shift));
 8623   effect(KILL cr);
 8624   format %{ "roll    $dst, $shift" %}
 8625   ins_encode %{
 8626     __ roll($dst$$Register, $shift$$constant);
 8627   %}
 8628   ins_pipe(ialu_reg);
 8629 %}
 8630 
 8631 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 8632 %{
 8633   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8634   match(Set dst (RotateLeft src shift));
 8635   format %{ "rolxl   $dst, $src, $shift" %}
 8636   ins_encode %{
 8637     int shift = 32 - ($shift$$constant & 31);
 8638     __ rorxl($dst$$Register, $src$$Register, shift);
 8639   %}
 8640   ins_pipe(ialu_reg_reg);
 8641 %}
 8642 
 8643 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8644 %{
 8645   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8646   match(Set dst (RotateLeft (LoadI src) shift));
 8647   ins_cost(175);
 8648   format %{ "rolxl   $dst, $src, $shift" %}
 8649   ins_encode %{
 8650     int shift = 32 - ($shift$$constant & 31);
 8651     __ rorxl($dst$$Register, $src$$Address, shift);
 8652   %}
 8653   ins_pipe(ialu_reg_mem);
 8654 %}
 8655 
 8656 // Rotate Left by variable
 8657 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8658 %{
 8659   predicate(n->bottom_type()->basic_type() == T_INT);
 8660   match(Set dst (RotateLeft dst shift));
 8661   effect(KILL cr);
 8662   format %{ "roll    $dst, $shift" %}
 8663   ins_encode %{
 8664     __ roll($dst$$Register);
 8665   %}
 8666   ins_pipe(ialu_reg_reg);
 8667 %}
 8668 
 8669 // Rotate Right by constant.
 8670 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8671 %{
 8672   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8673   match(Set dst (RotateRight dst shift));
 8674   effect(KILL cr);
 8675   format %{ "rorl    $dst, $shift" %}
 8676   ins_encode %{
 8677     __ rorl($dst$$Register, $shift$$constant);
 8678   %}
 8679   ins_pipe(ialu_reg);
 8680 %}
 8681 
 8682 // Rotate Right by constant.
 8683 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 8684 %{
 8685   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8686   match(Set dst (RotateRight src shift));
 8687   format %{ "rorxl   $dst, $src, $shift" %}
 8688   ins_encode %{
 8689     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 8690   %}
 8691   ins_pipe(ialu_reg_reg);
 8692 %}
 8693 
 8694 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8695 %{
 8696   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8697   match(Set dst (RotateRight (LoadI src) shift));
 8698   ins_cost(175);
 8699   format %{ "rorxl   $dst, $src, $shift" %}
 8700   ins_encode %{
 8701     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 8702   %}
 8703   ins_pipe(ialu_reg_mem);
 8704 %}
 8705 
 8706 // Rotate Right by variable
 8707 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8708 %{
 8709   predicate(n->bottom_type()->basic_type() == T_INT);
 8710   match(Set dst (RotateRight dst shift));
 8711   effect(KILL cr);
 8712   format %{ "rorl    $dst, $shift" %}
 8713   ins_encode %{
 8714     __ rorl($dst$$Register);
 8715   %}
 8716   ins_pipe(ialu_reg_reg);
 8717 %}
 8718 
 8719 // Rotate Left by constant.
 8720 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8721 %{
 8722   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8723   match(Set dst (RotateLeft dst shift));
 8724   effect(KILL cr);
 8725   format %{ "rolq    $dst, $shift" %}
 8726   ins_encode %{
 8727     __ rolq($dst$$Register, $shift$$constant);
 8728   %}
 8729   ins_pipe(ialu_reg);
 8730 %}
 8731 
 8732 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 8733 %{
 8734   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8735   match(Set dst (RotateLeft src shift));
 8736   format %{ "rolxq   $dst, $src, $shift" %}
 8737   ins_encode %{
 8738     int shift = 64 - ($shift$$constant & 63);
 8739     __ rorxq($dst$$Register, $src$$Register, shift);
 8740   %}
 8741   ins_pipe(ialu_reg_reg);
 8742 %}
 8743 
 8744 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8745 %{
 8746   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8747   match(Set dst (RotateLeft (LoadL src) shift));
 8748   ins_cost(175);
 8749   format %{ "rolxq   $dst, $src, $shift" %}
 8750   ins_encode %{
 8751     int shift = 64 - ($shift$$constant & 63);
 8752     __ rorxq($dst$$Register, $src$$Address, shift);
 8753   %}
 8754   ins_pipe(ialu_reg_mem);
 8755 %}
 8756 
 8757 // Rotate Left by variable
 8758 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8759 %{
 8760   predicate(n->bottom_type()->basic_type() == T_LONG);
 8761   match(Set dst (RotateLeft dst shift));
 8762   effect(KILL cr);
 8763   format %{ "rolq    $dst, $shift" %}
 8764   ins_encode %{
 8765     __ rolq($dst$$Register);
 8766   %}
 8767   ins_pipe(ialu_reg_reg);
 8768 %}
 8769 
 8770 // Rotate Right by constant.
 8771 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8772 %{
 8773   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8774   match(Set dst (RotateRight dst shift));
 8775   effect(KILL cr);
 8776   format %{ "rorq    $dst, $shift" %}
 8777   ins_encode %{
 8778     __ rorq($dst$$Register, $shift$$constant);
 8779   %}
 8780   ins_pipe(ialu_reg);
 8781 %}
 8782 
 8783 // Rotate Right by constant
 8784 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 8785 %{
 8786   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8787   match(Set dst (RotateRight src shift));
 8788   format %{ "rorxq   $dst, $src, $shift" %}
 8789   ins_encode %{
 8790     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 8791   %}
 8792   ins_pipe(ialu_reg_reg);
 8793 %}
 8794 
 8795 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8796 %{
 8797   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8798   match(Set dst (RotateRight (LoadL src) shift));
 8799   ins_cost(175);
 8800   format %{ "rorxq   $dst, $src, $shift" %}
 8801   ins_encode %{
 8802     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 8803   %}
 8804   ins_pipe(ialu_reg_mem);
 8805 %}
 8806 
 8807 // Rotate Right by variable
 8808 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8809 %{
 8810   predicate(n->bottom_type()->basic_type() == T_LONG);
 8811   match(Set dst (RotateRight dst shift));
 8812   effect(KILL cr);
 8813   format %{ "rorq    $dst, $shift" %}
 8814   ins_encode %{
 8815     __ rorq($dst$$Register);
 8816   %}
 8817   ins_pipe(ialu_reg_reg);
 8818 %}
 8819 
 8820 //----------------------------- CompressBits/ExpandBits ------------------------
 8821 
 8822 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8823   predicate(n->bottom_type()->isa_long());
 8824   match(Set dst (CompressBits src mask));
 8825   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8826   ins_encode %{
 8827     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 8828   %}
 8829   ins_pipe( pipe_slow );
 8830 %}
 8831 
 8832 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8833   predicate(n->bottom_type()->isa_long());
 8834   match(Set dst (ExpandBits src mask));
 8835   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8836   ins_encode %{
 8837     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 8838   %}
 8839   ins_pipe( pipe_slow );
 8840 %}
 8841 
 8842 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8843   predicate(n->bottom_type()->isa_long());
 8844   match(Set dst (CompressBits src (LoadL mask)));
 8845   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8846   ins_encode %{
 8847     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 8848   %}
 8849   ins_pipe( pipe_slow );
 8850 %}
 8851 
 8852 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8853   predicate(n->bottom_type()->isa_long());
 8854   match(Set dst (ExpandBits src (LoadL mask)));
 8855   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8856   ins_encode %{
 8857     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 8858   %}
 8859   ins_pipe( pipe_slow );
 8860 %}
 8861 
 8862 
 8863 // Logical Instructions
 8864 
 8865 // Integer Logical Instructions
 8866 
 8867 // And Instructions
 8868 // And Register with Register
 8869 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8870 %{
 8871   match(Set dst (AndI dst src));
 8872   effect(KILL cr);
 8873   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8874 
 8875   format %{ "andl    $dst, $src\t# int" %}
 8876   ins_encode %{
 8877     __ andl($dst$$Register, $src$$Register);
 8878   %}
 8879   ins_pipe(ialu_reg_reg);
 8880 %}
 8881 
 8882 // And Register with Immediate 255
 8883 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 8884 %{
 8885   match(Set dst (AndI src mask));
 8886 
 8887   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 8888   ins_encode %{
 8889     __ movzbl($dst$$Register, $src$$Register);
 8890   %}
 8891   ins_pipe(ialu_reg);
 8892 %}
 8893 
 8894 // And Register with Immediate 255 and promote to long
 8895 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 8896 %{
 8897   match(Set dst (ConvI2L (AndI src mask)));
 8898 
 8899   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 8900   ins_encode %{
 8901     __ movzbl($dst$$Register, $src$$Register);
 8902   %}
 8903   ins_pipe(ialu_reg);
 8904 %}
 8905 
 8906 // And Register with Immediate 65535
 8907 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 8908 %{
 8909   match(Set dst (AndI src mask));
 8910 
 8911   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 8912   ins_encode %{
 8913     __ movzwl($dst$$Register, $src$$Register);
 8914   %}
 8915   ins_pipe(ialu_reg);
 8916 %}
 8917 
 8918 // And Register with Immediate 65535 and promote to long
 8919 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 8920 %{
 8921   match(Set dst (ConvI2L (AndI src mask)));
 8922 
 8923   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 8924   ins_encode %{
 8925     __ movzwl($dst$$Register, $src$$Register);
 8926   %}
 8927   ins_pipe(ialu_reg);
 8928 %}
 8929 
 8930 // Can skip int2long conversions after AND with small bitmask
 8931 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 8932 %{
 8933   predicate(VM_Version::supports_bmi2());
 8934   ins_cost(125);
 8935   effect(TEMP tmp, KILL cr);
 8936   match(Set dst (ConvI2L (AndI src mask)));
 8937   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 8938   ins_encode %{
 8939     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 8940     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 8941   %}
 8942   ins_pipe(ialu_reg_reg);
 8943 %}
 8944 
 8945 // And Register with Immediate
 8946 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8947 %{
 8948   match(Set dst (AndI dst src));
 8949   effect(KILL cr);
 8950   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8951 
 8952   format %{ "andl    $dst, $src\t# int" %}
 8953   ins_encode %{
 8954     __ andl($dst$$Register, $src$$constant);
 8955   %}
 8956   ins_pipe(ialu_reg);
 8957 %}
 8958 
 8959 // And Register with Memory
 8960 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8961 %{
 8962   match(Set dst (AndI dst (LoadI src)));
 8963   effect(KILL cr);
 8964   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8965 
 8966   ins_cost(150);
 8967   format %{ "andl    $dst, $src\t# int" %}
 8968   ins_encode %{
 8969     __ andl($dst$$Register, $src$$Address);
 8970   %}
 8971   ins_pipe(ialu_reg_mem);
 8972 %}
 8973 
 8974 // And Memory with Register
 8975 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8976 %{
 8977   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 8978   effect(KILL cr);
 8979   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8980 
 8981   ins_cost(150);
 8982   format %{ "andb    $dst, $src\t# byte" %}
 8983   ins_encode %{
 8984     __ andb($dst$$Address, $src$$Register);
 8985   %}
 8986   ins_pipe(ialu_mem_reg);
 8987 %}
 8988 
 8989 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8990 %{
 8991   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 8992   effect(KILL cr);
 8993   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8994 
 8995   ins_cost(150);
 8996   format %{ "andl    $dst, $src\t# int" %}
 8997   ins_encode %{
 8998     __ andl($dst$$Address, $src$$Register);
 8999   %}
 9000   ins_pipe(ialu_mem_reg);
 9001 %}
 9002 
 9003 // And Memory with Immediate
 9004 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9005 %{
 9006   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9007   effect(KILL cr);
 9008   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9009 
 9010   ins_cost(125);
 9011   format %{ "andl    $dst, $src\t# int" %}
 9012   ins_encode %{
 9013     __ andl($dst$$Address, $src$$constant);
 9014   %}
 9015   ins_pipe(ialu_mem_imm);
 9016 %}
 9017 
 9018 // BMI1 instructions
 9019 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9020   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9021   predicate(UseBMI1Instructions);
 9022   effect(KILL cr);
 9023   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9024 
 9025   ins_cost(125);
 9026   format %{ "andnl  $dst, $src1, $src2" %}
 9027 
 9028   ins_encode %{
 9029     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9030   %}
 9031   ins_pipe(ialu_reg_mem);
 9032 %}
 9033 
 9034 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9035   match(Set dst (AndI (XorI src1 minus_1) src2));
 9036   predicate(UseBMI1Instructions);
 9037   effect(KILL cr);
 9038   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9039 
 9040   format %{ "andnl  $dst, $src1, $src2" %}
 9041 
 9042   ins_encode %{
 9043     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9044   %}
 9045   ins_pipe(ialu_reg);
 9046 %}
 9047 
 9048 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9049   match(Set dst (AndI (SubI imm_zero src) src));
 9050   predicate(UseBMI1Instructions);
 9051   effect(KILL cr);
 9052   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9053 
 9054   format %{ "blsil  $dst, $src" %}
 9055 
 9056   ins_encode %{
 9057     __ blsil($dst$$Register, $src$$Register);
 9058   %}
 9059   ins_pipe(ialu_reg);
 9060 %}
 9061 
 9062 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9063   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9064   predicate(UseBMI1Instructions);
 9065   effect(KILL cr);
 9066   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9067 
 9068   ins_cost(125);
 9069   format %{ "blsil  $dst, $src" %}
 9070 
 9071   ins_encode %{
 9072     __ blsil($dst$$Register, $src$$Address);
 9073   %}
 9074   ins_pipe(ialu_reg_mem);
 9075 %}
 9076 
 9077 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9078 %{
 9079   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9080   predicate(UseBMI1Instructions);
 9081   effect(KILL cr);
 9082   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9083 
 9084   ins_cost(125);
 9085   format %{ "blsmskl $dst, $src" %}
 9086 
 9087   ins_encode %{
 9088     __ blsmskl($dst$$Register, $src$$Address);
 9089   %}
 9090   ins_pipe(ialu_reg_mem);
 9091 %}
 9092 
 9093 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9094 %{
 9095   match(Set dst (XorI (AddI src minus_1) src));
 9096   predicate(UseBMI1Instructions);
 9097   effect(KILL cr);
 9098   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9099 
 9100   format %{ "blsmskl $dst, $src" %}
 9101 
 9102   ins_encode %{
 9103     __ blsmskl($dst$$Register, $src$$Register);
 9104   %}
 9105 
 9106   ins_pipe(ialu_reg);
 9107 %}
 9108 
 9109 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9110 %{
 9111   match(Set dst (AndI (AddI src minus_1) src) );
 9112   predicate(UseBMI1Instructions);
 9113   effect(KILL cr);
 9114   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9115 
 9116   format %{ "blsrl  $dst, $src" %}
 9117 
 9118   ins_encode %{
 9119     __ blsrl($dst$$Register, $src$$Register);
 9120   %}
 9121 
 9122   ins_pipe(ialu_reg_mem);
 9123 %}
 9124 
 9125 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9126 %{
 9127   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9128   predicate(UseBMI1Instructions);
 9129   effect(KILL cr);
 9130   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9131 
 9132   ins_cost(125);
 9133   format %{ "blsrl  $dst, $src" %}
 9134 
 9135   ins_encode %{
 9136     __ blsrl($dst$$Register, $src$$Address);
 9137   %}
 9138 
 9139   ins_pipe(ialu_reg);
 9140 %}
 9141 
 9142 // Or Instructions
 9143 // Or Register with Register
 9144 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9145 %{
 9146   match(Set dst (OrI dst src));
 9147   effect(KILL cr);
 9148   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9149 
 9150   format %{ "orl     $dst, $src\t# int" %}
 9151   ins_encode %{
 9152     __ orl($dst$$Register, $src$$Register);
 9153   %}
 9154   ins_pipe(ialu_reg_reg);
 9155 %}
 9156 
 9157 // Or Register with Immediate
 9158 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9159 %{
 9160   match(Set dst (OrI dst src));
 9161   effect(KILL cr);
 9162   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9163 
 9164   format %{ "orl     $dst, $src\t# int" %}
 9165   ins_encode %{
 9166     __ orl($dst$$Register, $src$$constant);
 9167   %}
 9168   ins_pipe(ialu_reg);
 9169 %}
 9170 
 9171 // Or Register with Memory
 9172 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9173 %{
 9174   match(Set dst (OrI dst (LoadI src)));
 9175   effect(KILL cr);
 9176   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9177 
 9178   ins_cost(150);
 9179   format %{ "orl     $dst, $src\t# int" %}
 9180   ins_encode %{
 9181     __ orl($dst$$Register, $src$$Address);
 9182   %}
 9183   ins_pipe(ialu_reg_mem);
 9184 %}
 9185 
 9186 // Or Memory with Register
 9187 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9188 %{
 9189   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9190   effect(KILL cr);
 9191   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9192 
 9193   ins_cost(150);
 9194   format %{ "orb    $dst, $src\t# byte" %}
 9195   ins_encode %{
 9196     __ orb($dst$$Address, $src$$Register);
 9197   %}
 9198   ins_pipe(ialu_mem_reg);
 9199 %}
 9200 
 9201 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9202 %{
 9203   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9204   effect(KILL cr);
 9205   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9206 
 9207   ins_cost(150);
 9208   format %{ "orl     $dst, $src\t# int" %}
 9209   ins_encode %{
 9210     __ orl($dst$$Address, $src$$Register);
 9211   %}
 9212   ins_pipe(ialu_mem_reg);
 9213 %}
 9214 
 9215 // Or Memory with Immediate
 9216 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9217 %{
 9218   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9219   effect(KILL cr);
 9220   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9221 
 9222   ins_cost(125);
 9223   format %{ "orl     $dst, $src\t# int" %}
 9224   ins_encode %{
 9225     __ orl($dst$$Address, $src$$constant);
 9226   %}
 9227   ins_pipe(ialu_mem_imm);
 9228 %}
 9229 
 9230 // Xor Instructions
 9231 // Xor Register with Register
 9232 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9233 %{
 9234   match(Set dst (XorI dst src));
 9235   effect(KILL cr);
 9236   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9237 
 9238   format %{ "xorl    $dst, $src\t# int" %}
 9239   ins_encode %{
 9240     __ xorl($dst$$Register, $src$$Register);
 9241   %}
 9242   ins_pipe(ialu_reg_reg);
 9243 %}
 9244 
 9245 // Xor Register with Immediate -1
 9246 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9247   match(Set dst (XorI dst imm));
 9248 
 9249   format %{ "not    $dst" %}
 9250   ins_encode %{
 9251      __ notl($dst$$Register);
 9252   %}
 9253   ins_pipe(ialu_reg);
 9254 %}
 9255 
 9256 // Xor Register with Immediate
 9257 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9258 %{
 9259   match(Set dst (XorI dst src));
 9260   effect(KILL cr);
 9261   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9262 
 9263   format %{ "xorl    $dst, $src\t# int" %}
 9264   ins_encode %{
 9265     __ xorl($dst$$Register, $src$$constant);
 9266   %}
 9267   ins_pipe(ialu_reg);
 9268 %}
 9269 
 9270 // Xor Register with Memory
 9271 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9272 %{
 9273   match(Set dst (XorI dst (LoadI src)));
 9274   effect(KILL cr);
 9275   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9276 
 9277   ins_cost(150);
 9278   format %{ "xorl    $dst, $src\t# int" %}
 9279   ins_encode %{
 9280     __ xorl($dst$$Register, $src$$Address);
 9281   %}
 9282   ins_pipe(ialu_reg_mem);
 9283 %}
 9284 
 9285 // Xor Memory with Register
 9286 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9287 %{
 9288   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9289   effect(KILL cr);
 9290   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9291 
 9292   ins_cost(150);
 9293   format %{ "xorb    $dst, $src\t# byte" %}
 9294   ins_encode %{
 9295     __ xorb($dst$$Address, $src$$Register);
 9296   %}
 9297   ins_pipe(ialu_mem_reg);
 9298 %}
 9299 
 9300 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9301 %{
 9302   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9303   effect(KILL cr);
 9304   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9305 
 9306   ins_cost(150);
 9307   format %{ "xorl    $dst, $src\t# int" %}
 9308   ins_encode %{
 9309     __ xorl($dst$$Address, $src$$Register);
 9310   %}
 9311   ins_pipe(ialu_mem_reg);
 9312 %}
 9313 
 9314 // Xor Memory with Immediate
 9315 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9316 %{
 9317   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9318   effect(KILL cr);
 9319   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9320 
 9321   ins_cost(125);
 9322   format %{ "xorl    $dst, $src\t# int" %}
 9323   ins_encode %{
 9324     __ xorl($dst$$Address, $src$$constant);
 9325   %}
 9326   ins_pipe(ialu_mem_imm);
 9327 %}
 9328 
 9329 
 9330 // Long Logical Instructions
 9331 
 9332 // And Instructions
 9333 // And Register with Register
 9334 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9335 %{
 9336   match(Set dst (AndL dst src));
 9337   effect(KILL cr);
 9338   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9339 
 9340   format %{ "andq    $dst, $src\t# long" %}
 9341   ins_encode %{
 9342     __ andq($dst$$Register, $src$$Register);
 9343   %}
 9344   ins_pipe(ialu_reg_reg);
 9345 %}
 9346 
 9347 // And Register with Immediate 255
 9348 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
 9349 %{
 9350   match(Set dst (AndL src mask));
 9351 
 9352   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
 9353   ins_encode %{
 9354     // movzbl zeroes out the upper 32-bit and does not need REX.W
 9355     __ movzbl($dst$$Register, $src$$Register);
 9356   %}
 9357   ins_pipe(ialu_reg);
 9358 %}
 9359 
 9360 // And Register with Immediate 65535
 9361 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
 9362 %{
 9363   match(Set dst (AndL src mask));
 9364 
 9365   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
 9366   ins_encode %{
 9367     // movzwl zeroes out the upper 32-bit and does not need REX.W
 9368     __ movzwl($dst$$Register, $src$$Register);
 9369   %}
 9370   ins_pipe(ialu_reg);
 9371 %}
 9372 
 9373 // And Register with Immediate
 9374 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9375 %{
 9376   match(Set dst (AndL dst src));
 9377   effect(KILL cr);
 9378   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9379 
 9380   format %{ "andq    $dst, $src\t# long" %}
 9381   ins_encode %{
 9382     __ andq($dst$$Register, $src$$constant);
 9383   %}
 9384   ins_pipe(ialu_reg);
 9385 %}
 9386 
 9387 // And Register with Memory
 9388 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9389 %{
 9390   match(Set dst (AndL dst (LoadL src)));
 9391   effect(KILL cr);
 9392   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9393 
 9394   ins_cost(150);
 9395   format %{ "andq    $dst, $src\t# long" %}
 9396   ins_encode %{
 9397     __ andq($dst$$Register, $src$$Address);
 9398   %}
 9399   ins_pipe(ialu_reg_mem);
 9400 %}
 9401 
 9402 // And Memory with Register
 9403 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9404 %{
 9405   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9406   effect(KILL cr);
 9407   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9408 
 9409   ins_cost(150);
 9410   format %{ "andq    $dst, $src\t# long" %}
 9411   ins_encode %{
 9412     __ andq($dst$$Address, $src$$Register);
 9413   %}
 9414   ins_pipe(ialu_mem_reg);
 9415 %}
 9416 
 9417 // And Memory with Immediate
 9418 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9419 %{
 9420   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9421   effect(KILL cr);
 9422   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9423 
 9424   ins_cost(125);
 9425   format %{ "andq    $dst, $src\t# long" %}
 9426   ins_encode %{
 9427     __ andq($dst$$Address, $src$$constant);
 9428   %}
 9429   ins_pipe(ialu_mem_imm);
 9430 %}
 9431 
 9432 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9433 %{
 9434   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9435   // because AND/OR works well enough for 8/32-bit values.
 9436   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9437 
 9438   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9439   effect(KILL cr);
 9440 
 9441   ins_cost(125);
 9442   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9443   ins_encode %{
 9444     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9445   %}
 9446   ins_pipe(ialu_mem_imm);
 9447 %}
 9448 
 9449 // BMI1 instructions
 9450 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9451   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9452   predicate(UseBMI1Instructions);
 9453   effect(KILL cr);
 9454   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9455 
 9456   ins_cost(125);
 9457   format %{ "andnq  $dst, $src1, $src2" %}
 9458 
 9459   ins_encode %{
 9460     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9461   %}
 9462   ins_pipe(ialu_reg_mem);
 9463 %}
 9464 
 9465 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9466   match(Set dst (AndL (XorL src1 minus_1) src2));
 9467   predicate(UseBMI1Instructions);
 9468   effect(KILL cr);
 9469   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9470 
 9471   format %{ "andnq  $dst, $src1, $src2" %}
 9472 
 9473   ins_encode %{
 9474   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9475   %}
 9476   ins_pipe(ialu_reg_mem);
 9477 %}
 9478 
 9479 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9480   match(Set dst (AndL (SubL imm_zero src) src));
 9481   predicate(UseBMI1Instructions);
 9482   effect(KILL cr);
 9483   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9484 
 9485   format %{ "blsiq  $dst, $src" %}
 9486 
 9487   ins_encode %{
 9488     __ blsiq($dst$$Register, $src$$Register);
 9489   %}
 9490   ins_pipe(ialu_reg);
 9491 %}
 9492 
 9493 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9494   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9495   predicate(UseBMI1Instructions);
 9496   effect(KILL cr);
 9497   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9498 
 9499   ins_cost(125);
 9500   format %{ "blsiq  $dst, $src" %}
 9501 
 9502   ins_encode %{
 9503     __ blsiq($dst$$Register, $src$$Address);
 9504   %}
 9505   ins_pipe(ialu_reg_mem);
 9506 %}
 9507 
 9508 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9509 %{
 9510   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9511   predicate(UseBMI1Instructions);
 9512   effect(KILL cr);
 9513   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9514 
 9515   ins_cost(125);
 9516   format %{ "blsmskq $dst, $src" %}
 9517 
 9518   ins_encode %{
 9519     __ blsmskq($dst$$Register, $src$$Address);
 9520   %}
 9521   ins_pipe(ialu_reg_mem);
 9522 %}
 9523 
 9524 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9525 %{
 9526   match(Set dst (XorL (AddL src minus_1) src));
 9527   predicate(UseBMI1Instructions);
 9528   effect(KILL cr);
 9529   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9530 
 9531   format %{ "blsmskq $dst, $src" %}
 9532 
 9533   ins_encode %{
 9534     __ blsmskq($dst$$Register, $src$$Register);
 9535   %}
 9536 
 9537   ins_pipe(ialu_reg);
 9538 %}
 9539 
 9540 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9541 %{
 9542   match(Set dst (AndL (AddL src minus_1) src) );
 9543   predicate(UseBMI1Instructions);
 9544   effect(KILL cr);
 9545   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9546 
 9547   format %{ "blsrq  $dst, $src" %}
 9548 
 9549   ins_encode %{
 9550     __ blsrq($dst$$Register, $src$$Register);
 9551   %}
 9552 
 9553   ins_pipe(ialu_reg);
 9554 %}
 9555 
 9556 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9557 %{
 9558   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9559   predicate(UseBMI1Instructions);
 9560   effect(KILL cr);
 9561   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9562 
 9563   ins_cost(125);
 9564   format %{ "blsrq  $dst, $src" %}
 9565 
 9566   ins_encode %{
 9567     __ blsrq($dst$$Register, $src$$Address);
 9568   %}
 9569 
 9570   ins_pipe(ialu_reg);
 9571 %}
 9572 
 9573 // Or Instructions
 9574 // Or Register with Register
 9575 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9576 %{
 9577   match(Set dst (OrL dst src));
 9578   effect(KILL cr);
 9579   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9580 
 9581   format %{ "orq     $dst, $src\t# long" %}
 9582   ins_encode %{
 9583     __ orq($dst$$Register, $src$$Register);
 9584   %}
 9585   ins_pipe(ialu_reg_reg);
 9586 %}
 9587 
 9588 // Use any_RegP to match R15 (TLS register) without spilling.
 9589 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 9590   match(Set dst (OrL dst (CastP2X src)));
 9591   effect(KILL cr);
 9592   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9593 
 9594   format %{ "orq     $dst, $src\t# long" %}
 9595   ins_encode %{
 9596     __ orq($dst$$Register, $src$$Register);
 9597   %}
 9598   ins_pipe(ialu_reg_reg);
 9599 %}
 9600 
 9601 
 9602 // Or Register with Immediate
 9603 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9604 %{
 9605   match(Set dst (OrL dst src));
 9606   effect(KILL cr);
 9607   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9608 
 9609   format %{ "orq     $dst, $src\t# long" %}
 9610   ins_encode %{
 9611     __ orq($dst$$Register, $src$$constant);
 9612   %}
 9613   ins_pipe(ialu_reg);
 9614 %}
 9615 
 9616 // Or Register with Memory
 9617 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9618 %{
 9619   match(Set dst (OrL dst (LoadL src)));
 9620   effect(KILL cr);
 9621   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9622 
 9623   ins_cost(150);
 9624   format %{ "orq     $dst, $src\t# long" %}
 9625   ins_encode %{
 9626     __ orq($dst$$Register, $src$$Address);
 9627   %}
 9628   ins_pipe(ialu_reg_mem);
 9629 %}
 9630 
 9631 // Or Memory with Register
 9632 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9633 %{
 9634   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9635   effect(KILL cr);
 9636   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9637 
 9638   ins_cost(150);
 9639   format %{ "orq     $dst, $src\t# long" %}
 9640   ins_encode %{
 9641     __ orq($dst$$Address, $src$$Register);
 9642   %}
 9643   ins_pipe(ialu_mem_reg);
 9644 %}
 9645 
 9646 // Or Memory with Immediate
 9647 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9648 %{
 9649   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9650   effect(KILL cr);
 9651   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9652 
 9653   ins_cost(125);
 9654   format %{ "orq     $dst, $src\t# long" %}
 9655   ins_encode %{
 9656     __ orq($dst$$Address, $src$$constant);
 9657   %}
 9658   ins_pipe(ialu_mem_imm);
 9659 %}
 9660 
 9661 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
 9662 %{
 9663   // con should be a pure 64-bit power of 2 immediate
 9664   // because AND/OR works well enough for 8/32-bit values.
 9665   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
 9666 
 9667   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
 9668   effect(KILL cr);
 9669 
 9670   ins_cost(125);
 9671   format %{ "btsq    $dst, log2($con)\t# long" %}
 9672   ins_encode %{
 9673     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
 9674   %}
 9675   ins_pipe(ialu_mem_imm);
 9676 %}
 9677 
 9678 // Xor Instructions
 9679 // Xor Register with Register
 9680 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9681 %{
 9682   match(Set dst (XorL dst src));
 9683   effect(KILL cr);
 9684   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9685 
 9686   format %{ "xorq    $dst, $src\t# long" %}
 9687   ins_encode %{
 9688     __ xorq($dst$$Register, $src$$Register);
 9689   %}
 9690   ins_pipe(ialu_reg_reg);
 9691 %}
 9692 
 9693 // Xor Register with Immediate -1
 9694 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 9695   match(Set dst (XorL dst imm));
 9696 
 9697   format %{ "notq   $dst" %}
 9698   ins_encode %{
 9699      __ notq($dst$$Register);
 9700   %}
 9701   ins_pipe(ialu_reg);
 9702 %}
 9703 
 9704 // Xor Register with Immediate
 9705 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9706 %{
 9707   match(Set dst (XorL dst src));
 9708   effect(KILL cr);
 9709   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9710 
 9711   format %{ "xorq    $dst, $src\t# long" %}
 9712   ins_encode %{
 9713     __ xorq($dst$$Register, $src$$constant);
 9714   %}
 9715   ins_pipe(ialu_reg);
 9716 %}
 9717 
 9718 // Xor Register with Memory
 9719 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9720 %{
 9721   match(Set dst (XorL dst (LoadL src)));
 9722   effect(KILL cr);
 9723   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9724 
 9725   ins_cost(150);
 9726   format %{ "xorq    $dst, $src\t# long" %}
 9727   ins_encode %{
 9728     __ xorq($dst$$Register, $src$$Address);
 9729   %}
 9730   ins_pipe(ialu_reg_mem);
 9731 %}
 9732 
 9733 // Xor Memory with Register
 9734 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9735 %{
 9736   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9737   effect(KILL cr);
 9738   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9739 
 9740   ins_cost(150);
 9741   format %{ "xorq    $dst, $src\t# long" %}
 9742   ins_encode %{
 9743     __ xorq($dst$$Address, $src$$Register);
 9744   %}
 9745   ins_pipe(ialu_mem_reg);
 9746 %}
 9747 
 9748 // Xor Memory with Immediate
 9749 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9750 %{
 9751   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9752   effect(KILL cr);
 9753   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9754 
 9755   ins_cost(125);
 9756   format %{ "xorq    $dst, $src\t# long" %}
 9757   ins_encode %{
 9758     __ xorq($dst$$Address, $src$$constant);
 9759   %}
 9760   ins_pipe(ialu_mem_imm);
 9761 %}
 9762 
 9763 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 9764 %{
 9765   match(Set dst (CmpLTMask p q));
 9766   effect(KILL cr);
 9767 
 9768   ins_cost(400);
 9769   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 9770             "setlt   $dst\n\t"
 9771             "movzbl  $dst, $dst\n\t"
 9772             "negl    $dst" %}
 9773   ins_encode %{
 9774     __ cmpl($p$$Register, $q$$Register);
 9775     __ setb(Assembler::less, $dst$$Register);
 9776     __ movzbl($dst$$Register, $dst$$Register);
 9777     __ negl($dst$$Register);
 9778   %}
 9779   ins_pipe(pipe_slow);
 9780 %}
 9781 
 9782 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
 9783 %{
 9784   match(Set dst (CmpLTMask dst zero));
 9785   effect(KILL cr);
 9786 
 9787   ins_cost(100);
 9788   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 9789   ins_encode %{
 9790     __ sarl($dst$$Register, 31);
 9791   %}
 9792   ins_pipe(ialu_reg);
 9793 %}
 9794 
 9795 /* Better to save a register than avoid a branch */
 9796 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9797 %{
 9798   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 9799   effect(KILL cr);
 9800   ins_cost(300);
 9801   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
 9802             "jge     done\n\t"
 9803             "addl    $p,$y\n"
 9804             "done:   " %}
 9805   ins_encode %{
 9806     Register Rp = $p$$Register;
 9807     Register Rq = $q$$Register;
 9808     Register Ry = $y$$Register;
 9809     Label done;
 9810     __ subl(Rp, Rq);
 9811     __ jccb(Assembler::greaterEqual, done);
 9812     __ addl(Rp, Ry);
 9813     __ bind(done);
 9814   %}
 9815   ins_pipe(pipe_cmplt);
 9816 %}
 9817 
 9818 /* Better to save a register than avoid a branch */
 9819 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9820 %{
 9821   match(Set y (AndI (CmpLTMask p q) y));
 9822   effect(KILL cr);
 9823 
 9824   ins_cost(300);
 9825 
 9826   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
 9827             "jlt     done\n\t"
 9828             "xorl    $y, $y\n"
 9829             "done:   " %}
 9830   ins_encode %{
 9831     Register Rp = $p$$Register;
 9832     Register Rq = $q$$Register;
 9833     Register Ry = $y$$Register;
 9834     Label done;
 9835     __ cmpl(Rp, Rq);
 9836     __ jccb(Assembler::less, done);
 9837     __ xorl(Ry, Ry);
 9838     __ bind(done);
 9839   %}
 9840   ins_pipe(pipe_cmplt);
 9841 %}
 9842 
 9843 
 9844 //---------- FP Instructions------------------------------------------------
 9845 
 9846 // Really expensive, avoid
 9847 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 9848 %{
 9849   match(Set cr (CmpF src1 src2));
 9850 
 9851   ins_cost(500);
 9852   format %{ "ucomiss $src1, $src2\n\t"
 9853             "jnp,s   exit\n\t"
 9854             "pushfq\t# saw NaN, set CF\n\t"
 9855             "andq    [rsp], #0xffffff2b\n\t"
 9856             "popfq\n"
 9857     "exit:" %}
 9858   ins_encode %{
 9859     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9860     emit_cmpfp_fixup(_masm);
 9861   %}
 9862   ins_pipe(pipe_slow);
 9863 %}
 9864 
 9865 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 9866   match(Set cr (CmpF src1 src2));
 9867 
 9868   ins_cost(100);
 9869   format %{ "ucomiss $src1, $src2" %}
 9870   ins_encode %{
 9871     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9872   %}
 9873   ins_pipe(pipe_slow);
 9874 %}
 9875 
 9876 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 9877   match(Set cr (CmpF src1 (LoadF src2)));
 9878 
 9879   ins_cost(100);
 9880   format %{ "ucomiss $src1, $src2" %}
 9881   ins_encode %{
 9882     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9883   %}
 9884   ins_pipe(pipe_slow);
 9885 %}
 9886 
 9887 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
 9888   match(Set cr (CmpF src con));
 9889   ins_cost(100);
 9890   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
 9891   ins_encode %{
 9892     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9893   %}
 9894   ins_pipe(pipe_slow);
 9895 %}
 9896 
 9897 // Really expensive, avoid
 9898 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 9899 %{
 9900   match(Set cr (CmpD src1 src2));
 9901 
 9902   ins_cost(500);
 9903   format %{ "ucomisd $src1, $src2\n\t"
 9904             "jnp,s   exit\n\t"
 9905             "pushfq\t# saw NaN, set CF\n\t"
 9906             "andq    [rsp], #0xffffff2b\n\t"
 9907             "popfq\n"
 9908     "exit:" %}
 9909   ins_encode %{
 9910     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9911     emit_cmpfp_fixup(_masm);
 9912   %}
 9913   ins_pipe(pipe_slow);
 9914 %}
 9915 
 9916 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 9917   match(Set cr (CmpD src1 src2));
 9918 
 9919   ins_cost(100);
 9920   format %{ "ucomisd $src1, $src2 test" %}
 9921   ins_encode %{
 9922     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9923   %}
 9924   ins_pipe(pipe_slow);
 9925 %}
 9926 
 9927 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 9928   match(Set cr (CmpD src1 (LoadD src2)));
 9929 
 9930   ins_cost(100);
 9931   format %{ "ucomisd $src1, $src2" %}
 9932   ins_encode %{
 9933     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9934   %}
 9935   ins_pipe(pipe_slow);
 9936 %}
 9937 
 9938 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
 9939   match(Set cr (CmpD src con));
 9940   ins_cost(100);
 9941   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
 9942   ins_encode %{
 9943     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9944   %}
 9945   ins_pipe(pipe_slow);
 9946 %}
 9947 
 9948 // Compare into -1,0,1
 9949 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
 9950 %{
 9951   match(Set dst (CmpF3 src1 src2));
 9952   effect(KILL cr);
 9953 
 9954   ins_cost(275);
 9955   format %{ "ucomiss $src1, $src2\n\t"
 9956             "movl    $dst, #-1\n\t"
 9957             "jp,s    done\n\t"
 9958             "jb,s    done\n\t"
 9959             "setne   $dst\n\t"
 9960             "movzbl  $dst, $dst\n"
 9961     "done:" %}
 9962   ins_encode %{
 9963     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9964     emit_cmpfp3(_masm, $dst$$Register);
 9965   %}
 9966   ins_pipe(pipe_slow);
 9967 %}
 9968 
 9969 // Compare into -1,0,1
 9970 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 9971 %{
 9972   match(Set dst (CmpF3 src1 (LoadF src2)));
 9973   effect(KILL cr);
 9974 
 9975   ins_cost(275);
 9976   format %{ "ucomiss $src1, $src2\n\t"
 9977             "movl    $dst, #-1\n\t"
 9978             "jp,s    done\n\t"
 9979             "jb,s    done\n\t"
 9980             "setne   $dst\n\t"
 9981             "movzbl  $dst, $dst\n"
 9982     "done:" %}
 9983   ins_encode %{
 9984     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9985     emit_cmpfp3(_masm, $dst$$Register);
 9986   %}
 9987   ins_pipe(pipe_slow);
 9988 %}
 9989 
 9990 // Compare into -1,0,1
 9991 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
 9992   match(Set dst (CmpF3 src con));
 9993   effect(KILL cr);
 9994 
 9995   ins_cost(275);
 9996   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
 9997             "movl    $dst, #-1\n\t"
 9998             "jp,s    done\n\t"
 9999             "jb,s    done\n\t"
10000             "setne   $dst\n\t"
10001             "movzbl  $dst, $dst\n"
10002     "done:" %}
10003   ins_encode %{
10004     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10005     emit_cmpfp3(_masm, $dst$$Register);
10006   %}
10007   ins_pipe(pipe_slow);
10008 %}
10009 
10010 // Compare into -1,0,1
10011 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10012 %{
10013   match(Set dst (CmpD3 src1 src2));
10014   effect(KILL cr);
10015 
10016   ins_cost(275);
10017   format %{ "ucomisd $src1, $src2\n\t"
10018             "movl    $dst, #-1\n\t"
10019             "jp,s    done\n\t"
10020             "jb,s    done\n\t"
10021             "setne   $dst\n\t"
10022             "movzbl  $dst, $dst\n"
10023     "done:" %}
10024   ins_encode %{
10025     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10026     emit_cmpfp3(_masm, $dst$$Register);
10027   %}
10028   ins_pipe(pipe_slow);
10029 %}
10030 
10031 // Compare into -1,0,1
10032 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10033 %{
10034   match(Set dst (CmpD3 src1 (LoadD src2)));
10035   effect(KILL cr);
10036 
10037   ins_cost(275);
10038   format %{ "ucomisd $src1, $src2\n\t"
10039             "movl    $dst, #-1\n\t"
10040             "jp,s    done\n\t"
10041             "jb,s    done\n\t"
10042             "setne   $dst\n\t"
10043             "movzbl  $dst, $dst\n"
10044     "done:" %}
10045   ins_encode %{
10046     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10047     emit_cmpfp3(_masm, $dst$$Register);
10048   %}
10049   ins_pipe(pipe_slow);
10050 %}
10051 
10052 // Compare into -1,0,1
10053 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10054   match(Set dst (CmpD3 src con));
10055   effect(KILL cr);
10056 
10057   ins_cost(275);
10058   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10059             "movl    $dst, #-1\n\t"
10060             "jp,s    done\n\t"
10061             "jb,s    done\n\t"
10062             "setne   $dst\n\t"
10063             "movzbl  $dst, $dst\n"
10064     "done:" %}
10065   ins_encode %{
10066     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10067     emit_cmpfp3(_masm, $dst$$Register);
10068   %}
10069   ins_pipe(pipe_slow);
10070 %}
10071 
10072 //----------Arithmetic Conversion Instructions---------------------------------
10073 
10074 instruct convF2D_reg_reg(regD dst, regF src)
10075 %{
10076   match(Set dst (ConvF2D src));
10077 
10078   format %{ "cvtss2sd $dst, $src" %}
10079   ins_encode %{
10080     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10081   %}
10082   ins_pipe(pipe_slow); // XXX
10083 %}
10084 
10085 instruct convF2D_reg_mem(regD dst, memory src)
10086 %{
10087   match(Set dst (ConvF2D (LoadF src)));
10088 
10089   format %{ "cvtss2sd $dst, $src" %}
10090   ins_encode %{
10091     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10092   %}
10093   ins_pipe(pipe_slow); // XXX
10094 %}
10095 
10096 instruct convD2F_reg_reg(regF dst, regD src)
10097 %{
10098   match(Set dst (ConvD2F src));
10099 
10100   format %{ "cvtsd2ss $dst, $src" %}
10101   ins_encode %{
10102     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10103   %}
10104   ins_pipe(pipe_slow); // XXX
10105 %}
10106 
10107 instruct convD2F_reg_mem(regF dst, memory src)
10108 %{
10109   match(Set dst (ConvD2F (LoadD src)));
10110 
10111   format %{ "cvtsd2ss $dst, $src" %}
10112   ins_encode %{
10113     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10114   %}
10115   ins_pipe(pipe_slow); // XXX
10116 %}
10117 
10118 // XXX do mem variants
10119 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10120 %{
10121   match(Set dst (ConvF2I src));
10122   effect(KILL cr);
10123   format %{ "convert_f2i $dst, $src" %}
10124   ins_encode %{
10125     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10126   %}
10127   ins_pipe(pipe_slow);
10128 %}
10129 
10130 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10131 %{
10132   match(Set dst (ConvF2L src));
10133   effect(KILL cr);
10134   format %{ "convert_f2l $dst, $src"%}
10135   ins_encode %{
10136     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10137   %}
10138   ins_pipe(pipe_slow);
10139 %}
10140 
10141 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10142 %{
10143   match(Set dst (ConvD2I src));
10144   effect(KILL cr);
10145   format %{ "convert_d2i $dst, $src"%}
10146   ins_encode %{
10147     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10148   %}
10149   ins_pipe(pipe_slow);
10150 %}
10151 
10152 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10153 %{
10154   match(Set dst (ConvD2L src));
10155   effect(KILL cr);
10156   format %{ "convert_d2l $dst, $src"%}
10157   ins_encode %{
10158     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10159   %}
10160   ins_pipe(pipe_slow);
10161 %}
10162 
10163 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10164 %{
10165   match(Set dst (RoundD src));
10166   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10167   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10168   ins_encode %{
10169     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10170   %}
10171   ins_pipe(pipe_slow);
10172 %}
10173 
10174 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10175 %{
10176   match(Set dst (RoundF src));
10177   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10178   format %{ "round_float $dst,$src" %}
10179   ins_encode %{
10180     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10181   %}
10182   ins_pipe(pipe_slow);
10183 %}
10184 
10185 instruct convI2F_reg_reg(regF dst, rRegI src)
10186 %{
10187   predicate(!UseXmmI2F);
10188   match(Set dst (ConvI2F src));
10189 
10190   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10191   ins_encode %{
10192     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10193   %}
10194   ins_pipe(pipe_slow); // XXX
10195 %}
10196 
10197 instruct convI2F_reg_mem(regF dst, memory src)
10198 %{
10199   match(Set dst (ConvI2F (LoadI src)));
10200 
10201   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10202   ins_encode %{
10203     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10204   %}
10205   ins_pipe(pipe_slow); // XXX
10206 %}
10207 
10208 instruct convI2D_reg_reg(regD dst, rRegI src)
10209 %{
10210   predicate(!UseXmmI2D);
10211   match(Set dst (ConvI2D src));
10212 
10213   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10214   ins_encode %{
10215     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10216   %}
10217   ins_pipe(pipe_slow); // XXX
10218 %}
10219 
10220 instruct convI2D_reg_mem(regD dst, memory src)
10221 %{
10222   match(Set dst (ConvI2D (LoadI src)));
10223 
10224   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10225   ins_encode %{
10226     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10227   %}
10228   ins_pipe(pipe_slow); // XXX
10229 %}
10230 
10231 instruct convXI2F_reg(regF dst, rRegI src)
10232 %{
10233   predicate(UseXmmI2F);
10234   match(Set dst (ConvI2F src));
10235 
10236   format %{ "movdl $dst, $src\n\t"
10237             "cvtdq2psl $dst, $dst\t# i2f" %}
10238   ins_encode %{
10239     __ movdl($dst$$XMMRegister, $src$$Register);
10240     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10241   %}
10242   ins_pipe(pipe_slow); // XXX
10243 %}
10244 
10245 instruct convXI2D_reg(regD dst, rRegI src)
10246 %{
10247   predicate(UseXmmI2D);
10248   match(Set dst (ConvI2D src));
10249 
10250   format %{ "movdl $dst, $src\n\t"
10251             "cvtdq2pdl $dst, $dst\t# i2d" %}
10252   ins_encode %{
10253     __ movdl($dst$$XMMRegister, $src$$Register);
10254     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10255   %}
10256   ins_pipe(pipe_slow); // XXX
10257 %}
10258 
10259 instruct convL2F_reg_reg(regF dst, rRegL src)
10260 %{
10261   match(Set dst (ConvL2F src));
10262 
10263   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10264   ins_encode %{
10265     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10266   %}
10267   ins_pipe(pipe_slow); // XXX
10268 %}
10269 
10270 instruct convL2F_reg_mem(regF dst, memory src)
10271 %{
10272   match(Set dst (ConvL2F (LoadL src)));
10273 
10274   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10275   ins_encode %{
10276     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10277   %}
10278   ins_pipe(pipe_slow); // XXX
10279 %}
10280 
10281 instruct convL2D_reg_reg(regD dst, rRegL src)
10282 %{
10283   match(Set dst (ConvL2D src));
10284 
10285   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10286   ins_encode %{
10287     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10288   %}
10289   ins_pipe(pipe_slow); // XXX
10290 %}
10291 
10292 instruct convL2D_reg_mem(regD dst, memory src)
10293 %{
10294   match(Set dst (ConvL2D (LoadL src)));
10295 
10296   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10297   ins_encode %{
10298     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10299   %}
10300   ins_pipe(pipe_slow); // XXX
10301 %}
10302 
10303 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10304 %{
10305   match(Set dst (ConvI2L src));
10306 
10307   ins_cost(125);
10308   format %{ "movslq  $dst, $src\t# i2l" %}
10309   ins_encode %{
10310     __ movslq($dst$$Register, $src$$Register);
10311   %}
10312   ins_pipe(ialu_reg_reg);
10313 %}
10314 
10315 // Zero-extend convert int to long
10316 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10317 %{
10318   match(Set dst (AndL (ConvI2L src) mask));
10319 
10320   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10321   ins_encode %{
10322     if ($dst$$reg != $src$$reg) {
10323       __ movl($dst$$Register, $src$$Register);
10324     }
10325   %}
10326   ins_pipe(ialu_reg_reg);
10327 %}
10328 
10329 // Zero-extend convert int to long
10330 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10331 %{
10332   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10333 
10334   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10335   ins_encode %{
10336     __ movl($dst$$Register, $src$$Address);
10337   %}
10338   ins_pipe(ialu_reg_mem);
10339 %}
10340 
10341 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10342 %{
10343   match(Set dst (AndL src mask));
10344 
10345   format %{ "movl    $dst, $src\t# zero-extend long" %}
10346   ins_encode %{
10347     __ movl($dst$$Register, $src$$Register);
10348   %}
10349   ins_pipe(ialu_reg_reg);
10350 %}
10351 
10352 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10353 %{
10354   match(Set dst (ConvL2I src));
10355 
10356   format %{ "movl    $dst, $src\t# l2i" %}
10357   ins_encode %{
10358     __ movl($dst$$Register, $src$$Register);
10359   %}
10360   ins_pipe(ialu_reg_reg);
10361 %}
10362 
10363 
10364 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10365   match(Set dst (MoveF2I src));
10366   effect(DEF dst, USE src);
10367 
10368   ins_cost(125);
10369   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10370   ins_encode %{
10371     __ movl($dst$$Register, Address(rsp, $src$$disp));
10372   %}
10373   ins_pipe(ialu_reg_mem);
10374 %}
10375 
10376 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10377   match(Set dst (MoveI2F src));
10378   effect(DEF dst, USE src);
10379 
10380   ins_cost(125);
10381   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10382   ins_encode %{
10383     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10384   %}
10385   ins_pipe(pipe_slow);
10386 %}
10387 
10388 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10389   match(Set dst (MoveD2L src));
10390   effect(DEF dst, USE src);
10391 
10392   ins_cost(125);
10393   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10394   ins_encode %{
10395     __ movq($dst$$Register, Address(rsp, $src$$disp));
10396   %}
10397   ins_pipe(ialu_reg_mem);
10398 %}
10399 
10400 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10401   predicate(!UseXmmLoadAndClearUpper);
10402   match(Set dst (MoveL2D src));
10403   effect(DEF dst, USE src);
10404 
10405   ins_cost(125);
10406   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10407   ins_encode %{
10408     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10409   %}
10410   ins_pipe(pipe_slow);
10411 %}
10412 
10413 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10414   predicate(UseXmmLoadAndClearUpper);
10415   match(Set dst (MoveL2D src));
10416   effect(DEF dst, USE src);
10417 
10418   ins_cost(125);
10419   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10420   ins_encode %{
10421     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10422   %}
10423   ins_pipe(pipe_slow);
10424 %}
10425 
10426 
10427 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10428   match(Set dst (MoveF2I src));
10429   effect(DEF dst, USE src);
10430 
10431   ins_cost(95); // XXX
10432   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10433   ins_encode %{
10434     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10435   %}
10436   ins_pipe(pipe_slow);
10437 %}
10438 
10439 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10440   match(Set dst (MoveI2F src));
10441   effect(DEF dst, USE src);
10442 
10443   ins_cost(100);
10444   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10445   ins_encode %{
10446     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10447   %}
10448   ins_pipe( ialu_mem_reg );
10449 %}
10450 
10451 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10452   match(Set dst (MoveD2L src));
10453   effect(DEF dst, USE src);
10454 
10455   ins_cost(95); // XXX
10456   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10457   ins_encode %{
10458     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10459   %}
10460   ins_pipe(pipe_slow);
10461 %}
10462 
10463 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10464   match(Set dst (MoveL2D src));
10465   effect(DEF dst, USE src);
10466 
10467   ins_cost(100);
10468   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10469   ins_encode %{
10470     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10471   %}
10472   ins_pipe(ialu_mem_reg);
10473 %}
10474 
10475 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10476   match(Set dst (MoveF2I src));
10477   effect(DEF dst, USE src);
10478   ins_cost(85);
10479   format %{ "movd    $dst,$src\t# MoveF2I" %}
10480   ins_encode %{
10481     __ movdl($dst$$Register, $src$$XMMRegister);
10482   %}
10483   ins_pipe( pipe_slow );
10484 %}
10485 
10486 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10487   match(Set dst (MoveD2L src));
10488   effect(DEF dst, USE src);
10489   ins_cost(85);
10490   format %{ "movd    $dst,$src\t# MoveD2L" %}
10491   ins_encode %{
10492     __ movdq($dst$$Register, $src$$XMMRegister);
10493   %}
10494   ins_pipe( pipe_slow );
10495 %}
10496 
10497 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10498   match(Set dst (MoveI2F src));
10499   effect(DEF dst, USE src);
10500   ins_cost(100);
10501   format %{ "movd    $dst,$src\t# MoveI2F" %}
10502   ins_encode %{
10503     __ movdl($dst$$XMMRegister, $src$$Register);
10504   %}
10505   ins_pipe( pipe_slow );
10506 %}
10507 
10508 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10509   match(Set dst (MoveL2D src));
10510   effect(DEF dst, USE src);
10511   ins_cost(100);
10512   format %{ "movd    $dst,$src\t# MoveL2D" %}
10513   ins_encode %{
10514      __ movdq($dst$$XMMRegister, $src$$Register);
10515   %}
10516   ins_pipe( pipe_slow );
10517 %}
10518 
10519 // Fast clearing of an array
10520 // Small ClearArray non-AVX512.
10521 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10522                   Universe dummy, rFlagsReg cr)
10523 %{
10524   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
10525   match(Set dummy (ClearArray cnt base));
10526   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10527 
10528   format %{ $$template
10529     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10530     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10531     $$emit$$"jg      LARGE\n\t"
10532     $$emit$$"dec     rcx\n\t"
10533     $$emit$$"js      DONE\t# Zero length\n\t"
10534     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10535     $$emit$$"dec     rcx\n\t"
10536     $$emit$$"jge     LOOP\n\t"
10537     $$emit$$"jmp     DONE\n\t"
10538     $$emit$$"# LARGE:\n\t"
10539     if (UseFastStosb) {
10540        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10541        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10542     } else if (UseXMMForObjInit) {
10543        $$emit$$"mov     rdi,rax\n\t"
10544        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10545        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10546        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10547        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10548        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10549        $$emit$$"add     0x40,rax\n\t"
10550        $$emit$$"# L_zero_64_bytes:\n\t"
10551        $$emit$$"sub     0x8,rcx\n\t"
10552        $$emit$$"jge     L_loop\n\t"
10553        $$emit$$"add     0x4,rcx\n\t"
10554        $$emit$$"jl      L_tail\n\t"
10555        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10556        $$emit$$"add     0x20,rax\n\t"
10557        $$emit$$"sub     0x4,rcx\n\t"
10558        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10559        $$emit$$"add     0x4,rcx\n\t"
10560        $$emit$$"jle     L_end\n\t"
10561        $$emit$$"dec     rcx\n\t"
10562        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10563        $$emit$$"vmovq   xmm0,(rax)\n\t"
10564        $$emit$$"add     0x8,rax\n\t"
10565        $$emit$$"dec     rcx\n\t"
10566        $$emit$$"jge     L_sloop\n\t"
10567        $$emit$$"# L_end:\n\t"
10568     } else {
10569        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10570     }
10571     $$emit$$"# DONE"
10572   %}
10573   ins_encode %{
10574     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10575                  $tmp$$XMMRegister, false, knoreg);
10576   %}
10577   ins_pipe(pipe_slow);
10578 %}
10579 
10580 // Small ClearArray AVX512 non-constant length.
10581 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10582                        Universe dummy, rFlagsReg cr)
10583 %{
10584   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
10585   match(Set dummy (ClearArray cnt base));
10586   ins_cost(125);
10587   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10588 
10589   format %{ $$template
10590     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10591     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10592     $$emit$$"jg      LARGE\n\t"
10593     $$emit$$"dec     rcx\n\t"
10594     $$emit$$"js      DONE\t# Zero length\n\t"
10595     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10596     $$emit$$"dec     rcx\n\t"
10597     $$emit$$"jge     LOOP\n\t"
10598     $$emit$$"jmp     DONE\n\t"
10599     $$emit$$"# LARGE:\n\t"
10600     if (UseFastStosb) {
10601        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10602        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10603     } else if (UseXMMForObjInit) {
10604        $$emit$$"mov     rdi,rax\n\t"
10605        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10606        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10607        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10608        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10609        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10610        $$emit$$"add     0x40,rax\n\t"
10611        $$emit$$"# L_zero_64_bytes:\n\t"
10612        $$emit$$"sub     0x8,rcx\n\t"
10613        $$emit$$"jge     L_loop\n\t"
10614        $$emit$$"add     0x4,rcx\n\t"
10615        $$emit$$"jl      L_tail\n\t"
10616        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10617        $$emit$$"add     0x20,rax\n\t"
10618        $$emit$$"sub     0x4,rcx\n\t"
10619        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10620        $$emit$$"add     0x4,rcx\n\t"
10621        $$emit$$"jle     L_end\n\t"
10622        $$emit$$"dec     rcx\n\t"
10623        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10624        $$emit$$"vmovq   xmm0,(rax)\n\t"
10625        $$emit$$"add     0x8,rax\n\t"
10626        $$emit$$"dec     rcx\n\t"
10627        $$emit$$"jge     L_sloop\n\t"
10628        $$emit$$"# L_end:\n\t"
10629     } else {
10630        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10631     }
10632     $$emit$$"# DONE"
10633   %}
10634   ins_encode %{
10635     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10636                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
10637   %}
10638   ins_pipe(pipe_slow);
10639 %}
10640 
10641 // Large ClearArray non-AVX512.
10642 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10643                         Universe dummy, rFlagsReg cr)
10644 %{
10645   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
10646   match(Set dummy (ClearArray cnt base));
10647   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10648 
10649   format %{ $$template
10650     if (UseFastStosb) {
10651        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10652        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10653        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10654     } else if (UseXMMForObjInit) {
10655        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10656        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10657        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10658        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10659        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10660        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10661        $$emit$$"add     0x40,rax\n\t"
10662        $$emit$$"# L_zero_64_bytes:\n\t"
10663        $$emit$$"sub     0x8,rcx\n\t"
10664        $$emit$$"jge     L_loop\n\t"
10665        $$emit$$"add     0x4,rcx\n\t"
10666        $$emit$$"jl      L_tail\n\t"
10667        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10668        $$emit$$"add     0x20,rax\n\t"
10669        $$emit$$"sub     0x4,rcx\n\t"
10670        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10671        $$emit$$"add     0x4,rcx\n\t"
10672        $$emit$$"jle     L_end\n\t"
10673        $$emit$$"dec     rcx\n\t"
10674        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10675        $$emit$$"vmovq   xmm0,(rax)\n\t"
10676        $$emit$$"add     0x8,rax\n\t"
10677        $$emit$$"dec     rcx\n\t"
10678        $$emit$$"jge     L_sloop\n\t"
10679        $$emit$$"# L_end:\n\t"
10680     } else {
10681        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10682        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10683     }
10684   %}
10685   ins_encode %{
10686     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10687                  $tmp$$XMMRegister, true, knoreg);
10688   %}
10689   ins_pipe(pipe_slow);
10690 %}
10691 
10692 // Large ClearArray AVX512.
10693 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10694                              Universe dummy, rFlagsReg cr)
10695 %{
10696   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
10697   match(Set dummy (ClearArray cnt base));
10698   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10699 
10700   format %{ $$template
10701     if (UseFastStosb) {
10702        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10703        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10704        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10705     } else if (UseXMMForObjInit) {
10706        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10707        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10708        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10709        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10710        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10711        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10712        $$emit$$"add     0x40,rax\n\t"
10713        $$emit$$"# L_zero_64_bytes:\n\t"
10714        $$emit$$"sub     0x8,rcx\n\t"
10715        $$emit$$"jge     L_loop\n\t"
10716        $$emit$$"add     0x4,rcx\n\t"
10717        $$emit$$"jl      L_tail\n\t"
10718        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10719        $$emit$$"add     0x20,rax\n\t"
10720        $$emit$$"sub     0x4,rcx\n\t"
10721        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10722        $$emit$$"add     0x4,rcx\n\t"
10723        $$emit$$"jle     L_end\n\t"
10724        $$emit$$"dec     rcx\n\t"
10725        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10726        $$emit$$"vmovq   xmm0,(rax)\n\t"
10727        $$emit$$"add     0x8,rax\n\t"
10728        $$emit$$"dec     rcx\n\t"
10729        $$emit$$"jge     L_sloop\n\t"
10730        $$emit$$"# L_end:\n\t"
10731     } else {
10732        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10733        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10734     }
10735   %}
10736   ins_encode %{
10737     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10738                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
10739   %}
10740   ins_pipe(pipe_slow);
10741 %}
10742 
10743 // Small ClearArray AVX512 constant length.
10744 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
10745 %{
10746   predicate(!((ClearArrayNode*)n)->is_large() &&
10747               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
10748   match(Set dummy (ClearArray cnt base));
10749   ins_cost(100);
10750   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
10751   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
10752   ins_encode %{
10753    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10754   %}
10755   ins_pipe(pipe_slow);
10756 %}
10757 
10758 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10759                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10760 %{
10761   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10762   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10763   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10764 
10765   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10766   ins_encode %{
10767     __ string_compare($str1$$Register, $str2$$Register,
10768                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10769                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
10770   %}
10771   ins_pipe( pipe_slow );
10772 %}
10773 
10774 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10775                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10776 %{
10777   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10778   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10779   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10780 
10781   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10782   ins_encode %{
10783     __ string_compare($str1$$Register, $str2$$Register,
10784                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10785                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
10786   %}
10787   ins_pipe( pipe_slow );
10788 %}
10789 
10790 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10791                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10792 %{
10793   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10794   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10795   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10796 
10797   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10798   ins_encode %{
10799     __ string_compare($str1$$Register, $str2$$Register,
10800                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10801                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
10802   %}
10803   ins_pipe( pipe_slow );
10804 %}
10805 
10806 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10807                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10808 %{
10809   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10810   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10811   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10812 
10813   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10814   ins_encode %{
10815     __ string_compare($str1$$Register, $str2$$Register,
10816                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10817                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
10818   %}
10819   ins_pipe( pipe_slow );
10820 %}
10821 
10822 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10823                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10824 %{
10825   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10826   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10827   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10828 
10829   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10830   ins_encode %{
10831     __ string_compare($str1$$Register, $str2$$Register,
10832                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10833                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
10834   %}
10835   ins_pipe( pipe_slow );
10836 %}
10837 
10838 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10839                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10840 %{
10841   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10842   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10843   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10844 
10845   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10846   ins_encode %{
10847     __ string_compare($str1$$Register, $str2$$Register,
10848                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10849                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
10850   %}
10851   ins_pipe( pipe_slow );
10852 %}
10853 
10854 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10855                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10856 %{
10857   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10858   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10859   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10860 
10861   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10862   ins_encode %{
10863     __ string_compare($str2$$Register, $str1$$Register,
10864                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10865                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
10866   %}
10867   ins_pipe( pipe_slow );
10868 %}
10869 
10870 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10871                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10872 %{
10873   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10874   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10875   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10876 
10877   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10878   ins_encode %{
10879     __ string_compare($str2$$Register, $str1$$Register,
10880                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10881                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
10882   %}
10883   ins_pipe( pipe_slow );
10884 %}
10885 
10886 // fast search of substring with known size.
10887 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10888                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10889 %{
10890   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10891   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10892   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10893 
10894   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10895   ins_encode %{
10896     int icnt2 = (int)$int_cnt2$$constant;
10897     if (icnt2 >= 16) {
10898       // IndexOf for constant substrings with size >= 16 elements
10899       // which don't need to be loaded through stack.
10900       __ string_indexofC8($str1$$Register, $str2$$Register,
10901                           $cnt1$$Register, $cnt2$$Register,
10902                           icnt2, $result$$Register,
10903                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10904     } else {
10905       // Small strings are loaded through stack if they cross page boundary.
10906       __ string_indexof($str1$$Register, $str2$$Register,
10907                         $cnt1$$Register, $cnt2$$Register,
10908                         icnt2, $result$$Register,
10909                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10910     }
10911   %}
10912   ins_pipe( pipe_slow );
10913 %}
10914 
10915 // fast search of substring with known size.
10916 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10917                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10918 %{
10919   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10920   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10921   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10922 
10923   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10924   ins_encode %{
10925     int icnt2 = (int)$int_cnt2$$constant;
10926     if (icnt2 >= 8) {
10927       // IndexOf for constant substrings with size >= 8 elements
10928       // which don't need to be loaded through stack.
10929       __ string_indexofC8($str1$$Register, $str2$$Register,
10930                           $cnt1$$Register, $cnt2$$Register,
10931                           icnt2, $result$$Register,
10932                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10933     } else {
10934       // Small strings are loaded through stack if they cross page boundary.
10935       __ string_indexof($str1$$Register, $str2$$Register,
10936                         $cnt1$$Register, $cnt2$$Register,
10937                         icnt2, $result$$Register,
10938                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10939     }
10940   %}
10941   ins_pipe( pipe_slow );
10942 %}
10943 
10944 // fast search of substring with known size.
10945 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10946                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10947 %{
10948   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10949   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10950   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10951 
10952   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10953   ins_encode %{
10954     int icnt2 = (int)$int_cnt2$$constant;
10955     if (icnt2 >= 8) {
10956       // IndexOf for constant substrings with size >= 8 elements
10957       // which don't need to be loaded through stack.
10958       __ string_indexofC8($str1$$Register, $str2$$Register,
10959                           $cnt1$$Register, $cnt2$$Register,
10960                           icnt2, $result$$Register,
10961                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10962     } else {
10963       // Small strings are loaded through stack if they cross page boundary.
10964       __ string_indexof($str1$$Register, $str2$$Register,
10965                         $cnt1$$Register, $cnt2$$Register,
10966                         icnt2, $result$$Register,
10967                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10968     }
10969   %}
10970   ins_pipe( pipe_slow );
10971 %}
10972 
10973 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10974                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10975 %{
10976   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10977   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10978   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10979 
10980   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10981   ins_encode %{
10982     __ string_indexof($str1$$Register, $str2$$Register,
10983                       $cnt1$$Register, $cnt2$$Register,
10984                       (-1), $result$$Register,
10985                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10986   %}
10987   ins_pipe( pipe_slow );
10988 %}
10989 
10990 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10991                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10992 %{
10993   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10994   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10995   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10996 
10997   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10998   ins_encode %{
10999     __ string_indexof($str1$$Register, $str2$$Register,
11000                       $cnt1$$Register, $cnt2$$Register,
11001                       (-1), $result$$Register,
11002                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11003   %}
11004   ins_pipe( pipe_slow );
11005 %}
11006 
11007 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11008                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11009 %{
11010   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11011   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11012   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11013 
11014   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11015   ins_encode %{
11016     __ string_indexof($str1$$Register, $str2$$Register,
11017                       $cnt1$$Register, $cnt2$$Register,
11018                       (-1), $result$$Register,
11019                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11020   %}
11021   ins_pipe( pipe_slow );
11022 %}
11023 
11024 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11025                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11026 %{
11027   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11028   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11029   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11030   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11031   ins_encode %{
11032     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11033                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11034   %}
11035   ins_pipe( pipe_slow );
11036 %}
11037 
11038 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11039                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11040 %{
11041   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11042   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11043   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11044   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11045   ins_encode %{
11046     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11047                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11048   %}
11049   ins_pipe( pipe_slow );
11050 %}
11051 
11052 // fast string equals
11053 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11054                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11055 %{
11056   predicate(!VM_Version::supports_avx512vlbw());
11057   match(Set result (StrEquals (Binary str1 str2) cnt));
11058   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11059 
11060   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11061   ins_encode %{
11062     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11063                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11064                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11065   %}
11066   ins_pipe( pipe_slow );
11067 %}
11068 
11069 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11070                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11071 %{
11072   predicate(VM_Version::supports_avx512vlbw());
11073   match(Set result (StrEquals (Binary str1 str2) cnt));
11074   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11075 
11076   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11077   ins_encode %{
11078     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11079                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11080                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11081   %}
11082   ins_pipe( pipe_slow );
11083 %}
11084 
11085 // fast array equals
11086 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11087                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11088 %{
11089   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11090   match(Set result (AryEq ary1 ary2));
11091   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11092 
11093   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11094   ins_encode %{
11095     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11096                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11097                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11098   %}
11099   ins_pipe( pipe_slow );
11100 %}
11101 
11102 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11103                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11104 %{
11105   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11106   match(Set result (AryEq ary1 ary2));
11107   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11108 
11109   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11110   ins_encode %{
11111     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11112                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11113                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11114   %}
11115   ins_pipe( pipe_slow );
11116 %}
11117 
11118 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11119                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11120 %{
11121   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11122   match(Set result (AryEq ary1 ary2));
11123   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11124 
11125   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11126   ins_encode %{
11127     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11128                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11129                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11130   %}
11131   ins_pipe( pipe_slow );
11132 %}
11133 
11134 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11135                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11136 %{
11137   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11138   match(Set result (AryEq ary1 ary2));
11139   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11140 
11141   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11142   ins_encode %{
11143     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11144                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11145                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11146   %}
11147   ins_pipe( pipe_slow );
11148 %}
11149 
11150 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11151                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11152                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11153                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11154                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11155 %{
11156   predicate(UseAVX >= 2);
11157   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11158   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11159          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11160          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11161          USE basic_type, KILL cr);
11162 
11163   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
11164   ins_encode %{
11165     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11166                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11167                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11168                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11169                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11170                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11171                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11172   %}
11173   ins_pipe( pipe_slow );
11174 %}
11175 
11176 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11177                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11178 %{
11179   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11180   match(Set result (CountPositives ary1 len));
11181   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11182 
11183   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11184   ins_encode %{
11185     __ count_positives($ary1$$Register, $len$$Register,
11186                        $result$$Register, $tmp3$$Register,
11187                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11188   %}
11189   ins_pipe( pipe_slow );
11190 %}
11191 
11192 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11193                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11194 %{
11195   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11196   match(Set result (CountPositives ary1 len));
11197   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11198 
11199   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11200   ins_encode %{
11201     __ count_positives($ary1$$Register, $len$$Register,
11202                        $result$$Register, $tmp3$$Register,
11203                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11204   %}
11205   ins_pipe( pipe_slow );
11206 %}
11207 
11208 // fast char[] to byte[] compression
11209 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11210                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11211   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11212   match(Set result (StrCompressedCopy src (Binary dst len)));
11213   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11214          USE_KILL len, KILL tmp5, KILL cr);
11215 
11216   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11217   ins_encode %{
11218     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11219                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11220                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11221                            knoreg, knoreg);
11222   %}
11223   ins_pipe( pipe_slow );
11224 %}
11225 
11226 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11227                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11228   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11229   match(Set result (StrCompressedCopy src (Binary dst len)));
11230   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11231          USE_KILL len, KILL tmp5, KILL cr);
11232 
11233   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11234   ins_encode %{
11235     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11236                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11237                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11238                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11239   %}
11240   ins_pipe( pipe_slow );
11241 %}
11242 // fast byte[] to char[] inflation
11243 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11244                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11245   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11246   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11247   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11248 
11249   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11250   ins_encode %{
11251     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11252                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11253   %}
11254   ins_pipe( pipe_slow );
11255 %}
11256 
11257 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11258                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11259   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11260   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11261   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11262 
11263   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11264   ins_encode %{
11265     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11266                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11267   %}
11268   ins_pipe( pipe_slow );
11269 %}
11270 
11271 // encode char[] to byte[] in ISO_8859_1
11272 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11273                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11274                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11275   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11276   match(Set result (EncodeISOArray src (Binary dst len)));
11277   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11278 
11279   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11280   ins_encode %{
11281     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11282                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11283                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11284   %}
11285   ins_pipe( pipe_slow );
11286 %}
11287 
11288 // encode char[] to byte[] in ASCII
11289 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11290                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11291                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11292   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11293   match(Set result (EncodeISOArray src (Binary dst len)));
11294   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11295 
11296   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11297   ins_encode %{
11298     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11299                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11300                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11301   %}
11302   ins_pipe( pipe_slow );
11303 %}
11304 
11305 //----------Overflow Math Instructions-----------------------------------------
11306 
11307 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11308 %{
11309   match(Set cr (OverflowAddI op1 op2));
11310   effect(DEF cr, USE_KILL op1, USE op2);
11311 
11312   format %{ "addl    $op1, $op2\t# overflow check int" %}
11313 
11314   ins_encode %{
11315     __ addl($op1$$Register, $op2$$Register);
11316   %}
11317   ins_pipe(ialu_reg_reg);
11318 %}
11319 
11320 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11321 %{
11322   match(Set cr (OverflowAddI op1 op2));
11323   effect(DEF cr, USE_KILL op1, USE op2);
11324 
11325   format %{ "addl    $op1, $op2\t# overflow check int" %}
11326 
11327   ins_encode %{
11328     __ addl($op1$$Register, $op2$$constant);
11329   %}
11330   ins_pipe(ialu_reg_reg);
11331 %}
11332 
11333 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11334 %{
11335   match(Set cr (OverflowAddL op1 op2));
11336   effect(DEF cr, USE_KILL op1, USE op2);
11337 
11338   format %{ "addq    $op1, $op2\t# overflow check long" %}
11339   ins_encode %{
11340     __ addq($op1$$Register, $op2$$Register);
11341   %}
11342   ins_pipe(ialu_reg_reg);
11343 %}
11344 
11345 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11346 %{
11347   match(Set cr (OverflowAddL op1 op2));
11348   effect(DEF cr, USE_KILL op1, USE op2);
11349 
11350   format %{ "addq    $op1, $op2\t# overflow check long" %}
11351   ins_encode %{
11352     __ addq($op1$$Register, $op2$$constant);
11353   %}
11354   ins_pipe(ialu_reg_reg);
11355 %}
11356 
11357 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11358 %{
11359   match(Set cr (OverflowSubI op1 op2));
11360 
11361   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11362   ins_encode %{
11363     __ cmpl($op1$$Register, $op2$$Register);
11364   %}
11365   ins_pipe(ialu_reg_reg);
11366 %}
11367 
11368 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11369 %{
11370   match(Set cr (OverflowSubI op1 op2));
11371 
11372   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11373   ins_encode %{
11374     __ cmpl($op1$$Register, $op2$$constant);
11375   %}
11376   ins_pipe(ialu_reg_reg);
11377 %}
11378 
11379 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11380 %{
11381   match(Set cr (OverflowSubL op1 op2));
11382 
11383   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11384   ins_encode %{
11385     __ cmpq($op1$$Register, $op2$$Register);
11386   %}
11387   ins_pipe(ialu_reg_reg);
11388 %}
11389 
11390 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11391 %{
11392   match(Set cr (OverflowSubL op1 op2));
11393 
11394   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11395   ins_encode %{
11396     __ cmpq($op1$$Register, $op2$$constant);
11397   %}
11398   ins_pipe(ialu_reg_reg);
11399 %}
11400 
11401 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11402 %{
11403   match(Set cr (OverflowSubI zero op2));
11404   effect(DEF cr, USE_KILL op2);
11405 
11406   format %{ "negl    $op2\t# overflow check int" %}
11407   ins_encode %{
11408     __ negl($op2$$Register);
11409   %}
11410   ins_pipe(ialu_reg_reg);
11411 %}
11412 
11413 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11414 %{
11415   match(Set cr (OverflowSubL zero op2));
11416   effect(DEF cr, USE_KILL op2);
11417 
11418   format %{ "negq    $op2\t# overflow check long" %}
11419   ins_encode %{
11420     __ negq($op2$$Register);
11421   %}
11422   ins_pipe(ialu_reg_reg);
11423 %}
11424 
11425 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11426 %{
11427   match(Set cr (OverflowMulI op1 op2));
11428   effect(DEF cr, USE_KILL op1, USE op2);
11429 
11430   format %{ "imull    $op1, $op2\t# overflow check int" %}
11431   ins_encode %{
11432     __ imull($op1$$Register, $op2$$Register);
11433   %}
11434   ins_pipe(ialu_reg_reg_alu0);
11435 %}
11436 
11437 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11438 %{
11439   match(Set cr (OverflowMulI op1 op2));
11440   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11441 
11442   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11443   ins_encode %{
11444     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11445   %}
11446   ins_pipe(ialu_reg_reg_alu0);
11447 %}
11448 
11449 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11450 %{
11451   match(Set cr (OverflowMulL op1 op2));
11452   effect(DEF cr, USE_KILL op1, USE op2);
11453 
11454   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11455   ins_encode %{
11456     __ imulq($op1$$Register, $op2$$Register);
11457   %}
11458   ins_pipe(ialu_reg_reg_alu0);
11459 %}
11460 
11461 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11462 %{
11463   match(Set cr (OverflowMulL op1 op2));
11464   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11465 
11466   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11467   ins_encode %{
11468     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11469   %}
11470   ins_pipe(ialu_reg_reg_alu0);
11471 %}
11472 
11473 
11474 //----------Control Flow Instructions------------------------------------------
11475 // Signed compare Instructions
11476 
11477 // XXX more variants!!
11478 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11479 %{
11480   match(Set cr (CmpI op1 op2));
11481   effect(DEF cr, USE op1, USE op2);
11482 
11483   format %{ "cmpl    $op1, $op2" %}
11484   ins_encode %{
11485     __ cmpl($op1$$Register, $op2$$Register);
11486   %}
11487   ins_pipe(ialu_cr_reg_reg);
11488 %}
11489 
11490 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11491 %{
11492   match(Set cr (CmpI op1 op2));
11493 
11494   format %{ "cmpl    $op1, $op2" %}
11495   ins_encode %{
11496     __ cmpl($op1$$Register, $op2$$constant);
11497   %}
11498   ins_pipe(ialu_cr_reg_imm);
11499 %}
11500 
11501 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11502 %{
11503   match(Set cr (CmpI op1 (LoadI op2)));
11504 
11505   ins_cost(500); // XXX
11506   format %{ "cmpl    $op1, $op2" %}
11507   ins_encode %{
11508     __ cmpl($op1$$Register, $op2$$Address);
11509   %}
11510   ins_pipe(ialu_cr_reg_mem);
11511 %}
11512 
11513 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11514 %{
11515   match(Set cr (CmpI src zero));
11516 
11517   format %{ "testl   $src, $src" %}
11518   ins_encode %{
11519     __ testl($src$$Register, $src$$Register);
11520   %}
11521   ins_pipe(ialu_cr_reg_imm);
11522 %}
11523 
11524 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11525 %{
11526   match(Set cr (CmpI (AndI src con) zero));
11527 
11528   format %{ "testl   $src, $con" %}
11529   ins_encode %{
11530     __ testl($src$$Register, $con$$constant);
11531   %}
11532   ins_pipe(ialu_cr_reg_imm);
11533 %}
11534 
11535 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11536 %{
11537   match(Set cr (CmpI (AndI src1 src2) zero));
11538 
11539   format %{ "testl   $src1, $src2" %}
11540   ins_encode %{
11541     __ testl($src1$$Register, $src2$$Register);
11542   %}
11543   ins_pipe(ialu_cr_reg_imm);
11544 %}
11545 
11546 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11547 %{
11548   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11549 
11550   format %{ "testl   $src, $mem" %}
11551   ins_encode %{
11552     __ testl($src$$Register, $mem$$Address);
11553   %}
11554   ins_pipe(ialu_cr_reg_mem);
11555 %}
11556 
11557 // Unsigned compare Instructions; really, same as signed except they
11558 // produce an rFlagsRegU instead of rFlagsReg.
11559 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11560 %{
11561   match(Set cr (CmpU op1 op2));
11562 
11563   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11564   ins_encode %{
11565     __ cmpl($op1$$Register, $op2$$Register);
11566   %}
11567   ins_pipe(ialu_cr_reg_reg);
11568 %}
11569 
11570 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11571 %{
11572   match(Set cr (CmpU op1 op2));
11573 
11574   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11575   ins_encode %{
11576     __ cmpl($op1$$Register, $op2$$constant);
11577   %}
11578   ins_pipe(ialu_cr_reg_imm);
11579 %}
11580 
11581 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11582 %{
11583   match(Set cr (CmpU op1 (LoadI op2)));
11584 
11585   ins_cost(500); // XXX
11586   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11587   ins_encode %{
11588     __ cmpl($op1$$Register, $op2$$Address);
11589   %}
11590   ins_pipe(ialu_cr_reg_mem);
11591 %}
11592 
11593 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11594 %{
11595   match(Set cr (CmpU src zero));
11596 
11597   format %{ "testl   $src, $src\t# unsigned" %}
11598   ins_encode %{
11599     __ testl($src$$Register, $src$$Register);
11600   %}
11601   ins_pipe(ialu_cr_reg_imm);
11602 %}
11603 
11604 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11605 %{
11606   match(Set cr (CmpP op1 op2));
11607 
11608   format %{ "cmpq    $op1, $op2\t# ptr" %}
11609   ins_encode %{
11610     __ cmpq($op1$$Register, $op2$$Register);
11611   %}
11612   ins_pipe(ialu_cr_reg_reg);
11613 %}
11614 
11615 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11616 %{
11617   match(Set cr (CmpP op1 (LoadP op2)));
11618   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11619 
11620   ins_cost(500); // XXX
11621   format %{ "cmpq    $op1, $op2\t# ptr" %}
11622   ins_encode %{
11623     __ cmpq($op1$$Register, $op2$$Address);
11624   %}
11625   ins_pipe(ialu_cr_reg_mem);
11626 %}
11627 
11628 // XXX this is generalized by compP_rReg_mem???
11629 // Compare raw pointer (used in out-of-heap check).
11630 // Only works because non-oop pointers must be raw pointers
11631 // and raw pointers have no anti-dependencies.
11632 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11633 %{
11634   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11635             n->in(2)->as_Load()->barrier_data() == 0);
11636   match(Set cr (CmpP op1 (LoadP op2)));
11637 
11638   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11639   ins_encode %{
11640     __ cmpq($op1$$Register, $op2$$Address);
11641   %}
11642   ins_pipe(ialu_cr_reg_mem);
11643 %}
11644 
11645 // This will generate a signed flags result. This should be OK since
11646 // any compare to a zero should be eq/neq.
11647 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11648 %{
11649   match(Set cr (CmpP src zero));
11650 
11651   format %{ "testq   $src, $src\t# ptr" %}
11652   ins_encode %{
11653     __ testq($src$$Register, $src$$Register);
11654   %}
11655   ins_pipe(ialu_cr_reg_imm);
11656 %}
11657 
11658 // This will generate a signed flags result. This should be OK since
11659 // any compare to a zero should be eq/neq.
11660 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11661 %{
11662   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
11663             n->in(1)->as_Load()->barrier_data() == 0);
11664   match(Set cr (CmpP (LoadP op) zero));
11665 
11666   ins_cost(500); // XXX
11667   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11668   ins_encode %{
11669     __ testq($op$$Address, 0xFFFFFFFF);
11670   %}
11671   ins_pipe(ialu_cr_reg_imm);
11672 %}
11673 
11674 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11675 %{
11676   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
11677             n->in(1)->as_Load()->barrier_data() == 0);
11678   match(Set cr (CmpP (LoadP mem) zero));
11679 
11680   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11681   ins_encode %{
11682     __ cmpq(r12, $mem$$Address);
11683   %}
11684   ins_pipe(ialu_cr_reg_mem);
11685 %}
11686 
11687 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11688 %{
11689   match(Set cr (CmpN op1 op2));
11690 
11691   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11692   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11693   ins_pipe(ialu_cr_reg_reg);
11694 %}
11695 
11696 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11697 %{
11698   match(Set cr (CmpN src (LoadN mem)));
11699 
11700   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11701   ins_encode %{
11702     __ cmpl($src$$Register, $mem$$Address);
11703   %}
11704   ins_pipe(ialu_cr_reg_mem);
11705 %}
11706 
11707 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11708   match(Set cr (CmpN op1 op2));
11709 
11710   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11711   ins_encode %{
11712     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11713   %}
11714   ins_pipe(ialu_cr_reg_imm);
11715 %}
11716 
11717 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11718 %{
11719   match(Set cr (CmpN src (LoadN mem)));
11720 
11721   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11722   ins_encode %{
11723     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11724   %}
11725   ins_pipe(ialu_cr_reg_mem);
11726 %}
11727 
11728 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11729   match(Set cr (CmpN op1 op2));
11730 
11731   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11732   ins_encode %{
11733     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11734   %}
11735   ins_pipe(ialu_cr_reg_imm);
11736 %}
11737 
11738 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11739 %{
11740   match(Set cr (CmpN src (LoadNKlass mem)));
11741 
11742   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11743   ins_encode %{
11744     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11745   %}
11746   ins_pipe(ialu_cr_reg_mem);
11747 %}
11748 
11749 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11750   match(Set cr (CmpN src zero));
11751 
11752   format %{ "testl   $src, $src\t# compressed ptr" %}
11753   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11754   ins_pipe(ialu_cr_reg_imm);
11755 %}
11756 
11757 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11758 %{
11759   predicate(CompressedOops::base() != nullptr);
11760   match(Set cr (CmpN (LoadN mem) zero));
11761 
11762   ins_cost(500); // XXX
11763   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11764   ins_encode %{
11765     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11766   %}
11767   ins_pipe(ialu_cr_reg_mem);
11768 %}
11769 
11770 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11771 %{
11772   predicate(CompressedOops::base() == nullptr);
11773   match(Set cr (CmpN (LoadN mem) zero));
11774 
11775   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11776   ins_encode %{
11777     __ cmpl(r12, $mem$$Address);
11778   %}
11779   ins_pipe(ialu_cr_reg_mem);
11780 %}
11781 
11782 // Yanked all unsigned pointer compare operations.
11783 // Pointer compares are done with CmpP which is already unsigned.
11784 
11785 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11786 %{
11787   match(Set cr (CmpL op1 op2));
11788 
11789   format %{ "cmpq    $op1, $op2" %}
11790   ins_encode %{
11791     __ cmpq($op1$$Register, $op2$$Register);
11792   %}
11793   ins_pipe(ialu_cr_reg_reg);
11794 %}
11795 
11796 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11797 %{
11798   match(Set cr (CmpL op1 op2));
11799 
11800   format %{ "cmpq    $op1, $op2" %}
11801   ins_encode %{
11802     __ cmpq($op1$$Register, $op2$$constant);
11803   %}
11804   ins_pipe(ialu_cr_reg_imm);
11805 %}
11806 
11807 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11808 %{
11809   match(Set cr (CmpL op1 (LoadL op2)));
11810 
11811   format %{ "cmpq    $op1, $op2" %}
11812   ins_encode %{
11813     __ cmpq($op1$$Register, $op2$$Address);
11814   %}
11815   ins_pipe(ialu_cr_reg_mem);
11816 %}
11817 
11818 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11819 %{
11820   match(Set cr (CmpL src zero));
11821 
11822   format %{ "testq   $src, $src" %}
11823   ins_encode %{
11824     __ testq($src$$Register, $src$$Register);
11825   %}
11826   ins_pipe(ialu_cr_reg_imm);
11827 %}
11828 
11829 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11830 %{
11831   match(Set cr (CmpL (AndL src con) zero));
11832 
11833   format %{ "testq   $src, $con\t# long" %}
11834   ins_encode %{
11835     __ testq($src$$Register, $con$$constant);
11836   %}
11837   ins_pipe(ialu_cr_reg_imm);
11838 %}
11839 
11840 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
11841 %{
11842   match(Set cr (CmpL (AndL src1 src2) zero));
11843 
11844   format %{ "testq   $src1, $src2\t# long" %}
11845   ins_encode %{
11846     __ testq($src1$$Register, $src2$$Register);
11847   %}
11848   ins_pipe(ialu_cr_reg_imm);
11849 %}
11850 
11851 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11852 %{
11853   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11854 
11855   format %{ "testq   $src, $mem" %}
11856   ins_encode %{
11857     __ testq($src$$Register, $mem$$Address);
11858   %}
11859   ins_pipe(ialu_cr_reg_mem);
11860 %}
11861 
11862 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11863 %{
11864   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11865 
11866   format %{ "testq   $src, $mem" %}
11867   ins_encode %{
11868     __ testq($src$$Register, $mem$$Address);
11869   %}
11870   ins_pipe(ialu_cr_reg_mem);
11871 %}
11872 
11873 // Manifest a CmpU result in an integer register.  Very painful.
11874 // This is the test to avoid.
11875 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
11876 %{
11877   match(Set dst (CmpU3 src1 src2));
11878   effect(KILL flags);
11879 
11880   ins_cost(275); // XXX
11881   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
11882             "movl    $dst, -1\n\t"
11883             "jb,u    done\n\t"
11884             "setne   $dst\n\t"
11885             "movzbl  $dst, $dst\n\t"
11886     "done:" %}
11887   ins_encode %{
11888     Label done;
11889     __ cmpl($src1$$Register, $src2$$Register);
11890     __ movl($dst$$Register, -1);
11891     __ jccb(Assembler::below, done);
11892     __ setb(Assembler::notZero, $dst$$Register);
11893     __ movzbl($dst$$Register, $dst$$Register);
11894     __ bind(done);
11895   %}
11896   ins_pipe(pipe_slow);
11897 %}
11898 
11899 // Manifest a CmpL result in an integer register.  Very painful.
11900 // This is the test to avoid.
11901 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11902 %{
11903   match(Set dst (CmpL3 src1 src2));
11904   effect(KILL flags);
11905 
11906   ins_cost(275); // XXX
11907   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11908             "movl    $dst, -1\n\t"
11909             "jl,s    done\n\t"
11910             "setne   $dst\n\t"
11911             "movzbl  $dst, $dst\n\t"
11912     "done:" %}
11913   ins_encode %{
11914     Label done;
11915     __ cmpq($src1$$Register, $src2$$Register);
11916     __ movl($dst$$Register, -1);
11917     __ jccb(Assembler::less, done);
11918     __ setb(Assembler::notZero, $dst$$Register);
11919     __ movzbl($dst$$Register, $dst$$Register);
11920     __ bind(done);
11921   %}
11922   ins_pipe(pipe_slow);
11923 %}
11924 
11925 // Manifest a CmpUL result in an integer register.  Very painful.
11926 // This is the test to avoid.
11927 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11928 %{
11929   match(Set dst (CmpUL3 src1 src2));
11930   effect(KILL flags);
11931 
11932   ins_cost(275); // XXX
11933   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11934             "movl    $dst, -1\n\t"
11935             "jb,u    done\n\t"
11936             "setne   $dst\n\t"
11937             "movzbl  $dst, $dst\n\t"
11938     "done:" %}
11939   ins_encode %{
11940     Label done;
11941     __ cmpq($src1$$Register, $src2$$Register);
11942     __ movl($dst$$Register, -1);
11943     __ jccb(Assembler::below, done);
11944     __ setb(Assembler::notZero, $dst$$Register);
11945     __ movzbl($dst$$Register, $dst$$Register);
11946     __ bind(done);
11947   %}
11948   ins_pipe(pipe_slow);
11949 %}
11950 
11951 // Unsigned long compare Instructions; really, same as signed long except they
11952 // produce an rFlagsRegU instead of rFlagsReg.
11953 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11954 %{
11955   match(Set cr (CmpUL op1 op2));
11956 
11957   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11958   ins_encode %{
11959     __ cmpq($op1$$Register, $op2$$Register);
11960   %}
11961   ins_pipe(ialu_cr_reg_reg);
11962 %}
11963 
11964 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11965 %{
11966   match(Set cr (CmpUL op1 op2));
11967 
11968   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11969   ins_encode %{
11970     __ cmpq($op1$$Register, $op2$$constant);
11971   %}
11972   ins_pipe(ialu_cr_reg_imm);
11973 %}
11974 
11975 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11976 %{
11977   match(Set cr (CmpUL op1 (LoadL op2)));
11978 
11979   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11980   ins_encode %{
11981     __ cmpq($op1$$Register, $op2$$Address);
11982   %}
11983   ins_pipe(ialu_cr_reg_mem);
11984 %}
11985 
11986 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11987 %{
11988   match(Set cr (CmpUL src zero));
11989 
11990   format %{ "testq   $src, $src\t# unsigned" %}
11991   ins_encode %{
11992     __ testq($src$$Register, $src$$Register);
11993   %}
11994   ins_pipe(ialu_cr_reg_imm);
11995 %}
11996 
11997 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11998 %{
11999   match(Set cr (CmpI (LoadB mem) imm));
12000 
12001   ins_cost(125);
12002   format %{ "cmpb    $mem, $imm" %}
12003   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12004   ins_pipe(ialu_cr_reg_mem);
12005 %}
12006 
12007 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12008 %{
12009   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12010 
12011   ins_cost(125);
12012   format %{ "testb   $mem, $imm\t# ubyte" %}
12013   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12014   ins_pipe(ialu_cr_reg_mem);
12015 %}
12016 
12017 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12018 %{
12019   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12020 
12021   ins_cost(125);
12022   format %{ "testb   $mem, $imm\t# byte" %}
12023   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12024   ins_pipe(ialu_cr_reg_mem);
12025 %}
12026 
12027 //----------Max and Min--------------------------------------------------------
12028 // Min Instructions
12029 
12030 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12031 %{
12032   effect(USE_DEF dst, USE src, USE cr);
12033 
12034   format %{ "cmovlgt $dst, $src\t# min" %}
12035   ins_encode %{
12036     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12037   %}
12038   ins_pipe(pipe_cmov_reg);
12039 %}
12040 
12041 
12042 instruct minI_rReg(rRegI dst, rRegI src)
12043 %{
12044   match(Set dst (MinI dst src));
12045 
12046   ins_cost(200);
12047   expand %{
12048     rFlagsReg cr;
12049     compI_rReg(cr, dst, src);
12050     cmovI_reg_g(dst, src, cr);
12051   %}
12052 %}
12053 
12054 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12055 %{
12056   effect(USE_DEF dst, USE src, USE cr);
12057 
12058   format %{ "cmovllt $dst, $src\t# max" %}
12059   ins_encode %{
12060     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12061   %}
12062   ins_pipe(pipe_cmov_reg);
12063 %}
12064 
12065 
12066 instruct maxI_rReg(rRegI dst, rRegI src)
12067 %{
12068   match(Set dst (MaxI dst src));
12069 
12070   ins_cost(200);
12071   expand %{
12072     rFlagsReg cr;
12073     compI_rReg(cr, dst, src);
12074     cmovI_reg_l(dst, src, cr);
12075   %}
12076 %}
12077 
12078 // ============================================================================
12079 // Branch Instructions
12080 
12081 // Jump Direct - Label defines a relative address from JMP+1
12082 instruct jmpDir(label labl)
12083 %{
12084   match(Goto);
12085   effect(USE labl);
12086 
12087   ins_cost(300);
12088   format %{ "jmp     $labl" %}
12089   size(5);
12090   ins_encode %{
12091     Label* L = $labl$$label;
12092     __ jmp(*L, false); // Always long jump
12093   %}
12094   ins_pipe(pipe_jmp);
12095 %}
12096 
12097 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12098 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12099 %{
12100   match(If cop cr);
12101   effect(USE labl);
12102 
12103   ins_cost(300);
12104   format %{ "j$cop     $labl" %}
12105   size(6);
12106   ins_encode %{
12107     Label* L = $labl$$label;
12108     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12109   %}
12110   ins_pipe(pipe_jcc);
12111 %}
12112 
12113 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12114 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12115 %{
12116   match(CountedLoopEnd cop cr);
12117   effect(USE labl);
12118 
12119   ins_cost(300);
12120   format %{ "j$cop     $labl\t# loop end" %}
12121   size(6);
12122   ins_encode %{
12123     Label* L = $labl$$label;
12124     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12125   %}
12126   ins_pipe(pipe_jcc);
12127 %}
12128 
12129 // Jump Direct Conditional - using unsigned comparison
12130 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12131   match(If cop cmp);
12132   effect(USE labl);
12133 
12134   ins_cost(300);
12135   format %{ "j$cop,u   $labl" %}
12136   size(6);
12137   ins_encode %{
12138     Label* L = $labl$$label;
12139     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12140   %}
12141   ins_pipe(pipe_jcc);
12142 %}
12143 
12144 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12145   match(If cop cmp);
12146   effect(USE labl);
12147 
12148   ins_cost(200);
12149   format %{ "j$cop,u   $labl" %}
12150   size(6);
12151   ins_encode %{
12152     Label* L = $labl$$label;
12153     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12154   %}
12155   ins_pipe(pipe_jcc);
12156 %}
12157 
12158 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12159   match(If cop cmp);
12160   effect(USE labl);
12161 
12162   ins_cost(200);
12163   format %{ $$template
12164     if ($cop$$cmpcode == Assembler::notEqual) {
12165       $$emit$$"jp,u    $labl\n\t"
12166       $$emit$$"j$cop,u   $labl"
12167     } else {
12168       $$emit$$"jp,u    done\n\t"
12169       $$emit$$"j$cop,u   $labl\n\t"
12170       $$emit$$"done:"
12171     }
12172   %}
12173   ins_encode %{
12174     Label* l = $labl$$label;
12175     if ($cop$$cmpcode == Assembler::notEqual) {
12176       __ jcc(Assembler::parity, *l, false);
12177       __ jcc(Assembler::notEqual, *l, false);
12178     } else if ($cop$$cmpcode == Assembler::equal) {
12179       Label done;
12180       __ jccb(Assembler::parity, done);
12181       __ jcc(Assembler::equal, *l, false);
12182       __ bind(done);
12183     } else {
12184        ShouldNotReachHere();
12185     }
12186   %}
12187   ins_pipe(pipe_jcc);
12188 %}
12189 
12190 // ============================================================================
12191 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12192 // superklass array for an instance of the superklass.  Set a hidden
12193 // internal cache on a hit (cache is checked with exposed code in
12194 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12195 // encoding ALSO sets flags.
12196 
12197 instruct partialSubtypeCheck(rdi_RegP result,
12198                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12199                              rFlagsReg cr)
12200 %{
12201   match(Set result (PartialSubtypeCheck sub super));
12202   effect(KILL rcx, KILL cr);
12203 
12204   ins_cost(1100);  // slightly larger than the next version
12205   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12206             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12207             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12208             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12209             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12210             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12211             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12212     "miss:\t" %}
12213 
12214   opcode(0x1); // Force a XOR of RDI
12215   ins_encode(enc_PartialSubtypeCheck());
12216   ins_pipe(pipe_slow);
12217 %}
12218 
12219 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12220                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12221                                      immP0 zero,
12222                                      rdi_RegP result)
12223 %{
12224   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12225   effect(KILL rcx, KILL result);
12226 
12227   ins_cost(1000);
12228   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12229             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12230             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12231             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12232             "jne,s   miss\t\t# Missed: flags nz\n\t"
12233             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12234     "miss:\t" %}
12235 
12236   opcode(0x0); // No need to XOR RDI
12237   ins_encode(enc_PartialSubtypeCheck());
12238   ins_pipe(pipe_slow);
12239 %}
12240 
12241 // ============================================================================
12242 // Branch Instructions -- short offset versions
12243 //
12244 // These instructions are used to replace jumps of a long offset (the default
12245 // match) with jumps of a shorter offset.  These instructions are all tagged
12246 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12247 // match rules in general matching.  Instead, the ADLC generates a conversion
12248 // method in the MachNode which can be used to do in-place replacement of the
12249 // long variant with the shorter variant.  The compiler will determine if a
12250 // branch can be taken by the is_short_branch_offset() predicate in the machine
12251 // specific code section of the file.
12252 
12253 // Jump Direct - Label defines a relative address from JMP+1
12254 instruct jmpDir_short(label labl) %{
12255   match(Goto);
12256   effect(USE labl);
12257 
12258   ins_cost(300);
12259   format %{ "jmp,s   $labl" %}
12260   size(2);
12261   ins_encode %{
12262     Label* L = $labl$$label;
12263     __ jmpb(*L);
12264   %}
12265   ins_pipe(pipe_jmp);
12266   ins_short_branch(1);
12267 %}
12268 
12269 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12270 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12271   match(If cop cr);
12272   effect(USE labl);
12273 
12274   ins_cost(300);
12275   format %{ "j$cop,s   $labl" %}
12276   size(2);
12277   ins_encode %{
12278     Label* L = $labl$$label;
12279     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12280   %}
12281   ins_pipe(pipe_jcc);
12282   ins_short_branch(1);
12283 %}
12284 
12285 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12286 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12287   match(CountedLoopEnd cop cr);
12288   effect(USE labl);
12289 
12290   ins_cost(300);
12291   format %{ "j$cop,s   $labl\t# loop end" %}
12292   size(2);
12293   ins_encode %{
12294     Label* L = $labl$$label;
12295     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12296   %}
12297   ins_pipe(pipe_jcc);
12298   ins_short_branch(1);
12299 %}
12300 
12301 // Jump Direct Conditional - using unsigned comparison
12302 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12303   match(If cop cmp);
12304   effect(USE labl);
12305 
12306   ins_cost(300);
12307   format %{ "j$cop,us  $labl" %}
12308   size(2);
12309   ins_encode %{
12310     Label* L = $labl$$label;
12311     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12312   %}
12313   ins_pipe(pipe_jcc);
12314   ins_short_branch(1);
12315 %}
12316 
12317 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12318   match(If cop cmp);
12319   effect(USE labl);
12320 
12321   ins_cost(300);
12322   format %{ "j$cop,us  $labl" %}
12323   size(2);
12324   ins_encode %{
12325     Label* L = $labl$$label;
12326     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12327   %}
12328   ins_pipe(pipe_jcc);
12329   ins_short_branch(1);
12330 %}
12331 
12332 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12333   match(If cop cmp);
12334   effect(USE labl);
12335 
12336   ins_cost(300);
12337   format %{ $$template
12338     if ($cop$$cmpcode == Assembler::notEqual) {
12339       $$emit$$"jp,u,s  $labl\n\t"
12340       $$emit$$"j$cop,u,s  $labl"
12341     } else {
12342       $$emit$$"jp,u,s  done\n\t"
12343       $$emit$$"j$cop,u,s  $labl\n\t"
12344       $$emit$$"done:"
12345     }
12346   %}
12347   size(4);
12348   ins_encode %{
12349     Label* l = $labl$$label;
12350     if ($cop$$cmpcode == Assembler::notEqual) {
12351       __ jccb(Assembler::parity, *l);
12352       __ jccb(Assembler::notEqual, *l);
12353     } else if ($cop$$cmpcode == Assembler::equal) {
12354       Label done;
12355       __ jccb(Assembler::parity, done);
12356       __ jccb(Assembler::equal, *l);
12357       __ bind(done);
12358     } else {
12359        ShouldNotReachHere();
12360     }
12361   %}
12362   ins_pipe(pipe_jcc);
12363   ins_short_branch(1);
12364 %}
12365 
12366 // ============================================================================
12367 // inlined locking and unlocking
12368 
12369 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12370   predicate(Compile::current()->use_rtm());
12371   match(Set cr (FastLock object box));
12372   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12373   ins_cost(300);
12374   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12375   ins_encode %{
12376     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12377                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
12378                  _rtm_counters, _stack_rtm_counters,
12379                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12380                  true, ra_->C->profile_rtm());
12381   %}
12382   ins_pipe(pipe_slow);
12383 %}
12384 
12385 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12386   predicate(LockingMode != LM_LIGHTWEIGHT && !Compile::current()->use_rtm());
12387   match(Set cr (FastLock object box));
12388   effect(TEMP tmp, TEMP scr, USE_KILL box);
12389   ins_cost(300);
12390   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12391   ins_encode %{
12392     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12393                  $scr$$Register, noreg, noreg, r15_thread, nullptr, nullptr, nullptr, false, false);
12394   %}
12395   ins_pipe(pipe_slow);
12396 %}
12397 
12398 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp, rRegP scr) %{
12399   predicate(LockingMode != LM_LIGHTWEIGHT);
12400   match(Set cr (FastUnlock object box));
12401   effect(TEMP tmp, TEMP scr, USE_KILL box);
12402   ins_cost(300);
12403   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12404   ins_encode %{
12405     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $scr$$Register, ra_->C->use_rtm());
12406   %}
12407   ins_pipe(pipe_slow);
12408 %}
12409 
12410 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
12411   predicate(LockingMode == LM_LIGHTWEIGHT);
12412   match(Set cr (FastLock object box));
12413   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
12414   ins_cost(300);
12415   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
12416   ins_encode %{
12417     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12418   %}
12419   ins_pipe(pipe_slow);
12420 %}
12421 
12422 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp, rRegP scr) %{
12423   predicate(LockingMode == LM_LIGHTWEIGHT);
12424   match(Set cr (FastUnlock object rax_reg));
12425   effect(TEMP tmp, TEMP scr, USE_KILL rax_reg);
12426   ins_cost(300);
12427   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
12428   ins_encode %{
12429     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, $scr$$Register, r15_thread);
12430   %}
12431   ins_pipe(pipe_slow);
12432 %}
12433 
12434 
12435 // ============================================================================
12436 // Safepoint Instructions
12437 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12438 %{
12439   match(SafePoint poll);
12440   effect(KILL cr, USE poll);
12441 
12442   format %{ "testl   rax, [$poll]\t"
12443             "# Safepoint: poll for GC" %}
12444   ins_cost(125);
12445   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12446   ins_encode %{
12447     __ relocate(relocInfo::poll_type);
12448     address pre_pc = __ pc();
12449     __ testl(rax, Address($poll$$Register, 0));
12450     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12451   %}
12452   ins_pipe(ialu_reg_mem);
12453 %}
12454 
12455 instruct mask_all_evexL(kReg dst, rRegL src) %{
12456   match(Set dst (MaskAll src));
12457   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12458   ins_encode %{
12459     int mask_len = Matcher::vector_length(this);
12460     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12461   %}
12462   ins_pipe( pipe_slow );
12463 %}
12464 
12465 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12466   predicate(Matcher::vector_length(n) > 32);
12467   match(Set dst (MaskAll src));
12468   effect(TEMP tmp);
12469   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12470   ins_encode %{
12471     int mask_len = Matcher::vector_length(this);
12472     __ movslq($tmp$$Register, $src$$Register);
12473     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12474   %}
12475   ins_pipe( pipe_slow );
12476 %}
12477 
12478 // ============================================================================
12479 // Procedure Call/Return Instructions
12480 // Call Java Static Instruction
12481 // Note: If this code changes, the corresponding ret_addr_offset() and
12482 //       compute_padding() functions will have to be adjusted.
12483 instruct CallStaticJavaDirect(method meth) %{
12484   match(CallStaticJava);
12485   effect(USE meth);
12486 
12487   ins_cost(300);
12488   format %{ "call,static " %}
12489   opcode(0xE8); /* E8 cd */
12490   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12491   ins_pipe(pipe_slow);
12492   ins_alignment(4);
12493 %}
12494 
12495 // Call Java Dynamic Instruction
12496 // Note: If this code changes, the corresponding ret_addr_offset() and
12497 //       compute_padding() functions will have to be adjusted.
12498 instruct CallDynamicJavaDirect(method meth)
12499 %{
12500   match(CallDynamicJava);
12501   effect(USE meth);
12502 
12503   ins_cost(300);
12504   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12505             "call,dynamic " %}
12506   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12507   ins_pipe(pipe_slow);
12508   ins_alignment(4);
12509 %}
12510 
12511 // Call Runtime Instruction
12512 instruct CallRuntimeDirect(method meth)
12513 %{
12514   match(CallRuntime);
12515   effect(USE meth);
12516 
12517   ins_cost(300);
12518   format %{ "call,runtime " %}
12519   ins_encode(clear_avx, Java_To_Runtime(meth));
12520   ins_pipe(pipe_slow);
12521 %}
12522 
12523 // Call runtime without safepoint
12524 instruct CallLeafDirect(method meth)
12525 %{
12526   match(CallLeaf);
12527   effect(USE meth);
12528 
12529   ins_cost(300);
12530   format %{ "call_leaf,runtime " %}
12531   ins_encode(clear_avx, Java_To_Runtime(meth));
12532   ins_pipe(pipe_slow);
12533 %}
12534 
12535 // Call runtime without safepoint and with vector arguments
12536 instruct CallLeafDirectVector(method meth)
12537 %{
12538   match(CallLeafVector);
12539   effect(USE meth);
12540 
12541   ins_cost(300);
12542   format %{ "call_leaf,vector " %}
12543   ins_encode(Java_To_Runtime(meth));
12544   ins_pipe(pipe_slow);
12545 %}
12546 
12547 // Call runtime without safepoint
12548 instruct CallLeafNoFPDirect(method meth)
12549 %{
12550   match(CallLeafNoFP);
12551   effect(USE meth);
12552 
12553   ins_cost(300);
12554   format %{ "call_leaf_nofp,runtime " %}
12555   ins_encode(clear_avx, Java_To_Runtime(meth));
12556   ins_pipe(pipe_slow);
12557 %}
12558 
12559 // Return Instruction
12560 // Remove the return address & jump to it.
12561 // Notice: We always emit a nop after a ret to make sure there is room
12562 // for safepoint patching
12563 instruct Ret()
12564 %{
12565   match(Return);
12566 
12567   format %{ "ret" %}
12568   ins_encode %{
12569     __ ret(0);
12570   %}
12571   ins_pipe(pipe_jmp);
12572 %}
12573 
12574 // Tail Call; Jump from runtime stub to Java code.
12575 // Also known as an 'interprocedural jump'.
12576 // Target of jump will eventually return to caller.
12577 // TailJump below removes the return address.
12578 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12579 %{
12580   match(TailCall jump_target method_ptr);
12581 
12582   ins_cost(300);
12583   format %{ "jmp     $jump_target\t# rbx holds method" %}
12584   ins_encode %{
12585     __ jmp($jump_target$$Register);
12586   %}
12587   ins_pipe(pipe_jmp);
12588 %}
12589 
12590 // Tail Jump; remove the return address; jump to target.
12591 // TailCall above leaves the return address around.
12592 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12593 %{
12594   match(TailJump jump_target ex_oop);
12595 
12596   ins_cost(300);
12597   format %{ "popq    rdx\t# pop return address\n\t"
12598             "jmp     $jump_target" %}
12599   ins_encode %{
12600     __ popq(as_Register(RDX_enc));
12601     __ jmp($jump_target$$Register);
12602   %}
12603   ins_pipe(pipe_jmp);
12604 %}
12605 
12606 // Create exception oop: created by stack-crawling runtime code.
12607 // Created exception is now available to this handler, and is setup
12608 // just prior to jumping to this handler.  No code emitted.
12609 instruct CreateException(rax_RegP ex_oop)
12610 %{
12611   match(Set ex_oop (CreateEx));
12612 
12613   size(0);
12614   // use the following format syntax
12615   format %{ "# exception oop is in rax; no code emitted" %}
12616   ins_encode();
12617   ins_pipe(empty);
12618 %}
12619 
12620 // Rethrow exception:
12621 // The exception oop will come in the first argument position.
12622 // Then JUMP (not call) to the rethrow stub code.
12623 instruct RethrowException()
12624 %{
12625   match(Rethrow);
12626 
12627   // use the following format syntax
12628   format %{ "jmp     rethrow_stub" %}
12629   ins_encode %{
12630     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12631   %}
12632   ins_pipe(pipe_jmp);
12633 %}
12634 
12635 // ============================================================================
12636 // This name is KNOWN by the ADLC and cannot be changed.
12637 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12638 // for this guy.
12639 instruct tlsLoadP(r15_RegP dst) %{
12640   match(Set dst (ThreadLocal));
12641   effect(DEF dst);
12642 
12643   size(0);
12644   format %{ "# TLS is in R15" %}
12645   ins_encode( /*empty encoding*/ );
12646   ins_pipe(ialu_reg_reg);
12647 %}
12648 
12649 
12650 //----------PEEPHOLE RULES-----------------------------------------------------
12651 // These must follow all instruction definitions as they use the names
12652 // defined in the instructions definitions.
12653 //
12654 // peeppredicate ( rule_predicate );
12655 // // the predicate unless which the peephole rule will be ignored
12656 //
12657 // peepmatch ( root_instr_name [preceding_instruction]* );
12658 //
12659 // peepprocedure ( procedure_name );
12660 // // provide a procedure name to perform the optimization, the procedure should
12661 // // reside in the architecture dependent peephole file, the method has the
12662 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12663 // // with the arguments being the basic block, the current node index inside the
12664 // // block, the register allocator, the functions upon invoked return a new node
12665 // // defined in peepreplace, and the rules of the nodes appearing in the
12666 // // corresponding peepmatch, the function return true if successful, else
12667 // // return false
12668 //
12669 // peepconstraint %{
12670 // (instruction_number.operand_name relational_op instruction_number.operand_name
12671 //  [, ...] );
12672 // // instruction numbers are zero-based using left to right order in peepmatch
12673 //
12674 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12675 // // provide an instruction_number.operand_name for each operand that appears
12676 // // in the replacement instruction's match rule
12677 //
12678 // ---------VM FLAGS---------------------------------------------------------
12679 //
12680 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12681 //
12682 // Each peephole rule is given an identifying number starting with zero and
12683 // increasing by one in the order seen by the parser.  An individual peephole
12684 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12685 // on the command-line.
12686 //
12687 // ---------CURRENT LIMITATIONS----------------------------------------------
12688 //
12689 // Only transformations inside a basic block (do we need more for peephole)
12690 //
12691 // ---------EXAMPLE----------------------------------------------------------
12692 //
12693 // // pertinent parts of existing instructions in architecture description
12694 // instruct movI(rRegI dst, rRegI src)
12695 // %{
12696 //   match(Set dst (CopyI src));
12697 // %}
12698 //
12699 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12700 // %{
12701 //   match(Set dst (AddI dst src));
12702 //   effect(KILL cr);
12703 // %}
12704 //
12705 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12706 // %{
12707 //   match(Set dst (AddI dst src));
12708 // %}
12709 //
12710 // 1. Simple replacement
12711 // - Only match adjacent instructions in same basic block
12712 // - Only equality constraints
12713 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
12714 // - Only one replacement instruction
12715 //
12716 // // Change (inc mov) to lea
12717 // peephole %{
12718 //   // lea should only be emitted when beneficial
12719 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12720 //   // increment preceded by register-register move
12721 //   peepmatch ( incI_rReg movI );
12722 //   // require that the destination register of the increment
12723 //   // match the destination register of the move
12724 //   peepconstraint ( 0.dst == 1.dst );
12725 //   // construct a replacement instruction that sets
12726 //   // the destination to ( move's source register + one )
12727 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12728 // %}
12729 //
12730 // 2. Procedural replacement
12731 // - More flexible finding relevent nodes
12732 // - More flexible constraints
12733 // - More flexible transformations
12734 // - May utilise architecture-dependent API more effectively
12735 // - Currently only one replacement instruction due to adlc parsing capabilities
12736 //
12737 // // Change (inc mov) to lea
12738 // peephole %{
12739 //   // lea should only be emitted when beneficial
12740 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12741 //   // the rule numbers of these nodes inside are passed into the function below
12742 //   peepmatch ( incI_rReg movI );
12743 //   // the method that takes the responsibility of transformation
12744 //   peepprocedure ( inc_mov_to_lea );
12745 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
12746 //   // node is passed into the function above
12747 //   peepreplace ( leaI_rReg_immI() );
12748 // %}
12749 
12750 // These instructions is not matched by the matcher but used by the peephole
12751 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
12752 %{
12753   predicate(false);
12754   match(Set dst (AddI src1 src2));
12755   format %{ "leal    $dst, [$src1 + $src2]" %}
12756   ins_encode %{
12757     Register dst = $dst$$Register;
12758     Register src1 = $src1$$Register;
12759     Register src2 = $src2$$Register;
12760     if (src1 != rbp && src1 != r13) {
12761       __ leal(dst, Address(src1, src2, Address::times_1));
12762     } else {
12763       assert(src2 != rbp && src2 != r13, "");
12764       __ leal(dst, Address(src2, src1, Address::times_1));
12765     }
12766   %}
12767   ins_pipe(ialu_reg_reg);
12768 %}
12769 
12770 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
12771 %{
12772   predicate(false);
12773   match(Set dst (AddI src1 src2));
12774   format %{ "leal    $dst, [$src1 + $src2]" %}
12775   ins_encode %{
12776     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
12777   %}
12778   ins_pipe(ialu_reg_reg);
12779 %}
12780 
12781 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
12782 %{
12783   predicate(false);
12784   match(Set dst (LShiftI src shift));
12785   format %{ "leal    $dst, [$src << $shift]" %}
12786   ins_encode %{
12787     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12788     Register src = $src$$Register;
12789     if (scale == Address::times_2 && src != rbp && src != r13) {
12790       __ leal($dst$$Register, Address(src, src, Address::times_1));
12791     } else {
12792       __ leal($dst$$Register, Address(noreg, src, scale));
12793     }
12794   %}
12795   ins_pipe(ialu_reg_reg);
12796 %}
12797 
12798 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
12799 %{
12800   predicate(false);
12801   match(Set dst (AddL src1 src2));
12802   format %{ "leaq    $dst, [$src1 + $src2]" %}
12803   ins_encode %{
12804     Register dst = $dst$$Register;
12805     Register src1 = $src1$$Register;
12806     Register src2 = $src2$$Register;
12807     if (src1 != rbp && src1 != r13) {
12808       __ leaq(dst, Address(src1, src2, Address::times_1));
12809     } else {
12810       assert(src2 != rbp && src2 != r13, "");
12811       __ leaq(dst, Address(src2, src1, Address::times_1));
12812     }
12813   %}
12814   ins_pipe(ialu_reg_reg);
12815 %}
12816 
12817 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
12818 %{
12819   predicate(false);
12820   match(Set dst (AddL src1 src2));
12821   format %{ "leaq    $dst, [$src1 + $src2]" %}
12822   ins_encode %{
12823     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
12824   %}
12825   ins_pipe(ialu_reg_reg);
12826 %}
12827 
12828 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
12829 %{
12830   predicate(false);
12831   match(Set dst (LShiftL src shift));
12832   format %{ "leaq    $dst, [$src << $shift]" %}
12833   ins_encode %{
12834     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12835     Register src = $src$$Register;
12836     if (scale == Address::times_2 && src != rbp && src != r13) {
12837       __ leaq($dst$$Register, Address(src, src, Address::times_1));
12838     } else {
12839       __ leaq($dst$$Register, Address(noreg, src, scale));
12840     }
12841   %}
12842   ins_pipe(ialu_reg_reg);
12843 %}
12844 
12845 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
12846 // sal}) with lea instructions. The {add, sal} rules are beneficial in
12847 // processors with at least partial ALU support for lea
12848 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
12849 // beneficial for processors with full ALU support
12850 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
12851 
12852 peephole
12853 %{
12854   peeppredicate(VM_Version::supports_fast_2op_lea());
12855   peepmatch (addI_rReg);
12856   peepprocedure (lea_coalesce_reg);
12857   peepreplace (leaI_rReg_rReg_peep());
12858 %}
12859 
12860 peephole
12861 %{
12862   peeppredicate(VM_Version::supports_fast_2op_lea());
12863   peepmatch (addI_rReg_imm);
12864   peepprocedure (lea_coalesce_imm);
12865   peepreplace (leaI_rReg_immI_peep());
12866 %}
12867 
12868 peephole
12869 %{
12870   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12871                 VM_Version::is_intel_cascade_lake());
12872   peepmatch (incI_rReg);
12873   peepprocedure (lea_coalesce_imm);
12874   peepreplace (leaI_rReg_immI_peep());
12875 %}
12876 
12877 peephole
12878 %{
12879   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12880                 VM_Version::is_intel_cascade_lake());
12881   peepmatch (decI_rReg);
12882   peepprocedure (lea_coalesce_imm);
12883   peepreplace (leaI_rReg_immI_peep());
12884 %}
12885 
12886 peephole
12887 %{
12888   peeppredicate(VM_Version::supports_fast_2op_lea());
12889   peepmatch (salI_rReg_immI2);
12890   peepprocedure (lea_coalesce_imm);
12891   peepreplace (leaI_rReg_immI2_peep());
12892 %}
12893 
12894 peephole
12895 %{
12896   peeppredicate(VM_Version::supports_fast_2op_lea());
12897   peepmatch (addL_rReg);
12898   peepprocedure (lea_coalesce_reg);
12899   peepreplace (leaL_rReg_rReg_peep());
12900 %}
12901 
12902 peephole
12903 %{
12904   peeppredicate(VM_Version::supports_fast_2op_lea());
12905   peepmatch (addL_rReg_imm);
12906   peepprocedure (lea_coalesce_imm);
12907   peepreplace (leaL_rReg_immL32_peep());
12908 %}
12909 
12910 peephole
12911 %{
12912   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12913                 VM_Version::is_intel_cascade_lake());
12914   peepmatch (incL_rReg);
12915   peepprocedure (lea_coalesce_imm);
12916   peepreplace (leaL_rReg_immL32_peep());
12917 %}
12918 
12919 peephole
12920 %{
12921   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12922                 VM_Version::is_intel_cascade_lake());
12923   peepmatch (decL_rReg);
12924   peepprocedure (lea_coalesce_imm);
12925   peepreplace (leaL_rReg_immL32_peep());
12926 %}
12927 
12928 peephole
12929 %{
12930   peeppredicate(VM_Version::supports_fast_2op_lea());
12931   peepmatch (salL_rReg_immI2);
12932   peepprocedure (lea_coalesce_imm);
12933   peepreplace (leaL_rReg_immI2_peep());
12934 %}
12935 
12936 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
12937 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
12938 
12939 //int variant
12940 peephole
12941 %{
12942   peepmatch (testI_reg);
12943   peepprocedure (test_may_remove);
12944 %}
12945 
12946 //long variant
12947 peephole
12948 %{
12949   peepmatch (testL_reg);
12950   peepprocedure (test_may_remove);
12951 %}
12952 
12953 
12954 //----------SMARTSPILL RULES---------------------------------------------------
12955 // These must follow all instruction definitions as they use the names
12956 // defined in the instructions definitions.