1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RDX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RDI int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 
  323 source_hpp %{
  324 
  325 #include "peephole_x86_64.hpp"
  326 
  327 %}
  328 
  329 // Register masks
  330 source_hpp %{
  331 
  332 extern RegMask _ANY_REG_mask;
  333 extern RegMask _PTR_REG_mask;
  334 extern RegMask _PTR_REG_NO_RBP_mask;
  335 extern RegMask _PTR_NO_RAX_REG_mask;
  336 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  337 extern RegMask _LONG_REG_mask;
  338 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  339 extern RegMask _LONG_NO_RCX_REG_mask;
  340 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  341 extern RegMask _INT_REG_mask;
  342 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  343 extern RegMask _INT_NO_RCX_REG_mask;
  344 extern RegMask _INT_NO_RBP_R13_REG_mask;
  345 extern RegMask _FLOAT_REG_mask;
  346 
  347 extern RegMask _STACK_OR_PTR_REG_mask;
  348 extern RegMask _STACK_OR_LONG_REG_mask;
  349 extern RegMask _STACK_OR_INT_REG_mask;
  350 
  351 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  352 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  353 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_INT_REG_mask;
  380 
  381 static bool need_r12_heapbase() {
  382   return UseCompressedOops;
  383 }
  384 
  385 void reg_mask_init() {
  386   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  387   // We derive a number of subsets from it.
  388   _ANY_REG_mask = _ALL_REG_mask;
  389 
  390   if (PreserveFramePointer) {
  391     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  392     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  393   }
  394   if (need_r12_heapbase()) {
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  396     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  397   }
  398 
  399   _PTR_REG_mask = _ANY_REG_mask;
  400   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  401   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  404 
  405   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  406   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  407 
  408   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  409   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  410   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  411 
  412   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  413   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  414   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  415 
  416   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  417   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  418   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  419 
  420   _LONG_REG_mask = _PTR_REG_mask;
  421   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  422   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  423 
  424   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  425   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  426   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  429 
  430   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  431   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  432   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  433 
  434   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  435   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  436   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  439 
  440   _INT_REG_mask = _ALL_INT_REG_mask;
  441   if (PreserveFramePointer) {
  442     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  443   }
  444   if (need_r12_heapbase()) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  446   }
  447 
  448   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  449   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  450 
  451   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  452   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  453   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  454 
  455   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  456   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  457 
  458   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  459   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  460   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  461 
  462   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  463   // from the float_reg_legacy/float_reg_evex register class.
  464   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  465 }
  466 
  467 static bool generate_vzeroupper(Compile* C) {
  468   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  469 }
  470 
  471 static int clear_avx_size() {
  472   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  473 }
  474 
  475 // !!!!! Special hack to get all types of calls to specify the byte offset
  476 //       from the start of the call to the point where the return address
  477 //       will point.
  478 int MachCallStaticJavaNode::ret_addr_offset()
  479 {
  480   int offset = 5; // 5 bytes from start of call to where return address points
  481   offset += clear_avx_size();
  482   return offset;
  483 }
  484 
  485 int MachCallDynamicJavaNode::ret_addr_offset()
  486 {
  487   int offset = 15; // 15 bytes from start of call to where return address points
  488   offset += clear_avx_size();
  489   return offset;
  490 }
  491 
  492 int MachCallRuntimeNode::ret_addr_offset() {
  493   int offset = 13; // movq r10,#addr; callq (r10)
  494   if (this->ideal_Opcode() != Op_CallLeafVector) {
  495     offset += clear_avx_size();
  496   }
  497   return offset;
  498 }
  499 //
  500 // Compute padding required for nodes which need alignment
  501 //
  502 
  503 // The address of the call instruction needs to be 4-byte aligned to
  504 // ensure that it does not span a cache line so that it can be patched.
  505 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  506 {
  507   current_offset += clear_avx_size(); // skip vzeroupper
  508   current_offset += 1; // skip call opcode byte
  509   return align_up(current_offset, alignment_required()) - current_offset;
  510 }
  511 
  512 // The address of the call instruction needs to be 4-byte aligned to
  513 // ensure that it does not span a cache line so that it can be patched.
  514 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  515 {
  516   current_offset += clear_avx_size(); // skip vzeroupper
  517   current_offset += 11; // skip movq instruction + call opcode byte
  518   return align_up(current_offset, alignment_required()) - current_offset;
  519 }
  520 
  521 // This could be in MacroAssembler but it's fairly C2 specific
  522 static void emit_cmpfp_fixup(MacroAssembler& _masm) {
  523   Label exit;
  524   __ jccb(Assembler::noParity, exit);
  525   __ pushf();
  526   //
  527   // comiss/ucomiss instructions set ZF,PF,CF flags and
  528   // zero OF,AF,SF for NaN values.
  529   // Fixup flags by zeroing ZF,PF so that compare of NaN
  530   // values returns 'less than' result (CF is set).
  531   // Leave the rest of flags unchanged.
  532   //
  533   //    7 6 5 4 3 2 1 0
  534   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  535   //    0 0 1 0 1 0 1 1   (0x2B)
  536   //
  537   __ andq(Address(rsp, 0), 0xffffff2b);
  538   __ popf();
  539   __ bind(exit);
  540 }
  541 
  542 static void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  543   Label done;
  544   __ movl(dst, -1);
  545   __ jcc(Assembler::parity, done);
  546   __ jcc(Assembler::below, done);
  547   __ setb(Assembler::notEqual, dst);
  548   __ movzbl(dst, dst);
  549   __ bind(done);
  550 }
  551 
  552 // Math.min()    # Math.max()
  553 // --------------------------
  554 // ucomis[s/d]   #
  555 // ja   -> b     # a
  556 // jp   -> NaN   # NaN
  557 // jb   -> a     # b
  558 // je            #
  559 // |-jz -> a | b # a & b
  560 // |    -> a     #
  561 static void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  562                             XMMRegister a, XMMRegister b,
  563                             XMMRegister xmmt, Register rt,
  564                             bool min, bool single) {
  565 
  566   Label nan, zero, below, above, done;
  567 
  568   if (single)
  569     __ ucomiss(a, b);
  570   else
  571     __ ucomisd(a, b);
  572 
  573   if (dst->encoding() != (min ? b : a)->encoding())
  574     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  575   else
  576     __ jccb(Assembler::above, done);
  577 
  578   __ jccb(Assembler::parity, nan);  // PF=1
  579   __ jccb(Assembler::below, below); // CF=1
  580 
  581   // equal
  582   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  583   if (single) {
  584     __ ucomiss(a, xmmt);
  585     __ jccb(Assembler::equal, zero);
  586 
  587     __ movflt(dst, a);
  588     __ jmp(done);
  589   }
  590   else {
  591     __ ucomisd(a, xmmt);
  592     __ jccb(Assembler::equal, zero);
  593 
  594     __ movdbl(dst, a);
  595     __ jmp(done);
  596   }
  597 
  598   __ bind(zero);
  599   if (min)
  600     __ vpor(dst, a, b, Assembler::AVX_128bit);
  601   else
  602     __ vpand(dst, a, b, Assembler::AVX_128bit);
  603 
  604   __ jmp(done);
  605 
  606   __ bind(above);
  607   if (single)
  608     __ movflt(dst, min ? b : a);
  609   else
  610     __ movdbl(dst, min ? b : a);
  611 
  612   __ jmp(done);
  613 
  614   __ bind(nan);
  615   if (single) {
  616     __ movl(rt, 0x7fc00000); // Float.NaN
  617     __ movdl(dst, rt);
  618   }
  619   else {
  620     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  621     __ movdq(dst, rt);
  622   }
  623   __ jmp(done);
  624 
  625   __ bind(below);
  626   if (single)
  627     __ movflt(dst, min ? a : b);
  628   else
  629     __ movdbl(dst, min ? a : b);
  630 
  631   __ bind(done);
  632 }
  633 
  634 //=============================================================================
  635 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  636 
  637 int ConstantTable::calculate_table_base_offset() const {
  638   return 0;  // absolute addressing, no offset
  639 }
  640 
  641 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  642 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  643   ShouldNotReachHere();
  644 }
  645 
  646 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  647   // Empty encoding
  648 }
  649 
  650 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  651   return 0;
  652 }
  653 
  654 #ifndef PRODUCT
  655 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  656   st->print("# MachConstantBaseNode (empty encoding)");
  657 }
  658 #endif
  659 
  660 
  661 //=============================================================================
  662 #ifndef PRODUCT
  663 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  664   Compile* C = ra_->C;
  665 
  666   int framesize = C->output()->frame_size_in_bytes();
  667   int bangsize = C->output()->bang_size_in_bytes();
  668   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  669   // Remove wordSize for return addr which is already pushed.
  670   framesize -= wordSize;
  671 
  672   if (C->output()->need_stack_bang(bangsize)) {
  673     framesize -= wordSize;
  674     st->print("# stack bang (%d bytes)", bangsize);
  675     st->print("\n\t");
  676     st->print("pushq   rbp\t# Save rbp");
  677     if (PreserveFramePointer) {
  678         st->print("\n\t");
  679         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  680     }
  681     if (framesize) {
  682       st->print("\n\t");
  683       st->print("subq    rsp, #%d\t# Create frame",framesize);
  684     }
  685   } else {
  686     st->print("subq    rsp, #%d\t# Create frame",framesize);
  687     st->print("\n\t");
  688     framesize -= wordSize;
  689     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  690     if (PreserveFramePointer) {
  691       st->print("\n\t");
  692       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  693       if (framesize > 0) {
  694         st->print("\n\t");
  695         st->print("addq    rbp, #%d", framesize);
  696       }
  697     }
  698   }
  699 
  700   if (VerifyStackAtCalls) {
  701     st->print("\n\t");
  702     framesize -= wordSize;
  703     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  704 #ifdef ASSERT
  705     st->print("\n\t");
  706     st->print("# stack alignment check");
  707 #endif
  708   }
  709   if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
  710     st->print("\n\t");
  711     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  712     st->print("\n\t");
  713     st->print("je      fast_entry\t");
  714     st->print("\n\t");
  715     st->print("call    #nmethod_entry_barrier_stub\t");
  716     st->print("\n\tfast_entry:");
  717   }
  718   st->cr();
  719 }
  720 #endif
  721 
  722 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  723   Compile* C = ra_->C;
  724   C2_MacroAssembler _masm(&cbuf);
  725 
  726   int framesize = C->output()->frame_size_in_bytes();
  727   int bangsize = C->output()->bang_size_in_bytes();
  728 
  729   if (C->clinit_barrier_on_entry()) {
  730     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  731     assert(!C->method()->holder()->is_not_initialized() || C->do_clinit_barriers(), "initialization should have been started");
  732 
  733     Label L_skip_barrier;
  734     Register klass = rscratch1;
  735 
  736     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  737     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  738 
  739     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  740 
  741     __ bind(L_skip_barrier);
  742   }
  743 
  744   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  745 
  746   C->output()->set_frame_complete(cbuf.insts_size());
  747 
  748   if (C->has_mach_constant_base_node()) {
  749     // NOTE: We set the table base offset here because users might be
  750     // emitted before MachConstantBaseNode.
  751     ConstantTable& constant_table = C->output()->constant_table();
  752     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  753   }
  754 }
  755 
  756 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  757 {
  758   return MachNode::size(ra_); // too many variables; just compute it
  759                               // the hard way
  760 }
  761 
  762 int MachPrologNode::reloc() const
  763 {
  764   return 0; // a large enough number
  765 }
  766 
  767 //=============================================================================
  768 #ifndef PRODUCT
  769 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  770 {
  771   Compile* C = ra_->C;
  772   if (generate_vzeroupper(C)) {
  773     st->print("vzeroupper");
  774     st->cr(); st->print("\t");
  775   }
  776 
  777   int framesize = C->output()->frame_size_in_bytes();
  778   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  779   // Remove word for return adr already pushed
  780   // and RBP
  781   framesize -= 2*wordSize;
  782 
  783   if (framesize) {
  784     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  785     st->print("\t");
  786   }
  787 
  788   st->print_cr("popq    rbp");
  789   if (do_polling() && C->is_method_compilation()) {
  790     st->print("\t");
  791     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  792                  "ja      #safepoint_stub\t"
  793                  "# Safepoint: poll for GC");
  794   }
  795 }
  796 #endif
  797 
  798 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  799 {
  800   Compile* C = ra_->C;
  801   MacroAssembler _masm(&cbuf);
  802 
  803   if (generate_vzeroupper(C)) {
  804     // Clear upper bits of YMM registers when current compiled code uses
  805     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  806     __ vzeroupper();
  807   }
  808 
  809   int framesize = C->output()->frame_size_in_bytes();
  810   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  811   // Remove word for return adr already pushed
  812   // and RBP
  813   framesize -= 2*wordSize;
  814 
  815   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  816 
  817   if (framesize) {
  818     __ addq(rsp, framesize);
  819   }
  820 
  821   __ popq(rbp);
  822 
  823   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  824     __ reserved_stack_check();
  825   }
  826 
  827   if (do_polling() && C->is_method_compilation()) {
  828     MacroAssembler _masm(&cbuf);
  829     Label dummy_label;
  830     Label* code_stub = &dummy_label;
  831     if (!C->output()->in_scratch_emit_size()) {
  832       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  833       C->output()->add_stub(stub);
  834       code_stub = &stub->entry();
  835     }
  836     __ relocate(relocInfo::poll_return_type);
  837     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  838   }
  839 }
  840 
  841 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  842 {
  843   return MachNode::size(ra_); // too many variables; just compute it
  844                               // the hard way
  845 }
  846 
  847 int MachEpilogNode::reloc() const
  848 {
  849   return 2; // a large enough number
  850 }
  851 
  852 const Pipeline* MachEpilogNode::pipeline() const
  853 {
  854   return MachNode::pipeline_class();
  855 }
  856 
  857 //=============================================================================
  858 
  859 enum RC {
  860   rc_bad,
  861   rc_int,
  862   rc_kreg,
  863   rc_float,
  864   rc_stack
  865 };
  866 
  867 static enum RC rc_class(OptoReg::Name reg)
  868 {
  869   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  870 
  871   if (OptoReg::is_stack(reg)) return rc_stack;
  872 
  873   VMReg r = OptoReg::as_VMReg(reg);
  874 
  875   if (r->is_Register()) return rc_int;
  876 
  877   if (r->is_KRegister()) return rc_kreg;
  878 
  879   assert(r->is_XMMRegister(), "must be");
  880   return rc_float;
  881 }
  882 
  883 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  884 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
  885                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  886 
  887 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
  888                      int stack_offset, int reg, uint ireg, outputStream* st);
  889 
  890 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
  891                                       int dst_offset, uint ireg, outputStream* st) {
  892   if (cbuf) {
  893     MacroAssembler _masm(cbuf);
  894     switch (ireg) {
  895     case Op_VecS:
  896       __ movq(Address(rsp, -8), rax);
  897       __ movl(rax, Address(rsp, src_offset));
  898       __ movl(Address(rsp, dst_offset), rax);
  899       __ movq(rax, Address(rsp, -8));
  900       break;
  901     case Op_VecD:
  902       __ pushq(Address(rsp, src_offset));
  903       __ popq (Address(rsp, dst_offset));
  904       break;
  905     case Op_VecX:
  906       __ pushq(Address(rsp, src_offset));
  907       __ popq (Address(rsp, dst_offset));
  908       __ pushq(Address(rsp, src_offset+8));
  909       __ popq (Address(rsp, dst_offset+8));
  910       break;
  911     case Op_VecY:
  912       __ vmovdqu(Address(rsp, -32), xmm0);
  913       __ vmovdqu(xmm0, Address(rsp, src_offset));
  914       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  915       __ vmovdqu(xmm0, Address(rsp, -32));
  916       break;
  917     case Op_VecZ:
  918       __ evmovdquq(Address(rsp, -64), xmm0, 2);
  919       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
  920       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
  921       __ evmovdquq(xmm0, Address(rsp, -64), 2);
  922       break;
  923     default:
  924       ShouldNotReachHere();
  925     }
  926 #ifndef PRODUCT
  927   } else {
  928     switch (ireg) {
  929     case Op_VecS:
  930       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
  931                 "movl    rax, [rsp + #%d]\n\t"
  932                 "movl    [rsp + #%d], rax\n\t"
  933                 "movq    rax, [rsp - #8]",
  934                 src_offset, dst_offset);
  935       break;
  936     case Op_VecD:
  937       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
  938                 "popq    [rsp + #%d]",
  939                 src_offset, dst_offset);
  940       break;
  941      case Op_VecX:
  942       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
  943                 "popq    [rsp + #%d]\n\t"
  944                 "pushq   [rsp + #%d]\n\t"
  945                 "popq    [rsp + #%d]",
  946                 src_offset, dst_offset, src_offset+8, dst_offset+8);
  947       break;
  948     case Op_VecY:
  949       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
  950                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  951                 "vmovdqu [rsp + #%d], xmm0\n\t"
  952                 "vmovdqu xmm0, [rsp - #32]",
  953                 src_offset, dst_offset);
  954       break;
  955     case Op_VecZ:
  956       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
  957                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  958                 "vmovdqu [rsp + #%d], xmm0\n\t"
  959                 "vmovdqu xmm0, [rsp - #64]",
  960                 src_offset, dst_offset);
  961       break;
  962     default:
  963       ShouldNotReachHere();
  964     }
  965 #endif
  966   }
  967 }
  968 
  969 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
  970                                        PhaseRegAlloc* ra_,
  971                                        bool do_size,
  972                                        outputStream* st) const {
  973   assert(cbuf != nullptr || st  != nullptr, "sanity");
  974   // Get registers to move
  975   OptoReg::Name src_second = ra_->get_reg_second(in(1));
  976   OptoReg::Name src_first = ra_->get_reg_first(in(1));
  977   OptoReg::Name dst_second = ra_->get_reg_second(this);
  978   OptoReg::Name dst_first = ra_->get_reg_first(this);
  979 
  980   enum RC src_second_rc = rc_class(src_second);
  981   enum RC src_first_rc = rc_class(src_first);
  982   enum RC dst_second_rc = rc_class(dst_second);
  983   enum RC dst_first_rc = rc_class(dst_first);
  984 
  985   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
  986          "must move at least 1 register" );
  987 
  988   if (src_first == dst_first && src_second == dst_second) {
  989     // Self copy, no move
  990     return 0;
  991   }
  992   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
  993     uint ireg = ideal_reg();
  994     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
  995     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
  996     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
  997       // mem -> mem
  998       int src_offset = ra_->reg2offset(src_first);
  999       int dst_offset = ra_->reg2offset(dst_first);
 1000       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1001     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1002       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1003     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1004       int stack_offset = ra_->reg2offset(dst_first);
 1005       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1006     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1007       int stack_offset = ra_->reg2offset(src_first);
 1008       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1009     } else {
 1010       ShouldNotReachHere();
 1011     }
 1012     return 0;
 1013   }
 1014   if (src_first_rc == rc_stack) {
 1015     // mem ->
 1016     if (dst_first_rc == rc_stack) {
 1017       // mem -> mem
 1018       assert(src_second != dst_first, "overlap");
 1019       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1020           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1021         // 64-bit
 1022         int src_offset = ra_->reg2offset(src_first);
 1023         int dst_offset = ra_->reg2offset(dst_first);
 1024         if (cbuf) {
 1025           MacroAssembler _masm(cbuf);
 1026           __ pushq(Address(rsp, src_offset));
 1027           __ popq (Address(rsp, dst_offset));
 1028 #ifndef PRODUCT
 1029         } else {
 1030           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1031                     "popq    [rsp + #%d]",
 1032                      src_offset, dst_offset);
 1033 #endif
 1034         }
 1035       } else {
 1036         // 32-bit
 1037         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1038         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1039         // No pushl/popl, so:
 1040         int src_offset = ra_->reg2offset(src_first);
 1041         int dst_offset = ra_->reg2offset(dst_first);
 1042         if (cbuf) {
 1043           MacroAssembler _masm(cbuf);
 1044           __ movq(Address(rsp, -8), rax);
 1045           __ movl(rax, Address(rsp, src_offset));
 1046           __ movl(Address(rsp, dst_offset), rax);
 1047           __ movq(rax, Address(rsp, -8));
 1048 #ifndef PRODUCT
 1049         } else {
 1050           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                     "movl    rax, [rsp + #%d]\n\t"
 1052                     "movl    [rsp + #%d], rax\n\t"
 1053                     "movq    rax, [rsp - #8]",
 1054                      src_offset, dst_offset);
 1055 #endif
 1056         }
 1057       }
 1058       return 0;
 1059     } else if (dst_first_rc == rc_int) {
 1060       // mem -> gpr
 1061       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1062           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1063         // 64-bit
 1064         int offset = ra_->reg2offset(src_first);
 1065         if (cbuf) {
 1066           MacroAssembler _masm(cbuf);
 1067           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1068 #ifndef PRODUCT
 1069         } else {
 1070           st->print("movq    %s, [rsp + #%d]\t# spill",
 1071                      Matcher::regName[dst_first],
 1072                      offset);
 1073 #endif
 1074         }
 1075       } else {
 1076         // 32-bit
 1077         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1078         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1079         int offset = ra_->reg2offset(src_first);
 1080         if (cbuf) {
 1081           MacroAssembler _masm(cbuf);
 1082           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1083 #ifndef PRODUCT
 1084         } else {
 1085           st->print("movl    %s, [rsp + #%d]\t# spill",
 1086                      Matcher::regName[dst_first],
 1087                      offset);
 1088 #endif
 1089         }
 1090       }
 1091       return 0;
 1092     } else if (dst_first_rc == rc_float) {
 1093       // mem-> xmm
 1094       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1095           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1096         // 64-bit
 1097         int offset = ra_->reg2offset(src_first);
 1098         if (cbuf) {
 1099           MacroAssembler _masm(cbuf);
 1100           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1101 #ifndef PRODUCT
 1102         } else {
 1103           st->print("%s  %s, [rsp + #%d]\t# spill",
 1104                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1105                      Matcher::regName[dst_first],
 1106                      offset);
 1107 #endif
 1108         }
 1109       } else {
 1110         // 32-bit
 1111         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1112         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1113         int offset = ra_->reg2offset(src_first);
 1114         if (cbuf) {
 1115           MacroAssembler _masm(cbuf);
 1116           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1117 #ifndef PRODUCT
 1118         } else {
 1119           st->print("movss   %s, [rsp + #%d]\t# spill",
 1120                      Matcher::regName[dst_first],
 1121                      offset);
 1122 #endif
 1123         }
 1124       }
 1125       return 0;
 1126     } else if (dst_first_rc == rc_kreg) {
 1127       // mem -> kreg
 1128       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1129           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1130         // 64-bit
 1131         int offset = ra_->reg2offset(src_first);
 1132         if (cbuf) {
 1133           MacroAssembler _masm(cbuf);
 1134           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1135 #ifndef PRODUCT
 1136         } else {
 1137           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1138                      Matcher::regName[dst_first],
 1139                      offset);
 1140 #endif
 1141         }
 1142       }
 1143       return 0;
 1144     }
 1145   } else if (src_first_rc == rc_int) {
 1146     // gpr ->
 1147     if (dst_first_rc == rc_stack) {
 1148       // gpr -> mem
 1149       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1150           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1151         // 64-bit
 1152         int offset = ra_->reg2offset(dst_first);
 1153         if (cbuf) {
 1154           MacroAssembler _masm(cbuf);
 1155           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1156 #ifndef PRODUCT
 1157         } else {
 1158           st->print("movq    [rsp + #%d], %s\t# spill",
 1159                      offset,
 1160                      Matcher::regName[src_first]);
 1161 #endif
 1162         }
 1163       } else {
 1164         // 32-bit
 1165         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1166         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1167         int offset = ra_->reg2offset(dst_first);
 1168         if (cbuf) {
 1169           MacroAssembler _masm(cbuf);
 1170           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1171 #ifndef PRODUCT
 1172         } else {
 1173           st->print("movl    [rsp + #%d], %s\t# spill",
 1174                      offset,
 1175                      Matcher::regName[src_first]);
 1176 #endif
 1177         }
 1178       }
 1179       return 0;
 1180     } else if (dst_first_rc == rc_int) {
 1181       // gpr -> gpr
 1182       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1183           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1184         // 64-bit
 1185         if (cbuf) {
 1186           MacroAssembler _masm(cbuf);
 1187           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1188                   as_Register(Matcher::_regEncode[src_first]));
 1189 #ifndef PRODUCT
 1190         } else {
 1191           st->print("movq    %s, %s\t# spill",
 1192                      Matcher::regName[dst_first],
 1193                      Matcher::regName[src_first]);
 1194 #endif
 1195         }
 1196         return 0;
 1197       } else {
 1198         // 32-bit
 1199         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1200         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1201         if (cbuf) {
 1202           MacroAssembler _masm(cbuf);
 1203           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1204                   as_Register(Matcher::_regEncode[src_first]));
 1205 #ifndef PRODUCT
 1206         } else {
 1207           st->print("movl    %s, %s\t# spill",
 1208                      Matcher::regName[dst_first],
 1209                      Matcher::regName[src_first]);
 1210 #endif
 1211         }
 1212         return 0;
 1213       }
 1214     } else if (dst_first_rc == rc_float) {
 1215       // gpr -> xmm
 1216       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1217           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1218         // 64-bit
 1219         if (cbuf) {
 1220           MacroAssembler _masm(cbuf);
 1221           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1222 #ifndef PRODUCT
 1223         } else {
 1224           st->print("movdq   %s, %s\t# spill",
 1225                      Matcher::regName[dst_first],
 1226                      Matcher::regName[src_first]);
 1227 #endif
 1228         }
 1229       } else {
 1230         // 32-bit
 1231         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1232         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1233         if (cbuf) {
 1234           MacroAssembler _masm(cbuf);
 1235           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1236 #ifndef PRODUCT
 1237         } else {
 1238           st->print("movdl   %s, %s\t# spill",
 1239                      Matcher::regName[dst_first],
 1240                      Matcher::regName[src_first]);
 1241 #endif
 1242         }
 1243       }
 1244       return 0;
 1245     } else if (dst_first_rc == rc_kreg) {
 1246       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1247           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1248         // 64-bit
 1249         if (cbuf) {
 1250           MacroAssembler _masm(cbuf);
 1251           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1252   #ifndef PRODUCT
 1253         } else {
 1254            st->print("kmovq   %s, %s\t# spill",
 1255                        Matcher::regName[dst_first],
 1256                        Matcher::regName[src_first]);
 1257   #endif
 1258         }
 1259       }
 1260       Unimplemented();
 1261       return 0;
 1262     }
 1263   } else if (src_first_rc == rc_float) {
 1264     // xmm ->
 1265     if (dst_first_rc == rc_stack) {
 1266       // xmm -> mem
 1267       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1268           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1269         // 64-bit
 1270         int offset = ra_->reg2offset(dst_first);
 1271         if (cbuf) {
 1272           MacroAssembler _masm(cbuf);
 1273           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1274 #ifndef PRODUCT
 1275         } else {
 1276           st->print("movsd   [rsp + #%d], %s\t# spill",
 1277                      offset,
 1278                      Matcher::regName[src_first]);
 1279 #endif
 1280         }
 1281       } else {
 1282         // 32-bit
 1283         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1284         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1285         int offset = ra_->reg2offset(dst_first);
 1286         if (cbuf) {
 1287           MacroAssembler _masm(cbuf);
 1288           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1289 #ifndef PRODUCT
 1290         } else {
 1291           st->print("movss   [rsp + #%d], %s\t# spill",
 1292                      offset,
 1293                      Matcher::regName[src_first]);
 1294 #endif
 1295         }
 1296       }
 1297       return 0;
 1298     } else if (dst_first_rc == rc_int) {
 1299       // xmm -> gpr
 1300       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1301           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1302         // 64-bit
 1303         if (cbuf) {
 1304           MacroAssembler _masm(cbuf);
 1305           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1306 #ifndef PRODUCT
 1307         } else {
 1308           st->print("movdq   %s, %s\t# spill",
 1309                      Matcher::regName[dst_first],
 1310                      Matcher::regName[src_first]);
 1311 #endif
 1312         }
 1313       } else {
 1314         // 32-bit
 1315         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1316         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1317         if (cbuf) {
 1318           MacroAssembler _masm(cbuf);
 1319           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1320 #ifndef PRODUCT
 1321         } else {
 1322           st->print("movdl   %s, %s\t# spill",
 1323                      Matcher::regName[dst_first],
 1324                      Matcher::regName[src_first]);
 1325 #endif
 1326         }
 1327       }
 1328       return 0;
 1329     } else if (dst_first_rc == rc_float) {
 1330       // xmm -> xmm
 1331       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1332           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1333         // 64-bit
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("%s  %s, %s\t# spill",
 1340                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1341                      Matcher::regName[dst_first],
 1342                      Matcher::regName[src_first]);
 1343 #endif
 1344         }
 1345       } else {
 1346         // 32-bit
 1347         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1348         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("%s  %s, %s\t# spill",
 1355                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1356                      Matcher::regName[dst_first],
 1357                      Matcher::regName[src_first]);
 1358 #endif
 1359         }
 1360       }
 1361       return 0;
 1362     } else if (dst_first_rc == rc_kreg) {
 1363       assert(false, "Illegal spilling");
 1364       return 0;
 1365     }
 1366   } else if (src_first_rc == rc_kreg) {
 1367     if (dst_first_rc == rc_stack) {
 1368       // mem -> kreg
 1369       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1370           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1371         // 64-bit
 1372         int offset = ra_->reg2offset(dst_first);
 1373         if (cbuf) {
 1374           MacroAssembler _masm(cbuf);
 1375           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1376 #ifndef PRODUCT
 1377         } else {
 1378           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1379                      offset,
 1380                      Matcher::regName[src_first]);
 1381 #endif
 1382         }
 1383       }
 1384       return 0;
 1385     } else if (dst_first_rc == rc_int) {
 1386       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1387           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1388         // 64-bit
 1389         if (cbuf) {
 1390           MacroAssembler _masm(cbuf);
 1391           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1392 #ifndef PRODUCT
 1393         } else {
 1394          st->print("kmovq   %s, %s\t# spill",
 1395                      Matcher::regName[dst_first],
 1396                      Matcher::regName[src_first]);
 1397 #endif
 1398         }
 1399       }
 1400       Unimplemented();
 1401       return 0;
 1402     } else if (dst_first_rc == rc_kreg) {
 1403       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1404           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1405         // 64-bit
 1406         if (cbuf) {
 1407           MacroAssembler _masm(cbuf);
 1408           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411          st->print("kmovq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       }
 1417       return 0;
 1418     } else if (dst_first_rc == rc_float) {
 1419       assert(false, "Illegal spill");
 1420       return 0;
 1421     }
 1422   }
 1423 
 1424   assert(0," foo ");
 1425   Unimplemented();
 1426   return 0;
 1427 }
 1428 
 1429 #ifndef PRODUCT
 1430 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1431   implementation(nullptr, ra_, false, st);
 1432 }
 1433 #endif
 1434 
 1435 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1436   implementation(&cbuf, ra_, false, nullptr);
 1437 }
 1438 
 1439 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1440   return MachNode::size(ra_);
 1441 }
 1442 
 1443 //=============================================================================
 1444 #ifndef PRODUCT
 1445 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1446 {
 1447   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1448   int reg = ra_->get_reg_first(this);
 1449   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1450             Matcher::regName[reg], offset);
 1451 }
 1452 #endif
 1453 
 1454 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1455 {
 1456   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1457   int reg = ra_->get_encode(this);
 1458 
 1459   MacroAssembler masm(&cbuf);
 1460   masm.lea(as_Register(reg), Address(rsp, offset));
 1461 }
 1462 
 1463 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1464 {
 1465   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1466   return (offset < 0x80) ? 5 : 8; // REX
 1467 }
 1468 
 1469 //=============================================================================
 1470 #ifndef PRODUCT
 1471 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1472 {
 1473   if (UseCompressedClassPointers) {
 1474     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1475     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1476   } else {
 1477     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1478     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1479   }
 1480   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1481 }
 1482 #endif
 1483 
 1484 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1485 {
 1486   MacroAssembler masm(&cbuf);
 1487   masm.ic_check(InteriorEntryAlignment);
 1488 }
 1489 
 1490 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1491 {
 1492   return MachNode::size(ra_); // too many variables; just compute it
 1493                               // the hard way
 1494 }
 1495 
 1496 
 1497 //=============================================================================
 1498 
 1499 bool Matcher::supports_vector_calling_convention(void) {
 1500   if (EnableVectorSupport && UseVectorStubs) {
 1501     return true;
 1502   }
 1503   return false;
 1504 }
 1505 
 1506 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1507   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1508   int lo = XMM0_num;
 1509   int hi = XMM0b_num;
 1510   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1511   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1512   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1513   return OptoRegPair(hi, lo);
 1514 }
 1515 
 1516 // Is this branch offset short enough that a short branch can be used?
 1517 //
 1518 // NOTE: If the platform does not provide any short branch variants, then
 1519 //       this method should return false for offset 0.
 1520 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1521   // The passed offset is relative to address of the branch.
 1522   // On 86 a branch displacement is calculated relative to address
 1523   // of a next instruction.
 1524   offset -= br_size;
 1525 
 1526   // the short version of jmpConUCF2 contains multiple branches,
 1527   // making the reach slightly less
 1528   if (rule == jmpConUCF2_rule)
 1529     return (-126 <= offset && offset <= 125);
 1530   return (-128 <= offset && offset <= 127);
 1531 }
 1532 
 1533 // Return whether or not this register is ever used as an argument.
 1534 // This function is used on startup to build the trampoline stubs in
 1535 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1536 // call in the trampoline, and arguments in those registers not be
 1537 // available to the callee.
 1538 bool Matcher::can_be_java_arg(int reg)
 1539 {
 1540   return
 1541     reg ==  RDI_num || reg == RDI_H_num ||
 1542     reg ==  RSI_num || reg == RSI_H_num ||
 1543     reg ==  RDX_num || reg == RDX_H_num ||
 1544     reg ==  RCX_num || reg == RCX_H_num ||
 1545     reg ==   R8_num || reg ==  R8_H_num ||
 1546     reg ==   R9_num || reg ==  R9_H_num ||
 1547     reg ==  R12_num || reg == R12_H_num ||
 1548     reg == XMM0_num || reg == XMM0b_num ||
 1549     reg == XMM1_num || reg == XMM1b_num ||
 1550     reg == XMM2_num || reg == XMM2b_num ||
 1551     reg == XMM3_num || reg == XMM3b_num ||
 1552     reg == XMM4_num || reg == XMM4b_num ||
 1553     reg == XMM5_num || reg == XMM5b_num ||
 1554     reg == XMM6_num || reg == XMM6b_num ||
 1555     reg == XMM7_num || reg == XMM7b_num;
 1556 }
 1557 
 1558 bool Matcher::is_spillable_arg(int reg)
 1559 {
 1560   return can_be_java_arg(reg);
 1561 }
 1562 
 1563 uint Matcher::int_pressure_limit()
 1564 {
 1565   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1566 }
 1567 
 1568 uint Matcher::float_pressure_limit()
 1569 {
 1570   // After experiment around with different values, the following default threshold
 1571   // works best for LCM's register pressure scheduling on x64.
 1572   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1573   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1574   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1575 }
 1576 
 1577 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1578   // In 64 bit mode a code which use multiply when
 1579   // devisor is constant is faster than hardware
 1580   // DIV instruction (it uses MulHiL).
 1581   return false;
 1582 }
 1583 
 1584 // Register for DIVI projection of divmodI
 1585 RegMask Matcher::divI_proj_mask() {
 1586   return INT_RAX_REG_mask();
 1587 }
 1588 
 1589 // Register for MODI projection of divmodI
 1590 RegMask Matcher::modI_proj_mask() {
 1591   return INT_RDX_REG_mask();
 1592 }
 1593 
 1594 // Register for DIVL projection of divmodL
 1595 RegMask Matcher::divL_proj_mask() {
 1596   return LONG_RAX_REG_mask();
 1597 }
 1598 
 1599 // Register for MODL projection of divmodL
 1600 RegMask Matcher::modL_proj_mask() {
 1601   return LONG_RDX_REG_mask();
 1602 }
 1603 
 1604 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1605 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1606     return NO_REG_mask();
 1607 }
 1608 
 1609 %}
 1610 
 1611 //----------ENCODING BLOCK-----------------------------------------------------
 1612 // This block specifies the encoding classes used by the compiler to
 1613 // output byte streams.  Encoding classes are parameterized macros
 1614 // used by Machine Instruction Nodes in order to generate the bit
 1615 // encoding of the instruction.  Operands specify their base encoding
 1616 // interface with the interface keyword.  There are currently
 1617 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1618 // COND_INTER.  REG_INTER causes an operand to generate a function
 1619 // which returns its register number when queried.  CONST_INTER causes
 1620 // an operand to generate a function which returns the value of the
 1621 // constant when queried.  MEMORY_INTER causes an operand to generate
 1622 // four functions which return the Base Register, the Index Register,
 1623 // the Scale Value, and the Offset Value of the operand when queried.
 1624 // COND_INTER causes an operand to generate six functions which return
 1625 // the encoding code (ie - encoding bits for the instruction)
 1626 // associated with each basic boolean condition for a conditional
 1627 // instruction.
 1628 //
 1629 // Instructions specify two basic values for encoding.  Again, a
 1630 // function is available to check if the constant displacement is an
 1631 // oop. They use the ins_encode keyword to specify their encoding
 1632 // classes (which must be a sequence of enc_class names, and their
 1633 // parameters, specified in the encoding block), and they use the
 1634 // opcode keyword to specify, in order, their primary, secondary, and
 1635 // tertiary opcode.  Only the opcode sections which a particular
 1636 // instruction needs for encoding need to be specified.
 1637 encode %{
 1638   enc_class cdql_enc(no_rax_rdx_RegI div)
 1639   %{
 1640     // Full implementation of Java idiv and irem; checks for
 1641     // special case as described in JVM spec., p.243 & p.271.
 1642     //
 1643     //         normal case                           special case
 1644     //
 1645     // input : rax: dividend                         min_int
 1646     //         reg: divisor                          -1
 1647     //
 1648     // output: rax: quotient  (= rax idiv reg)       min_int
 1649     //         rdx: remainder (= rax irem reg)       0
 1650     //
 1651     //  Code sequnce:
 1652     //
 1653     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1654     //    5:   75 07/08                jne    e <normal>
 1655     //    7:   33 d2                   xor    %edx,%edx
 1656     //  [div >= 8 -> offset + 1]
 1657     //  [REX_B]
 1658     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1659     //    c:   74 03/04                je     11 <done>
 1660     // 000000000000000e <normal>:
 1661     //    e:   99                      cltd
 1662     //  [div >= 8 -> offset + 1]
 1663     //  [REX_B]
 1664     //    f:   f7 f9                   idiv   $div
 1665     // 0000000000000011 <done>:
 1666     MacroAssembler _masm(&cbuf);
 1667     Label normal;
 1668     Label done;
 1669 
 1670     // cmp    $0x80000000,%eax
 1671     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1672 
 1673     // jne    e <normal>
 1674     __ jccb(Assembler::notEqual, normal);
 1675 
 1676     // xor    %edx,%edx
 1677     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1678 
 1679     // cmp    $0xffffffffffffffff,%ecx
 1680     __ cmpl($div$$Register, -1);
 1681 
 1682     // je     11 <done>
 1683     __ jccb(Assembler::equal, done);
 1684 
 1685     // <normal>
 1686     // cltd
 1687     __ bind(normal);
 1688     __ cdql();
 1689 
 1690     // idivl
 1691     // <done>
 1692     __ idivl($div$$Register);
 1693     __ bind(done);
 1694   %}
 1695 
 1696   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1697   %{
 1698     // Full implementation of Java ldiv and lrem; checks for
 1699     // special case as described in JVM spec., p.243 & p.271.
 1700     //
 1701     //         normal case                           special case
 1702     //
 1703     // input : rax: dividend                         min_long
 1704     //         reg: divisor                          -1
 1705     //
 1706     // output: rax: quotient  (= rax idiv reg)       min_long
 1707     //         rdx: remainder (= rax irem reg)       0
 1708     //
 1709     //  Code sequnce:
 1710     //
 1711     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1712     //    7:   00 00 80
 1713     //    a:   48 39 d0                cmp    %rdx,%rax
 1714     //    d:   75 08                   jne    17 <normal>
 1715     //    f:   33 d2                   xor    %edx,%edx
 1716     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1717     //   15:   74 05                   je     1c <done>
 1718     // 0000000000000017 <normal>:
 1719     //   17:   48 99                   cqto
 1720     //   19:   48 f7 f9                idiv   $div
 1721     // 000000000000001c <done>:
 1722     MacroAssembler _masm(&cbuf);
 1723     Label normal;
 1724     Label done;
 1725 
 1726     // mov    $0x8000000000000000,%rdx
 1727     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1728 
 1729     // cmp    %rdx,%rax
 1730     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1731 
 1732     // jne    17 <normal>
 1733     __ jccb(Assembler::notEqual, normal);
 1734 
 1735     // xor    %edx,%edx
 1736     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1737 
 1738     // cmp    $0xffffffffffffffff,$div
 1739     __ cmpq($div$$Register, -1);
 1740 
 1741     // je     1e <done>
 1742     __ jccb(Assembler::equal, done);
 1743 
 1744     // <normal>
 1745     // cqto
 1746     __ bind(normal);
 1747     __ cdqq();
 1748 
 1749     // idivq (note: must be emitted by the user of this rule)
 1750     // <done>
 1751     __ idivq($div$$Register);
 1752     __ bind(done);
 1753   %}
 1754 
 1755   enc_class enc_PartialSubtypeCheck()
 1756   %{
 1757     Register Rrdi = as_Register(RDI_enc); // result register
 1758     Register Rrax = as_Register(RAX_enc); // super class
 1759     Register Rrcx = as_Register(RCX_enc); // killed
 1760     Register Rrsi = as_Register(RSI_enc); // sub class
 1761     Label miss;
 1762     const bool set_cond_codes = true;
 1763 
 1764     MacroAssembler _masm(&cbuf);
 1765     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 1766                                      nullptr, &miss,
 1767                                      /*set_cond_codes:*/ true);
 1768     if ($primary) {
 1769       __ xorptr(Rrdi, Rrdi);
 1770     }
 1771     __ bind(miss);
 1772   %}
 1773 
 1774   enc_class clear_avx %{
 1775     debug_only(int off0 = cbuf.insts_size());
 1776     if (generate_vzeroupper(Compile::current())) {
 1777       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1778       // Clear upper bits of YMM registers when current compiled code uses
 1779       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1780       MacroAssembler _masm(&cbuf);
 1781       __ vzeroupper();
 1782     }
 1783     debug_only(int off1 = cbuf.insts_size());
 1784     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1785   %}
 1786 
 1787   enc_class Java_To_Runtime(method meth) %{
 1788     // No relocation needed
 1789     MacroAssembler _masm(&cbuf);
 1790     if (SCCache::is_on_for_write()) {
 1791       // Created runtime_call_type relocation when caching code
 1792       __ lea(r10, RuntimeAddress((address)$meth$$method));
 1793     } else {
 1794       __ mov64(r10, (int64_t) $meth$$method);
 1795     }
 1796     __ call(r10);
 1797     __ post_call_nop();
 1798   %}
 1799 
 1800   enc_class Java_Static_Call(method meth)
 1801   %{
 1802     // JAVA STATIC CALL
 1803     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1804     // determine who we intended to call.
 1805     MacroAssembler _masm(&cbuf);
 1806 
 1807     if (!_method) {
 1808       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1809     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1810       // The NOP here is purely to ensure that eliding a call to
 1811       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1812       __ addr_nop_5();
 1813       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1814     } else {
 1815       int method_index = resolved_method_index(cbuf);
 1816       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1817                                                   : static_call_Relocation::spec(method_index);
 1818       address mark = __ pc();
 1819       int call_offset = __ offset();
 1820       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1821       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1822         // Calls of the same statically bound method can share
 1823         // a stub to the interpreter.
 1824         cbuf.shared_stub_to_interp_for(_method, call_offset);
 1825       } else {
 1826         // Emit stubs for static call.
 1827         address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, mark);
 1828         if (stub == nullptr) {
 1829           ciEnv::current()->record_failure("CodeCache is full");
 1830           return;
 1831         }
 1832       }
 1833     }
 1834     __ post_call_nop();
 1835   %}
 1836 
 1837   enc_class Java_Dynamic_Call(method meth) %{
 1838     MacroAssembler _masm(&cbuf);
 1839     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 1840     __ post_call_nop();
 1841   %}
 1842 
 1843 %}
 1844 
 1845 
 1846 
 1847 //----------FRAME--------------------------------------------------------------
 1848 // Definition of frame structure and management information.
 1849 //
 1850 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1851 //                             |   (to get allocators register number
 1852 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1853 //  r   CALLER     |        |
 1854 //  o     |        +--------+      pad to even-align allocators stack-slot
 1855 //  w     V        |  pad0  |        numbers; owned by CALLER
 1856 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1857 //  h     ^        |   in   |  5
 1858 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1859 //  |     |        |        |  3
 1860 //  |     |        +--------+
 1861 //  V     |        | old out|      Empty on Intel, window on Sparc
 1862 //        |    old |preserve|      Must be even aligned.
 1863 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1864 //        |        |   in   |  3   area for Intel ret address
 1865 //     Owned by    |preserve|      Empty on Sparc.
 1866 //       SELF      +--------+
 1867 //        |        |  pad2  |  2   pad to align old SP
 1868 //        |        +--------+  1
 1869 //        |        | locks  |  0
 1870 //        |        +--------+----> OptoReg::stack0(), even aligned
 1871 //        |        |  pad1  | 11   pad to align new SP
 1872 //        |        +--------+
 1873 //        |        |        | 10
 1874 //        |        | spills |  9   spills
 1875 //        V        |        |  8   (pad0 slot for callee)
 1876 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1877 //        ^        |  out   |  7
 1878 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1879 //     Owned by    +--------+
 1880 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1881 //        |    new |preserve|      Must be even-aligned.
 1882 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1883 //        |        |        |
 1884 //
 1885 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1886 //         known from SELF's arguments and the Java calling convention.
 1887 //         Region 6-7 is determined per call site.
 1888 // Note 2: If the calling convention leaves holes in the incoming argument
 1889 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1890 //         are owned by the CALLEE.  Holes should not be necessary in the
 1891 //         incoming area, as the Java calling convention is completely under
 1892 //         the control of the AD file.  Doubles can be sorted and packed to
 1893 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1894 //         varargs C calling conventions.
 1895 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1896 //         even aligned with pad0 as needed.
 1897 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1898 //         region 6-11 is even aligned; it may be padded out more so that
 1899 //         the region from SP to FP meets the minimum stack alignment.
 1900 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1901 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1902 //         SP meets the minimum alignment.
 1903 
 1904 frame
 1905 %{
 1906   // These three registers define part of the calling convention
 1907   // between compiled code and the interpreter.
 1908   inline_cache_reg(RAX);                // Inline Cache Register
 1909 
 1910   // Optional: name the operand used by cisc-spilling to access
 1911   // [stack_pointer + offset]
 1912   cisc_spilling_operand_name(indOffset32);
 1913 
 1914   // Number of stack slots consumed by locking an object
 1915   sync_stack_slots(2);
 1916 
 1917   // Compiled code's Frame Pointer
 1918   frame_pointer(RSP);
 1919 
 1920   // Interpreter stores its frame pointer in a register which is
 1921   // stored to the stack by I2CAdaptors.
 1922   // I2CAdaptors convert from interpreted java to compiled java.
 1923   interpreter_frame_pointer(RBP);
 1924 
 1925   // Stack alignment requirement
 1926   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1927 
 1928   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1929   // for calls to C.  Supports the var-args backing area for register parms.
 1930   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1931 
 1932   // The after-PROLOG location of the return address.  Location of
 1933   // return address specifies a type (REG or STACK) and a number
 1934   // representing the register number (i.e. - use a register name) or
 1935   // stack slot.
 1936   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1937   // Otherwise, it is above the locks and verification slot and alignment word
 1938   return_addr(STACK - 2 +
 1939               align_up((Compile::current()->in_preserve_stack_slots() +
 1940                         Compile::current()->fixed_slots()),
 1941                        stack_alignment_in_slots()));
 1942 
 1943   // Location of compiled Java return values.  Same as C for now.
 1944   return_value
 1945   %{
 1946     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 1947            "only return normal values");
 1948 
 1949     static const int lo[Op_RegL + 1] = {
 1950       0,
 1951       0,
 1952       RAX_num,  // Op_RegN
 1953       RAX_num,  // Op_RegI
 1954       RAX_num,  // Op_RegP
 1955       XMM0_num, // Op_RegF
 1956       XMM0_num, // Op_RegD
 1957       RAX_num   // Op_RegL
 1958     };
 1959     static const int hi[Op_RegL + 1] = {
 1960       0,
 1961       0,
 1962       OptoReg::Bad, // Op_RegN
 1963       OptoReg::Bad, // Op_RegI
 1964       RAX_H_num,    // Op_RegP
 1965       OptoReg::Bad, // Op_RegF
 1966       XMM0b_num,    // Op_RegD
 1967       RAX_H_num     // Op_RegL
 1968     };
 1969     // Excluded flags and vector registers.
 1970     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 1971     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 1972   %}
 1973 %}
 1974 
 1975 //----------ATTRIBUTES---------------------------------------------------------
 1976 //----------Operand Attributes-------------------------------------------------
 1977 op_attrib op_cost(0);        // Required cost attribute
 1978 
 1979 //----------Instruction Attributes---------------------------------------------
 1980 ins_attrib ins_cost(100);       // Required cost attribute
 1981 ins_attrib ins_size(8);         // Required size attribute (in bits)
 1982 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 1983                                 // a non-matching short branch variant
 1984                                 // of some long branch?
 1985 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 1986                                 // be a power of 2) specifies the
 1987                                 // alignment that some part of the
 1988                                 // instruction (not necessarily the
 1989                                 // start) requires.  If > 1, a
 1990                                 // compute_padding() function must be
 1991                                 // provided for the instruction
 1992 
 1993 //----------OPERANDS-----------------------------------------------------------
 1994 // Operand definitions must precede instruction definitions for correct parsing
 1995 // in the ADLC because operands constitute user defined types which are used in
 1996 // instruction definitions.
 1997 
 1998 //----------Simple Operands----------------------------------------------------
 1999 // Immediate Operands
 2000 // Integer Immediate
 2001 operand immI()
 2002 %{
 2003   match(ConI);
 2004 
 2005   op_cost(10);
 2006   format %{ %}
 2007   interface(CONST_INTER);
 2008 %}
 2009 
 2010 // Constant for test vs zero
 2011 operand immI_0()
 2012 %{
 2013   predicate(n->get_int() == 0);
 2014   match(ConI);
 2015 
 2016   op_cost(0);
 2017   format %{ %}
 2018   interface(CONST_INTER);
 2019 %}
 2020 
 2021 // Constant for increment
 2022 operand immI_1()
 2023 %{
 2024   predicate(n->get_int() == 1);
 2025   match(ConI);
 2026 
 2027   op_cost(0);
 2028   format %{ %}
 2029   interface(CONST_INTER);
 2030 %}
 2031 
 2032 // Constant for decrement
 2033 operand immI_M1()
 2034 %{
 2035   predicate(n->get_int() == -1);
 2036   match(ConI);
 2037 
 2038   op_cost(0);
 2039   format %{ %}
 2040   interface(CONST_INTER);
 2041 %}
 2042 
 2043 operand immI_2()
 2044 %{
 2045   predicate(n->get_int() == 2);
 2046   match(ConI);
 2047 
 2048   op_cost(0);
 2049   format %{ %}
 2050   interface(CONST_INTER);
 2051 %}
 2052 
 2053 operand immI_4()
 2054 %{
 2055   predicate(n->get_int() == 4);
 2056   match(ConI);
 2057 
 2058   op_cost(0);
 2059   format %{ %}
 2060   interface(CONST_INTER);
 2061 %}
 2062 
 2063 operand immI_8()
 2064 %{
 2065   predicate(n->get_int() == 8);
 2066   match(ConI);
 2067 
 2068   op_cost(0);
 2069   format %{ %}
 2070   interface(CONST_INTER);
 2071 %}
 2072 
 2073 // Valid scale values for addressing modes
 2074 operand immI2()
 2075 %{
 2076   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2077   match(ConI);
 2078 
 2079   format %{ %}
 2080   interface(CONST_INTER);
 2081 %}
 2082 
 2083 operand immU7()
 2084 %{
 2085   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2086   match(ConI);
 2087 
 2088   op_cost(5);
 2089   format %{ %}
 2090   interface(CONST_INTER);
 2091 %}
 2092 
 2093 operand immI8()
 2094 %{
 2095   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2096   match(ConI);
 2097 
 2098   op_cost(5);
 2099   format %{ %}
 2100   interface(CONST_INTER);
 2101 %}
 2102 
 2103 operand immU8()
 2104 %{
 2105   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2106   match(ConI);
 2107 
 2108   op_cost(5);
 2109   format %{ %}
 2110   interface(CONST_INTER);
 2111 %}
 2112 
 2113 operand immI16()
 2114 %{
 2115   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2116   match(ConI);
 2117 
 2118   op_cost(10);
 2119   format %{ %}
 2120   interface(CONST_INTER);
 2121 %}
 2122 
 2123 // Int Immediate non-negative
 2124 operand immU31()
 2125 %{
 2126   predicate(n->get_int() >= 0);
 2127   match(ConI);
 2128 
 2129   op_cost(0);
 2130   format %{ %}
 2131   interface(CONST_INTER);
 2132 %}
 2133 
 2134 // Constant for long shifts
 2135 operand immI_32()
 2136 %{
 2137   predicate( n->get_int() == 32 );
 2138   match(ConI);
 2139 
 2140   op_cost(0);
 2141   format %{ %}
 2142   interface(CONST_INTER);
 2143 %}
 2144 
 2145 // Constant for long shifts
 2146 operand immI_64()
 2147 %{
 2148   predicate( n->get_int() == 64 );
 2149   match(ConI);
 2150 
 2151   op_cost(0);
 2152   format %{ %}
 2153   interface(CONST_INTER);
 2154 %}
 2155 
 2156 // Pointer Immediate
 2157 operand immP()
 2158 %{
 2159   match(ConP);
 2160 
 2161   op_cost(10);
 2162   format %{ %}
 2163   interface(CONST_INTER);
 2164 %}
 2165 
 2166 // Null Pointer Immediate
 2167 operand immP0()
 2168 %{
 2169   predicate(n->get_ptr() == 0);
 2170   match(ConP);
 2171 
 2172   op_cost(5);
 2173   format %{ %}
 2174   interface(CONST_INTER);
 2175 %}
 2176 
 2177 // Pointer Immediate
 2178 operand immN() %{
 2179   match(ConN);
 2180 
 2181   op_cost(10);
 2182   format %{ %}
 2183   interface(CONST_INTER);
 2184 %}
 2185 
 2186 operand immNKlass() %{
 2187   match(ConNKlass);
 2188 
 2189   op_cost(10);
 2190   format %{ %}
 2191   interface(CONST_INTER);
 2192 %}
 2193 
 2194 // Null Pointer Immediate
 2195 operand immN0() %{
 2196   predicate(n->get_narrowcon() == 0);
 2197   match(ConN);
 2198 
 2199   op_cost(5);
 2200   format %{ %}
 2201   interface(CONST_INTER);
 2202 %}
 2203 
 2204 operand immP31()
 2205 %{
 2206   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2207             && (n->get_ptr() >> 31) == 0);
 2208   match(ConP);
 2209 
 2210   op_cost(5);
 2211   format %{ %}
 2212   interface(CONST_INTER);
 2213 %}
 2214 
 2215 
 2216 // Long Immediate
 2217 operand immL()
 2218 %{
 2219   match(ConL);
 2220 
 2221   op_cost(20);
 2222   format %{ %}
 2223   interface(CONST_INTER);
 2224 %}
 2225 
 2226 // Long Immediate 8-bit
 2227 operand immL8()
 2228 %{
 2229   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2230   match(ConL);
 2231 
 2232   op_cost(5);
 2233   format %{ %}
 2234   interface(CONST_INTER);
 2235 %}
 2236 
 2237 // Long Immediate 32-bit unsigned
 2238 operand immUL32()
 2239 %{
 2240   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2241   match(ConL);
 2242 
 2243   op_cost(10);
 2244   format %{ %}
 2245   interface(CONST_INTER);
 2246 %}
 2247 
 2248 // Long Immediate 32-bit signed
 2249 operand immL32()
 2250 %{
 2251   predicate(n->get_long() == (int) (n->get_long()));
 2252   match(ConL);
 2253 
 2254   op_cost(15);
 2255   format %{ %}
 2256   interface(CONST_INTER);
 2257 %}
 2258 
 2259 operand immL_Pow2()
 2260 %{
 2261   predicate(is_power_of_2((julong)n->get_long()));
 2262   match(ConL);
 2263 
 2264   op_cost(15);
 2265   format %{ %}
 2266   interface(CONST_INTER);
 2267 %}
 2268 
 2269 operand immL_NotPow2()
 2270 %{
 2271   predicate(is_power_of_2((julong)~n->get_long()));
 2272   match(ConL);
 2273 
 2274   op_cost(15);
 2275   format %{ %}
 2276   interface(CONST_INTER);
 2277 %}
 2278 
 2279 // Long Immediate zero
 2280 operand immL0()
 2281 %{
 2282   predicate(n->get_long() == 0L);
 2283   match(ConL);
 2284 
 2285   op_cost(10);
 2286   format %{ %}
 2287   interface(CONST_INTER);
 2288 %}
 2289 
 2290 // Constant for increment
 2291 operand immL1()
 2292 %{
 2293   predicate(n->get_long() == 1);
 2294   match(ConL);
 2295 
 2296   format %{ %}
 2297   interface(CONST_INTER);
 2298 %}
 2299 
 2300 // Constant for decrement
 2301 operand immL_M1()
 2302 %{
 2303   predicate(n->get_long() == -1);
 2304   match(ConL);
 2305 
 2306   format %{ %}
 2307   interface(CONST_INTER);
 2308 %}
 2309 
 2310 // Long Immediate: the value 10
 2311 operand immL10()
 2312 %{
 2313   predicate(n->get_long() == 10);
 2314   match(ConL);
 2315 
 2316   format %{ %}
 2317   interface(CONST_INTER);
 2318 %}
 2319 
 2320 // Long immediate from 0 to 127.
 2321 // Used for a shorter form of long mul by 10.
 2322 operand immL_127()
 2323 %{
 2324   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 2325   match(ConL);
 2326 
 2327   op_cost(10);
 2328   format %{ %}
 2329   interface(CONST_INTER);
 2330 %}
 2331 
 2332 // Long Immediate: low 32-bit mask
 2333 operand immL_32bits()
 2334 %{
 2335   predicate(n->get_long() == 0xFFFFFFFFL);
 2336   match(ConL);
 2337   op_cost(20);
 2338 
 2339   format %{ %}
 2340   interface(CONST_INTER);
 2341 %}
 2342 
 2343 // Int Immediate: 2^n-1, positive
 2344 operand immI_Pow2M1()
 2345 %{
 2346   predicate((n->get_int() > 0)
 2347             && is_power_of_2((juint)n->get_int() + 1));
 2348   match(ConI);
 2349 
 2350   op_cost(20);
 2351   format %{ %}
 2352   interface(CONST_INTER);
 2353 %}
 2354 
 2355 // Float Immediate zero
 2356 operand immF0()
 2357 %{
 2358   predicate(jint_cast(n->getf()) == 0);
 2359   match(ConF);
 2360 
 2361   op_cost(5);
 2362   format %{ %}
 2363   interface(CONST_INTER);
 2364 %}
 2365 
 2366 // Float Immediate
 2367 operand immF()
 2368 %{
 2369   match(ConF);
 2370 
 2371   op_cost(15);
 2372   format %{ %}
 2373   interface(CONST_INTER);
 2374 %}
 2375 
 2376 // Double Immediate zero
 2377 operand immD0()
 2378 %{
 2379   predicate(jlong_cast(n->getd()) == 0);
 2380   match(ConD);
 2381 
 2382   op_cost(5);
 2383   format %{ %}
 2384   interface(CONST_INTER);
 2385 %}
 2386 
 2387 // Double Immediate
 2388 operand immD()
 2389 %{
 2390   match(ConD);
 2391 
 2392   op_cost(15);
 2393   format %{ %}
 2394   interface(CONST_INTER);
 2395 %}
 2396 
 2397 // Immediates for special shifts (sign extend)
 2398 
 2399 // Constants for increment
 2400 operand immI_16()
 2401 %{
 2402   predicate(n->get_int() == 16);
 2403   match(ConI);
 2404 
 2405   format %{ %}
 2406   interface(CONST_INTER);
 2407 %}
 2408 
 2409 operand immI_24()
 2410 %{
 2411   predicate(n->get_int() == 24);
 2412   match(ConI);
 2413 
 2414   format %{ %}
 2415   interface(CONST_INTER);
 2416 %}
 2417 
 2418 // Constant for byte-wide masking
 2419 operand immI_255()
 2420 %{
 2421   predicate(n->get_int() == 255);
 2422   match(ConI);
 2423 
 2424   format %{ %}
 2425   interface(CONST_INTER);
 2426 %}
 2427 
 2428 // Constant for short-wide masking
 2429 operand immI_65535()
 2430 %{
 2431   predicate(n->get_int() == 65535);
 2432   match(ConI);
 2433 
 2434   format %{ %}
 2435   interface(CONST_INTER);
 2436 %}
 2437 
 2438 // Constant for byte-wide masking
 2439 operand immL_255()
 2440 %{
 2441   predicate(n->get_long() == 255);
 2442   match(ConL);
 2443 
 2444   format %{ %}
 2445   interface(CONST_INTER);
 2446 %}
 2447 
 2448 // Constant for short-wide masking
 2449 operand immL_65535()
 2450 %{
 2451   predicate(n->get_long() == 65535);
 2452   match(ConL);
 2453 
 2454   format %{ %}
 2455   interface(CONST_INTER);
 2456 %}
 2457 
 2458 operand kReg()
 2459 %{
 2460   constraint(ALLOC_IN_RC(vectmask_reg));
 2461   match(RegVectMask);
 2462   format %{%}
 2463   interface(REG_INTER);
 2464 %}
 2465 
 2466 operand kReg_K1()
 2467 %{
 2468   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 2469   match(RegVectMask);
 2470   format %{%}
 2471   interface(REG_INTER);
 2472 %}
 2473 
 2474 operand kReg_K2()
 2475 %{
 2476   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 2477   match(RegVectMask);
 2478   format %{%}
 2479   interface(REG_INTER);
 2480 %}
 2481 
 2482 // Special Registers
 2483 operand kReg_K3()
 2484 %{
 2485   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 2486   match(RegVectMask);
 2487   format %{%}
 2488   interface(REG_INTER);
 2489 %}
 2490 
 2491 operand kReg_K4()
 2492 %{
 2493   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 2494   match(RegVectMask);
 2495   format %{%}
 2496   interface(REG_INTER);
 2497 %}
 2498 
 2499 operand kReg_K5()
 2500 %{
 2501   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 2502   match(RegVectMask);
 2503   format %{%}
 2504   interface(REG_INTER);
 2505 %}
 2506 
 2507 operand kReg_K6()
 2508 %{
 2509   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 2510   match(RegVectMask);
 2511   format %{%}
 2512   interface(REG_INTER);
 2513 %}
 2514 
 2515 // Special Registers
 2516 operand kReg_K7()
 2517 %{
 2518   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 2519   match(RegVectMask);
 2520   format %{%}
 2521   interface(REG_INTER);
 2522 %}
 2523 
 2524 // Register Operands
 2525 // Integer Register
 2526 operand rRegI()
 2527 %{
 2528   constraint(ALLOC_IN_RC(int_reg));
 2529   match(RegI);
 2530 
 2531   match(rax_RegI);
 2532   match(rbx_RegI);
 2533   match(rcx_RegI);
 2534   match(rdx_RegI);
 2535   match(rdi_RegI);
 2536 
 2537   format %{ %}
 2538   interface(REG_INTER);
 2539 %}
 2540 
 2541 // Special Registers
 2542 operand rax_RegI()
 2543 %{
 2544   constraint(ALLOC_IN_RC(int_rax_reg));
 2545   match(RegI);
 2546   match(rRegI);
 2547 
 2548   format %{ "RAX" %}
 2549   interface(REG_INTER);
 2550 %}
 2551 
 2552 // Special Registers
 2553 operand rbx_RegI()
 2554 %{
 2555   constraint(ALLOC_IN_RC(int_rbx_reg));
 2556   match(RegI);
 2557   match(rRegI);
 2558 
 2559   format %{ "RBX" %}
 2560   interface(REG_INTER);
 2561 %}
 2562 
 2563 operand rcx_RegI()
 2564 %{
 2565   constraint(ALLOC_IN_RC(int_rcx_reg));
 2566   match(RegI);
 2567   match(rRegI);
 2568 
 2569   format %{ "RCX" %}
 2570   interface(REG_INTER);
 2571 %}
 2572 
 2573 operand rdx_RegI()
 2574 %{
 2575   constraint(ALLOC_IN_RC(int_rdx_reg));
 2576   match(RegI);
 2577   match(rRegI);
 2578 
 2579   format %{ "RDX" %}
 2580   interface(REG_INTER);
 2581 %}
 2582 
 2583 operand rdi_RegI()
 2584 %{
 2585   constraint(ALLOC_IN_RC(int_rdi_reg));
 2586   match(RegI);
 2587   match(rRegI);
 2588 
 2589   format %{ "RDI" %}
 2590   interface(REG_INTER);
 2591 %}
 2592 
 2593 operand no_rax_rdx_RegI()
 2594 %{
 2595   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2596   match(RegI);
 2597   match(rbx_RegI);
 2598   match(rcx_RegI);
 2599   match(rdi_RegI);
 2600 
 2601   format %{ %}
 2602   interface(REG_INTER);
 2603 %}
 2604 
 2605 operand no_rbp_r13_RegI()
 2606 %{
 2607   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2608   match(RegI);
 2609   match(rRegI);
 2610   match(rax_RegI);
 2611   match(rbx_RegI);
 2612   match(rcx_RegI);
 2613   match(rdx_RegI);
 2614   match(rdi_RegI);
 2615 
 2616   format %{ %}
 2617   interface(REG_INTER);
 2618 %}
 2619 
 2620 // Pointer Register
 2621 operand any_RegP()
 2622 %{
 2623   constraint(ALLOC_IN_RC(any_reg));
 2624   match(RegP);
 2625   match(rax_RegP);
 2626   match(rbx_RegP);
 2627   match(rdi_RegP);
 2628   match(rsi_RegP);
 2629   match(rbp_RegP);
 2630   match(r15_RegP);
 2631   match(rRegP);
 2632 
 2633   format %{ %}
 2634   interface(REG_INTER);
 2635 %}
 2636 
 2637 operand rRegP()
 2638 %{
 2639   constraint(ALLOC_IN_RC(ptr_reg));
 2640   match(RegP);
 2641   match(rax_RegP);
 2642   match(rbx_RegP);
 2643   match(rdi_RegP);
 2644   match(rsi_RegP);
 2645   match(rbp_RegP);  // See Q&A below about
 2646   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2647 
 2648   format %{ %}
 2649   interface(REG_INTER);
 2650 %}
 2651 
 2652 operand rRegN() %{
 2653   constraint(ALLOC_IN_RC(int_reg));
 2654   match(RegN);
 2655 
 2656   format %{ %}
 2657   interface(REG_INTER);
 2658 %}
 2659 
 2660 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2661 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2662 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2663 // The output of an instruction is controlled by the allocator, which respects
 2664 // register class masks, not match rules.  Unless an instruction mentions
 2665 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2666 // by the allocator as an input.
 2667 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2668 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2669 // result, RBP is not included in the output of the instruction either.
 2670 
 2671 operand no_rax_RegP()
 2672 %{
 2673   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 2674   match(RegP);
 2675   match(rbx_RegP);
 2676   match(rsi_RegP);
 2677   match(rdi_RegP);
 2678 
 2679   format %{ %}
 2680   interface(REG_INTER);
 2681 %}
 2682 
 2683 // This operand is not allowed to use RBP even if
 2684 // RBP is not used to hold the frame pointer.
 2685 operand no_rbp_RegP()
 2686 %{
 2687   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2688   match(RegP);
 2689   match(rbx_RegP);
 2690   match(rsi_RegP);
 2691   match(rdi_RegP);
 2692 
 2693   format %{ %}
 2694   interface(REG_INTER);
 2695 %}
 2696 
 2697 operand no_rax_rbx_RegP()
 2698 %{
 2699   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 2700   match(RegP);
 2701   match(rsi_RegP);
 2702   match(rdi_RegP);
 2703 
 2704   format %{ %}
 2705   interface(REG_INTER);
 2706 %}
 2707 
 2708 // Special Registers
 2709 // Return a pointer value
 2710 operand rax_RegP()
 2711 %{
 2712   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2713   match(RegP);
 2714   match(rRegP);
 2715 
 2716   format %{ %}
 2717   interface(REG_INTER);
 2718 %}
 2719 
 2720 // Special Registers
 2721 // Return a compressed pointer value
 2722 operand rax_RegN()
 2723 %{
 2724   constraint(ALLOC_IN_RC(int_rax_reg));
 2725   match(RegN);
 2726   match(rRegN);
 2727 
 2728   format %{ %}
 2729   interface(REG_INTER);
 2730 %}
 2731 
 2732 // Used in AtomicAdd
 2733 operand rbx_RegP()
 2734 %{
 2735   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2736   match(RegP);
 2737   match(rRegP);
 2738 
 2739   format %{ %}
 2740   interface(REG_INTER);
 2741 %}
 2742 
 2743 operand rsi_RegP()
 2744 %{
 2745   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2746   match(RegP);
 2747   match(rRegP);
 2748 
 2749   format %{ %}
 2750   interface(REG_INTER);
 2751 %}
 2752 
 2753 operand rbp_RegP()
 2754 %{
 2755   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2756   match(RegP);
 2757   match(rRegP);
 2758 
 2759   format %{ %}
 2760   interface(REG_INTER);
 2761 %}
 2762 
 2763 // Used in rep stosq
 2764 operand rdi_RegP()
 2765 %{
 2766   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2767   match(RegP);
 2768   match(rRegP);
 2769 
 2770   format %{ %}
 2771   interface(REG_INTER);
 2772 %}
 2773 
 2774 operand r15_RegP()
 2775 %{
 2776   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2777   match(RegP);
 2778   match(rRegP);
 2779 
 2780   format %{ %}
 2781   interface(REG_INTER);
 2782 %}
 2783 
 2784 operand rRegL()
 2785 %{
 2786   constraint(ALLOC_IN_RC(long_reg));
 2787   match(RegL);
 2788   match(rax_RegL);
 2789   match(rdx_RegL);
 2790 
 2791   format %{ %}
 2792   interface(REG_INTER);
 2793 %}
 2794 
 2795 // Special Registers
 2796 operand no_rax_rdx_RegL()
 2797 %{
 2798   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2799   match(RegL);
 2800   match(rRegL);
 2801 
 2802   format %{ %}
 2803   interface(REG_INTER);
 2804 %}
 2805 
 2806 operand rax_RegL()
 2807 %{
 2808   constraint(ALLOC_IN_RC(long_rax_reg));
 2809   match(RegL);
 2810   match(rRegL);
 2811 
 2812   format %{ "RAX" %}
 2813   interface(REG_INTER);
 2814 %}
 2815 
 2816 operand rcx_RegL()
 2817 %{
 2818   constraint(ALLOC_IN_RC(long_rcx_reg));
 2819   match(RegL);
 2820   match(rRegL);
 2821 
 2822   format %{ %}
 2823   interface(REG_INTER);
 2824 %}
 2825 
 2826 operand rdx_RegL()
 2827 %{
 2828   constraint(ALLOC_IN_RC(long_rdx_reg));
 2829   match(RegL);
 2830   match(rRegL);
 2831 
 2832   format %{ %}
 2833   interface(REG_INTER);
 2834 %}
 2835 
 2836 operand no_rbp_r13_RegL()
 2837 %{
 2838   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2839   match(RegL);
 2840   match(rRegL);
 2841   match(rax_RegL);
 2842   match(rcx_RegL);
 2843   match(rdx_RegL);
 2844 
 2845   format %{ %}
 2846   interface(REG_INTER);
 2847 %}
 2848 
 2849 // Flags register, used as output of compare instructions
 2850 operand rFlagsReg()
 2851 %{
 2852   constraint(ALLOC_IN_RC(int_flags));
 2853   match(RegFlags);
 2854 
 2855   format %{ "RFLAGS" %}
 2856   interface(REG_INTER);
 2857 %}
 2858 
 2859 // Flags register, used as output of FLOATING POINT compare instructions
 2860 operand rFlagsRegU()
 2861 %{
 2862   constraint(ALLOC_IN_RC(int_flags));
 2863   match(RegFlags);
 2864 
 2865   format %{ "RFLAGS_U" %}
 2866   interface(REG_INTER);
 2867 %}
 2868 
 2869 operand rFlagsRegUCF() %{
 2870   constraint(ALLOC_IN_RC(int_flags));
 2871   match(RegFlags);
 2872   predicate(false);
 2873 
 2874   format %{ "RFLAGS_U_CF" %}
 2875   interface(REG_INTER);
 2876 %}
 2877 
 2878 // Float register operands
 2879 operand regF() %{
 2880    constraint(ALLOC_IN_RC(float_reg));
 2881    match(RegF);
 2882 
 2883    format %{ %}
 2884    interface(REG_INTER);
 2885 %}
 2886 
 2887 // Float register operands
 2888 operand legRegF() %{
 2889    constraint(ALLOC_IN_RC(float_reg_legacy));
 2890    match(RegF);
 2891 
 2892    format %{ %}
 2893    interface(REG_INTER);
 2894 %}
 2895 
 2896 // Float register operands
 2897 operand vlRegF() %{
 2898    constraint(ALLOC_IN_RC(float_reg_vl));
 2899    match(RegF);
 2900 
 2901    format %{ %}
 2902    interface(REG_INTER);
 2903 %}
 2904 
 2905 // Double register operands
 2906 operand regD() %{
 2907    constraint(ALLOC_IN_RC(double_reg));
 2908    match(RegD);
 2909 
 2910    format %{ %}
 2911    interface(REG_INTER);
 2912 %}
 2913 
 2914 // Double register operands
 2915 operand legRegD() %{
 2916    constraint(ALLOC_IN_RC(double_reg_legacy));
 2917    match(RegD);
 2918 
 2919    format %{ %}
 2920    interface(REG_INTER);
 2921 %}
 2922 
 2923 // Double register operands
 2924 operand vlRegD() %{
 2925    constraint(ALLOC_IN_RC(double_reg_vl));
 2926    match(RegD);
 2927 
 2928    format %{ %}
 2929    interface(REG_INTER);
 2930 %}
 2931 
 2932 //----------Memory Operands----------------------------------------------------
 2933 // Direct Memory Operand
 2934 // operand direct(immP addr)
 2935 // %{
 2936 //   match(addr);
 2937 
 2938 //   format %{ "[$addr]" %}
 2939 //   interface(MEMORY_INTER) %{
 2940 //     base(0xFFFFFFFF);
 2941 //     index(0x4);
 2942 //     scale(0x0);
 2943 //     disp($addr);
 2944 //   %}
 2945 // %}
 2946 
 2947 // Indirect Memory Operand
 2948 operand indirect(any_RegP reg)
 2949 %{
 2950   constraint(ALLOC_IN_RC(ptr_reg));
 2951   match(reg);
 2952 
 2953   format %{ "[$reg]" %}
 2954   interface(MEMORY_INTER) %{
 2955     base($reg);
 2956     index(0x4);
 2957     scale(0x0);
 2958     disp(0x0);
 2959   %}
 2960 %}
 2961 
 2962 // Indirect Memory Plus Short Offset Operand
 2963 operand indOffset8(any_RegP reg, immL8 off)
 2964 %{
 2965   constraint(ALLOC_IN_RC(ptr_reg));
 2966   match(AddP reg off);
 2967 
 2968   format %{ "[$reg + $off (8-bit)]" %}
 2969   interface(MEMORY_INTER) %{
 2970     base($reg);
 2971     index(0x4);
 2972     scale(0x0);
 2973     disp($off);
 2974   %}
 2975 %}
 2976 
 2977 // Indirect Memory Plus Long Offset Operand
 2978 operand indOffset32(any_RegP reg, immL32 off)
 2979 %{
 2980   constraint(ALLOC_IN_RC(ptr_reg));
 2981   match(AddP reg off);
 2982 
 2983   format %{ "[$reg + $off (32-bit)]" %}
 2984   interface(MEMORY_INTER) %{
 2985     base($reg);
 2986     index(0x4);
 2987     scale(0x0);
 2988     disp($off);
 2989   %}
 2990 %}
 2991 
 2992 // Indirect Memory Plus Index Register Plus Offset Operand
 2993 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2994 %{
 2995   constraint(ALLOC_IN_RC(ptr_reg));
 2996   match(AddP (AddP reg lreg) off);
 2997 
 2998   op_cost(10);
 2999   format %{"[$reg + $off + $lreg]" %}
 3000   interface(MEMORY_INTER) %{
 3001     base($reg);
 3002     index($lreg);
 3003     scale(0x0);
 3004     disp($off);
 3005   %}
 3006 %}
 3007 
 3008 // Indirect Memory Plus Index Register Plus Offset Operand
 3009 operand indIndex(any_RegP reg, rRegL lreg)
 3010 %{
 3011   constraint(ALLOC_IN_RC(ptr_reg));
 3012   match(AddP reg lreg);
 3013 
 3014   op_cost(10);
 3015   format %{"[$reg + $lreg]" %}
 3016   interface(MEMORY_INTER) %{
 3017     base($reg);
 3018     index($lreg);
 3019     scale(0x0);
 3020     disp(0x0);
 3021   %}
 3022 %}
 3023 
 3024 // Indirect Memory Times Scale Plus Index Register
 3025 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3026 %{
 3027   constraint(ALLOC_IN_RC(ptr_reg));
 3028   match(AddP reg (LShiftL lreg scale));
 3029 
 3030   op_cost(10);
 3031   format %{"[$reg + $lreg << $scale]" %}
 3032   interface(MEMORY_INTER) %{
 3033     base($reg);
 3034     index($lreg);
 3035     scale($scale);
 3036     disp(0x0);
 3037   %}
 3038 %}
 3039 
 3040 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3041 %{
 3042   constraint(ALLOC_IN_RC(ptr_reg));
 3043   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3044   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3045 
 3046   op_cost(10);
 3047   format %{"[$reg + pos $idx << $scale]" %}
 3048   interface(MEMORY_INTER) %{
 3049     base($reg);
 3050     index($idx);
 3051     scale($scale);
 3052     disp(0x0);
 3053   %}
 3054 %}
 3055 
 3056 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3057 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3058 %{
 3059   constraint(ALLOC_IN_RC(ptr_reg));
 3060   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3061 
 3062   op_cost(10);
 3063   format %{"[$reg + $off + $lreg << $scale]" %}
 3064   interface(MEMORY_INTER) %{
 3065     base($reg);
 3066     index($lreg);
 3067     scale($scale);
 3068     disp($off);
 3069   %}
 3070 %}
 3071 
 3072 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3073 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3074 %{
 3075   constraint(ALLOC_IN_RC(ptr_reg));
 3076   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3077   match(AddP (AddP reg (ConvI2L idx)) off);
 3078 
 3079   op_cost(10);
 3080   format %{"[$reg + $off + $idx]" %}
 3081   interface(MEMORY_INTER) %{
 3082     base($reg);
 3083     index($idx);
 3084     scale(0x0);
 3085     disp($off);
 3086   %}
 3087 %}
 3088 
 3089 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3090 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3091 %{
 3092   constraint(ALLOC_IN_RC(ptr_reg));
 3093   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3094   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3095 
 3096   op_cost(10);
 3097   format %{"[$reg + $off + $idx << $scale]" %}
 3098   interface(MEMORY_INTER) %{
 3099     base($reg);
 3100     index($idx);
 3101     scale($scale);
 3102     disp($off);
 3103   %}
 3104 %}
 3105 
 3106 // Indirect Narrow Oop Plus Offset Operand
 3107 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3108 // we can't free r12 even with CompressedOops::base() == nullptr.
 3109 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3110   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3111   constraint(ALLOC_IN_RC(ptr_reg));
 3112   match(AddP (DecodeN reg) off);
 3113 
 3114   op_cost(10);
 3115   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3116   interface(MEMORY_INTER) %{
 3117     base(0xc); // R12
 3118     index($reg);
 3119     scale(0x3);
 3120     disp($off);
 3121   %}
 3122 %}
 3123 
 3124 // Indirect Memory Operand
 3125 operand indirectNarrow(rRegN reg)
 3126 %{
 3127   predicate(CompressedOops::shift() == 0);
 3128   constraint(ALLOC_IN_RC(ptr_reg));
 3129   match(DecodeN reg);
 3130 
 3131   format %{ "[$reg]" %}
 3132   interface(MEMORY_INTER) %{
 3133     base($reg);
 3134     index(0x4);
 3135     scale(0x0);
 3136     disp(0x0);
 3137   %}
 3138 %}
 3139 
 3140 // Indirect Memory Plus Short Offset Operand
 3141 operand indOffset8Narrow(rRegN reg, immL8 off)
 3142 %{
 3143   predicate(CompressedOops::shift() == 0);
 3144   constraint(ALLOC_IN_RC(ptr_reg));
 3145   match(AddP (DecodeN reg) off);
 3146 
 3147   format %{ "[$reg + $off (8-bit)]" %}
 3148   interface(MEMORY_INTER) %{
 3149     base($reg);
 3150     index(0x4);
 3151     scale(0x0);
 3152     disp($off);
 3153   %}
 3154 %}
 3155 
 3156 // Indirect Memory Plus Long Offset Operand
 3157 operand indOffset32Narrow(rRegN reg, immL32 off)
 3158 %{
 3159   predicate(CompressedOops::shift() == 0);
 3160   constraint(ALLOC_IN_RC(ptr_reg));
 3161   match(AddP (DecodeN reg) off);
 3162 
 3163   format %{ "[$reg + $off (32-bit)]" %}
 3164   interface(MEMORY_INTER) %{
 3165     base($reg);
 3166     index(0x4);
 3167     scale(0x0);
 3168     disp($off);
 3169   %}
 3170 %}
 3171 
 3172 // Indirect Memory Plus Index Register Plus Offset Operand
 3173 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3174 %{
 3175   predicate(CompressedOops::shift() == 0);
 3176   constraint(ALLOC_IN_RC(ptr_reg));
 3177   match(AddP (AddP (DecodeN reg) lreg) off);
 3178 
 3179   op_cost(10);
 3180   format %{"[$reg + $off + $lreg]" %}
 3181   interface(MEMORY_INTER) %{
 3182     base($reg);
 3183     index($lreg);
 3184     scale(0x0);
 3185     disp($off);
 3186   %}
 3187 %}
 3188 
 3189 // Indirect Memory Plus Index Register Plus Offset Operand
 3190 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3191 %{
 3192   predicate(CompressedOops::shift() == 0);
 3193   constraint(ALLOC_IN_RC(ptr_reg));
 3194   match(AddP (DecodeN reg) lreg);
 3195 
 3196   op_cost(10);
 3197   format %{"[$reg + $lreg]" %}
 3198   interface(MEMORY_INTER) %{
 3199     base($reg);
 3200     index($lreg);
 3201     scale(0x0);
 3202     disp(0x0);
 3203   %}
 3204 %}
 3205 
 3206 // Indirect Memory Times Scale Plus Index Register
 3207 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3208 %{
 3209   predicate(CompressedOops::shift() == 0);
 3210   constraint(ALLOC_IN_RC(ptr_reg));
 3211   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3212 
 3213   op_cost(10);
 3214   format %{"[$reg + $lreg << $scale]" %}
 3215   interface(MEMORY_INTER) %{
 3216     base($reg);
 3217     index($lreg);
 3218     scale($scale);
 3219     disp(0x0);
 3220   %}
 3221 %}
 3222 
 3223 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3224 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3225 %{
 3226   predicate(CompressedOops::shift() == 0);
 3227   constraint(ALLOC_IN_RC(ptr_reg));
 3228   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3229 
 3230   op_cost(10);
 3231   format %{"[$reg + $off + $lreg << $scale]" %}
 3232   interface(MEMORY_INTER) %{
 3233     base($reg);
 3234     index($lreg);
 3235     scale($scale);
 3236     disp($off);
 3237   %}
 3238 %}
 3239 
 3240 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3241 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3242 %{
 3243   constraint(ALLOC_IN_RC(ptr_reg));
 3244   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3245   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3246 
 3247   op_cost(10);
 3248   format %{"[$reg + $off + $idx]" %}
 3249   interface(MEMORY_INTER) %{
 3250     base($reg);
 3251     index($idx);
 3252     scale(0x0);
 3253     disp($off);
 3254   %}
 3255 %}
 3256 
 3257 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3258 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3259 %{
 3260   constraint(ALLOC_IN_RC(ptr_reg));
 3261   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3262   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3263 
 3264   op_cost(10);
 3265   format %{"[$reg + $off + $idx << $scale]" %}
 3266   interface(MEMORY_INTER) %{
 3267     base($reg);
 3268     index($idx);
 3269     scale($scale);
 3270     disp($off);
 3271   %}
 3272 %}
 3273 
 3274 //----------Special Memory Operands--------------------------------------------
 3275 // Stack Slot Operand - This operand is used for loading and storing temporary
 3276 //                      values on the stack where a match requires a value to
 3277 //                      flow through memory.
 3278 operand stackSlotP(sRegP reg)
 3279 %{
 3280   constraint(ALLOC_IN_RC(stack_slots));
 3281   // No match rule because this operand is only generated in matching
 3282 
 3283   format %{ "[$reg]" %}
 3284   interface(MEMORY_INTER) %{
 3285     base(0x4);   // RSP
 3286     index(0x4);  // No Index
 3287     scale(0x0);  // No Scale
 3288     disp($reg);  // Stack Offset
 3289   %}
 3290 %}
 3291 
 3292 operand stackSlotI(sRegI reg)
 3293 %{
 3294   constraint(ALLOC_IN_RC(stack_slots));
 3295   // No match rule because this operand is only generated in matching
 3296 
 3297   format %{ "[$reg]" %}
 3298   interface(MEMORY_INTER) %{
 3299     base(0x4);   // RSP
 3300     index(0x4);  // No Index
 3301     scale(0x0);  // No Scale
 3302     disp($reg);  // Stack Offset
 3303   %}
 3304 %}
 3305 
 3306 operand stackSlotF(sRegF reg)
 3307 %{
 3308   constraint(ALLOC_IN_RC(stack_slots));
 3309   // No match rule because this operand is only generated in matching
 3310 
 3311   format %{ "[$reg]" %}
 3312   interface(MEMORY_INTER) %{
 3313     base(0x4);   // RSP
 3314     index(0x4);  // No Index
 3315     scale(0x0);  // No Scale
 3316     disp($reg);  // Stack Offset
 3317   %}
 3318 %}
 3319 
 3320 operand stackSlotD(sRegD reg)
 3321 %{
 3322   constraint(ALLOC_IN_RC(stack_slots));
 3323   // No match rule because this operand is only generated in matching
 3324 
 3325   format %{ "[$reg]" %}
 3326   interface(MEMORY_INTER) %{
 3327     base(0x4);   // RSP
 3328     index(0x4);  // No Index
 3329     scale(0x0);  // No Scale
 3330     disp($reg);  // Stack Offset
 3331   %}
 3332 %}
 3333 operand stackSlotL(sRegL reg)
 3334 %{
 3335   constraint(ALLOC_IN_RC(stack_slots));
 3336   // No match rule because this operand is only generated in matching
 3337 
 3338   format %{ "[$reg]" %}
 3339   interface(MEMORY_INTER) %{
 3340     base(0x4);   // RSP
 3341     index(0x4);  // No Index
 3342     scale(0x0);  // No Scale
 3343     disp($reg);  // Stack Offset
 3344   %}
 3345 %}
 3346 
 3347 //----------Conditional Branch Operands----------------------------------------
 3348 // Comparison Op  - This is the operation of the comparison, and is limited to
 3349 //                  the following set of codes:
 3350 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3351 //
 3352 // Other attributes of the comparison, such as unsignedness, are specified
 3353 // by the comparison instruction that sets a condition code flags register.
 3354 // That result is represented by a flags operand whose subtype is appropriate
 3355 // to the unsignedness (etc.) of the comparison.
 3356 //
 3357 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3358 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3359 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3360 
 3361 // Comparison Code
 3362 operand cmpOp()
 3363 %{
 3364   match(Bool);
 3365 
 3366   format %{ "" %}
 3367   interface(COND_INTER) %{
 3368     equal(0x4, "e");
 3369     not_equal(0x5, "ne");
 3370     less(0xC, "l");
 3371     greater_equal(0xD, "ge");
 3372     less_equal(0xE, "le");
 3373     greater(0xF, "g");
 3374     overflow(0x0, "o");
 3375     no_overflow(0x1, "no");
 3376   %}
 3377 %}
 3378 
 3379 // Comparison Code, unsigned compare.  Used by FP also, with
 3380 // C2 (unordered) turned into GT or LT already.  The other bits
 3381 // C0 and C3 are turned into Carry & Zero flags.
 3382 operand cmpOpU()
 3383 %{
 3384   match(Bool);
 3385 
 3386   format %{ "" %}
 3387   interface(COND_INTER) %{
 3388     equal(0x4, "e");
 3389     not_equal(0x5, "ne");
 3390     less(0x2, "b");
 3391     greater_equal(0x3, "ae");
 3392     less_equal(0x6, "be");
 3393     greater(0x7, "a");
 3394     overflow(0x0, "o");
 3395     no_overflow(0x1, "no");
 3396   %}
 3397 %}
 3398 
 3399 
 3400 // Floating comparisons that don't require any fixup for the unordered case,
 3401 // If both inputs of the comparison are the same, ZF is always set so we
 3402 // don't need to use cmpOpUCF2 for eq/ne
 3403 operand cmpOpUCF() %{
 3404   match(Bool);
 3405   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3406             n->as_Bool()->_test._test == BoolTest::ge ||
 3407             n->as_Bool()->_test._test == BoolTest::le ||
 3408             n->as_Bool()->_test._test == BoolTest::gt ||
 3409             n->in(1)->in(1) == n->in(1)->in(2));
 3410   format %{ "" %}
 3411   interface(COND_INTER) %{
 3412     equal(0xb, "np");
 3413     not_equal(0xa, "p");
 3414     less(0x2, "b");
 3415     greater_equal(0x3, "ae");
 3416     less_equal(0x6, "be");
 3417     greater(0x7, "a");
 3418     overflow(0x0, "o");
 3419     no_overflow(0x1, "no");
 3420   %}
 3421 %}
 3422 
 3423 
 3424 // Floating comparisons that can be fixed up with extra conditional jumps
 3425 operand cmpOpUCF2() %{
 3426   match(Bool);
 3427   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3428              n->as_Bool()->_test._test == BoolTest::eq) &&
 3429             n->in(1)->in(1) != n->in(1)->in(2));
 3430   format %{ "" %}
 3431   interface(COND_INTER) %{
 3432     equal(0x4, "e");
 3433     not_equal(0x5, "ne");
 3434     less(0x2, "b");
 3435     greater_equal(0x3, "ae");
 3436     less_equal(0x6, "be");
 3437     greater(0x7, "a");
 3438     overflow(0x0, "o");
 3439     no_overflow(0x1, "no");
 3440   %}
 3441 %}
 3442 
 3443 //----------OPERAND CLASSES----------------------------------------------------
 3444 // Operand Classes are groups of operands that are used as to simplify
 3445 // instruction definitions by not requiring the AD writer to specify separate
 3446 // instructions for every form of operand when the instruction accepts
 3447 // multiple operand types with the same basic encoding and format.  The classic
 3448 // case of this is memory operands.
 3449 
 3450 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3451                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3452                indCompressedOopOffset,
 3453                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3454                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3455                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3456 
 3457 //----------PIPELINE-----------------------------------------------------------
 3458 // Rules which define the behavior of the target architectures pipeline.
 3459 pipeline %{
 3460 
 3461 //----------ATTRIBUTES---------------------------------------------------------
 3462 attributes %{
 3463   variable_size_instructions;        // Fixed size instructions
 3464   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3465   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3466   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3467   instruction_fetch_units = 1;       // of 16 bytes
 3468 
 3469   // List of nop instructions
 3470   nops( MachNop );
 3471 %}
 3472 
 3473 //----------RESOURCES----------------------------------------------------------
 3474 // Resources are the functional units available to the machine
 3475 
 3476 // Generic P2/P3 pipeline
 3477 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3478 // 3 instructions decoded per cycle.
 3479 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3480 // 3 ALU op, only ALU0 handles mul instructions.
 3481 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3482            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3483            BR, FPU,
 3484            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3485 
 3486 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3487 // Pipeline Description specifies the stages in the machine's pipeline
 3488 
 3489 // Generic P2/P3 pipeline
 3490 pipe_desc(S0, S1, S2, S3, S4, S5);
 3491 
 3492 //----------PIPELINE CLASSES---------------------------------------------------
 3493 // Pipeline Classes describe the stages in which input and output are
 3494 // referenced by the hardware pipeline.
 3495 
 3496 // Naming convention: ialu or fpu
 3497 // Then: _reg
 3498 // Then: _reg if there is a 2nd register
 3499 // Then: _long if it's a pair of instructions implementing a long
 3500 // Then: _fat if it requires the big decoder
 3501 //   Or: _mem if it requires the big decoder and a memory unit.
 3502 
 3503 // Integer ALU reg operation
 3504 pipe_class ialu_reg(rRegI dst)
 3505 %{
 3506     single_instruction;
 3507     dst    : S4(write);
 3508     dst    : S3(read);
 3509     DECODE : S0;        // any decoder
 3510     ALU    : S3;        // any alu
 3511 %}
 3512 
 3513 // Long ALU reg operation
 3514 pipe_class ialu_reg_long(rRegL dst)
 3515 %{
 3516     instruction_count(2);
 3517     dst    : S4(write);
 3518     dst    : S3(read);
 3519     DECODE : S0(2);     // any 2 decoders
 3520     ALU    : S3(2);     // both alus
 3521 %}
 3522 
 3523 // Integer ALU reg operation using big decoder
 3524 pipe_class ialu_reg_fat(rRegI dst)
 3525 %{
 3526     single_instruction;
 3527     dst    : S4(write);
 3528     dst    : S3(read);
 3529     D0     : S0;        // big decoder only
 3530     ALU    : S3;        // any alu
 3531 %}
 3532 
 3533 // Integer ALU reg-reg operation
 3534 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3535 %{
 3536     single_instruction;
 3537     dst    : S4(write);
 3538     src    : S3(read);
 3539     DECODE : S0;        // any decoder
 3540     ALU    : S3;        // any alu
 3541 %}
 3542 
 3543 // Integer ALU reg-reg operation
 3544 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3545 %{
 3546     single_instruction;
 3547     dst    : S4(write);
 3548     src    : S3(read);
 3549     D0     : S0;        // big decoder only
 3550     ALU    : S3;        // any alu
 3551 %}
 3552 
 3553 // Integer ALU reg-mem operation
 3554 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3555 %{
 3556     single_instruction;
 3557     dst    : S5(write);
 3558     mem    : S3(read);
 3559     D0     : S0;        // big decoder only
 3560     ALU    : S4;        // any alu
 3561     MEM    : S3;        // any mem
 3562 %}
 3563 
 3564 // Integer mem operation (prefetch)
 3565 pipe_class ialu_mem(memory mem)
 3566 %{
 3567     single_instruction;
 3568     mem    : S3(read);
 3569     D0     : S0;        // big decoder only
 3570     MEM    : S3;        // any mem
 3571 %}
 3572 
 3573 // Integer Store to Memory
 3574 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3575 %{
 3576     single_instruction;
 3577     mem    : S3(read);
 3578     src    : S5(read);
 3579     D0     : S0;        // big decoder only
 3580     ALU    : S4;        // any alu
 3581     MEM    : S3;
 3582 %}
 3583 
 3584 // // Long Store to Memory
 3585 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3586 // %{
 3587 //     instruction_count(2);
 3588 //     mem    : S3(read);
 3589 //     src    : S5(read);
 3590 //     D0     : S0(2);          // big decoder only; twice
 3591 //     ALU    : S4(2);     // any 2 alus
 3592 //     MEM    : S3(2);  // Both mems
 3593 // %}
 3594 
 3595 // Integer Store to Memory
 3596 pipe_class ialu_mem_imm(memory mem)
 3597 %{
 3598     single_instruction;
 3599     mem    : S3(read);
 3600     D0     : S0;        // big decoder only
 3601     ALU    : S4;        // any alu
 3602     MEM    : S3;
 3603 %}
 3604 
 3605 // Integer ALU0 reg-reg operation
 3606 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3607 %{
 3608     single_instruction;
 3609     dst    : S4(write);
 3610     src    : S3(read);
 3611     D0     : S0;        // Big decoder only
 3612     ALU0   : S3;        // only alu0
 3613 %}
 3614 
 3615 // Integer ALU0 reg-mem operation
 3616 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3617 %{
 3618     single_instruction;
 3619     dst    : S5(write);
 3620     mem    : S3(read);
 3621     D0     : S0;        // big decoder only
 3622     ALU0   : S4;        // ALU0 only
 3623     MEM    : S3;        // any mem
 3624 %}
 3625 
 3626 // Integer ALU reg-reg operation
 3627 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3628 %{
 3629     single_instruction;
 3630     cr     : S4(write);
 3631     src1   : S3(read);
 3632     src2   : S3(read);
 3633     DECODE : S0;        // any decoder
 3634     ALU    : S3;        // any alu
 3635 %}
 3636 
 3637 // Integer ALU reg-imm operation
 3638 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3639 %{
 3640     single_instruction;
 3641     cr     : S4(write);
 3642     src1   : S3(read);
 3643     DECODE : S0;        // any decoder
 3644     ALU    : S3;        // any alu
 3645 %}
 3646 
 3647 // Integer ALU reg-mem operation
 3648 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3649 %{
 3650     single_instruction;
 3651     cr     : S4(write);
 3652     src1   : S3(read);
 3653     src2   : S3(read);
 3654     D0     : S0;        // big decoder only
 3655     ALU    : S4;        // any alu
 3656     MEM    : S3;
 3657 %}
 3658 
 3659 // Conditional move reg-reg
 3660 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3661 %{
 3662     instruction_count(4);
 3663     y      : S4(read);
 3664     q      : S3(read);
 3665     p      : S3(read);
 3666     DECODE : S0(4);     // any decoder
 3667 %}
 3668 
 3669 // Conditional move reg-reg
 3670 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3671 %{
 3672     single_instruction;
 3673     dst    : S4(write);
 3674     src    : S3(read);
 3675     cr     : S3(read);
 3676     DECODE : S0;        // any decoder
 3677 %}
 3678 
 3679 // Conditional move reg-mem
 3680 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3681 %{
 3682     single_instruction;
 3683     dst    : S4(write);
 3684     src    : S3(read);
 3685     cr     : S3(read);
 3686     DECODE : S0;        // any decoder
 3687     MEM    : S3;
 3688 %}
 3689 
 3690 // Conditional move reg-reg long
 3691 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3692 %{
 3693     single_instruction;
 3694     dst    : S4(write);
 3695     src    : S3(read);
 3696     cr     : S3(read);
 3697     DECODE : S0(2);     // any 2 decoders
 3698 %}
 3699 
 3700 // Float reg-reg operation
 3701 pipe_class fpu_reg(regD dst)
 3702 %{
 3703     instruction_count(2);
 3704     dst    : S3(read);
 3705     DECODE : S0(2);     // any 2 decoders
 3706     FPU    : S3;
 3707 %}
 3708 
 3709 // Float reg-reg operation
 3710 pipe_class fpu_reg_reg(regD dst, regD src)
 3711 %{
 3712     instruction_count(2);
 3713     dst    : S4(write);
 3714     src    : S3(read);
 3715     DECODE : S0(2);     // any 2 decoders
 3716     FPU    : S3;
 3717 %}
 3718 
 3719 // Float reg-reg operation
 3720 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3721 %{
 3722     instruction_count(3);
 3723     dst    : S4(write);
 3724     src1   : S3(read);
 3725     src2   : S3(read);
 3726     DECODE : S0(3);     // any 3 decoders
 3727     FPU    : S3(2);
 3728 %}
 3729 
 3730 // Float reg-reg operation
 3731 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3732 %{
 3733     instruction_count(4);
 3734     dst    : S4(write);
 3735     src1   : S3(read);
 3736     src2   : S3(read);
 3737     src3   : S3(read);
 3738     DECODE : S0(4);     // any 3 decoders
 3739     FPU    : S3(2);
 3740 %}
 3741 
 3742 // Float reg-reg operation
 3743 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3744 %{
 3745     instruction_count(4);
 3746     dst    : S4(write);
 3747     src1   : S3(read);
 3748     src2   : S3(read);
 3749     src3   : S3(read);
 3750     DECODE : S1(3);     // any 3 decoders
 3751     D0     : S0;        // Big decoder only
 3752     FPU    : S3(2);
 3753     MEM    : S3;
 3754 %}
 3755 
 3756 // Float reg-mem operation
 3757 pipe_class fpu_reg_mem(regD dst, memory mem)
 3758 %{
 3759     instruction_count(2);
 3760     dst    : S5(write);
 3761     mem    : S3(read);
 3762     D0     : S0;        // big decoder only
 3763     DECODE : S1;        // any decoder for FPU POP
 3764     FPU    : S4;
 3765     MEM    : S3;        // any mem
 3766 %}
 3767 
 3768 // Float reg-mem operation
 3769 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3770 %{
 3771     instruction_count(3);
 3772     dst    : S5(write);
 3773     src1   : S3(read);
 3774     mem    : S3(read);
 3775     D0     : S0;        // big decoder only
 3776     DECODE : S1(2);     // any decoder for FPU POP
 3777     FPU    : S4;
 3778     MEM    : S3;        // any mem
 3779 %}
 3780 
 3781 // Float mem-reg operation
 3782 pipe_class fpu_mem_reg(memory mem, regD src)
 3783 %{
 3784     instruction_count(2);
 3785     src    : S5(read);
 3786     mem    : S3(read);
 3787     DECODE : S0;        // any decoder for FPU PUSH
 3788     D0     : S1;        // big decoder only
 3789     FPU    : S4;
 3790     MEM    : S3;        // any mem
 3791 %}
 3792 
 3793 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3794 %{
 3795     instruction_count(3);
 3796     src1   : S3(read);
 3797     src2   : S3(read);
 3798     mem    : S3(read);
 3799     DECODE : S0(2);     // any decoder for FPU PUSH
 3800     D0     : S1;        // big decoder only
 3801     FPU    : S4;
 3802     MEM    : S3;        // any mem
 3803 %}
 3804 
 3805 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3806 %{
 3807     instruction_count(3);
 3808     src1   : S3(read);
 3809     src2   : S3(read);
 3810     mem    : S4(read);
 3811     DECODE : S0;        // any decoder for FPU PUSH
 3812     D0     : S0(2);     // big decoder only
 3813     FPU    : S4;
 3814     MEM    : S3(2);     // any mem
 3815 %}
 3816 
 3817 pipe_class fpu_mem_mem(memory dst, memory src1)
 3818 %{
 3819     instruction_count(2);
 3820     src1   : S3(read);
 3821     dst    : S4(read);
 3822     D0     : S0(2);     // big decoder only
 3823     MEM    : S3(2);     // any mem
 3824 %}
 3825 
 3826 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3827 %{
 3828     instruction_count(3);
 3829     src1   : S3(read);
 3830     src2   : S3(read);
 3831     dst    : S4(read);
 3832     D0     : S0(3);     // big decoder only
 3833     FPU    : S4;
 3834     MEM    : S3(3);     // any mem
 3835 %}
 3836 
 3837 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3838 %{
 3839     instruction_count(3);
 3840     src1   : S4(read);
 3841     mem    : S4(read);
 3842     DECODE : S0;        // any decoder for FPU PUSH
 3843     D0     : S0(2);     // big decoder only
 3844     FPU    : S4;
 3845     MEM    : S3(2);     // any mem
 3846 %}
 3847 
 3848 // Float load constant
 3849 pipe_class fpu_reg_con(regD dst)
 3850 %{
 3851     instruction_count(2);
 3852     dst    : S5(write);
 3853     D0     : S0;        // big decoder only for the load
 3854     DECODE : S1;        // any decoder for FPU POP
 3855     FPU    : S4;
 3856     MEM    : S3;        // any mem
 3857 %}
 3858 
 3859 // Float load constant
 3860 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3861 %{
 3862     instruction_count(3);
 3863     dst    : S5(write);
 3864     src    : S3(read);
 3865     D0     : S0;        // big decoder only for the load
 3866     DECODE : S1(2);     // any decoder for FPU POP
 3867     FPU    : S4;
 3868     MEM    : S3;        // any mem
 3869 %}
 3870 
 3871 // UnConditional branch
 3872 pipe_class pipe_jmp(label labl)
 3873 %{
 3874     single_instruction;
 3875     BR   : S3;
 3876 %}
 3877 
 3878 // Conditional branch
 3879 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3880 %{
 3881     single_instruction;
 3882     cr    : S1(read);
 3883     BR    : S3;
 3884 %}
 3885 
 3886 // Allocation idiom
 3887 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3888 %{
 3889     instruction_count(1); force_serialization;
 3890     fixed_latency(6);
 3891     heap_ptr : S3(read);
 3892     DECODE   : S0(3);
 3893     D0       : S2;
 3894     MEM      : S3;
 3895     ALU      : S3(2);
 3896     dst      : S5(write);
 3897     BR       : S5;
 3898 %}
 3899 
 3900 // Generic big/slow expanded idiom
 3901 pipe_class pipe_slow()
 3902 %{
 3903     instruction_count(10); multiple_bundles; force_serialization;
 3904     fixed_latency(100);
 3905     D0  : S0(2);
 3906     MEM : S3(2);
 3907 %}
 3908 
 3909 // The real do-nothing guy
 3910 pipe_class empty()
 3911 %{
 3912     instruction_count(0);
 3913 %}
 3914 
 3915 // Define the class for the Nop node
 3916 define
 3917 %{
 3918    MachNop = empty;
 3919 %}
 3920 
 3921 %}
 3922 
 3923 //----------INSTRUCTIONS-------------------------------------------------------
 3924 //
 3925 // match      -- States which machine-independent subtree may be replaced
 3926 //               by this instruction.
 3927 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3928 //               selection to identify a minimum cost tree of machine
 3929 //               instructions that matches a tree of machine-independent
 3930 //               instructions.
 3931 // format     -- A string providing the disassembly for this instruction.
 3932 //               The value of an instruction's operand may be inserted
 3933 //               by referring to it with a '$' prefix.
 3934 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3935 //               to within an encode class as $primary, $secondary, and $tertiary
 3936 //               rrspectively.  The primary opcode is commonly used to
 3937 //               indicate the type of machine instruction, while secondary
 3938 //               and tertiary are often used for prefix options or addressing
 3939 //               modes.
 3940 // ins_encode -- A list of encode classes with parameters. The encode class
 3941 //               name must have been defined in an 'enc_class' specification
 3942 //               in the encode section of the architecture description.
 3943 
 3944 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3945 // Load Float
 3946 instruct MoveF2VL(vlRegF dst, regF src) %{
 3947   match(Set dst src);
 3948   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3949   ins_encode %{
 3950     ShouldNotReachHere();
 3951   %}
 3952   ins_pipe( fpu_reg_reg );
 3953 %}
 3954 
 3955 // Load Float
 3956 instruct MoveF2LEG(legRegF dst, regF src) %{
 3957   match(Set dst src);
 3958   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3959   ins_encode %{
 3960     ShouldNotReachHere();
 3961   %}
 3962   ins_pipe( fpu_reg_reg );
 3963 %}
 3964 
 3965 // Load Float
 3966 instruct MoveVL2F(regF dst, vlRegF src) %{
 3967   match(Set dst src);
 3968   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3969   ins_encode %{
 3970     ShouldNotReachHere();
 3971   %}
 3972   ins_pipe( fpu_reg_reg );
 3973 %}
 3974 
 3975 // Load Float
 3976 instruct MoveLEG2F(regF dst, legRegF src) %{
 3977   match(Set dst src);
 3978   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3979   ins_encode %{
 3980     ShouldNotReachHere();
 3981   %}
 3982   ins_pipe( fpu_reg_reg );
 3983 %}
 3984 
 3985 // Load Double
 3986 instruct MoveD2VL(vlRegD dst, regD src) %{
 3987   match(Set dst src);
 3988   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3989   ins_encode %{
 3990     ShouldNotReachHere();
 3991   %}
 3992   ins_pipe( fpu_reg_reg );
 3993 %}
 3994 
 3995 // Load Double
 3996 instruct MoveD2LEG(legRegD dst, regD src) %{
 3997   match(Set dst src);
 3998   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3999   ins_encode %{
 4000     ShouldNotReachHere();
 4001   %}
 4002   ins_pipe( fpu_reg_reg );
 4003 %}
 4004 
 4005 // Load Double
 4006 instruct MoveVL2D(regD dst, vlRegD src) %{
 4007   match(Set dst src);
 4008   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4009   ins_encode %{
 4010     ShouldNotReachHere();
 4011   %}
 4012   ins_pipe( fpu_reg_reg );
 4013 %}
 4014 
 4015 // Load Double
 4016 instruct MoveLEG2D(regD dst, legRegD src) %{
 4017   match(Set dst src);
 4018   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4019   ins_encode %{
 4020     ShouldNotReachHere();
 4021   %}
 4022   ins_pipe( fpu_reg_reg );
 4023 %}
 4024 
 4025 //----------Load/Store/Move Instructions---------------------------------------
 4026 //----------Load Instructions--------------------------------------------------
 4027 
 4028 // Load Byte (8 bit signed)
 4029 instruct loadB(rRegI dst, memory mem)
 4030 %{
 4031   match(Set dst (LoadB mem));
 4032 
 4033   ins_cost(125);
 4034   format %{ "movsbl  $dst, $mem\t# byte" %}
 4035 
 4036   ins_encode %{
 4037     __ movsbl($dst$$Register, $mem$$Address);
 4038   %}
 4039 
 4040   ins_pipe(ialu_reg_mem);
 4041 %}
 4042 
 4043 // Load Byte (8 bit signed) into Long Register
 4044 instruct loadB2L(rRegL dst, memory mem)
 4045 %{
 4046   match(Set dst (ConvI2L (LoadB mem)));
 4047 
 4048   ins_cost(125);
 4049   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4050 
 4051   ins_encode %{
 4052     __ movsbq($dst$$Register, $mem$$Address);
 4053   %}
 4054 
 4055   ins_pipe(ialu_reg_mem);
 4056 %}
 4057 
 4058 // Load Unsigned Byte (8 bit UNsigned)
 4059 instruct loadUB(rRegI dst, memory mem)
 4060 %{
 4061   match(Set dst (LoadUB mem));
 4062 
 4063   ins_cost(125);
 4064   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4065 
 4066   ins_encode %{
 4067     __ movzbl($dst$$Register, $mem$$Address);
 4068   %}
 4069 
 4070   ins_pipe(ialu_reg_mem);
 4071 %}
 4072 
 4073 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4074 instruct loadUB2L(rRegL dst, memory mem)
 4075 %{
 4076   match(Set dst (ConvI2L (LoadUB mem)));
 4077 
 4078   ins_cost(125);
 4079   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4080 
 4081   ins_encode %{
 4082     __ movzbq($dst$$Register, $mem$$Address);
 4083   %}
 4084 
 4085   ins_pipe(ialu_reg_mem);
 4086 %}
 4087 
 4088 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4089 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4090   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4091   effect(KILL cr);
 4092 
 4093   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4094             "andl    $dst, right_n_bits($mask, 8)" %}
 4095   ins_encode %{
 4096     Register Rdst = $dst$$Register;
 4097     __ movzbq(Rdst, $mem$$Address);
 4098     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4099   %}
 4100   ins_pipe(ialu_reg_mem);
 4101 %}
 4102 
 4103 // Load Short (16 bit signed)
 4104 instruct loadS(rRegI dst, memory mem)
 4105 %{
 4106   match(Set dst (LoadS mem));
 4107 
 4108   ins_cost(125);
 4109   format %{ "movswl $dst, $mem\t# short" %}
 4110 
 4111   ins_encode %{
 4112     __ movswl($dst$$Register, $mem$$Address);
 4113   %}
 4114 
 4115   ins_pipe(ialu_reg_mem);
 4116 %}
 4117 
 4118 // Load Short (16 bit signed) to Byte (8 bit signed)
 4119 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4120   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4121 
 4122   ins_cost(125);
 4123   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4124   ins_encode %{
 4125     __ movsbl($dst$$Register, $mem$$Address);
 4126   %}
 4127   ins_pipe(ialu_reg_mem);
 4128 %}
 4129 
 4130 // Load Short (16 bit signed) into Long Register
 4131 instruct loadS2L(rRegL dst, memory mem)
 4132 %{
 4133   match(Set dst (ConvI2L (LoadS mem)));
 4134 
 4135   ins_cost(125);
 4136   format %{ "movswq $dst, $mem\t# short -> long" %}
 4137 
 4138   ins_encode %{
 4139     __ movswq($dst$$Register, $mem$$Address);
 4140   %}
 4141 
 4142   ins_pipe(ialu_reg_mem);
 4143 %}
 4144 
 4145 // Load Unsigned Short/Char (16 bit UNsigned)
 4146 instruct loadUS(rRegI dst, memory mem)
 4147 %{
 4148   match(Set dst (LoadUS mem));
 4149 
 4150   ins_cost(125);
 4151   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4152 
 4153   ins_encode %{
 4154     __ movzwl($dst$$Register, $mem$$Address);
 4155   %}
 4156 
 4157   ins_pipe(ialu_reg_mem);
 4158 %}
 4159 
 4160 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4161 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4162   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4163 
 4164   ins_cost(125);
 4165   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4166   ins_encode %{
 4167     __ movsbl($dst$$Register, $mem$$Address);
 4168   %}
 4169   ins_pipe(ialu_reg_mem);
 4170 %}
 4171 
 4172 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4173 instruct loadUS2L(rRegL dst, memory mem)
 4174 %{
 4175   match(Set dst (ConvI2L (LoadUS mem)));
 4176 
 4177   ins_cost(125);
 4178   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4179 
 4180   ins_encode %{
 4181     __ movzwq($dst$$Register, $mem$$Address);
 4182   %}
 4183 
 4184   ins_pipe(ialu_reg_mem);
 4185 %}
 4186 
 4187 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4188 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4189   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4190 
 4191   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4192   ins_encode %{
 4193     __ movzbq($dst$$Register, $mem$$Address);
 4194   %}
 4195   ins_pipe(ialu_reg_mem);
 4196 %}
 4197 
 4198 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4199 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4200   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4201   effect(KILL cr);
 4202 
 4203   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4204             "andl    $dst, right_n_bits($mask, 16)" %}
 4205   ins_encode %{
 4206     Register Rdst = $dst$$Register;
 4207     __ movzwq(Rdst, $mem$$Address);
 4208     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4209   %}
 4210   ins_pipe(ialu_reg_mem);
 4211 %}
 4212 
 4213 // Load Integer
 4214 instruct loadI(rRegI dst, memory mem)
 4215 %{
 4216   match(Set dst (LoadI mem));
 4217 
 4218   ins_cost(125);
 4219   format %{ "movl    $dst, $mem\t# int" %}
 4220 
 4221   ins_encode %{
 4222     __ movl($dst$$Register, $mem$$Address);
 4223   %}
 4224 
 4225   ins_pipe(ialu_reg_mem);
 4226 %}
 4227 
 4228 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4229 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4230   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4231 
 4232   ins_cost(125);
 4233   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4234   ins_encode %{
 4235     __ movsbl($dst$$Register, $mem$$Address);
 4236   %}
 4237   ins_pipe(ialu_reg_mem);
 4238 %}
 4239 
 4240 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4241 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4242   match(Set dst (AndI (LoadI mem) mask));
 4243 
 4244   ins_cost(125);
 4245   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4246   ins_encode %{
 4247     __ movzbl($dst$$Register, $mem$$Address);
 4248   %}
 4249   ins_pipe(ialu_reg_mem);
 4250 %}
 4251 
 4252 // Load Integer (32 bit signed) to Short (16 bit signed)
 4253 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4254   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4255 
 4256   ins_cost(125);
 4257   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4258   ins_encode %{
 4259     __ movswl($dst$$Register, $mem$$Address);
 4260   %}
 4261   ins_pipe(ialu_reg_mem);
 4262 %}
 4263 
 4264 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4265 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4266   match(Set dst (AndI (LoadI mem) mask));
 4267 
 4268   ins_cost(125);
 4269   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4270   ins_encode %{
 4271     __ movzwl($dst$$Register, $mem$$Address);
 4272   %}
 4273   ins_pipe(ialu_reg_mem);
 4274 %}
 4275 
 4276 // Load Integer into Long Register
 4277 instruct loadI2L(rRegL dst, memory mem)
 4278 %{
 4279   match(Set dst (ConvI2L (LoadI mem)));
 4280 
 4281   ins_cost(125);
 4282   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4283 
 4284   ins_encode %{
 4285     __ movslq($dst$$Register, $mem$$Address);
 4286   %}
 4287 
 4288   ins_pipe(ialu_reg_mem);
 4289 %}
 4290 
 4291 // Load Integer with mask 0xFF into Long Register
 4292 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4293   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4294 
 4295   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4296   ins_encode %{
 4297     __ movzbq($dst$$Register, $mem$$Address);
 4298   %}
 4299   ins_pipe(ialu_reg_mem);
 4300 %}
 4301 
 4302 // Load Integer with mask 0xFFFF into Long Register
 4303 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4304   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4305 
 4306   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4307   ins_encode %{
 4308     __ movzwq($dst$$Register, $mem$$Address);
 4309   %}
 4310   ins_pipe(ialu_reg_mem);
 4311 %}
 4312 
 4313 // Load Integer with a 31-bit mask into Long Register
 4314 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4315   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4316   effect(KILL cr);
 4317 
 4318   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4319             "andl    $dst, $mask" %}
 4320   ins_encode %{
 4321     Register Rdst = $dst$$Register;
 4322     __ movl(Rdst, $mem$$Address);
 4323     __ andl(Rdst, $mask$$constant);
 4324   %}
 4325   ins_pipe(ialu_reg_mem);
 4326 %}
 4327 
 4328 // Load Unsigned Integer into Long Register
 4329 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4330 %{
 4331   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4332 
 4333   ins_cost(125);
 4334   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4335 
 4336   ins_encode %{
 4337     __ movl($dst$$Register, $mem$$Address);
 4338   %}
 4339 
 4340   ins_pipe(ialu_reg_mem);
 4341 %}
 4342 
 4343 // Load Long
 4344 instruct loadL(rRegL dst, memory mem)
 4345 %{
 4346   match(Set dst (LoadL mem));
 4347 
 4348   ins_cost(125);
 4349   format %{ "movq    $dst, $mem\t# long" %}
 4350 
 4351   ins_encode %{
 4352     __ movq($dst$$Register, $mem$$Address);
 4353   %}
 4354 
 4355   ins_pipe(ialu_reg_mem); // XXX
 4356 %}
 4357 
 4358 // Load Range
 4359 instruct loadRange(rRegI dst, memory mem)
 4360 %{
 4361   match(Set dst (LoadRange mem));
 4362 
 4363   ins_cost(125); // XXX
 4364   format %{ "movl    $dst, $mem\t# range" %}
 4365   ins_encode %{
 4366     __ movl($dst$$Register, $mem$$Address);
 4367   %}
 4368   ins_pipe(ialu_reg_mem);
 4369 %}
 4370 
 4371 // Load Pointer
 4372 instruct loadP(rRegP dst, memory mem)
 4373 %{
 4374   match(Set dst (LoadP mem));
 4375   predicate(n->as_Load()->barrier_data() == 0);
 4376 
 4377   ins_cost(125); // XXX
 4378   format %{ "movq    $dst, $mem\t# ptr" %}
 4379   ins_encode %{
 4380     __ movq($dst$$Register, $mem$$Address);
 4381   %}
 4382   ins_pipe(ialu_reg_mem); // XXX
 4383 %}
 4384 
 4385 // Load Compressed Pointer
 4386 instruct loadN(rRegN dst, memory mem)
 4387 %{
 4388    match(Set dst (LoadN mem));
 4389 
 4390    ins_cost(125); // XXX
 4391    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4392    ins_encode %{
 4393      __ movl($dst$$Register, $mem$$Address);
 4394    %}
 4395    ins_pipe(ialu_reg_mem); // XXX
 4396 %}
 4397 
 4398 
 4399 // Load Klass Pointer
 4400 instruct loadKlass(rRegP dst, memory mem)
 4401 %{
 4402   match(Set dst (LoadKlass mem));
 4403 
 4404   ins_cost(125); // XXX
 4405   format %{ "movq    $dst, $mem\t# class" %}
 4406   ins_encode %{
 4407     __ movq($dst$$Register, $mem$$Address);
 4408   %}
 4409   ins_pipe(ialu_reg_mem); // XXX
 4410 %}
 4411 
 4412 // Load narrow Klass Pointer
 4413 instruct loadNKlass(rRegN dst, memory mem)
 4414 %{
 4415   match(Set dst (LoadNKlass mem));
 4416 
 4417   ins_cost(125); // XXX
 4418   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4419   ins_encode %{
 4420     __ movl($dst$$Register, $mem$$Address);
 4421   %}
 4422   ins_pipe(ialu_reg_mem); // XXX
 4423 %}
 4424 
 4425 // Load Float
 4426 instruct loadF(regF dst, memory mem)
 4427 %{
 4428   match(Set dst (LoadF mem));
 4429 
 4430   ins_cost(145); // XXX
 4431   format %{ "movss   $dst, $mem\t# float" %}
 4432   ins_encode %{
 4433     __ movflt($dst$$XMMRegister, $mem$$Address);
 4434   %}
 4435   ins_pipe(pipe_slow); // XXX
 4436 %}
 4437 
 4438 // Load Double
 4439 instruct loadD_partial(regD dst, memory mem)
 4440 %{
 4441   predicate(!UseXmmLoadAndClearUpper);
 4442   match(Set dst (LoadD mem));
 4443 
 4444   ins_cost(145); // XXX
 4445   format %{ "movlpd  $dst, $mem\t# double" %}
 4446   ins_encode %{
 4447     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4448   %}
 4449   ins_pipe(pipe_slow); // XXX
 4450 %}
 4451 
 4452 instruct loadD(regD dst, memory mem)
 4453 %{
 4454   predicate(UseXmmLoadAndClearUpper);
 4455   match(Set dst (LoadD mem));
 4456 
 4457   ins_cost(145); // XXX
 4458   format %{ "movsd   $dst, $mem\t# double" %}
 4459   ins_encode %{
 4460     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4461   %}
 4462   ins_pipe(pipe_slow); // XXX
 4463 %}
 4464 
 4465 // max = java.lang.Math.max(float a, float b)
 4466 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4467   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4468   match(Set dst (MaxF a b));
 4469   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4470   format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4471   ins_encode %{
 4472     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4473   %}
 4474   ins_pipe( pipe_slow );
 4475 %}
 4476 
 4477 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4478   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4479   match(Set dst (MaxF a b));
 4480   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4481 
 4482   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 4483   ins_encode %{
 4484     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4485                     false /*min*/, true /*single*/);
 4486   %}
 4487   ins_pipe( pipe_slow );
 4488 %}
 4489 
 4490 // max = java.lang.Math.max(double a, double b)
 4491 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4492   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4493   match(Set dst (MaxD a b));
 4494   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4495   format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4496   ins_encode %{
 4497     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4498   %}
 4499   ins_pipe( pipe_slow );
 4500 %}
 4501 
 4502 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4503   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4504   match(Set dst (MaxD a b));
 4505   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4506 
 4507   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 4508   ins_encode %{
 4509     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4510                     false /*min*/, false /*single*/);
 4511   %}
 4512   ins_pipe( pipe_slow );
 4513 %}
 4514 
 4515 // min = java.lang.Math.min(float a, float b)
 4516 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4517   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4518   match(Set dst (MinF a b));
 4519   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4520   format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4521   ins_encode %{
 4522     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4523   %}
 4524   ins_pipe( pipe_slow );
 4525 %}
 4526 
 4527 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4528   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4529   match(Set dst (MinF a b));
 4530   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4531 
 4532   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 4533   ins_encode %{
 4534     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4535                     true /*min*/, true /*single*/);
 4536   %}
 4537   ins_pipe( pipe_slow );
 4538 %}
 4539 
 4540 // min = java.lang.Math.min(double a, double b)
 4541 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4542   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4543   match(Set dst (MinD a b));
 4544   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4545     format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
 4546   ins_encode %{
 4547     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4548   %}
 4549   ins_pipe( pipe_slow );
 4550 %}
 4551 
 4552 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4553   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4554   match(Set dst (MinD a b));
 4555   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4556 
 4557   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 4558   ins_encode %{
 4559     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4560                     true /*min*/, false /*single*/);
 4561   %}
 4562   ins_pipe( pipe_slow );
 4563 %}
 4564 
 4565 // Load Effective Address
 4566 instruct leaP8(rRegP dst, indOffset8 mem)
 4567 %{
 4568   match(Set dst mem);
 4569 
 4570   ins_cost(110); // XXX
 4571   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4572   ins_encode %{
 4573     __ leaq($dst$$Register, $mem$$Address);
 4574   %}
 4575   ins_pipe(ialu_reg_reg_fat);
 4576 %}
 4577 
 4578 instruct leaP32(rRegP dst, indOffset32 mem)
 4579 %{
 4580   match(Set dst mem);
 4581 
 4582   ins_cost(110);
 4583   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4584   ins_encode %{
 4585     __ leaq($dst$$Register, $mem$$Address);
 4586   %}
 4587   ins_pipe(ialu_reg_reg_fat);
 4588 %}
 4589 
 4590 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4591 %{
 4592   match(Set dst mem);
 4593 
 4594   ins_cost(110);
 4595   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4596   ins_encode %{
 4597     __ leaq($dst$$Register, $mem$$Address);
 4598   %}
 4599   ins_pipe(ialu_reg_reg_fat);
 4600 %}
 4601 
 4602 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4603 %{
 4604   match(Set dst mem);
 4605 
 4606   ins_cost(110);
 4607   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4608   ins_encode %{
 4609     __ leaq($dst$$Register, $mem$$Address);
 4610   %}
 4611   ins_pipe(ialu_reg_reg_fat);
 4612 %}
 4613 
 4614 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4615 %{
 4616   match(Set dst mem);
 4617 
 4618   ins_cost(110);
 4619   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4620   ins_encode %{
 4621     __ leaq($dst$$Register, $mem$$Address);
 4622   %}
 4623   ins_pipe(ialu_reg_reg_fat);
 4624 %}
 4625 
 4626 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4627 %{
 4628   match(Set dst mem);
 4629 
 4630   ins_cost(110);
 4631   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4632   ins_encode %{
 4633     __ leaq($dst$$Register, $mem$$Address);
 4634   %}
 4635   ins_pipe(ialu_reg_reg_fat);
 4636 %}
 4637 
 4638 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4639 %{
 4640   match(Set dst mem);
 4641 
 4642   ins_cost(110);
 4643   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4644   ins_encode %{
 4645     __ leaq($dst$$Register, $mem$$Address);
 4646   %}
 4647   ins_pipe(ialu_reg_reg_fat);
 4648 %}
 4649 
 4650 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4651 %{
 4652   match(Set dst mem);
 4653 
 4654   ins_cost(110);
 4655   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4656   ins_encode %{
 4657     __ leaq($dst$$Register, $mem$$Address);
 4658   %}
 4659   ins_pipe(ialu_reg_reg_fat);
 4660 %}
 4661 
 4662 // Load Effective Address which uses Narrow (32-bits) oop
 4663 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4664 %{
 4665   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4666   match(Set dst mem);
 4667 
 4668   ins_cost(110);
 4669   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4670   ins_encode %{
 4671     __ leaq($dst$$Register, $mem$$Address);
 4672   %}
 4673   ins_pipe(ialu_reg_reg_fat);
 4674 %}
 4675 
 4676 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4677 %{
 4678   predicate(CompressedOops::shift() == 0);
 4679   match(Set dst mem);
 4680 
 4681   ins_cost(110); // XXX
 4682   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4683   ins_encode %{
 4684     __ leaq($dst$$Register, $mem$$Address);
 4685   %}
 4686   ins_pipe(ialu_reg_reg_fat);
 4687 %}
 4688 
 4689 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4690 %{
 4691   predicate(CompressedOops::shift() == 0);
 4692   match(Set dst mem);
 4693 
 4694   ins_cost(110);
 4695   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4696   ins_encode %{
 4697     __ leaq($dst$$Register, $mem$$Address);
 4698   %}
 4699   ins_pipe(ialu_reg_reg_fat);
 4700 %}
 4701 
 4702 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4703 %{
 4704   predicate(CompressedOops::shift() == 0);
 4705   match(Set dst mem);
 4706 
 4707   ins_cost(110);
 4708   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4709   ins_encode %{
 4710     __ leaq($dst$$Register, $mem$$Address);
 4711   %}
 4712   ins_pipe(ialu_reg_reg_fat);
 4713 %}
 4714 
 4715 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4716 %{
 4717   predicate(CompressedOops::shift() == 0);
 4718   match(Set dst mem);
 4719 
 4720   ins_cost(110);
 4721   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4722   ins_encode %{
 4723     __ leaq($dst$$Register, $mem$$Address);
 4724   %}
 4725   ins_pipe(ialu_reg_reg_fat);
 4726 %}
 4727 
 4728 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4729 %{
 4730   predicate(CompressedOops::shift() == 0);
 4731   match(Set dst mem);
 4732 
 4733   ins_cost(110);
 4734   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4735   ins_encode %{
 4736     __ leaq($dst$$Register, $mem$$Address);
 4737   %}
 4738   ins_pipe(ialu_reg_reg_fat);
 4739 %}
 4740 
 4741 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4742 %{
 4743   predicate(CompressedOops::shift() == 0);
 4744   match(Set dst mem);
 4745 
 4746   ins_cost(110);
 4747   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4748   ins_encode %{
 4749     __ leaq($dst$$Register, $mem$$Address);
 4750   %}
 4751   ins_pipe(ialu_reg_reg_fat);
 4752 %}
 4753 
 4754 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4755 %{
 4756   predicate(CompressedOops::shift() == 0);
 4757   match(Set dst mem);
 4758 
 4759   ins_cost(110);
 4760   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4761   ins_encode %{
 4762     __ leaq($dst$$Register, $mem$$Address);
 4763   %}
 4764   ins_pipe(ialu_reg_reg_fat);
 4765 %}
 4766 
 4767 instruct loadConI(rRegI dst, immI src)
 4768 %{
 4769   match(Set dst src);
 4770 
 4771   format %{ "movl    $dst, $src\t# int" %}
 4772   ins_encode %{
 4773     __ movl($dst$$Register, $src$$constant);
 4774   %}
 4775   ins_pipe(ialu_reg_fat); // XXX
 4776 %}
 4777 
 4778 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4779 %{
 4780   match(Set dst src);
 4781   effect(KILL cr);
 4782 
 4783   ins_cost(50);
 4784   format %{ "xorl    $dst, $dst\t# int" %}
 4785   ins_encode %{
 4786     __ xorl($dst$$Register, $dst$$Register);
 4787   %}
 4788   ins_pipe(ialu_reg);
 4789 %}
 4790 
 4791 instruct loadConL(rRegL dst, immL src)
 4792 %{
 4793   match(Set dst src);
 4794 
 4795   ins_cost(150);
 4796   format %{ "movq    $dst, $src\t# long" %}
 4797   ins_encode %{
 4798     __ mov64($dst$$Register, $src$$constant);
 4799   %}
 4800   ins_pipe(ialu_reg);
 4801 %}
 4802 
 4803 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4804 %{
 4805   match(Set dst src);
 4806   effect(KILL cr);
 4807 
 4808   ins_cost(50);
 4809   format %{ "xorl    $dst, $dst\t# long" %}
 4810   ins_encode %{
 4811     __ xorl($dst$$Register, $dst$$Register);
 4812   %}
 4813   ins_pipe(ialu_reg); // XXX
 4814 %}
 4815 
 4816 instruct loadConUL32(rRegL dst, immUL32 src)
 4817 %{
 4818   match(Set dst src);
 4819 
 4820   ins_cost(60);
 4821   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4822   ins_encode %{
 4823     __ movl($dst$$Register, $src$$constant);
 4824   %}
 4825   ins_pipe(ialu_reg);
 4826 %}
 4827 
 4828 instruct loadConL32(rRegL dst, immL32 src)
 4829 %{
 4830   match(Set dst src);
 4831 
 4832   ins_cost(70);
 4833   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4834   ins_encode %{
 4835     __ movq($dst$$Register, $src$$constant);
 4836   %}
 4837   ins_pipe(ialu_reg);
 4838 %}
 4839 
 4840 instruct loadConP(rRegP dst, immP con) %{
 4841   match(Set dst con);
 4842 
 4843   format %{ "movq    $dst, $con\t# ptr" %}
 4844   ins_encode %{
 4845     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4846   %}
 4847   ins_pipe(ialu_reg_fat); // XXX
 4848 %}
 4849 
 4850 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4851 %{
 4852   match(Set dst src);
 4853   effect(KILL cr);
 4854 
 4855   ins_cost(50);
 4856   format %{ "xorl    $dst, $dst\t# ptr" %}
 4857   ins_encode %{
 4858     __ xorl($dst$$Register, $dst$$Register);
 4859   %}
 4860   ins_pipe(ialu_reg);
 4861 %}
 4862 
 4863 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4864 %{
 4865   match(Set dst src);
 4866   effect(KILL cr);
 4867 
 4868   ins_cost(60);
 4869   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4870   ins_encode %{
 4871     __ movl($dst$$Register, $src$$constant);
 4872   %}
 4873   ins_pipe(ialu_reg);
 4874 %}
 4875 
 4876 instruct loadConF(regF dst, immF con) %{
 4877   match(Set dst con);
 4878   ins_cost(125);
 4879   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4880   ins_encode %{
 4881     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4882   %}
 4883   ins_pipe(pipe_slow);
 4884 %}
 4885 
 4886 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4887   match(Set dst src);
 4888   effect(KILL cr);
 4889   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4890   ins_encode %{
 4891     __ xorq($dst$$Register, $dst$$Register);
 4892   %}
 4893   ins_pipe(ialu_reg);
 4894 %}
 4895 
 4896 instruct loadConN(rRegN dst, immN src) %{
 4897   match(Set dst src);
 4898 
 4899   ins_cost(125);
 4900   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4901   ins_encode %{
 4902     address con = (address)$src$$constant;
 4903     if (con == nullptr) {
 4904       ShouldNotReachHere();
 4905     } else {
 4906       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4907     }
 4908   %}
 4909   ins_pipe(ialu_reg_fat); // XXX
 4910 %}
 4911 
 4912 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4913   match(Set dst src);
 4914 
 4915   ins_cost(125);
 4916   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4917   ins_encode %{
 4918     address con = (address)$src$$constant;
 4919     if (con == nullptr) {
 4920       ShouldNotReachHere();
 4921     } else {
 4922       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4923     }
 4924   %}
 4925   ins_pipe(ialu_reg_fat); // XXX
 4926 %}
 4927 
 4928 instruct loadConF0(regF dst, immF0 src)
 4929 %{
 4930   match(Set dst src);
 4931   ins_cost(100);
 4932 
 4933   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4934   ins_encode %{
 4935     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4936   %}
 4937   ins_pipe(pipe_slow);
 4938 %}
 4939 
 4940 // Use the same format since predicate() can not be used here.
 4941 instruct loadConD(regD dst, immD con) %{
 4942   match(Set dst con);
 4943   ins_cost(125);
 4944   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4945   ins_encode %{
 4946     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4947   %}
 4948   ins_pipe(pipe_slow);
 4949 %}
 4950 
 4951 instruct loadConD0(regD dst, immD0 src)
 4952 %{
 4953   match(Set dst src);
 4954   ins_cost(100);
 4955 
 4956   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4957   ins_encode %{
 4958     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4959   %}
 4960   ins_pipe(pipe_slow);
 4961 %}
 4962 
 4963 instruct loadSSI(rRegI dst, stackSlotI src)
 4964 %{
 4965   match(Set dst src);
 4966 
 4967   ins_cost(125);
 4968   format %{ "movl    $dst, $src\t# int stk" %}
 4969   ins_encode %{
 4970     __ movl($dst$$Register, $src$$Address);
 4971   %}
 4972   ins_pipe(ialu_reg_mem);
 4973 %}
 4974 
 4975 instruct loadSSL(rRegL dst, stackSlotL src)
 4976 %{
 4977   match(Set dst src);
 4978 
 4979   ins_cost(125);
 4980   format %{ "movq    $dst, $src\t# long stk" %}
 4981   ins_encode %{
 4982     __ movq($dst$$Register, $src$$Address);
 4983   %}
 4984   ins_pipe(ialu_reg_mem);
 4985 %}
 4986 
 4987 instruct loadSSP(rRegP dst, stackSlotP src)
 4988 %{
 4989   match(Set dst src);
 4990 
 4991   ins_cost(125);
 4992   format %{ "movq    $dst, $src\t# ptr stk" %}
 4993   ins_encode %{
 4994     __ movq($dst$$Register, $src$$Address);
 4995   %}
 4996   ins_pipe(ialu_reg_mem);
 4997 %}
 4998 
 4999 instruct loadSSF(regF dst, stackSlotF src)
 5000 %{
 5001   match(Set dst src);
 5002 
 5003   ins_cost(125);
 5004   format %{ "movss   $dst, $src\t# float stk" %}
 5005   ins_encode %{
 5006     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5007   %}
 5008   ins_pipe(pipe_slow); // XXX
 5009 %}
 5010 
 5011 // Use the same format since predicate() can not be used here.
 5012 instruct loadSSD(regD dst, stackSlotD src)
 5013 %{
 5014   match(Set dst src);
 5015 
 5016   ins_cost(125);
 5017   format %{ "movsd   $dst, $src\t# double stk" %}
 5018   ins_encode  %{
 5019     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5020   %}
 5021   ins_pipe(pipe_slow); // XXX
 5022 %}
 5023 
 5024 // Prefetch instructions for allocation.
 5025 // Must be safe to execute with invalid address (cannot fault).
 5026 
 5027 instruct prefetchAlloc( memory mem ) %{
 5028   predicate(AllocatePrefetchInstr==3);
 5029   match(PrefetchAllocation mem);
 5030   ins_cost(125);
 5031 
 5032   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5033   ins_encode %{
 5034     __ prefetchw($mem$$Address);
 5035   %}
 5036   ins_pipe(ialu_mem);
 5037 %}
 5038 
 5039 instruct prefetchAllocNTA( memory mem ) %{
 5040   predicate(AllocatePrefetchInstr==0);
 5041   match(PrefetchAllocation mem);
 5042   ins_cost(125);
 5043 
 5044   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5045   ins_encode %{
 5046     __ prefetchnta($mem$$Address);
 5047   %}
 5048   ins_pipe(ialu_mem);
 5049 %}
 5050 
 5051 instruct prefetchAllocT0( memory mem ) %{
 5052   predicate(AllocatePrefetchInstr==1);
 5053   match(PrefetchAllocation mem);
 5054   ins_cost(125);
 5055 
 5056   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5057   ins_encode %{
 5058     __ prefetcht0($mem$$Address);
 5059   %}
 5060   ins_pipe(ialu_mem);
 5061 %}
 5062 
 5063 instruct prefetchAllocT2( memory mem ) %{
 5064   predicate(AllocatePrefetchInstr==2);
 5065   match(PrefetchAllocation mem);
 5066   ins_cost(125);
 5067 
 5068   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5069   ins_encode %{
 5070     __ prefetcht2($mem$$Address);
 5071   %}
 5072   ins_pipe(ialu_mem);
 5073 %}
 5074 
 5075 //----------Store Instructions-------------------------------------------------
 5076 
 5077 // Store Byte
 5078 instruct storeB(memory mem, rRegI src)
 5079 %{
 5080   match(Set mem (StoreB mem src));
 5081 
 5082   ins_cost(125); // XXX
 5083   format %{ "movb    $mem, $src\t# byte" %}
 5084   ins_encode %{
 5085     __ movb($mem$$Address, $src$$Register);
 5086   %}
 5087   ins_pipe(ialu_mem_reg);
 5088 %}
 5089 
 5090 // Store Char/Short
 5091 instruct storeC(memory mem, rRegI src)
 5092 %{
 5093   match(Set mem (StoreC mem src));
 5094 
 5095   ins_cost(125); // XXX
 5096   format %{ "movw    $mem, $src\t# char/short" %}
 5097   ins_encode %{
 5098     __ movw($mem$$Address, $src$$Register);
 5099   %}
 5100   ins_pipe(ialu_mem_reg);
 5101 %}
 5102 
 5103 // Store Integer
 5104 instruct storeI(memory mem, rRegI src)
 5105 %{
 5106   match(Set mem (StoreI mem src));
 5107 
 5108   ins_cost(125); // XXX
 5109   format %{ "movl    $mem, $src\t# int" %}
 5110   ins_encode %{
 5111     __ movl($mem$$Address, $src$$Register);
 5112   %}
 5113   ins_pipe(ialu_mem_reg);
 5114 %}
 5115 
 5116 // Store Long
 5117 instruct storeL(memory mem, rRegL src)
 5118 %{
 5119   match(Set mem (StoreL mem src));
 5120 
 5121   ins_cost(125); // XXX
 5122   format %{ "movq    $mem, $src\t# long" %}
 5123   ins_encode %{
 5124     __ movq($mem$$Address, $src$$Register);
 5125   %}
 5126   ins_pipe(ialu_mem_reg); // XXX
 5127 %}
 5128 
 5129 // Store Pointer
 5130 instruct storeP(memory mem, any_RegP src)
 5131 %{
 5132   predicate(n->as_Store()->barrier_data() == 0);
 5133   match(Set mem (StoreP mem src));
 5134 
 5135   ins_cost(125); // XXX
 5136   format %{ "movq    $mem, $src\t# ptr" %}
 5137   ins_encode %{
 5138     __ movq($mem$$Address, $src$$Register);
 5139   %}
 5140   ins_pipe(ialu_mem_reg);
 5141 %}
 5142 
 5143 instruct storeImmP0(memory mem, immP0 zero)
 5144 %{
 5145   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5146   match(Set mem (StoreP mem zero));
 5147 
 5148   ins_cost(125); // XXX
 5149   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5150   ins_encode %{
 5151     __ movq($mem$$Address, r12);
 5152   %}
 5153   ins_pipe(ialu_mem_reg);
 5154 %}
 5155 
 5156 // Store Null Pointer, mark word, or other simple pointer constant.
 5157 instruct storeImmP(memory mem, immP31 src)
 5158 %{
 5159   predicate(n->as_Store()->barrier_data() == 0);
 5160   match(Set mem (StoreP mem src));
 5161 
 5162   ins_cost(150); // XXX
 5163   format %{ "movq    $mem, $src\t# ptr" %}
 5164   ins_encode %{
 5165     __ movq($mem$$Address, $src$$constant);
 5166   %}
 5167   ins_pipe(ialu_mem_imm);
 5168 %}
 5169 
 5170 // Store Compressed Pointer
 5171 instruct storeN(memory mem, rRegN src)
 5172 %{
 5173   match(Set mem (StoreN mem src));
 5174 
 5175   ins_cost(125); // XXX
 5176   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5177   ins_encode %{
 5178     __ movl($mem$$Address, $src$$Register);
 5179   %}
 5180   ins_pipe(ialu_mem_reg);
 5181 %}
 5182 
 5183 instruct storeNKlass(memory mem, rRegN src)
 5184 %{
 5185   match(Set mem (StoreNKlass mem src));
 5186 
 5187   ins_cost(125); // XXX
 5188   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5189   ins_encode %{
 5190     __ movl($mem$$Address, $src$$Register);
 5191   %}
 5192   ins_pipe(ialu_mem_reg);
 5193 %}
 5194 
 5195 instruct storeImmN0(memory mem, immN0 zero)
 5196 %{
 5197   predicate(CompressedOops::base() == nullptr);
 5198   match(Set mem (StoreN mem zero));
 5199 
 5200   ins_cost(125); // XXX
 5201   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5202   ins_encode %{
 5203     __ movl($mem$$Address, r12);
 5204   %}
 5205   ins_pipe(ialu_mem_reg);
 5206 %}
 5207 
 5208 instruct storeImmN(memory mem, immN src)
 5209 %{
 5210   match(Set mem (StoreN mem src));
 5211 
 5212   ins_cost(150); // XXX
 5213   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5214   ins_encode %{
 5215     address con = (address)$src$$constant;
 5216     if (con == nullptr) {
 5217       __ movl($mem$$Address, 0);
 5218     } else {
 5219       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5220     }
 5221   %}
 5222   ins_pipe(ialu_mem_imm);
 5223 %}
 5224 
 5225 instruct storeImmNKlass(memory mem, immNKlass src)
 5226 %{
 5227   match(Set mem (StoreNKlass mem src));
 5228 
 5229   ins_cost(150); // XXX
 5230   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5231   ins_encode %{
 5232     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5233   %}
 5234   ins_pipe(ialu_mem_imm);
 5235 %}
 5236 
 5237 // Store Integer Immediate
 5238 instruct storeImmI0(memory mem, immI_0 zero)
 5239 %{
 5240   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5241   match(Set mem (StoreI mem zero));
 5242 
 5243   ins_cost(125); // XXX
 5244   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5245   ins_encode %{
 5246     __ movl($mem$$Address, r12);
 5247   %}
 5248   ins_pipe(ialu_mem_reg);
 5249 %}
 5250 
 5251 instruct storeImmI(memory mem, immI src)
 5252 %{
 5253   match(Set mem (StoreI mem src));
 5254 
 5255   ins_cost(150);
 5256   format %{ "movl    $mem, $src\t# int" %}
 5257   ins_encode %{
 5258     __ movl($mem$$Address, $src$$constant);
 5259   %}
 5260   ins_pipe(ialu_mem_imm);
 5261 %}
 5262 
 5263 // Store Long Immediate
 5264 instruct storeImmL0(memory mem, immL0 zero)
 5265 %{
 5266   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5267   match(Set mem (StoreL mem zero));
 5268 
 5269   ins_cost(125); // XXX
 5270   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5271   ins_encode %{
 5272     __ movq($mem$$Address, r12);
 5273   %}
 5274   ins_pipe(ialu_mem_reg);
 5275 %}
 5276 
 5277 instruct storeImmL(memory mem, immL32 src)
 5278 %{
 5279   match(Set mem (StoreL mem src));
 5280 
 5281   ins_cost(150);
 5282   format %{ "movq    $mem, $src\t# long" %}
 5283   ins_encode %{
 5284     __ movq($mem$$Address, $src$$constant);
 5285   %}
 5286   ins_pipe(ialu_mem_imm);
 5287 %}
 5288 
 5289 // Store Short/Char Immediate
 5290 instruct storeImmC0(memory mem, immI_0 zero)
 5291 %{
 5292   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5293   match(Set mem (StoreC mem zero));
 5294 
 5295   ins_cost(125); // XXX
 5296   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5297   ins_encode %{
 5298     __ movw($mem$$Address, r12);
 5299   %}
 5300   ins_pipe(ialu_mem_reg);
 5301 %}
 5302 
 5303 instruct storeImmI16(memory mem, immI16 src)
 5304 %{
 5305   predicate(UseStoreImmI16);
 5306   match(Set mem (StoreC mem src));
 5307 
 5308   ins_cost(150);
 5309   format %{ "movw    $mem, $src\t# short/char" %}
 5310   ins_encode %{
 5311     __ movw($mem$$Address, $src$$constant);
 5312   %}
 5313   ins_pipe(ialu_mem_imm);
 5314 %}
 5315 
 5316 // Store Byte Immediate
 5317 instruct storeImmB0(memory mem, immI_0 zero)
 5318 %{
 5319   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5320   match(Set mem (StoreB mem zero));
 5321 
 5322   ins_cost(125); // XXX
 5323   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5324   ins_encode %{
 5325     __ movb($mem$$Address, r12);
 5326   %}
 5327   ins_pipe(ialu_mem_reg);
 5328 %}
 5329 
 5330 instruct storeImmB(memory mem, immI8 src)
 5331 %{
 5332   match(Set mem (StoreB mem src));
 5333 
 5334   ins_cost(150); // XXX
 5335   format %{ "movb    $mem, $src\t# byte" %}
 5336   ins_encode %{
 5337     __ movb($mem$$Address, $src$$constant);
 5338   %}
 5339   ins_pipe(ialu_mem_imm);
 5340 %}
 5341 
 5342 // Store CMS card-mark Immediate
 5343 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 5344 %{
 5345   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5346   match(Set mem (StoreCM mem zero));
 5347 
 5348   ins_cost(125); // XXX
 5349   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 5350   ins_encode %{
 5351     __ movb($mem$$Address, r12);
 5352   %}
 5353   ins_pipe(ialu_mem_reg);
 5354 %}
 5355 
 5356 instruct storeImmCM0(memory mem, immI_0 src)
 5357 %{
 5358   match(Set mem (StoreCM mem src));
 5359 
 5360   ins_cost(150); // XXX
 5361   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 5362   ins_encode %{
 5363     __ movb($mem$$Address, $src$$constant);
 5364   %}
 5365   ins_pipe(ialu_mem_imm);
 5366 %}
 5367 
 5368 // Store Float
 5369 instruct storeF(memory mem, regF src)
 5370 %{
 5371   match(Set mem (StoreF mem src));
 5372 
 5373   ins_cost(95); // XXX
 5374   format %{ "movss   $mem, $src\t# float" %}
 5375   ins_encode %{
 5376     __ movflt($mem$$Address, $src$$XMMRegister);
 5377   %}
 5378   ins_pipe(pipe_slow); // XXX
 5379 %}
 5380 
 5381 // Store immediate Float value (it is faster than store from XMM register)
 5382 instruct storeF0(memory mem, immF0 zero)
 5383 %{
 5384   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5385   match(Set mem (StoreF mem zero));
 5386 
 5387   ins_cost(25); // XXX
 5388   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5389   ins_encode %{
 5390     __ movl($mem$$Address, r12);
 5391   %}
 5392   ins_pipe(ialu_mem_reg);
 5393 %}
 5394 
 5395 instruct storeF_imm(memory mem, immF src)
 5396 %{
 5397   match(Set mem (StoreF mem src));
 5398 
 5399   ins_cost(50);
 5400   format %{ "movl    $mem, $src\t# float" %}
 5401   ins_encode %{
 5402     __ movl($mem$$Address, jint_cast($src$$constant));
 5403   %}
 5404   ins_pipe(ialu_mem_imm);
 5405 %}
 5406 
 5407 // Store Double
 5408 instruct storeD(memory mem, regD src)
 5409 %{
 5410   match(Set mem (StoreD mem src));
 5411 
 5412   ins_cost(95); // XXX
 5413   format %{ "movsd   $mem, $src\t# double" %}
 5414   ins_encode %{
 5415     __ movdbl($mem$$Address, $src$$XMMRegister);
 5416   %}
 5417   ins_pipe(pipe_slow); // XXX
 5418 %}
 5419 
 5420 // Store immediate double 0.0 (it is faster than store from XMM register)
 5421 instruct storeD0_imm(memory mem, immD0 src)
 5422 %{
 5423   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5424   match(Set mem (StoreD mem src));
 5425 
 5426   ins_cost(50);
 5427   format %{ "movq    $mem, $src\t# double 0." %}
 5428   ins_encode %{
 5429     __ movq($mem$$Address, $src$$constant);
 5430   %}
 5431   ins_pipe(ialu_mem_imm);
 5432 %}
 5433 
 5434 instruct storeD0(memory mem, immD0 zero)
 5435 %{
 5436   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5437   match(Set mem (StoreD mem zero));
 5438 
 5439   ins_cost(25); // XXX
 5440   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5441   ins_encode %{
 5442     __ movq($mem$$Address, r12);
 5443   %}
 5444   ins_pipe(ialu_mem_reg);
 5445 %}
 5446 
 5447 instruct storeSSI(stackSlotI dst, rRegI src)
 5448 %{
 5449   match(Set dst src);
 5450 
 5451   ins_cost(100);
 5452   format %{ "movl    $dst, $src\t# int stk" %}
 5453   ins_encode %{
 5454     __ movl($dst$$Address, $src$$Register);
 5455   %}
 5456   ins_pipe( ialu_mem_reg );
 5457 %}
 5458 
 5459 instruct storeSSL(stackSlotL dst, rRegL src)
 5460 %{
 5461   match(Set dst src);
 5462 
 5463   ins_cost(100);
 5464   format %{ "movq    $dst, $src\t# long stk" %}
 5465   ins_encode %{
 5466     __ movq($dst$$Address, $src$$Register);
 5467   %}
 5468   ins_pipe(ialu_mem_reg);
 5469 %}
 5470 
 5471 instruct storeSSP(stackSlotP dst, rRegP src)
 5472 %{
 5473   match(Set dst src);
 5474 
 5475   ins_cost(100);
 5476   format %{ "movq    $dst, $src\t# ptr stk" %}
 5477   ins_encode %{
 5478     __ movq($dst$$Address, $src$$Register);
 5479   %}
 5480   ins_pipe(ialu_mem_reg);
 5481 %}
 5482 
 5483 instruct storeSSF(stackSlotF dst, regF src)
 5484 %{
 5485   match(Set dst src);
 5486 
 5487   ins_cost(95); // XXX
 5488   format %{ "movss   $dst, $src\t# float stk" %}
 5489   ins_encode %{
 5490     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5491   %}
 5492   ins_pipe(pipe_slow); // XXX
 5493 %}
 5494 
 5495 instruct storeSSD(stackSlotD dst, regD src)
 5496 %{
 5497   match(Set dst src);
 5498 
 5499   ins_cost(95); // XXX
 5500   format %{ "movsd   $dst, $src\t# double stk" %}
 5501   ins_encode %{
 5502     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5503   %}
 5504   ins_pipe(pipe_slow); // XXX
 5505 %}
 5506 
 5507 instruct cacheWB(indirect addr)
 5508 %{
 5509   predicate(VM_Version::supports_data_cache_line_flush());
 5510   match(CacheWB addr);
 5511 
 5512   ins_cost(100);
 5513   format %{"cache wb $addr" %}
 5514   ins_encode %{
 5515     assert($addr->index_position() < 0, "should be");
 5516     assert($addr$$disp == 0, "should be");
 5517     __ cache_wb(Address($addr$$base$$Register, 0));
 5518   %}
 5519   ins_pipe(pipe_slow); // XXX
 5520 %}
 5521 
 5522 instruct cacheWBPreSync()
 5523 %{
 5524   predicate(VM_Version::supports_data_cache_line_flush());
 5525   match(CacheWBPreSync);
 5526 
 5527   ins_cost(100);
 5528   format %{"cache wb presync" %}
 5529   ins_encode %{
 5530     __ cache_wbsync(true);
 5531   %}
 5532   ins_pipe(pipe_slow); // XXX
 5533 %}
 5534 
 5535 instruct cacheWBPostSync()
 5536 %{
 5537   predicate(VM_Version::supports_data_cache_line_flush());
 5538   match(CacheWBPostSync);
 5539 
 5540   ins_cost(100);
 5541   format %{"cache wb postsync" %}
 5542   ins_encode %{
 5543     __ cache_wbsync(false);
 5544   %}
 5545   ins_pipe(pipe_slow); // XXX
 5546 %}
 5547 
 5548 //----------BSWAP Instructions-------------------------------------------------
 5549 instruct bytes_reverse_int(rRegI dst) %{
 5550   match(Set dst (ReverseBytesI dst));
 5551 
 5552   format %{ "bswapl  $dst" %}
 5553   ins_encode %{
 5554     __ bswapl($dst$$Register);
 5555   %}
 5556   ins_pipe( ialu_reg );
 5557 %}
 5558 
 5559 instruct bytes_reverse_long(rRegL dst) %{
 5560   match(Set dst (ReverseBytesL dst));
 5561 
 5562   format %{ "bswapq  $dst" %}
 5563   ins_encode %{
 5564     __ bswapq($dst$$Register);
 5565   %}
 5566   ins_pipe( ialu_reg);
 5567 %}
 5568 
 5569 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5570   match(Set dst (ReverseBytesUS dst));
 5571   effect(KILL cr);
 5572 
 5573   format %{ "bswapl  $dst\n\t"
 5574             "shrl    $dst,16\n\t" %}
 5575   ins_encode %{
 5576     __ bswapl($dst$$Register);
 5577     __ shrl($dst$$Register, 16);
 5578   %}
 5579   ins_pipe( ialu_reg );
 5580 %}
 5581 
 5582 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5583   match(Set dst (ReverseBytesS dst));
 5584   effect(KILL cr);
 5585 
 5586   format %{ "bswapl  $dst\n\t"
 5587             "sar     $dst,16\n\t" %}
 5588   ins_encode %{
 5589     __ bswapl($dst$$Register);
 5590     __ sarl($dst$$Register, 16);
 5591   %}
 5592   ins_pipe( ialu_reg );
 5593 %}
 5594 
 5595 //---------- Zeros Count Instructions ------------------------------------------
 5596 
 5597 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5598   predicate(UseCountLeadingZerosInstruction);
 5599   match(Set dst (CountLeadingZerosI src));
 5600   effect(KILL cr);
 5601 
 5602   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5603   ins_encode %{
 5604     __ lzcntl($dst$$Register, $src$$Register);
 5605   %}
 5606   ins_pipe(ialu_reg);
 5607 %}
 5608 
 5609 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5610   predicate(UseCountLeadingZerosInstruction);
 5611   match(Set dst (CountLeadingZerosI (LoadI src)));
 5612   effect(KILL cr);
 5613   ins_cost(175);
 5614   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5615   ins_encode %{
 5616     __ lzcntl($dst$$Register, $src$$Address);
 5617   %}
 5618   ins_pipe(ialu_reg_mem);
 5619 %}
 5620 
 5621 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5622   predicate(!UseCountLeadingZerosInstruction);
 5623   match(Set dst (CountLeadingZerosI src));
 5624   effect(KILL cr);
 5625 
 5626   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5627             "jnz     skip\n\t"
 5628             "movl    $dst, -1\n"
 5629       "skip:\n\t"
 5630             "negl    $dst\n\t"
 5631             "addl    $dst, 31" %}
 5632   ins_encode %{
 5633     Register Rdst = $dst$$Register;
 5634     Register Rsrc = $src$$Register;
 5635     Label skip;
 5636     __ bsrl(Rdst, Rsrc);
 5637     __ jccb(Assembler::notZero, skip);
 5638     __ movl(Rdst, -1);
 5639     __ bind(skip);
 5640     __ negl(Rdst);
 5641     __ addl(Rdst, BitsPerInt - 1);
 5642   %}
 5643   ins_pipe(ialu_reg);
 5644 %}
 5645 
 5646 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5647   predicate(UseCountLeadingZerosInstruction);
 5648   match(Set dst (CountLeadingZerosL src));
 5649   effect(KILL cr);
 5650 
 5651   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5652   ins_encode %{
 5653     __ lzcntq($dst$$Register, $src$$Register);
 5654   %}
 5655   ins_pipe(ialu_reg);
 5656 %}
 5657 
 5658 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5659   predicate(UseCountLeadingZerosInstruction);
 5660   match(Set dst (CountLeadingZerosL (LoadL src)));
 5661   effect(KILL cr);
 5662   ins_cost(175);
 5663   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5664   ins_encode %{
 5665     __ lzcntq($dst$$Register, $src$$Address);
 5666   %}
 5667   ins_pipe(ialu_reg_mem);
 5668 %}
 5669 
 5670 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5671   predicate(!UseCountLeadingZerosInstruction);
 5672   match(Set dst (CountLeadingZerosL src));
 5673   effect(KILL cr);
 5674 
 5675   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5676             "jnz     skip\n\t"
 5677             "movl    $dst, -1\n"
 5678       "skip:\n\t"
 5679             "negl    $dst\n\t"
 5680             "addl    $dst, 63" %}
 5681   ins_encode %{
 5682     Register Rdst = $dst$$Register;
 5683     Register Rsrc = $src$$Register;
 5684     Label skip;
 5685     __ bsrq(Rdst, Rsrc);
 5686     __ jccb(Assembler::notZero, skip);
 5687     __ movl(Rdst, -1);
 5688     __ bind(skip);
 5689     __ negl(Rdst);
 5690     __ addl(Rdst, BitsPerLong - 1);
 5691   %}
 5692   ins_pipe(ialu_reg);
 5693 %}
 5694 
 5695 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5696   predicate(UseCountTrailingZerosInstruction);
 5697   match(Set dst (CountTrailingZerosI src));
 5698   effect(KILL cr);
 5699 
 5700   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5701   ins_encode %{
 5702     __ tzcntl($dst$$Register, $src$$Register);
 5703   %}
 5704   ins_pipe(ialu_reg);
 5705 %}
 5706 
 5707 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5708   predicate(UseCountTrailingZerosInstruction);
 5709   match(Set dst (CountTrailingZerosI (LoadI src)));
 5710   effect(KILL cr);
 5711   ins_cost(175);
 5712   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5713   ins_encode %{
 5714     __ tzcntl($dst$$Register, $src$$Address);
 5715   %}
 5716   ins_pipe(ialu_reg_mem);
 5717 %}
 5718 
 5719 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5720   predicate(!UseCountTrailingZerosInstruction);
 5721   match(Set dst (CountTrailingZerosI src));
 5722   effect(KILL cr);
 5723 
 5724   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5725             "jnz     done\n\t"
 5726             "movl    $dst, 32\n"
 5727       "done:" %}
 5728   ins_encode %{
 5729     Register Rdst = $dst$$Register;
 5730     Label done;
 5731     __ bsfl(Rdst, $src$$Register);
 5732     __ jccb(Assembler::notZero, done);
 5733     __ movl(Rdst, BitsPerInt);
 5734     __ bind(done);
 5735   %}
 5736   ins_pipe(ialu_reg);
 5737 %}
 5738 
 5739 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5740   predicate(UseCountTrailingZerosInstruction);
 5741   match(Set dst (CountTrailingZerosL src));
 5742   effect(KILL cr);
 5743 
 5744   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5745   ins_encode %{
 5746     __ tzcntq($dst$$Register, $src$$Register);
 5747   %}
 5748   ins_pipe(ialu_reg);
 5749 %}
 5750 
 5751 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5752   predicate(UseCountTrailingZerosInstruction);
 5753   match(Set dst (CountTrailingZerosL (LoadL src)));
 5754   effect(KILL cr);
 5755   ins_cost(175);
 5756   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5757   ins_encode %{
 5758     __ tzcntq($dst$$Register, $src$$Address);
 5759   %}
 5760   ins_pipe(ialu_reg_mem);
 5761 %}
 5762 
 5763 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5764   predicate(!UseCountTrailingZerosInstruction);
 5765   match(Set dst (CountTrailingZerosL src));
 5766   effect(KILL cr);
 5767 
 5768   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5769             "jnz     done\n\t"
 5770             "movl    $dst, 64\n"
 5771       "done:" %}
 5772   ins_encode %{
 5773     Register Rdst = $dst$$Register;
 5774     Label done;
 5775     __ bsfq(Rdst, $src$$Register);
 5776     __ jccb(Assembler::notZero, done);
 5777     __ movl(Rdst, BitsPerLong);
 5778     __ bind(done);
 5779   %}
 5780   ins_pipe(ialu_reg);
 5781 %}
 5782 
 5783 //--------------- Reverse Operation Instructions ----------------
 5784 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5785   predicate(!VM_Version::supports_gfni());
 5786   match(Set dst (ReverseI src));
 5787   effect(TEMP dst, TEMP rtmp, KILL cr);
 5788   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5789   ins_encode %{
 5790     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5791   %}
 5792   ins_pipe( ialu_reg );
 5793 %}
 5794 
 5795 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5796   predicate(VM_Version::supports_gfni());
 5797   match(Set dst (ReverseI src));
 5798   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5799   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5800   ins_encode %{
 5801     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5802   %}
 5803   ins_pipe( ialu_reg );
 5804 %}
 5805 
 5806 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5807   predicate(!VM_Version::supports_gfni());
 5808   match(Set dst (ReverseL src));
 5809   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5810   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5811   ins_encode %{
 5812     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5813   %}
 5814   ins_pipe( ialu_reg );
 5815 %}
 5816 
 5817 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5818   predicate(VM_Version::supports_gfni());
 5819   match(Set dst (ReverseL src));
 5820   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5821   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5822   ins_encode %{
 5823     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5824   %}
 5825   ins_pipe( ialu_reg );
 5826 %}
 5827 
 5828 //---------- Population Count Instructions -------------------------------------
 5829 
 5830 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5831   predicate(UsePopCountInstruction);
 5832   match(Set dst (PopCountI src));
 5833   effect(KILL cr);
 5834 
 5835   format %{ "popcnt  $dst, $src" %}
 5836   ins_encode %{
 5837     __ popcntl($dst$$Register, $src$$Register);
 5838   %}
 5839   ins_pipe(ialu_reg);
 5840 %}
 5841 
 5842 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5843   predicate(UsePopCountInstruction);
 5844   match(Set dst (PopCountI (LoadI mem)));
 5845   effect(KILL cr);
 5846 
 5847   format %{ "popcnt  $dst, $mem" %}
 5848   ins_encode %{
 5849     __ popcntl($dst$$Register, $mem$$Address);
 5850   %}
 5851   ins_pipe(ialu_reg);
 5852 %}
 5853 
 5854 // Note: Long.bitCount(long) returns an int.
 5855 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5856   predicate(UsePopCountInstruction);
 5857   match(Set dst (PopCountL src));
 5858   effect(KILL cr);
 5859 
 5860   format %{ "popcnt  $dst, $src" %}
 5861   ins_encode %{
 5862     __ popcntq($dst$$Register, $src$$Register);
 5863   %}
 5864   ins_pipe(ialu_reg);
 5865 %}
 5866 
 5867 // Note: Long.bitCount(long) returns an int.
 5868 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5869   predicate(UsePopCountInstruction);
 5870   match(Set dst (PopCountL (LoadL mem)));
 5871   effect(KILL cr);
 5872 
 5873   format %{ "popcnt  $dst, $mem" %}
 5874   ins_encode %{
 5875     __ popcntq($dst$$Register, $mem$$Address);
 5876   %}
 5877   ins_pipe(ialu_reg);
 5878 %}
 5879 
 5880 
 5881 //----------MemBar Instructions-----------------------------------------------
 5882 // Memory barrier flavors
 5883 
 5884 instruct membar_acquire()
 5885 %{
 5886   match(MemBarAcquire);
 5887   match(LoadFence);
 5888   ins_cost(0);
 5889 
 5890   size(0);
 5891   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5892   ins_encode();
 5893   ins_pipe(empty);
 5894 %}
 5895 
 5896 instruct membar_acquire_lock()
 5897 %{
 5898   match(MemBarAcquireLock);
 5899   ins_cost(0);
 5900 
 5901   size(0);
 5902   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5903   ins_encode();
 5904   ins_pipe(empty);
 5905 %}
 5906 
 5907 instruct membar_release()
 5908 %{
 5909   match(MemBarRelease);
 5910   match(StoreFence);
 5911   ins_cost(0);
 5912 
 5913   size(0);
 5914   format %{ "MEMBAR-release ! (empty encoding)" %}
 5915   ins_encode();
 5916   ins_pipe(empty);
 5917 %}
 5918 
 5919 instruct membar_release_lock()
 5920 %{
 5921   match(MemBarReleaseLock);
 5922   ins_cost(0);
 5923 
 5924   size(0);
 5925   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5926   ins_encode();
 5927   ins_pipe(empty);
 5928 %}
 5929 
 5930 instruct membar_volatile(rFlagsReg cr) %{
 5931   match(MemBarVolatile);
 5932   effect(KILL cr);
 5933   ins_cost(400);
 5934 
 5935   format %{
 5936     $$template
 5937     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5938   %}
 5939   ins_encode %{
 5940     __ membar(Assembler::StoreLoad);
 5941   %}
 5942   ins_pipe(pipe_slow);
 5943 %}
 5944 
 5945 instruct unnecessary_membar_volatile()
 5946 %{
 5947   match(MemBarVolatile);
 5948   predicate(Matcher::post_store_load_barrier(n));
 5949   ins_cost(0);
 5950 
 5951   size(0);
 5952   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5953   ins_encode();
 5954   ins_pipe(empty);
 5955 %}
 5956 
 5957 instruct membar_storestore() %{
 5958   match(MemBarStoreStore);
 5959   match(StoreStoreFence);
 5960   ins_cost(0);
 5961 
 5962   size(0);
 5963   format %{ "MEMBAR-storestore (empty encoding)" %}
 5964   ins_encode( );
 5965   ins_pipe(empty);
 5966 %}
 5967 
 5968 //----------Move Instructions--------------------------------------------------
 5969 
 5970 instruct castX2P(rRegP dst, rRegL src)
 5971 %{
 5972   match(Set dst (CastX2P src));
 5973 
 5974   format %{ "movq    $dst, $src\t# long->ptr" %}
 5975   ins_encode %{
 5976     if ($dst$$reg != $src$$reg) {
 5977       __ movptr($dst$$Register, $src$$Register);
 5978     }
 5979   %}
 5980   ins_pipe(ialu_reg_reg); // XXX
 5981 %}
 5982 
 5983 instruct castP2X(rRegL dst, rRegP src)
 5984 %{
 5985   match(Set dst (CastP2X src));
 5986 
 5987   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5988   ins_encode %{
 5989     if ($dst$$reg != $src$$reg) {
 5990       __ movptr($dst$$Register, $src$$Register);
 5991     }
 5992   %}
 5993   ins_pipe(ialu_reg_reg); // XXX
 5994 %}
 5995 
 5996 // Convert oop into int for vectors alignment masking
 5997 instruct convP2I(rRegI dst, rRegP src)
 5998 %{
 5999   match(Set dst (ConvL2I (CastP2X src)));
 6000 
 6001   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6002   ins_encode %{
 6003     __ movl($dst$$Register, $src$$Register);
 6004   %}
 6005   ins_pipe(ialu_reg_reg); // XXX
 6006 %}
 6007 
 6008 // Convert compressed oop into int for vectors alignment masking
 6009 // in case of 32bit oops (heap < 4Gb).
 6010 instruct convN2I(rRegI dst, rRegN src)
 6011 %{
 6012   predicate(CompressedOops::shift() == 0);
 6013   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6014 
 6015   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6016   ins_encode %{
 6017     __ movl($dst$$Register, $src$$Register);
 6018   %}
 6019   ins_pipe(ialu_reg_reg); // XXX
 6020 %}
 6021 
 6022 // Convert oop pointer into compressed form
 6023 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6024   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6025   match(Set dst (EncodeP src));
 6026   effect(KILL cr);
 6027   format %{ "encode_heap_oop $dst,$src" %}
 6028   ins_encode %{
 6029     Register s = $src$$Register;
 6030     Register d = $dst$$Register;
 6031     if (s != d) {
 6032       __ movq(d, s);
 6033     }
 6034     __ encode_heap_oop(d);
 6035   %}
 6036   ins_pipe(ialu_reg_long);
 6037 %}
 6038 
 6039 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6040   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6041   match(Set dst (EncodeP src));
 6042   effect(KILL cr);
 6043   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6044   ins_encode %{
 6045     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6046   %}
 6047   ins_pipe(ialu_reg_long);
 6048 %}
 6049 
 6050 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6051   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6052             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6053   match(Set dst (DecodeN src));
 6054   effect(KILL cr);
 6055   format %{ "decode_heap_oop $dst,$src" %}
 6056   ins_encode %{
 6057     Register s = $src$$Register;
 6058     Register d = $dst$$Register;
 6059     if (s != d) {
 6060       __ movq(d, s);
 6061     }
 6062     __ decode_heap_oop(d);
 6063   %}
 6064   ins_pipe(ialu_reg_long);
 6065 %}
 6066 
 6067 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6068   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6069             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6070   match(Set dst (DecodeN src));
 6071   effect(KILL cr);
 6072   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6073   ins_encode %{
 6074     Register s = $src$$Register;
 6075     Register d = $dst$$Register;
 6076     if (s != d) {
 6077       __ decode_heap_oop_not_null(d, s);
 6078     } else {
 6079       __ decode_heap_oop_not_null(d);
 6080     }
 6081   %}
 6082   ins_pipe(ialu_reg_long);
 6083 %}
 6084 
 6085 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6086   match(Set dst (EncodePKlass src));
 6087   effect(TEMP dst, KILL cr);
 6088   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6089   ins_encode %{
 6090     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6091   %}
 6092   ins_pipe(ialu_reg_long);
 6093 %}
 6094 
 6095 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6096   match(Set dst (DecodeNKlass src));
 6097   effect(TEMP dst, KILL cr);
 6098   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6099   ins_encode %{
 6100     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6101   %}
 6102   ins_pipe(ialu_reg_long);
 6103 %}
 6104 
 6105 //----------Conditional Move---------------------------------------------------
 6106 // Jump
 6107 // dummy instruction for generating temp registers
 6108 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6109   match(Jump (LShiftL switch_val shift));
 6110   ins_cost(350);
 6111   predicate(false);
 6112   effect(TEMP dest);
 6113 
 6114   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6115             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6116   ins_encode %{
 6117     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6118     // to do that and the compiler is using that register as one it can allocate.
 6119     // So we build it all by hand.
 6120     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6121     // ArrayAddress dispatch(table, index);
 6122     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6123     __ lea($dest$$Register, $constantaddress);
 6124     __ jmp(dispatch);
 6125   %}
 6126   ins_pipe(pipe_jmp);
 6127 %}
 6128 
 6129 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6130   match(Jump (AddL (LShiftL switch_val shift) offset));
 6131   ins_cost(350);
 6132   effect(TEMP dest);
 6133 
 6134   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6135             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6136   ins_encode %{
 6137     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6138     // to do that and the compiler is using that register as one it can allocate.
 6139     // So we build it all by hand.
 6140     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6141     // ArrayAddress dispatch(table, index);
 6142     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6143     __ lea($dest$$Register, $constantaddress);
 6144     __ jmp(dispatch);
 6145   %}
 6146   ins_pipe(pipe_jmp);
 6147 %}
 6148 
 6149 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6150   match(Jump switch_val);
 6151   ins_cost(350);
 6152   effect(TEMP dest);
 6153 
 6154   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6155             "jmp     [$dest + $switch_val]\n\t" %}
 6156   ins_encode %{
 6157     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6158     // to do that and the compiler is using that register as one it can allocate.
 6159     // So we build it all by hand.
 6160     // Address index(noreg, switch_reg, Address::times_1);
 6161     // ArrayAddress dispatch(table, index);
 6162     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6163     __ lea($dest$$Register, $constantaddress);
 6164     __ jmp(dispatch);
 6165   %}
 6166   ins_pipe(pipe_jmp);
 6167 %}
 6168 
 6169 // Conditional move
 6170 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6171 %{
 6172   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6173   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6174 
 6175   ins_cost(100); // XXX
 6176   format %{ "setbn$cop $dst\t# signed, int" %}
 6177   ins_encode %{
 6178     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6179     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6180   %}
 6181   ins_pipe(ialu_reg);
 6182 %}
 6183 
 6184 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6185 %{
 6186   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6187 
 6188   ins_cost(200); // XXX
 6189   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6190   ins_encode %{
 6191     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6192   %}
 6193   ins_pipe(pipe_cmov_reg);
 6194 %}
 6195 
 6196 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6197 %{
 6198   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6199   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6200 
 6201   ins_cost(100); // XXX
 6202   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6203   ins_encode %{
 6204     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6205     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6206   %}
 6207   ins_pipe(ialu_reg);
 6208 %}
 6209 
 6210 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6211   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6212 
 6213   ins_cost(200); // XXX
 6214   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6215   ins_encode %{
 6216     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6217   %}
 6218   ins_pipe(pipe_cmov_reg);
 6219 %}
 6220 
 6221 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6222 %{
 6223   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6224   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6225 
 6226   ins_cost(100); // XXX
 6227   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6228   ins_encode %{
 6229     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6230     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6231   %}
 6232   ins_pipe(ialu_reg);
 6233 %}
 6234 
 6235 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6236   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6237   ins_cost(200);
 6238   expand %{
 6239     cmovI_regU(cop, cr, dst, src);
 6240   %}
 6241 %}
 6242 
 6243 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6244   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6245   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6246 
 6247   ins_cost(200); // XXX
 6248   format %{ "cmovpl  $dst, $src\n\t"
 6249             "cmovnel $dst, $src" %}
 6250   ins_encode %{
 6251     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6252     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6253   %}
 6254   ins_pipe(pipe_cmov_reg);
 6255 %}
 6256 
 6257 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6258 // inputs of the CMove
 6259 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6260   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6261   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6262 
 6263   ins_cost(200); // XXX
 6264   format %{ "cmovpl  $dst, $src\n\t"
 6265             "cmovnel $dst, $src" %}
 6266   ins_encode %{
 6267     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6268     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6269   %}
 6270   ins_pipe(pipe_cmov_reg);
 6271 %}
 6272 
 6273 // Conditional move
 6274 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6275   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6276 
 6277   ins_cost(250); // XXX
 6278   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6279   ins_encode %{
 6280     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6281   %}
 6282   ins_pipe(pipe_cmov_mem);
 6283 %}
 6284 
 6285 // Conditional move
 6286 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6287 %{
 6288   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6289 
 6290   ins_cost(250); // XXX
 6291   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6292   ins_encode %{
 6293     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6294   %}
 6295   ins_pipe(pipe_cmov_mem);
 6296 %}
 6297 
 6298 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6299   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6300   ins_cost(250);
 6301   expand %{
 6302     cmovI_memU(cop, cr, dst, src);
 6303   %}
 6304 %}
 6305 
 6306 // Conditional move
 6307 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6308 %{
 6309   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6310 
 6311   ins_cost(200); // XXX
 6312   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6313   ins_encode %{
 6314     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6315   %}
 6316   ins_pipe(pipe_cmov_reg);
 6317 %}
 6318 
 6319 // Conditional move
 6320 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6321 %{
 6322   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6323 
 6324   ins_cost(200); // XXX
 6325   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6326   ins_encode %{
 6327     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6328   %}
 6329   ins_pipe(pipe_cmov_reg);
 6330 %}
 6331 
 6332 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6333   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6334   ins_cost(200);
 6335   expand %{
 6336     cmovN_regU(cop, cr, dst, src);
 6337   %}
 6338 %}
 6339 
 6340 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6341   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6342   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6343 
 6344   ins_cost(200); // XXX
 6345   format %{ "cmovpl  $dst, $src\n\t"
 6346             "cmovnel $dst, $src" %}
 6347   ins_encode %{
 6348     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6349     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6350   %}
 6351   ins_pipe(pipe_cmov_reg);
 6352 %}
 6353 
 6354 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6355 // inputs of the CMove
 6356 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6357   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6358   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6359 
 6360   ins_cost(200); // XXX
 6361   format %{ "cmovpl  $dst, $src\n\t"
 6362             "cmovnel $dst, $src" %}
 6363   ins_encode %{
 6364     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6365     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6366   %}
 6367   ins_pipe(pipe_cmov_reg);
 6368 %}
 6369 
 6370 // Conditional move
 6371 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6372 %{
 6373   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6374 
 6375   ins_cost(200); // XXX
 6376   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6377   ins_encode %{
 6378     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6379   %}
 6380   ins_pipe(pipe_cmov_reg);  // XXX
 6381 %}
 6382 
 6383 // Conditional move
 6384 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6385 %{
 6386   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6387 
 6388   ins_cost(200); // XXX
 6389   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6390   ins_encode %{
 6391     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6392   %}
 6393   ins_pipe(pipe_cmov_reg); // XXX
 6394 %}
 6395 
 6396 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6397   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6398   ins_cost(200);
 6399   expand %{
 6400     cmovP_regU(cop, cr, dst, src);
 6401   %}
 6402 %}
 6403 
 6404 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6405   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6406   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6407 
 6408   ins_cost(200); // XXX
 6409   format %{ "cmovpq  $dst, $src\n\t"
 6410             "cmovneq $dst, $src" %}
 6411   ins_encode %{
 6412     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6413     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6414   %}
 6415   ins_pipe(pipe_cmov_reg);
 6416 %}
 6417 
 6418 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6419 // inputs of the CMove
 6420 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6421   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6422   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6423 
 6424   ins_cost(200); // XXX
 6425   format %{ "cmovpq  $dst, $src\n\t"
 6426             "cmovneq $dst, $src" %}
 6427   ins_encode %{
 6428     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6429     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6430   %}
 6431   ins_pipe(pipe_cmov_reg);
 6432 %}
 6433 
 6434 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6435 %{
 6436   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6437   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6438 
 6439   ins_cost(100); // XXX
 6440   format %{ "setbn$cop $dst\t# signed, long" %}
 6441   ins_encode %{
 6442     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6443     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6444   %}
 6445   ins_pipe(ialu_reg);
 6446 %}
 6447 
 6448 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6449 %{
 6450   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6451 
 6452   ins_cost(200); // XXX
 6453   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6454   ins_encode %{
 6455     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6456   %}
 6457   ins_pipe(pipe_cmov_reg);  // XXX
 6458 %}
 6459 
 6460 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6461 %{
 6462   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6463 
 6464   ins_cost(200); // XXX
 6465   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6466   ins_encode %{
 6467     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6468   %}
 6469   ins_pipe(pipe_cmov_mem);  // XXX
 6470 %}
 6471 
 6472 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6473 %{
 6474   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6475   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6476 
 6477   ins_cost(100); // XXX
 6478   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6479   ins_encode %{
 6480     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6481     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6482   %}
 6483   ins_pipe(ialu_reg);
 6484 %}
 6485 
 6486 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6487 %{
 6488   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6489 
 6490   ins_cost(200); // XXX
 6491   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6492   ins_encode %{
 6493     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6494   %}
 6495   ins_pipe(pipe_cmov_reg); // XXX
 6496 %}
 6497 
 6498 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6499 %{
 6500   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6501   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6502 
 6503   ins_cost(100); // XXX
 6504   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6505   ins_encode %{
 6506     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6507     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6508   %}
 6509   ins_pipe(ialu_reg);
 6510 %}
 6511 
 6512 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6513   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6514   ins_cost(200);
 6515   expand %{
 6516     cmovL_regU(cop, cr, dst, src);
 6517   %}
 6518 %}
 6519 
 6520 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6521   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6522   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6523 
 6524   ins_cost(200); // XXX
 6525   format %{ "cmovpq  $dst, $src\n\t"
 6526             "cmovneq $dst, $src" %}
 6527   ins_encode %{
 6528     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6529     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6530   %}
 6531   ins_pipe(pipe_cmov_reg);
 6532 %}
 6533 
 6534 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6535 // inputs of the CMove
 6536 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6537   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6538   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6539 
 6540   ins_cost(200); // XXX
 6541   format %{ "cmovpq  $dst, $src\n\t"
 6542             "cmovneq $dst, $src" %}
 6543   ins_encode %{
 6544     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6545     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6546   %}
 6547   ins_pipe(pipe_cmov_reg);
 6548 %}
 6549 
 6550 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6551 %{
 6552   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6553 
 6554   ins_cost(200); // XXX
 6555   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6556   ins_encode %{
 6557     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6558   %}
 6559   ins_pipe(pipe_cmov_mem); // XXX
 6560 %}
 6561 
 6562 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6563   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6564   ins_cost(200);
 6565   expand %{
 6566     cmovL_memU(cop, cr, dst, src);
 6567   %}
 6568 %}
 6569 
 6570 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6571 %{
 6572   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6573 
 6574   ins_cost(200); // XXX
 6575   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6576             "movss     $dst, $src\n"
 6577     "skip:" %}
 6578   ins_encode %{
 6579     Label Lskip;
 6580     // Invert sense of branch from sense of CMOV
 6581     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6582     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6583     __ bind(Lskip);
 6584   %}
 6585   ins_pipe(pipe_slow);
 6586 %}
 6587 
 6588 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6589 %{
 6590   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6591 
 6592   ins_cost(200); // XXX
 6593   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6594             "movss     $dst, $src\n"
 6595     "skip:" %}
 6596   ins_encode %{
 6597     Label Lskip;
 6598     // Invert sense of branch from sense of CMOV
 6599     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6600     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6601     __ bind(Lskip);
 6602   %}
 6603   ins_pipe(pipe_slow);
 6604 %}
 6605 
 6606 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6607   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6608   ins_cost(200);
 6609   expand %{
 6610     cmovF_regU(cop, cr, dst, src);
 6611   %}
 6612 %}
 6613 
 6614 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6615 %{
 6616   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6617 
 6618   ins_cost(200); // XXX
 6619   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6620             "movsd     $dst, $src\n"
 6621     "skip:" %}
 6622   ins_encode %{
 6623     Label Lskip;
 6624     // Invert sense of branch from sense of CMOV
 6625     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6626     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6627     __ bind(Lskip);
 6628   %}
 6629   ins_pipe(pipe_slow);
 6630 %}
 6631 
 6632 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6633 %{
 6634   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6635 
 6636   ins_cost(200); // XXX
 6637   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6638             "movsd     $dst, $src\n"
 6639     "skip:" %}
 6640   ins_encode %{
 6641     Label Lskip;
 6642     // Invert sense of branch from sense of CMOV
 6643     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6644     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6645     __ bind(Lskip);
 6646   %}
 6647   ins_pipe(pipe_slow);
 6648 %}
 6649 
 6650 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6651   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6652   ins_cost(200);
 6653   expand %{
 6654     cmovD_regU(cop, cr, dst, src);
 6655   %}
 6656 %}
 6657 
 6658 //----------Arithmetic Instructions--------------------------------------------
 6659 //----------Addition Instructions----------------------------------------------
 6660 
 6661 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6662 %{
 6663   match(Set dst (AddI dst src));
 6664   effect(KILL cr);
 6665   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6666   format %{ "addl    $dst, $src\t# int" %}
 6667   ins_encode %{
 6668     __ addl($dst$$Register, $src$$Register);
 6669   %}
 6670   ins_pipe(ialu_reg_reg);
 6671 %}
 6672 
 6673 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 6674 %{
 6675   match(Set dst (AddI dst src));
 6676   effect(KILL cr);
 6677   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6678 
 6679   format %{ "addl    $dst, $src\t# int" %}
 6680   ins_encode %{
 6681     __ addl($dst$$Register, $src$$constant);
 6682   %}
 6683   ins_pipe( ialu_reg );
 6684 %}
 6685 
 6686 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 6687 %{
 6688   match(Set dst (AddI dst (LoadI src)));
 6689   effect(KILL cr);
 6690   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6691 
 6692   ins_cost(150); // XXX
 6693   format %{ "addl    $dst, $src\t# int" %}
 6694   ins_encode %{
 6695     __ addl($dst$$Register, $src$$Address);
 6696   %}
 6697   ins_pipe(ialu_reg_mem);
 6698 %}
 6699 
 6700 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 6701 %{
 6702   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6703   effect(KILL cr);
 6704   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6705 
 6706   ins_cost(150); // XXX
 6707   format %{ "addl    $dst, $src\t# int" %}
 6708   ins_encode %{
 6709     __ addl($dst$$Address, $src$$Register);
 6710   %}
 6711   ins_pipe(ialu_mem_reg);
 6712 %}
 6713 
 6714 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 6715 %{
 6716   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6717   effect(KILL cr);
 6718   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6719 
 6720 
 6721   ins_cost(125); // XXX
 6722   format %{ "addl    $dst, $src\t# int" %}
 6723   ins_encode %{
 6724     __ addl($dst$$Address, $src$$constant);
 6725   %}
 6726   ins_pipe(ialu_mem_imm);
 6727 %}
 6728 
 6729 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 6730 %{
 6731   predicate(UseIncDec);
 6732   match(Set dst (AddI dst src));
 6733   effect(KILL cr);
 6734 
 6735   format %{ "incl    $dst\t# int" %}
 6736   ins_encode %{
 6737     __ incrementl($dst$$Register);
 6738   %}
 6739   ins_pipe(ialu_reg);
 6740 %}
 6741 
 6742 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 6743 %{
 6744   predicate(UseIncDec);
 6745   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6746   effect(KILL cr);
 6747 
 6748   ins_cost(125); // XXX
 6749   format %{ "incl    $dst\t# int" %}
 6750   ins_encode %{
 6751     __ incrementl($dst$$Address);
 6752   %}
 6753   ins_pipe(ialu_mem_imm);
 6754 %}
 6755 
 6756 // XXX why does that use AddI
 6757 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 6758 %{
 6759   predicate(UseIncDec);
 6760   match(Set dst (AddI dst src));
 6761   effect(KILL cr);
 6762 
 6763   format %{ "decl    $dst\t# int" %}
 6764   ins_encode %{
 6765     __ decrementl($dst$$Register);
 6766   %}
 6767   ins_pipe(ialu_reg);
 6768 %}
 6769 
 6770 // XXX why does that use AddI
 6771 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 6772 %{
 6773   predicate(UseIncDec);
 6774   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6775   effect(KILL cr);
 6776 
 6777   ins_cost(125); // XXX
 6778   format %{ "decl    $dst\t# int" %}
 6779   ins_encode %{
 6780     __ decrementl($dst$$Address);
 6781   %}
 6782   ins_pipe(ialu_mem_imm);
 6783 %}
 6784 
 6785 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 6786 %{
 6787   predicate(VM_Version::supports_fast_2op_lea());
 6788   match(Set dst (AddI (LShiftI index scale) disp));
 6789 
 6790   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 6791   ins_encode %{
 6792     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6793     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6794   %}
 6795   ins_pipe(ialu_reg_reg);
 6796 %}
 6797 
 6798 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 6799 %{
 6800   predicate(VM_Version::supports_fast_3op_lea());
 6801   match(Set dst (AddI (AddI base index) disp));
 6802 
 6803   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 6804   ins_encode %{
 6805     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6806   %}
 6807   ins_pipe(ialu_reg_reg);
 6808 %}
 6809 
 6810 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 6811 %{
 6812   predicate(VM_Version::supports_fast_2op_lea());
 6813   match(Set dst (AddI base (LShiftI index scale)));
 6814 
 6815   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 6816   ins_encode %{
 6817     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6818     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6819   %}
 6820   ins_pipe(ialu_reg_reg);
 6821 %}
 6822 
 6823 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 6824 %{
 6825   predicate(VM_Version::supports_fast_3op_lea());
 6826   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 6827 
 6828   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 6829   ins_encode %{
 6830     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6831     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6832   %}
 6833   ins_pipe(ialu_reg_reg);
 6834 %}
 6835 
 6836 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 6837 %{
 6838   match(Set dst (AddL dst src));
 6839   effect(KILL cr);
 6840   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6841 
 6842   format %{ "addq    $dst, $src\t# long" %}
 6843   ins_encode %{
 6844     __ addq($dst$$Register, $src$$Register);
 6845   %}
 6846   ins_pipe(ialu_reg_reg);
 6847 %}
 6848 
 6849 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 6850 %{
 6851   match(Set dst (AddL dst src));
 6852   effect(KILL cr);
 6853   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6854 
 6855   format %{ "addq    $dst, $src\t# long" %}
 6856   ins_encode %{
 6857     __ addq($dst$$Register, $src$$constant);
 6858   %}
 6859   ins_pipe( ialu_reg );
 6860 %}
 6861 
 6862 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 6863 %{
 6864   match(Set dst (AddL dst (LoadL src)));
 6865   effect(KILL cr);
 6866   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6867 
 6868   ins_cost(150); // XXX
 6869   format %{ "addq    $dst, $src\t# long" %}
 6870   ins_encode %{
 6871     __ addq($dst$$Register, $src$$Address);
 6872   %}
 6873   ins_pipe(ialu_reg_mem);
 6874 %}
 6875 
 6876 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 6877 %{
 6878   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6879   effect(KILL cr);
 6880   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6881 
 6882   ins_cost(150); // XXX
 6883   format %{ "addq    $dst, $src\t# long" %}
 6884   ins_encode %{
 6885     __ addq($dst$$Address, $src$$Register);
 6886   %}
 6887   ins_pipe(ialu_mem_reg);
 6888 %}
 6889 
 6890 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 6891 %{
 6892   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6893   effect(KILL cr);
 6894   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6895 
 6896   ins_cost(125); // XXX
 6897   format %{ "addq    $dst, $src\t# long" %}
 6898   ins_encode %{
 6899     __ addq($dst$$Address, $src$$constant);
 6900   %}
 6901   ins_pipe(ialu_mem_imm);
 6902 %}
 6903 
 6904 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 6905 %{
 6906   predicate(UseIncDec);
 6907   match(Set dst (AddL dst src));
 6908   effect(KILL cr);
 6909 
 6910   format %{ "incq    $dst\t# long" %}
 6911   ins_encode %{
 6912     __ incrementq($dst$$Register);
 6913   %}
 6914   ins_pipe(ialu_reg);
 6915 %}
 6916 
 6917 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 6918 %{
 6919   predicate(UseIncDec);
 6920   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6921   effect(KILL cr);
 6922 
 6923   ins_cost(125); // XXX
 6924   format %{ "incq    $dst\t# long" %}
 6925   ins_encode %{
 6926     __ incrementq($dst$$Address);
 6927   %}
 6928   ins_pipe(ialu_mem_imm);
 6929 %}
 6930 
 6931 // XXX why does that use AddL
 6932 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 6933 %{
 6934   predicate(UseIncDec);
 6935   match(Set dst (AddL dst src));
 6936   effect(KILL cr);
 6937 
 6938   format %{ "decq    $dst\t# long" %}
 6939   ins_encode %{
 6940     __ decrementq($dst$$Register);
 6941   %}
 6942   ins_pipe(ialu_reg);
 6943 %}
 6944 
 6945 // XXX why does that use AddL
 6946 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 6947 %{
 6948   predicate(UseIncDec);
 6949   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6950   effect(KILL cr);
 6951 
 6952   ins_cost(125); // XXX
 6953   format %{ "decq    $dst\t# long" %}
 6954   ins_encode %{
 6955     __ decrementq($dst$$Address);
 6956   %}
 6957   ins_pipe(ialu_mem_imm);
 6958 %}
 6959 
 6960 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 6961 %{
 6962   predicate(VM_Version::supports_fast_2op_lea());
 6963   match(Set dst (AddL (LShiftL index scale) disp));
 6964 
 6965   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 6966   ins_encode %{
 6967     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6968     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6969   %}
 6970   ins_pipe(ialu_reg_reg);
 6971 %}
 6972 
 6973 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 6974 %{
 6975   predicate(VM_Version::supports_fast_3op_lea());
 6976   match(Set dst (AddL (AddL base index) disp));
 6977 
 6978   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 6979   ins_encode %{
 6980     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6981   %}
 6982   ins_pipe(ialu_reg_reg);
 6983 %}
 6984 
 6985 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 6986 %{
 6987   predicate(VM_Version::supports_fast_2op_lea());
 6988   match(Set dst (AddL base (LShiftL index scale)));
 6989 
 6990   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 6991   ins_encode %{
 6992     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6993     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6994   %}
 6995   ins_pipe(ialu_reg_reg);
 6996 %}
 6997 
 6998 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 6999 %{
 7000   predicate(VM_Version::supports_fast_3op_lea());
 7001   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7002 
 7003   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7004   ins_encode %{
 7005     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7006     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7007   %}
 7008   ins_pipe(ialu_reg_reg);
 7009 %}
 7010 
 7011 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7012 %{
 7013   match(Set dst (AddP dst src));
 7014   effect(KILL cr);
 7015   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7016 
 7017   format %{ "addq    $dst, $src\t# ptr" %}
 7018   ins_encode %{
 7019     __ addq($dst$$Register, $src$$Register);
 7020   %}
 7021   ins_pipe(ialu_reg_reg);
 7022 %}
 7023 
 7024 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7025 %{
 7026   match(Set dst (AddP dst src));
 7027   effect(KILL cr);
 7028   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7029 
 7030   format %{ "addq    $dst, $src\t# ptr" %}
 7031   ins_encode %{
 7032     __ addq($dst$$Register, $src$$constant);
 7033   %}
 7034   ins_pipe( ialu_reg );
 7035 %}
 7036 
 7037 // XXX addP mem ops ????
 7038 
 7039 instruct checkCastPP(rRegP dst)
 7040 %{
 7041   match(Set dst (CheckCastPP dst));
 7042 
 7043   size(0);
 7044   format %{ "# checkcastPP of $dst" %}
 7045   ins_encode(/* empty encoding */);
 7046   ins_pipe(empty);
 7047 %}
 7048 
 7049 instruct castPP(rRegP dst)
 7050 %{
 7051   match(Set dst (CastPP dst));
 7052 
 7053   size(0);
 7054   format %{ "# castPP of $dst" %}
 7055   ins_encode(/* empty encoding */);
 7056   ins_pipe(empty);
 7057 %}
 7058 
 7059 instruct castII(rRegI dst)
 7060 %{
 7061   match(Set dst (CastII dst));
 7062 
 7063   size(0);
 7064   format %{ "# castII of $dst" %}
 7065   ins_encode(/* empty encoding */);
 7066   ins_cost(0);
 7067   ins_pipe(empty);
 7068 %}
 7069 
 7070 instruct castLL(rRegL dst)
 7071 %{
 7072   match(Set dst (CastLL dst));
 7073 
 7074   size(0);
 7075   format %{ "# castLL of $dst" %}
 7076   ins_encode(/* empty encoding */);
 7077   ins_cost(0);
 7078   ins_pipe(empty);
 7079 %}
 7080 
 7081 instruct castFF(regF dst)
 7082 %{
 7083   match(Set dst (CastFF dst));
 7084 
 7085   size(0);
 7086   format %{ "# castFF of $dst" %}
 7087   ins_encode(/* empty encoding */);
 7088   ins_cost(0);
 7089   ins_pipe(empty);
 7090 %}
 7091 
 7092 instruct castDD(regD dst)
 7093 %{
 7094   match(Set dst (CastDD dst));
 7095 
 7096   size(0);
 7097   format %{ "# castDD of $dst" %}
 7098   ins_encode(/* empty encoding */);
 7099   ins_cost(0);
 7100   ins_pipe(empty);
 7101 %}
 7102 
 7103 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7104 instruct compareAndSwapP(rRegI res,
 7105                          memory mem_ptr,
 7106                          rax_RegP oldval, rRegP newval,
 7107                          rFlagsReg cr)
 7108 %{
 7109   predicate(n->as_LoadStore()->barrier_data() == 0);
 7110   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7111   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7112   effect(KILL cr, KILL oldval);
 7113 
 7114   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7115             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7116             "sete    $res\n\t"
 7117             "movzbl  $res, $res" %}
 7118   ins_encode %{
 7119     __ lock();
 7120     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7121     __ setb(Assembler::equal, $res$$Register);
 7122     __ movzbl($res$$Register, $res$$Register);
 7123   %}
 7124   ins_pipe( pipe_cmpxchg );
 7125 %}
 7126 
 7127 instruct compareAndSwapL(rRegI res,
 7128                          memory mem_ptr,
 7129                          rax_RegL oldval, rRegL newval,
 7130                          rFlagsReg cr)
 7131 %{
 7132   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7133   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7134   effect(KILL cr, KILL oldval);
 7135 
 7136   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7137             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7138             "sete    $res\n\t"
 7139             "movzbl  $res, $res" %}
 7140   ins_encode %{
 7141     __ lock();
 7142     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7143     __ setb(Assembler::equal, $res$$Register);
 7144     __ movzbl($res$$Register, $res$$Register);
 7145   %}
 7146   ins_pipe( pipe_cmpxchg );
 7147 %}
 7148 
 7149 instruct compareAndSwapI(rRegI res,
 7150                          memory mem_ptr,
 7151                          rax_RegI oldval, rRegI newval,
 7152                          rFlagsReg cr)
 7153 %{
 7154   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7155   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7156   effect(KILL cr, KILL oldval);
 7157 
 7158   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7159             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7160             "sete    $res\n\t"
 7161             "movzbl  $res, $res" %}
 7162   ins_encode %{
 7163     __ lock();
 7164     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7165     __ setb(Assembler::equal, $res$$Register);
 7166     __ movzbl($res$$Register, $res$$Register);
 7167   %}
 7168   ins_pipe( pipe_cmpxchg );
 7169 %}
 7170 
 7171 instruct compareAndSwapB(rRegI res,
 7172                          memory mem_ptr,
 7173                          rax_RegI oldval, rRegI newval,
 7174                          rFlagsReg cr)
 7175 %{
 7176   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7177   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7178   effect(KILL cr, KILL oldval);
 7179 
 7180   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7181             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7182             "sete    $res\n\t"
 7183             "movzbl  $res, $res" %}
 7184   ins_encode %{
 7185     __ lock();
 7186     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7187     __ setb(Assembler::equal, $res$$Register);
 7188     __ movzbl($res$$Register, $res$$Register);
 7189   %}
 7190   ins_pipe( pipe_cmpxchg );
 7191 %}
 7192 
 7193 instruct compareAndSwapS(rRegI res,
 7194                          memory mem_ptr,
 7195                          rax_RegI oldval, rRegI newval,
 7196                          rFlagsReg cr)
 7197 %{
 7198   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7199   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7200   effect(KILL cr, KILL oldval);
 7201 
 7202   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7203             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7204             "sete    $res\n\t"
 7205             "movzbl  $res, $res" %}
 7206   ins_encode %{
 7207     __ lock();
 7208     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7209     __ setb(Assembler::equal, $res$$Register);
 7210     __ movzbl($res$$Register, $res$$Register);
 7211   %}
 7212   ins_pipe( pipe_cmpxchg );
 7213 %}
 7214 
 7215 instruct compareAndSwapN(rRegI res,
 7216                           memory mem_ptr,
 7217                           rax_RegN oldval, rRegN newval,
 7218                           rFlagsReg cr) %{
 7219   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7220   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7221   effect(KILL cr, KILL oldval);
 7222 
 7223   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7224             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7225             "sete    $res\n\t"
 7226             "movzbl  $res, $res" %}
 7227   ins_encode %{
 7228     __ lock();
 7229     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7230     __ setb(Assembler::equal, $res$$Register);
 7231     __ movzbl($res$$Register, $res$$Register);
 7232   %}
 7233   ins_pipe( pipe_cmpxchg );
 7234 %}
 7235 
 7236 instruct compareAndExchangeB(
 7237                          memory mem_ptr,
 7238                          rax_RegI oldval, rRegI newval,
 7239                          rFlagsReg cr)
 7240 %{
 7241   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7242   effect(KILL cr);
 7243 
 7244   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7245             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7246   ins_encode %{
 7247     __ lock();
 7248     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7249   %}
 7250   ins_pipe( pipe_cmpxchg );
 7251 %}
 7252 
 7253 instruct compareAndExchangeS(
 7254                          memory mem_ptr,
 7255                          rax_RegI oldval, rRegI newval,
 7256                          rFlagsReg cr)
 7257 %{
 7258   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7259   effect(KILL cr);
 7260 
 7261   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7262             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7263   ins_encode %{
 7264     __ lock();
 7265     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7266   %}
 7267   ins_pipe( pipe_cmpxchg );
 7268 %}
 7269 
 7270 instruct compareAndExchangeI(
 7271                          memory mem_ptr,
 7272                          rax_RegI oldval, rRegI newval,
 7273                          rFlagsReg cr)
 7274 %{
 7275   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7276   effect(KILL cr);
 7277 
 7278   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7279             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7280   ins_encode %{
 7281     __ lock();
 7282     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7283   %}
 7284   ins_pipe( pipe_cmpxchg );
 7285 %}
 7286 
 7287 instruct compareAndExchangeL(
 7288                          memory mem_ptr,
 7289                          rax_RegL oldval, rRegL newval,
 7290                          rFlagsReg cr)
 7291 %{
 7292   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7293   effect(KILL cr);
 7294 
 7295   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7296             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7297   ins_encode %{
 7298     __ lock();
 7299     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7300   %}
 7301   ins_pipe( pipe_cmpxchg );
 7302 %}
 7303 
 7304 instruct compareAndExchangeN(
 7305                           memory mem_ptr,
 7306                           rax_RegN oldval, rRegN newval,
 7307                           rFlagsReg cr) %{
 7308   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7309   effect(KILL cr);
 7310 
 7311   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7312             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7313   ins_encode %{
 7314     __ lock();
 7315     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7316   %}
 7317   ins_pipe( pipe_cmpxchg );
 7318 %}
 7319 
 7320 instruct compareAndExchangeP(
 7321                          memory mem_ptr,
 7322                          rax_RegP oldval, rRegP newval,
 7323                          rFlagsReg cr)
 7324 %{
 7325   predicate(n->as_LoadStore()->barrier_data() == 0);
 7326   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7327   effect(KILL cr);
 7328 
 7329   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7330             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7331   ins_encode %{
 7332     __ lock();
 7333     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7334   %}
 7335   ins_pipe( pipe_cmpxchg );
 7336 %}
 7337 
 7338 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7339   predicate(n->as_LoadStore()->result_not_used());
 7340   match(Set dummy (GetAndAddB mem add));
 7341   effect(KILL cr);
 7342   format %{ "addb_lock   $mem, $add" %}
 7343   ins_encode %{
 7344     __ lock();
 7345     __ addb($mem$$Address, $add$$Register);
 7346   %}
 7347   ins_pipe(pipe_cmpxchg);
 7348 %}
 7349 
 7350 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7351   predicate(n->as_LoadStore()->result_not_used());
 7352   match(Set dummy (GetAndAddB mem add));
 7353   effect(KILL cr);
 7354   format %{ "addb_lock   $mem, $add" %}
 7355   ins_encode %{
 7356     __ lock();
 7357     __ addb($mem$$Address, $add$$constant);
 7358   %}
 7359   ins_pipe(pipe_cmpxchg);
 7360 %}
 7361 
 7362 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7363   predicate(!n->as_LoadStore()->result_not_used());
 7364   match(Set newval (GetAndAddB mem newval));
 7365   effect(KILL cr);
 7366   format %{ "xaddb_lock  $mem, $newval" %}
 7367   ins_encode %{
 7368     __ lock();
 7369     __ xaddb($mem$$Address, $newval$$Register);
 7370   %}
 7371   ins_pipe(pipe_cmpxchg);
 7372 %}
 7373 
 7374 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7375   predicate(n->as_LoadStore()->result_not_used());
 7376   match(Set dummy (GetAndAddS mem add));
 7377   effect(KILL cr);
 7378   format %{ "addw_lock   $mem, $add" %}
 7379   ins_encode %{
 7380     __ lock();
 7381     __ addw($mem$$Address, $add$$Register);
 7382   %}
 7383   ins_pipe(pipe_cmpxchg);
 7384 %}
 7385 
 7386 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7387   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7388   match(Set dummy (GetAndAddS mem add));
 7389   effect(KILL cr);
 7390   format %{ "addw_lock   $mem, $add" %}
 7391   ins_encode %{
 7392     __ lock();
 7393     __ addw($mem$$Address, $add$$constant);
 7394   %}
 7395   ins_pipe(pipe_cmpxchg);
 7396 %}
 7397 
 7398 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7399   predicate(!n->as_LoadStore()->result_not_used());
 7400   match(Set newval (GetAndAddS mem newval));
 7401   effect(KILL cr);
 7402   format %{ "xaddw_lock  $mem, $newval" %}
 7403   ins_encode %{
 7404     __ lock();
 7405     __ xaddw($mem$$Address, $newval$$Register);
 7406   %}
 7407   ins_pipe(pipe_cmpxchg);
 7408 %}
 7409 
 7410 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7411   predicate(n->as_LoadStore()->result_not_used());
 7412   match(Set dummy (GetAndAddI mem add));
 7413   effect(KILL cr);
 7414   format %{ "addl_lock   $mem, $add" %}
 7415   ins_encode %{
 7416     __ lock();
 7417     __ addl($mem$$Address, $add$$Register);
 7418   %}
 7419   ins_pipe(pipe_cmpxchg);
 7420 %}
 7421 
 7422 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7423   predicate(n->as_LoadStore()->result_not_used());
 7424   match(Set dummy (GetAndAddI mem add));
 7425   effect(KILL cr);
 7426   format %{ "addl_lock   $mem, $add" %}
 7427   ins_encode %{
 7428     __ lock();
 7429     __ addl($mem$$Address, $add$$constant);
 7430   %}
 7431   ins_pipe(pipe_cmpxchg);
 7432 %}
 7433 
 7434 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 7435   predicate(!n->as_LoadStore()->result_not_used());
 7436   match(Set newval (GetAndAddI mem newval));
 7437   effect(KILL cr);
 7438   format %{ "xaddl_lock  $mem, $newval" %}
 7439   ins_encode %{
 7440     __ lock();
 7441     __ xaddl($mem$$Address, $newval$$Register);
 7442   %}
 7443   ins_pipe(pipe_cmpxchg);
 7444 %}
 7445 
 7446 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 7447   predicate(n->as_LoadStore()->result_not_used());
 7448   match(Set dummy (GetAndAddL mem add));
 7449   effect(KILL cr);
 7450   format %{ "addq_lock   $mem, $add" %}
 7451   ins_encode %{
 7452     __ lock();
 7453     __ addq($mem$$Address, $add$$Register);
 7454   %}
 7455   ins_pipe(pipe_cmpxchg);
 7456 %}
 7457 
 7458 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 7459   predicate(n->as_LoadStore()->result_not_used());
 7460   match(Set dummy (GetAndAddL mem add));
 7461   effect(KILL cr);
 7462   format %{ "addq_lock   $mem, $add" %}
 7463   ins_encode %{
 7464     __ lock();
 7465     __ addq($mem$$Address, $add$$constant);
 7466   %}
 7467   ins_pipe(pipe_cmpxchg);
 7468 %}
 7469 
 7470 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 7471   predicate(!n->as_LoadStore()->result_not_used());
 7472   match(Set newval (GetAndAddL mem newval));
 7473   effect(KILL cr);
 7474   format %{ "xaddq_lock  $mem, $newval" %}
 7475   ins_encode %{
 7476     __ lock();
 7477     __ xaddq($mem$$Address, $newval$$Register);
 7478   %}
 7479   ins_pipe(pipe_cmpxchg);
 7480 %}
 7481 
 7482 instruct xchgB( memory mem, rRegI newval) %{
 7483   match(Set newval (GetAndSetB mem newval));
 7484   format %{ "XCHGB  $newval,[$mem]" %}
 7485   ins_encode %{
 7486     __ xchgb($newval$$Register, $mem$$Address);
 7487   %}
 7488   ins_pipe( pipe_cmpxchg );
 7489 %}
 7490 
 7491 instruct xchgS( memory mem, rRegI newval) %{
 7492   match(Set newval (GetAndSetS mem newval));
 7493   format %{ "XCHGW  $newval,[$mem]" %}
 7494   ins_encode %{
 7495     __ xchgw($newval$$Register, $mem$$Address);
 7496   %}
 7497   ins_pipe( pipe_cmpxchg );
 7498 %}
 7499 
 7500 instruct xchgI( memory mem, rRegI newval) %{
 7501   match(Set newval (GetAndSetI mem newval));
 7502   format %{ "XCHGL  $newval,[$mem]" %}
 7503   ins_encode %{
 7504     __ xchgl($newval$$Register, $mem$$Address);
 7505   %}
 7506   ins_pipe( pipe_cmpxchg );
 7507 %}
 7508 
 7509 instruct xchgL( memory mem, rRegL newval) %{
 7510   match(Set newval (GetAndSetL mem newval));
 7511   format %{ "XCHGL  $newval,[$mem]" %}
 7512   ins_encode %{
 7513     __ xchgq($newval$$Register, $mem$$Address);
 7514   %}
 7515   ins_pipe( pipe_cmpxchg );
 7516 %}
 7517 
 7518 instruct xchgP( memory mem, rRegP newval) %{
 7519   match(Set newval (GetAndSetP mem newval));
 7520   predicate(n->as_LoadStore()->barrier_data() == 0);
 7521   format %{ "XCHGQ  $newval,[$mem]" %}
 7522   ins_encode %{
 7523     __ xchgq($newval$$Register, $mem$$Address);
 7524   %}
 7525   ins_pipe( pipe_cmpxchg );
 7526 %}
 7527 
 7528 instruct xchgN( memory mem, rRegN newval) %{
 7529   match(Set newval (GetAndSetN mem newval));
 7530   format %{ "XCHGL  $newval,$mem]" %}
 7531   ins_encode %{
 7532     __ xchgl($newval$$Register, $mem$$Address);
 7533   %}
 7534   ins_pipe( pipe_cmpxchg );
 7535 %}
 7536 
 7537 //----------Abs Instructions-------------------------------------------
 7538 
 7539 // Integer Absolute Instructions
 7540 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7541 %{
 7542   match(Set dst (AbsI src));
 7543   effect(TEMP dst, KILL cr);
 7544   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 7545             "subl    $dst, $src\n\t"
 7546             "cmovll  $dst, $src" %}
 7547   ins_encode %{
 7548     __ xorl($dst$$Register, $dst$$Register);
 7549     __ subl($dst$$Register, $src$$Register);
 7550     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 7551   %}
 7552 
 7553   ins_pipe(ialu_reg_reg);
 7554 %}
 7555 
 7556 // Long Absolute Instructions
 7557 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7558 %{
 7559   match(Set dst (AbsL src));
 7560   effect(TEMP dst, KILL cr);
 7561   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 7562             "subq    $dst, $src\n\t"
 7563             "cmovlq  $dst, $src" %}
 7564   ins_encode %{
 7565     __ xorl($dst$$Register, $dst$$Register);
 7566     __ subq($dst$$Register, $src$$Register);
 7567     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 7568   %}
 7569 
 7570   ins_pipe(ialu_reg_reg);
 7571 %}
 7572 
 7573 //----------Subtraction Instructions-------------------------------------------
 7574 
 7575 // Integer Subtraction Instructions
 7576 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7577 %{
 7578   match(Set dst (SubI dst src));
 7579   effect(KILL cr);
 7580   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7581 
 7582   format %{ "subl    $dst, $src\t# int" %}
 7583   ins_encode %{
 7584     __ subl($dst$$Register, $src$$Register);
 7585   %}
 7586   ins_pipe(ialu_reg_reg);
 7587 %}
 7588 
 7589 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7590 %{
 7591   match(Set dst (SubI dst (LoadI src)));
 7592   effect(KILL cr);
 7593   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7594 
 7595   ins_cost(150);
 7596   format %{ "subl    $dst, $src\t# int" %}
 7597   ins_encode %{
 7598     __ subl($dst$$Register, $src$$Address);
 7599   %}
 7600   ins_pipe(ialu_reg_mem);
 7601 %}
 7602 
 7603 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7604 %{
 7605   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 7606   effect(KILL cr);
 7607   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7608 
 7609   ins_cost(150);
 7610   format %{ "subl    $dst, $src\t# int" %}
 7611   ins_encode %{
 7612     __ subl($dst$$Address, $src$$Register);
 7613   %}
 7614   ins_pipe(ialu_mem_reg);
 7615 %}
 7616 
 7617 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7618 %{
 7619   match(Set dst (SubL dst src));
 7620   effect(KILL cr);
 7621   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7622 
 7623   format %{ "subq    $dst, $src\t# long" %}
 7624   ins_encode %{
 7625     __ subq($dst$$Register, $src$$Register);
 7626   %}
 7627   ins_pipe(ialu_reg_reg);
 7628 %}
 7629 
 7630 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7631 %{
 7632   match(Set dst (SubL dst (LoadL src)));
 7633   effect(KILL cr);
 7634   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7635 
 7636   ins_cost(150);
 7637   format %{ "subq    $dst, $src\t# long" %}
 7638   ins_encode %{
 7639     __ subq($dst$$Register, $src$$Address);
 7640   %}
 7641   ins_pipe(ialu_reg_mem);
 7642 %}
 7643 
 7644 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7645 %{
 7646   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 7647   effect(KILL cr);
 7648   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7649 
 7650   ins_cost(150);
 7651   format %{ "subq    $dst, $src\t# long" %}
 7652   ins_encode %{
 7653     __ subq($dst$$Address, $src$$Register);
 7654   %}
 7655   ins_pipe(ialu_mem_reg);
 7656 %}
 7657 
 7658 // Subtract from a pointer
 7659 // XXX hmpf???
 7660 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 7661 %{
 7662   match(Set dst (AddP dst (SubI zero src)));
 7663   effect(KILL cr);
 7664 
 7665   format %{ "subq    $dst, $src\t# ptr - int" %}
 7666   ins_encode %{
 7667     __ subq($dst$$Register, $src$$Register);
 7668   %}
 7669   ins_pipe(ialu_reg_reg);
 7670 %}
 7671 
 7672 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 7673 %{
 7674   match(Set dst (SubI zero dst));
 7675   effect(KILL cr);
 7676   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7677 
 7678   format %{ "negl    $dst\t# int" %}
 7679   ins_encode %{
 7680     __ negl($dst$$Register);
 7681   %}
 7682   ins_pipe(ialu_reg);
 7683 %}
 7684 
 7685 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 7686 %{
 7687   match(Set dst (NegI dst));
 7688   effect(KILL cr);
 7689   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7690 
 7691   format %{ "negl    $dst\t# int" %}
 7692   ins_encode %{
 7693     __ negl($dst$$Register);
 7694   %}
 7695   ins_pipe(ialu_reg);
 7696 %}
 7697 
 7698 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 7699 %{
 7700   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 7701   effect(KILL cr);
 7702   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7703 
 7704   format %{ "negl    $dst\t# int" %}
 7705   ins_encode %{
 7706     __ negl($dst$$Address);
 7707   %}
 7708   ins_pipe(ialu_reg);
 7709 %}
 7710 
 7711 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 7712 %{
 7713   match(Set dst (SubL zero dst));
 7714   effect(KILL cr);
 7715   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7716 
 7717   format %{ "negq    $dst\t# long" %}
 7718   ins_encode %{
 7719     __ negq($dst$$Register);
 7720   %}
 7721   ins_pipe(ialu_reg);
 7722 %}
 7723 
 7724 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 7725 %{
 7726   match(Set dst (NegL dst));
 7727   effect(KILL cr);
 7728   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7729 
 7730   format %{ "negq    $dst\t# int" %}
 7731   ins_encode %{
 7732     __ negq($dst$$Register);
 7733   %}
 7734   ins_pipe(ialu_reg);
 7735 %}
 7736 
 7737 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 7738 %{
 7739   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 7740   effect(KILL cr);
 7741   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7742 
 7743   format %{ "negq    $dst\t# long" %}
 7744   ins_encode %{
 7745     __ negq($dst$$Address);
 7746   %}
 7747   ins_pipe(ialu_reg);
 7748 %}
 7749 
 7750 //----------Multiplication/Division Instructions-------------------------------
 7751 // Integer Multiplication Instructions
 7752 // Multiply Register
 7753 
 7754 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7755 %{
 7756   match(Set dst (MulI dst src));
 7757   effect(KILL cr);
 7758 
 7759   ins_cost(300);
 7760   format %{ "imull   $dst, $src\t# int" %}
 7761   ins_encode %{
 7762     __ imull($dst$$Register, $src$$Register);
 7763   %}
 7764   ins_pipe(ialu_reg_reg_alu0);
 7765 %}
 7766 
 7767 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 7768 %{
 7769   match(Set dst (MulI src imm));
 7770   effect(KILL cr);
 7771 
 7772   ins_cost(300);
 7773   format %{ "imull   $dst, $src, $imm\t# int" %}
 7774   ins_encode %{
 7775     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 7776   %}
 7777   ins_pipe(ialu_reg_reg_alu0);
 7778 %}
 7779 
 7780 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 7781 %{
 7782   match(Set dst (MulI dst (LoadI src)));
 7783   effect(KILL cr);
 7784 
 7785   ins_cost(350);
 7786   format %{ "imull   $dst, $src\t# int" %}
 7787   ins_encode %{
 7788     __ imull($dst$$Register, $src$$Address);
 7789   %}
 7790   ins_pipe(ialu_reg_mem_alu0);
 7791 %}
 7792 
 7793 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 7794 %{
 7795   match(Set dst (MulI (LoadI src) imm));
 7796   effect(KILL cr);
 7797 
 7798   ins_cost(300);
 7799   format %{ "imull   $dst, $src, $imm\t# int" %}
 7800   ins_encode %{
 7801     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 7802   %}
 7803   ins_pipe(ialu_reg_mem_alu0);
 7804 %}
 7805 
 7806 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 7807 %{
 7808   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 7809   effect(KILL cr, KILL src2);
 7810 
 7811   expand %{ mulI_rReg(dst, src1, cr);
 7812            mulI_rReg(src2, src3, cr);
 7813            addI_rReg(dst, src2, cr); %}
 7814 %}
 7815 
 7816 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7817 %{
 7818   match(Set dst (MulL dst src));
 7819   effect(KILL cr);
 7820 
 7821   ins_cost(300);
 7822   format %{ "imulq   $dst, $src\t# long" %}
 7823   ins_encode %{
 7824     __ imulq($dst$$Register, $src$$Register);
 7825   %}
 7826   ins_pipe(ialu_reg_reg_alu0);
 7827 %}
 7828 
 7829 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 7830 %{
 7831   match(Set dst (MulL src imm));
 7832   effect(KILL cr);
 7833 
 7834   ins_cost(300);
 7835   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7836   ins_encode %{
 7837     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 7838   %}
 7839   ins_pipe(ialu_reg_reg_alu0);
 7840 %}
 7841 
 7842 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 7843 %{
 7844   match(Set dst (MulL dst (LoadL src)));
 7845   effect(KILL cr);
 7846 
 7847   ins_cost(350);
 7848   format %{ "imulq   $dst, $src\t# long" %}
 7849   ins_encode %{
 7850     __ imulq($dst$$Register, $src$$Address);
 7851   %}
 7852   ins_pipe(ialu_reg_mem_alu0);
 7853 %}
 7854 
 7855 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 7856 %{
 7857   match(Set dst (MulL (LoadL src) imm));
 7858   effect(KILL cr);
 7859 
 7860   ins_cost(300);
 7861   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7862   ins_encode %{
 7863     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 7864   %}
 7865   ins_pipe(ialu_reg_mem_alu0);
 7866 %}
 7867 
 7868 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7869 %{
 7870   match(Set dst (MulHiL src rax));
 7871   effect(USE_KILL rax, KILL cr);
 7872 
 7873   ins_cost(300);
 7874   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 7875   ins_encode %{
 7876     __ imulq($src$$Register);
 7877   %}
 7878   ins_pipe(ialu_reg_reg_alu0);
 7879 %}
 7880 
 7881 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7882 %{
 7883   match(Set dst (UMulHiL src rax));
 7884   effect(USE_KILL rax, KILL cr);
 7885 
 7886   ins_cost(300);
 7887   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 7888   ins_encode %{
 7889     __ mulq($src$$Register);
 7890   %}
 7891   ins_pipe(ialu_reg_reg_alu0);
 7892 %}
 7893 
 7894 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7895                    rFlagsReg cr)
 7896 %{
 7897   match(Set rax (DivI rax div));
 7898   effect(KILL rdx, KILL cr);
 7899 
 7900   ins_cost(30*100+10*100); // XXX
 7901   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7902             "jne,s   normal\n\t"
 7903             "xorl    rdx, rdx\n\t"
 7904             "cmpl    $div, -1\n\t"
 7905             "je,s    done\n"
 7906     "normal: cdql\n\t"
 7907             "idivl   $div\n"
 7908     "done:"        %}
 7909   ins_encode(cdql_enc(div));
 7910   ins_pipe(ialu_reg_reg_alu0);
 7911 %}
 7912 
 7913 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7914                    rFlagsReg cr)
 7915 %{
 7916   match(Set rax (DivL rax div));
 7917   effect(KILL rdx, KILL cr);
 7918 
 7919   ins_cost(30*100+10*100); // XXX
 7920   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7921             "cmpq    rax, rdx\n\t"
 7922             "jne,s   normal\n\t"
 7923             "xorl    rdx, rdx\n\t"
 7924             "cmpq    $div, -1\n\t"
 7925             "je,s    done\n"
 7926     "normal: cdqq\n\t"
 7927             "idivq   $div\n"
 7928     "done:"        %}
 7929   ins_encode(cdqq_enc(div));
 7930   ins_pipe(ialu_reg_reg_alu0);
 7931 %}
 7932 
 7933 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 7934 %{
 7935   match(Set rax (UDivI rax div));
 7936   effect(KILL rdx, KILL cr);
 7937 
 7938   ins_cost(300);
 7939   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 7940   ins_encode %{
 7941     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 7942   %}
 7943   ins_pipe(ialu_reg_reg_alu0);
 7944 %}
 7945 
 7946 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 7947 %{
 7948   match(Set rax (UDivL rax div));
 7949   effect(KILL rdx, KILL cr);
 7950 
 7951   ins_cost(300);
 7952   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 7953   ins_encode %{
 7954      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 7955   %}
 7956   ins_pipe(ialu_reg_reg_alu0);
 7957 %}
 7958 
 7959 // Integer DIVMOD with Register, both quotient and mod results
 7960 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7961                              rFlagsReg cr)
 7962 %{
 7963   match(DivModI rax div);
 7964   effect(KILL cr);
 7965 
 7966   ins_cost(30*100+10*100); // XXX
 7967   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7968             "jne,s   normal\n\t"
 7969             "xorl    rdx, rdx\n\t"
 7970             "cmpl    $div, -1\n\t"
 7971             "je,s    done\n"
 7972     "normal: cdql\n\t"
 7973             "idivl   $div\n"
 7974     "done:"        %}
 7975   ins_encode(cdql_enc(div));
 7976   ins_pipe(pipe_slow);
 7977 %}
 7978 
 7979 // Long DIVMOD with Register, both quotient and mod results
 7980 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7981                              rFlagsReg cr)
 7982 %{
 7983   match(DivModL rax div);
 7984   effect(KILL cr);
 7985 
 7986   ins_cost(30*100+10*100); // XXX
 7987   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7988             "cmpq    rax, rdx\n\t"
 7989             "jne,s   normal\n\t"
 7990             "xorl    rdx, rdx\n\t"
 7991             "cmpq    $div, -1\n\t"
 7992             "je,s    done\n"
 7993     "normal: cdqq\n\t"
 7994             "idivq   $div\n"
 7995     "done:"        %}
 7996   ins_encode(cdqq_enc(div));
 7997   ins_pipe(pipe_slow);
 7998 %}
 7999 
 8000 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8001 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8002                               no_rax_rdx_RegI div, rFlagsReg cr)
 8003 %{
 8004   match(UDivModI rax div);
 8005   effect(TEMP tmp, KILL cr);
 8006 
 8007   ins_cost(300);
 8008   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8009             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8010           %}
 8011   ins_encode %{
 8012     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8013   %}
 8014   ins_pipe(pipe_slow);
 8015 %}
 8016 
 8017 // Unsigned long DIVMOD with Register, both quotient and mod results
 8018 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8019                               no_rax_rdx_RegL div, rFlagsReg cr)
 8020 %{
 8021   match(UDivModL rax div);
 8022   effect(TEMP tmp, KILL cr);
 8023 
 8024   ins_cost(300);
 8025   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8026             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8027           %}
 8028   ins_encode %{
 8029     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8030   %}
 8031   ins_pipe(pipe_slow);
 8032 %}
 8033 
 8034 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8035                    rFlagsReg cr)
 8036 %{
 8037   match(Set rdx (ModI rax div));
 8038   effect(KILL rax, KILL cr);
 8039 
 8040   ins_cost(300); // XXX
 8041   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8042             "jne,s   normal\n\t"
 8043             "xorl    rdx, rdx\n\t"
 8044             "cmpl    $div, -1\n\t"
 8045             "je,s    done\n"
 8046     "normal: cdql\n\t"
 8047             "idivl   $div\n"
 8048     "done:"        %}
 8049   ins_encode(cdql_enc(div));
 8050   ins_pipe(ialu_reg_reg_alu0);
 8051 %}
 8052 
 8053 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8054                    rFlagsReg cr)
 8055 %{
 8056   match(Set rdx (ModL rax div));
 8057   effect(KILL rax, KILL cr);
 8058 
 8059   ins_cost(300); // XXX
 8060   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8061             "cmpq    rax, rdx\n\t"
 8062             "jne,s   normal\n\t"
 8063             "xorl    rdx, rdx\n\t"
 8064             "cmpq    $div, -1\n\t"
 8065             "je,s    done\n"
 8066     "normal: cdqq\n\t"
 8067             "idivq   $div\n"
 8068     "done:"        %}
 8069   ins_encode(cdqq_enc(div));
 8070   ins_pipe(ialu_reg_reg_alu0);
 8071 %}
 8072 
 8073 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8074 %{
 8075   match(Set rdx (UModI rax div));
 8076   effect(KILL rax, KILL cr);
 8077 
 8078   ins_cost(300);
 8079   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8080   ins_encode %{
 8081     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8082   %}
 8083   ins_pipe(ialu_reg_reg_alu0);
 8084 %}
 8085 
 8086 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8087 %{
 8088   match(Set rdx (UModL rax div));
 8089   effect(KILL rax, KILL cr);
 8090 
 8091   ins_cost(300);
 8092   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8093   ins_encode %{
 8094     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8095   %}
 8096   ins_pipe(ialu_reg_reg_alu0);
 8097 %}
 8098 
 8099 // Integer Shift Instructions
 8100 // Shift Left by one, two, three
 8101 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8102 %{
 8103   match(Set dst (LShiftI dst shift));
 8104   effect(KILL cr);
 8105 
 8106   format %{ "sall    $dst, $shift" %}
 8107   ins_encode %{
 8108     __ sall($dst$$Register, $shift$$constant);
 8109   %}
 8110   ins_pipe(ialu_reg);
 8111 %}
 8112 
 8113 // Shift Left by 8-bit immediate
 8114 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8115 %{
 8116   match(Set dst (LShiftI dst shift));
 8117   effect(KILL cr);
 8118 
 8119   format %{ "sall    $dst, $shift" %}
 8120   ins_encode %{
 8121     __ sall($dst$$Register, $shift$$constant);
 8122   %}
 8123   ins_pipe(ialu_reg);
 8124 %}
 8125 
 8126 // Shift Left by 8-bit immediate
 8127 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8128 %{
 8129   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8130   effect(KILL cr);
 8131 
 8132   format %{ "sall    $dst, $shift" %}
 8133   ins_encode %{
 8134     __ sall($dst$$Address, $shift$$constant);
 8135   %}
 8136   ins_pipe(ialu_mem_imm);
 8137 %}
 8138 
 8139 // Shift Left by variable
 8140 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8141 %{
 8142   predicate(!VM_Version::supports_bmi2());
 8143   match(Set dst (LShiftI dst shift));
 8144   effect(KILL cr);
 8145 
 8146   format %{ "sall    $dst, $shift" %}
 8147   ins_encode %{
 8148     __ sall($dst$$Register);
 8149   %}
 8150   ins_pipe(ialu_reg_reg);
 8151 %}
 8152 
 8153 // Shift Left by variable
 8154 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8155 %{
 8156   predicate(!VM_Version::supports_bmi2());
 8157   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8158   effect(KILL cr);
 8159 
 8160   format %{ "sall    $dst, $shift" %}
 8161   ins_encode %{
 8162     __ sall($dst$$Address);
 8163   %}
 8164   ins_pipe(ialu_mem_reg);
 8165 %}
 8166 
 8167 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8168 %{
 8169   predicate(VM_Version::supports_bmi2());
 8170   match(Set dst (LShiftI src shift));
 8171 
 8172   format %{ "shlxl   $dst, $src, $shift" %}
 8173   ins_encode %{
 8174     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 8175   %}
 8176   ins_pipe(ialu_reg_reg);
 8177 %}
 8178 
 8179 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8180 %{
 8181   predicate(VM_Version::supports_bmi2());
 8182   match(Set dst (LShiftI (LoadI src) shift));
 8183   ins_cost(175);
 8184   format %{ "shlxl   $dst, $src, $shift" %}
 8185   ins_encode %{
 8186     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 8187   %}
 8188   ins_pipe(ialu_reg_mem);
 8189 %}
 8190 
 8191 // Arithmetic Shift Right by 8-bit immediate
 8192 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8193 %{
 8194   match(Set dst (RShiftI dst shift));
 8195   effect(KILL cr);
 8196 
 8197   format %{ "sarl    $dst, $shift" %}
 8198   ins_encode %{
 8199     __ sarl($dst$$Register, $shift$$constant);
 8200   %}
 8201   ins_pipe(ialu_mem_imm);
 8202 %}
 8203 
 8204 // Arithmetic Shift Right by 8-bit immediate
 8205 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8206 %{
 8207   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8208   effect(KILL cr);
 8209 
 8210   format %{ "sarl    $dst, $shift" %}
 8211   ins_encode %{
 8212     __ sarl($dst$$Address, $shift$$constant);
 8213   %}
 8214   ins_pipe(ialu_mem_imm);
 8215 %}
 8216 
 8217 // Arithmetic Shift Right by variable
 8218 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8219 %{
 8220   predicate(!VM_Version::supports_bmi2());
 8221   match(Set dst (RShiftI dst shift));
 8222   effect(KILL cr);
 8223 
 8224   format %{ "sarl    $dst, $shift" %}
 8225   ins_encode %{
 8226     __ sarl($dst$$Register);
 8227   %}
 8228   ins_pipe(ialu_reg_reg);
 8229 %}
 8230 
 8231 // Arithmetic Shift Right by variable
 8232 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8233 %{
 8234   predicate(!VM_Version::supports_bmi2());
 8235   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8236   effect(KILL cr);
 8237 
 8238   format %{ "sarl    $dst, $shift" %}
 8239   ins_encode %{
 8240     __ sarl($dst$$Address);
 8241   %}
 8242   ins_pipe(ialu_mem_reg);
 8243 %}
 8244 
 8245 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8246 %{
 8247   predicate(VM_Version::supports_bmi2());
 8248   match(Set dst (RShiftI src shift));
 8249 
 8250   format %{ "sarxl   $dst, $src, $shift" %}
 8251   ins_encode %{
 8252     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 8253   %}
 8254   ins_pipe(ialu_reg_reg);
 8255 %}
 8256 
 8257 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8258 %{
 8259   predicate(VM_Version::supports_bmi2());
 8260   match(Set dst (RShiftI (LoadI src) shift));
 8261   ins_cost(175);
 8262   format %{ "sarxl   $dst, $src, $shift" %}
 8263   ins_encode %{
 8264     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 8265   %}
 8266   ins_pipe(ialu_reg_mem);
 8267 %}
 8268 
 8269 // Logical Shift Right by 8-bit immediate
 8270 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8271 %{
 8272   match(Set dst (URShiftI dst shift));
 8273   effect(KILL cr);
 8274 
 8275   format %{ "shrl    $dst, $shift" %}
 8276   ins_encode %{
 8277     __ shrl($dst$$Register, $shift$$constant);
 8278   %}
 8279   ins_pipe(ialu_reg);
 8280 %}
 8281 
 8282 // Logical Shift Right by 8-bit immediate
 8283 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8284 %{
 8285   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8286   effect(KILL cr);
 8287 
 8288   format %{ "shrl    $dst, $shift" %}
 8289   ins_encode %{
 8290     __ shrl($dst$$Address, $shift$$constant);
 8291   %}
 8292   ins_pipe(ialu_mem_imm);
 8293 %}
 8294 
 8295 // Logical Shift Right by variable
 8296 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8297 %{
 8298   predicate(!VM_Version::supports_bmi2());
 8299   match(Set dst (URShiftI dst shift));
 8300   effect(KILL cr);
 8301 
 8302   format %{ "shrl    $dst, $shift" %}
 8303   ins_encode %{
 8304     __ shrl($dst$$Register);
 8305   %}
 8306   ins_pipe(ialu_reg_reg);
 8307 %}
 8308 
 8309 // Logical Shift Right by variable
 8310 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8311 %{
 8312   predicate(!VM_Version::supports_bmi2());
 8313   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8314   effect(KILL cr);
 8315 
 8316   format %{ "shrl    $dst, $shift" %}
 8317   ins_encode %{
 8318     __ shrl($dst$$Address);
 8319   %}
 8320   ins_pipe(ialu_mem_reg);
 8321 %}
 8322 
 8323 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8324 %{
 8325   predicate(VM_Version::supports_bmi2());
 8326   match(Set dst (URShiftI src shift));
 8327 
 8328   format %{ "shrxl   $dst, $src, $shift" %}
 8329   ins_encode %{
 8330     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 8331   %}
 8332   ins_pipe(ialu_reg_reg);
 8333 %}
 8334 
 8335 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8336 %{
 8337   predicate(VM_Version::supports_bmi2());
 8338   match(Set dst (URShiftI (LoadI src) shift));
 8339   ins_cost(175);
 8340   format %{ "shrxl   $dst, $src, $shift" %}
 8341   ins_encode %{
 8342     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 8343   %}
 8344   ins_pipe(ialu_reg_mem);
 8345 %}
 8346 
 8347 // Long Shift Instructions
 8348 // Shift Left by one, two, three
 8349 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 8350 %{
 8351   match(Set dst (LShiftL dst shift));
 8352   effect(KILL cr);
 8353 
 8354   format %{ "salq    $dst, $shift" %}
 8355   ins_encode %{
 8356     __ salq($dst$$Register, $shift$$constant);
 8357   %}
 8358   ins_pipe(ialu_reg);
 8359 %}
 8360 
 8361 // Shift Left by 8-bit immediate
 8362 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8363 %{
 8364   match(Set dst (LShiftL dst shift));
 8365   effect(KILL cr);
 8366 
 8367   format %{ "salq    $dst, $shift" %}
 8368   ins_encode %{
 8369     __ salq($dst$$Register, $shift$$constant);
 8370   %}
 8371   ins_pipe(ialu_reg);
 8372 %}
 8373 
 8374 // Shift Left by 8-bit immediate
 8375 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8376 %{
 8377   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8378   effect(KILL cr);
 8379 
 8380   format %{ "salq    $dst, $shift" %}
 8381   ins_encode %{
 8382     __ salq($dst$$Address, $shift$$constant);
 8383   %}
 8384   ins_pipe(ialu_mem_imm);
 8385 %}
 8386 
 8387 // Shift Left by variable
 8388 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8389 %{
 8390   predicate(!VM_Version::supports_bmi2());
 8391   match(Set dst (LShiftL dst shift));
 8392   effect(KILL cr);
 8393 
 8394   format %{ "salq    $dst, $shift" %}
 8395   ins_encode %{
 8396     __ salq($dst$$Register);
 8397   %}
 8398   ins_pipe(ialu_reg_reg);
 8399 %}
 8400 
 8401 // Shift Left by variable
 8402 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8403 %{
 8404   predicate(!VM_Version::supports_bmi2());
 8405   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8406   effect(KILL cr);
 8407 
 8408   format %{ "salq    $dst, $shift" %}
 8409   ins_encode %{
 8410     __ salq($dst$$Address);
 8411   %}
 8412   ins_pipe(ialu_mem_reg);
 8413 %}
 8414 
 8415 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8416 %{
 8417   predicate(VM_Version::supports_bmi2());
 8418   match(Set dst (LShiftL src shift));
 8419 
 8420   format %{ "shlxq   $dst, $src, $shift" %}
 8421   ins_encode %{
 8422     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 8423   %}
 8424   ins_pipe(ialu_reg_reg);
 8425 %}
 8426 
 8427 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8428 %{
 8429   predicate(VM_Version::supports_bmi2());
 8430   match(Set dst (LShiftL (LoadL src) shift));
 8431   ins_cost(175);
 8432   format %{ "shlxq   $dst, $src, $shift" %}
 8433   ins_encode %{
 8434     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 8435   %}
 8436   ins_pipe(ialu_reg_mem);
 8437 %}
 8438 
 8439 // Arithmetic Shift Right by 8-bit immediate
 8440 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 8441 %{
 8442   match(Set dst (RShiftL dst shift));
 8443   effect(KILL cr);
 8444 
 8445   format %{ "sarq    $dst, $shift" %}
 8446   ins_encode %{
 8447     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 8448   %}
 8449   ins_pipe(ialu_mem_imm);
 8450 %}
 8451 
 8452 // Arithmetic Shift Right by 8-bit immediate
 8453 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 8454 %{
 8455   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8456   effect(KILL cr);
 8457 
 8458   format %{ "sarq    $dst, $shift" %}
 8459   ins_encode %{
 8460     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 8461   %}
 8462   ins_pipe(ialu_mem_imm);
 8463 %}
 8464 
 8465 // Arithmetic Shift Right by variable
 8466 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8467 %{
 8468   predicate(!VM_Version::supports_bmi2());
 8469   match(Set dst (RShiftL dst shift));
 8470   effect(KILL cr);
 8471 
 8472   format %{ "sarq    $dst, $shift" %}
 8473   ins_encode %{
 8474     __ sarq($dst$$Register);
 8475   %}
 8476   ins_pipe(ialu_reg_reg);
 8477 %}
 8478 
 8479 // Arithmetic Shift Right by variable
 8480 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8481 %{
 8482   predicate(!VM_Version::supports_bmi2());
 8483   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8484   effect(KILL cr);
 8485 
 8486   format %{ "sarq    $dst, $shift" %}
 8487   ins_encode %{
 8488     __ sarq($dst$$Address);
 8489   %}
 8490   ins_pipe(ialu_mem_reg);
 8491 %}
 8492 
 8493 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8494 %{
 8495   predicate(VM_Version::supports_bmi2());
 8496   match(Set dst (RShiftL src shift));
 8497 
 8498   format %{ "sarxq   $dst, $src, $shift" %}
 8499   ins_encode %{
 8500     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 8501   %}
 8502   ins_pipe(ialu_reg_reg);
 8503 %}
 8504 
 8505 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8506 %{
 8507   predicate(VM_Version::supports_bmi2());
 8508   match(Set dst (RShiftL (LoadL src) shift));
 8509   ins_cost(175);
 8510   format %{ "sarxq   $dst, $src, $shift" %}
 8511   ins_encode %{
 8512     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 8513   %}
 8514   ins_pipe(ialu_reg_mem);
 8515 %}
 8516 
 8517 // Logical Shift Right by 8-bit immediate
 8518 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8519 %{
 8520   match(Set dst (URShiftL dst shift));
 8521   effect(KILL cr);
 8522 
 8523   format %{ "shrq    $dst, $shift" %}
 8524   ins_encode %{
 8525     __ shrq($dst$$Register, $shift$$constant);
 8526   %}
 8527   ins_pipe(ialu_reg);
 8528 %}
 8529 
 8530 // Logical Shift Right by 8-bit immediate
 8531 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8532 %{
 8533   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8534   effect(KILL cr);
 8535 
 8536   format %{ "shrq    $dst, $shift" %}
 8537   ins_encode %{
 8538     __ shrq($dst$$Address, $shift$$constant);
 8539   %}
 8540   ins_pipe(ialu_mem_imm);
 8541 %}
 8542 
 8543 // Logical Shift Right by variable
 8544 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8545 %{
 8546   predicate(!VM_Version::supports_bmi2());
 8547   match(Set dst (URShiftL dst shift));
 8548   effect(KILL cr);
 8549 
 8550   format %{ "shrq    $dst, $shift" %}
 8551   ins_encode %{
 8552     __ shrq($dst$$Register);
 8553   %}
 8554   ins_pipe(ialu_reg_reg);
 8555 %}
 8556 
 8557 // Logical Shift Right by variable
 8558 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8559 %{
 8560   predicate(!VM_Version::supports_bmi2());
 8561   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8562   effect(KILL cr);
 8563 
 8564   format %{ "shrq    $dst, $shift" %}
 8565   ins_encode %{
 8566     __ shrq($dst$$Address);
 8567   %}
 8568   ins_pipe(ialu_mem_reg);
 8569 %}
 8570 
 8571 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8572 %{
 8573   predicate(VM_Version::supports_bmi2());
 8574   match(Set dst (URShiftL src shift));
 8575 
 8576   format %{ "shrxq   $dst, $src, $shift" %}
 8577   ins_encode %{
 8578     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 8579   %}
 8580   ins_pipe(ialu_reg_reg);
 8581 %}
 8582 
 8583 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8584 %{
 8585   predicate(VM_Version::supports_bmi2());
 8586   match(Set dst (URShiftL (LoadL src) shift));
 8587   ins_cost(175);
 8588   format %{ "shrxq   $dst, $src, $shift" %}
 8589   ins_encode %{
 8590     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 8591   %}
 8592   ins_pipe(ialu_reg_mem);
 8593 %}
 8594 
 8595 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 8596 // This idiom is used by the compiler for the i2b bytecode.
 8597 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 8598 %{
 8599   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 8600 
 8601   format %{ "movsbl  $dst, $src\t# i2b" %}
 8602   ins_encode %{
 8603     __ movsbl($dst$$Register, $src$$Register);
 8604   %}
 8605   ins_pipe(ialu_reg_reg);
 8606 %}
 8607 
 8608 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 8609 // This idiom is used by the compiler the i2s bytecode.
 8610 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 8611 %{
 8612   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 8613 
 8614   format %{ "movswl  $dst, $src\t# i2s" %}
 8615   ins_encode %{
 8616     __ movswl($dst$$Register, $src$$Register);
 8617   %}
 8618   ins_pipe(ialu_reg_reg);
 8619 %}
 8620 
 8621 // ROL/ROR instructions
 8622 
 8623 // Rotate left by constant.
 8624 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8625 %{
 8626   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8627   match(Set dst (RotateLeft dst shift));
 8628   effect(KILL cr);
 8629   format %{ "roll    $dst, $shift" %}
 8630   ins_encode %{
 8631     __ roll($dst$$Register, $shift$$constant);
 8632   %}
 8633   ins_pipe(ialu_reg);
 8634 %}
 8635 
 8636 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 8637 %{
 8638   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8639   match(Set dst (RotateLeft src shift));
 8640   format %{ "rolxl   $dst, $src, $shift" %}
 8641   ins_encode %{
 8642     int shift = 32 - ($shift$$constant & 31);
 8643     __ rorxl($dst$$Register, $src$$Register, shift);
 8644   %}
 8645   ins_pipe(ialu_reg_reg);
 8646 %}
 8647 
 8648 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8649 %{
 8650   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8651   match(Set dst (RotateLeft (LoadI src) shift));
 8652   ins_cost(175);
 8653   format %{ "rolxl   $dst, $src, $shift" %}
 8654   ins_encode %{
 8655     int shift = 32 - ($shift$$constant & 31);
 8656     __ rorxl($dst$$Register, $src$$Address, shift);
 8657   %}
 8658   ins_pipe(ialu_reg_mem);
 8659 %}
 8660 
 8661 // Rotate Left by variable
 8662 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8663 %{
 8664   predicate(n->bottom_type()->basic_type() == T_INT);
 8665   match(Set dst (RotateLeft dst shift));
 8666   effect(KILL cr);
 8667   format %{ "roll    $dst, $shift" %}
 8668   ins_encode %{
 8669     __ roll($dst$$Register);
 8670   %}
 8671   ins_pipe(ialu_reg_reg);
 8672 %}
 8673 
 8674 // Rotate Right by constant.
 8675 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8676 %{
 8677   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8678   match(Set dst (RotateRight dst shift));
 8679   effect(KILL cr);
 8680   format %{ "rorl    $dst, $shift" %}
 8681   ins_encode %{
 8682     __ rorl($dst$$Register, $shift$$constant);
 8683   %}
 8684   ins_pipe(ialu_reg);
 8685 %}
 8686 
 8687 // Rotate Right by constant.
 8688 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 8689 %{
 8690   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8691   match(Set dst (RotateRight src shift));
 8692   format %{ "rorxl   $dst, $src, $shift" %}
 8693   ins_encode %{
 8694     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 8695   %}
 8696   ins_pipe(ialu_reg_reg);
 8697 %}
 8698 
 8699 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8700 %{
 8701   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8702   match(Set dst (RotateRight (LoadI src) shift));
 8703   ins_cost(175);
 8704   format %{ "rorxl   $dst, $src, $shift" %}
 8705   ins_encode %{
 8706     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 8707   %}
 8708   ins_pipe(ialu_reg_mem);
 8709 %}
 8710 
 8711 // Rotate Right by variable
 8712 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8713 %{
 8714   predicate(n->bottom_type()->basic_type() == T_INT);
 8715   match(Set dst (RotateRight dst shift));
 8716   effect(KILL cr);
 8717   format %{ "rorl    $dst, $shift" %}
 8718   ins_encode %{
 8719     __ rorl($dst$$Register);
 8720   %}
 8721   ins_pipe(ialu_reg_reg);
 8722 %}
 8723 
 8724 // Rotate Left by constant.
 8725 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8726 %{
 8727   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8728   match(Set dst (RotateLeft dst shift));
 8729   effect(KILL cr);
 8730   format %{ "rolq    $dst, $shift" %}
 8731   ins_encode %{
 8732     __ rolq($dst$$Register, $shift$$constant);
 8733   %}
 8734   ins_pipe(ialu_reg);
 8735 %}
 8736 
 8737 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 8738 %{
 8739   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8740   match(Set dst (RotateLeft src shift));
 8741   format %{ "rolxq   $dst, $src, $shift" %}
 8742   ins_encode %{
 8743     int shift = 64 - ($shift$$constant & 63);
 8744     __ rorxq($dst$$Register, $src$$Register, shift);
 8745   %}
 8746   ins_pipe(ialu_reg_reg);
 8747 %}
 8748 
 8749 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8750 %{
 8751   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8752   match(Set dst (RotateLeft (LoadL src) shift));
 8753   ins_cost(175);
 8754   format %{ "rolxq   $dst, $src, $shift" %}
 8755   ins_encode %{
 8756     int shift = 64 - ($shift$$constant & 63);
 8757     __ rorxq($dst$$Register, $src$$Address, shift);
 8758   %}
 8759   ins_pipe(ialu_reg_mem);
 8760 %}
 8761 
 8762 // Rotate Left by variable
 8763 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8764 %{
 8765   predicate(n->bottom_type()->basic_type() == T_LONG);
 8766   match(Set dst (RotateLeft dst shift));
 8767   effect(KILL cr);
 8768   format %{ "rolq    $dst, $shift" %}
 8769   ins_encode %{
 8770     __ rolq($dst$$Register);
 8771   %}
 8772   ins_pipe(ialu_reg_reg);
 8773 %}
 8774 
 8775 // Rotate Right by constant.
 8776 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8777 %{
 8778   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8779   match(Set dst (RotateRight dst shift));
 8780   effect(KILL cr);
 8781   format %{ "rorq    $dst, $shift" %}
 8782   ins_encode %{
 8783     __ rorq($dst$$Register, $shift$$constant);
 8784   %}
 8785   ins_pipe(ialu_reg);
 8786 %}
 8787 
 8788 // Rotate Right by constant
 8789 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 8790 %{
 8791   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8792   match(Set dst (RotateRight src shift));
 8793   format %{ "rorxq   $dst, $src, $shift" %}
 8794   ins_encode %{
 8795     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 8796   %}
 8797   ins_pipe(ialu_reg_reg);
 8798 %}
 8799 
 8800 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8801 %{
 8802   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8803   match(Set dst (RotateRight (LoadL src) shift));
 8804   ins_cost(175);
 8805   format %{ "rorxq   $dst, $src, $shift" %}
 8806   ins_encode %{
 8807     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 8808   %}
 8809   ins_pipe(ialu_reg_mem);
 8810 %}
 8811 
 8812 // Rotate Right by variable
 8813 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8814 %{
 8815   predicate(n->bottom_type()->basic_type() == T_LONG);
 8816   match(Set dst (RotateRight dst shift));
 8817   effect(KILL cr);
 8818   format %{ "rorq    $dst, $shift" %}
 8819   ins_encode %{
 8820     __ rorq($dst$$Register);
 8821   %}
 8822   ins_pipe(ialu_reg_reg);
 8823 %}
 8824 
 8825 //----------------------------- CompressBits/ExpandBits ------------------------
 8826 
 8827 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8828   predicate(n->bottom_type()->isa_long());
 8829   match(Set dst (CompressBits src mask));
 8830   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8831   ins_encode %{
 8832     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 8833   %}
 8834   ins_pipe( pipe_slow );
 8835 %}
 8836 
 8837 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8838   predicate(n->bottom_type()->isa_long());
 8839   match(Set dst (ExpandBits src mask));
 8840   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8841   ins_encode %{
 8842     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 8843   %}
 8844   ins_pipe( pipe_slow );
 8845 %}
 8846 
 8847 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8848   predicate(n->bottom_type()->isa_long());
 8849   match(Set dst (CompressBits src (LoadL mask)));
 8850   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8851   ins_encode %{
 8852     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 8853   %}
 8854   ins_pipe( pipe_slow );
 8855 %}
 8856 
 8857 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8858   predicate(n->bottom_type()->isa_long());
 8859   match(Set dst (ExpandBits src (LoadL mask)));
 8860   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8861   ins_encode %{
 8862     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 8863   %}
 8864   ins_pipe( pipe_slow );
 8865 %}
 8866 
 8867 
 8868 // Logical Instructions
 8869 
 8870 // Integer Logical Instructions
 8871 
 8872 // And Instructions
 8873 // And Register with Register
 8874 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8875 %{
 8876   match(Set dst (AndI dst src));
 8877   effect(KILL cr);
 8878   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8879 
 8880   format %{ "andl    $dst, $src\t# int" %}
 8881   ins_encode %{
 8882     __ andl($dst$$Register, $src$$Register);
 8883   %}
 8884   ins_pipe(ialu_reg_reg);
 8885 %}
 8886 
 8887 // And Register with Immediate 255
 8888 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 8889 %{
 8890   match(Set dst (AndI src mask));
 8891 
 8892   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 8893   ins_encode %{
 8894     __ movzbl($dst$$Register, $src$$Register);
 8895   %}
 8896   ins_pipe(ialu_reg);
 8897 %}
 8898 
 8899 // And Register with Immediate 255 and promote to long
 8900 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 8901 %{
 8902   match(Set dst (ConvI2L (AndI src mask)));
 8903 
 8904   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 8905   ins_encode %{
 8906     __ movzbl($dst$$Register, $src$$Register);
 8907   %}
 8908   ins_pipe(ialu_reg);
 8909 %}
 8910 
 8911 // And Register with Immediate 65535
 8912 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 8913 %{
 8914   match(Set dst (AndI src mask));
 8915 
 8916   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 8917   ins_encode %{
 8918     __ movzwl($dst$$Register, $src$$Register);
 8919   %}
 8920   ins_pipe(ialu_reg);
 8921 %}
 8922 
 8923 // And Register with Immediate 65535 and promote to long
 8924 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 8925 %{
 8926   match(Set dst (ConvI2L (AndI src mask)));
 8927 
 8928   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 8929   ins_encode %{
 8930     __ movzwl($dst$$Register, $src$$Register);
 8931   %}
 8932   ins_pipe(ialu_reg);
 8933 %}
 8934 
 8935 // Can skip int2long conversions after AND with small bitmask
 8936 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 8937 %{
 8938   predicate(VM_Version::supports_bmi2());
 8939   ins_cost(125);
 8940   effect(TEMP tmp, KILL cr);
 8941   match(Set dst (ConvI2L (AndI src mask)));
 8942   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 8943   ins_encode %{
 8944     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 8945     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 8946   %}
 8947   ins_pipe(ialu_reg_reg);
 8948 %}
 8949 
 8950 // And Register with Immediate
 8951 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8952 %{
 8953   match(Set dst (AndI dst src));
 8954   effect(KILL cr);
 8955   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8956 
 8957   format %{ "andl    $dst, $src\t# int" %}
 8958   ins_encode %{
 8959     __ andl($dst$$Register, $src$$constant);
 8960   %}
 8961   ins_pipe(ialu_reg);
 8962 %}
 8963 
 8964 // And Register with Memory
 8965 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8966 %{
 8967   match(Set dst (AndI dst (LoadI src)));
 8968   effect(KILL cr);
 8969   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8970 
 8971   ins_cost(150);
 8972   format %{ "andl    $dst, $src\t# int" %}
 8973   ins_encode %{
 8974     __ andl($dst$$Register, $src$$Address);
 8975   %}
 8976   ins_pipe(ialu_reg_mem);
 8977 %}
 8978 
 8979 // And Memory with Register
 8980 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8981 %{
 8982   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 8983   effect(KILL cr);
 8984   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8985 
 8986   ins_cost(150);
 8987   format %{ "andb    $dst, $src\t# byte" %}
 8988   ins_encode %{
 8989     __ andb($dst$$Address, $src$$Register);
 8990   %}
 8991   ins_pipe(ialu_mem_reg);
 8992 %}
 8993 
 8994 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8995 %{
 8996   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 8997   effect(KILL cr);
 8998   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8999 
 9000   ins_cost(150);
 9001   format %{ "andl    $dst, $src\t# int" %}
 9002   ins_encode %{
 9003     __ andl($dst$$Address, $src$$Register);
 9004   %}
 9005   ins_pipe(ialu_mem_reg);
 9006 %}
 9007 
 9008 // And Memory with Immediate
 9009 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9010 %{
 9011   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9012   effect(KILL cr);
 9013   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9014 
 9015   ins_cost(125);
 9016   format %{ "andl    $dst, $src\t# int" %}
 9017   ins_encode %{
 9018     __ andl($dst$$Address, $src$$constant);
 9019   %}
 9020   ins_pipe(ialu_mem_imm);
 9021 %}
 9022 
 9023 // BMI1 instructions
 9024 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9025   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9026   predicate(UseBMI1Instructions);
 9027   effect(KILL cr);
 9028   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9029 
 9030   ins_cost(125);
 9031   format %{ "andnl  $dst, $src1, $src2" %}
 9032 
 9033   ins_encode %{
 9034     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9035   %}
 9036   ins_pipe(ialu_reg_mem);
 9037 %}
 9038 
 9039 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9040   match(Set dst (AndI (XorI src1 minus_1) src2));
 9041   predicate(UseBMI1Instructions);
 9042   effect(KILL cr);
 9043   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9044 
 9045   format %{ "andnl  $dst, $src1, $src2" %}
 9046 
 9047   ins_encode %{
 9048     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9049   %}
 9050   ins_pipe(ialu_reg);
 9051 %}
 9052 
 9053 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9054   match(Set dst (AndI (SubI imm_zero src) src));
 9055   predicate(UseBMI1Instructions);
 9056   effect(KILL cr);
 9057   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9058 
 9059   format %{ "blsil  $dst, $src" %}
 9060 
 9061   ins_encode %{
 9062     __ blsil($dst$$Register, $src$$Register);
 9063   %}
 9064   ins_pipe(ialu_reg);
 9065 %}
 9066 
 9067 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9068   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9069   predicate(UseBMI1Instructions);
 9070   effect(KILL cr);
 9071   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9072 
 9073   ins_cost(125);
 9074   format %{ "blsil  $dst, $src" %}
 9075 
 9076   ins_encode %{
 9077     __ blsil($dst$$Register, $src$$Address);
 9078   %}
 9079   ins_pipe(ialu_reg_mem);
 9080 %}
 9081 
 9082 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9083 %{
 9084   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9085   predicate(UseBMI1Instructions);
 9086   effect(KILL cr);
 9087   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9088 
 9089   ins_cost(125);
 9090   format %{ "blsmskl $dst, $src" %}
 9091 
 9092   ins_encode %{
 9093     __ blsmskl($dst$$Register, $src$$Address);
 9094   %}
 9095   ins_pipe(ialu_reg_mem);
 9096 %}
 9097 
 9098 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9099 %{
 9100   match(Set dst (XorI (AddI src minus_1) src));
 9101   predicate(UseBMI1Instructions);
 9102   effect(KILL cr);
 9103   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9104 
 9105   format %{ "blsmskl $dst, $src" %}
 9106 
 9107   ins_encode %{
 9108     __ blsmskl($dst$$Register, $src$$Register);
 9109   %}
 9110 
 9111   ins_pipe(ialu_reg);
 9112 %}
 9113 
 9114 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9115 %{
 9116   match(Set dst (AndI (AddI src minus_1) src) );
 9117   predicate(UseBMI1Instructions);
 9118   effect(KILL cr);
 9119   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9120 
 9121   format %{ "blsrl  $dst, $src" %}
 9122 
 9123   ins_encode %{
 9124     __ blsrl($dst$$Register, $src$$Register);
 9125   %}
 9126 
 9127   ins_pipe(ialu_reg_mem);
 9128 %}
 9129 
 9130 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9131 %{
 9132   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9133   predicate(UseBMI1Instructions);
 9134   effect(KILL cr);
 9135   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9136 
 9137   ins_cost(125);
 9138   format %{ "blsrl  $dst, $src" %}
 9139 
 9140   ins_encode %{
 9141     __ blsrl($dst$$Register, $src$$Address);
 9142   %}
 9143 
 9144   ins_pipe(ialu_reg);
 9145 %}
 9146 
 9147 // Or Instructions
 9148 // Or Register with Register
 9149 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9150 %{
 9151   match(Set dst (OrI dst src));
 9152   effect(KILL cr);
 9153   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9154 
 9155   format %{ "orl     $dst, $src\t# int" %}
 9156   ins_encode %{
 9157     __ orl($dst$$Register, $src$$Register);
 9158   %}
 9159   ins_pipe(ialu_reg_reg);
 9160 %}
 9161 
 9162 // Or Register with Immediate
 9163 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9164 %{
 9165   match(Set dst (OrI dst src));
 9166   effect(KILL cr);
 9167   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9168 
 9169   format %{ "orl     $dst, $src\t# int" %}
 9170   ins_encode %{
 9171     __ orl($dst$$Register, $src$$constant);
 9172   %}
 9173   ins_pipe(ialu_reg);
 9174 %}
 9175 
 9176 // Or Register with Memory
 9177 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9178 %{
 9179   match(Set dst (OrI dst (LoadI src)));
 9180   effect(KILL cr);
 9181   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9182 
 9183   ins_cost(150);
 9184   format %{ "orl     $dst, $src\t# int" %}
 9185   ins_encode %{
 9186     __ orl($dst$$Register, $src$$Address);
 9187   %}
 9188   ins_pipe(ialu_reg_mem);
 9189 %}
 9190 
 9191 // Or Memory with Register
 9192 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9193 %{
 9194   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9195   effect(KILL cr);
 9196   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9197 
 9198   ins_cost(150);
 9199   format %{ "orb    $dst, $src\t# byte" %}
 9200   ins_encode %{
 9201     __ orb($dst$$Address, $src$$Register);
 9202   %}
 9203   ins_pipe(ialu_mem_reg);
 9204 %}
 9205 
 9206 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9207 %{
 9208   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9209   effect(KILL cr);
 9210   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9211 
 9212   ins_cost(150);
 9213   format %{ "orl     $dst, $src\t# int" %}
 9214   ins_encode %{
 9215     __ orl($dst$$Address, $src$$Register);
 9216   %}
 9217   ins_pipe(ialu_mem_reg);
 9218 %}
 9219 
 9220 // Or Memory with Immediate
 9221 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9222 %{
 9223   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9224   effect(KILL cr);
 9225   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9226 
 9227   ins_cost(125);
 9228   format %{ "orl     $dst, $src\t# int" %}
 9229   ins_encode %{
 9230     __ orl($dst$$Address, $src$$constant);
 9231   %}
 9232   ins_pipe(ialu_mem_imm);
 9233 %}
 9234 
 9235 // Xor Instructions
 9236 // Xor Register with Register
 9237 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9238 %{
 9239   match(Set dst (XorI dst src));
 9240   effect(KILL cr);
 9241   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9242 
 9243   format %{ "xorl    $dst, $src\t# int" %}
 9244   ins_encode %{
 9245     __ xorl($dst$$Register, $src$$Register);
 9246   %}
 9247   ins_pipe(ialu_reg_reg);
 9248 %}
 9249 
 9250 // Xor Register with Immediate -1
 9251 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9252   match(Set dst (XorI dst imm));
 9253 
 9254   format %{ "not    $dst" %}
 9255   ins_encode %{
 9256      __ notl($dst$$Register);
 9257   %}
 9258   ins_pipe(ialu_reg);
 9259 %}
 9260 
 9261 // Xor Register with Immediate
 9262 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9263 %{
 9264   match(Set dst (XorI dst src));
 9265   effect(KILL cr);
 9266   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9267 
 9268   format %{ "xorl    $dst, $src\t# int" %}
 9269   ins_encode %{
 9270     __ xorl($dst$$Register, $src$$constant);
 9271   %}
 9272   ins_pipe(ialu_reg);
 9273 %}
 9274 
 9275 // Xor Register with Memory
 9276 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9277 %{
 9278   match(Set dst (XorI dst (LoadI src)));
 9279   effect(KILL cr);
 9280   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9281 
 9282   ins_cost(150);
 9283   format %{ "xorl    $dst, $src\t# int" %}
 9284   ins_encode %{
 9285     __ xorl($dst$$Register, $src$$Address);
 9286   %}
 9287   ins_pipe(ialu_reg_mem);
 9288 %}
 9289 
 9290 // Xor Memory with Register
 9291 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9292 %{
 9293   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9294   effect(KILL cr);
 9295   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9296 
 9297   ins_cost(150);
 9298   format %{ "xorb    $dst, $src\t# byte" %}
 9299   ins_encode %{
 9300     __ xorb($dst$$Address, $src$$Register);
 9301   %}
 9302   ins_pipe(ialu_mem_reg);
 9303 %}
 9304 
 9305 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9306 %{
 9307   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9308   effect(KILL cr);
 9309   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9310 
 9311   ins_cost(150);
 9312   format %{ "xorl    $dst, $src\t# int" %}
 9313   ins_encode %{
 9314     __ xorl($dst$$Address, $src$$Register);
 9315   %}
 9316   ins_pipe(ialu_mem_reg);
 9317 %}
 9318 
 9319 // Xor Memory with Immediate
 9320 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9321 %{
 9322   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9323   effect(KILL cr);
 9324   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9325 
 9326   ins_cost(125);
 9327   format %{ "xorl    $dst, $src\t# int" %}
 9328   ins_encode %{
 9329     __ xorl($dst$$Address, $src$$constant);
 9330   %}
 9331   ins_pipe(ialu_mem_imm);
 9332 %}
 9333 
 9334 
 9335 // Long Logical Instructions
 9336 
 9337 // And Instructions
 9338 // And Register with Register
 9339 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9340 %{
 9341   match(Set dst (AndL dst src));
 9342   effect(KILL cr);
 9343   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9344 
 9345   format %{ "andq    $dst, $src\t# long" %}
 9346   ins_encode %{
 9347     __ andq($dst$$Register, $src$$Register);
 9348   %}
 9349   ins_pipe(ialu_reg_reg);
 9350 %}
 9351 
 9352 // And Register with Immediate 255
 9353 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
 9354 %{
 9355   match(Set dst (AndL src mask));
 9356 
 9357   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
 9358   ins_encode %{
 9359     // movzbl zeroes out the upper 32-bit and does not need REX.W
 9360     __ movzbl($dst$$Register, $src$$Register);
 9361   %}
 9362   ins_pipe(ialu_reg);
 9363 %}
 9364 
 9365 // And Register with Immediate 65535
 9366 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
 9367 %{
 9368   match(Set dst (AndL src mask));
 9369 
 9370   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
 9371   ins_encode %{
 9372     // movzwl zeroes out the upper 32-bit and does not need REX.W
 9373     __ movzwl($dst$$Register, $src$$Register);
 9374   %}
 9375   ins_pipe(ialu_reg);
 9376 %}
 9377 
 9378 // And Register with Immediate
 9379 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9380 %{
 9381   match(Set dst (AndL dst src));
 9382   effect(KILL cr);
 9383   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9384 
 9385   format %{ "andq    $dst, $src\t# long" %}
 9386   ins_encode %{
 9387     __ andq($dst$$Register, $src$$constant);
 9388   %}
 9389   ins_pipe(ialu_reg);
 9390 %}
 9391 
 9392 // And Register with Memory
 9393 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9394 %{
 9395   match(Set dst (AndL dst (LoadL src)));
 9396   effect(KILL cr);
 9397   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9398 
 9399   ins_cost(150);
 9400   format %{ "andq    $dst, $src\t# long" %}
 9401   ins_encode %{
 9402     __ andq($dst$$Register, $src$$Address);
 9403   %}
 9404   ins_pipe(ialu_reg_mem);
 9405 %}
 9406 
 9407 // And Memory with Register
 9408 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9409 %{
 9410   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9411   effect(KILL cr);
 9412   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9413 
 9414   ins_cost(150);
 9415   format %{ "andq    $dst, $src\t# long" %}
 9416   ins_encode %{
 9417     __ andq($dst$$Address, $src$$Register);
 9418   %}
 9419   ins_pipe(ialu_mem_reg);
 9420 %}
 9421 
 9422 // And Memory with Immediate
 9423 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9424 %{
 9425   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9426   effect(KILL cr);
 9427   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9428 
 9429   ins_cost(125);
 9430   format %{ "andq    $dst, $src\t# long" %}
 9431   ins_encode %{
 9432     __ andq($dst$$Address, $src$$constant);
 9433   %}
 9434   ins_pipe(ialu_mem_imm);
 9435 %}
 9436 
 9437 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9438 %{
 9439   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9440   // because AND/OR works well enough for 8/32-bit values.
 9441   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9442 
 9443   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9444   effect(KILL cr);
 9445 
 9446   ins_cost(125);
 9447   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9448   ins_encode %{
 9449     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9450   %}
 9451   ins_pipe(ialu_mem_imm);
 9452 %}
 9453 
 9454 // BMI1 instructions
 9455 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9456   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9457   predicate(UseBMI1Instructions);
 9458   effect(KILL cr);
 9459   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9460 
 9461   ins_cost(125);
 9462   format %{ "andnq  $dst, $src1, $src2" %}
 9463 
 9464   ins_encode %{
 9465     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9466   %}
 9467   ins_pipe(ialu_reg_mem);
 9468 %}
 9469 
 9470 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9471   match(Set dst (AndL (XorL src1 minus_1) src2));
 9472   predicate(UseBMI1Instructions);
 9473   effect(KILL cr);
 9474   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9475 
 9476   format %{ "andnq  $dst, $src1, $src2" %}
 9477 
 9478   ins_encode %{
 9479   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9480   %}
 9481   ins_pipe(ialu_reg_mem);
 9482 %}
 9483 
 9484 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9485   match(Set dst (AndL (SubL imm_zero src) src));
 9486   predicate(UseBMI1Instructions);
 9487   effect(KILL cr);
 9488   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9489 
 9490   format %{ "blsiq  $dst, $src" %}
 9491 
 9492   ins_encode %{
 9493     __ blsiq($dst$$Register, $src$$Register);
 9494   %}
 9495   ins_pipe(ialu_reg);
 9496 %}
 9497 
 9498 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9499   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9500   predicate(UseBMI1Instructions);
 9501   effect(KILL cr);
 9502   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9503 
 9504   ins_cost(125);
 9505   format %{ "blsiq  $dst, $src" %}
 9506 
 9507   ins_encode %{
 9508     __ blsiq($dst$$Register, $src$$Address);
 9509   %}
 9510   ins_pipe(ialu_reg_mem);
 9511 %}
 9512 
 9513 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9514 %{
 9515   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9516   predicate(UseBMI1Instructions);
 9517   effect(KILL cr);
 9518   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9519 
 9520   ins_cost(125);
 9521   format %{ "blsmskq $dst, $src" %}
 9522 
 9523   ins_encode %{
 9524     __ blsmskq($dst$$Register, $src$$Address);
 9525   %}
 9526   ins_pipe(ialu_reg_mem);
 9527 %}
 9528 
 9529 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9530 %{
 9531   match(Set dst (XorL (AddL src minus_1) src));
 9532   predicate(UseBMI1Instructions);
 9533   effect(KILL cr);
 9534   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9535 
 9536   format %{ "blsmskq $dst, $src" %}
 9537 
 9538   ins_encode %{
 9539     __ blsmskq($dst$$Register, $src$$Register);
 9540   %}
 9541 
 9542   ins_pipe(ialu_reg);
 9543 %}
 9544 
 9545 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9546 %{
 9547   match(Set dst (AndL (AddL src minus_1) src) );
 9548   predicate(UseBMI1Instructions);
 9549   effect(KILL cr);
 9550   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9551 
 9552   format %{ "blsrq  $dst, $src" %}
 9553 
 9554   ins_encode %{
 9555     __ blsrq($dst$$Register, $src$$Register);
 9556   %}
 9557 
 9558   ins_pipe(ialu_reg);
 9559 %}
 9560 
 9561 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9562 %{
 9563   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9564   predicate(UseBMI1Instructions);
 9565   effect(KILL cr);
 9566   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9567 
 9568   ins_cost(125);
 9569   format %{ "blsrq  $dst, $src" %}
 9570 
 9571   ins_encode %{
 9572     __ blsrq($dst$$Register, $src$$Address);
 9573   %}
 9574 
 9575   ins_pipe(ialu_reg);
 9576 %}
 9577 
 9578 // Or Instructions
 9579 // Or Register with Register
 9580 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9581 %{
 9582   match(Set dst (OrL dst src));
 9583   effect(KILL cr);
 9584   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9585 
 9586   format %{ "orq     $dst, $src\t# long" %}
 9587   ins_encode %{
 9588     __ orq($dst$$Register, $src$$Register);
 9589   %}
 9590   ins_pipe(ialu_reg_reg);
 9591 %}
 9592 
 9593 // Use any_RegP to match R15 (TLS register) without spilling.
 9594 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 9595   match(Set dst (OrL dst (CastP2X src)));
 9596   effect(KILL cr);
 9597   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9598 
 9599   format %{ "orq     $dst, $src\t# long" %}
 9600   ins_encode %{
 9601     __ orq($dst$$Register, $src$$Register);
 9602   %}
 9603   ins_pipe(ialu_reg_reg);
 9604 %}
 9605 
 9606 
 9607 // Or Register with Immediate
 9608 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9609 %{
 9610   match(Set dst (OrL dst src));
 9611   effect(KILL cr);
 9612   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9613 
 9614   format %{ "orq     $dst, $src\t# long" %}
 9615   ins_encode %{
 9616     __ orq($dst$$Register, $src$$constant);
 9617   %}
 9618   ins_pipe(ialu_reg);
 9619 %}
 9620 
 9621 // Or Register with Memory
 9622 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9623 %{
 9624   match(Set dst (OrL dst (LoadL src)));
 9625   effect(KILL cr);
 9626   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9627 
 9628   ins_cost(150);
 9629   format %{ "orq     $dst, $src\t# long" %}
 9630   ins_encode %{
 9631     __ orq($dst$$Register, $src$$Address);
 9632   %}
 9633   ins_pipe(ialu_reg_mem);
 9634 %}
 9635 
 9636 // Or Memory with Register
 9637 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9638 %{
 9639   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9640   effect(KILL cr);
 9641   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9642 
 9643   ins_cost(150);
 9644   format %{ "orq     $dst, $src\t# long" %}
 9645   ins_encode %{
 9646     __ orq($dst$$Address, $src$$Register);
 9647   %}
 9648   ins_pipe(ialu_mem_reg);
 9649 %}
 9650 
 9651 // Or Memory with Immediate
 9652 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9653 %{
 9654   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9655   effect(KILL cr);
 9656   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9657 
 9658   ins_cost(125);
 9659   format %{ "orq     $dst, $src\t# long" %}
 9660   ins_encode %{
 9661     __ orq($dst$$Address, $src$$constant);
 9662   %}
 9663   ins_pipe(ialu_mem_imm);
 9664 %}
 9665 
 9666 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
 9667 %{
 9668   // con should be a pure 64-bit power of 2 immediate
 9669   // because AND/OR works well enough for 8/32-bit values.
 9670   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
 9671 
 9672   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
 9673   effect(KILL cr);
 9674 
 9675   ins_cost(125);
 9676   format %{ "btsq    $dst, log2($con)\t# long" %}
 9677   ins_encode %{
 9678     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
 9679   %}
 9680   ins_pipe(ialu_mem_imm);
 9681 %}
 9682 
 9683 // Xor Instructions
 9684 // Xor Register with Register
 9685 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9686 %{
 9687   match(Set dst (XorL dst src));
 9688   effect(KILL cr);
 9689   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9690 
 9691   format %{ "xorq    $dst, $src\t# long" %}
 9692   ins_encode %{
 9693     __ xorq($dst$$Register, $src$$Register);
 9694   %}
 9695   ins_pipe(ialu_reg_reg);
 9696 %}
 9697 
 9698 // Xor Register with Immediate -1
 9699 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 9700   match(Set dst (XorL dst imm));
 9701 
 9702   format %{ "notq   $dst" %}
 9703   ins_encode %{
 9704      __ notq($dst$$Register);
 9705   %}
 9706   ins_pipe(ialu_reg);
 9707 %}
 9708 
 9709 // Xor Register with Immediate
 9710 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9711 %{
 9712   match(Set dst (XorL dst src));
 9713   effect(KILL cr);
 9714   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9715 
 9716   format %{ "xorq    $dst, $src\t# long" %}
 9717   ins_encode %{
 9718     __ xorq($dst$$Register, $src$$constant);
 9719   %}
 9720   ins_pipe(ialu_reg);
 9721 %}
 9722 
 9723 // Xor Register with Memory
 9724 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9725 %{
 9726   match(Set dst (XorL dst (LoadL src)));
 9727   effect(KILL cr);
 9728   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9729 
 9730   ins_cost(150);
 9731   format %{ "xorq    $dst, $src\t# long" %}
 9732   ins_encode %{
 9733     __ xorq($dst$$Register, $src$$Address);
 9734   %}
 9735   ins_pipe(ialu_reg_mem);
 9736 %}
 9737 
 9738 // Xor Memory with Register
 9739 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9740 %{
 9741   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9742   effect(KILL cr);
 9743   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9744 
 9745   ins_cost(150);
 9746   format %{ "xorq    $dst, $src\t# long" %}
 9747   ins_encode %{
 9748     __ xorq($dst$$Address, $src$$Register);
 9749   %}
 9750   ins_pipe(ialu_mem_reg);
 9751 %}
 9752 
 9753 // Xor Memory with Immediate
 9754 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9755 %{
 9756   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9757   effect(KILL cr);
 9758   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9759 
 9760   ins_cost(125);
 9761   format %{ "xorq    $dst, $src\t# long" %}
 9762   ins_encode %{
 9763     __ xorq($dst$$Address, $src$$constant);
 9764   %}
 9765   ins_pipe(ialu_mem_imm);
 9766 %}
 9767 
 9768 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 9769 %{
 9770   match(Set dst (CmpLTMask p q));
 9771   effect(KILL cr);
 9772 
 9773   ins_cost(400);
 9774   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 9775             "setlt   $dst\n\t"
 9776             "movzbl  $dst, $dst\n\t"
 9777             "negl    $dst" %}
 9778   ins_encode %{
 9779     __ cmpl($p$$Register, $q$$Register);
 9780     __ setb(Assembler::less, $dst$$Register);
 9781     __ movzbl($dst$$Register, $dst$$Register);
 9782     __ negl($dst$$Register);
 9783   %}
 9784   ins_pipe(pipe_slow);
 9785 %}
 9786 
 9787 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
 9788 %{
 9789   match(Set dst (CmpLTMask dst zero));
 9790   effect(KILL cr);
 9791 
 9792   ins_cost(100);
 9793   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 9794   ins_encode %{
 9795     __ sarl($dst$$Register, 31);
 9796   %}
 9797   ins_pipe(ialu_reg);
 9798 %}
 9799 
 9800 /* Better to save a register than avoid a branch */
 9801 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9802 %{
 9803   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 9804   effect(KILL cr);
 9805   ins_cost(300);
 9806   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
 9807             "jge     done\n\t"
 9808             "addl    $p,$y\n"
 9809             "done:   " %}
 9810   ins_encode %{
 9811     Register Rp = $p$$Register;
 9812     Register Rq = $q$$Register;
 9813     Register Ry = $y$$Register;
 9814     Label done;
 9815     __ subl(Rp, Rq);
 9816     __ jccb(Assembler::greaterEqual, done);
 9817     __ addl(Rp, Ry);
 9818     __ bind(done);
 9819   %}
 9820   ins_pipe(pipe_cmplt);
 9821 %}
 9822 
 9823 /* Better to save a register than avoid a branch */
 9824 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9825 %{
 9826   match(Set y (AndI (CmpLTMask p q) y));
 9827   effect(KILL cr);
 9828 
 9829   ins_cost(300);
 9830 
 9831   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
 9832             "jlt     done\n\t"
 9833             "xorl    $y, $y\n"
 9834             "done:   " %}
 9835   ins_encode %{
 9836     Register Rp = $p$$Register;
 9837     Register Rq = $q$$Register;
 9838     Register Ry = $y$$Register;
 9839     Label done;
 9840     __ cmpl(Rp, Rq);
 9841     __ jccb(Assembler::less, done);
 9842     __ xorl(Ry, Ry);
 9843     __ bind(done);
 9844   %}
 9845   ins_pipe(pipe_cmplt);
 9846 %}
 9847 
 9848 
 9849 //---------- FP Instructions------------------------------------------------
 9850 
 9851 // Really expensive, avoid
 9852 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 9853 %{
 9854   match(Set cr (CmpF src1 src2));
 9855 
 9856   ins_cost(500);
 9857   format %{ "ucomiss $src1, $src2\n\t"
 9858             "jnp,s   exit\n\t"
 9859             "pushfq\t# saw NaN, set CF\n\t"
 9860             "andq    [rsp], #0xffffff2b\n\t"
 9861             "popfq\n"
 9862     "exit:" %}
 9863   ins_encode %{
 9864     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9865     emit_cmpfp_fixup(_masm);
 9866   %}
 9867   ins_pipe(pipe_slow);
 9868 %}
 9869 
 9870 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 9871   match(Set cr (CmpF src1 src2));
 9872 
 9873   ins_cost(100);
 9874   format %{ "ucomiss $src1, $src2" %}
 9875   ins_encode %{
 9876     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9877   %}
 9878   ins_pipe(pipe_slow);
 9879 %}
 9880 
 9881 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 9882   match(Set cr (CmpF src1 (LoadF src2)));
 9883 
 9884   ins_cost(100);
 9885   format %{ "ucomiss $src1, $src2" %}
 9886   ins_encode %{
 9887     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9888   %}
 9889   ins_pipe(pipe_slow);
 9890 %}
 9891 
 9892 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
 9893   match(Set cr (CmpF src con));
 9894   ins_cost(100);
 9895   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
 9896   ins_encode %{
 9897     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9898   %}
 9899   ins_pipe(pipe_slow);
 9900 %}
 9901 
 9902 // Really expensive, avoid
 9903 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 9904 %{
 9905   match(Set cr (CmpD src1 src2));
 9906 
 9907   ins_cost(500);
 9908   format %{ "ucomisd $src1, $src2\n\t"
 9909             "jnp,s   exit\n\t"
 9910             "pushfq\t# saw NaN, set CF\n\t"
 9911             "andq    [rsp], #0xffffff2b\n\t"
 9912             "popfq\n"
 9913     "exit:" %}
 9914   ins_encode %{
 9915     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9916     emit_cmpfp_fixup(_masm);
 9917   %}
 9918   ins_pipe(pipe_slow);
 9919 %}
 9920 
 9921 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 9922   match(Set cr (CmpD src1 src2));
 9923 
 9924   ins_cost(100);
 9925   format %{ "ucomisd $src1, $src2 test" %}
 9926   ins_encode %{
 9927     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9928   %}
 9929   ins_pipe(pipe_slow);
 9930 %}
 9931 
 9932 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 9933   match(Set cr (CmpD src1 (LoadD src2)));
 9934 
 9935   ins_cost(100);
 9936   format %{ "ucomisd $src1, $src2" %}
 9937   ins_encode %{
 9938     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9939   %}
 9940   ins_pipe(pipe_slow);
 9941 %}
 9942 
 9943 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
 9944   match(Set cr (CmpD src con));
 9945   ins_cost(100);
 9946   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
 9947   ins_encode %{
 9948     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9949   %}
 9950   ins_pipe(pipe_slow);
 9951 %}
 9952 
 9953 // Compare into -1,0,1
 9954 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
 9955 %{
 9956   match(Set dst (CmpF3 src1 src2));
 9957   effect(KILL cr);
 9958 
 9959   ins_cost(275);
 9960   format %{ "ucomiss $src1, $src2\n\t"
 9961             "movl    $dst, #-1\n\t"
 9962             "jp,s    done\n\t"
 9963             "jb,s    done\n\t"
 9964             "setne   $dst\n\t"
 9965             "movzbl  $dst, $dst\n"
 9966     "done:" %}
 9967   ins_encode %{
 9968     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9969     emit_cmpfp3(_masm, $dst$$Register);
 9970   %}
 9971   ins_pipe(pipe_slow);
 9972 %}
 9973 
 9974 // Compare into -1,0,1
 9975 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 9976 %{
 9977   match(Set dst (CmpF3 src1 (LoadF src2)));
 9978   effect(KILL cr);
 9979 
 9980   ins_cost(275);
 9981   format %{ "ucomiss $src1, $src2\n\t"
 9982             "movl    $dst, #-1\n\t"
 9983             "jp,s    done\n\t"
 9984             "jb,s    done\n\t"
 9985             "setne   $dst\n\t"
 9986             "movzbl  $dst, $dst\n"
 9987     "done:" %}
 9988   ins_encode %{
 9989     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9990     emit_cmpfp3(_masm, $dst$$Register);
 9991   %}
 9992   ins_pipe(pipe_slow);
 9993 %}
 9994 
 9995 // Compare into -1,0,1
 9996 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
 9997   match(Set dst (CmpF3 src con));
 9998   effect(KILL cr);
 9999 
10000   ins_cost(275);
10001   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10002             "movl    $dst, #-1\n\t"
10003             "jp,s    done\n\t"
10004             "jb,s    done\n\t"
10005             "setne   $dst\n\t"
10006             "movzbl  $dst, $dst\n"
10007     "done:" %}
10008   ins_encode %{
10009     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10010     emit_cmpfp3(_masm, $dst$$Register);
10011   %}
10012   ins_pipe(pipe_slow);
10013 %}
10014 
10015 // Compare into -1,0,1
10016 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10017 %{
10018   match(Set dst (CmpD3 src1 src2));
10019   effect(KILL cr);
10020 
10021   ins_cost(275);
10022   format %{ "ucomisd $src1, $src2\n\t"
10023             "movl    $dst, #-1\n\t"
10024             "jp,s    done\n\t"
10025             "jb,s    done\n\t"
10026             "setne   $dst\n\t"
10027             "movzbl  $dst, $dst\n"
10028     "done:" %}
10029   ins_encode %{
10030     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10031     emit_cmpfp3(_masm, $dst$$Register);
10032   %}
10033   ins_pipe(pipe_slow);
10034 %}
10035 
10036 // Compare into -1,0,1
10037 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10038 %{
10039   match(Set dst (CmpD3 src1 (LoadD src2)));
10040   effect(KILL cr);
10041 
10042   ins_cost(275);
10043   format %{ "ucomisd $src1, $src2\n\t"
10044             "movl    $dst, #-1\n\t"
10045             "jp,s    done\n\t"
10046             "jb,s    done\n\t"
10047             "setne   $dst\n\t"
10048             "movzbl  $dst, $dst\n"
10049     "done:" %}
10050   ins_encode %{
10051     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10052     emit_cmpfp3(_masm, $dst$$Register);
10053   %}
10054   ins_pipe(pipe_slow);
10055 %}
10056 
10057 // Compare into -1,0,1
10058 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10059   match(Set dst (CmpD3 src con));
10060   effect(KILL cr);
10061 
10062   ins_cost(275);
10063   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10064             "movl    $dst, #-1\n\t"
10065             "jp,s    done\n\t"
10066             "jb,s    done\n\t"
10067             "setne   $dst\n\t"
10068             "movzbl  $dst, $dst\n"
10069     "done:" %}
10070   ins_encode %{
10071     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10072     emit_cmpfp3(_masm, $dst$$Register);
10073   %}
10074   ins_pipe(pipe_slow);
10075 %}
10076 
10077 //----------Arithmetic Conversion Instructions---------------------------------
10078 
10079 instruct convF2D_reg_reg(regD dst, regF src)
10080 %{
10081   match(Set dst (ConvF2D src));
10082 
10083   format %{ "cvtss2sd $dst, $src" %}
10084   ins_encode %{
10085     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10086   %}
10087   ins_pipe(pipe_slow); // XXX
10088 %}
10089 
10090 instruct convF2D_reg_mem(regD dst, memory src)
10091 %{
10092   match(Set dst (ConvF2D (LoadF src)));
10093 
10094   format %{ "cvtss2sd $dst, $src" %}
10095   ins_encode %{
10096     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10097   %}
10098   ins_pipe(pipe_slow); // XXX
10099 %}
10100 
10101 instruct convD2F_reg_reg(regF dst, regD src)
10102 %{
10103   match(Set dst (ConvD2F src));
10104 
10105   format %{ "cvtsd2ss $dst, $src" %}
10106   ins_encode %{
10107     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10108   %}
10109   ins_pipe(pipe_slow); // XXX
10110 %}
10111 
10112 instruct convD2F_reg_mem(regF dst, memory src)
10113 %{
10114   match(Set dst (ConvD2F (LoadD src)));
10115 
10116   format %{ "cvtsd2ss $dst, $src" %}
10117   ins_encode %{
10118     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10119   %}
10120   ins_pipe(pipe_slow); // XXX
10121 %}
10122 
10123 // XXX do mem variants
10124 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10125 %{
10126   match(Set dst (ConvF2I src));
10127   effect(KILL cr);
10128   format %{ "convert_f2i $dst, $src" %}
10129   ins_encode %{
10130     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10131   %}
10132   ins_pipe(pipe_slow);
10133 %}
10134 
10135 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10136 %{
10137   match(Set dst (ConvF2L src));
10138   effect(KILL cr);
10139   format %{ "convert_f2l $dst, $src"%}
10140   ins_encode %{
10141     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10142   %}
10143   ins_pipe(pipe_slow);
10144 %}
10145 
10146 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10147 %{
10148   match(Set dst (ConvD2I src));
10149   effect(KILL cr);
10150   format %{ "convert_d2i $dst, $src"%}
10151   ins_encode %{
10152     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10153   %}
10154   ins_pipe(pipe_slow);
10155 %}
10156 
10157 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10158 %{
10159   match(Set dst (ConvD2L src));
10160   effect(KILL cr);
10161   format %{ "convert_d2l $dst, $src"%}
10162   ins_encode %{
10163     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10164   %}
10165   ins_pipe(pipe_slow);
10166 %}
10167 
10168 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10169 %{
10170   match(Set dst (RoundD src));
10171   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10172   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10173   ins_encode %{
10174     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10175   %}
10176   ins_pipe(pipe_slow);
10177 %}
10178 
10179 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10180 %{
10181   match(Set dst (RoundF src));
10182   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10183   format %{ "round_float $dst,$src" %}
10184   ins_encode %{
10185     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10186   %}
10187   ins_pipe(pipe_slow);
10188 %}
10189 
10190 instruct convI2F_reg_reg(regF dst, rRegI src)
10191 %{
10192   predicate(!UseXmmI2F);
10193   match(Set dst (ConvI2F src));
10194 
10195   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10196   ins_encode %{
10197     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10198   %}
10199   ins_pipe(pipe_slow); // XXX
10200 %}
10201 
10202 instruct convI2F_reg_mem(regF dst, memory src)
10203 %{
10204   match(Set dst (ConvI2F (LoadI src)));
10205 
10206   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10207   ins_encode %{
10208     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10209   %}
10210   ins_pipe(pipe_slow); // XXX
10211 %}
10212 
10213 instruct convI2D_reg_reg(regD dst, rRegI src)
10214 %{
10215   predicate(!UseXmmI2D);
10216   match(Set dst (ConvI2D src));
10217 
10218   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10219   ins_encode %{
10220     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10221   %}
10222   ins_pipe(pipe_slow); // XXX
10223 %}
10224 
10225 instruct convI2D_reg_mem(regD dst, memory src)
10226 %{
10227   match(Set dst (ConvI2D (LoadI src)));
10228 
10229   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10230   ins_encode %{
10231     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10232   %}
10233   ins_pipe(pipe_slow); // XXX
10234 %}
10235 
10236 instruct convXI2F_reg(regF dst, rRegI src)
10237 %{
10238   predicate(UseXmmI2F);
10239   match(Set dst (ConvI2F src));
10240 
10241   format %{ "movdl $dst, $src\n\t"
10242             "cvtdq2psl $dst, $dst\t# i2f" %}
10243   ins_encode %{
10244     __ movdl($dst$$XMMRegister, $src$$Register);
10245     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10246   %}
10247   ins_pipe(pipe_slow); // XXX
10248 %}
10249 
10250 instruct convXI2D_reg(regD dst, rRegI src)
10251 %{
10252   predicate(UseXmmI2D);
10253   match(Set dst (ConvI2D src));
10254 
10255   format %{ "movdl $dst, $src\n\t"
10256             "cvtdq2pdl $dst, $dst\t# i2d" %}
10257   ins_encode %{
10258     __ movdl($dst$$XMMRegister, $src$$Register);
10259     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10260   %}
10261   ins_pipe(pipe_slow); // XXX
10262 %}
10263 
10264 instruct convL2F_reg_reg(regF dst, rRegL src)
10265 %{
10266   match(Set dst (ConvL2F src));
10267 
10268   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10269   ins_encode %{
10270     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10271   %}
10272   ins_pipe(pipe_slow); // XXX
10273 %}
10274 
10275 instruct convL2F_reg_mem(regF dst, memory src)
10276 %{
10277   match(Set dst (ConvL2F (LoadL src)));
10278 
10279   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10280   ins_encode %{
10281     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10282   %}
10283   ins_pipe(pipe_slow); // XXX
10284 %}
10285 
10286 instruct convL2D_reg_reg(regD dst, rRegL src)
10287 %{
10288   match(Set dst (ConvL2D src));
10289 
10290   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10291   ins_encode %{
10292     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10293   %}
10294   ins_pipe(pipe_slow); // XXX
10295 %}
10296 
10297 instruct convL2D_reg_mem(regD dst, memory src)
10298 %{
10299   match(Set dst (ConvL2D (LoadL src)));
10300 
10301   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10302   ins_encode %{
10303     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10304   %}
10305   ins_pipe(pipe_slow); // XXX
10306 %}
10307 
10308 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10309 %{
10310   match(Set dst (ConvI2L src));
10311 
10312   ins_cost(125);
10313   format %{ "movslq  $dst, $src\t# i2l" %}
10314   ins_encode %{
10315     __ movslq($dst$$Register, $src$$Register);
10316   %}
10317   ins_pipe(ialu_reg_reg);
10318 %}
10319 
10320 // Zero-extend convert int to long
10321 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10322 %{
10323   match(Set dst (AndL (ConvI2L src) mask));
10324 
10325   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10326   ins_encode %{
10327     if ($dst$$reg != $src$$reg) {
10328       __ movl($dst$$Register, $src$$Register);
10329     }
10330   %}
10331   ins_pipe(ialu_reg_reg);
10332 %}
10333 
10334 // Zero-extend convert int to long
10335 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10336 %{
10337   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10338 
10339   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10340   ins_encode %{
10341     __ movl($dst$$Register, $src$$Address);
10342   %}
10343   ins_pipe(ialu_reg_mem);
10344 %}
10345 
10346 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10347 %{
10348   match(Set dst (AndL src mask));
10349 
10350   format %{ "movl    $dst, $src\t# zero-extend long" %}
10351   ins_encode %{
10352     __ movl($dst$$Register, $src$$Register);
10353   %}
10354   ins_pipe(ialu_reg_reg);
10355 %}
10356 
10357 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10358 %{
10359   match(Set dst (ConvL2I src));
10360 
10361   format %{ "movl    $dst, $src\t# l2i" %}
10362   ins_encode %{
10363     __ movl($dst$$Register, $src$$Register);
10364   %}
10365   ins_pipe(ialu_reg_reg);
10366 %}
10367 
10368 
10369 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10370   match(Set dst (MoveF2I src));
10371   effect(DEF dst, USE src);
10372 
10373   ins_cost(125);
10374   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10375   ins_encode %{
10376     __ movl($dst$$Register, Address(rsp, $src$$disp));
10377   %}
10378   ins_pipe(ialu_reg_mem);
10379 %}
10380 
10381 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10382   match(Set dst (MoveI2F src));
10383   effect(DEF dst, USE src);
10384 
10385   ins_cost(125);
10386   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10387   ins_encode %{
10388     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10389   %}
10390   ins_pipe(pipe_slow);
10391 %}
10392 
10393 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10394   match(Set dst (MoveD2L src));
10395   effect(DEF dst, USE src);
10396 
10397   ins_cost(125);
10398   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10399   ins_encode %{
10400     __ movq($dst$$Register, Address(rsp, $src$$disp));
10401   %}
10402   ins_pipe(ialu_reg_mem);
10403 %}
10404 
10405 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10406   predicate(!UseXmmLoadAndClearUpper);
10407   match(Set dst (MoveL2D src));
10408   effect(DEF dst, USE src);
10409 
10410   ins_cost(125);
10411   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10412   ins_encode %{
10413     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10414   %}
10415   ins_pipe(pipe_slow);
10416 %}
10417 
10418 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10419   predicate(UseXmmLoadAndClearUpper);
10420   match(Set dst (MoveL2D src));
10421   effect(DEF dst, USE src);
10422 
10423   ins_cost(125);
10424   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10425   ins_encode %{
10426     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10427   %}
10428   ins_pipe(pipe_slow);
10429 %}
10430 
10431 
10432 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10433   match(Set dst (MoveF2I src));
10434   effect(DEF dst, USE src);
10435 
10436   ins_cost(95); // XXX
10437   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10438   ins_encode %{
10439     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10440   %}
10441   ins_pipe(pipe_slow);
10442 %}
10443 
10444 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10445   match(Set dst (MoveI2F src));
10446   effect(DEF dst, USE src);
10447 
10448   ins_cost(100);
10449   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10450   ins_encode %{
10451     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10452   %}
10453   ins_pipe( ialu_mem_reg );
10454 %}
10455 
10456 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10457   match(Set dst (MoveD2L src));
10458   effect(DEF dst, USE src);
10459 
10460   ins_cost(95); // XXX
10461   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10462   ins_encode %{
10463     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10464   %}
10465   ins_pipe(pipe_slow);
10466 %}
10467 
10468 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10469   match(Set dst (MoveL2D src));
10470   effect(DEF dst, USE src);
10471 
10472   ins_cost(100);
10473   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10474   ins_encode %{
10475     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10476   %}
10477   ins_pipe(ialu_mem_reg);
10478 %}
10479 
10480 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10481   match(Set dst (MoveF2I src));
10482   effect(DEF dst, USE src);
10483   ins_cost(85);
10484   format %{ "movd    $dst,$src\t# MoveF2I" %}
10485   ins_encode %{
10486     __ movdl($dst$$Register, $src$$XMMRegister);
10487   %}
10488   ins_pipe( pipe_slow );
10489 %}
10490 
10491 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10492   match(Set dst (MoveD2L src));
10493   effect(DEF dst, USE src);
10494   ins_cost(85);
10495   format %{ "movd    $dst,$src\t# MoveD2L" %}
10496   ins_encode %{
10497     __ movdq($dst$$Register, $src$$XMMRegister);
10498   %}
10499   ins_pipe( pipe_slow );
10500 %}
10501 
10502 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10503   match(Set dst (MoveI2F src));
10504   effect(DEF dst, USE src);
10505   ins_cost(100);
10506   format %{ "movd    $dst,$src\t# MoveI2F" %}
10507   ins_encode %{
10508     __ movdl($dst$$XMMRegister, $src$$Register);
10509   %}
10510   ins_pipe( pipe_slow );
10511 %}
10512 
10513 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10514   match(Set dst (MoveL2D src));
10515   effect(DEF dst, USE src);
10516   ins_cost(100);
10517   format %{ "movd    $dst,$src\t# MoveL2D" %}
10518   ins_encode %{
10519      __ movdq($dst$$XMMRegister, $src$$Register);
10520   %}
10521   ins_pipe( pipe_slow );
10522 %}
10523 
10524 // Fast clearing of an array
10525 // Small ClearArray non-AVX512.
10526 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10527                   Universe dummy, rFlagsReg cr)
10528 %{
10529   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
10530   match(Set dummy (ClearArray cnt base));
10531   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10532 
10533   format %{ $$template
10534     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10535     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10536     $$emit$$"jg      LARGE\n\t"
10537     $$emit$$"dec     rcx\n\t"
10538     $$emit$$"js      DONE\t# Zero length\n\t"
10539     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10540     $$emit$$"dec     rcx\n\t"
10541     $$emit$$"jge     LOOP\n\t"
10542     $$emit$$"jmp     DONE\n\t"
10543     $$emit$$"# LARGE:\n\t"
10544     if (UseFastStosb) {
10545        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10546        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10547     } else if (UseXMMForObjInit) {
10548        $$emit$$"mov     rdi,rax\n\t"
10549        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10550        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10551        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10552        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10553        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10554        $$emit$$"add     0x40,rax\n\t"
10555        $$emit$$"# L_zero_64_bytes:\n\t"
10556        $$emit$$"sub     0x8,rcx\n\t"
10557        $$emit$$"jge     L_loop\n\t"
10558        $$emit$$"add     0x4,rcx\n\t"
10559        $$emit$$"jl      L_tail\n\t"
10560        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10561        $$emit$$"add     0x20,rax\n\t"
10562        $$emit$$"sub     0x4,rcx\n\t"
10563        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10564        $$emit$$"add     0x4,rcx\n\t"
10565        $$emit$$"jle     L_end\n\t"
10566        $$emit$$"dec     rcx\n\t"
10567        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10568        $$emit$$"vmovq   xmm0,(rax)\n\t"
10569        $$emit$$"add     0x8,rax\n\t"
10570        $$emit$$"dec     rcx\n\t"
10571        $$emit$$"jge     L_sloop\n\t"
10572        $$emit$$"# L_end:\n\t"
10573     } else {
10574        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10575     }
10576     $$emit$$"# DONE"
10577   %}
10578   ins_encode %{
10579     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10580                  $tmp$$XMMRegister, false, knoreg);
10581   %}
10582   ins_pipe(pipe_slow);
10583 %}
10584 
10585 // Small ClearArray AVX512 non-constant length.
10586 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10587                        Universe dummy, rFlagsReg cr)
10588 %{
10589   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
10590   match(Set dummy (ClearArray cnt base));
10591   ins_cost(125);
10592   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10593 
10594   format %{ $$template
10595     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10596     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10597     $$emit$$"jg      LARGE\n\t"
10598     $$emit$$"dec     rcx\n\t"
10599     $$emit$$"js      DONE\t# Zero length\n\t"
10600     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10601     $$emit$$"dec     rcx\n\t"
10602     $$emit$$"jge     LOOP\n\t"
10603     $$emit$$"jmp     DONE\n\t"
10604     $$emit$$"# LARGE:\n\t"
10605     if (UseFastStosb) {
10606        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10607        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10608     } else if (UseXMMForObjInit) {
10609        $$emit$$"mov     rdi,rax\n\t"
10610        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10611        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10612        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10613        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10614        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10615        $$emit$$"add     0x40,rax\n\t"
10616        $$emit$$"# L_zero_64_bytes:\n\t"
10617        $$emit$$"sub     0x8,rcx\n\t"
10618        $$emit$$"jge     L_loop\n\t"
10619        $$emit$$"add     0x4,rcx\n\t"
10620        $$emit$$"jl      L_tail\n\t"
10621        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10622        $$emit$$"add     0x20,rax\n\t"
10623        $$emit$$"sub     0x4,rcx\n\t"
10624        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10625        $$emit$$"add     0x4,rcx\n\t"
10626        $$emit$$"jle     L_end\n\t"
10627        $$emit$$"dec     rcx\n\t"
10628        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10629        $$emit$$"vmovq   xmm0,(rax)\n\t"
10630        $$emit$$"add     0x8,rax\n\t"
10631        $$emit$$"dec     rcx\n\t"
10632        $$emit$$"jge     L_sloop\n\t"
10633        $$emit$$"# L_end:\n\t"
10634     } else {
10635        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10636     }
10637     $$emit$$"# DONE"
10638   %}
10639   ins_encode %{
10640     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10641                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
10642   %}
10643   ins_pipe(pipe_slow);
10644 %}
10645 
10646 // Large ClearArray non-AVX512.
10647 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10648                         Universe dummy, rFlagsReg cr)
10649 %{
10650   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
10651   match(Set dummy (ClearArray cnt base));
10652   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10653 
10654   format %{ $$template
10655     if (UseFastStosb) {
10656        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10657        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10658        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10659     } else if (UseXMMForObjInit) {
10660        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10661        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10662        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10663        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10664        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10665        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10666        $$emit$$"add     0x40,rax\n\t"
10667        $$emit$$"# L_zero_64_bytes:\n\t"
10668        $$emit$$"sub     0x8,rcx\n\t"
10669        $$emit$$"jge     L_loop\n\t"
10670        $$emit$$"add     0x4,rcx\n\t"
10671        $$emit$$"jl      L_tail\n\t"
10672        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10673        $$emit$$"add     0x20,rax\n\t"
10674        $$emit$$"sub     0x4,rcx\n\t"
10675        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10676        $$emit$$"add     0x4,rcx\n\t"
10677        $$emit$$"jle     L_end\n\t"
10678        $$emit$$"dec     rcx\n\t"
10679        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10680        $$emit$$"vmovq   xmm0,(rax)\n\t"
10681        $$emit$$"add     0x8,rax\n\t"
10682        $$emit$$"dec     rcx\n\t"
10683        $$emit$$"jge     L_sloop\n\t"
10684        $$emit$$"# L_end:\n\t"
10685     } else {
10686        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10687        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10688     }
10689   %}
10690   ins_encode %{
10691     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10692                  $tmp$$XMMRegister, true, knoreg);
10693   %}
10694   ins_pipe(pipe_slow);
10695 %}
10696 
10697 // Large ClearArray AVX512.
10698 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10699                              Universe dummy, rFlagsReg cr)
10700 %{
10701   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
10702   match(Set dummy (ClearArray cnt base));
10703   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10704 
10705   format %{ $$template
10706     if (UseFastStosb) {
10707        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10708        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10709        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10710     } else if (UseXMMForObjInit) {
10711        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10712        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10713        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10714        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10715        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10716        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10717        $$emit$$"add     0x40,rax\n\t"
10718        $$emit$$"# L_zero_64_bytes:\n\t"
10719        $$emit$$"sub     0x8,rcx\n\t"
10720        $$emit$$"jge     L_loop\n\t"
10721        $$emit$$"add     0x4,rcx\n\t"
10722        $$emit$$"jl      L_tail\n\t"
10723        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10724        $$emit$$"add     0x20,rax\n\t"
10725        $$emit$$"sub     0x4,rcx\n\t"
10726        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10727        $$emit$$"add     0x4,rcx\n\t"
10728        $$emit$$"jle     L_end\n\t"
10729        $$emit$$"dec     rcx\n\t"
10730        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10731        $$emit$$"vmovq   xmm0,(rax)\n\t"
10732        $$emit$$"add     0x8,rax\n\t"
10733        $$emit$$"dec     rcx\n\t"
10734        $$emit$$"jge     L_sloop\n\t"
10735        $$emit$$"# L_end:\n\t"
10736     } else {
10737        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10738        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10739     }
10740   %}
10741   ins_encode %{
10742     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10743                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
10744   %}
10745   ins_pipe(pipe_slow);
10746 %}
10747 
10748 // Small ClearArray AVX512 constant length.
10749 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
10750 %{
10751   predicate(!((ClearArrayNode*)n)->is_large() &&
10752               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
10753   match(Set dummy (ClearArray cnt base));
10754   ins_cost(100);
10755   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
10756   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
10757   ins_encode %{
10758    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10759   %}
10760   ins_pipe(pipe_slow);
10761 %}
10762 
10763 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10764                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10765 %{
10766   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10767   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10768   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10769 
10770   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10771   ins_encode %{
10772     __ string_compare($str1$$Register, $str2$$Register,
10773                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10774                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
10775   %}
10776   ins_pipe( pipe_slow );
10777 %}
10778 
10779 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10780                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10781 %{
10782   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10783   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10784   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10785 
10786   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10787   ins_encode %{
10788     __ string_compare($str1$$Register, $str2$$Register,
10789                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10790                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
10791   %}
10792   ins_pipe( pipe_slow );
10793 %}
10794 
10795 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10796                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10797 %{
10798   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10799   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10800   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10801 
10802   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10803   ins_encode %{
10804     __ string_compare($str1$$Register, $str2$$Register,
10805                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10806                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
10807   %}
10808   ins_pipe( pipe_slow );
10809 %}
10810 
10811 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10812                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10813 %{
10814   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10815   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10816   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10817 
10818   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10819   ins_encode %{
10820     __ string_compare($str1$$Register, $str2$$Register,
10821                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10822                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
10823   %}
10824   ins_pipe( pipe_slow );
10825 %}
10826 
10827 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10828                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10829 %{
10830   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10831   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10832   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10833 
10834   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10835   ins_encode %{
10836     __ string_compare($str1$$Register, $str2$$Register,
10837                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10838                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
10839   %}
10840   ins_pipe( pipe_slow );
10841 %}
10842 
10843 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10844                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10845 %{
10846   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10847   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10848   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10849 
10850   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10851   ins_encode %{
10852     __ string_compare($str1$$Register, $str2$$Register,
10853                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10854                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
10855   %}
10856   ins_pipe( pipe_slow );
10857 %}
10858 
10859 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10860                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10861 %{
10862   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10863   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10864   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10865 
10866   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10867   ins_encode %{
10868     __ string_compare($str2$$Register, $str1$$Register,
10869                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10870                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
10871   %}
10872   ins_pipe( pipe_slow );
10873 %}
10874 
10875 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10876                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10877 %{
10878   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10879   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10880   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10881 
10882   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10883   ins_encode %{
10884     __ string_compare($str2$$Register, $str1$$Register,
10885                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10886                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
10887   %}
10888   ins_pipe( pipe_slow );
10889 %}
10890 
10891 // fast search of substring with known size.
10892 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10893                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10894 %{
10895   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10896   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10897   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10898 
10899   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10900   ins_encode %{
10901     int icnt2 = (int)$int_cnt2$$constant;
10902     if (icnt2 >= 16) {
10903       // IndexOf for constant substrings with size >= 16 elements
10904       // which don't need to be loaded through stack.
10905       __ string_indexofC8($str1$$Register, $str2$$Register,
10906                           $cnt1$$Register, $cnt2$$Register,
10907                           icnt2, $result$$Register,
10908                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10909     } else {
10910       // Small strings are loaded through stack if they cross page boundary.
10911       __ string_indexof($str1$$Register, $str2$$Register,
10912                         $cnt1$$Register, $cnt2$$Register,
10913                         icnt2, $result$$Register,
10914                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10915     }
10916   %}
10917   ins_pipe( pipe_slow );
10918 %}
10919 
10920 // fast search of substring with known size.
10921 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10922                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10923 %{
10924   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10925   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10926   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10927 
10928   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10929   ins_encode %{
10930     int icnt2 = (int)$int_cnt2$$constant;
10931     if (icnt2 >= 8) {
10932       // IndexOf for constant substrings with size >= 8 elements
10933       // which don't need to be loaded through stack.
10934       __ string_indexofC8($str1$$Register, $str2$$Register,
10935                           $cnt1$$Register, $cnt2$$Register,
10936                           icnt2, $result$$Register,
10937                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10938     } else {
10939       // Small strings are loaded through stack if they cross page boundary.
10940       __ string_indexof($str1$$Register, $str2$$Register,
10941                         $cnt1$$Register, $cnt2$$Register,
10942                         icnt2, $result$$Register,
10943                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10944     }
10945   %}
10946   ins_pipe( pipe_slow );
10947 %}
10948 
10949 // fast search of substring with known size.
10950 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10951                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10952 %{
10953   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10954   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10955   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10956 
10957   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10958   ins_encode %{
10959     int icnt2 = (int)$int_cnt2$$constant;
10960     if (icnt2 >= 8) {
10961       // IndexOf for constant substrings with size >= 8 elements
10962       // which don't need to be loaded through stack.
10963       __ string_indexofC8($str1$$Register, $str2$$Register,
10964                           $cnt1$$Register, $cnt2$$Register,
10965                           icnt2, $result$$Register,
10966                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10967     } else {
10968       // Small strings are loaded through stack if they cross page boundary.
10969       __ string_indexof($str1$$Register, $str2$$Register,
10970                         $cnt1$$Register, $cnt2$$Register,
10971                         icnt2, $result$$Register,
10972                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10973     }
10974   %}
10975   ins_pipe( pipe_slow );
10976 %}
10977 
10978 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10979                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10980 %{
10981   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10982   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10983   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10984 
10985   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10986   ins_encode %{
10987     __ string_indexof($str1$$Register, $str2$$Register,
10988                       $cnt1$$Register, $cnt2$$Register,
10989                       (-1), $result$$Register,
10990                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10991   %}
10992   ins_pipe( pipe_slow );
10993 %}
10994 
10995 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10996                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
10997 %{
10998   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10999   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11000   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11001 
11002   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11003   ins_encode %{
11004     __ string_indexof($str1$$Register, $str2$$Register,
11005                       $cnt1$$Register, $cnt2$$Register,
11006                       (-1), $result$$Register,
11007                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11008   %}
11009   ins_pipe( pipe_slow );
11010 %}
11011 
11012 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11013                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11014 %{
11015   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11016   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11017   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11018 
11019   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11020   ins_encode %{
11021     __ string_indexof($str1$$Register, $str2$$Register,
11022                       $cnt1$$Register, $cnt2$$Register,
11023                       (-1), $result$$Register,
11024                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11025   %}
11026   ins_pipe( pipe_slow );
11027 %}
11028 
11029 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11030                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11031 %{
11032   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11033   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11034   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11035   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11036   ins_encode %{
11037     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11038                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11039   %}
11040   ins_pipe( pipe_slow );
11041 %}
11042 
11043 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11044                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11045 %{
11046   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11047   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11048   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11049   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11050   ins_encode %{
11051     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11052                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11053   %}
11054   ins_pipe( pipe_slow );
11055 %}
11056 
11057 // fast string equals
11058 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11059                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11060 %{
11061   predicate(!VM_Version::supports_avx512vlbw());
11062   match(Set result (StrEquals (Binary str1 str2) cnt));
11063   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11064 
11065   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11066   ins_encode %{
11067     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11068                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11069                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11070   %}
11071   ins_pipe( pipe_slow );
11072 %}
11073 
11074 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11075                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11076 %{
11077   predicate(VM_Version::supports_avx512vlbw());
11078   match(Set result (StrEquals (Binary str1 str2) cnt));
11079   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11080 
11081   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11082   ins_encode %{
11083     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11084                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11085                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11086   %}
11087   ins_pipe( pipe_slow );
11088 %}
11089 
11090 // fast array equals
11091 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11092                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11093 %{
11094   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11095   match(Set result (AryEq ary1 ary2));
11096   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11097 
11098   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11099   ins_encode %{
11100     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11101                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11102                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11103   %}
11104   ins_pipe( pipe_slow );
11105 %}
11106 
11107 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11108                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11109 %{
11110   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11111   match(Set result (AryEq ary1 ary2));
11112   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11113 
11114   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11115   ins_encode %{
11116     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11117                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11118                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11119   %}
11120   ins_pipe( pipe_slow );
11121 %}
11122 
11123 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11124                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11125 %{
11126   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11127   match(Set result (AryEq ary1 ary2));
11128   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11129 
11130   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11131   ins_encode %{
11132     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11133                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11134                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11135   %}
11136   ins_pipe( pipe_slow );
11137 %}
11138 
11139 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11140                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11141 %{
11142   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11143   match(Set result (AryEq ary1 ary2));
11144   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11145 
11146   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11147   ins_encode %{
11148     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11149                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11150                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11151   %}
11152   ins_pipe( pipe_slow );
11153 %}
11154 
11155 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11156                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11157                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11158                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11159                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11160 %{
11161   predicate(UseAVX >= 2);
11162   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11163   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11164          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11165          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11166          USE basic_type, KILL cr);
11167 
11168   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
11169   ins_encode %{
11170     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11171                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11172                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11173                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11174                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11175                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11176                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11177   %}
11178   ins_pipe( pipe_slow );
11179 %}
11180 
11181 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11182                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11183 %{
11184   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11185   match(Set result (CountPositives ary1 len));
11186   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11187 
11188   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11189   ins_encode %{
11190     __ count_positives($ary1$$Register, $len$$Register,
11191                        $result$$Register, $tmp3$$Register,
11192                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11193   %}
11194   ins_pipe( pipe_slow );
11195 %}
11196 
11197 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11198                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11199 %{
11200   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11201   match(Set result (CountPositives ary1 len));
11202   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11203 
11204   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11205   ins_encode %{
11206     __ count_positives($ary1$$Register, $len$$Register,
11207                        $result$$Register, $tmp3$$Register,
11208                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11209   %}
11210   ins_pipe( pipe_slow );
11211 %}
11212 
11213 // fast char[] to byte[] compression
11214 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11215                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11216   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11217   match(Set result (StrCompressedCopy src (Binary dst len)));
11218   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11219          USE_KILL len, KILL tmp5, KILL cr);
11220 
11221   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11222   ins_encode %{
11223     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11224                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11225                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11226                            knoreg, knoreg);
11227   %}
11228   ins_pipe( pipe_slow );
11229 %}
11230 
11231 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11232                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11233   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11234   match(Set result (StrCompressedCopy src (Binary dst len)));
11235   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11236          USE_KILL len, KILL tmp5, KILL cr);
11237 
11238   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11239   ins_encode %{
11240     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11241                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11242                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11243                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11244   %}
11245   ins_pipe( pipe_slow );
11246 %}
11247 // fast byte[] to char[] inflation
11248 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11249                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11250   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11251   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11252   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11253 
11254   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11255   ins_encode %{
11256     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11257                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11258   %}
11259   ins_pipe( pipe_slow );
11260 %}
11261 
11262 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11263                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11264   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11265   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11266   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11267 
11268   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11269   ins_encode %{
11270     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11271                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11272   %}
11273   ins_pipe( pipe_slow );
11274 %}
11275 
11276 // encode char[] to byte[] in ISO_8859_1
11277 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11278                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11279                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11280   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11281   match(Set result (EncodeISOArray src (Binary dst len)));
11282   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11283 
11284   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11285   ins_encode %{
11286     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11287                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11288                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11289   %}
11290   ins_pipe( pipe_slow );
11291 %}
11292 
11293 // encode char[] to byte[] in ASCII
11294 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11295                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11296                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11297   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11298   match(Set result (EncodeISOArray src (Binary dst len)));
11299   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11300 
11301   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11302   ins_encode %{
11303     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11304                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11305                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11306   %}
11307   ins_pipe( pipe_slow );
11308 %}
11309 
11310 //----------Overflow Math Instructions-----------------------------------------
11311 
11312 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11313 %{
11314   match(Set cr (OverflowAddI op1 op2));
11315   effect(DEF cr, USE_KILL op1, USE op2);
11316 
11317   format %{ "addl    $op1, $op2\t# overflow check int" %}
11318 
11319   ins_encode %{
11320     __ addl($op1$$Register, $op2$$Register);
11321   %}
11322   ins_pipe(ialu_reg_reg);
11323 %}
11324 
11325 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11326 %{
11327   match(Set cr (OverflowAddI op1 op2));
11328   effect(DEF cr, USE_KILL op1, USE op2);
11329 
11330   format %{ "addl    $op1, $op2\t# overflow check int" %}
11331 
11332   ins_encode %{
11333     __ addl($op1$$Register, $op2$$constant);
11334   %}
11335   ins_pipe(ialu_reg_reg);
11336 %}
11337 
11338 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11339 %{
11340   match(Set cr (OverflowAddL op1 op2));
11341   effect(DEF cr, USE_KILL op1, USE op2);
11342 
11343   format %{ "addq    $op1, $op2\t# overflow check long" %}
11344   ins_encode %{
11345     __ addq($op1$$Register, $op2$$Register);
11346   %}
11347   ins_pipe(ialu_reg_reg);
11348 %}
11349 
11350 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11351 %{
11352   match(Set cr (OverflowAddL op1 op2));
11353   effect(DEF cr, USE_KILL op1, USE op2);
11354 
11355   format %{ "addq    $op1, $op2\t# overflow check long" %}
11356   ins_encode %{
11357     __ addq($op1$$Register, $op2$$constant);
11358   %}
11359   ins_pipe(ialu_reg_reg);
11360 %}
11361 
11362 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11363 %{
11364   match(Set cr (OverflowSubI op1 op2));
11365 
11366   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11367   ins_encode %{
11368     __ cmpl($op1$$Register, $op2$$Register);
11369   %}
11370   ins_pipe(ialu_reg_reg);
11371 %}
11372 
11373 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11374 %{
11375   match(Set cr (OverflowSubI op1 op2));
11376 
11377   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11378   ins_encode %{
11379     __ cmpl($op1$$Register, $op2$$constant);
11380   %}
11381   ins_pipe(ialu_reg_reg);
11382 %}
11383 
11384 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11385 %{
11386   match(Set cr (OverflowSubL op1 op2));
11387 
11388   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11389   ins_encode %{
11390     __ cmpq($op1$$Register, $op2$$Register);
11391   %}
11392   ins_pipe(ialu_reg_reg);
11393 %}
11394 
11395 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11396 %{
11397   match(Set cr (OverflowSubL op1 op2));
11398 
11399   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11400   ins_encode %{
11401     __ cmpq($op1$$Register, $op2$$constant);
11402   %}
11403   ins_pipe(ialu_reg_reg);
11404 %}
11405 
11406 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11407 %{
11408   match(Set cr (OverflowSubI zero op2));
11409   effect(DEF cr, USE_KILL op2);
11410 
11411   format %{ "negl    $op2\t# overflow check int" %}
11412   ins_encode %{
11413     __ negl($op2$$Register);
11414   %}
11415   ins_pipe(ialu_reg_reg);
11416 %}
11417 
11418 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11419 %{
11420   match(Set cr (OverflowSubL zero op2));
11421   effect(DEF cr, USE_KILL op2);
11422 
11423   format %{ "negq    $op2\t# overflow check long" %}
11424   ins_encode %{
11425     __ negq($op2$$Register);
11426   %}
11427   ins_pipe(ialu_reg_reg);
11428 %}
11429 
11430 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11431 %{
11432   match(Set cr (OverflowMulI op1 op2));
11433   effect(DEF cr, USE_KILL op1, USE op2);
11434 
11435   format %{ "imull    $op1, $op2\t# overflow check int" %}
11436   ins_encode %{
11437     __ imull($op1$$Register, $op2$$Register);
11438   %}
11439   ins_pipe(ialu_reg_reg_alu0);
11440 %}
11441 
11442 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11443 %{
11444   match(Set cr (OverflowMulI op1 op2));
11445   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11446 
11447   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11448   ins_encode %{
11449     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11450   %}
11451   ins_pipe(ialu_reg_reg_alu0);
11452 %}
11453 
11454 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11455 %{
11456   match(Set cr (OverflowMulL op1 op2));
11457   effect(DEF cr, USE_KILL op1, USE op2);
11458 
11459   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11460   ins_encode %{
11461     __ imulq($op1$$Register, $op2$$Register);
11462   %}
11463   ins_pipe(ialu_reg_reg_alu0);
11464 %}
11465 
11466 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11467 %{
11468   match(Set cr (OverflowMulL op1 op2));
11469   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11470 
11471   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11472   ins_encode %{
11473     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11474   %}
11475   ins_pipe(ialu_reg_reg_alu0);
11476 %}
11477 
11478 
11479 //----------Control Flow Instructions------------------------------------------
11480 // Signed compare Instructions
11481 
11482 // XXX more variants!!
11483 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11484 %{
11485   match(Set cr (CmpI op1 op2));
11486   effect(DEF cr, USE op1, USE op2);
11487 
11488   format %{ "cmpl    $op1, $op2" %}
11489   ins_encode %{
11490     __ cmpl($op1$$Register, $op2$$Register);
11491   %}
11492   ins_pipe(ialu_cr_reg_reg);
11493 %}
11494 
11495 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11496 %{
11497   match(Set cr (CmpI op1 op2));
11498 
11499   format %{ "cmpl    $op1, $op2" %}
11500   ins_encode %{
11501     __ cmpl($op1$$Register, $op2$$constant);
11502   %}
11503   ins_pipe(ialu_cr_reg_imm);
11504 %}
11505 
11506 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11507 %{
11508   match(Set cr (CmpI op1 (LoadI op2)));
11509 
11510   ins_cost(500); // XXX
11511   format %{ "cmpl    $op1, $op2" %}
11512   ins_encode %{
11513     __ cmpl($op1$$Register, $op2$$Address);
11514   %}
11515   ins_pipe(ialu_cr_reg_mem);
11516 %}
11517 
11518 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11519 %{
11520   match(Set cr (CmpI src zero));
11521 
11522   format %{ "testl   $src, $src" %}
11523   ins_encode %{
11524     __ testl($src$$Register, $src$$Register);
11525   %}
11526   ins_pipe(ialu_cr_reg_imm);
11527 %}
11528 
11529 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11530 %{
11531   match(Set cr (CmpI (AndI src con) zero));
11532 
11533   format %{ "testl   $src, $con" %}
11534   ins_encode %{
11535     __ testl($src$$Register, $con$$constant);
11536   %}
11537   ins_pipe(ialu_cr_reg_imm);
11538 %}
11539 
11540 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11541 %{
11542   match(Set cr (CmpI (AndI src1 src2) zero));
11543 
11544   format %{ "testl   $src1, $src2" %}
11545   ins_encode %{
11546     __ testl($src1$$Register, $src2$$Register);
11547   %}
11548   ins_pipe(ialu_cr_reg_imm);
11549 %}
11550 
11551 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11552 %{
11553   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11554 
11555   format %{ "testl   $src, $mem" %}
11556   ins_encode %{
11557     __ testl($src$$Register, $mem$$Address);
11558   %}
11559   ins_pipe(ialu_cr_reg_mem);
11560 %}
11561 
11562 // Unsigned compare Instructions; really, same as signed except they
11563 // produce an rFlagsRegU instead of rFlagsReg.
11564 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11565 %{
11566   match(Set cr (CmpU op1 op2));
11567 
11568   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11569   ins_encode %{
11570     __ cmpl($op1$$Register, $op2$$Register);
11571   %}
11572   ins_pipe(ialu_cr_reg_reg);
11573 %}
11574 
11575 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11576 %{
11577   match(Set cr (CmpU op1 op2));
11578 
11579   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11580   ins_encode %{
11581     __ cmpl($op1$$Register, $op2$$constant);
11582   %}
11583   ins_pipe(ialu_cr_reg_imm);
11584 %}
11585 
11586 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11587 %{
11588   match(Set cr (CmpU op1 (LoadI op2)));
11589 
11590   ins_cost(500); // XXX
11591   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11592   ins_encode %{
11593     __ cmpl($op1$$Register, $op2$$Address);
11594   %}
11595   ins_pipe(ialu_cr_reg_mem);
11596 %}
11597 
11598 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11599 %{
11600   match(Set cr (CmpU src zero));
11601 
11602   format %{ "testl   $src, $src\t# unsigned" %}
11603   ins_encode %{
11604     __ testl($src$$Register, $src$$Register);
11605   %}
11606   ins_pipe(ialu_cr_reg_imm);
11607 %}
11608 
11609 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11610 %{
11611   match(Set cr (CmpP op1 op2));
11612 
11613   format %{ "cmpq    $op1, $op2\t# ptr" %}
11614   ins_encode %{
11615     __ cmpq($op1$$Register, $op2$$Register);
11616   %}
11617   ins_pipe(ialu_cr_reg_reg);
11618 %}
11619 
11620 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11621 %{
11622   match(Set cr (CmpP op1 (LoadP op2)));
11623   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11624 
11625   ins_cost(500); // XXX
11626   format %{ "cmpq    $op1, $op2\t# ptr" %}
11627   ins_encode %{
11628     __ cmpq($op1$$Register, $op2$$Address);
11629   %}
11630   ins_pipe(ialu_cr_reg_mem);
11631 %}
11632 
11633 // XXX this is generalized by compP_rReg_mem???
11634 // Compare raw pointer (used in out-of-heap check).
11635 // Only works because non-oop pointers must be raw pointers
11636 // and raw pointers have no anti-dependencies.
11637 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11638 %{
11639   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11640             n->in(2)->as_Load()->barrier_data() == 0);
11641   match(Set cr (CmpP op1 (LoadP op2)));
11642 
11643   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11644   ins_encode %{
11645     __ cmpq($op1$$Register, $op2$$Address);
11646   %}
11647   ins_pipe(ialu_cr_reg_mem);
11648 %}
11649 
11650 // This will generate a signed flags result. This should be OK since
11651 // any compare to a zero should be eq/neq.
11652 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11653 %{
11654   match(Set cr (CmpP src zero));
11655 
11656   format %{ "testq   $src, $src\t# ptr" %}
11657   ins_encode %{
11658     __ testq($src$$Register, $src$$Register);
11659   %}
11660   ins_pipe(ialu_cr_reg_imm);
11661 %}
11662 
11663 // This will generate a signed flags result. This should be OK since
11664 // any compare to a zero should be eq/neq.
11665 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11666 %{
11667   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
11668             n->in(1)->as_Load()->barrier_data() == 0);
11669   match(Set cr (CmpP (LoadP op) zero));
11670 
11671   ins_cost(500); // XXX
11672   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11673   ins_encode %{
11674     __ testq($op$$Address, 0xFFFFFFFF);
11675   %}
11676   ins_pipe(ialu_cr_reg_imm);
11677 %}
11678 
11679 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11680 %{
11681   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
11682             n->in(1)->as_Load()->barrier_data() == 0);
11683   match(Set cr (CmpP (LoadP mem) zero));
11684 
11685   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11686   ins_encode %{
11687     __ cmpq(r12, $mem$$Address);
11688   %}
11689   ins_pipe(ialu_cr_reg_mem);
11690 %}
11691 
11692 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11693 %{
11694   match(Set cr (CmpN op1 op2));
11695 
11696   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11697   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11698   ins_pipe(ialu_cr_reg_reg);
11699 %}
11700 
11701 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11702 %{
11703   match(Set cr (CmpN src (LoadN mem)));
11704 
11705   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11706   ins_encode %{
11707     __ cmpl($src$$Register, $mem$$Address);
11708   %}
11709   ins_pipe(ialu_cr_reg_mem);
11710 %}
11711 
11712 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11713   match(Set cr (CmpN op1 op2));
11714 
11715   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11716   ins_encode %{
11717     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11718   %}
11719   ins_pipe(ialu_cr_reg_imm);
11720 %}
11721 
11722 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11723 %{
11724   match(Set cr (CmpN src (LoadN mem)));
11725 
11726   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11727   ins_encode %{
11728     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11729   %}
11730   ins_pipe(ialu_cr_reg_mem);
11731 %}
11732 
11733 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11734   match(Set cr (CmpN op1 op2));
11735 
11736   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11737   ins_encode %{
11738     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11739   %}
11740   ins_pipe(ialu_cr_reg_imm);
11741 %}
11742 
11743 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11744 %{
11745   match(Set cr (CmpN src (LoadNKlass mem)));
11746 
11747   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11748   ins_encode %{
11749     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11750   %}
11751   ins_pipe(ialu_cr_reg_mem);
11752 %}
11753 
11754 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11755   match(Set cr (CmpN src zero));
11756 
11757   format %{ "testl   $src, $src\t# compressed ptr" %}
11758   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11759   ins_pipe(ialu_cr_reg_imm);
11760 %}
11761 
11762 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11763 %{
11764   predicate(CompressedOops::base() != nullptr);
11765   match(Set cr (CmpN (LoadN mem) zero));
11766 
11767   ins_cost(500); // XXX
11768   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11769   ins_encode %{
11770     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11771   %}
11772   ins_pipe(ialu_cr_reg_mem);
11773 %}
11774 
11775 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11776 %{
11777   predicate(CompressedOops::base() == nullptr);
11778   match(Set cr (CmpN (LoadN mem) zero));
11779 
11780   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11781   ins_encode %{
11782     __ cmpl(r12, $mem$$Address);
11783   %}
11784   ins_pipe(ialu_cr_reg_mem);
11785 %}
11786 
11787 // Yanked all unsigned pointer compare operations.
11788 // Pointer compares are done with CmpP which is already unsigned.
11789 
11790 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11791 %{
11792   match(Set cr (CmpL op1 op2));
11793 
11794   format %{ "cmpq    $op1, $op2" %}
11795   ins_encode %{
11796     __ cmpq($op1$$Register, $op2$$Register);
11797   %}
11798   ins_pipe(ialu_cr_reg_reg);
11799 %}
11800 
11801 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11802 %{
11803   match(Set cr (CmpL op1 op2));
11804 
11805   format %{ "cmpq    $op1, $op2" %}
11806   ins_encode %{
11807     __ cmpq($op1$$Register, $op2$$constant);
11808   %}
11809   ins_pipe(ialu_cr_reg_imm);
11810 %}
11811 
11812 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11813 %{
11814   match(Set cr (CmpL op1 (LoadL op2)));
11815 
11816   format %{ "cmpq    $op1, $op2" %}
11817   ins_encode %{
11818     __ cmpq($op1$$Register, $op2$$Address);
11819   %}
11820   ins_pipe(ialu_cr_reg_mem);
11821 %}
11822 
11823 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11824 %{
11825   match(Set cr (CmpL src zero));
11826 
11827   format %{ "testq   $src, $src" %}
11828   ins_encode %{
11829     __ testq($src$$Register, $src$$Register);
11830   %}
11831   ins_pipe(ialu_cr_reg_imm);
11832 %}
11833 
11834 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11835 %{
11836   match(Set cr (CmpL (AndL src con) zero));
11837 
11838   format %{ "testq   $src, $con\t# long" %}
11839   ins_encode %{
11840     __ testq($src$$Register, $con$$constant);
11841   %}
11842   ins_pipe(ialu_cr_reg_imm);
11843 %}
11844 
11845 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
11846 %{
11847   match(Set cr (CmpL (AndL src1 src2) zero));
11848 
11849   format %{ "testq   $src1, $src2\t# long" %}
11850   ins_encode %{
11851     __ testq($src1$$Register, $src2$$Register);
11852   %}
11853   ins_pipe(ialu_cr_reg_imm);
11854 %}
11855 
11856 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11857 %{
11858   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11859 
11860   format %{ "testq   $src, $mem" %}
11861   ins_encode %{
11862     __ testq($src$$Register, $mem$$Address);
11863   %}
11864   ins_pipe(ialu_cr_reg_mem);
11865 %}
11866 
11867 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11868 %{
11869   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11870 
11871   format %{ "testq   $src, $mem" %}
11872   ins_encode %{
11873     __ testq($src$$Register, $mem$$Address);
11874   %}
11875   ins_pipe(ialu_cr_reg_mem);
11876 %}
11877 
11878 // Manifest a CmpU result in an integer register.  Very painful.
11879 // This is the test to avoid.
11880 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
11881 %{
11882   match(Set dst (CmpU3 src1 src2));
11883   effect(KILL flags);
11884 
11885   ins_cost(275); // XXX
11886   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
11887             "movl    $dst, -1\n\t"
11888             "jb,u    done\n\t"
11889             "setne   $dst\n\t"
11890             "movzbl  $dst, $dst\n\t"
11891     "done:" %}
11892   ins_encode %{
11893     Label done;
11894     __ cmpl($src1$$Register, $src2$$Register);
11895     __ movl($dst$$Register, -1);
11896     __ jccb(Assembler::below, done);
11897     __ setb(Assembler::notZero, $dst$$Register);
11898     __ movzbl($dst$$Register, $dst$$Register);
11899     __ bind(done);
11900   %}
11901   ins_pipe(pipe_slow);
11902 %}
11903 
11904 // Manifest a CmpL result in an integer register.  Very painful.
11905 // This is the test to avoid.
11906 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11907 %{
11908   match(Set dst (CmpL3 src1 src2));
11909   effect(KILL flags);
11910 
11911   ins_cost(275); // XXX
11912   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11913             "movl    $dst, -1\n\t"
11914             "jl,s    done\n\t"
11915             "setne   $dst\n\t"
11916             "movzbl  $dst, $dst\n\t"
11917     "done:" %}
11918   ins_encode %{
11919     Label done;
11920     __ cmpq($src1$$Register, $src2$$Register);
11921     __ movl($dst$$Register, -1);
11922     __ jccb(Assembler::less, done);
11923     __ setb(Assembler::notZero, $dst$$Register);
11924     __ movzbl($dst$$Register, $dst$$Register);
11925     __ bind(done);
11926   %}
11927   ins_pipe(pipe_slow);
11928 %}
11929 
11930 // Manifest a CmpUL result in an integer register.  Very painful.
11931 // This is the test to avoid.
11932 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11933 %{
11934   match(Set dst (CmpUL3 src1 src2));
11935   effect(KILL flags);
11936 
11937   ins_cost(275); // XXX
11938   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11939             "movl    $dst, -1\n\t"
11940             "jb,u    done\n\t"
11941             "setne   $dst\n\t"
11942             "movzbl  $dst, $dst\n\t"
11943     "done:" %}
11944   ins_encode %{
11945     Label done;
11946     __ cmpq($src1$$Register, $src2$$Register);
11947     __ movl($dst$$Register, -1);
11948     __ jccb(Assembler::below, done);
11949     __ setb(Assembler::notZero, $dst$$Register);
11950     __ movzbl($dst$$Register, $dst$$Register);
11951     __ bind(done);
11952   %}
11953   ins_pipe(pipe_slow);
11954 %}
11955 
11956 // Unsigned long compare Instructions; really, same as signed long except they
11957 // produce an rFlagsRegU instead of rFlagsReg.
11958 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11959 %{
11960   match(Set cr (CmpUL op1 op2));
11961 
11962   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11963   ins_encode %{
11964     __ cmpq($op1$$Register, $op2$$Register);
11965   %}
11966   ins_pipe(ialu_cr_reg_reg);
11967 %}
11968 
11969 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11970 %{
11971   match(Set cr (CmpUL op1 op2));
11972 
11973   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11974   ins_encode %{
11975     __ cmpq($op1$$Register, $op2$$constant);
11976   %}
11977   ins_pipe(ialu_cr_reg_imm);
11978 %}
11979 
11980 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11981 %{
11982   match(Set cr (CmpUL op1 (LoadL op2)));
11983 
11984   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11985   ins_encode %{
11986     __ cmpq($op1$$Register, $op2$$Address);
11987   %}
11988   ins_pipe(ialu_cr_reg_mem);
11989 %}
11990 
11991 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11992 %{
11993   match(Set cr (CmpUL src zero));
11994 
11995   format %{ "testq   $src, $src\t# unsigned" %}
11996   ins_encode %{
11997     __ testq($src$$Register, $src$$Register);
11998   %}
11999   ins_pipe(ialu_cr_reg_imm);
12000 %}
12001 
12002 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12003 %{
12004   match(Set cr (CmpI (LoadB mem) imm));
12005 
12006   ins_cost(125);
12007   format %{ "cmpb    $mem, $imm" %}
12008   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12009   ins_pipe(ialu_cr_reg_mem);
12010 %}
12011 
12012 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12013 %{
12014   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12015 
12016   ins_cost(125);
12017   format %{ "testb   $mem, $imm\t# ubyte" %}
12018   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12019   ins_pipe(ialu_cr_reg_mem);
12020 %}
12021 
12022 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12023 %{
12024   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12025 
12026   ins_cost(125);
12027   format %{ "testb   $mem, $imm\t# byte" %}
12028   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12029   ins_pipe(ialu_cr_reg_mem);
12030 %}
12031 
12032 //----------Max and Min--------------------------------------------------------
12033 // Min Instructions
12034 
12035 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12036 %{
12037   effect(USE_DEF dst, USE src, USE cr);
12038 
12039   format %{ "cmovlgt $dst, $src\t# min" %}
12040   ins_encode %{
12041     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12042   %}
12043   ins_pipe(pipe_cmov_reg);
12044 %}
12045 
12046 
12047 instruct minI_rReg(rRegI dst, rRegI src)
12048 %{
12049   match(Set dst (MinI dst src));
12050 
12051   ins_cost(200);
12052   expand %{
12053     rFlagsReg cr;
12054     compI_rReg(cr, dst, src);
12055     cmovI_reg_g(dst, src, cr);
12056   %}
12057 %}
12058 
12059 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12060 %{
12061   effect(USE_DEF dst, USE src, USE cr);
12062 
12063   format %{ "cmovllt $dst, $src\t# max" %}
12064   ins_encode %{
12065     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12066   %}
12067   ins_pipe(pipe_cmov_reg);
12068 %}
12069 
12070 
12071 instruct maxI_rReg(rRegI dst, rRegI src)
12072 %{
12073   match(Set dst (MaxI dst src));
12074 
12075   ins_cost(200);
12076   expand %{
12077     rFlagsReg cr;
12078     compI_rReg(cr, dst, src);
12079     cmovI_reg_l(dst, src, cr);
12080   %}
12081 %}
12082 
12083 // ============================================================================
12084 // Branch Instructions
12085 
12086 // Jump Direct - Label defines a relative address from JMP+1
12087 instruct jmpDir(label labl)
12088 %{
12089   match(Goto);
12090   effect(USE labl);
12091 
12092   ins_cost(300);
12093   format %{ "jmp     $labl" %}
12094   size(5);
12095   ins_encode %{
12096     Label* L = $labl$$label;
12097     __ jmp(*L, false); // Always long jump
12098   %}
12099   ins_pipe(pipe_jmp);
12100 %}
12101 
12102 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12103 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12104 %{
12105   match(If cop cr);
12106   effect(USE labl);
12107 
12108   ins_cost(300);
12109   format %{ "j$cop     $labl" %}
12110   size(6);
12111   ins_encode %{
12112     Label* L = $labl$$label;
12113     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12114   %}
12115   ins_pipe(pipe_jcc);
12116 %}
12117 
12118 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12119 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12120 %{
12121   match(CountedLoopEnd cop cr);
12122   effect(USE labl);
12123 
12124   ins_cost(300);
12125   format %{ "j$cop     $labl\t# loop end" %}
12126   size(6);
12127   ins_encode %{
12128     Label* L = $labl$$label;
12129     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12130   %}
12131   ins_pipe(pipe_jcc);
12132 %}
12133 
12134 // Jump Direct Conditional - using unsigned comparison
12135 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12136   match(If cop cmp);
12137   effect(USE labl);
12138 
12139   ins_cost(300);
12140   format %{ "j$cop,u   $labl" %}
12141   size(6);
12142   ins_encode %{
12143     Label* L = $labl$$label;
12144     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12145   %}
12146   ins_pipe(pipe_jcc);
12147 %}
12148 
12149 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12150   match(If cop cmp);
12151   effect(USE labl);
12152 
12153   ins_cost(200);
12154   format %{ "j$cop,u   $labl" %}
12155   size(6);
12156   ins_encode %{
12157     Label* L = $labl$$label;
12158     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12159   %}
12160   ins_pipe(pipe_jcc);
12161 %}
12162 
12163 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12164   match(If cop cmp);
12165   effect(USE labl);
12166 
12167   ins_cost(200);
12168   format %{ $$template
12169     if ($cop$$cmpcode == Assembler::notEqual) {
12170       $$emit$$"jp,u    $labl\n\t"
12171       $$emit$$"j$cop,u   $labl"
12172     } else {
12173       $$emit$$"jp,u    done\n\t"
12174       $$emit$$"j$cop,u   $labl\n\t"
12175       $$emit$$"done:"
12176     }
12177   %}
12178   ins_encode %{
12179     Label* l = $labl$$label;
12180     if ($cop$$cmpcode == Assembler::notEqual) {
12181       __ jcc(Assembler::parity, *l, false);
12182       __ jcc(Assembler::notEqual, *l, false);
12183     } else if ($cop$$cmpcode == Assembler::equal) {
12184       Label done;
12185       __ jccb(Assembler::parity, done);
12186       __ jcc(Assembler::equal, *l, false);
12187       __ bind(done);
12188     } else {
12189        ShouldNotReachHere();
12190     }
12191   %}
12192   ins_pipe(pipe_jcc);
12193 %}
12194 
12195 // ============================================================================
12196 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12197 // superklass array for an instance of the superklass.  Set a hidden
12198 // internal cache on a hit (cache is checked with exposed code in
12199 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12200 // encoding ALSO sets flags.
12201 
12202 instruct partialSubtypeCheck(rdi_RegP result,
12203                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12204                              rFlagsReg cr)
12205 %{
12206   match(Set result (PartialSubtypeCheck sub super));
12207   effect(KILL rcx, KILL cr);
12208 
12209   ins_cost(1100);  // slightly larger than the next version
12210   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12211             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12212             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12213             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12214             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12215             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12216             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12217     "miss:\t" %}
12218 
12219   opcode(0x1); // Force a XOR of RDI
12220   ins_encode(enc_PartialSubtypeCheck());
12221   ins_pipe(pipe_slow);
12222 %}
12223 
12224 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12225                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12226                                      immP0 zero,
12227                                      rdi_RegP result)
12228 %{
12229   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12230   effect(KILL rcx, KILL result);
12231 
12232   ins_cost(1000);
12233   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12234             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12235             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12236             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12237             "jne,s   miss\t\t# Missed: flags nz\n\t"
12238             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12239     "miss:\t" %}
12240 
12241   opcode(0x0); // No need to XOR RDI
12242   ins_encode(enc_PartialSubtypeCheck());
12243   ins_pipe(pipe_slow);
12244 %}
12245 
12246 // ============================================================================
12247 // Branch Instructions -- short offset versions
12248 //
12249 // These instructions are used to replace jumps of a long offset (the default
12250 // match) with jumps of a shorter offset.  These instructions are all tagged
12251 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12252 // match rules in general matching.  Instead, the ADLC generates a conversion
12253 // method in the MachNode which can be used to do in-place replacement of the
12254 // long variant with the shorter variant.  The compiler will determine if a
12255 // branch can be taken by the is_short_branch_offset() predicate in the machine
12256 // specific code section of the file.
12257 
12258 // Jump Direct - Label defines a relative address from JMP+1
12259 instruct jmpDir_short(label labl) %{
12260   match(Goto);
12261   effect(USE labl);
12262 
12263   ins_cost(300);
12264   format %{ "jmp,s   $labl" %}
12265   size(2);
12266   ins_encode %{
12267     Label* L = $labl$$label;
12268     __ jmpb(*L);
12269   %}
12270   ins_pipe(pipe_jmp);
12271   ins_short_branch(1);
12272 %}
12273 
12274 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12275 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12276   match(If cop cr);
12277   effect(USE labl);
12278 
12279   ins_cost(300);
12280   format %{ "j$cop,s   $labl" %}
12281   size(2);
12282   ins_encode %{
12283     Label* L = $labl$$label;
12284     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12285   %}
12286   ins_pipe(pipe_jcc);
12287   ins_short_branch(1);
12288 %}
12289 
12290 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12291 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12292   match(CountedLoopEnd cop cr);
12293   effect(USE labl);
12294 
12295   ins_cost(300);
12296   format %{ "j$cop,s   $labl\t# loop end" %}
12297   size(2);
12298   ins_encode %{
12299     Label* L = $labl$$label;
12300     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12301   %}
12302   ins_pipe(pipe_jcc);
12303   ins_short_branch(1);
12304 %}
12305 
12306 // Jump Direct Conditional - using unsigned comparison
12307 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12308   match(If cop cmp);
12309   effect(USE labl);
12310 
12311   ins_cost(300);
12312   format %{ "j$cop,us  $labl" %}
12313   size(2);
12314   ins_encode %{
12315     Label* L = $labl$$label;
12316     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12317   %}
12318   ins_pipe(pipe_jcc);
12319   ins_short_branch(1);
12320 %}
12321 
12322 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12323   match(If cop cmp);
12324   effect(USE labl);
12325 
12326   ins_cost(300);
12327   format %{ "j$cop,us  $labl" %}
12328   size(2);
12329   ins_encode %{
12330     Label* L = $labl$$label;
12331     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12332   %}
12333   ins_pipe(pipe_jcc);
12334   ins_short_branch(1);
12335 %}
12336 
12337 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12338   match(If cop cmp);
12339   effect(USE labl);
12340 
12341   ins_cost(300);
12342   format %{ $$template
12343     if ($cop$$cmpcode == Assembler::notEqual) {
12344       $$emit$$"jp,u,s  $labl\n\t"
12345       $$emit$$"j$cop,u,s  $labl"
12346     } else {
12347       $$emit$$"jp,u,s  done\n\t"
12348       $$emit$$"j$cop,u,s  $labl\n\t"
12349       $$emit$$"done:"
12350     }
12351   %}
12352   size(4);
12353   ins_encode %{
12354     Label* l = $labl$$label;
12355     if ($cop$$cmpcode == Assembler::notEqual) {
12356       __ jccb(Assembler::parity, *l);
12357       __ jccb(Assembler::notEqual, *l);
12358     } else if ($cop$$cmpcode == Assembler::equal) {
12359       Label done;
12360       __ jccb(Assembler::parity, done);
12361       __ jccb(Assembler::equal, *l);
12362       __ bind(done);
12363     } else {
12364        ShouldNotReachHere();
12365     }
12366   %}
12367   ins_pipe(pipe_jcc);
12368   ins_short_branch(1);
12369 %}
12370 
12371 // ============================================================================
12372 // inlined locking and unlocking
12373 
12374 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12375   predicate(Compile::current()->use_rtm());
12376   match(Set cr (FastLock object box));
12377   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12378   ins_cost(300);
12379   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12380   ins_encode %{
12381     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12382                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
12383                  _rtm_counters, _stack_rtm_counters,
12384                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12385                  true, ra_->C->profile_rtm());
12386   %}
12387   ins_pipe(pipe_slow);
12388 %}
12389 
12390 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12391   predicate(LockingMode != LM_LIGHTWEIGHT && !Compile::current()->use_rtm());
12392   match(Set cr (FastLock object box));
12393   effect(TEMP tmp, TEMP scr, USE_KILL box);
12394   ins_cost(300);
12395   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12396   ins_encode %{
12397     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12398                  $scr$$Register, noreg, noreg, r15_thread, nullptr, nullptr, nullptr, false, false);
12399   %}
12400   ins_pipe(pipe_slow);
12401 %}
12402 
12403 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12404   predicate(LockingMode != LM_LIGHTWEIGHT);
12405   match(Set cr (FastUnlock object box));
12406   effect(TEMP tmp, USE_KILL box);
12407   ins_cost(300);
12408   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12409   ins_encode %{
12410     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12411   %}
12412   ins_pipe(pipe_slow);
12413 %}
12414 
12415 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
12416   predicate(LockingMode == LM_LIGHTWEIGHT);
12417   match(Set cr (FastLock object box));
12418   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
12419   ins_cost(300);
12420   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
12421   ins_encode %{
12422     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12423   %}
12424   ins_pipe(pipe_slow);
12425 %}
12426 
12427 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
12428   predicate(LockingMode == LM_LIGHTWEIGHT);
12429   match(Set cr (FastUnlock object rax_reg));
12430   effect(TEMP tmp, USE_KILL rax_reg);
12431   ins_cost(300);
12432   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
12433   ins_encode %{
12434     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12435   %}
12436   ins_pipe(pipe_slow);
12437 %}
12438 
12439 
12440 // ============================================================================
12441 // Safepoint Instructions
12442 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12443 %{
12444   match(SafePoint poll);
12445   effect(KILL cr, USE poll);
12446 
12447   format %{ "testl   rax, [$poll]\t"
12448             "# Safepoint: poll for GC" %}
12449   ins_cost(125);
12450   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12451   ins_encode %{
12452     __ relocate(relocInfo::poll_type);
12453     address pre_pc = __ pc();
12454     __ testl(rax, Address($poll$$Register, 0));
12455     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12456   %}
12457   ins_pipe(ialu_reg_mem);
12458 %}
12459 
12460 instruct mask_all_evexL(kReg dst, rRegL src) %{
12461   match(Set dst (MaskAll src));
12462   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12463   ins_encode %{
12464     int mask_len = Matcher::vector_length(this);
12465     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12466   %}
12467   ins_pipe( pipe_slow );
12468 %}
12469 
12470 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12471   predicate(Matcher::vector_length(n) > 32);
12472   match(Set dst (MaskAll src));
12473   effect(TEMP tmp);
12474   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12475   ins_encode %{
12476     int mask_len = Matcher::vector_length(this);
12477     __ movslq($tmp$$Register, $src$$Register);
12478     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12479   %}
12480   ins_pipe( pipe_slow );
12481 %}
12482 
12483 // ============================================================================
12484 // Procedure Call/Return Instructions
12485 // Call Java Static Instruction
12486 // Note: If this code changes, the corresponding ret_addr_offset() and
12487 //       compute_padding() functions will have to be adjusted.
12488 instruct CallStaticJavaDirect(method meth) %{
12489   match(CallStaticJava);
12490   effect(USE meth);
12491 
12492   ins_cost(300);
12493   format %{ "call,static " %}
12494   opcode(0xE8); /* E8 cd */
12495   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12496   ins_pipe(pipe_slow);
12497   ins_alignment(4);
12498 %}
12499 
12500 // Call Java Dynamic Instruction
12501 // Note: If this code changes, the corresponding ret_addr_offset() and
12502 //       compute_padding() functions will have to be adjusted.
12503 instruct CallDynamicJavaDirect(method meth)
12504 %{
12505   match(CallDynamicJava);
12506   effect(USE meth);
12507 
12508   ins_cost(300);
12509   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12510             "call,dynamic " %}
12511   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12512   ins_pipe(pipe_slow);
12513   ins_alignment(4);
12514 %}
12515 
12516 // Call Runtime Instruction
12517 instruct CallRuntimeDirect(method meth)
12518 %{
12519   match(CallRuntime);
12520   effect(USE meth);
12521 
12522   ins_cost(300);
12523   format %{ "call,runtime " %}
12524   ins_encode(clear_avx, Java_To_Runtime(meth));
12525   ins_pipe(pipe_slow);
12526 %}
12527 
12528 // Call runtime without safepoint
12529 instruct CallLeafDirect(method meth)
12530 %{
12531   match(CallLeaf);
12532   effect(USE meth);
12533 
12534   ins_cost(300);
12535   format %{ "call_leaf,runtime " %}
12536   ins_encode(clear_avx, Java_To_Runtime(meth));
12537   ins_pipe(pipe_slow);
12538 %}
12539 
12540 // Call runtime without safepoint and with vector arguments
12541 instruct CallLeafDirectVector(method meth)
12542 %{
12543   match(CallLeafVector);
12544   effect(USE meth);
12545 
12546   ins_cost(300);
12547   format %{ "call_leaf,vector " %}
12548   ins_encode(Java_To_Runtime(meth));
12549   ins_pipe(pipe_slow);
12550 %}
12551 
12552 // Call runtime without safepoint
12553 instruct CallLeafNoFPDirect(method meth)
12554 %{
12555   match(CallLeafNoFP);
12556   effect(USE meth);
12557 
12558   ins_cost(300);
12559   format %{ "call_leaf_nofp,runtime " %}
12560   ins_encode(clear_avx, Java_To_Runtime(meth));
12561   ins_pipe(pipe_slow);
12562 %}
12563 
12564 // Return Instruction
12565 // Remove the return address & jump to it.
12566 // Notice: We always emit a nop after a ret to make sure there is room
12567 // for safepoint patching
12568 instruct Ret()
12569 %{
12570   match(Return);
12571 
12572   format %{ "ret" %}
12573   ins_encode %{
12574     __ ret(0);
12575   %}
12576   ins_pipe(pipe_jmp);
12577 %}
12578 
12579 // Tail Call; Jump from runtime stub to Java code.
12580 // Also known as an 'interprocedural jump'.
12581 // Target of jump will eventually return to caller.
12582 // TailJump below removes the return address.
12583 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12584 %{
12585   match(TailCall jump_target method_ptr);
12586 
12587   ins_cost(300);
12588   format %{ "jmp     $jump_target\t# rbx holds method" %}
12589   ins_encode %{
12590     __ jmp($jump_target$$Register);
12591   %}
12592   ins_pipe(pipe_jmp);
12593 %}
12594 
12595 // Tail Jump; remove the return address; jump to target.
12596 // TailCall above leaves the return address around.
12597 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12598 %{
12599   match(TailJump jump_target ex_oop);
12600 
12601   ins_cost(300);
12602   format %{ "popq    rdx\t# pop return address\n\t"
12603             "jmp     $jump_target" %}
12604   ins_encode %{
12605     __ popq(as_Register(RDX_enc));
12606     __ jmp($jump_target$$Register);
12607   %}
12608   ins_pipe(pipe_jmp);
12609 %}
12610 
12611 // Create exception oop: created by stack-crawling runtime code.
12612 // Created exception is now available to this handler, and is setup
12613 // just prior to jumping to this handler.  No code emitted.
12614 instruct CreateException(rax_RegP ex_oop)
12615 %{
12616   match(Set ex_oop (CreateEx));
12617 
12618   size(0);
12619   // use the following format syntax
12620   format %{ "# exception oop is in rax; no code emitted" %}
12621   ins_encode();
12622   ins_pipe(empty);
12623 %}
12624 
12625 // Rethrow exception:
12626 // The exception oop will come in the first argument position.
12627 // Then JUMP (not call) to the rethrow stub code.
12628 instruct RethrowException()
12629 %{
12630   match(Rethrow);
12631 
12632   // use the following format syntax
12633   format %{ "jmp     rethrow_stub" %}
12634   ins_encode %{
12635     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12636   %}
12637   ins_pipe(pipe_jmp);
12638 %}
12639 
12640 // ============================================================================
12641 // This name is KNOWN by the ADLC and cannot be changed.
12642 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12643 // for this guy.
12644 instruct tlsLoadP(r15_RegP dst) %{
12645   match(Set dst (ThreadLocal));
12646   effect(DEF dst);
12647 
12648   size(0);
12649   format %{ "# TLS is in R15" %}
12650   ins_encode( /*empty encoding*/ );
12651   ins_pipe(ialu_reg_reg);
12652 %}
12653 
12654 
12655 //----------PEEPHOLE RULES-----------------------------------------------------
12656 // These must follow all instruction definitions as they use the names
12657 // defined in the instructions definitions.
12658 //
12659 // peeppredicate ( rule_predicate );
12660 // // the predicate unless which the peephole rule will be ignored
12661 //
12662 // peepmatch ( root_instr_name [preceding_instruction]* );
12663 //
12664 // peepprocedure ( procedure_name );
12665 // // provide a procedure name to perform the optimization, the procedure should
12666 // // reside in the architecture dependent peephole file, the method has the
12667 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12668 // // with the arguments being the basic block, the current node index inside the
12669 // // block, the register allocator, the functions upon invoked return a new node
12670 // // defined in peepreplace, and the rules of the nodes appearing in the
12671 // // corresponding peepmatch, the function return true if successful, else
12672 // // return false
12673 //
12674 // peepconstraint %{
12675 // (instruction_number.operand_name relational_op instruction_number.operand_name
12676 //  [, ...] );
12677 // // instruction numbers are zero-based using left to right order in peepmatch
12678 //
12679 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12680 // // provide an instruction_number.operand_name for each operand that appears
12681 // // in the replacement instruction's match rule
12682 //
12683 // ---------VM FLAGS---------------------------------------------------------
12684 //
12685 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12686 //
12687 // Each peephole rule is given an identifying number starting with zero and
12688 // increasing by one in the order seen by the parser.  An individual peephole
12689 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12690 // on the command-line.
12691 //
12692 // ---------CURRENT LIMITATIONS----------------------------------------------
12693 //
12694 // Only transformations inside a basic block (do we need more for peephole)
12695 //
12696 // ---------EXAMPLE----------------------------------------------------------
12697 //
12698 // // pertinent parts of existing instructions in architecture description
12699 // instruct movI(rRegI dst, rRegI src)
12700 // %{
12701 //   match(Set dst (CopyI src));
12702 // %}
12703 //
12704 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12705 // %{
12706 //   match(Set dst (AddI dst src));
12707 //   effect(KILL cr);
12708 // %}
12709 //
12710 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12711 // %{
12712 //   match(Set dst (AddI dst src));
12713 // %}
12714 //
12715 // 1. Simple replacement
12716 // - Only match adjacent instructions in same basic block
12717 // - Only equality constraints
12718 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
12719 // - Only one replacement instruction
12720 //
12721 // // Change (inc mov) to lea
12722 // peephole %{
12723 //   // lea should only be emitted when beneficial
12724 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12725 //   // increment preceded by register-register move
12726 //   peepmatch ( incI_rReg movI );
12727 //   // require that the destination register of the increment
12728 //   // match the destination register of the move
12729 //   peepconstraint ( 0.dst == 1.dst );
12730 //   // construct a replacement instruction that sets
12731 //   // the destination to ( move's source register + one )
12732 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12733 // %}
12734 //
12735 // 2. Procedural replacement
12736 // - More flexible finding relevent nodes
12737 // - More flexible constraints
12738 // - More flexible transformations
12739 // - May utilise architecture-dependent API more effectively
12740 // - Currently only one replacement instruction due to adlc parsing capabilities
12741 //
12742 // // Change (inc mov) to lea
12743 // peephole %{
12744 //   // lea should only be emitted when beneficial
12745 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12746 //   // the rule numbers of these nodes inside are passed into the function below
12747 //   peepmatch ( incI_rReg movI );
12748 //   // the method that takes the responsibility of transformation
12749 //   peepprocedure ( inc_mov_to_lea );
12750 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
12751 //   // node is passed into the function above
12752 //   peepreplace ( leaI_rReg_immI() );
12753 // %}
12754 
12755 // These instructions is not matched by the matcher but used by the peephole
12756 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
12757 %{
12758   predicate(false);
12759   match(Set dst (AddI src1 src2));
12760   format %{ "leal    $dst, [$src1 + $src2]" %}
12761   ins_encode %{
12762     Register dst = $dst$$Register;
12763     Register src1 = $src1$$Register;
12764     Register src2 = $src2$$Register;
12765     if (src1 != rbp && src1 != r13) {
12766       __ leal(dst, Address(src1, src2, Address::times_1));
12767     } else {
12768       assert(src2 != rbp && src2 != r13, "");
12769       __ leal(dst, Address(src2, src1, Address::times_1));
12770     }
12771   %}
12772   ins_pipe(ialu_reg_reg);
12773 %}
12774 
12775 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
12776 %{
12777   predicate(false);
12778   match(Set dst (AddI src1 src2));
12779   format %{ "leal    $dst, [$src1 + $src2]" %}
12780   ins_encode %{
12781     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
12782   %}
12783   ins_pipe(ialu_reg_reg);
12784 %}
12785 
12786 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
12787 %{
12788   predicate(false);
12789   match(Set dst (LShiftI src shift));
12790   format %{ "leal    $dst, [$src << $shift]" %}
12791   ins_encode %{
12792     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12793     Register src = $src$$Register;
12794     if (scale == Address::times_2 && src != rbp && src != r13) {
12795       __ leal($dst$$Register, Address(src, src, Address::times_1));
12796     } else {
12797       __ leal($dst$$Register, Address(noreg, src, scale));
12798     }
12799   %}
12800   ins_pipe(ialu_reg_reg);
12801 %}
12802 
12803 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
12804 %{
12805   predicate(false);
12806   match(Set dst (AddL src1 src2));
12807   format %{ "leaq    $dst, [$src1 + $src2]" %}
12808   ins_encode %{
12809     Register dst = $dst$$Register;
12810     Register src1 = $src1$$Register;
12811     Register src2 = $src2$$Register;
12812     if (src1 != rbp && src1 != r13) {
12813       __ leaq(dst, Address(src1, src2, Address::times_1));
12814     } else {
12815       assert(src2 != rbp && src2 != r13, "");
12816       __ leaq(dst, Address(src2, src1, Address::times_1));
12817     }
12818   %}
12819   ins_pipe(ialu_reg_reg);
12820 %}
12821 
12822 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
12823 %{
12824   predicate(false);
12825   match(Set dst (AddL src1 src2));
12826   format %{ "leaq    $dst, [$src1 + $src2]" %}
12827   ins_encode %{
12828     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
12829   %}
12830   ins_pipe(ialu_reg_reg);
12831 %}
12832 
12833 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
12834 %{
12835   predicate(false);
12836   match(Set dst (LShiftL src shift));
12837   format %{ "leaq    $dst, [$src << $shift]" %}
12838   ins_encode %{
12839     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12840     Register src = $src$$Register;
12841     if (scale == Address::times_2 && src != rbp && src != r13) {
12842       __ leaq($dst$$Register, Address(src, src, Address::times_1));
12843     } else {
12844       __ leaq($dst$$Register, Address(noreg, src, scale));
12845     }
12846   %}
12847   ins_pipe(ialu_reg_reg);
12848 %}
12849 
12850 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
12851 // sal}) with lea instructions. The {add, sal} rules are beneficial in
12852 // processors with at least partial ALU support for lea
12853 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
12854 // beneficial for processors with full ALU support
12855 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
12856 
12857 peephole
12858 %{
12859   peeppredicate(VM_Version::supports_fast_2op_lea());
12860   peepmatch (addI_rReg);
12861   peepprocedure (lea_coalesce_reg);
12862   peepreplace (leaI_rReg_rReg_peep());
12863 %}
12864 
12865 peephole
12866 %{
12867   peeppredicate(VM_Version::supports_fast_2op_lea());
12868   peepmatch (addI_rReg_imm);
12869   peepprocedure (lea_coalesce_imm);
12870   peepreplace (leaI_rReg_immI_peep());
12871 %}
12872 
12873 peephole
12874 %{
12875   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12876                 VM_Version::is_intel_cascade_lake());
12877   peepmatch (incI_rReg);
12878   peepprocedure (lea_coalesce_imm);
12879   peepreplace (leaI_rReg_immI_peep());
12880 %}
12881 
12882 peephole
12883 %{
12884   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12885                 VM_Version::is_intel_cascade_lake());
12886   peepmatch (decI_rReg);
12887   peepprocedure (lea_coalesce_imm);
12888   peepreplace (leaI_rReg_immI_peep());
12889 %}
12890 
12891 peephole
12892 %{
12893   peeppredicate(VM_Version::supports_fast_2op_lea());
12894   peepmatch (salI_rReg_immI2);
12895   peepprocedure (lea_coalesce_imm);
12896   peepreplace (leaI_rReg_immI2_peep());
12897 %}
12898 
12899 peephole
12900 %{
12901   peeppredicate(VM_Version::supports_fast_2op_lea());
12902   peepmatch (addL_rReg);
12903   peepprocedure (lea_coalesce_reg);
12904   peepreplace (leaL_rReg_rReg_peep());
12905 %}
12906 
12907 peephole
12908 %{
12909   peeppredicate(VM_Version::supports_fast_2op_lea());
12910   peepmatch (addL_rReg_imm);
12911   peepprocedure (lea_coalesce_imm);
12912   peepreplace (leaL_rReg_immL32_peep());
12913 %}
12914 
12915 peephole
12916 %{
12917   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12918                 VM_Version::is_intel_cascade_lake());
12919   peepmatch (incL_rReg);
12920   peepprocedure (lea_coalesce_imm);
12921   peepreplace (leaL_rReg_immL32_peep());
12922 %}
12923 
12924 peephole
12925 %{
12926   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12927                 VM_Version::is_intel_cascade_lake());
12928   peepmatch (decL_rReg);
12929   peepprocedure (lea_coalesce_imm);
12930   peepreplace (leaL_rReg_immL32_peep());
12931 %}
12932 
12933 peephole
12934 %{
12935   peeppredicate(VM_Version::supports_fast_2op_lea());
12936   peepmatch (salL_rReg_immI2);
12937   peepprocedure (lea_coalesce_imm);
12938   peepreplace (leaL_rReg_immI2_peep());
12939 %}
12940 
12941 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
12942 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
12943 
12944 //int variant
12945 peephole
12946 %{
12947   peepmatch (testI_reg);
12948   peepprocedure (test_may_remove);
12949 %}
12950 
12951 //long variant
12952 peephole
12953 %{
12954   peepmatch (testL_reg);
12955   peepprocedure (test_may_remove);
12956 %}
12957 
12958 
12959 //----------SMARTSPILL RULES---------------------------------------------------
12960 // These must follow all instruction definitions as they use the names
12961 // defined in the instructions definitions.