1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RDX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RDI int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 
  323 source_hpp %{
  324 
  325 #include "peephole_x86_64.hpp"
  326 
  327 %}
  328 
  329 // Register masks
  330 source_hpp %{
  331 
  332 extern RegMask _ANY_REG_mask;
  333 extern RegMask _PTR_REG_mask;
  334 extern RegMask _PTR_REG_NO_RBP_mask;
  335 extern RegMask _PTR_NO_RAX_REG_mask;
  336 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  337 extern RegMask _LONG_REG_mask;
  338 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  339 extern RegMask _LONG_NO_RCX_REG_mask;
  340 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  341 extern RegMask _INT_REG_mask;
  342 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  343 extern RegMask _INT_NO_RCX_REG_mask;
  344 extern RegMask _INT_NO_RBP_R13_REG_mask;
  345 extern RegMask _FLOAT_REG_mask;
  346 
  347 extern RegMask _STACK_OR_PTR_REG_mask;
  348 extern RegMask _STACK_OR_LONG_REG_mask;
  349 extern RegMask _STACK_OR_INT_REG_mask;
  350 
  351 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  352 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  353 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  354 
  355 %}
  356 
  357 source %{
  358 #define   RELOC_IMM64    Assembler::imm_operand
  359 #define   RELOC_DISP32   Assembler::disp32_operand
  360 
  361 #define __ _masm.
  362 
  363 RegMask _ANY_REG_mask;
  364 RegMask _PTR_REG_mask;
  365 RegMask _PTR_REG_NO_RBP_mask;
  366 RegMask _PTR_NO_RAX_REG_mask;
  367 RegMask _PTR_NO_RAX_RBX_REG_mask;
  368 RegMask _LONG_REG_mask;
  369 RegMask _LONG_NO_RAX_RDX_REG_mask;
  370 RegMask _LONG_NO_RCX_REG_mask;
  371 RegMask _LONG_NO_RBP_R13_REG_mask;
  372 RegMask _INT_REG_mask;
  373 RegMask _INT_NO_RAX_RDX_REG_mask;
  374 RegMask _INT_NO_RCX_REG_mask;
  375 RegMask _INT_NO_RBP_R13_REG_mask;
  376 RegMask _FLOAT_REG_mask;
  377 RegMask _STACK_OR_PTR_REG_mask;
  378 RegMask _STACK_OR_LONG_REG_mask;
  379 RegMask _STACK_OR_INT_REG_mask;
  380 
  381 static bool need_r12_heapbase() {
  382   return UseCompressedOops;
  383 }
  384 
  385 void reg_mask_init() {
  386   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  387   // We derive a number of subsets from it.
  388   _ANY_REG_mask = _ALL_REG_mask;
  389 
  390   if (PreserveFramePointer) {
  391     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  392     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  393   }
  394   if (need_r12_heapbase()) {
  395     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  396     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  397   }
  398 
  399   _PTR_REG_mask = _ANY_REG_mask;
  400   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  401   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  402   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  403   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  404 
  405   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  406   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  407 
  408   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  409   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  410   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  411 
  412   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  413   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  414   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  415 
  416   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  417   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  418   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  419 
  420   _LONG_REG_mask = _PTR_REG_mask;
  421   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  422   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  423 
  424   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  425   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  426   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  427   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  428   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  429 
  430   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  431   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  432   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  433 
  434   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  435   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  436   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  437   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  438   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  439 
  440   _INT_REG_mask = _ALL_INT_REG_mask;
  441   if (PreserveFramePointer) {
  442     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  443   }
  444   if (need_r12_heapbase()) {
  445     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  446   }
  447 
  448   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  449   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  450 
  451   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  452   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  453   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  454 
  455   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  456   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  457 
  458   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  459   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  460   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  461 
  462   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  463   // from the float_reg_legacy/float_reg_evex register class.
  464   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  465 }
  466 
  467 static bool generate_vzeroupper(Compile* C) {
  468   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  469 }
  470 
  471 static int clear_avx_size() {
  472   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  473 }
  474 
  475 // !!!!! Special hack to get all types of calls to specify the byte offset
  476 //       from the start of the call to the point where the return address
  477 //       will point.
  478 int MachCallStaticJavaNode::ret_addr_offset()
  479 {
  480   int offset = 5; // 5 bytes from start of call to where return address points
  481   offset += clear_avx_size();
  482   return offset;
  483 }
  484 
  485 int MachCallDynamicJavaNode::ret_addr_offset()
  486 {
  487   int offset = 15; // 15 bytes from start of call to where return address points
  488   offset += clear_avx_size();
  489   return offset;
  490 }
  491 
  492 int MachCallRuntimeNode::ret_addr_offset() {
  493   int offset = 13; // movq r10,#addr; callq (r10)
  494   if (this->ideal_Opcode() != Op_CallLeafVector) {
  495     offset += clear_avx_size();
  496   }
  497   return offset;
  498 }
  499 //
  500 // Compute padding required for nodes which need alignment
  501 //
  502 
  503 // The address of the call instruction needs to be 4-byte aligned to
  504 // ensure that it does not span a cache line so that it can be patched.
  505 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  506 {
  507   current_offset += clear_avx_size(); // skip vzeroupper
  508   current_offset += 1; // skip call opcode byte
  509   return align_up(current_offset, alignment_required()) - current_offset;
  510 }
  511 
  512 // The address of the call instruction needs to be 4-byte aligned to
  513 // ensure that it does not span a cache line so that it can be patched.
  514 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  515 {
  516   current_offset += clear_avx_size(); // skip vzeroupper
  517   current_offset += 11; // skip movq instruction + call opcode byte
  518   return align_up(current_offset, alignment_required()) - current_offset;
  519 }
  520 
  521 // This could be in MacroAssembler but it's fairly C2 specific
  522 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  523   Label exit;
  524   __ jccb(Assembler::noParity, exit);
  525   __ pushf();
  526   //
  527   // comiss/ucomiss instructions set ZF,PF,CF flags and
  528   // zero OF,AF,SF for NaN values.
  529   // Fixup flags by zeroing ZF,PF so that compare of NaN
  530   // values returns 'less than' result (CF is set).
  531   // Leave the rest of flags unchanged.
  532   //
  533   //    7 6 5 4 3 2 1 0
  534   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  535   //    0 0 1 0 1 0 1 1   (0x2B)
  536   //
  537   __ andq(Address(rsp, 0), 0xffffff2b);
  538   __ popf();
  539   __ bind(exit);
  540 }
  541 
  542 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  543   Label done;
  544   __ movl(dst, -1);
  545   __ jcc(Assembler::parity, done);
  546   __ jcc(Assembler::below, done);
  547   __ setb(Assembler::notEqual, dst);
  548   __ movzbl(dst, dst);
  549   __ bind(done);
  550 }
  551 
  552 // Math.min()    # Math.max()
  553 // --------------------------
  554 // ucomis[s/d]   #
  555 // ja   -> b     # a
  556 // jp   -> NaN   # NaN
  557 // jb   -> a     # b
  558 // je            #
  559 // |-jz -> a | b # a & b
  560 // |    -> a     #
  561 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  562                      XMMRegister a, XMMRegister b,
  563                      XMMRegister xmmt, Register rt,
  564                      bool min, bool single) {
  565 
  566   Label nan, zero, below, above, done;
  567 
  568   if (single)
  569     __ ucomiss(a, b);
  570   else
  571     __ ucomisd(a, b);
  572 
  573   if (dst->encoding() != (min ? b : a)->encoding())
  574     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  575   else
  576     __ jccb(Assembler::above, done);
  577 
  578   __ jccb(Assembler::parity, nan);  // PF=1
  579   __ jccb(Assembler::below, below); // CF=1
  580 
  581   // equal
  582   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  583   if (single) {
  584     __ ucomiss(a, xmmt);
  585     __ jccb(Assembler::equal, zero);
  586 
  587     __ movflt(dst, a);
  588     __ jmp(done);
  589   }
  590   else {
  591     __ ucomisd(a, xmmt);
  592     __ jccb(Assembler::equal, zero);
  593 
  594     __ movdbl(dst, a);
  595     __ jmp(done);
  596   }
  597 
  598   __ bind(zero);
  599   if (min)
  600     __ vpor(dst, a, b, Assembler::AVX_128bit);
  601   else
  602     __ vpand(dst, a, b, Assembler::AVX_128bit);
  603 
  604   __ jmp(done);
  605 
  606   __ bind(above);
  607   if (single)
  608     __ movflt(dst, min ? b : a);
  609   else
  610     __ movdbl(dst, min ? b : a);
  611 
  612   __ jmp(done);
  613 
  614   __ bind(nan);
  615   if (single) {
  616     __ movl(rt, 0x7fc00000); // Float.NaN
  617     __ movdl(dst, rt);
  618   }
  619   else {
  620     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  621     __ movdq(dst, rt);
  622   }
  623   __ jmp(done);
  624 
  625   __ bind(below);
  626   if (single)
  627     __ movflt(dst, min ? a : b);
  628   else
  629     __ movdbl(dst, min ? a : b);
  630 
  631   __ bind(done);
  632 }
  633 
  634 //=============================================================================
  635 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  636 
  637 int ConstantTable::calculate_table_base_offset() const {
  638   return 0;  // absolute addressing, no offset
  639 }
  640 
  641 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  642 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  643   ShouldNotReachHere();
  644 }
  645 
  646 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  647   // Empty encoding
  648 }
  649 
  650 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  651   return 0;
  652 }
  653 
  654 #ifndef PRODUCT
  655 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  656   st->print("# MachConstantBaseNode (empty encoding)");
  657 }
  658 #endif
  659 
  660 
  661 //=============================================================================
  662 #ifndef PRODUCT
  663 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  664   Compile* C = ra_->C;
  665 
  666   int framesize = C->output()->frame_size_in_bytes();
  667   int bangsize = C->output()->bang_size_in_bytes();
  668   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  669   // Remove wordSize for return addr which is already pushed.
  670   framesize -= wordSize;
  671 
  672   if (C->output()->need_stack_bang(bangsize)) {
  673     framesize -= wordSize;
  674     st->print("# stack bang (%d bytes)", bangsize);
  675     st->print("\n\t");
  676     st->print("pushq   rbp\t# Save rbp");
  677     if (PreserveFramePointer) {
  678         st->print("\n\t");
  679         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  680     }
  681     if (framesize) {
  682       st->print("\n\t");
  683       st->print("subq    rsp, #%d\t# Create frame",framesize);
  684     }
  685   } else {
  686     st->print("subq    rsp, #%d\t# Create frame",framesize);
  687     st->print("\n\t");
  688     framesize -= wordSize;
  689     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  690     if (PreserveFramePointer) {
  691       st->print("\n\t");
  692       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  693       if (framesize > 0) {
  694         st->print("\n\t");
  695         st->print("addq    rbp, #%d", framesize);
  696       }
  697     }
  698   }
  699 
  700   if (VerifyStackAtCalls) {
  701     st->print("\n\t");
  702     framesize -= wordSize;
  703     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  704 #ifdef ASSERT
  705     st->print("\n\t");
  706     st->print("# stack alignment check");
  707 #endif
  708   }
  709   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  710     st->print("\n\t");
  711     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  712     st->print("\n\t");
  713     st->print("je      fast_entry\t");
  714     st->print("\n\t");
  715     st->print("call    #nmethod_entry_barrier_stub\t");
  716     st->print("\n\tfast_entry:");
  717   }
  718   st->cr();
  719 }
  720 #endif
  721 
  722 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  723   Compile* C = ra_->C;
  724   C2_MacroAssembler _masm(&cbuf);
  725 
  726   int framesize = C->output()->frame_size_in_bytes();
  727   int bangsize = C->output()->bang_size_in_bytes();
  728 
  729   if (C->clinit_barrier_on_entry()) {
  730     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  731     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  732 
  733     Label L_skip_barrier;
  734     Register klass = rscratch1;
  735 
  736     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  737     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  738 
  739     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  740 
  741     __ bind(L_skip_barrier);
  742   }
  743 
  744   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  745 
  746   C->output()->set_frame_complete(cbuf.insts_size());
  747 
  748   if (C->has_mach_constant_base_node()) {
  749     // NOTE: We set the table base offset here because users might be
  750     // emitted before MachConstantBaseNode.
  751     ConstantTable& constant_table = C->output()->constant_table();
  752     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  753   }
  754 }
  755 
  756 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  757 {
  758   return MachNode::size(ra_); // too many variables; just compute it
  759                               // the hard way
  760 }
  761 
  762 int MachPrologNode::reloc() const
  763 {
  764   return 0; // a large enough number
  765 }
  766 
  767 //=============================================================================
  768 #ifndef PRODUCT
  769 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  770 {
  771   Compile* C = ra_->C;
  772   if (generate_vzeroupper(C)) {
  773     st->print("vzeroupper");
  774     st->cr(); st->print("\t");
  775   }
  776 
  777   int framesize = C->output()->frame_size_in_bytes();
  778   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  779   // Remove word for return adr already pushed
  780   // and RBP
  781   framesize -= 2*wordSize;
  782 
  783   if (framesize) {
  784     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  785     st->print("\t");
  786   }
  787 
  788   st->print_cr("popq    rbp");
  789   if (do_polling() && C->is_method_compilation()) {
  790     st->print("\t");
  791     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  792                  "ja      #safepoint_stub\t"
  793                  "# Safepoint: poll for GC");
  794   }
  795 }
  796 #endif
  797 
  798 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  799 {
  800   Compile* C = ra_->C;
  801   MacroAssembler _masm(&cbuf);
  802 
  803   if (generate_vzeroupper(C)) {
  804     // Clear upper bits of YMM registers when current compiled code uses
  805     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  806     __ vzeroupper();
  807   }
  808 
  809   int framesize = C->output()->frame_size_in_bytes();
  810   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  811   // Remove word for return adr already pushed
  812   // and RBP
  813   framesize -= 2*wordSize;
  814 
  815   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  816 
  817   if (framesize) {
  818     __ addq(rsp, framesize);
  819   }
  820 
  821   __ popq(rbp);
  822 
  823   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  824     __ reserved_stack_check();
  825   }
  826 
  827   if (do_polling() && C->is_method_compilation()) {
  828     MacroAssembler _masm(&cbuf);
  829     Label dummy_label;
  830     Label* code_stub = &dummy_label;
  831     if (!C->output()->in_scratch_emit_size()) {
  832       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  833       C->output()->add_stub(stub);
  834       code_stub = &stub->entry();
  835     }
  836     __ relocate(relocInfo::poll_return_type);
  837     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  838   }
  839 }
  840 
  841 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  842 {
  843   return MachNode::size(ra_); // too many variables; just compute it
  844                               // the hard way
  845 }
  846 
  847 int MachEpilogNode::reloc() const
  848 {
  849   return 2; // a large enough number
  850 }
  851 
  852 const Pipeline* MachEpilogNode::pipeline() const
  853 {
  854   return MachNode::pipeline_class();
  855 }
  856 
  857 //=============================================================================
  858 
  859 enum RC {
  860   rc_bad,
  861   rc_int,
  862   rc_kreg,
  863   rc_float,
  864   rc_stack
  865 };
  866 
  867 static enum RC rc_class(OptoReg::Name reg)
  868 {
  869   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  870 
  871   if (OptoReg::is_stack(reg)) return rc_stack;
  872 
  873   VMReg r = OptoReg::as_VMReg(reg);
  874 
  875   if (r->is_Register()) return rc_int;
  876 
  877   if (r->is_KRegister()) return rc_kreg;
  878 
  879   assert(r->is_XMMRegister(), "must be");
  880   return rc_float;
  881 }
  882 
  883 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  884 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
  885                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  886 
  887 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
  888                      int stack_offset, int reg, uint ireg, outputStream* st);
  889 
  890 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
  891                                       int dst_offset, uint ireg, outputStream* st) {
  892   if (cbuf) {
  893     MacroAssembler _masm(cbuf);
  894     switch (ireg) {
  895     case Op_VecS:
  896       __ movq(Address(rsp, -8), rax);
  897       __ movl(rax, Address(rsp, src_offset));
  898       __ movl(Address(rsp, dst_offset), rax);
  899       __ movq(rax, Address(rsp, -8));
  900       break;
  901     case Op_VecD:
  902       __ pushq(Address(rsp, src_offset));
  903       __ popq (Address(rsp, dst_offset));
  904       break;
  905     case Op_VecX:
  906       __ pushq(Address(rsp, src_offset));
  907       __ popq (Address(rsp, dst_offset));
  908       __ pushq(Address(rsp, src_offset+8));
  909       __ popq (Address(rsp, dst_offset+8));
  910       break;
  911     case Op_VecY:
  912       __ vmovdqu(Address(rsp, -32), xmm0);
  913       __ vmovdqu(xmm0, Address(rsp, src_offset));
  914       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  915       __ vmovdqu(xmm0, Address(rsp, -32));
  916       break;
  917     case Op_VecZ:
  918       __ evmovdquq(Address(rsp, -64), xmm0, 2);
  919       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
  920       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
  921       __ evmovdquq(xmm0, Address(rsp, -64), 2);
  922       break;
  923     default:
  924       ShouldNotReachHere();
  925     }
  926 #ifndef PRODUCT
  927   } else {
  928     switch (ireg) {
  929     case Op_VecS:
  930       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
  931                 "movl    rax, [rsp + #%d]\n\t"
  932                 "movl    [rsp + #%d], rax\n\t"
  933                 "movq    rax, [rsp - #8]",
  934                 src_offset, dst_offset);
  935       break;
  936     case Op_VecD:
  937       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
  938                 "popq    [rsp + #%d]",
  939                 src_offset, dst_offset);
  940       break;
  941      case Op_VecX:
  942       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
  943                 "popq    [rsp + #%d]\n\t"
  944                 "pushq   [rsp + #%d]\n\t"
  945                 "popq    [rsp + #%d]",
  946                 src_offset, dst_offset, src_offset+8, dst_offset+8);
  947       break;
  948     case Op_VecY:
  949       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
  950                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  951                 "vmovdqu [rsp + #%d], xmm0\n\t"
  952                 "vmovdqu xmm0, [rsp - #32]",
  953                 src_offset, dst_offset);
  954       break;
  955     case Op_VecZ:
  956       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
  957                 "vmovdqu xmm0, [rsp + #%d]\n\t"
  958                 "vmovdqu [rsp + #%d], xmm0\n\t"
  959                 "vmovdqu xmm0, [rsp - #64]",
  960                 src_offset, dst_offset);
  961       break;
  962     default:
  963       ShouldNotReachHere();
  964     }
  965 #endif
  966   }
  967 }
  968 
  969 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
  970                                        PhaseRegAlloc* ra_,
  971                                        bool do_size,
  972                                        outputStream* st) const {
  973   assert(cbuf != NULL || st  != NULL, "sanity");
  974   // Get registers to move
  975   OptoReg::Name src_second = ra_->get_reg_second(in(1));
  976   OptoReg::Name src_first = ra_->get_reg_first(in(1));
  977   OptoReg::Name dst_second = ra_->get_reg_second(this);
  978   OptoReg::Name dst_first = ra_->get_reg_first(this);
  979 
  980   enum RC src_second_rc = rc_class(src_second);
  981   enum RC src_first_rc = rc_class(src_first);
  982   enum RC dst_second_rc = rc_class(dst_second);
  983   enum RC dst_first_rc = rc_class(dst_first);
  984 
  985   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
  986          "must move at least 1 register" );
  987 
  988   if (src_first == dst_first && src_second == dst_second) {
  989     // Self copy, no move
  990     return 0;
  991   }
  992   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
  993     uint ireg = ideal_reg();
  994     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
  995     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
  996     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
  997       // mem -> mem
  998       int src_offset = ra_->reg2offset(src_first);
  999       int dst_offset = ra_->reg2offset(dst_first);
 1000       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1001     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1002       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1003     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1004       int stack_offset = ra_->reg2offset(dst_first);
 1005       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1006     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1007       int stack_offset = ra_->reg2offset(src_first);
 1008       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1009     } else {
 1010       ShouldNotReachHere();
 1011     }
 1012     return 0;
 1013   }
 1014   if (src_first_rc == rc_stack) {
 1015     // mem ->
 1016     if (dst_first_rc == rc_stack) {
 1017       // mem -> mem
 1018       assert(src_second != dst_first, "overlap");
 1019       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1020           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1021         // 64-bit
 1022         int src_offset = ra_->reg2offset(src_first);
 1023         int dst_offset = ra_->reg2offset(dst_first);
 1024         if (cbuf) {
 1025           MacroAssembler _masm(cbuf);
 1026           __ pushq(Address(rsp, src_offset));
 1027           __ popq (Address(rsp, dst_offset));
 1028 #ifndef PRODUCT
 1029         } else {
 1030           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1031                     "popq    [rsp + #%d]",
 1032                      src_offset, dst_offset);
 1033 #endif
 1034         }
 1035       } else {
 1036         // 32-bit
 1037         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1038         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1039         // No pushl/popl, so:
 1040         int src_offset = ra_->reg2offset(src_first);
 1041         int dst_offset = ra_->reg2offset(dst_first);
 1042         if (cbuf) {
 1043           MacroAssembler _masm(cbuf);
 1044           __ movq(Address(rsp, -8), rax);
 1045           __ movl(rax, Address(rsp, src_offset));
 1046           __ movl(Address(rsp, dst_offset), rax);
 1047           __ movq(rax, Address(rsp, -8));
 1048 #ifndef PRODUCT
 1049         } else {
 1050           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                     "movl    rax, [rsp + #%d]\n\t"
 1052                     "movl    [rsp + #%d], rax\n\t"
 1053                     "movq    rax, [rsp - #8]",
 1054                      src_offset, dst_offset);
 1055 #endif
 1056         }
 1057       }
 1058       return 0;
 1059     } else if (dst_first_rc == rc_int) {
 1060       // mem -> gpr
 1061       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1062           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1063         // 64-bit
 1064         int offset = ra_->reg2offset(src_first);
 1065         if (cbuf) {
 1066           MacroAssembler _masm(cbuf);
 1067           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1068 #ifndef PRODUCT
 1069         } else {
 1070           st->print("movq    %s, [rsp + #%d]\t# spill",
 1071                      Matcher::regName[dst_first],
 1072                      offset);
 1073 #endif
 1074         }
 1075       } else {
 1076         // 32-bit
 1077         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1078         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1079         int offset = ra_->reg2offset(src_first);
 1080         if (cbuf) {
 1081           MacroAssembler _masm(cbuf);
 1082           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1083 #ifndef PRODUCT
 1084         } else {
 1085           st->print("movl    %s, [rsp + #%d]\t# spill",
 1086                      Matcher::regName[dst_first],
 1087                      offset);
 1088 #endif
 1089         }
 1090       }
 1091       return 0;
 1092     } else if (dst_first_rc == rc_float) {
 1093       // mem-> xmm
 1094       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1095           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1096         // 64-bit
 1097         int offset = ra_->reg2offset(src_first);
 1098         if (cbuf) {
 1099           MacroAssembler _masm(cbuf);
 1100           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1101 #ifndef PRODUCT
 1102         } else {
 1103           st->print("%s  %s, [rsp + #%d]\t# spill",
 1104                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1105                      Matcher::regName[dst_first],
 1106                      offset);
 1107 #endif
 1108         }
 1109       } else {
 1110         // 32-bit
 1111         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1112         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1113         int offset = ra_->reg2offset(src_first);
 1114         if (cbuf) {
 1115           MacroAssembler _masm(cbuf);
 1116           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1117 #ifndef PRODUCT
 1118         } else {
 1119           st->print("movss   %s, [rsp + #%d]\t# spill",
 1120                      Matcher::regName[dst_first],
 1121                      offset);
 1122 #endif
 1123         }
 1124       }
 1125       return 0;
 1126     } else if (dst_first_rc == rc_kreg) {
 1127       // mem -> kreg
 1128       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1129           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1130         // 64-bit
 1131         int offset = ra_->reg2offset(src_first);
 1132         if (cbuf) {
 1133           MacroAssembler _masm(cbuf);
 1134           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1135 #ifndef PRODUCT
 1136         } else {
 1137           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1138                      Matcher::regName[dst_first],
 1139                      offset);
 1140 #endif
 1141         }
 1142       }
 1143       return 0;
 1144     }
 1145   } else if (src_first_rc == rc_int) {
 1146     // gpr ->
 1147     if (dst_first_rc == rc_stack) {
 1148       // gpr -> mem
 1149       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1150           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1151         // 64-bit
 1152         int offset = ra_->reg2offset(dst_first);
 1153         if (cbuf) {
 1154           MacroAssembler _masm(cbuf);
 1155           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1156 #ifndef PRODUCT
 1157         } else {
 1158           st->print("movq    [rsp + #%d], %s\t# spill",
 1159                      offset,
 1160                      Matcher::regName[src_first]);
 1161 #endif
 1162         }
 1163       } else {
 1164         // 32-bit
 1165         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1166         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1167         int offset = ra_->reg2offset(dst_first);
 1168         if (cbuf) {
 1169           MacroAssembler _masm(cbuf);
 1170           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1171 #ifndef PRODUCT
 1172         } else {
 1173           st->print("movl    [rsp + #%d], %s\t# spill",
 1174                      offset,
 1175                      Matcher::regName[src_first]);
 1176 #endif
 1177         }
 1178       }
 1179       return 0;
 1180     } else if (dst_first_rc == rc_int) {
 1181       // gpr -> gpr
 1182       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1183           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1184         // 64-bit
 1185         if (cbuf) {
 1186           MacroAssembler _masm(cbuf);
 1187           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1188                   as_Register(Matcher::_regEncode[src_first]));
 1189 #ifndef PRODUCT
 1190         } else {
 1191           st->print("movq    %s, %s\t# spill",
 1192                      Matcher::regName[dst_first],
 1193                      Matcher::regName[src_first]);
 1194 #endif
 1195         }
 1196         return 0;
 1197       } else {
 1198         // 32-bit
 1199         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1200         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1201         if (cbuf) {
 1202           MacroAssembler _masm(cbuf);
 1203           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1204                   as_Register(Matcher::_regEncode[src_first]));
 1205 #ifndef PRODUCT
 1206         } else {
 1207           st->print("movl    %s, %s\t# spill",
 1208                      Matcher::regName[dst_first],
 1209                      Matcher::regName[src_first]);
 1210 #endif
 1211         }
 1212         return 0;
 1213       }
 1214     } else if (dst_first_rc == rc_float) {
 1215       // gpr -> xmm
 1216       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1217           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1218         // 64-bit
 1219         if (cbuf) {
 1220           MacroAssembler _masm(cbuf);
 1221           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1222 #ifndef PRODUCT
 1223         } else {
 1224           st->print("movdq   %s, %s\t# spill",
 1225                      Matcher::regName[dst_first],
 1226                      Matcher::regName[src_first]);
 1227 #endif
 1228         }
 1229       } else {
 1230         // 32-bit
 1231         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1232         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1233         if (cbuf) {
 1234           MacroAssembler _masm(cbuf);
 1235           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1236 #ifndef PRODUCT
 1237         } else {
 1238           st->print("movdl   %s, %s\t# spill",
 1239                      Matcher::regName[dst_first],
 1240                      Matcher::regName[src_first]);
 1241 #endif
 1242         }
 1243       }
 1244       return 0;
 1245     } else if (dst_first_rc == rc_kreg) {
 1246       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1247           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1248         // 64-bit
 1249         if (cbuf) {
 1250           MacroAssembler _masm(cbuf);
 1251           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1252   #ifndef PRODUCT
 1253         } else {
 1254            st->print("kmovq   %s, %s\t# spill",
 1255                        Matcher::regName[dst_first],
 1256                        Matcher::regName[src_first]);
 1257   #endif
 1258         }
 1259       }
 1260       Unimplemented();
 1261       return 0;
 1262     }
 1263   } else if (src_first_rc == rc_float) {
 1264     // xmm ->
 1265     if (dst_first_rc == rc_stack) {
 1266       // xmm -> mem
 1267       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1268           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1269         // 64-bit
 1270         int offset = ra_->reg2offset(dst_first);
 1271         if (cbuf) {
 1272           MacroAssembler _masm(cbuf);
 1273           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1274 #ifndef PRODUCT
 1275         } else {
 1276           st->print("movsd   [rsp + #%d], %s\t# spill",
 1277                      offset,
 1278                      Matcher::regName[src_first]);
 1279 #endif
 1280         }
 1281       } else {
 1282         // 32-bit
 1283         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1284         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1285         int offset = ra_->reg2offset(dst_first);
 1286         if (cbuf) {
 1287           MacroAssembler _masm(cbuf);
 1288           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1289 #ifndef PRODUCT
 1290         } else {
 1291           st->print("movss   [rsp + #%d], %s\t# spill",
 1292                      offset,
 1293                      Matcher::regName[src_first]);
 1294 #endif
 1295         }
 1296       }
 1297       return 0;
 1298     } else if (dst_first_rc == rc_int) {
 1299       // xmm -> gpr
 1300       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1301           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1302         // 64-bit
 1303         if (cbuf) {
 1304           MacroAssembler _masm(cbuf);
 1305           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1306 #ifndef PRODUCT
 1307         } else {
 1308           st->print("movdq   %s, %s\t# spill",
 1309                      Matcher::regName[dst_first],
 1310                      Matcher::regName[src_first]);
 1311 #endif
 1312         }
 1313       } else {
 1314         // 32-bit
 1315         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1316         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1317         if (cbuf) {
 1318           MacroAssembler _masm(cbuf);
 1319           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1320 #ifndef PRODUCT
 1321         } else {
 1322           st->print("movdl   %s, %s\t# spill",
 1323                      Matcher::regName[dst_first],
 1324                      Matcher::regName[src_first]);
 1325 #endif
 1326         }
 1327       }
 1328       return 0;
 1329     } else if (dst_first_rc == rc_float) {
 1330       // xmm -> xmm
 1331       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1332           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1333         // 64-bit
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("%s  %s, %s\t# spill",
 1340                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1341                      Matcher::regName[dst_first],
 1342                      Matcher::regName[src_first]);
 1343 #endif
 1344         }
 1345       } else {
 1346         // 32-bit
 1347         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1348         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("%s  %s, %s\t# spill",
 1355                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1356                      Matcher::regName[dst_first],
 1357                      Matcher::regName[src_first]);
 1358 #endif
 1359         }
 1360       }
 1361       return 0;
 1362     } else if (dst_first_rc == rc_kreg) {
 1363       assert(false, "Illegal spilling");
 1364       return 0;
 1365     }
 1366   } else if (src_first_rc == rc_kreg) {
 1367     if (dst_first_rc == rc_stack) {
 1368       // mem -> kreg
 1369       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1370           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1371         // 64-bit
 1372         int offset = ra_->reg2offset(dst_first);
 1373         if (cbuf) {
 1374           MacroAssembler _masm(cbuf);
 1375           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1376 #ifndef PRODUCT
 1377         } else {
 1378           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1379                      offset,
 1380                      Matcher::regName[src_first]);
 1381 #endif
 1382         }
 1383       }
 1384       return 0;
 1385     } else if (dst_first_rc == rc_int) {
 1386       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1387           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1388         // 64-bit
 1389         if (cbuf) {
 1390           MacroAssembler _masm(cbuf);
 1391           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1392 #ifndef PRODUCT
 1393         } else {
 1394          st->print("kmovq   %s, %s\t# spill",
 1395                      Matcher::regName[dst_first],
 1396                      Matcher::regName[src_first]);
 1397 #endif
 1398         }
 1399       }
 1400       Unimplemented();
 1401       return 0;
 1402     } else if (dst_first_rc == rc_kreg) {
 1403       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1404           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1405         // 64-bit
 1406         if (cbuf) {
 1407           MacroAssembler _masm(cbuf);
 1408           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411          st->print("kmovq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       }
 1417       return 0;
 1418     } else if (dst_first_rc == rc_float) {
 1419       assert(false, "Illegal spill");
 1420       return 0;
 1421     }
 1422   }
 1423 
 1424   assert(0," foo ");
 1425   Unimplemented();
 1426   return 0;
 1427 }
 1428 
 1429 #ifndef PRODUCT
 1430 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1431   implementation(NULL, ra_, false, st);
 1432 }
 1433 #endif
 1434 
 1435 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1436   implementation(&cbuf, ra_, false, NULL);
 1437 }
 1438 
 1439 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1440   return MachNode::size(ra_);
 1441 }
 1442 
 1443 //=============================================================================
 1444 #ifndef PRODUCT
 1445 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1446 {
 1447   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1448   int reg = ra_->get_reg_first(this);
 1449   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1450             Matcher::regName[reg], offset);
 1451 }
 1452 #endif
 1453 
 1454 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1455 {
 1456   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1457   int reg = ra_->get_encode(this);
 1458 
 1459   MacroAssembler masm(&cbuf);
 1460   masm.lea(as_Register(reg), Address(rsp, offset));
 1461 }
 1462 
 1463 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1464 {
 1465   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1466   return (offset < 0x80) ? 5 : 8; // REX
 1467 }
 1468 
 1469 //=============================================================================
 1470 #ifndef PRODUCT
 1471 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1472 {
 1473   if (UseCompressedClassPointers) {
 1474     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1475     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1476     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1477   } else {
 1478     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1479                  "# Inline cache check");
 1480   }
 1481   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1482   st->print_cr("\tnop\t# nops to align entry point");
 1483 }
 1484 #endif
 1485 
 1486 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1487 {
 1488   MacroAssembler masm(&cbuf);
 1489   uint insts_size = cbuf.insts_size();
 1490   if (UseCompressedClassPointers) {
 1491     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1492     masm.cmpptr(rax, rscratch1);
 1493   } else {
 1494     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1495   }
 1496 
 1497   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1498 
 1499   /* WARNING these NOPs are critical so that verified entry point is properly
 1500      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1501   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1502   if (OptoBreakpoint) {
 1503     // Leave space for int3
 1504     nops_cnt -= 1;
 1505   }
 1506   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1507   if (nops_cnt > 0)
 1508     masm.nop(nops_cnt);
 1509 }
 1510 
 1511 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1512 {
 1513   return MachNode::size(ra_); // too many variables; just compute it
 1514                               // the hard way
 1515 }
 1516 
 1517 
 1518 //=============================================================================
 1519 
 1520 bool Matcher::supports_vector_calling_convention(void) {
 1521   if (EnableVectorSupport && UseVectorStubs) {
 1522     return true;
 1523   }
 1524   return false;
 1525 }
 1526 
 1527 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1528   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1529   int lo = XMM0_num;
 1530   int hi = XMM0b_num;
 1531   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1532   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1533   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1534   return OptoRegPair(hi, lo);
 1535 }
 1536 
 1537 // Is this branch offset short enough that a short branch can be used?
 1538 //
 1539 // NOTE: If the platform does not provide any short branch variants, then
 1540 //       this method should return false for offset 0.
 1541 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1542   // The passed offset is relative to address of the branch.
 1543   // On 86 a branch displacement is calculated relative to address
 1544   // of a next instruction.
 1545   offset -= br_size;
 1546 
 1547   // the short version of jmpConUCF2 contains multiple branches,
 1548   // making the reach slightly less
 1549   if (rule == jmpConUCF2_rule)
 1550     return (-126 <= offset && offset <= 125);
 1551   return (-128 <= offset && offset <= 127);
 1552 }
 1553 
 1554 // Return whether or not this register is ever used as an argument.
 1555 // This function is used on startup to build the trampoline stubs in
 1556 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1557 // call in the trampoline, and arguments in those registers not be
 1558 // available to the callee.
 1559 bool Matcher::can_be_java_arg(int reg)
 1560 {
 1561   return
 1562     reg ==  RDI_num || reg == RDI_H_num ||
 1563     reg ==  RSI_num || reg == RSI_H_num ||
 1564     reg ==  RDX_num || reg == RDX_H_num ||
 1565     reg ==  RCX_num || reg == RCX_H_num ||
 1566     reg ==   R8_num || reg ==  R8_H_num ||
 1567     reg ==   R9_num || reg ==  R9_H_num ||
 1568     reg ==  R12_num || reg == R12_H_num ||
 1569     reg == XMM0_num || reg == XMM0b_num ||
 1570     reg == XMM1_num || reg == XMM1b_num ||
 1571     reg == XMM2_num || reg == XMM2b_num ||
 1572     reg == XMM3_num || reg == XMM3b_num ||
 1573     reg == XMM4_num || reg == XMM4b_num ||
 1574     reg == XMM5_num || reg == XMM5b_num ||
 1575     reg == XMM6_num || reg == XMM6b_num ||
 1576     reg == XMM7_num || reg == XMM7b_num;
 1577 }
 1578 
 1579 bool Matcher::is_spillable_arg(int reg)
 1580 {
 1581   return can_be_java_arg(reg);
 1582 }
 1583 
 1584 uint Matcher::int_pressure_limit()
 1585 {
 1586   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1587 }
 1588 
 1589 uint Matcher::float_pressure_limit()
 1590 {
 1591   // After experiment around with different values, the following default threshold
 1592   // works best for LCM's register pressure scheduling on x64.
 1593   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1594   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1595   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1596 }
 1597 
 1598 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1599   // In 64 bit mode a code which use multiply when
 1600   // devisor is constant is faster than hardware
 1601   // DIV instruction (it uses MulHiL).
 1602   return false;
 1603 }
 1604 
 1605 // Register for DIVI projection of divmodI
 1606 RegMask Matcher::divI_proj_mask() {
 1607   return INT_RAX_REG_mask();
 1608 }
 1609 
 1610 // Register for MODI projection of divmodI
 1611 RegMask Matcher::modI_proj_mask() {
 1612   return INT_RDX_REG_mask();
 1613 }
 1614 
 1615 // Register for DIVL projection of divmodL
 1616 RegMask Matcher::divL_proj_mask() {
 1617   return LONG_RAX_REG_mask();
 1618 }
 1619 
 1620 // Register for MODL projection of divmodL
 1621 RegMask Matcher::modL_proj_mask() {
 1622   return LONG_RDX_REG_mask();
 1623 }
 1624 
 1625 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1626 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1627     return NO_REG_mask();
 1628 }
 1629 
 1630 %}
 1631 
 1632 //----------ENCODING BLOCK-----------------------------------------------------
 1633 // This block specifies the encoding classes used by the compiler to
 1634 // output byte streams.  Encoding classes are parameterized macros
 1635 // used by Machine Instruction Nodes in order to generate the bit
 1636 // encoding of the instruction.  Operands specify their base encoding
 1637 // interface with the interface keyword.  There are currently
 1638 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1639 // COND_INTER.  REG_INTER causes an operand to generate a function
 1640 // which returns its register number when queried.  CONST_INTER causes
 1641 // an operand to generate a function which returns the value of the
 1642 // constant when queried.  MEMORY_INTER causes an operand to generate
 1643 // four functions which return the Base Register, the Index Register,
 1644 // the Scale Value, and the Offset Value of the operand when queried.
 1645 // COND_INTER causes an operand to generate six functions which return
 1646 // the encoding code (ie - encoding bits for the instruction)
 1647 // associated with each basic boolean condition for a conditional
 1648 // instruction.
 1649 //
 1650 // Instructions specify two basic values for encoding.  Again, a
 1651 // function is available to check if the constant displacement is an
 1652 // oop. They use the ins_encode keyword to specify their encoding
 1653 // classes (which must be a sequence of enc_class names, and their
 1654 // parameters, specified in the encoding block), and they use the
 1655 // opcode keyword to specify, in order, their primary, secondary, and
 1656 // tertiary opcode.  Only the opcode sections which a particular
 1657 // instruction needs for encoding need to be specified.
 1658 encode %{
 1659   enc_class cdql_enc(no_rax_rdx_RegI div)
 1660   %{
 1661     // Full implementation of Java idiv and irem; checks for
 1662     // special case as described in JVM spec., p.243 & p.271.
 1663     //
 1664     //         normal case                           special case
 1665     //
 1666     // input : rax: dividend                         min_int
 1667     //         reg: divisor                          -1
 1668     //
 1669     // output: rax: quotient  (= rax idiv reg)       min_int
 1670     //         rdx: remainder (= rax irem reg)       0
 1671     //
 1672     //  Code sequnce:
 1673     //
 1674     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1675     //    5:   75 07/08                jne    e <normal>
 1676     //    7:   33 d2                   xor    %edx,%edx
 1677     //  [div >= 8 -> offset + 1]
 1678     //  [REX_B]
 1679     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1680     //    c:   74 03/04                je     11 <done>
 1681     // 000000000000000e <normal>:
 1682     //    e:   99                      cltd
 1683     //  [div >= 8 -> offset + 1]
 1684     //  [REX_B]
 1685     //    f:   f7 f9                   idiv   $div
 1686     // 0000000000000011 <done>:
 1687     MacroAssembler _masm(&cbuf);
 1688     Label normal;
 1689     Label done;
 1690 
 1691     // cmp    $0x80000000,%eax
 1692     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1693 
 1694     // jne    e <normal>
 1695     __ jccb(Assembler::notEqual, normal);
 1696 
 1697     // xor    %edx,%edx
 1698     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1699 
 1700     // cmp    $0xffffffffffffffff,%ecx
 1701     __ cmpl($div$$Register, -1);
 1702 
 1703     // je     11 <done>
 1704     __ jccb(Assembler::equal, done);
 1705 
 1706     // <normal>
 1707     // cltd
 1708     __ bind(normal);
 1709     __ cdql();
 1710 
 1711     // idivl
 1712     // <done>
 1713     __ idivl($div$$Register);
 1714     __ bind(done);
 1715   %}
 1716 
 1717   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1718   %{
 1719     // Full implementation of Java ldiv and lrem; checks for
 1720     // special case as described in JVM spec., p.243 & p.271.
 1721     //
 1722     //         normal case                           special case
 1723     //
 1724     // input : rax: dividend                         min_long
 1725     //         reg: divisor                          -1
 1726     //
 1727     // output: rax: quotient  (= rax idiv reg)       min_long
 1728     //         rdx: remainder (= rax irem reg)       0
 1729     //
 1730     //  Code sequnce:
 1731     //
 1732     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1733     //    7:   00 00 80
 1734     //    a:   48 39 d0                cmp    %rdx,%rax
 1735     //    d:   75 08                   jne    17 <normal>
 1736     //    f:   33 d2                   xor    %edx,%edx
 1737     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1738     //   15:   74 05                   je     1c <done>
 1739     // 0000000000000017 <normal>:
 1740     //   17:   48 99                   cqto
 1741     //   19:   48 f7 f9                idiv   $div
 1742     // 000000000000001c <done>:
 1743     MacroAssembler _masm(&cbuf);
 1744     Label normal;
 1745     Label done;
 1746 
 1747     // mov    $0x8000000000000000,%rdx
 1748     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1749 
 1750     // cmp    %rdx,%rax
 1751     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1752 
 1753     // jne    17 <normal>
 1754     __ jccb(Assembler::notEqual, normal);
 1755 
 1756     // xor    %edx,%edx
 1757     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1758 
 1759     // cmp    $0xffffffffffffffff,$div
 1760     __ cmpq($div$$Register, -1);
 1761 
 1762     // je     1e <done>
 1763     __ jccb(Assembler::equal, done);
 1764 
 1765     // <normal>
 1766     // cqto
 1767     __ bind(normal);
 1768     __ cdqq();
 1769 
 1770     // idivq (note: must be emitted by the user of this rule)
 1771     // <done>
 1772     __ idivq($div$$Register);
 1773     __ bind(done);
 1774   %}
 1775 
 1776   enc_class enc_PartialSubtypeCheck()
 1777   %{
 1778     Register Rrdi = as_Register(RDI_enc); // result register
 1779     Register Rrax = as_Register(RAX_enc); // super class
 1780     Register Rrcx = as_Register(RCX_enc); // killed
 1781     Register Rrsi = as_Register(RSI_enc); // sub class
 1782     Label miss;
 1783     const bool set_cond_codes = true;
 1784 
 1785     MacroAssembler _masm(&cbuf);
 1786     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 1787                                      NULL, &miss,
 1788                                      /*set_cond_codes:*/ true);
 1789     if ($primary) {
 1790       __ xorptr(Rrdi, Rrdi);
 1791     }
 1792     __ bind(miss);
 1793   %}
 1794 
 1795   enc_class clear_avx %{
 1796     debug_only(int off0 = cbuf.insts_size());
 1797     if (generate_vzeroupper(Compile::current())) {
 1798       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1799       // Clear upper bits of YMM registers when current compiled code uses
 1800       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1801       MacroAssembler _masm(&cbuf);
 1802       __ vzeroupper();
 1803     }
 1804     debug_only(int off1 = cbuf.insts_size());
 1805     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1806   %}
 1807 
 1808   enc_class Java_To_Runtime(method meth) %{
 1809     // No relocation needed
 1810     MacroAssembler _masm(&cbuf);
 1811     __ mov64(r10, (int64_t) $meth$$method);
 1812     __ call(r10);
 1813     __ post_call_nop();
 1814   %}
 1815 
 1816   enc_class Java_Static_Call(method meth)
 1817   %{
 1818     // JAVA STATIC CALL
 1819     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1820     // determine who we intended to call.
 1821     MacroAssembler _masm(&cbuf);
 1822 
 1823     if (!_method) {
 1824       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1825     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1826       // The NOP here is purely to ensure that eliding a call to
 1827       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1828       __ addr_nop_5();
 1829       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1830     } else {
 1831       int method_index = resolved_method_index(cbuf);
 1832       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1833                                                   : static_call_Relocation::spec(method_index);
 1834       address mark = __ pc();
 1835       int call_offset = __ offset();
 1836       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1837       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1838         // Calls of the same statically bound method can share
 1839         // a stub to the interpreter.
 1840         cbuf.shared_stub_to_interp_for(_method, call_offset);
 1841       } else {
 1842         // Emit stubs for static call.
 1843         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 1844         if (stub == NULL) {
 1845           ciEnv::current()->record_failure("CodeCache is full");
 1846           return;
 1847         }
 1848       }
 1849     }
 1850     __ post_call_nop();
 1851   %}
 1852 
 1853   enc_class Java_Dynamic_Call(method meth) %{
 1854     MacroAssembler _masm(&cbuf);
 1855     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 1856     __ post_call_nop();
 1857   %}
 1858 
 1859 %}
 1860 
 1861 
 1862 
 1863 //----------FRAME--------------------------------------------------------------
 1864 // Definition of frame structure and management information.
 1865 //
 1866 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1867 //                             |   (to get allocators register number
 1868 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1869 //  r   CALLER     |        |
 1870 //  o     |        +--------+      pad to even-align allocators stack-slot
 1871 //  w     V        |  pad0  |        numbers; owned by CALLER
 1872 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1873 //  h     ^        |   in   |  5
 1874 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1875 //  |     |        |        |  3
 1876 //  |     |        +--------+
 1877 //  V     |        | old out|      Empty on Intel, window on Sparc
 1878 //        |    old |preserve|      Must be even aligned.
 1879 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1880 //        |        |   in   |  3   area for Intel ret address
 1881 //     Owned by    |preserve|      Empty on Sparc.
 1882 //       SELF      +--------+
 1883 //        |        |  pad2  |  2   pad to align old SP
 1884 //        |        +--------+  1
 1885 //        |        | locks  |  0
 1886 //        |        +--------+----> OptoReg::stack0(), even aligned
 1887 //        |        |  pad1  | 11   pad to align new SP
 1888 //        |        +--------+
 1889 //        |        |        | 10
 1890 //        |        | spills |  9   spills
 1891 //        V        |        |  8   (pad0 slot for callee)
 1892 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1893 //        ^        |  out   |  7
 1894 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1895 //     Owned by    +--------+
 1896 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1897 //        |    new |preserve|      Must be even-aligned.
 1898 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1899 //        |        |        |
 1900 //
 1901 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1902 //         known from SELF's arguments and the Java calling convention.
 1903 //         Region 6-7 is determined per call site.
 1904 // Note 2: If the calling convention leaves holes in the incoming argument
 1905 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1906 //         are owned by the CALLEE.  Holes should not be necessary in the
 1907 //         incoming area, as the Java calling convention is completely under
 1908 //         the control of the AD file.  Doubles can be sorted and packed to
 1909 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1910 //         varargs C calling conventions.
 1911 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1912 //         even aligned with pad0 as needed.
 1913 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1914 //         region 6-11 is even aligned; it may be padded out more so that
 1915 //         the region from SP to FP meets the minimum stack alignment.
 1916 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1917 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1918 //         SP meets the minimum alignment.
 1919 
 1920 frame
 1921 %{
 1922   // These three registers define part of the calling convention
 1923   // between compiled code and the interpreter.
 1924   inline_cache_reg(RAX);                // Inline Cache Register
 1925 
 1926   // Optional: name the operand used by cisc-spilling to access
 1927   // [stack_pointer + offset]
 1928   cisc_spilling_operand_name(indOffset32);
 1929 
 1930   // Number of stack slots consumed by locking an object
 1931   sync_stack_slots(2);
 1932 
 1933   // Compiled code's Frame Pointer
 1934   frame_pointer(RSP);
 1935 
 1936   // Interpreter stores its frame pointer in a register which is
 1937   // stored to the stack by I2CAdaptors.
 1938   // I2CAdaptors convert from interpreted java to compiled java.
 1939   interpreter_frame_pointer(RBP);
 1940 
 1941   // Stack alignment requirement
 1942   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1943 
 1944   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1945   // for calls to C.  Supports the var-args backing area for register parms.
 1946   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1947 
 1948   // The after-PROLOG location of the return address.  Location of
 1949   // return address specifies a type (REG or STACK) and a number
 1950   // representing the register number (i.e. - use a register name) or
 1951   // stack slot.
 1952   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1953   // Otherwise, it is above the locks and verification slot and alignment word
 1954   return_addr(STACK - 2 +
 1955               align_up((Compile::current()->in_preserve_stack_slots() +
 1956                         Compile::current()->fixed_slots()),
 1957                        stack_alignment_in_slots()));
 1958 
 1959   // Location of compiled Java return values.  Same as C for now.
 1960   return_value
 1961   %{
 1962     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 1963            "only return normal values");
 1964 
 1965     static const int lo[Op_RegL + 1] = {
 1966       0,
 1967       0,
 1968       RAX_num,  // Op_RegN
 1969       RAX_num,  // Op_RegI
 1970       RAX_num,  // Op_RegP
 1971       XMM0_num, // Op_RegF
 1972       XMM0_num, // Op_RegD
 1973       RAX_num   // Op_RegL
 1974     };
 1975     static const int hi[Op_RegL + 1] = {
 1976       0,
 1977       0,
 1978       OptoReg::Bad, // Op_RegN
 1979       OptoReg::Bad, // Op_RegI
 1980       RAX_H_num,    // Op_RegP
 1981       OptoReg::Bad, // Op_RegF
 1982       XMM0b_num,    // Op_RegD
 1983       RAX_H_num     // Op_RegL
 1984     };
 1985     // Excluded flags and vector registers.
 1986     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 1987     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 1988   %}
 1989 %}
 1990 
 1991 //----------ATTRIBUTES---------------------------------------------------------
 1992 //----------Operand Attributes-------------------------------------------------
 1993 op_attrib op_cost(0);        // Required cost attribute
 1994 
 1995 //----------Instruction Attributes---------------------------------------------
 1996 ins_attrib ins_cost(100);       // Required cost attribute
 1997 ins_attrib ins_size(8);         // Required size attribute (in bits)
 1998 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 1999                                 // a non-matching short branch variant
 2000                                 // of some long branch?
 2001 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2002                                 // be a power of 2) specifies the
 2003                                 // alignment that some part of the
 2004                                 // instruction (not necessarily the
 2005                                 // start) requires.  If > 1, a
 2006                                 // compute_padding() function must be
 2007                                 // provided for the instruction
 2008 
 2009 //----------OPERANDS-----------------------------------------------------------
 2010 // Operand definitions must precede instruction definitions for correct parsing
 2011 // in the ADLC because operands constitute user defined types which are used in
 2012 // instruction definitions.
 2013 
 2014 //----------Simple Operands----------------------------------------------------
 2015 // Immediate Operands
 2016 // Integer Immediate
 2017 operand immI()
 2018 %{
 2019   match(ConI);
 2020 
 2021   op_cost(10);
 2022   format %{ %}
 2023   interface(CONST_INTER);
 2024 %}
 2025 
 2026 // Constant for test vs zero
 2027 operand immI_0()
 2028 %{
 2029   predicate(n->get_int() == 0);
 2030   match(ConI);
 2031 
 2032   op_cost(0);
 2033   format %{ %}
 2034   interface(CONST_INTER);
 2035 %}
 2036 
 2037 // Constant for increment
 2038 operand immI_1()
 2039 %{
 2040   predicate(n->get_int() == 1);
 2041   match(ConI);
 2042 
 2043   op_cost(0);
 2044   format %{ %}
 2045   interface(CONST_INTER);
 2046 %}
 2047 
 2048 // Constant for decrement
 2049 operand immI_M1()
 2050 %{
 2051   predicate(n->get_int() == -1);
 2052   match(ConI);
 2053 
 2054   op_cost(0);
 2055   format %{ %}
 2056   interface(CONST_INTER);
 2057 %}
 2058 
 2059 operand immI_2()
 2060 %{
 2061   predicate(n->get_int() == 2);
 2062   match(ConI);
 2063 
 2064   op_cost(0);
 2065   format %{ %}
 2066   interface(CONST_INTER);
 2067 %}
 2068 
 2069 operand immI_4()
 2070 %{
 2071   predicate(n->get_int() == 4);
 2072   match(ConI);
 2073 
 2074   op_cost(0);
 2075   format %{ %}
 2076   interface(CONST_INTER);
 2077 %}
 2078 
 2079 operand immI_8()
 2080 %{
 2081   predicate(n->get_int() == 8);
 2082   match(ConI);
 2083 
 2084   op_cost(0);
 2085   format %{ %}
 2086   interface(CONST_INTER);
 2087 %}
 2088 
 2089 // Valid scale values for addressing modes
 2090 operand immI2()
 2091 %{
 2092   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2093   match(ConI);
 2094 
 2095   format %{ %}
 2096   interface(CONST_INTER);
 2097 %}
 2098 
 2099 operand immU7()
 2100 %{
 2101   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2102   match(ConI);
 2103 
 2104   op_cost(5);
 2105   format %{ %}
 2106   interface(CONST_INTER);
 2107 %}
 2108 
 2109 operand immI8()
 2110 %{
 2111   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2112   match(ConI);
 2113 
 2114   op_cost(5);
 2115   format %{ %}
 2116   interface(CONST_INTER);
 2117 %}
 2118 
 2119 operand immU8()
 2120 %{
 2121   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2122   match(ConI);
 2123 
 2124   op_cost(5);
 2125   format %{ %}
 2126   interface(CONST_INTER);
 2127 %}
 2128 
 2129 operand immI16()
 2130 %{
 2131   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2132   match(ConI);
 2133 
 2134   op_cost(10);
 2135   format %{ %}
 2136   interface(CONST_INTER);
 2137 %}
 2138 
 2139 // Int Immediate non-negative
 2140 operand immU31()
 2141 %{
 2142   predicate(n->get_int() >= 0);
 2143   match(ConI);
 2144 
 2145   op_cost(0);
 2146   format %{ %}
 2147   interface(CONST_INTER);
 2148 %}
 2149 
 2150 // Constant for long shifts
 2151 operand immI_32()
 2152 %{
 2153   predicate( n->get_int() == 32 );
 2154   match(ConI);
 2155 
 2156   op_cost(0);
 2157   format %{ %}
 2158   interface(CONST_INTER);
 2159 %}
 2160 
 2161 // Constant for long shifts
 2162 operand immI_64()
 2163 %{
 2164   predicate( n->get_int() == 64 );
 2165   match(ConI);
 2166 
 2167   op_cost(0);
 2168   format %{ %}
 2169   interface(CONST_INTER);
 2170 %}
 2171 
 2172 // Pointer Immediate
 2173 operand immP()
 2174 %{
 2175   match(ConP);
 2176 
 2177   op_cost(10);
 2178   format %{ %}
 2179   interface(CONST_INTER);
 2180 %}
 2181 
 2182 // NULL Pointer Immediate
 2183 operand immP0()
 2184 %{
 2185   predicate(n->get_ptr() == 0);
 2186   match(ConP);
 2187 
 2188   op_cost(5);
 2189   format %{ %}
 2190   interface(CONST_INTER);
 2191 %}
 2192 
 2193 // Pointer Immediate
 2194 operand immN() %{
 2195   match(ConN);
 2196 
 2197   op_cost(10);
 2198   format %{ %}
 2199   interface(CONST_INTER);
 2200 %}
 2201 
 2202 operand immNKlass() %{
 2203   match(ConNKlass);
 2204 
 2205   op_cost(10);
 2206   format %{ %}
 2207   interface(CONST_INTER);
 2208 %}
 2209 
 2210 // NULL Pointer Immediate
 2211 operand immN0() %{
 2212   predicate(n->get_narrowcon() == 0);
 2213   match(ConN);
 2214 
 2215   op_cost(5);
 2216   format %{ %}
 2217   interface(CONST_INTER);
 2218 %}
 2219 
 2220 operand immP31()
 2221 %{
 2222   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2223             && (n->get_ptr() >> 31) == 0);
 2224   match(ConP);
 2225 
 2226   op_cost(5);
 2227   format %{ %}
 2228   interface(CONST_INTER);
 2229 %}
 2230 
 2231 
 2232 // Long Immediate
 2233 operand immL()
 2234 %{
 2235   match(ConL);
 2236 
 2237   op_cost(20);
 2238   format %{ %}
 2239   interface(CONST_INTER);
 2240 %}
 2241 
 2242 // Long Immediate 8-bit
 2243 operand immL8()
 2244 %{
 2245   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2246   match(ConL);
 2247 
 2248   op_cost(5);
 2249   format %{ %}
 2250   interface(CONST_INTER);
 2251 %}
 2252 
 2253 // Long Immediate 32-bit unsigned
 2254 operand immUL32()
 2255 %{
 2256   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2257   match(ConL);
 2258 
 2259   op_cost(10);
 2260   format %{ %}
 2261   interface(CONST_INTER);
 2262 %}
 2263 
 2264 // Long Immediate 32-bit signed
 2265 operand immL32()
 2266 %{
 2267   predicate(n->get_long() == (int) (n->get_long()));
 2268   match(ConL);
 2269 
 2270   op_cost(15);
 2271   format %{ %}
 2272   interface(CONST_INTER);
 2273 %}
 2274 
 2275 operand immL_Pow2()
 2276 %{
 2277   predicate(is_power_of_2((julong)n->get_long()));
 2278   match(ConL);
 2279 
 2280   op_cost(15);
 2281   format %{ %}
 2282   interface(CONST_INTER);
 2283 %}
 2284 
 2285 operand immL_NotPow2()
 2286 %{
 2287   predicate(is_power_of_2((julong)~n->get_long()));
 2288   match(ConL);
 2289 
 2290   op_cost(15);
 2291   format %{ %}
 2292   interface(CONST_INTER);
 2293 %}
 2294 
 2295 // Long Immediate zero
 2296 operand immL0()
 2297 %{
 2298   predicate(n->get_long() == 0L);
 2299   match(ConL);
 2300 
 2301   op_cost(10);
 2302   format %{ %}
 2303   interface(CONST_INTER);
 2304 %}
 2305 
 2306 // Constant for increment
 2307 operand immL1()
 2308 %{
 2309   predicate(n->get_long() == 1);
 2310   match(ConL);
 2311 
 2312   format %{ %}
 2313   interface(CONST_INTER);
 2314 %}
 2315 
 2316 // Constant for decrement
 2317 operand immL_M1()
 2318 %{
 2319   predicate(n->get_long() == -1);
 2320   match(ConL);
 2321 
 2322   format %{ %}
 2323   interface(CONST_INTER);
 2324 %}
 2325 
 2326 // Long Immediate: the value 10
 2327 operand immL10()
 2328 %{
 2329   predicate(n->get_long() == 10);
 2330   match(ConL);
 2331 
 2332   format %{ %}
 2333   interface(CONST_INTER);
 2334 %}
 2335 
 2336 // Long immediate from 0 to 127.
 2337 // Used for a shorter form of long mul by 10.
 2338 operand immL_127()
 2339 %{
 2340   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 2341   match(ConL);
 2342 
 2343   op_cost(10);
 2344   format %{ %}
 2345   interface(CONST_INTER);
 2346 %}
 2347 
 2348 // Long Immediate: low 32-bit mask
 2349 operand immL_32bits()
 2350 %{
 2351   predicate(n->get_long() == 0xFFFFFFFFL);
 2352   match(ConL);
 2353   op_cost(20);
 2354 
 2355   format %{ %}
 2356   interface(CONST_INTER);
 2357 %}
 2358 
 2359 // Int Immediate: 2^n-1, positive
 2360 operand immI_Pow2M1()
 2361 %{
 2362   predicate((n->get_int() > 0)
 2363             && is_power_of_2((juint)n->get_int() + 1));
 2364   match(ConI);
 2365 
 2366   op_cost(20);
 2367   format %{ %}
 2368   interface(CONST_INTER);
 2369 %}
 2370 
 2371 // Float Immediate zero
 2372 operand immF0()
 2373 %{
 2374   predicate(jint_cast(n->getf()) == 0);
 2375   match(ConF);
 2376 
 2377   op_cost(5);
 2378   format %{ %}
 2379   interface(CONST_INTER);
 2380 %}
 2381 
 2382 // Float Immediate
 2383 operand immF()
 2384 %{
 2385   match(ConF);
 2386 
 2387   op_cost(15);
 2388   format %{ %}
 2389   interface(CONST_INTER);
 2390 %}
 2391 
 2392 // Double Immediate zero
 2393 operand immD0()
 2394 %{
 2395   predicate(jlong_cast(n->getd()) == 0);
 2396   match(ConD);
 2397 
 2398   op_cost(5);
 2399   format %{ %}
 2400   interface(CONST_INTER);
 2401 %}
 2402 
 2403 // Double Immediate
 2404 operand immD()
 2405 %{
 2406   match(ConD);
 2407 
 2408   op_cost(15);
 2409   format %{ %}
 2410   interface(CONST_INTER);
 2411 %}
 2412 
 2413 // Immediates for special shifts (sign extend)
 2414 
 2415 // Constants for increment
 2416 operand immI_16()
 2417 %{
 2418   predicate(n->get_int() == 16);
 2419   match(ConI);
 2420 
 2421   format %{ %}
 2422   interface(CONST_INTER);
 2423 %}
 2424 
 2425 operand immI_24()
 2426 %{
 2427   predicate(n->get_int() == 24);
 2428   match(ConI);
 2429 
 2430   format %{ %}
 2431   interface(CONST_INTER);
 2432 %}
 2433 
 2434 // Constant for byte-wide masking
 2435 operand immI_255()
 2436 %{
 2437   predicate(n->get_int() == 255);
 2438   match(ConI);
 2439 
 2440   format %{ %}
 2441   interface(CONST_INTER);
 2442 %}
 2443 
 2444 // Constant for short-wide masking
 2445 operand immI_65535()
 2446 %{
 2447   predicate(n->get_int() == 65535);
 2448   match(ConI);
 2449 
 2450   format %{ %}
 2451   interface(CONST_INTER);
 2452 %}
 2453 
 2454 // Constant for byte-wide masking
 2455 operand immL_255()
 2456 %{
 2457   predicate(n->get_long() == 255);
 2458   match(ConL);
 2459 
 2460   format %{ %}
 2461   interface(CONST_INTER);
 2462 %}
 2463 
 2464 // Constant for short-wide masking
 2465 operand immL_65535()
 2466 %{
 2467   predicate(n->get_long() == 65535);
 2468   match(ConL);
 2469 
 2470   format %{ %}
 2471   interface(CONST_INTER);
 2472 %}
 2473 
 2474 operand kReg()
 2475 %{
 2476   constraint(ALLOC_IN_RC(vectmask_reg));
 2477   match(RegVectMask);
 2478   format %{%}
 2479   interface(REG_INTER);
 2480 %}
 2481 
 2482 operand kReg_K1()
 2483 %{
 2484   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 2485   match(RegVectMask);
 2486   format %{%}
 2487   interface(REG_INTER);
 2488 %}
 2489 
 2490 operand kReg_K2()
 2491 %{
 2492   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 2493   match(RegVectMask);
 2494   format %{%}
 2495   interface(REG_INTER);
 2496 %}
 2497 
 2498 // Special Registers
 2499 operand kReg_K3()
 2500 %{
 2501   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 2502   match(RegVectMask);
 2503   format %{%}
 2504   interface(REG_INTER);
 2505 %}
 2506 
 2507 operand kReg_K4()
 2508 %{
 2509   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 2510   match(RegVectMask);
 2511   format %{%}
 2512   interface(REG_INTER);
 2513 %}
 2514 
 2515 operand kReg_K5()
 2516 %{
 2517   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 2518   match(RegVectMask);
 2519   format %{%}
 2520   interface(REG_INTER);
 2521 %}
 2522 
 2523 operand kReg_K6()
 2524 %{
 2525   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 2526   match(RegVectMask);
 2527   format %{%}
 2528   interface(REG_INTER);
 2529 %}
 2530 
 2531 // Special Registers
 2532 operand kReg_K7()
 2533 %{
 2534   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 2535   match(RegVectMask);
 2536   format %{%}
 2537   interface(REG_INTER);
 2538 %}
 2539 
 2540 // Register Operands
 2541 // Integer Register
 2542 operand rRegI()
 2543 %{
 2544   constraint(ALLOC_IN_RC(int_reg));
 2545   match(RegI);
 2546 
 2547   match(rax_RegI);
 2548   match(rbx_RegI);
 2549   match(rcx_RegI);
 2550   match(rdx_RegI);
 2551   match(rdi_RegI);
 2552 
 2553   format %{ %}
 2554   interface(REG_INTER);
 2555 %}
 2556 
 2557 // Special Registers
 2558 operand rax_RegI()
 2559 %{
 2560   constraint(ALLOC_IN_RC(int_rax_reg));
 2561   match(RegI);
 2562   match(rRegI);
 2563 
 2564   format %{ "RAX" %}
 2565   interface(REG_INTER);
 2566 %}
 2567 
 2568 // Special Registers
 2569 operand rbx_RegI()
 2570 %{
 2571   constraint(ALLOC_IN_RC(int_rbx_reg));
 2572   match(RegI);
 2573   match(rRegI);
 2574 
 2575   format %{ "RBX" %}
 2576   interface(REG_INTER);
 2577 %}
 2578 
 2579 operand rcx_RegI()
 2580 %{
 2581   constraint(ALLOC_IN_RC(int_rcx_reg));
 2582   match(RegI);
 2583   match(rRegI);
 2584 
 2585   format %{ "RCX" %}
 2586   interface(REG_INTER);
 2587 %}
 2588 
 2589 operand rdx_RegI()
 2590 %{
 2591   constraint(ALLOC_IN_RC(int_rdx_reg));
 2592   match(RegI);
 2593   match(rRegI);
 2594 
 2595   format %{ "RDX" %}
 2596   interface(REG_INTER);
 2597 %}
 2598 
 2599 operand rdi_RegI()
 2600 %{
 2601   constraint(ALLOC_IN_RC(int_rdi_reg));
 2602   match(RegI);
 2603   match(rRegI);
 2604 
 2605   format %{ "RDI" %}
 2606   interface(REG_INTER);
 2607 %}
 2608 
 2609 operand no_rax_rdx_RegI()
 2610 %{
 2611   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2612   match(RegI);
 2613   match(rbx_RegI);
 2614   match(rcx_RegI);
 2615   match(rdi_RegI);
 2616 
 2617   format %{ %}
 2618   interface(REG_INTER);
 2619 %}
 2620 
 2621 operand no_rbp_r13_RegI()
 2622 %{
 2623   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2624   match(RegI);
 2625   match(rRegI);
 2626   match(rax_RegI);
 2627   match(rbx_RegI);
 2628   match(rcx_RegI);
 2629   match(rdx_RegI);
 2630   match(rdi_RegI);
 2631 
 2632   format %{ %}
 2633   interface(REG_INTER);
 2634 %}
 2635 
 2636 // Pointer Register
 2637 operand any_RegP()
 2638 %{
 2639   constraint(ALLOC_IN_RC(any_reg));
 2640   match(RegP);
 2641   match(rax_RegP);
 2642   match(rbx_RegP);
 2643   match(rdi_RegP);
 2644   match(rsi_RegP);
 2645   match(rbp_RegP);
 2646   match(r15_RegP);
 2647   match(rRegP);
 2648 
 2649   format %{ %}
 2650   interface(REG_INTER);
 2651 %}
 2652 
 2653 operand rRegP()
 2654 %{
 2655   constraint(ALLOC_IN_RC(ptr_reg));
 2656   match(RegP);
 2657   match(rax_RegP);
 2658   match(rbx_RegP);
 2659   match(rdi_RegP);
 2660   match(rsi_RegP);
 2661   match(rbp_RegP);  // See Q&A below about
 2662   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2663 
 2664   format %{ %}
 2665   interface(REG_INTER);
 2666 %}
 2667 
 2668 operand rRegN() %{
 2669   constraint(ALLOC_IN_RC(int_reg));
 2670   match(RegN);
 2671 
 2672   format %{ %}
 2673   interface(REG_INTER);
 2674 %}
 2675 
 2676 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2677 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2678 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2679 // The output of an instruction is controlled by the allocator, which respects
 2680 // register class masks, not match rules.  Unless an instruction mentions
 2681 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2682 // by the allocator as an input.
 2683 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2684 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2685 // result, RBP is not included in the output of the instruction either.
 2686 
 2687 operand no_rax_RegP()
 2688 %{
 2689   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 2690   match(RegP);
 2691   match(rbx_RegP);
 2692   match(rsi_RegP);
 2693   match(rdi_RegP);
 2694 
 2695   format %{ %}
 2696   interface(REG_INTER);
 2697 %}
 2698 
 2699 // This operand is not allowed to use RBP even if
 2700 // RBP is not used to hold the frame pointer.
 2701 operand no_rbp_RegP()
 2702 %{
 2703   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2704   match(RegP);
 2705   match(rbx_RegP);
 2706   match(rsi_RegP);
 2707   match(rdi_RegP);
 2708 
 2709   format %{ %}
 2710   interface(REG_INTER);
 2711 %}
 2712 
 2713 operand no_rax_rbx_RegP()
 2714 %{
 2715   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 2716   match(RegP);
 2717   match(rsi_RegP);
 2718   match(rdi_RegP);
 2719 
 2720   format %{ %}
 2721   interface(REG_INTER);
 2722 %}
 2723 
 2724 // Special Registers
 2725 // Return a pointer value
 2726 operand rax_RegP()
 2727 %{
 2728   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2729   match(RegP);
 2730   match(rRegP);
 2731 
 2732   format %{ %}
 2733   interface(REG_INTER);
 2734 %}
 2735 
 2736 // Special Registers
 2737 // Return a compressed pointer value
 2738 operand rax_RegN()
 2739 %{
 2740   constraint(ALLOC_IN_RC(int_rax_reg));
 2741   match(RegN);
 2742   match(rRegN);
 2743 
 2744   format %{ %}
 2745   interface(REG_INTER);
 2746 %}
 2747 
 2748 // Used in AtomicAdd
 2749 operand rbx_RegP()
 2750 %{
 2751   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2752   match(RegP);
 2753   match(rRegP);
 2754 
 2755   format %{ %}
 2756   interface(REG_INTER);
 2757 %}
 2758 
 2759 operand rsi_RegP()
 2760 %{
 2761   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2762   match(RegP);
 2763   match(rRegP);
 2764 
 2765   format %{ %}
 2766   interface(REG_INTER);
 2767 %}
 2768 
 2769 operand rbp_RegP()
 2770 %{
 2771   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2772   match(RegP);
 2773   match(rRegP);
 2774 
 2775   format %{ %}
 2776   interface(REG_INTER);
 2777 %}
 2778 
 2779 // Used in rep stosq
 2780 operand rdi_RegP()
 2781 %{
 2782   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2783   match(RegP);
 2784   match(rRegP);
 2785 
 2786   format %{ %}
 2787   interface(REG_INTER);
 2788 %}
 2789 
 2790 operand r15_RegP()
 2791 %{
 2792   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2793   match(RegP);
 2794   match(rRegP);
 2795 
 2796   format %{ %}
 2797   interface(REG_INTER);
 2798 %}
 2799 
 2800 operand rRegL()
 2801 %{
 2802   constraint(ALLOC_IN_RC(long_reg));
 2803   match(RegL);
 2804   match(rax_RegL);
 2805   match(rdx_RegL);
 2806 
 2807   format %{ %}
 2808   interface(REG_INTER);
 2809 %}
 2810 
 2811 // Special Registers
 2812 operand no_rax_rdx_RegL()
 2813 %{
 2814   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2815   match(RegL);
 2816   match(rRegL);
 2817 
 2818   format %{ %}
 2819   interface(REG_INTER);
 2820 %}
 2821 
 2822 operand rax_RegL()
 2823 %{
 2824   constraint(ALLOC_IN_RC(long_rax_reg));
 2825   match(RegL);
 2826   match(rRegL);
 2827 
 2828   format %{ "RAX" %}
 2829   interface(REG_INTER);
 2830 %}
 2831 
 2832 operand rcx_RegL()
 2833 %{
 2834   constraint(ALLOC_IN_RC(long_rcx_reg));
 2835   match(RegL);
 2836   match(rRegL);
 2837 
 2838   format %{ %}
 2839   interface(REG_INTER);
 2840 %}
 2841 
 2842 operand rdx_RegL()
 2843 %{
 2844   constraint(ALLOC_IN_RC(long_rdx_reg));
 2845   match(RegL);
 2846   match(rRegL);
 2847 
 2848   format %{ %}
 2849   interface(REG_INTER);
 2850 %}
 2851 
 2852 operand no_rbp_r13_RegL()
 2853 %{
 2854   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2855   match(RegL);
 2856   match(rRegL);
 2857   match(rax_RegL);
 2858   match(rcx_RegL);
 2859   match(rdx_RegL);
 2860 
 2861   format %{ %}
 2862   interface(REG_INTER);
 2863 %}
 2864 
 2865 // Flags register, used as output of compare instructions
 2866 operand rFlagsReg()
 2867 %{
 2868   constraint(ALLOC_IN_RC(int_flags));
 2869   match(RegFlags);
 2870 
 2871   format %{ "RFLAGS" %}
 2872   interface(REG_INTER);
 2873 %}
 2874 
 2875 // Flags register, used as output of FLOATING POINT compare instructions
 2876 operand rFlagsRegU()
 2877 %{
 2878   constraint(ALLOC_IN_RC(int_flags));
 2879   match(RegFlags);
 2880 
 2881   format %{ "RFLAGS_U" %}
 2882   interface(REG_INTER);
 2883 %}
 2884 
 2885 operand rFlagsRegUCF() %{
 2886   constraint(ALLOC_IN_RC(int_flags));
 2887   match(RegFlags);
 2888   predicate(false);
 2889 
 2890   format %{ "RFLAGS_U_CF" %}
 2891   interface(REG_INTER);
 2892 %}
 2893 
 2894 // Float register operands
 2895 operand regF() %{
 2896    constraint(ALLOC_IN_RC(float_reg));
 2897    match(RegF);
 2898 
 2899    format %{ %}
 2900    interface(REG_INTER);
 2901 %}
 2902 
 2903 // Float register operands
 2904 operand legRegF() %{
 2905    constraint(ALLOC_IN_RC(float_reg_legacy));
 2906    match(RegF);
 2907 
 2908    format %{ %}
 2909    interface(REG_INTER);
 2910 %}
 2911 
 2912 // Float register operands
 2913 operand vlRegF() %{
 2914    constraint(ALLOC_IN_RC(float_reg_vl));
 2915    match(RegF);
 2916 
 2917    format %{ %}
 2918    interface(REG_INTER);
 2919 %}
 2920 
 2921 // Double register operands
 2922 operand regD() %{
 2923    constraint(ALLOC_IN_RC(double_reg));
 2924    match(RegD);
 2925 
 2926    format %{ %}
 2927    interface(REG_INTER);
 2928 %}
 2929 
 2930 // Double register operands
 2931 operand legRegD() %{
 2932    constraint(ALLOC_IN_RC(double_reg_legacy));
 2933    match(RegD);
 2934 
 2935    format %{ %}
 2936    interface(REG_INTER);
 2937 %}
 2938 
 2939 // Double register operands
 2940 operand vlRegD() %{
 2941    constraint(ALLOC_IN_RC(double_reg_vl));
 2942    match(RegD);
 2943 
 2944    format %{ %}
 2945    interface(REG_INTER);
 2946 %}
 2947 
 2948 //----------Memory Operands----------------------------------------------------
 2949 // Direct Memory Operand
 2950 // operand direct(immP addr)
 2951 // %{
 2952 //   match(addr);
 2953 
 2954 //   format %{ "[$addr]" %}
 2955 //   interface(MEMORY_INTER) %{
 2956 //     base(0xFFFFFFFF);
 2957 //     index(0x4);
 2958 //     scale(0x0);
 2959 //     disp($addr);
 2960 //   %}
 2961 // %}
 2962 
 2963 // Indirect Memory Operand
 2964 operand indirect(any_RegP reg)
 2965 %{
 2966   constraint(ALLOC_IN_RC(ptr_reg));
 2967   match(reg);
 2968 
 2969   format %{ "[$reg]" %}
 2970   interface(MEMORY_INTER) %{
 2971     base($reg);
 2972     index(0x4);
 2973     scale(0x0);
 2974     disp(0x0);
 2975   %}
 2976 %}
 2977 
 2978 // Indirect Memory Plus Short Offset Operand
 2979 operand indOffset8(any_RegP reg, immL8 off)
 2980 %{
 2981   constraint(ALLOC_IN_RC(ptr_reg));
 2982   match(AddP reg off);
 2983 
 2984   format %{ "[$reg + $off (8-bit)]" %}
 2985   interface(MEMORY_INTER) %{
 2986     base($reg);
 2987     index(0x4);
 2988     scale(0x0);
 2989     disp($off);
 2990   %}
 2991 %}
 2992 
 2993 // Indirect Memory Plus Long Offset Operand
 2994 operand indOffset32(any_RegP reg, immL32 off)
 2995 %{
 2996   constraint(ALLOC_IN_RC(ptr_reg));
 2997   match(AddP reg off);
 2998 
 2999   format %{ "[$reg + $off (32-bit)]" %}
 3000   interface(MEMORY_INTER) %{
 3001     base($reg);
 3002     index(0x4);
 3003     scale(0x0);
 3004     disp($off);
 3005   %}
 3006 %}
 3007 
 3008 // Indirect Memory Plus Index Register Plus Offset Operand
 3009 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3010 %{
 3011   constraint(ALLOC_IN_RC(ptr_reg));
 3012   match(AddP (AddP reg lreg) off);
 3013 
 3014   op_cost(10);
 3015   format %{"[$reg + $off + $lreg]" %}
 3016   interface(MEMORY_INTER) %{
 3017     base($reg);
 3018     index($lreg);
 3019     scale(0x0);
 3020     disp($off);
 3021   %}
 3022 %}
 3023 
 3024 // Indirect Memory Plus Index Register Plus Offset Operand
 3025 operand indIndex(any_RegP reg, rRegL lreg)
 3026 %{
 3027   constraint(ALLOC_IN_RC(ptr_reg));
 3028   match(AddP reg lreg);
 3029 
 3030   op_cost(10);
 3031   format %{"[$reg + $lreg]" %}
 3032   interface(MEMORY_INTER) %{
 3033     base($reg);
 3034     index($lreg);
 3035     scale(0x0);
 3036     disp(0x0);
 3037   %}
 3038 %}
 3039 
 3040 // Indirect Memory Times Scale Plus Index Register
 3041 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3042 %{
 3043   constraint(ALLOC_IN_RC(ptr_reg));
 3044   match(AddP reg (LShiftL lreg scale));
 3045 
 3046   op_cost(10);
 3047   format %{"[$reg + $lreg << $scale]" %}
 3048   interface(MEMORY_INTER) %{
 3049     base($reg);
 3050     index($lreg);
 3051     scale($scale);
 3052     disp(0x0);
 3053   %}
 3054 %}
 3055 
 3056 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3057 %{
 3058   constraint(ALLOC_IN_RC(ptr_reg));
 3059   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3060   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3061 
 3062   op_cost(10);
 3063   format %{"[$reg + pos $idx << $scale]" %}
 3064   interface(MEMORY_INTER) %{
 3065     base($reg);
 3066     index($idx);
 3067     scale($scale);
 3068     disp(0x0);
 3069   %}
 3070 %}
 3071 
 3072 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3073 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3074 %{
 3075   constraint(ALLOC_IN_RC(ptr_reg));
 3076   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3077 
 3078   op_cost(10);
 3079   format %{"[$reg + $off + $lreg << $scale]" %}
 3080   interface(MEMORY_INTER) %{
 3081     base($reg);
 3082     index($lreg);
 3083     scale($scale);
 3084     disp($off);
 3085   %}
 3086 %}
 3087 
 3088 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3089 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3090 %{
 3091   constraint(ALLOC_IN_RC(ptr_reg));
 3092   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3093   match(AddP (AddP reg (ConvI2L idx)) off);
 3094 
 3095   op_cost(10);
 3096   format %{"[$reg + $off + $idx]" %}
 3097   interface(MEMORY_INTER) %{
 3098     base($reg);
 3099     index($idx);
 3100     scale(0x0);
 3101     disp($off);
 3102   %}
 3103 %}
 3104 
 3105 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3106 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3107 %{
 3108   constraint(ALLOC_IN_RC(ptr_reg));
 3109   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3110   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3111 
 3112   op_cost(10);
 3113   format %{"[$reg + $off + $idx << $scale]" %}
 3114   interface(MEMORY_INTER) %{
 3115     base($reg);
 3116     index($idx);
 3117     scale($scale);
 3118     disp($off);
 3119   %}
 3120 %}
 3121 
 3122 // Indirect Narrow Oop Plus Offset Operand
 3123 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3124 // we can't free r12 even with CompressedOops::base() == NULL.
 3125 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3126   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3127   constraint(ALLOC_IN_RC(ptr_reg));
 3128   match(AddP (DecodeN reg) off);
 3129 
 3130   op_cost(10);
 3131   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3132   interface(MEMORY_INTER) %{
 3133     base(0xc); // R12
 3134     index($reg);
 3135     scale(0x3);
 3136     disp($off);
 3137   %}
 3138 %}
 3139 
 3140 // Indirect Memory Operand
 3141 operand indirectNarrow(rRegN reg)
 3142 %{
 3143   predicate(CompressedOops::shift() == 0);
 3144   constraint(ALLOC_IN_RC(ptr_reg));
 3145   match(DecodeN reg);
 3146 
 3147   format %{ "[$reg]" %}
 3148   interface(MEMORY_INTER) %{
 3149     base($reg);
 3150     index(0x4);
 3151     scale(0x0);
 3152     disp(0x0);
 3153   %}
 3154 %}
 3155 
 3156 // Indirect Memory Plus Short Offset Operand
 3157 operand indOffset8Narrow(rRegN reg, immL8 off)
 3158 %{
 3159   predicate(CompressedOops::shift() == 0);
 3160   constraint(ALLOC_IN_RC(ptr_reg));
 3161   match(AddP (DecodeN reg) off);
 3162 
 3163   format %{ "[$reg + $off (8-bit)]" %}
 3164   interface(MEMORY_INTER) %{
 3165     base($reg);
 3166     index(0x4);
 3167     scale(0x0);
 3168     disp($off);
 3169   %}
 3170 %}
 3171 
 3172 // Indirect Memory Plus Long Offset Operand
 3173 operand indOffset32Narrow(rRegN reg, immL32 off)
 3174 %{
 3175   predicate(CompressedOops::shift() == 0);
 3176   constraint(ALLOC_IN_RC(ptr_reg));
 3177   match(AddP (DecodeN reg) off);
 3178 
 3179   format %{ "[$reg + $off (32-bit)]" %}
 3180   interface(MEMORY_INTER) %{
 3181     base($reg);
 3182     index(0x4);
 3183     scale(0x0);
 3184     disp($off);
 3185   %}
 3186 %}
 3187 
 3188 // Indirect Memory Plus Index Register Plus Offset Operand
 3189 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3190 %{
 3191   predicate(CompressedOops::shift() == 0);
 3192   constraint(ALLOC_IN_RC(ptr_reg));
 3193   match(AddP (AddP (DecodeN reg) lreg) off);
 3194 
 3195   op_cost(10);
 3196   format %{"[$reg + $off + $lreg]" %}
 3197   interface(MEMORY_INTER) %{
 3198     base($reg);
 3199     index($lreg);
 3200     scale(0x0);
 3201     disp($off);
 3202   %}
 3203 %}
 3204 
 3205 // Indirect Memory Plus Index Register Plus Offset Operand
 3206 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3207 %{
 3208   predicate(CompressedOops::shift() == 0);
 3209   constraint(ALLOC_IN_RC(ptr_reg));
 3210   match(AddP (DecodeN reg) lreg);
 3211 
 3212   op_cost(10);
 3213   format %{"[$reg + $lreg]" %}
 3214   interface(MEMORY_INTER) %{
 3215     base($reg);
 3216     index($lreg);
 3217     scale(0x0);
 3218     disp(0x0);
 3219   %}
 3220 %}
 3221 
 3222 // Indirect Memory Times Scale Plus Index Register
 3223 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3224 %{
 3225   predicate(CompressedOops::shift() == 0);
 3226   constraint(ALLOC_IN_RC(ptr_reg));
 3227   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3228 
 3229   op_cost(10);
 3230   format %{"[$reg + $lreg << $scale]" %}
 3231   interface(MEMORY_INTER) %{
 3232     base($reg);
 3233     index($lreg);
 3234     scale($scale);
 3235     disp(0x0);
 3236   %}
 3237 %}
 3238 
 3239 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3240 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3241 %{
 3242   predicate(CompressedOops::shift() == 0);
 3243   constraint(ALLOC_IN_RC(ptr_reg));
 3244   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3245 
 3246   op_cost(10);
 3247   format %{"[$reg + $off + $lreg << $scale]" %}
 3248   interface(MEMORY_INTER) %{
 3249     base($reg);
 3250     index($lreg);
 3251     scale($scale);
 3252     disp($off);
 3253   %}
 3254 %}
 3255 
 3256 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3257 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3258 %{
 3259   constraint(ALLOC_IN_RC(ptr_reg));
 3260   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3261   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3262 
 3263   op_cost(10);
 3264   format %{"[$reg + $off + $idx]" %}
 3265   interface(MEMORY_INTER) %{
 3266     base($reg);
 3267     index($idx);
 3268     scale(0x0);
 3269     disp($off);
 3270   %}
 3271 %}
 3272 
 3273 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3274 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3275 %{
 3276   constraint(ALLOC_IN_RC(ptr_reg));
 3277   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3278   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3279 
 3280   op_cost(10);
 3281   format %{"[$reg + $off + $idx << $scale]" %}
 3282   interface(MEMORY_INTER) %{
 3283     base($reg);
 3284     index($idx);
 3285     scale($scale);
 3286     disp($off);
 3287   %}
 3288 %}
 3289 
 3290 //----------Special Memory Operands--------------------------------------------
 3291 // Stack Slot Operand - This operand is used for loading and storing temporary
 3292 //                      values on the stack where a match requires a value to
 3293 //                      flow through memory.
 3294 operand stackSlotP(sRegP reg)
 3295 %{
 3296   constraint(ALLOC_IN_RC(stack_slots));
 3297   // No match rule because this operand is only generated in matching
 3298 
 3299   format %{ "[$reg]" %}
 3300   interface(MEMORY_INTER) %{
 3301     base(0x4);   // RSP
 3302     index(0x4);  // No Index
 3303     scale(0x0);  // No Scale
 3304     disp($reg);  // Stack Offset
 3305   %}
 3306 %}
 3307 
 3308 operand stackSlotI(sRegI reg)
 3309 %{
 3310   constraint(ALLOC_IN_RC(stack_slots));
 3311   // No match rule because this operand is only generated in matching
 3312 
 3313   format %{ "[$reg]" %}
 3314   interface(MEMORY_INTER) %{
 3315     base(0x4);   // RSP
 3316     index(0x4);  // No Index
 3317     scale(0x0);  // No Scale
 3318     disp($reg);  // Stack Offset
 3319   %}
 3320 %}
 3321 
 3322 operand stackSlotF(sRegF reg)
 3323 %{
 3324   constraint(ALLOC_IN_RC(stack_slots));
 3325   // No match rule because this operand is only generated in matching
 3326 
 3327   format %{ "[$reg]" %}
 3328   interface(MEMORY_INTER) %{
 3329     base(0x4);   // RSP
 3330     index(0x4);  // No Index
 3331     scale(0x0);  // No Scale
 3332     disp($reg);  // Stack Offset
 3333   %}
 3334 %}
 3335 
 3336 operand stackSlotD(sRegD reg)
 3337 %{
 3338   constraint(ALLOC_IN_RC(stack_slots));
 3339   // No match rule because this operand is only generated in matching
 3340 
 3341   format %{ "[$reg]" %}
 3342   interface(MEMORY_INTER) %{
 3343     base(0x4);   // RSP
 3344     index(0x4);  // No Index
 3345     scale(0x0);  // No Scale
 3346     disp($reg);  // Stack Offset
 3347   %}
 3348 %}
 3349 operand stackSlotL(sRegL reg)
 3350 %{
 3351   constraint(ALLOC_IN_RC(stack_slots));
 3352   // No match rule because this operand is only generated in matching
 3353 
 3354   format %{ "[$reg]" %}
 3355   interface(MEMORY_INTER) %{
 3356     base(0x4);   // RSP
 3357     index(0x4);  // No Index
 3358     scale(0x0);  // No Scale
 3359     disp($reg);  // Stack Offset
 3360   %}
 3361 %}
 3362 
 3363 //----------Conditional Branch Operands----------------------------------------
 3364 // Comparison Op  - This is the operation of the comparison, and is limited to
 3365 //                  the following set of codes:
 3366 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3367 //
 3368 // Other attributes of the comparison, such as unsignedness, are specified
 3369 // by the comparison instruction that sets a condition code flags register.
 3370 // That result is represented by a flags operand whose subtype is appropriate
 3371 // to the unsignedness (etc.) of the comparison.
 3372 //
 3373 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3374 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3375 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3376 
 3377 // Comparison Code
 3378 operand cmpOp()
 3379 %{
 3380   match(Bool);
 3381 
 3382   format %{ "" %}
 3383   interface(COND_INTER) %{
 3384     equal(0x4, "e");
 3385     not_equal(0x5, "ne");
 3386     less(0xC, "l");
 3387     greater_equal(0xD, "ge");
 3388     less_equal(0xE, "le");
 3389     greater(0xF, "g");
 3390     overflow(0x0, "o");
 3391     no_overflow(0x1, "no");
 3392   %}
 3393 %}
 3394 
 3395 // Comparison Code, unsigned compare.  Used by FP also, with
 3396 // C2 (unordered) turned into GT or LT already.  The other bits
 3397 // C0 and C3 are turned into Carry & Zero flags.
 3398 operand cmpOpU()
 3399 %{
 3400   match(Bool);
 3401 
 3402   format %{ "" %}
 3403   interface(COND_INTER) %{
 3404     equal(0x4, "e");
 3405     not_equal(0x5, "ne");
 3406     less(0x2, "b");
 3407     greater_equal(0x3, "ae");
 3408     less_equal(0x6, "be");
 3409     greater(0x7, "a");
 3410     overflow(0x0, "o");
 3411     no_overflow(0x1, "no");
 3412   %}
 3413 %}
 3414 
 3415 
 3416 // Floating comparisons that don't require any fixup for the unordered case,
 3417 // If both inputs of the comparison are the same, ZF is always set so we
 3418 // don't need to use cmpOpUCF2 for eq/ne
 3419 operand cmpOpUCF() %{
 3420   match(Bool);
 3421   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3422             n->as_Bool()->_test._test == BoolTest::ge ||
 3423             n->as_Bool()->_test._test == BoolTest::le ||
 3424             n->as_Bool()->_test._test == BoolTest::gt ||
 3425             n->in(1)->in(1) == n->in(1)->in(2));
 3426   format %{ "" %}
 3427   interface(COND_INTER) %{
 3428     equal(0xb, "np");
 3429     not_equal(0xa, "p");
 3430     less(0x2, "b");
 3431     greater_equal(0x3, "ae");
 3432     less_equal(0x6, "be");
 3433     greater(0x7, "a");
 3434     overflow(0x0, "o");
 3435     no_overflow(0x1, "no");
 3436   %}
 3437 %}
 3438 
 3439 
 3440 // Floating comparisons that can be fixed up with extra conditional jumps
 3441 operand cmpOpUCF2() %{
 3442   match(Bool);
 3443   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3444              n->as_Bool()->_test._test == BoolTest::eq) &&
 3445             n->in(1)->in(1) != n->in(1)->in(2));
 3446   format %{ "" %}
 3447   interface(COND_INTER) %{
 3448     equal(0x4, "e");
 3449     not_equal(0x5, "ne");
 3450     less(0x2, "b");
 3451     greater_equal(0x3, "ae");
 3452     less_equal(0x6, "be");
 3453     greater(0x7, "a");
 3454     overflow(0x0, "o");
 3455     no_overflow(0x1, "no");
 3456   %}
 3457 %}
 3458 
 3459 //----------OPERAND CLASSES----------------------------------------------------
 3460 // Operand Classes are groups of operands that are used as to simplify
 3461 // instruction definitions by not requiring the AD writer to specify separate
 3462 // instructions for every form of operand when the instruction accepts
 3463 // multiple operand types with the same basic encoding and format.  The classic
 3464 // case of this is memory operands.
 3465 
 3466 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3467                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3468                indCompressedOopOffset,
 3469                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3470                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3471                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3472 
 3473 //----------PIPELINE-----------------------------------------------------------
 3474 // Rules which define the behavior of the target architectures pipeline.
 3475 pipeline %{
 3476 
 3477 //----------ATTRIBUTES---------------------------------------------------------
 3478 attributes %{
 3479   variable_size_instructions;        // Fixed size instructions
 3480   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3481   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3482   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3483   instruction_fetch_units = 1;       // of 16 bytes
 3484 
 3485   // List of nop instructions
 3486   nops( MachNop );
 3487 %}
 3488 
 3489 //----------RESOURCES----------------------------------------------------------
 3490 // Resources are the functional units available to the machine
 3491 
 3492 // Generic P2/P3 pipeline
 3493 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3494 // 3 instructions decoded per cycle.
 3495 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3496 // 3 ALU op, only ALU0 handles mul instructions.
 3497 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3498            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3499            BR, FPU,
 3500            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3501 
 3502 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3503 // Pipeline Description specifies the stages in the machine's pipeline
 3504 
 3505 // Generic P2/P3 pipeline
 3506 pipe_desc(S0, S1, S2, S3, S4, S5);
 3507 
 3508 //----------PIPELINE CLASSES---------------------------------------------------
 3509 // Pipeline Classes describe the stages in which input and output are
 3510 // referenced by the hardware pipeline.
 3511 
 3512 // Naming convention: ialu or fpu
 3513 // Then: _reg
 3514 // Then: _reg if there is a 2nd register
 3515 // Then: _long if it's a pair of instructions implementing a long
 3516 // Then: _fat if it requires the big decoder
 3517 //   Or: _mem if it requires the big decoder and a memory unit.
 3518 
 3519 // Integer ALU reg operation
 3520 pipe_class ialu_reg(rRegI dst)
 3521 %{
 3522     single_instruction;
 3523     dst    : S4(write);
 3524     dst    : S3(read);
 3525     DECODE : S0;        // any decoder
 3526     ALU    : S3;        // any alu
 3527 %}
 3528 
 3529 // Long ALU reg operation
 3530 pipe_class ialu_reg_long(rRegL dst)
 3531 %{
 3532     instruction_count(2);
 3533     dst    : S4(write);
 3534     dst    : S3(read);
 3535     DECODE : S0(2);     // any 2 decoders
 3536     ALU    : S3(2);     // both alus
 3537 %}
 3538 
 3539 // Integer ALU reg operation using big decoder
 3540 pipe_class ialu_reg_fat(rRegI dst)
 3541 %{
 3542     single_instruction;
 3543     dst    : S4(write);
 3544     dst    : S3(read);
 3545     D0     : S0;        // big decoder only
 3546     ALU    : S3;        // any alu
 3547 %}
 3548 
 3549 // Integer ALU reg-reg operation
 3550 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3551 %{
 3552     single_instruction;
 3553     dst    : S4(write);
 3554     src    : S3(read);
 3555     DECODE : S0;        // any decoder
 3556     ALU    : S3;        // any alu
 3557 %}
 3558 
 3559 // Integer ALU reg-reg operation
 3560 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3561 %{
 3562     single_instruction;
 3563     dst    : S4(write);
 3564     src    : S3(read);
 3565     D0     : S0;        // big decoder only
 3566     ALU    : S3;        // any alu
 3567 %}
 3568 
 3569 // Integer ALU reg-mem operation
 3570 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3571 %{
 3572     single_instruction;
 3573     dst    : S5(write);
 3574     mem    : S3(read);
 3575     D0     : S0;        // big decoder only
 3576     ALU    : S4;        // any alu
 3577     MEM    : S3;        // any mem
 3578 %}
 3579 
 3580 // Integer mem operation (prefetch)
 3581 pipe_class ialu_mem(memory mem)
 3582 %{
 3583     single_instruction;
 3584     mem    : S3(read);
 3585     D0     : S0;        // big decoder only
 3586     MEM    : S3;        // any mem
 3587 %}
 3588 
 3589 // Integer Store to Memory
 3590 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3591 %{
 3592     single_instruction;
 3593     mem    : S3(read);
 3594     src    : S5(read);
 3595     D0     : S0;        // big decoder only
 3596     ALU    : S4;        // any alu
 3597     MEM    : S3;
 3598 %}
 3599 
 3600 // // Long Store to Memory
 3601 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3602 // %{
 3603 //     instruction_count(2);
 3604 //     mem    : S3(read);
 3605 //     src    : S5(read);
 3606 //     D0     : S0(2);          // big decoder only; twice
 3607 //     ALU    : S4(2);     // any 2 alus
 3608 //     MEM    : S3(2);  // Both mems
 3609 // %}
 3610 
 3611 // Integer Store to Memory
 3612 pipe_class ialu_mem_imm(memory mem)
 3613 %{
 3614     single_instruction;
 3615     mem    : S3(read);
 3616     D0     : S0;        // big decoder only
 3617     ALU    : S4;        // any alu
 3618     MEM    : S3;
 3619 %}
 3620 
 3621 // Integer ALU0 reg-reg operation
 3622 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3623 %{
 3624     single_instruction;
 3625     dst    : S4(write);
 3626     src    : S3(read);
 3627     D0     : S0;        // Big decoder only
 3628     ALU0   : S3;        // only alu0
 3629 %}
 3630 
 3631 // Integer ALU0 reg-mem operation
 3632 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3633 %{
 3634     single_instruction;
 3635     dst    : S5(write);
 3636     mem    : S3(read);
 3637     D0     : S0;        // big decoder only
 3638     ALU0   : S4;        // ALU0 only
 3639     MEM    : S3;        // any mem
 3640 %}
 3641 
 3642 // Integer ALU reg-reg operation
 3643 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3644 %{
 3645     single_instruction;
 3646     cr     : S4(write);
 3647     src1   : S3(read);
 3648     src2   : S3(read);
 3649     DECODE : S0;        // any decoder
 3650     ALU    : S3;        // any alu
 3651 %}
 3652 
 3653 // Integer ALU reg-imm operation
 3654 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3655 %{
 3656     single_instruction;
 3657     cr     : S4(write);
 3658     src1   : S3(read);
 3659     DECODE : S0;        // any decoder
 3660     ALU    : S3;        // any alu
 3661 %}
 3662 
 3663 // Integer ALU reg-mem operation
 3664 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3665 %{
 3666     single_instruction;
 3667     cr     : S4(write);
 3668     src1   : S3(read);
 3669     src2   : S3(read);
 3670     D0     : S0;        // big decoder only
 3671     ALU    : S4;        // any alu
 3672     MEM    : S3;
 3673 %}
 3674 
 3675 // Conditional move reg-reg
 3676 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3677 %{
 3678     instruction_count(4);
 3679     y      : S4(read);
 3680     q      : S3(read);
 3681     p      : S3(read);
 3682     DECODE : S0(4);     // any decoder
 3683 %}
 3684 
 3685 // Conditional move reg-reg
 3686 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3687 %{
 3688     single_instruction;
 3689     dst    : S4(write);
 3690     src    : S3(read);
 3691     cr     : S3(read);
 3692     DECODE : S0;        // any decoder
 3693 %}
 3694 
 3695 // Conditional move reg-mem
 3696 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3697 %{
 3698     single_instruction;
 3699     dst    : S4(write);
 3700     src    : S3(read);
 3701     cr     : S3(read);
 3702     DECODE : S0;        // any decoder
 3703     MEM    : S3;
 3704 %}
 3705 
 3706 // Conditional move reg-reg long
 3707 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3708 %{
 3709     single_instruction;
 3710     dst    : S4(write);
 3711     src    : S3(read);
 3712     cr     : S3(read);
 3713     DECODE : S0(2);     // any 2 decoders
 3714 %}
 3715 
 3716 // Float reg-reg operation
 3717 pipe_class fpu_reg(regD dst)
 3718 %{
 3719     instruction_count(2);
 3720     dst    : S3(read);
 3721     DECODE : S0(2);     // any 2 decoders
 3722     FPU    : S3;
 3723 %}
 3724 
 3725 // Float reg-reg operation
 3726 pipe_class fpu_reg_reg(regD dst, regD src)
 3727 %{
 3728     instruction_count(2);
 3729     dst    : S4(write);
 3730     src    : S3(read);
 3731     DECODE : S0(2);     // any 2 decoders
 3732     FPU    : S3;
 3733 %}
 3734 
 3735 // Float reg-reg operation
 3736 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3737 %{
 3738     instruction_count(3);
 3739     dst    : S4(write);
 3740     src1   : S3(read);
 3741     src2   : S3(read);
 3742     DECODE : S0(3);     // any 3 decoders
 3743     FPU    : S3(2);
 3744 %}
 3745 
 3746 // Float reg-reg operation
 3747 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3748 %{
 3749     instruction_count(4);
 3750     dst    : S4(write);
 3751     src1   : S3(read);
 3752     src2   : S3(read);
 3753     src3   : S3(read);
 3754     DECODE : S0(4);     // any 3 decoders
 3755     FPU    : S3(2);
 3756 %}
 3757 
 3758 // Float reg-reg operation
 3759 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3760 %{
 3761     instruction_count(4);
 3762     dst    : S4(write);
 3763     src1   : S3(read);
 3764     src2   : S3(read);
 3765     src3   : S3(read);
 3766     DECODE : S1(3);     // any 3 decoders
 3767     D0     : S0;        // Big decoder only
 3768     FPU    : S3(2);
 3769     MEM    : S3;
 3770 %}
 3771 
 3772 // Float reg-mem operation
 3773 pipe_class fpu_reg_mem(regD dst, memory mem)
 3774 %{
 3775     instruction_count(2);
 3776     dst    : S5(write);
 3777     mem    : S3(read);
 3778     D0     : S0;        // big decoder only
 3779     DECODE : S1;        // any decoder for FPU POP
 3780     FPU    : S4;
 3781     MEM    : S3;        // any mem
 3782 %}
 3783 
 3784 // Float reg-mem operation
 3785 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3786 %{
 3787     instruction_count(3);
 3788     dst    : S5(write);
 3789     src1   : S3(read);
 3790     mem    : S3(read);
 3791     D0     : S0;        // big decoder only
 3792     DECODE : S1(2);     // any decoder for FPU POP
 3793     FPU    : S4;
 3794     MEM    : S3;        // any mem
 3795 %}
 3796 
 3797 // Float mem-reg operation
 3798 pipe_class fpu_mem_reg(memory mem, regD src)
 3799 %{
 3800     instruction_count(2);
 3801     src    : S5(read);
 3802     mem    : S3(read);
 3803     DECODE : S0;        // any decoder for FPU PUSH
 3804     D0     : S1;        // big decoder only
 3805     FPU    : S4;
 3806     MEM    : S3;        // any mem
 3807 %}
 3808 
 3809 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3810 %{
 3811     instruction_count(3);
 3812     src1   : S3(read);
 3813     src2   : S3(read);
 3814     mem    : S3(read);
 3815     DECODE : S0(2);     // any decoder for FPU PUSH
 3816     D0     : S1;        // big decoder only
 3817     FPU    : S4;
 3818     MEM    : S3;        // any mem
 3819 %}
 3820 
 3821 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3822 %{
 3823     instruction_count(3);
 3824     src1   : S3(read);
 3825     src2   : S3(read);
 3826     mem    : S4(read);
 3827     DECODE : S0;        // any decoder for FPU PUSH
 3828     D0     : S0(2);     // big decoder only
 3829     FPU    : S4;
 3830     MEM    : S3(2);     // any mem
 3831 %}
 3832 
 3833 pipe_class fpu_mem_mem(memory dst, memory src1)
 3834 %{
 3835     instruction_count(2);
 3836     src1   : S3(read);
 3837     dst    : S4(read);
 3838     D0     : S0(2);     // big decoder only
 3839     MEM    : S3(2);     // any mem
 3840 %}
 3841 
 3842 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3843 %{
 3844     instruction_count(3);
 3845     src1   : S3(read);
 3846     src2   : S3(read);
 3847     dst    : S4(read);
 3848     D0     : S0(3);     // big decoder only
 3849     FPU    : S4;
 3850     MEM    : S3(3);     // any mem
 3851 %}
 3852 
 3853 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3854 %{
 3855     instruction_count(3);
 3856     src1   : S4(read);
 3857     mem    : S4(read);
 3858     DECODE : S0;        // any decoder for FPU PUSH
 3859     D0     : S0(2);     // big decoder only
 3860     FPU    : S4;
 3861     MEM    : S3(2);     // any mem
 3862 %}
 3863 
 3864 // Float load constant
 3865 pipe_class fpu_reg_con(regD dst)
 3866 %{
 3867     instruction_count(2);
 3868     dst    : S5(write);
 3869     D0     : S0;        // big decoder only for the load
 3870     DECODE : S1;        // any decoder for FPU POP
 3871     FPU    : S4;
 3872     MEM    : S3;        // any mem
 3873 %}
 3874 
 3875 // Float load constant
 3876 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3877 %{
 3878     instruction_count(3);
 3879     dst    : S5(write);
 3880     src    : S3(read);
 3881     D0     : S0;        // big decoder only for the load
 3882     DECODE : S1(2);     // any decoder for FPU POP
 3883     FPU    : S4;
 3884     MEM    : S3;        // any mem
 3885 %}
 3886 
 3887 // UnConditional branch
 3888 pipe_class pipe_jmp(label labl)
 3889 %{
 3890     single_instruction;
 3891     BR   : S3;
 3892 %}
 3893 
 3894 // Conditional branch
 3895 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3896 %{
 3897     single_instruction;
 3898     cr    : S1(read);
 3899     BR    : S3;
 3900 %}
 3901 
 3902 // Allocation idiom
 3903 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3904 %{
 3905     instruction_count(1); force_serialization;
 3906     fixed_latency(6);
 3907     heap_ptr : S3(read);
 3908     DECODE   : S0(3);
 3909     D0       : S2;
 3910     MEM      : S3;
 3911     ALU      : S3(2);
 3912     dst      : S5(write);
 3913     BR       : S5;
 3914 %}
 3915 
 3916 // Generic big/slow expanded idiom
 3917 pipe_class pipe_slow()
 3918 %{
 3919     instruction_count(10); multiple_bundles; force_serialization;
 3920     fixed_latency(100);
 3921     D0  : S0(2);
 3922     MEM : S3(2);
 3923 %}
 3924 
 3925 // The real do-nothing guy
 3926 pipe_class empty()
 3927 %{
 3928     instruction_count(0);
 3929 %}
 3930 
 3931 // Define the class for the Nop node
 3932 define
 3933 %{
 3934    MachNop = empty;
 3935 %}
 3936 
 3937 %}
 3938 
 3939 //----------INSTRUCTIONS-------------------------------------------------------
 3940 //
 3941 // match      -- States which machine-independent subtree may be replaced
 3942 //               by this instruction.
 3943 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3944 //               selection to identify a minimum cost tree of machine
 3945 //               instructions that matches a tree of machine-independent
 3946 //               instructions.
 3947 // format     -- A string providing the disassembly for this instruction.
 3948 //               The value of an instruction's operand may be inserted
 3949 //               by referring to it with a '$' prefix.
 3950 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3951 //               to within an encode class as $primary, $secondary, and $tertiary
 3952 //               rrspectively.  The primary opcode is commonly used to
 3953 //               indicate the type of machine instruction, while secondary
 3954 //               and tertiary are often used for prefix options or addressing
 3955 //               modes.
 3956 // ins_encode -- A list of encode classes with parameters. The encode class
 3957 //               name must have been defined in an 'enc_class' specification
 3958 //               in the encode section of the architecture description.
 3959 
 3960 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3961 // Load Float
 3962 instruct MoveF2VL(vlRegF dst, regF src) %{
 3963   match(Set dst src);
 3964   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3965   ins_encode %{
 3966     ShouldNotReachHere();
 3967   %}
 3968   ins_pipe( fpu_reg_reg );
 3969 %}
 3970 
 3971 // Load Float
 3972 instruct MoveF2LEG(legRegF dst, regF src) %{
 3973   match(Set dst src);
 3974   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3975   ins_encode %{
 3976     ShouldNotReachHere();
 3977   %}
 3978   ins_pipe( fpu_reg_reg );
 3979 %}
 3980 
 3981 // Load Float
 3982 instruct MoveVL2F(regF dst, vlRegF src) %{
 3983   match(Set dst src);
 3984   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3985   ins_encode %{
 3986     ShouldNotReachHere();
 3987   %}
 3988   ins_pipe( fpu_reg_reg );
 3989 %}
 3990 
 3991 // Load Float
 3992 instruct MoveLEG2F(regF dst, legRegF src) %{
 3993   match(Set dst src);
 3994   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3995   ins_encode %{
 3996     ShouldNotReachHere();
 3997   %}
 3998   ins_pipe( fpu_reg_reg );
 3999 %}
 4000 
 4001 // Load Double
 4002 instruct MoveD2VL(vlRegD dst, regD src) %{
 4003   match(Set dst src);
 4004   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4005   ins_encode %{
 4006     ShouldNotReachHere();
 4007   %}
 4008   ins_pipe( fpu_reg_reg );
 4009 %}
 4010 
 4011 // Load Double
 4012 instruct MoveD2LEG(legRegD dst, regD src) %{
 4013   match(Set dst src);
 4014   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4015   ins_encode %{
 4016     ShouldNotReachHere();
 4017   %}
 4018   ins_pipe( fpu_reg_reg );
 4019 %}
 4020 
 4021 // Load Double
 4022 instruct MoveVL2D(regD dst, vlRegD src) %{
 4023   match(Set dst src);
 4024   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4025   ins_encode %{
 4026     ShouldNotReachHere();
 4027   %}
 4028   ins_pipe( fpu_reg_reg );
 4029 %}
 4030 
 4031 // Load Double
 4032 instruct MoveLEG2D(regD dst, legRegD src) %{
 4033   match(Set dst src);
 4034   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4035   ins_encode %{
 4036     ShouldNotReachHere();
 4037   %}
 4038   ins_pipe( fpu_reg_reg );
 4039 %}
 4040 
 4041 //----------Load/Store/Move Instructions---------------------------------------
 4042 //----------Load Instructions--------------------------------------------------
 4043 
 4044 // Load Byte (8 bit signed)
 4045 instruct loadB(rRegI dst, memory mem)
 4046 %{
 4047   match(Set dst (LoadB mem));
 4048 
 4049   ins_cost(125);
 4050   format %{ "movsbl  $dst, $mem\t# byte" %}
 4051 
 4052   ins_encode %{
 4053     __ movsbl($dst$$Register, $mem$$Address);
 4054   %}
 4055 
 4056   ins_pipe(ialu_reg_mem);
 4057 %}
 4058 
 4059 // Load Byte (8 bit signed) into Long Register
 4060 instruct loadB2L(rRegL dst, memory mem)
 4061 %{
 4062   match(Set dst (ConvI2L (LoadB mem)));
 4063 
 4064   ins_cost(125);
 4065   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4066 
 4067   ins_encode %{
 4068     __ movsbq($dst$$Register, $mem$$Address);
 4069   %}
 4070 
 4071   ins_pipe(ialu_reg_mem);
 4072 %}
 4073 
 4074 // Load Unsigned Byte (8 bit UNsigned)
 4075 instruct loadUB(rRegI dst, memory mem)
 4076 %{
 4077   match(Set dst (LoadUB mem));
 4078 
 4079   ins_cost(125);
 4080   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4081 
 4082   ins_encode %{
 4083     __ movzbl($dst$$Register, $mem$$Address);
 4084   %}
 4085 
 4086   ins_pipe(ialu_reg_mem);
 4087 %}
 4088 
 4089 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4090 instruct loadUB2L(rRegL dst, memory mem)
 4091 %{
 4092   match(Set dst (ConvI2L (LoadUB mem)));
 4093 
 4094   ins_cost(125);
 4095   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4096 
 4097   ins_encode %{
 4098     __ movzbq($dst$$Register, $mem$$Address);
 4099   %}
 4100 
 4101   ins_pipe(ialu_reg_mem);
 4102 %}
 4103 
 4104 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4105 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4106   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4107   effect(KILL cr);
 4108 
 4109   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4110             "andl    $dst, right_n_bits($mask, 8)" %}
 4111   ins_encode %{
 4112     Register Rdst = $dst$$Register;
 4113     __ movzbq(Rdst, $mem$$Address);
 4114     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4115   %}
 4116   ins_pipe(ialu_reg_mem);
 4117 %}
 4118 
 4119 // Load Short (16 bit signed)
 4120 instruct loadS(rRegI dst, memory mem)
 4121 %{
 4122   match(Set dst (LoadS mem));
 4123 
 4124   ins_cost(125);
 4125   format %{ "movswl $dst, $mem\t# short" %}
 4126 
 4127   ins_encode %{
 4128     __ movswl($dst$$Register, $mem$$Address);
 4129   %}
 4130 
 4131   ins_pipe(ialu_reg_mem);
 4132 %}
 4133 
 4134 // Load Short (16 bit signed) to Byte (8 bit signed)
 4135 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4136   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4137 
 4138   ins_cost(125);
 4139   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4140   ins_encode %{
 4141     __ movsbl($dst$$Register, $mem$$Address);
 4142   %}
 4143   ins_pipe(ialu_reg_mem);
 4144 %}
 4145 
 4146 // Load Short (16 bit signed) into Long Register
 4147 instruct loadS2L(rRegL dst, memory mem)
 4148 %{
 4149   match(Set dst (ConvI2L (LoadS mem)));
 4150 
 4151   ins_cost(125);
 4152   format %{ "movswq $dst, $mem\t# short -> long" %}
 4153 
 4154   ins_encode %{
 4155     __ movswq($dst$$Register, $mem$$Address);
 4156   %}
 4157 
 4158   ins_pipe(ialu_reg_mem);
 4159 %}
 4160 
 4161 // Load Unsigned Short/Char (16 bit UNsigned)
 4162 instruct loadUS(rRegI dst, memory mem)
 4163 %{
 4164   match(Set dst (LoadUS mem));
 4165 
 4166   ins_cost(125);
 4167   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4168 
 4169   ins_encode %{
 4170     __ movzwl($dst$$Register, $mem$$Address);
 4171   %}
 4172 
 4173   ins_pipe(ialu_reg_mem);
 4174 %}
 4175 
 4176 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4177 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4178   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4179 
 4180   ins_cost(125);
 4181   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4182   ins_encode %{
 4183     __ movsbl($dst$$Register, $mem$$Address);
 4184   %}
 4185   ins_pipe(ialu_reg_mem);
 4186 %}
 4187 
 4188 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4189 instruct loadUS2L(rRegL dst, memory mem)
 4190 %{
 4191   match(Set dst (ConvI2L (LoadUS mem)));
 4192 
 4193   ins_cost(125);
 4194   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4195 
 4196   ins_encode %{
 4197     __ movzwq($dst$$Register, $mem$$Address);
 4198   %}
 4199 
 4200   ins_pipe(ialu_reg_mem);
 4201 %}
 4202 
 4203 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4204 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4205   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4206 
 4207   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4208   ins_encode %{
 4209     __ movzbq($dst$$Register, $mem$$Address);
 4210   %}
 4211   ins_pipe(ialu_reg_mem);
 4212 %}
 4213 
 4214 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4215 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4216   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4217   effect(KILL cr);
 4218 
 4219   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4220             "andl    $dst, right_n_bits($mask, 16)" %}
 4221   ins_encode %{
 4222     Register Rdst = $dst$$Register;
 4223     __ movzwq(Rdst, $mem$$Address);
 4224     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4225   %}
 4226   ins_pipe(ialu_reg_mem);
 4227 %}
 4228 
 4229 // Load Integer
 4230 instruct loadI(rRegI dst, memory mem)
 4231 %{
 4232   match(Set dst (LoadI mem));
 4233 
 4234   ins_cost(125);
 4235   format %{ "movl    $dst, $mem\t# int" %}
 4236 
 4237   ins_encode %{
 4238     __ movl($dst$$Register, $mem$$Address);
 4239   %}
 4240 
 4241   ins_pipe(ialu_reg_mem);
 4242 %}
 4243 
 4244 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4245 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4246   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4247 
 4248   ins_cost(125);
 4249   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4250   ins_encode %{
 4251     __ movsbl($dst$$Register, $mem$$Address);
 4252   %}
 4253   ins_pipe(ialu_reg_mem);
 4254 %}
 4255 
 4256 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4257 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4258   match(Set dst (AndI (LoadI mem) mask));
 4259 
 4260   ins_cost(125);
 4261   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4262   ins_encode %{
 4263     __ movzbl($dst$$Register, $mem$$Address);
 4264   %}
 4265   ins_pipe(ialu_reg_mem);
 4266 %}
 4267 
 4268 // Load Integer (32 bit signed) to Short (16 bit signed)
 4269 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4270   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4271 
 4272   ins_cost(125);
 4273   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4274   ins_encode %{
 4275     __ movswl($dst$$Register, $mem$$Address);
 4276   %}
 4277   ins_pipe(ialu_reg_mem);
 4278 %}
 4279 
 4280 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4281 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4282   match(Set dst (AndI (LoadI mem) mask));
 4283 
 4284   ins_cost(125);
 4285   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4286   ins_encode %{
 4287     __ movzwl($dst$$Register, $mem$$Address);
 4288   %}
 4289   ins_pipe(ialu_reg_mem);
 4290 %}
 4291 
 4292 // Load Integer into Long Register
 4293 instruct loadI2L(rRegL dst, memory mem)
 4294 %{
 4295   match(Set dst (ConvI2L (LoadI mem)));
 4296 
 4297   ins_cost(125);
 4298   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4299 
 4300   ins_encode %{
 4301     __ movslq($dst$$Register, $mem$$Address);
 4302   %}
 4303 
 4304   ins_pipe(ialu_reg_mem);
 4305 %}
 4306 
 4307 // Load Integer with mask 0xFF into Long Register
 4308 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4309   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4310 
 4311   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4312   ins_encode %{
 4313     __ movzbq($dst$$Register, $mem$$Address);
 4314   %}
 4315   ins_pipe(ialu_reg_mem);
 4316 %}
 4317 
 4318 // Load Integer with mask 0xFFFF into Long Register
 4319 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4320   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4321 
 4322   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4323   ins_encode %{
 4324     __ movzwq($dst$$Register, $mem$$Address);
 4325   %}
 4326   ins_pipe(ialu_reg_mem);
 4327 %}
 4328 
 4329 // Load Integer with a 31-bit mask into Long Register
 4330 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4331   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4332   effect(KILL cr);
 4333 
 4334   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4335             "andl    $dst, $mask" %}
 4336   ins_encode %{
 4337     Register Rdst = $dst$$Register;
 4338     __ movl(Rdst, $mem$$Address);
 4339     __ andl(Rdst, $mask$$constant);
 4340   %}
 4341   ins_pipe(ialu_reg_mem);
 4342 %}
 4343 
 4344 // Load Unsigned Integer into Long Register
 4345 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4346 %{
 4347   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4348 
 4349   ins_cost(125);
 4350   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4351 
 4352   ins_encode %{
 4353     __ movl($dst$$Register, $mem$$Address);
 4354   %}
 4355 
 4356   ins_pipe(ialu_reg_mem);
 4357 %}
 4358 
 4359 // Load Long
 4360 instruct loadL(rRegL dst, memory mem)
 4361 %{
 4362   match(Set dst (LoadL mem));
 4363 
 4364   ins_cost(125);
 4365   format %{ "movq    $dst, $mem\t# long" %}
 4366 
 4367   ins_encode %{
 4368     __ movq($dst$$Register, $mem$$Address);
 4369   %}
 4370 
 4371   ins_pipe(ialu_reg_mem); // XXX
 4372 %}
 4373 
 4374 // Load Range
 4375 instruct loadRange(rRegI dst, memory mem)
 4376 %{
 4377   match(Set dst (LoadRange mem));
 4378 
 4379   ins_cost(125); // XXX
 4380   format %{ "movl    $dst, $mem\t# range" %}
 4381   ins_encode %{
 4382     __ movl($dst$$Register, $mem$$Address);
 4383   %}
 4384   ins_pipe(ialu_reg_mem);
 4385 %}
 4386 
 4387 // Load Pointer
 4388 instruct loadP(rRegP dst, memory mem)
 4389 %{
 4390   match(Set dst (LoadP mem));
 4391   predicate(n->as_Load()->barrier_data() == 0);
 4392 
 4393   ins_cost(125); // XXX
 4394   format %{ "movq    $dst, $mem\t# ptr" %}
 4395   ins_encode %{
 4396     __ movq($dst$$Register, $mem$$Address);
 4397   %}
 4398   ins_pipe(ialu_reg_mem); // XXX
 4399 %}
 4400 
 4401 // Load Compressed Pointer
 4402 instruct loadN(rRegN dst, memory mem)
 4403 %{
 4404    match(Set dst (LoadN mem));
 4405 
 4406    ins_cost(125); // XXX
 4407    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4408    ins_encode %{
 4409      __ movl($dst$$Register, $mem$$Address);
 4410    %}
 4411    ins_pipe(ialu_reg_mem); // XXX
 4412 %}
 4413 
 4414 
 4415 // Load Klass Pointer
 4416 instruct loadKlass(rRegP dst, memory mem)
 4417 %{
 4418   match(Set dst (LoadKlass mem));
 4419 
 4420   ins_cost(125); // XXX
 4421   format %{ "movq    $dst, $mem\t# class" %}
 4422   ins_encode %{
 4423     __ movq($dst$$Register, $mem$$Address);
 4424   %}
 4425   ins_pipe(ialu_reg_mem); // XXX
 4426 %}
 4427 
 4428 // Load narrow Klass Pointer
 4429 instruct loadNKlass(rRegN dst, memory mem)
 4430 %{
 4431   match(Set dst (LoadNKlass mem));
 4432 
 4433   ins_cost(125); // XXX
 4434   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4435   ins_encode %{
 4436     __ movl($dst$$Register, $mem$$Address);
 4437   %}
 4438   ins_pipe(ialu_reg_mem); // XXX
 4439 %}
 4440 
 4441 // Load Float
 4442 instruct loadF(regF dst, memory mem)
 4443 %{
 4444   match(Set dst (LoadF mem));
 4445 
 4446   ins_cost(145); // XXX
 4447   format %{ "movss   $dst, $mem\t# float" %}
 4448   ins_encode %{
 4449     __ movflt($dst$$XMMRegister, $mem$$Address);
 4450   %}
 4451   ins_pipe(pipe_slow); // XXX
 4452 %}
 4453 
 4454 // Load Double
 4455 instruct loadD_partial(regD dst, memory mem)
 4456 %{
 4457   predicate(!UseXmmLoadAndClearUpper);
 4458   match(Set dst (LoadD mem));
 4459 
 4460   ins_cost(145); // XXX
 4461   format %{ "movlpd  $dst, $mem\t# double" %}
 4462   ins_encode %{
 4463     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4464   %}
 4465   ins_pipe(pipe_slow); // XXX
 4466 %}
 4467 
 4468 instruct loadD(regD dst, memory mem)
 4469 %{
 4470   predicate(UseXmmLoadAndClearUpper);
 4471   match(Set dst (LoadD mem));
 4472 
 4473   ins_cost(145); // XXX
 4474   format %{ "movsd   $dst, $mem\t# double" %}
 4475   ins_encode %{
 4476     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4477   %}
 4478   ins_pipe(pipe_slow); // XXX
 4479 %}
 4480 
 4481 
 4482 // Following pseudo code describes the algorithm for max[FD]:
 4483 // Min algorithm is on similar lines
 4484 //  btmp = (b < +0.0) ? a : b
 4485 //  atmp = (b < +0.0) ? b : a
 4486 //  Tmp  = Max_Float(atmp , btmp)
 4487 //  Res  = (atmp == NaN) ? atmp : Tmp
 4488 
 4489 // max = java.lang.Math.max(float a, float b)
 4490 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4491   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4492   match(Set dst (MaxF a b));
 4493   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4494   format %{
 4495      "vblendvps        $btmp,$b,$a,$b           \n\t"
 4496      "vblendvps        $atmp,$a,$b,$b           \n\t"
 4497      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 4498      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 4499      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 4500   %}
 4501   ins_encode %{
 4502     int vector_len = Assembler::AVX_128bit;
 4503     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 4504     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 4505     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 4506     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 4507     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 4508  %}
 4509   ins_pipe( pipe_slow );
 4510 %}
 4511 
 4512 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4513   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4514   match(Set dst (MaxF a b));
 4515   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4516 
 4517   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 4518   ins_encode %{
 4519     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4520                     false /*min*/, true /*single*/);
 4521   %}
 4522   ins_pipe( pipe_slow );
 4523 %}
 4524 
 4525 // max = java.lang.Math.max(double a, double b)
 4526 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4527   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4528   match(Set dst (MaxD a b));
 4529   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4530   format %{
 4531      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 4532      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 4533      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 4534      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 4535      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 4536   %}
 4537   ins_encode %{
 4538     int vector_len = Assembler::AVX_128bit;
 4539     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 4540     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 4541     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 4542     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 4543     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 4544   %}
 4545   ins_pipe( pipe_slow );
 4546 %}
 4547 
 4548 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4549   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4550   match(Set dst (MaxD a b));
 4551   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4552 
 4553   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 4554   ins_encode %{
 4555     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4556                     false /*min*/, false /*single*/);
 4557   %}
 4558   ins_pipe( pipe_slow );
 4559 %}
 4560 
 4561 // min = java.lang.Math.min(float a, float b)
 4562 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4563   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4564   match(Set dst (MinF a b));
 4565   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4566   format %{
 4567      "vblendvps        $atmp,$a,$b,$a             \n\t"
 4568      "vblendvps        $btmp,$b,$a,$a             \n\t"
 4569      "vminss           $tmp,$atmp,$btmp           \n\t"
 4570      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 4571      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 4572   %}
 4573   ins_encode %{
 4574     int vector_len = Assembler::AVX_128bit;
 4575     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 4576     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 4577     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 4578     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 4579     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 4580   %}
 4581   ins_pipe( pipe_slow );
 4582 %}
 4583 
 4584 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 4585   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4586   match(Set dst (MinF a b));
 4587   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4588 
 4589   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 4590   ins_encode %{
 4591     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4592                     true /*min*/, true /*single*/);
 4593   %}
 4594   ins_pipe( pipe_slow );
 4595 %}
 4596 
 4597 // min = java.lang.Math.min(double a, double b)
 4598 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4599   predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
 4600   match(Set dst (MinD a b));
 4601   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4602   format %{
 4603      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 4604      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 4605      "vminsd           $tmp,$atmp,$btmp         \n\t"
 4606      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 4607      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 4608   %}
 4609   ins_encode %{
 4610     int vector_len = Assembler::AVX_128bit;
 4611     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 4612     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 4613     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 4614     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 4615     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 4616   %}
 4617   ins_pipe( pipe_slow );
 4618 %}
 4619 
 4620 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 4621   predicate(UseAVX > 0 && SuperWord::is_reduction(n));
 4622   match(Set dst (MinD a b));
 4623   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 4624 
 4625   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 4626   ins_encode %{
 4627     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 4628                     true /*min*/, false /*single*/);
 4629   %}
 4630   ins_pipe( pipe_slow );
 4631 %}
 4632 
 4633 // Load Effective Address
 4634 instruct leaP8(rRegP dst, indOffset8 mem)
 4635 %{
 4636   match(Set dst mem);
 4637 
 4638   ins_cost(110); // XXX
 4639   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4640   ins_encode %{
 4641     __ leaq($dst$$Register, $mem$$Address);
 4642   %}
 4643   ins_pipe(ialu_reg_reg_fat);
 4644 %}
 4645 
 4646 instruct leaP32(rRegP dst, indOffset32 mem)
 4647 %{
 4648   match(Set dst mem);
 4649 
 4650   ins_cost(110);
 4651   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4652   ins_encode %{
 4653     __ leaq($dst$$Register, $mem$$Address);
 4654   %}
 4655   ins_pipe(ialu_reg_reg_fat);
 4656 %}
 4657 
 4658 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4659 %{
 4660   match(Set dst mem);
 4661 
 4662   ins_cost(110);
 4663   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4664   ins_encode %{
 4665     __ leaq($dst$$Register, $mem$$Address);
 4666   %}
 4667   ins_pipe(ialu_reg_reg_fat);
 4668 %}
 4669 
 4670 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4671 %{
 4672   match(Set dst mem);
 4673 
 4674   ins_cost(110);
 4675   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4676   ins_encode %{
 4677     __ leaq($dst$$Register, $mem$$Address);
 4678   %}
 4679   ins_pipe(ialu_reg_reg_fat);
 4680 %}
 4681 
 4682 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4683 %{
 4684   match(Set dst mem);
 4685 
 4686   ins_cost(110);
 4687   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4688   ins_encode %{
 4689     __ leaq($dst$$Register, $mem$$Address);
 4690   %}
 4691   ins_pipe(ialu_reg_reg_fat);
 4692 %}
 4693 
 4694 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4695 %{
 4696   match(Set dst mem);
 4697 
 4698   ins_cost(110);
 4699   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4700   ins_encode %{
 4701     __ leaq($dst$$Register, $mem$$Address);
 4702   %}
 4703   ins_pipe(ialu_reg_reg_fat);
 4704 %}
 4705 
 4706 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4707 %{
 4708   match(Set dst mem);
 4709 
 4710   ins_cost(110);
 4711   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4712   ins_encode %{
 4713     __ leaq($dst$$Register, $mem$$Address);
 4714   %}
 4715   ins_pipe(ialu_reg_reg_fat);
 4716 %}
 4717 
 4718 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4719 %{
 4720   match(Set dst mem);
 4721 
 4722   ins_cost(110);
 4723   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4724   ins_encode %{
 4725     __ leaq($dst$$Register, $mem$$Address);
 4726   %}
 4727   ins_pipe(ialu_reg_reg_fat);
 4728 %}
 4729 
 4730 // Load Effective Address which uses Narrow (32-bits) oop
 4731 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4732 %{
 4733   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4734   match(Set dst mem);
 4735 
 4736   ins_cost(110);
 4737   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4738   ins_encode %{
 4739     __ leaq($dst$$Register, $mem$$Address);
 4740   %}
 4741   ins_pipe(ialu_reg_reg_fat);
 4742 %}
 4743 
 4744 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4745 %{
 4746   predicate(CompressedOops::shift() == 0);
 4747   match(Set dst mem);
 4748 
 4749   ins_cost(110); // XXX
 4750   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4751   ins_encode %{
 4752     __ leaq($dst$$Register, $mem$$Address);
 4753   %}
 4754   ins_pipe(ialu_reg_reg_fat);
 4755 %}
 4756 
 4757 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4758 %{
 4759   predicate(CompressedOops::shift() == 0);
 4760   match(Set dst mem);
 4761 
 4762   ins_cost(110);
 4763   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4764   ins_encode %{
 4765     __ leaq($dst$$Register, $mem$$Address);
 4766   %}
 4767   ins_pipe(ialu_reg_reg_fat);
 4768 %}
 4769 
 4770 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4771 %{
 4772   predicate(CompressedOops::shift() == 0);
 4773   match(Set dst mem);
 4774 
 4775   ins_cost(110);
 4776   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4777   ins_encode %{
 4778     __ leaq($dst$$Register, $mem$$Address);
 4779   %}
 4780   ins_pipe(ialu_reg_reg_fat);
 4781 %}
 4782 
 4783 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4784 %{
 4785   predicate(CompressedOops::shift() == 0);
 4786   match(Set dst mem);
 4787 
 4788   ins_cost(110);
 4789   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4790   ins_encode %{
 4791     __ leaq($dst$$Register, $mem$$Address);
 4792   %}
 4793   ins_pipe(ialu_reg_reg_fat);
 4794 %}
 4795 
 4796 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4797 %{
 4798   predicate(CompressedOops::shift() == 0);
 4799   match(Set dst mem);
 4800 
 4801   ins_cost(110);
 4802   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4803   ins_encode %{
 4804     __ leaq($dst$$Register, $mem$$Address);
 4805   %}
 4806   ins_pipe(ialu_reg_reg_fat);
 4807 %}
 4808 
 4809 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4810 %{
 4811   predicate(CompressedOops::shift() == 0);
 4812   match(Set dst mem);
 4813 
 4814   ins_cost(110);
 4815   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4816   ins_encode %{
 4817     __ leaq($dst$$Register, $mem$$Address);
 4818   %}
 4819   ins_pipe(ialu_reg_reg_fat);
 4820 %}
 4821 
 4822 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4823 %{
 4824   predicate(CompressedOops::shift() == 0);
 4825   match(Set dst mem);
 4826 
 4827   ins_cost(110);
 4828   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4829   ins_encode %{
 4830     __ leaq($dst$$Register, $mem$$Address);
 4831   %}
 4832   ins_pipe(ialu_reg_reg_fat);
 4833 %}
 4834 
 4835 instruct loadConI(rRegI dst, immI src)
 4836 %{
 4837   match(Set dst src);
 4838 
 4839   format %{ "movl    $dst, $src\t# int" %}
 4840   ins_encode %{
 4841     __ movl($dst$$Register, $src$$constant);
 4842   %}
 4843   ins_pipe(ialu_reg_fat); // XXX
 4844 %}
 4845 
 4846 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4847 %{
 4848   match(Set dst src);
 4849   effect(KILL cr);
 4850 
 4851   ins_cost(50);
 4852   format %{ "xorl    $dst, $dst\t# int" %}
 4853   ins_encode %{
 4854     __ xorl($dst$$Register, $dst$$Register);
 4855   %}
 4856   ins_pipe(ialu_reg);
 4857 %}
 4858 
 4859 instruct loadConL(rRegL dst, immL src)
 4860 %{
 4861   match(Set dst src);
 4862 
 4863   ins_cost(150);
 4864   format %{ "movq    $dst, $src\t# long" %}
 4865   ins_encode %{
 4866     __ mov64($dst$$Register, $src$$constant);
 4867   %}
 4868   ins_pipe(ialu_reg);
 4869 %}
 4870 
 4871 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4872 %{
 4873   match(Set dst src);
 4874   effect(KILL cr);
 4875 
 4876   ins_cost(50);
 4877   format %{ "xorl    $dst, $dst\t# long" %}
 4878   ins_encode %{
 4879     __ xorl($dst$$Register, $dst$$Register);
 4880   %}
 4881   ins_pipe(ialu_reg); // XXX
 4882 %}
 4883 
 4884 instruct loadConUL32(rRegL dst, immUL32 src)
 4885 %{
 4886   match(Set dst src);
 4887 
 4888   ins_cost(60);
 4889   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4890   ins_encode %{
 4891     __ movl($dst$$Register, $src$$constant);
 4892   %}
 4893   ins_pipe(ialu_reg);
 4894 %}
 4895 
 4896 instruct loadConL32(rRegL dst, immL32 src)
 4897 %{
 4898   match(Set dst src);
 4899 
 4900   ins_cost(70);
 4901   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4902   ins_encode %{
 4903     __ movq($dst$$Register, $src$$constant);
 4904   %}
 4905   ins_pipe(ialu_reg);
 4906 %}
 4907 
 4908 instruct loadConP(rRegP dst, immP con) %{
 4909   match(Set dst con);
 4910 
 4911   format %{ "movq    $dst, $con\t# ptr" %}
 4912   ins_encode %{
 4913     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4914   %}
 4915   ins_pipe(ialu_reg_fat); // XXX
 4916 %}
 4917 
 4918 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4919 %{
 4920   match(Set dst src);
 4921   effect(KILL cr);
 4922 
 4923   ins_cost(50);
 4924   format %{ "xorl    $dst, $dst\t# ptr" %}
 4925   ins_encode %{
 4926     __ xorl($dst$$Register, $dst$$Register);
 4927   %}
 4928   ins_pipe(ialu_reg);
 4929 %}
 4930 
 4931 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4932 %{
 4933   match(Set dst src);
 4934   effect(KILL cr);
 4935 
 4936   ins_cost(60);
 4937   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4938   ins_encode %{
 4939     __ movl($dst$$Register, $src$$constant);
 4940   %}
 4941   ins_pipe(ialu_reg);
 4942 %}
 4943 
 4944 instruct loadConF(regF dst, immF con) %{
 4945   match(Set dst con);
 4946   ins_cost(125);
 4947   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4948   ins_encode %{
 4949     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4950   %}
 4951   ins_pipe(pipe_slow);
 4952 %}
 4953 
 4954 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4955   match(Set dst src);
 4956   effect(KILL cr);
 4957   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 4958   ins_encode %{
 4959     __ xorq($dst$$Register, $dst$$Register);
 4960   %}
 4961   ins_pipe(ialu_reg);
 4962 %}
 4963 
 4964 instruct loadConN(rRegN dst, immN src) %{
 4965   match(Set dst src);
 4966 
 4967   ins_cost(125);
 4968   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4969   ins_encode %{
 4970     address con = (address)$src$$constant;
 4971     if (con == NULL) {
 4972       ShouldNotReachHere();
 4973     } else {
 4974       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4975     }
 4976   %}
 4977   ins_pipe(ialu_reg_fat); // XXX
 4978 %}
 4979 
 4980 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4981   match(Set dst src);
 4982 
 4983   ins_cost(125);
 4984   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4985   ins_encode %{
 4986     address con = (address)$src$$constant;
 4987     if (con == NULL) {
 4988       ShouldNotReachHere();
 4989     } else {
 4990       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4991     }
 4992   %}
 4993   ins_pipe(ialu_reg_fat); // XXX
 4994 %}
 4995 
 4996 instruct loadConF0(regF dst, immF0 src)
 4997 %{
 4998   match(Set dst src);
 4999   ins_cost(100);
 5000 
 5001   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5002   ins_encode %{
 5003     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5004   %}
 5005   ins_pipe(pipe_slow);
 5006 %}
 5007 
 5008 // Use the same format since predicate() can not be used here.
 5009 instruct loadConD(regD dst, immD con) %{
 5010   match(Set dst con);
 5011   ins_cost(125);
 5012   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5013   ins_encode %{
 5014     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5015   %}
 5016   ins_pipe(pipe_slow);
 5017 %}
 5018 
 5019 instruct loadConD0(regD dst, immD0 src)
 5020 %{
 5021   match(Set dst src);
 5022   ins_cost(100);
 5023 
 5024   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5025   ins_encode %{
 5026     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 5027   %}
 5028   ins_pipe(pipe_slow);
 5029 %}
 5030 
 5031 instruct loadSSI(rRegI dst, stackSlotI src)
 5032 %{
 5033   match(Set dst src);
 5034 
 5035   ins_cost(125);
 5036   format %{ "movl    $dst, $src\t# int stk" %}
 5037   ins_encode %{
 5038     __ movl($dst$$Register, $src$$Address);
 5039   %}
 5040   ins_pipe(ialu_reg_mem);
 5041 %}
 5042 
 5043 instruct loadSSL(rRegL dst, stackSlotL src)
 5044 %{
 5045   match(Set dst src);
 5046 
 5047   ins_cost(125);
 5048   format %{ "movq    $dst, $src\t# long stk" %}
 5049   ins_encode %{
 5050     __ movq($dst$$Register, $src$$Address);
 5051   %}
 5052   ins_pipe(ialu_reg_mem);
 5053 %}
 5054 
 5055 instruct loadSSP(rRegP dst, stackSlotP src)
 5056 %{
 5057   match(Set dst src);
 5058 
 5059   ins_cost(125);
 5060   format %{ "movq    $dst, $src\t# ptr stk" %}
 5061   ins_encode %{
 5062     __ movq($dst$$Register, $src$$Address);
 5063   %}
 5064   ins_pipe(ialu_reg_mem);
 5065 %}
 5066 
 5067 instruct loadSSF(regF dst, stackSlotF src)
 5068 %{
 5069   match(Set dst src);
 5070 
 5071   ins_cost(125);
 5072   format %{ "movss   $dst, $src\t# float stk" %}
 5073   ins_encode %{
 5074     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5075   %}
 5076   ins_pipe(pipe_slow); // XXX
 5077 %}
 5078 
 5079 // Use the same format since predicate() can not be used here.
 5080 instruct loadSSD(regD dst, stackSlotD src)
 5081 %{
 5082   match(Set dst src);
 5083 
 5084   ins_cost(125);
 5085   format %{ "movsd   $dst, $src\t# double stk" %}
 5086   ins_encode  %{
 5087     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5088   %}
 5089   ins_pipe(pipe_slow); // XXX
 5090 %}
 5091 
 5092 // Prefetch instructions for allocation.
 5093 // Must be safe to execute with invalid address (cannot fault).
 5094 
 5095 instruct prefetchAlloc( memory mem ) %{
 5096   predicate(AllocatePrefetchInstr==3);
 5097   match(PrefetchAllocation mem);
 5098   ins_cost(125);
 5099 
 5100   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5101   ins_encode %{
 5102     __ prefetchw($mem$$Address);
 5103   %}
 5104   ins_pipe(ialu_mem);
 5105 %}
 5106 
 5107 instruct prefetchAllocNTA( memory mem ) %{
 5108   predicate(AllocatePrefetchInstr==0);
 5109   match(PrefetchAllocation mem);
 5110   ins_cost(125);
 5111 
 5112   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5113   ins_encode %{
 5114     __ prefetchnta($mem$$Address);
 5115   %}
 5116   ins_pipe(ialu_mem);
 5117 %}
 5118 
 5119 instruct prefetchAllocT0( memory mem ) %{
 5120   predicate(AllocatePrefetchInstr==1);
 5121   match(PrefetchAllocation mem);
 5122   ins_cost(125);
 5123 
 5124   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5125   ins_encode %{
 5126     __ prefetcht0($mem$$Address);
 5127   %}
 5128   ins_pipe(ialu_mem);
 5129 %}
 5130 
 5131 instruct prefetchAllocT2( memory mem ) %{
 5132   predicate(AllocatePrefetchInstr==2);
 5133   match(PrefetchAllocation mem);
 5134   ins_cost(125);
 5135 
 5136   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5137   ins_encode %{
 5138     __ prefetcht2($mem$$Address);
 5139   %}
 5140   ins_pipe(ialu_mem);
 5141 %}
 5142 
 5143 //----------Store Instructions-------------------------------------------------
 5144 
 5145 // Store Byte
 5146 instruct storeB(memory mem, rRegI src)
 5147 %{
 5148   match(Set mem (StoreB mem src));
 5149 
 5150   ins_cost(125); // XXX
 5151   format %{ "movb    $mem, $src\t# byte" %}
 5152   ins_encode %{
 5153     __ movb($mem$$Address, $src$$Register);
 5154   %}
 5155   ins_pipe(ialu_mem_reg);
 5156 %}
 5157 
 5158 // Store Char/Short
 5159 instruct storeC(memory mem, rRegI src)
 5160 %{
 5161   match(Set mem (StoreC mem src));
 5162 
 5163   ins_cost(125); // XXX
 5164   format %{ "movw    $mem, $src\t# char/short" %}
 5165   ins_encode %{
 5166     __ movw($mem$$Address, $src$$Register);
 5167   %}
 5168   ins_pipe(ialu_mem_reg);
 5169 %}
 5170 
 5171 // Store Integer
 5172 instruct storeI(memory mem, rRegI src)
 5173 %{
 5174   match(Set mem (StoreI mem src));
 5175 
 5176   ins_cost(125); // XXX
 5177   format %{ "movl    $mem, $src\t# int" %}
 5178   ins_encode %{
 5179     __ movl($mem$$Address, $src$$Register);
 5180   %}
 5181   ins_pipe(ialu_mem_reg);
 5182 %}
 5183 
 5184 // Store Long
 5185 instruct storeL(memory mem, rRegL src)
 5186 %{
 5187   match(Set mem (StoreL mem src));
 5188 
 5189   ins_cost(125); // XXX
 5190   format %{ "movq    $mem, $src\t# long" %}
 5191   ins_encode %{
 5192     __ movq($mem$$Address, $src$$Register);
 5193   %}
 5194   ins_pipe(ialu_mem_reg); // XXX
 5195 %}
 5196 
 5197 // Store Pointer
 5198 instruct storeP(memory mem, any_RegP src)
 5199 %{
 5200   predicate(n->as_Store()->barrier_data() == 0);
 5201   match(Set mem (StoreP mem src));
 5202 
 5203   ins_cost(125); // XXX
 5204   format %{ "movq    $mem, $src\t# ptr" %}
 5205   ins_encode %{
 5206     __ movq($mem$$Address, $src$$Register);
 5207   %}
 5208   ins_pipe(ialu_mem_reg);
 5209 %}
 5210 
 5211 instruct storeImmP0(memory mem, immP0 zero)
 5212 %{
 5213   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && n->as_Store()->barrier_data() == 0);
 5214   match(Set mem (StoreP mem zero));
 5215 
 5216   ins_cost(125); // XXX
 5217   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5218   ins_encode %{
 5219     __ movq($mem$$Address, r12);
 5220   %}
 5221   ins_pipe(ialu_mem_reg);
 5222 %}
 5223 
 5224 // Store NULL Pointer, mark word, or other simple pointer constant.
 5225 instruct storeImmP(memory mem, immP31 src)
 5226 %{
 5227   predicate(n->as_Store()->barrier_data() == 0);
 5228   match(Set mem (StoreP mem src));
 5229 
 5230   ins_cost(150); // XXX
 5231   format %{ "movq    $mem, $src\t# ptr" %}
 5232   ins_encode %{
 5233     __ movq($mem$$Address, $src$$constant);
 5234   %}
 5235   ins_pipe(ialu_mem_imm);
 5236 %}
 5237 
 5238 // Store Compressed Pointer
 5239 instruct storeN(memory mem, rRegN src)
 5240 %{
 5241   match(Set mem (StoreN mem src));
 5242 
 5243   ins_cost(125); // XXX
 5244   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5245   ins_encode %{
 5246     __ movl($mem$$Address, $src$$Register);
 5247   %}
 5248   ins_pipe(ialu_mem_reg);
 5249 %}
 5250 
 5251 instruct storeNKlass(memory mem, rRegN src)
 5252 %{
 5253   match(Set mem (StoreNKlass mem src));
 5254 
 5255   ins_cost(125); // XXX
 5256   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5257   ins_encode %{
 5258     __ movl($mem$$Address, $src$$Register);
 5259   %}
 5260   ins_pipe(ialu_mem_reg);
 5261 %}
 5262 
 5263 instruct storeImmN0(memory mem, immN0 zero)
 5264 %{
 5265   predicate(CompressedOops::base() == NULL);
 5266   match(Set mem (StoreN mem zero));
 5267 
 5268   ins_cost(125); // XXX
 5269   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5270   ins_encode %{
 5271     __ movl($mem$$Address, r12);
 5272   %}
 5273   ins_pipe(ialu_mem_reg);
 5274 %}
 5275 
 5276 instruct storeImmN(memory mem, immN src)
 5277 %{
 5278   match(Set mem (StoreN mem src));
 5279 
 5280   ins_cost(150); // XXX
 5281   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5282   ins_encode %{
 5283     address con = (address)$src$$constant;
 5284     if (con == NULL) {
 5285       __ movl($mem$$Address, 0);
 5286     } else {
 5287       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5288     }
 5289   %}
 5290   ins_pipe(ialu_mem_imm);
 5291 %}
 5292 
 5293 instruct storeImmNKlass(memory mem, immNKlass src)
 5294 %{
 5295   match(Set mem (StoreNKlass mem src));
 5296 
 5297   ins_cost(150); // XXX
 5298   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5299   ins_encode %{
 5300     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5301   %}
 5302   ins_pipe(ialu_mem_imm);
 5303 %}
 5304 
 5305 // Store Integer Immediate
 5306 instruct storeImmI0(memory mem, immI_0 zero)
 5307 %{
 5308   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5309   match(Set mem (StoreI mem zero));
 5310 
 5311   ins_cost(125); // XXX
 5312   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5313   ins_encode %{
 5314     __ movl($mem$$Address, r12);
 5315   %}
 5316   ins_pipe(ialu_mem_reg);
 5317 %}
 5318 
 5319 instruct storeImmI(memory mem, immI src)
 5320 %{
 5321   match(Set mem (StoreI mem src));
 5322 
 5323   ins_cost(150);
 5324   format %{ "movl    $mem, $src\t# int" %}
 5325   ins_encode %{
 5326     __ movl($mem$$Address, $src$$constant);
 5327   %}
 5328   ins_pipe(ialu_mem_imm);
 5329 %}
 5330 
 5331 // Store Long Immediate
 5332 instruct storeImmL0(memory mem, immL0 zero)
 5333 %{
 5334   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5335   match(Set mem (StoreL mem zero));
 5336 
 5337   ins_cost(125); // XXX
 5338   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5339   ins_encode %{
 5340     __ movq($mem$$Address, r12);
 5341   %}
 5342   ins_pipe(ialu_mem_reg);
 5343 %}
 5344 
 5345 instruct storeImmL(memory mem, immL32 src)
 5346 %{
 5347   match(Set mem (StoreL mem src));
 5348 
 5349   ins_cost(150);
 5350   format %{ "movq    $mem, $src\t# long" %}
 5351   ins_encode %{
 5352     __ movq($mem$$Address, $src$$constant);
 5353   %}
 5354   ins_pipe(ialu_mem_imm);
 5355 %}
 5356 
 5357 // Store Short/Char Immediate
 5358 instruct storeImmC0(memory mem, immI_0 zero)
 5359 %{
 5360   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5361   match(Set mem (StoreC mem zero));
 5362 
 5363   ins_cost(125); // XXX
 5364   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5365   ins_encode %{
 5366     __ movw($mem$$Address, r12);
 5367   %}
 5368   ins_pipe(ialu_mem_reg);
 5369 %}
 5370 
 5371 instruct storeImmI16(memory mem, immI16 src)
 5372 %{
 5373   predicate(UseStoreImmI16);
 5374   match(Set mem (StoreC mem src));
 5375 
 5376   ins_cost(150);
 5377   format %{ "movw    $mem, $src\t# short/char" %}
 5378   ins_encode %{
 5379     __ movw($mem$$Address, $src$$constant);
 5380   %}
 5381   ins_pipe(ialu_mem_imm);
 5382 %}
 5383 
 5384 // Store Byte Immediate
 5385 instruct storeImmB0(memory mem, immI_0 zero)
 5386 %{
 5387   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5388   match(Set mem (StoreB mem zero));
 5389 
 5390   ins_cost(125); // XXX
 5391   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5392   ins_encode %{
 5393     __ movb($mem$$Address, r12);
 5394   %}
 5395   ins_pipe(ialu_mem_reg);
 5396 %}
 5397 
 5398 instruct storeImmB(memory mem, immI8 src)
 5399 %{
 5400   match(Set mem (StoreB mem src));
 5401 
 5402   ins_cost(150); // XXX
 5403   format %{ "movb    $mem, $src\t# byte" %}
 5404   ins_encode %{
 5405     __ movb($mem$$Address, $src$$constant);
 5406   %}
 5407   ins_pipe(ialu_mem_imm);
 5408 %}
 5409 
 5410 // Store CMS card-mark Immediate
 5411 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 5412 %{
 5413   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5414   match(Set mem (StoreCM mem zero));
 5415 
 5416   ins_cost(125); // XXX
 5417   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 5418   ins_encode %{
 5419     __ movb($mem$$Address, r12);
 5420   %}
 5421   ins_pipe(ialu_mem_reg);
 5422 %}
 5423 
 5424 instruct storeImmCM0(memory mem, immI_0 src)
 5425 %{
 5426   match(Set mem (StoreCM mem src));
 5427 
 5428   ins_cost(150); // XXX
 5429   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 5430   ins_encode %{
 5431     __ movb($mem$$Address, $src$$constant);
 5432   %}
 5433   ins_pipe(ialu_mem_imm);
 5434 %}
 5435 
 5436 // Store Float
 5437 instruct storeF(memory mem, regF src)
 5438 %{
 5439   match(Set mem (StoreF mem src));
 5440 
 5441   ins_cost(95); // XXX
 5442   format %{ "movss   $mem, $src\t# float" %}
 5443   ins_encode %{
 5444     __ movflt($mem$$Address, $src$$XMMRegister);
 5445   %}
 5446   ins_pipe(pipe_slow); // XXX
 5447 %}
 5448 
 5449 // Store immediate Float value (it is faster than store from XMM register)
 5450 instruct storeF0(memory mem, immF0 zero)
 5451 %{
 5452   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5453   match(Set mem (StoreF mem zero));
 5454 
 5455   ins_cost(25); // XXX
 5456   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5457   ins_encode %{
 5458     __ movl($mem$$Address, r12);
 5459   %}
 5460   ins_pipe(ialu_mem_reg);
 5461 %}
 5462 
 5463 instruct storeF_imm(memory mem, immF src)
 5464 %{
 5465   match(Set mem (StoreF mem src));
 5466 
 5467   ins_cost(50);
 5468   format %{ "movl    $mem, $src\t# float" %}
 5469   ins_encode %{
 5470     __ movl($mem$$Address, jint_cast($src$$constant));
 5471   %}
 5472   ins_pipe(ialu_mem_imm);
 5473 %}
 5474 
 5475 // Store Double
 5476 instruct storeD(memory mem, regD src)
 5477 %{
 5478   match(Set mem (StoreD mem src));
 5479 
 5480   ins_cost(95); // XXX
 5481   format %{ "movsd   $mem, $src\t# double" %}
 5482   ins_encode %{
 5483     __ movdbl($mem$$Address, $src$$XMMRegister);
 5484   %}
 5485   ins_pipe(pipe_slow); // XXX
 5486 %}
 5487 
 5488 // Store immediate double 0.0 (it is faster than store from XMM register)
 5489 instruct storeD0_imm(memory mem, immD0 src)
 5490 %{
 5491   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 5492   match(Set mem (StoreD mem src));
 5493 
 5494   ins_cost(50);
 5495   format %{ "movq    $mem, $src\t# double 0." %}
 5496   ins_encode %{
 5497     __ movq($mem$$Address, $src$$constant);
 5498   %}
 5499   ins_pipe(ialu_mem_imm);
 5500 %}
 5501 
 5502 instruct storeD0(memory mem, immD0 zero)
 5503 %{
 5504   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 5505   match(Set mem (StoreD mem zero));
 5506 
 5507   ins_cost(25); // XXX
 5508   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5509   ins_encode %{
 5510     __ movq($mem$$Address, r12);
 5511   %}
 5512   ins_pipe(ialu_mem_reg);
 5513 %}
 5514 
 5515 instruct storeSSI(stackSlotI dst, rRegI src)
 5516 %{
 5517   match(Set dst src);
 5518 
 5519   ins_cost(100);
 5520   format %{ "movl    $dst, $src\t# int stk" %}
 5521   ins_encode %{
 5522     __ movl($dst$$Address, $src$$Register);
 5523   %}
 5524   ins_pipe( ialu_mem_reg );
 5525 %}
 5526 
 5527 instruct storeSSL(stackSlotL dst, rRegL src)
 5528 %{
 5529   match(Set dst src);
 5530 
 5531   ins_cost(100);
 5532   format %{ "movq    $dst, $src\t# long stk" %}
 5533   ins_encode %{
 5534     __ movq($dst$$Address, $src$$Register);
 5535   %}
 5536   ins_pipe(ialu_mem_reg);
 5537 %}
 5538 
 5539 instruct storeSSP(stackSlotP dst, rRegP src)
 5540 %{
 5541   match(Set dst src);
 5542 
 5543   ins_cost(100);
 5544   format %{ "movq    $dst, $src\t# ptr stk" %}
 5545   ins_encode %{
 5546     __ movq($dst$$Address, $src$$Register);
 5547   %}
 5548   ins_pipe(ialu_mem_reg);
 5549 %}
 5550 
 5551 instruct storeSSF(stackSlotF dst, regF src)
 5552 %{
 5553   match(Set dst src);
 5554 
 5555   ins_cost(95); // XXX
 5556   format %{ "movss   $dst, $src\t# float stk" %}
 5557   ins_encode %{
 5558     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5559   %}
 5560   ins_pipe(pipe_slow); // XXX
 5561 %}
 5562 
 5563 instruct storeSSD(stackSlotD dst, regD src)
 5564 %{
 5565   match(Set dst src);
 5566 
 5567   ins_cost(95); // XXX
 5568   format %{ "movsd   $dst, $src\t# double stk" %}
 5569   ins_encode %{
 5570     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5571   %}
 5572   ins_pipe(pipe_slow); // XXX
 5573 %}
 5574 
 5575 instruct cacheWB(indirect addr)
 5576 %{
 5577   predicate(VM_Version::supports_data_cache_line_flush());
 5578   match(CacheWB addr);
 5579 
 5580   ins_cost(100);
 5581   format %{"cache wb $addr" %}
 5582   ins_encode %{
 5583     assert($addr->index_position() < 0, "should be");
 5584     assert($addr$$disp == 0, "should be");
 5585     __ cache_wb(Address($addr$$base$$Register, 0));
 5586   %}
 5587   ins_pipe(pipe_slow); // XXX
 5588 %}
 5589 
 5590 instruct cacheWBPreSync()
 5591 %{
 5592   predicate(VM_Version::supports_data_cache_line_flush());
 5593   match(CacheWBPreSync);
 5594 
 5595   ins_cost(100);
 5596   format %{"cache wb presync" %}
 5597   ins_encode %{
 5598     __ cache_wbsync(true);
 5599   %}
 5600   ins_pipe(pipe_slow); // XXX
 5601 %}
 5602 
 5603 instruct cacheWBPostSync()
 5604 %{
 5605   predicate(VM_Version::supports_data_cache_line_flush());
 5606   match(CacheWBPostSync);
 5607 
 5608   ins_cost(100);
 5609   format %{"cache wb postsync" %}
 5610   ins_encode %{
 5611     __ cache_wbsync(false);
 5612   %}
 5613   ins_pipe(pipe_slow); // XXX
 5614 %}
 5615 
 5616 //----------BSWAP Instructions-------------------------------------------------
 5617 instruct bytes_reverse_int(rRegI dst) %{
 5618   match(Set dst (ReverseBytesI dst));
 5619 
 5620   format %{ "bswapl  $dst" %}
 5621   ins_encode %{
 5622     __ bswapl($dst$$Register);
 5623   %}
 5624   ins_pipe( ialu_reg );
 5625 %}
 5626 
 5627 instruct bytes_reverse_long(rRegL dst) %{
 5628   match(Set dst (ReverseBytesL dst));
 5629 
 5630   format %{ "bswapq  $dst" %}
 5631   ins_encode %{
 5632     __ bswapq($dst$$Register);
 5633   %}
 5634   ins_pipe( ialu_reg);
 5635 %}
 5636 
 5637 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5638   match(Set dst (ReverseBytesUS dst));
 5639   effect(KILL cr);
 5640 
 5641   format %{ "bswapl  $dst\n\t"
 5642             "shrl    $dst,16\n\t" %}
 5643   ins_encode %{
 5644     __ bswapl($dst$$Register);
 5645     __ shrl($dst$$Register, 16);
 5646   %}
 5647   ins_pipe( ialu_reg );
 5648 %}
 5649 
 5650 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5651   match(Set dst (ReverseBytesS dst));
 5652   effect(KILL cr);
 5653 
 5654   format %{ "bswapl  $dst\n\t"
 5655             "sar     $dst,16\n\t" %}
 5656   ins_encode %{
 5657     __ bswapl($dst$$Register);
 5658     __ sarl($dst$$Register, 16);
 5659   %}
 5660   ins_pipe( ialu_reg );
 5661 %}
 5662 
 5663 //---------- Zeros Count Instructions ------------------------------------------
 5664 
 5665 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5666   predicate(UseCountLeadingZerosInstruction);
 5667   match(Set dst (CountLeadingZerosI src));
 5668   effect(KILL cr);
 5669 
 5670   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5671   ins_encode %{
 5672     __ lzcntl($dst$$Register, $src$$Register);
 5673   %}
 5674   ins_pipe(ialu_reg);
 5675 %}
 5676 
 5677 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5678   predicate(UseCountLeadingZerosInstruction);
 5679   match(Set dst (CountLeadingZerosI (LoadI src)));
 5680   effect(KILL cr);
 5681   ins_cost(175);
 5682   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5683   ins_encode %{
 5684     __ lzcntl($dst$$Register, $src$$Address);
 5685   %}
 5686   ins_pipe(ialu_reg_mem);
 5687 %}
 5688 
 5689 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5690   predicate(!UseCountLeadingZerosInstruction);
 5691   match(Set dst (CountLeadingZerosI src));
 5692   effect(KILL cr);
 5693 
 5694   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5695             "jnz     skip\n\t"
 5696             "movl    $dst, -1\n"
 5697       "skip:\n\t"
 5698             "negl    $dst\n\t"
 5699             "addl    $dst, 31" %}
 5700   ins_encode %{
 5701     Register Rdst = $dst$$Register;
 5702     Register Rsrc = $src$$Register;
 5703     Label skip;
 5704     __ bsrl(Rdst, Rsrc);
 5705     __ jccb(Assembler::notZero, skip);
 5706     __ movl(Rdst, -1);
 5707     __ bind(skip);
 5708     __ negl(Rdst);
 5709     __ addl(Rdst, BitsPerInt - 1);
 5710   %}
 5711   ins_pipe(ialu_reg);
 5712 %}
 5713 
 5714 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5715   predicate(UseCountLeadingZerosInstruction);
 5716   match(Set dst (CountLeadingZerosL src));
 5717   effect(KILL cr);
 5718 
 5719   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5720   ins_encode %{
 5721     __ lzcntq($dst$$Register, $src$$Register);
 5722   %}
 5723   ins_pipe(ialu_reg);
 5724 %}
 5725 
 5726 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5727   predicate(UseCountLeadingZerosInstruction);
 5728   match(Set dst (CountLeadingZerosL (LoadL src)));
 5729   effect(KILL cr);
 5730   ins_cost(175);
 5731   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5732   ins_encode %{
 5733     __ lzcntq($dst$$Register, $src$$Address);
 5734   %}
 5735   ins_pipe(ialu_reg_mem);
 5736 %}
 5737 
 5738 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5739   predicate(!UseCountLeadingZerosInstruction);
 5740   match(Set dst (CountLeadingZerosL src));
 5741   effect(KILL cr);
 5742 
 5743   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5744             "jnz     skip\n\t"
 5745             "movl    $dst, -1\n"
 5746       "skip:\n\t"
 5747             "negl    $dst\n\t"
 5748             "addl    $dst, 63" %}
 5749   ins_encode %{
 5750     Register Rdst = $dst$$Register;
 5751     Register Rsrc = $src$$Register;
 5752     Label skip;
 5753     __ bsrq(Rdst, Rsrc);
 5754     __ jccb(Assembler::notZero, skip);
 5755     __ movl(Rdst, -1);
 5756     __ bind(skip);
 5757     __ negl(Rdst);
 5758     __ addl(Rdst, BitsPerLong - 1);
 5759   %}
 5760   ins_pipe(ialu_reg);
 5761 %}
 5762 
 5763 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5764   predicate(UseCountTrailingZerosInstruction);
 5765   match(Set dst (CountTrailingZerosI src));
 5766   effect(KILL cr);
 5767 
 5768   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5769   ins_encode %{
 5770     __ tzcntl($dst$$Register, $src$$Register);
 5771   %}
 5772   ins_pipe(ialu_reg);
 5773 %}
 5774 
 5775 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5776   predicate(UseCountTrailingZerosInstruction);
 5777   match(Set dst (CountTrailingZerosI (LoadI src)));
 5778   effect(KILL cr);
 5779   ins_cost(175);
 5780   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5781   ins_encode %{
 5782     __ tzcntl($dst$$Register, $src$$Address);
 5783   %}
 5784   ins_pipe(ialu_reg_mem);
 5785 %}
 5786 
 5787 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5788   predicate(!UseCountTrailingZerosInstruction);
 5789   match(Set dst (CountTrailingZerosI src));
 5790   effect(KILL cr);
 5791 
 5792   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5793             "jnz     done\n\t"
 5794             "movl    $dst, 32\n"
 5795       "done:" %}
 5796   ins_encode %{
 5797     Register Rdst = $dst$$Register;
 5798     Label done;
 5799     __ bsfl(Rdst, $src$$Register);
 5800     __ jccb(Assembler::notZero, done);
 5801     __ movl(Rdst, BitsPerInt);
 5802     __ bind(done);
 5803   %}
 5804   ins_pipe(ialu_reg);
 5805 %}
 5806 
 5807 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5808   predicate(UseCountTrailingZerosInstruction);
 5809   match(Set dst (CountTrailingZerosL src));
 5810   effect(KILL cr);
 5811 
 5812   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5813   ins_encode %{
 5814     __ tzcntq($dst$$Register, $src$$Register);
 5815   %}
 5816   ins_pipe(ialu_reg);
 5817 %}
 5818 
 5819 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5820   predicate(UseCountTrailingZerosInstruction);
 5821   match(Set dst (CountTrailingZerosL (LoadL src)));
 5822   effect(KILL cr);
 5823   ins_cost(175);
 5824   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5825   ins_encode %{
 5826     __ tzcntq($dst$$Register, $src$$Address);
 5827   %}
 5828   ins_pipe(ialu_reg_mem);
 5829 %}
 5830 
 5831 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5832   predicate(!UseCountTrailingZerosInstruction);
 5833   match(Set dst (CountTrailingZerosL src));
 5834   effect(KILL cr);
 5835 
 5836   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5837             "jnz     done\n\t"
 5838             "movl    $dst, 64\n"
 5839       "done:" %}
 5840   ins_encode %{
 5841     Register Rdst = $dst$$Register;
 5842     Label done;
 5843     __ bsfq(Rdst, $src$$Register);
 5844     __ jccb(Assembler::notZero, done);
 5845     __ movl(Rdst, BitsPerLong);
 5846     __ bind(done);
 5847   %}
 5848   ins_pipe(ialu_reg);
 5849 %}
 5850 
 5851 //--------------- Reverse Operation Instructions ----------------
 5852 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5853   predicate(!VM_Version::supports_gfni());
 5854   match(Set dst (ReverseI src));
 5855   effect(TEMP dst, TEMP rtmp, KILL cr);
 5856   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5857   ins_encode %{
 5858     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5859   %}
 5860   ins_pipe( ialu_reg );
 5861 %}
 5862 
 5863 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5864   predicate(VM_Version::supports_gfni());
 5865   match(Set dst (ReverseI src));
 5866   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5867   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5868   ins_encode %{
 5869     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5870   %}
 5871   ins_pipe( ialu_reg );
 5872 %}
 5873 
 5874 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5875   predicate(!VM_Version::supports_gfni());
 5876   match(Set dst (ReverseL src));
 5877   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5878   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5879   ins_encode %{
 5880     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5881   %}
 5882   ins_pipe( ialu_reg );
 5883 %}
 5884 
 5885 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5886   predicate(VM_Version::supports_gfni());
 5887   match(Set dst (ReverseL src));
 5888   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5889   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5890   ins_encode %{
 5891     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5892   %}
 5893   ins_pipe( ialu_reg );
 5894 %}
 5895 
 5896 //---------- Population Count Instructions -------------------------------------
 5897 
 5898 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5899   predicate(UsePopCountInstruction);
 5900   match(Set dst (PopCountI src));
 5901   effect(KILL cr);
 5902 
 5903   format %{ "popcnt  $dst, $src" %}
 5904   ins_encode %{
 5905     __ popcntl($dst$$Register, $src$$Register);
 5906   %}
 5907   ins_pipe(ialu_reg);
 5908 %}
 5909 
 5910 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5911   predicate(UsePopCountInstruction);
 5912   match(Set dst (PopCountI (LoadI mem)));
 5913   effect(KILL cr);
 5914 
 5915   format %{ "popcnt  $dst, $mem" %}
 5916   ins_encode %{
 5917     __ popcntl($dst$$Register, $mem$$Address);
 5918   %}
 5919   ins_pipe(ialu_reg);
 5920 %}
 5921 
 5922 // Note: Long.bitCount(long) returns an int.
 5923 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5924   predicate(UsePopCountInstruction);
 5925   match(Set dst (PopCountL src));
 5926   effect(KILL cr);
 5927 
 5928   format %{ "popcnt  $dst, $src" %}
 5929   ins_encode %{
 5930     __ popcntq($dst$$Register, $src$$Register);
 5931   %}
 5932   ins_pipe(ialu_reg);
 5933 %}
 5934 
 5935 // Note: Long.bitCount(long) returns an int.
 5936 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5937   predicate(UsePopCountInstruction);
 5938   match(Set dst (PopCountL (LoadL mem)));
 5939   effect(KILL cr);
 5940 
 5941   format %{ "popcnt  $dst, $mem" %}
 5942   ins_encode %{
 5943     __ popcntq($dst$$Register, $mem$$Address);
 5944   %}
 5945   ins_pipe(ialu_reg);
 5946 %}
 5947 
 5948 
 5949 //----------MemBar Instructions-----------------------------------------------
 5950 // Memory barrier flavors
 5951 
 5952 instruct membar_acquire()
 5953 %{
 5954   match(MemBarAcquire);
 5955   match(LoadFence);
 5956   ins_cost(0);
 5957 
 5958   size(0);
 5959   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5960   ins_encode();
 5961   ins_pipe(empty);
 5962 %}
 5963 
 5964 instruct membar_acquire_lock()
 5965 %{
 5966   match(MemBarAcquireLock);
 5967   ins_cost(0);
 5968 
 5969   size(0);
 5970   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5971   ins_encode();
 5972   ins_pipe(empty);
 5973 %}
 5974 
 5975 instruct membar_release()
 5976 %{
 5977   match(MemBarRelease);
 5978   match(StoreFence);
 5979   ins_cost(0);
 5980 
 5981   size(0);
 5982   format %{ "MEMBAR-release ! (empty encoding)" %}
 5983   ins_encode();
 5984   ins_pipe(empty);
 5985 %}
 5986 
 5987 instruct membar_release_lock()
 5988 %{
 5989   match(MemBarReleaseLock);
 5990   ins_cost(0);
 5991 
 5992   size(0);
 5993   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5994   ins_encode();
 5995   ins_pipe(empty);
 5996 %}
 5997 
 5998 instruct membar_volatile(rFlagsReg cr) %{
 5999   match(MemBarVolatile);
 6000   effect(KILL cr);
 6001   ins_cost(400);
 6002 
 6003   format %{
 6004     $$template
 6005     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6006   %}
 6007   ins_encode %{
 6008     __ membar(Assembler::StoreLoad);
 6009   %}
 6010   ins_pipe(pipe_slow);
 6011 %}
 6012 
 6013 instruct unnecessary_membar_volatile()
 6014 %{
 6015   match(MemBarVolatile);
 6016   predicate(Matcher::post_store_load_barrier(n));
 6017   ins_cost(0);
 6018 
 6019   size(0);
 6020   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6021   ins_encode();
 6022   ins_pipe(empty);
 6023 %}
 6024 
 6025 instruct membar_storestore() %{
 6026   match(MemBarStoreStore);
 6027   match(StoreStoreFence);
 6028   ins_cost(0);
 6029 
 6030   size(0);
 6031   format %{ "MEMBAR-storestore (empty encoding)" %}
 6032   ins_encode( );
 6033   ins_pipe(empty);
 6034 %}
 6035 
 6036 //----------Move Instructions--------------------------------------------------
 6037 
 6038 instruct castX2P(rRegP dst, rRegL src)
 6039 %{
 6040   match(Set dst (CastX2P src));
 6041 
 6042   format %{ "movq    $dst, $src\t# long->ptr" %}
 6043   ins_encode %{
 6044     if ($dst$$reg != $src$$reg) {
 6045       __ movptr($dst$$Register, $src$$Register);
 6046     }
 6047   %}
 6048   ins_pipe(ialu_reg_reg); // XXX
 6049 %}
 6050 
 6051 instruct castP2X(rRegL dst, rRegP src)
 6052 %{
 6053   match(Set dst (CastP2X src));
 6054 
 6055   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6056   ins_encode %{
 6057     if ($dst$$reg != $src$$reg) {
 6058       __ movptr($dst$$Register, $src$$Register);
 6059     }
 6060   %}
 6061   ins_pipe(ialu_reg_reg); // XXX
 6062 %}
 6063 
 6064 // Convert oop into int for vectors alignment masking
 6065 instruct convP2I(rRegI dst, rRegP src)
 6066 %{
 6067   match(Set dst (ConvL2I (CastP2X src)));
 6068 
 6069   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6070   ins_encode %{
 6071     __ movl($dst$$Register, $src$$Register);
 6072   %}
 6073   ins_pipe(ialu_reg_reg); // XXX
 6074 %}
 6075 
 6076 // Convert compressed oop into int for vectors alignment masking
 6077 // in case of 32bit oops (heap < 4Gb).
 6078 instruct convN2I(rRegI dst, rRegN src)
 6079 %{
 6080   predicate(CompressedOops::shift() == 0);
 6081   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6082 
 6083   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6084   ins_encode %{
 6085     __ movl($dst$$Register, $src$$Register);
 6086   %}
 6087   ins_pipe(ialu_reg_reg); // XXX
 6088 %}
 6089 
 6090 // Convert oop pointer into compressed form
 6091 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6092   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6093   match(Set dst (EncodeP src));
 6094   effect(KILL cr);
 6095   format %{ "encode_heap_oop $dst,$src" %}
 6096   ins_encode %{
 6097     Register s = $src$$Register;
 6098     Register d = $dst$$Register;
 6099     if (s != d) {
 6100       __ movq(d, s);
 6101     }
 6102     __ encode_heap_oop(d);
 6103   %}
 6104   ins_pipe(ialu_reg_long);
 6105 %}
 6106 
 6107 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6108   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6109   match(Set dst (EncodeP src));
 6110   effect(KILL cr);
 6111   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6112   ins_encode %{
 6113     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6114   %}
 6115   ins_pipe(ialu_reg_long);
 6116 %}
 6117 
 6118 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6119   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6120             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6121   match(Set dst (DecodeN src));
 6122   effect(KILL cr);
 6123   format %{ "decode_heap_oop $dst,$src" %}
 6124   ins_encode %{
 6125     Register s = $src$$Register;
 6126     Register d = $dst$$Register;
 6127     if (s != d) {
 6128       __ movq(d, s);
 6129     }
 6130     __ decode_heap_oop(d);
 6131   %}
 6132   ins_pipe(ialu_reg_long);
 6133 %}
 6134 
 6135 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6136   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6137             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6138   match(Set dst (DecodeN src));
 6139   effect(KILL cr);
 6140   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6141   ins_encode %{
 6142     Register s = $src$$Register;
 6143     Register d = $dst$$Register;
 6144     if (s != d) {
 6145       __ decode_heap_oop_not_null(d, s);
 6146     } else {
 6147       __ decode_heap_oop_not_null(d);
 6148     }
 6149   %}
 6150   ins_pipe(ialu_reg_long);
 6151 %}
 6152 
 6153 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6154   match(Set dst (EncodePKlass src));
 6155   effect(TEMP dst, KILL cr);
 6156   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6157   ins_encode %{
 6158     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6159   %}
 6160   ins_pipe(ialu_reg_long);
 6161 %}
 6162 
 6163 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6164   match(Set dst (DecodeNKlass src));
 6165   effect(TEMP dst, KILL cr);
 6166   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6167   ins_encode %{
 6168     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6169   %}
 6170   ins_pipe(ialu_reg_long);
 6171 %}
 6172 
 6173 //----------Conditional Move---------------------------------------------------
 6174 // Jump
 6175 // dummy instruction for generating temp registers
 6176 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6177   match(Jump (LShiftL switch_val shift));
 6178   ins_cost(350);
 6179   predicate(false);
 6180   effect(TEMP dest);
 6181 
 6182   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6183             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6184   ins_encode %{
 6185     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6186     // to do that and the compiler is using that register as one it can allocate.
 6187     // So we build it all by hand.
 6188     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6189     // ArrayAddress dispatch(table, index);
 6190     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6191     __ lea($dest$$Register, $constantaddress);
 6192     __ jmp(dispatch);
 6193   %}
 6194   ins_pipe(pipe_jmp);
 6195 %}
 6196 
 6197 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6198   match(Jump (AddL (LShiftL switch_val shift) offset));
 6199   ins_cost(350);
 6200   effect(TEMP dest);
 6201 
 6202   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6203             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6204   ins_encode %{
 6205     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6206     // to do that and the compiler is using that register as one it can allocate.
 6207     // So we build it all by hand.
 6208     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6209     // ArrayAddress dispatch(table, index);
 6210     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6211     __ lea($dest$$Register, $constantaddress);
 6212     __ jmp(dispatch);
 6213   %}
 6214   ins_pipe(pipe_jmp);
 6215 %}
 6216 
 6217 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6218   match(Jump switch_val);
 6219   ins_cost(350);
 6220   effect(TEMP dest);
 6221 
 6222   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6223             "jmp     [$dest + $switch_val]\n\t" %}
 6224   ins_encode %{
 6225     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6226     // to do that and the compiler is using that register as one it can allocate.
 6227     // So we build it all by hand.
 6228     // Address index(noreg, switch_reg, Address::times_1);
 6229     // ArrayAddress dispatch(table, index);
 6230     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6231     __ lea($dest$$Register, $constantaddress);
 6232     __ jmp(dispatch);
 6233   %}
 6234   ins_pipe(pipe_jmp);
 6235 %}
 6236 
 6237 // Conditional move
 6238 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6239 %{
 6240   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6241   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6242 
 6243   ins_cost(100); // XXX
 6244   format %{ "setbn$cop $dst\t# signed, int" %}
 6245   ins_encode %{
 6246     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6247     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6248   %}
 6249   ins_pipe(ialu_reg);
 6250 %}
 6251 
 6252 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6253 %{
 6254   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6255 
 6256   ins_cost(200); // XXX
 6257   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6258   ins_encode %{
 6259     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6260   %}
 6261   ins_pipe(pipe_cmov_reg);
 6262 %}
 6263 
 6264 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6265 %{
 6266   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6267   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6268 
 6269   ins_cost(100); // XXX
 6270   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6271   ins_encode %{
 6272     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6273     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6274   %}
 6275   ins_pipe(ialu_reg);
 6276 %}
 6277 
 6278 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6279   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6280 
 6281   ins_cost(200); // XXX
 6282   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6283   ins_encode %{
 6284     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6285   %}
 6286   ins_pipe(pipe_cmov_reg);
 6287 %}
 6288 
 6289 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6290 %{
 6291   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6292   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6293 
 6294   ins_cost(100); // XXX
 6295   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6296   ins_encode %{
 6297     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6298     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6299   %}
 6300   ins_pipe(ialu_reg);
 6301 %}
 6302 
 6303 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6304   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6305   ins_cost(200);
 6306   expand %{
 6307     cmovI_regU(cop, cr, dst, src);
 6308   %}
 6309 %}
 6310 
 6311 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6312   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6313   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6314 
 6315   ins_cost(200); // XXX
 6316   format %{ "cmovpl  $dst, $src\n\t"
 6317             "cmovnel $dst, $src" %}
 6318   ins_encode %{
 6319     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6320     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6321   %}
 6322   ins_pipe(pipe_cmov_reg);
 6323 %}
 6324 
 6325 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6326 // inputs of the CMove
 6327 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6328   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6329   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6330 
 6331   ins_cost(200); // XXX
 6332   format %{ "cmovpl  $dst, $src\n\t"
 6333             "cmovnel $dst, $src" %}
 6334   ins_encode %{
 6335     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6336     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6337   %}
 6338   ins_pipe(pipe_cmov_reg);
 6339 %}
 6340 
 6341 // Conditional move
 6342 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6343   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6344 
 6345   ins_cost(250); // XXX
 6346   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6347   ins_encode %{
 6348     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6349   %}
 6350   ins_pipe(pipe_cmov_mem);
 6351 %}
 6352 
 6353 // Conditional move
 6354 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6355 %{
 6356   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6357 
 6358   ins_cost(250); // XXX
 6359   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6360   ins_encode %{
 6361     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6362   %}
 6363   ins_pipe(pipe_cmov_mem);
 6364 %}
 6365 
 6366 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6367   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6368   ins_cost(250);
 6369   expand %{
 6370     cmovI_memU(cop, cr, dst, src);
 6371   %}
 6372 %}
 6373 
 6374 // Conditional move
 6375 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6376 %{
 6377   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6378 
 6379   ins_cost(200); // XXX
 6380   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6381   ins_encode %{
 6382     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6383   %}
 6384   ins_pipe(pipe_cmov_reg);
 6385 %}
 6386 
 6387 // Conditional move
 6388 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6389 %{
 6390   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6391 
 6392   ins_cost(200); // XXX
 6393   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6394   ins_encode %{
 6395     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6396   %}
 6397   ins_pipe(pipe_cmov_reg);
 6398 %}
 6399 
 6400 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6401   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6402   ins_cost(200);
 6403   expand %{
 6404     cmovN_regU(cop, cr, dst, src);
 6405   %}
 6406 %}
 6407 
 6408 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6409   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6410   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6411 
 6412   ins_cost(200); // XXX
 6413   format %{ "cmovpl  $dst, $src\n\t"
 6414             "cmovnel $dst, $src" %}
 6415   ins_encode %{
 6416     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6417     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6418   %}
 6419   ins_pipe(pipe_cmov_reg);
 6420 %}
 6421 
 6422 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6423 // inputs of the CMove
 6424 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6425   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6426   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6427 
 6428   ins_cost(200); // XXX
 6429   format %{ "cmovpl  $dst, $src\n\t"
 6430             "cmovnel $dst, $src" %}
 6431   ins_encode %{
 6432     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6433     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6434   %}
 6435   ins_pipe(pipe_cmov_reg);
 6436 %}
 6437 
 6438 // Conditional move
 6439 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6440 %{
 6441   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6442 
 6443   ins_cost(200); // XXX
 6444   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6445   ins_encode %{
 6446     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6447   %}
 6448   ins_pipe(pipe_cmov_reg);  // XXX
 6449 %}
 6450 
 6451 // Conditional move
 6452 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6453 %{
 6454   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6455 
 6456   ins_cost(200); // XXX
 6457   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6458   ins_encode %{
 6459     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6460   %}
 6461   ins_pipe(pipe_cmov_reg); // XXX
 6462 %}
 6463 
 6464 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6465   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6466   ins_cost(200);
 6467   expand %{
 6468     cmovP_regU(cop, cr, dst, src);
 6469   %}
 6470 %}
 6471 
 6472 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6473   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6474   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6475 
 6476   ins_cost(200); // XXX
 6477   format %{ "cmovpq  $dst, $src\n\t"
 6478             "cmovneq $dst, $src" %}
 6479   ins_encode %{
 6480     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6481     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6482   %}
 6483   ins_pipe(pipe_cmov_reg);
 6484 %}
 6485 
 6486 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6487 // inputs of the CMove
 6488 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6489   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6490   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6491 
 6492   ins_cost(200); // XXX
 6493   format %{ "cmovpq  $dst, $src\n\t"
 6494             "cmovneq $dst, $src" %}
 6495   ins_encode %{
 6496     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6497     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6498   %}
 6499   ins_pipe(pipe_cmov_reg);
 6500 %}
 6501 
 6502 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6503 %{
 6504   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6505   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6506 
 6507   ins_cost(100); // XXX
 6508   format %{ "setbn$cop $dst\t# signed, long" %}
 6509   ins_encode %{
 6510     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6511     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6512   %}
 6513   ins_pipe(ialu_reg);
 6514 %}
 6515 
 6516 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6517 %{
 6518   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6519 
 6520   ins_cost(200); // XXX
 6521   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6522   ins_encode %{
 6523     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6524   %}
 6525   ins_pipe(pipe_cmov_reg);  // XXX
 6526 %}
 6527 
 6528 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6529 %{
 6530   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6531 
 6532   ins_cost(200); // XXX
 6533   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6534   ins_encode %{
 6535     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6536   %}
 6537   ins_pipe(pipe_cmov_mem);  // XXX
 6538 %}
 6539 
 6540 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6541 %{
 6542   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6543   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6544 
 6545   ins_cost(100); // XXX
 6546   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6547   ins_encode %{
 6548     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6549     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6550   %}
 6551   ins_pipe(ialu_reg);
 6552 %}
 6553 
 6554 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6555 %{
 6556   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6557 
 6558   ins_cost(200); // XXX
 6559   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6560   ins_encode %{
 6561     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6562   %}
 6563   ins_pipe(pipe_cmov_reg); // XXX
 6564 %}
 6565 
 6566 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6567 %{
 6568   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6569   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6570 
 6571   ins_cost(100); // XXX
 6572   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6573   ins_encode %{
 6574     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6575     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6576   %}
 6577   ins_pipe(ialu_reg);
 6578 %}
 6579 
 6580 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6581   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6582   ins_cost(200);
 6583   expand %{
 6584     cmovL_regU(cop, cr, dst, src);
 6585   %}
 6586 %}
 6587 
 6588 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6589   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6590   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6591 
 6592   ins_cost(200); // XXX
 6593   format %{ "cmovpq  $dst, $src\n\t"
 6594             "cmovneq $dst, $src" %}
 6595   ins_encode %{
 6596     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6597     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6598   %}
 6599   ins_pipe(pipe_cmov_reg);
 6600 %}
 6601 
 6602 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6603 // inputs of the CMove
 6604 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6605   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6606   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6607 
 6608   ins_cost(200); // XXX
 6609   format %{ "cmovpq  $dst, $src\n\t"
 6610             "cmovneq $dst, $src" %}
 6611   ins_encode %{
 6612     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6613     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6614   %}
 6615   ins_pipe(pipe_cmov_reg);
 6616 %}
 6617 
 6618 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6619 %{
 6620   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6621 
 6622   ins_cost(200); // XXX
 6623   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6624   ins_encode %{
 6625     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6626   %}
 6627   ins_pipe(pipe_cmov_mem); // XXX
 6628 %}
 6629 
 6630 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6631   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6632   ins_cost(200);
 6633   expand %{
 6634     cmovL_memU(cop, cr, dst, src);
 6635   %}
 6636 %}
 6637 
 6638 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6639 %{
 6640   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6641 
 6642   ins_cost(200); // XXX
 6643   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6644             "movss     $dst, $src\n"
 6645     "skip:" %}
 6646   ins_encode %{
 6647     Label Lskip;
 6648     // Invert sense of branch from sense of CMOV
 6649     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6650     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6651     __ bind(Lskip);
 6652   %}
 6653   ins_pipe(pipe_slow);
 6654 %}
 6655 
 6656 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6657 %{
 6658   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6659 
 6660   ins_cost(200); // XXX
 6661   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6662             "movss     $dst, $src\n"
 6663     "skip:" %}
 6664   ins_encode %{
 6665     Label Lskip;
 6666     // Invert sense of branch from sense of CMOV
 6667     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6668     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6669     __ bind(Lskip);
 6670   %}
 6671   ins_pipe(pipe_slow);
 6672 %}
 6673 
 6674 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6675   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6676   ins_cost(200);
 6677   expand %{
 6678     cmovF_regU(cop, cr, dst, src);
 6679   %}
 6680 %}
 6681 
 6682 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6683 %{
 6684   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6685 
 6686   ins_cost(200); // XXX
 6687   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6688             "movsd     $dst, $src\n"
 6689     "skip:" %}
 6690   ins_encode %{
 6691     Label Lskip;
 6692     // Invert sense of branch from sense of CMOV
 6693     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6694     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6695     __ bind(Lskip);
 6696   %}
 6697   ins_pipe(pipe_slow);
 6698 %}
 6699 
 6700 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6701 %{
 6702   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6703 
 6704   ins_cost(200); // XXX
 6705   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6706             "movsd     $dst, $src\n"
 6707     "skip:" %}
 6708   ins_encode %{
 6709     Label Lskip;
 6710     // Invert sense of branch from sense of CMOV
 6711     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6712     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6713     __ bind(Lskip);
 6714   %}
 6715   ins_pipe(pipe_slow);
 6716 %}
 6717 
 6718 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6719   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6720   ins_cost(200);
 6721   expand %{
 6722     cmovD_regU(cop, cr, dst, src);
 6723   %}
 6724 %}
 6725 
 6726 //----------Arithmetic Instructions--------------------------------------------
 6727 //----------Addition Instructions----------------------------------------------
 6728 
 6729 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6730 %{
 6731   match(Set dst (AddI dst src));
 6732   effect(KILL cr);
 6733   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6734   format %{ "addl    $dst, $src\t# int" %}
 6735   ins_encode %{
 6736     __ addl($dst$$Register, $src$$Register);
 6737   %}
 6738   ins_pipe(ialu_reg_reg);
 6739 %}
 6740 
 6741 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 6742 %{
 6743   match(Set dst (AddI dst src));
 6744   effect(KILL cr);
 6745   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6746 
 6747   format %{ "addl    $dst, $src\t# int" %}
 6748   ins_encode %{
 6749     __ addl($dst$$Register, $src$$constant);
 6750   %}
 6751   ins_pipe( ialu_reg );
 6752 %}
 6753 
 6754 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 6755 %{
 6756   match(Set dst (AddI dst (LoadI src)));
 6757   effect(KILL cr);
 6758   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6759 
 6760   ins_cost(150); // XXX
 6761   format %{ "addl    $dst, $src\t# int" %}
 6762   ins_encode %{
 6763     __ addl($dst$$Register, $src$$Address);
 6764   %}
 6765   ins_pipe(ialu_reg_mem);
 6766 %}
 6767 
 6768 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 6769 %{
 6770   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6771   effect(KILL cr);
 6772   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6773 
 6774   ins_cost(150); // XXX
 6775   format %{ "addl    $dst, $src\t# int" %}
 6776   ins_encode %{
 6777     __ addl($dst$$Address, $src$$Register);
 6778   %}
 6779   ins_pipe(ialu_mem_reg);
 6780 %}
 6781 
 6782 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 6783 %{
 6784   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6785   effect(KILL cr);
 6786   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6787 
 6788 
 6789   ins_cost(125); // XXX
 6790   format %{ "addl    $dst, $src\t# int" %}
 6791   ins_encode %{
 6792     __ addl($dst$$Address, $src$$constant);
 6793   %}
 6794   ins_pipe(ialu_mem_imm);
 6795 %}
 6796 
 6797 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 6798 %{
 6799   predicate(UseIncDec);
 6800   match(Set dst (AddI dst src));
 6801   effect(KILL cr);
 6802 
 6803   format %{ "incl    $dst\t# int" %}
 6804   ins_encode %{
 6805     __ incrementl($dst$$Register);
 6806   %}
 6807   ins_pipe(ialu_reg);
 6808 %}
 6809 
 6810 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 6811 %{
 6812   predicate(UseIncDec);
 6813   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6814   effect(KILL cr);
 6815 
 6816   ins_cost(125); // XXX
 6817   format %{ "incl    $dst\t# int" %}
 6818   ins_encode %{
 6819     __ incrementl($dst$$Address);
 6820   %}
 6821   ins_pipe(ialu_mem_imm);
 6822 %}
 6823 
 6824 // XXX why does that use AddI
 6825 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 6826 %{
 6827   predicate(UseIncDec);
 6828   match(Set dst (AddI dst src));
 6829   effect(KILL cr);
 6830 
 6831   format %{ "decl    $dst\t# int" %}
 6832   ins_encode %{
 6833     __ decrementl($dst$$Register);
 6834   %}
 6835   ins_pipe(ialu_reg);
 6836 %}
 6837 
 6838 // XXX why does that use AddI
 6839 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 6840 %{
 6841   predicate(UseIncDec);
 6842   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6843   effect(KILL cr);
 6844 
 6845   ins_cost(125); // XXX
 6846   format %{ "decl    $dst\t# int" %}
 6847   ins_encode %{
 6848     __ decrementl($dst$$Address);
 6849   %}
 6850   ins_pipe(ialu_mem_imm);
 6851 %}
 6852 
 6853 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 6854 %{
 6855   predicate(VM_Version::supports_fast_2op_lea());
 6856   match(Set dst (AddI (LShiftI index scale) disp));
 6857 
 6858   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 6859   ins_encode %{
 6860     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6861     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6862   %}
 6863   ins_pipe(ialu_reg_reg);
 6864 %}
 6865 
 6866 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 6867 %{
 6868   predicate(VM_Version::supports_fast_3op_lea());
 6869   match(Set dst (AddI (AddI base index) disp));
 6870 
 6871   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 6872   ins_encode %{
 6873     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6874   %}
 6875   ins_pipe(ialu_reg_reg);
 6876 %}
 6877 
 6878 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 6879 %{
 6880   predicate(VM_Version::supports_fast_2op_lea());
 6881   match(Set dst (AddI base (LShiftI index scale)));
 6882 
 6883   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 6884   ins_encode %{
 6885     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6886     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6887   %}
 6888   ins_pipe(ialu_reg_reg);
 6889 %}
 6890 
 6891 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 6892 %{
 6893   predicate(VM_Version::supports_fast_3op_lea());
 6894   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 6895 
 6896   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 6897   ins_encode %{
 6898     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6899     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6900   %}
 6901   ins_pipe(ialu_reg_reg);
 6902 %}
 6903 
 6904 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 6905 %{
 6906   match(Set dst (AddL dst src));
 6907   effect(KILL cr);
 6908   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6909 
 6910   format %{ "addq    $dst, $src\t# long" %}
 6911   ins_encode %{
 6912     __ addq($dst$$Register, $src$$Register);
 6913   %}
 6914   ins_pipe(ialu_reg_reg);
 6915 %}
 6916 
 6917 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 6918 %{
 6919   match(Set dst (AddL dst src));
 6920   effect(KILL cr);
 6921   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6922 
 6923   format %{ "addq    $dst, $src\t# long" %}
 6924   ins_encode %{
 6925     __ addq($dst$$Register, $src$$constant);
 6926   %}
 6927   ins_pipe( ialu_reg );
 6928 %}
 6929 
 6930 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 6931 %{
 6932   match(Set dst (AddL dst (LoadL src)));
 6933   effect(KILL cr);
 6934   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6935 
 6936   ins_cost(150); // XXX
 6937   format %{ "addq    $dst, $src\t# long" %}
 6938   ins_encode %{
 6939     __ addq($dst$$Register, $src$$Address);
 6940   %}
 6941   ins_pipe(ialu_reg_mem);
 6942 %}
 6943 
 6944 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 6945 %{
 6946   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6947   effect(KILL cr);
 6948   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6949 
 6950   ins_cost(150); // XXX
 6951   format %{ "addq    $dst, $src\t# long" %}
 6952   ins_encode %{
 6953     __ addq($dst$$Address, $src$$Register);
 6954   %}
 6955   ins_pipe(ialu_mem_reg);
 6956 %}
 6957 
 6958 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 6959 %{
 6960   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6961   effect(KILL cr);
 6962   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6963 
 6964   ins_cost(125); // XXX
 6965   format %{ "addq    $dst, $src\t# long" %}
 6966   ins_encode %{
 6967     __ addq($dst$$Address, $src$$constant);
 6968   %}
 6969   ins_pipe(ialu_mem_imm);
 6970 %}
 6971 
 6972 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 6973 %{
 6974   predicate(UseIncDec);
 6975   match(Set dst (AddL dst src));
 6976   effect(KILL cr);
 6977 
 6978   format %{ "incq    $dst\t# long" %}
 6979   ins_encode %{
 6980     __ incrementq($dst$$Register);
 6981   %}
 6982   ins_pipe(ialu_reg);
 6983 %}
 6984 
 6985 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 6986 %{
 6987   predicate(UseIncDec);
 6988   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6989   effect(KILL cr);
 6990 
 6991   ins_cost(125); // XXX
 6992   format %{ "incq    $dst\t# long" %}
 6993   ins_encode %{
 6994     __ incrementq($dst$$Address);
 6995   %}
 6996   ins_pipe(ialu_mem_imm);
 6997 %}
 6998 
 6999 // XXX why does that use AddL
 7000 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7001 %{
 7002   predicate(UseIncDec);
 7003   match(Set dst (AddL dst src));
 7004   effect(KILL cr);
 7005 
 7006   format %{ "decq    $dst\t# long" %}
 7007   ins_encode %{
 7008     __ decrementq($dst$$Register);
 7009   %}
 7010   ins_pipe(ialu_reg);
 7011 %}
 7012 
 7013 // XXX why does that use AddL
 7014 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7015 %{
 7016   predicate(UseIncDec);
 7017   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7018   effect(KILL cr);
 7019 
 7020   ins_cost(125); // XXX
 7021   format %{ "decq    $dst\t# long" %}
 7022   ins_encode %{
 7023     __ decrementq($dst$$Address);
 7024   %}
 7025   ins_pipe(ialu_mem_imm);
 7026 %}
 7027 
 7028 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7029 %{
 7030   predicate(VM_Version::supports_fast_2op_lea());
 7031   match(Set dst (AddL (LShiftL index scale) disp));
 7032 
 7033   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7034   ins_encode %{
 7035     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7036     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7037   %}
 7038   ins_pipe(ialu_reg_reg);
 7039 %}
 7040 
 7041 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7042 %{
 7043   predicate(VM_Version::supports_fast_3op_lea());
 7044   match(Set dst (AddL (AddL base index) disp));
 7045 
 7046   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7047   ins_encode %{
 7048     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7049   %}
 7050   ins_pipe(ialu_reg_reg);
 7051 %}
 7052 
 7053 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7054 %{
 7055   predicate(VM_Version::supports_fast_2op_lea());
 7056   match(Set dst (AddL base (LShiftL index scale)));
 7057 
 7058   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7059   ins_encode %{
 7060     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7061     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7062   %}
 7063   ins_pipe(ialu_reg_reg);
 7064 %}
 7065 
 7066 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7067 %{
 7068   predicate(VM_Version::supports_fast_3op_lea());
 7069   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7070 
 7071   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7072   ins_encode %{
 7073     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7074     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7075   %}
 7076   ins_pipe(ialu_reg_reg);
 7077 %}
 7078 
 7079 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7080 %{
 7081   match(Set dst (AddP dst src));
 7082   effect(KILL cr);
 7083   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7084 
 7085   format %{ "addq    $dst, $src\t# ptr" %}
 7086   ins_encode %{
 7087     __ addq($dst$$Register, $src$$Register);
 7088   %}
 7089   ins_pipe(ialu_reg_reg);
 7090 %}
 7091 
 7092 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7093 %{
 7094   match(Set dst (AddP dst src));
 7095   effect(KILL cr);
 7096   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7097 
 7098   format %{ "addq    $dst, $src\t# ptr" %}
 7099   ins_encode %{
 7100     __ addq($dst$$Register, $src$$constant);
 7101   %}
 7102   ins_pipe( ialu_reg );
 7103 %}
 7104 
 7105 // XXX addP mem ops ????
 7106 
 7107 instruct checkCastPP(rRegP dst)
 7108 %{
 7109   match(Set dst (CheckCastPP dst));
 7110 
 7111   size(0);
 7112   format %{ "# checkcastPP of $dst" %}
 7113   ins_encode(/* empty encoding */);
 7114   ins_pipe(empty);
 7115 %}
 7116 
 7117 instruct castPP(rRegP dst)
 7118 %{
 7119   match(Set dst (CastPP dst));
 7120 
 7121   size(0);
 7122   format %{ "# castPP of $dst" %}
 7123   ins_encode(/* empty encoding */);
 7124   ins_pipe(empty);
 7125 %}
 7126 
 7127 instruct castII(rRegI dst)
 7128 %{
 7129   match(Set dst (CastII dst));
 7130 
 7131   size(0);
 7132   format %{ "# castII of $dst" %}
 7133   ins_encode(/* empty encoding */);
 7134   ins_cost(0);
 7135   ins_pipe(empty);
 7136 %}
 7137 
 7138 instruct castLL(rRegL dst)
 7139 %{
 7140   match(Set dst (CastLL dst));
 7141 
 7142   size(0);
 7143   format %{ "# castLL of $dst" %}
 7144   ins_encode(/* empty encoding */);
 7145   ins_cost(0);
 7146   ins_pipe(empty);
 7147 %}
 7148 
 7149 instruct castFF(regF dst)
 7150 %{
 7151   match(Set dst (CastFF dst));
 7152 
 7153   size(0);
 7154   format %{ "# castFF of $dst" %}
 7155   ins_encode(/* empty encoding */);
 7156   ins_cost(0);
 7157   ins_pipe(empty);
 7158 %}
 7159 
 7160 instruct castDD(regD dst)
 7161 %{
 7162   match(Set dst (CastDD dst));
 7163 
 7164   size(0);
 7165   format %{ "# castDD of $dst" %}
 7166   ins_encode(/* empty encoding */);
 7167   ins_cost(0);
 7168   ins_pipe(empty);
 7169 %}
 7170 
 7171 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7172 instruct compareAndSwapP(rRegI res,
 7173                          memory mem_ptr,
 7174                          rax_RegP oldval, rRegP newval,
 7175                          rFlagsReg cr)
 7176 %{
 7177   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7178   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7179   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7180   effect(KILL cr, KILL oldval);
 7181 
 7182   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7183             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7184             "sete    $res\n\t"
 7185             "movzbl  $res, $res" %}
 7186   ins_encode %{
 7187     __ lock();
 7188     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7189     __ setb(Assembler::equal, $res$$Register);
 7190     __ movzbl($res$$Register, $res$$Register);
 7191   %}
 7192   ins_pipe( pipe_cmpxchg );
 7193 %}
 7194 
 7195 instruct compareAndSwapL(rRegI res,
 7196                          memory mem_ptr,
 7197                          rax_RegL oldval, rRegL newval,
 7198                          rFlagsReg cr)
 7199 %{
 7200   predicate(VM_Version::supports_cx8());
 7201   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7202   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7203   effect(KILL cr, KILL oldval);
 7204 
 7205   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7206             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7207             "sete    $res\n\t"
 7208             "movzbl  $res, $res" %}
 7209   ins_encode %{
 7210     __ lock();
 7211     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7212     __ setb(Assembler::equal, $res$$Register);
 7213     __ movzbl($res$$Register, $res$$Register);
 7214   %}
 7215   ins_pipe( pipe_cmpxchg );
 7216 %}
 7217 
 7218 instruct compareAndSwapI(rRegI res,
 7219                          memory mem_ptr,
 7220                          rax_RegI oldval, rRegI newval,
 7221                          rFlagsReg cr)
 7222 %{
 7223   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7224   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7225   effect(KILL cr, KILL oldval);
 7226 
 7227   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7228             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7229             "sete    $res\n\t"
 7230             "movzbl  $res, $res" %}
 7231   ins_encode %{
 7232     __ lock();
 7233     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7234     __ setb(Assembler::equal, $res$$Register);
 7235     __ movzbl($res$$Register, $res$$Register);
 7236   %}
 7237   ins_pipe( pipe_cmpxchg );
 7238 %}
 7239 
 7240 instruct compareAndSwapB(rRegI res,
 7241                          memory mem_ptr,
 7242                          rax_RegI oldval, rRegI newval,
 7243                          rFlagsReg cr)
 7244 %{
 7245   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7246   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7247   effect(KILL cr, KILL oldval);
 7248 
 7249   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7250             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7251             "sete    $res\n\t"
 7252             "movzbl  $res, $res" %}
 7253   ins_encode %{
 7254     __ lock();
 7255     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7256     __ setb(Assembler::equal, $res$$Register);
 7257     __ movzbl($res$$Register, $res$$Register);
 7258   %}
 7259   ins_pipe( pipe_cmpxchg );
 7260 %}
 7261 
 7262 instruct compareAndSwapS(rRegI res,
 7263                          memory mem_ptr,
 7264                          rax_RegI oldval, rRegI newval,
 7265                          rFlagsReg cr)
 7266 %{
 7267   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7268   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7269   effect(KILL cr, KILL oldval);
 7270 
 7271   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7272             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7273             "sete    $res\n\t"
 7274             "movzbl  $res, $res" %}
 7275   ins_encode %{
 7276     __ lock();
 7277     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7278     __ setb(Assembler::equal, $res$$Register);
 7279     __ movzbl($res$$Register, $res$$Register);
 7280   %}
 7281   ins_pipe( pipe_cmpxchg );
 7282 %}
 7283 
 7284 instruct compareAndSwapN(rRegI res,
 7285                           memory mem_ptr,
 7286                           rax_RegN oldval, rRegN newval,
 7287                           rFlagsReg cr) %{
 7288   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7289   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7290   effect(KILL cr, KILL oldval);
 7291 
 7292   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7293             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7294             "sete    $res\n\t"
 7295             "movzbl  $res, $res" %}
 7296   ins_encode %{
 7297     __ lock();
 7298     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7299     __ setb(Assembler::equal, $res$$Register);
 7300     __ movzbl($res$$Register, $res$$Register);
 7301   %}
 7302   ins_pipe( pipe_cmpxchg );
 7303 %}
 7304 
 7305 instruct compareAndExchangeB(
 7306                          memory mem_ptr,
 7307                          rax_RegI oldval, rRegI newval,
 7308                          rFlagsReg cr)
 7309 %{
 7310   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7311   effect(KILL cr);
 7312 
 7313   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7314             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7315   ins_encode %{
 7316     __ lock();
 7317     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7318   %}
 7319   ins_pipe( pipe_cmpxchg );
 7320 %}
 7321 
 7322 instruct compareAndExchangeS(
 7323                          memory mem_ptr,
 7324                          rax_RegI oldval, rRegI newval,
 7325                          rFlagsReg cr)
 7326 %{
 7327   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7328   effect(KILL cr);
 7329 
 7330   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7331             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7332   ins_encode %{
 7333     __ lock();
 7334     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7335   %}
 7336   ins_pipe( pipe_cmpxchg );
 7337 %}
 7338 
 7339 instruct compareAndExchangeI(
 7340                          memory mem_ptr,
 7341                          rax_RegI oldval, rRegI newval,
 7342                          rFlagsReg cr)
 7343 %{
 7344   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7345   effect(KILL cr);
 7346 
 7347   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7348             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7349   ins_encode %{
 7350     __ lock();
 7351     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7352   %}
 7353   ins_pipe( pipe_cmpxchg );
 7354 %}
 7355 
 7356 instruct compareAndExchangeL(
 7357                          memory mem_ptr,
 7358                          rax_RegL oldval, rRegL newval,
 7359                          rFlagsReg cr)
 7360 %{
 7361   predicate(VM_Version::supports_cx8());
 7362   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7363   effect(KILL cr);
 7364 
 7365   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7366             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7367   ins_encode %{
 7368     __ lock();
 7369     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7370   %}
 7371   ins_pipe( pipe_cmpxchg );
 7372 %}
 7373 
 7374 instruct compareAndExchangeN(
 7375                           memory mem_ptr,
 7376                           rax_RegN oldval, rRegN newval,
 7377                           rFlagsReg cr) %{
 7378   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7379   effect(KILL cr);
 7380 
 7381   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7382             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7383   ins_encode %{
 7384     __ lock();
 7385     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7386   %}
 7387   ins_pipe( pipe_cmpxchg );
 7388 %}
 7389 
 7390 instruct compareAndExchangeP(
 7391                          memory mem_ptr,
 7392                          rax_RegP oldval, rRegP newval,
 7393                          rFlagsReg cr)
 7394 %{
 7395   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7396   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7397   effect(KILL cr);
 7398 
 7399   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7400             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7401   ins_encode %{
 7402     __ lock();
 7403     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7404   %}
 7405   ins_pipe( pipe_cmpxchg );
 7406 %}
 7407 
 7408 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7409   predicate(n->as_LoadStore()->result_not_used());
 7410   match(Set dummy (GetAndAddB mem add));
 7411   effect(KILL cr);
 7412   format %{ "addb_lock   $mem, $add" %}
 7413   ins_encode %{
 7414     __ lock();
 7415     __ addb($mem$$Address, $add$$Register);
 7416   %}
 7417   ins_pipe(pipe_cmpxchg);
 7418 %}
 7419 
 7420 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7421   predicate(n->as_LoadStore()->result_not_used());
 7422   match(Set dummy (GetAndAddB mem add));
 7423   effect(KILL cr);
 7424   format %{ "addb_lock   $mem, $add" %}
 7425   ins_encode %{
 7426     __ lock();
 7427     __ addb($mem$$Address, $add$$constant);
 7428   %}
 7429   ins_pipe(pipe_cmpxchg);
 7430 %}
 7431 
 7432 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7433   predicate(!n->as_LoadStore()->result_not_used());
 7434   match(Set newval (GetAndAddB mem newval));
 7435   effect(KILL cr);
 7436   format %{ "xaddb_lock  $mem, $newval" %}
 7437   ins_encode %{
 7438     __ lock();
 7439     __ xaddb($mem$$Address, $newval$$Register);
 7440   %}
 7441   ins_pipe(pipe_cmpxchg);
 7442 %}
 7443 
 7444 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7445   predicate(n->as_LoadStore()->result_not_used());
 7446   match(Set dummy (GetAndAddS mem add));
 7447   effect(KILL cr);
 7448   format %{ "addw_lock   $mem, $add" %}
 7449   ins_encode %{
 7450     __ lock();
 7451     __ addw($mem$$Address, $add$$Register);
 7452   %}
 7453   ins_pipe(pipe_cmpxchg);
 7454 %}
 7455 
 7456 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7457   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7458   match(Set dummy (GetAndAddS mem add));
 7459   effect(KILL cr);
 7460   format %{ "addw_lock   $mem, $add" %}
 7461   ins_encode %{
 7462     __ lock();
 7463     __ addw($mem$$Address, $add$$constant);
 7464   %}
 7465   ins_pipe(pipe_cmpxchg);
 7466 %}
 7467 
 7468 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7469   predicate(!n->as_LoadStore()->result_not_used());
 7470   match(Set newval (GetAndAddS mem newval));
 7471   effect(KILL cr);
 7472   format %{ "xaddw_lock  $mem, $newval" %}
 7473   ins_encode %{
 7474     __ lock();
 7475     __ xaddw($mem$$Address, $newval$$Register);
 7476   %}
 7477   ins_pipe(pipe_cmpxchg);
 7478 %}
 7479 
 7480 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7481   predicate(n->as_LoadStore()->result_not_used());
 7482   match(Set dummy (GetAndAddI mem add));
 7483   effect(KILL cr);
 7484   format %{ "addl_lock   $mem, $add" %}
 7485   ins_encode %{
 7486     __ lock();
 7487     __ addl($mem$$Address, $add$$Register);
 7488   %}
 7489   ins_pipe(pipe_cmpxchg);
 7490 %}
 7491 
 7492 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7493   predicate(n->as_LoadStore()->result_not_used());
 7494   match(Set dummy (GetAndAddI mem add));
 7495   effect(KILL cr);
 7496   format %{ "addl_lock   $mem, $add" %}
 7497   ins_encode %{
 7498     __ lock();
 7499     __ addl($mem$$Address, $add$$constant);
 7500   %}
 7501   ins_pipe(pipe_cmpxchg);
 7502 %}
 7503 
 7504 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 7505   predicate(!n->as_LoadStore()->result_not_used());
 7506   match(Set newval (GetAndAddI mem newval));
 7507   effect(KILL cr);
 7508   format %{ "xaddl_lock  $mem, $newval" %}
 7509   ins_encode %{
 7510     __ lock();
 7511     __ xaddl($mem$$Address, $newval$$Register);
 7512   %}
 7513   ins_pipe(pipe_cmpxchg);
 7514 %}
 7515 
 7516 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 7517   predicate(n->as_LoadStore()->result_not_used());
 7518   match(Set dummy (GetAndAddL mem add));
 7519   effect(KILL cr);
 7520   format %{ "addq_lock   $mem, $add" %}
 7521   ins_encode %{
 7522     __ lock();
 7523     __ addq($mem$$Address, $add$$Register);
 7524   %}
 7525   ins_pipe(pipe_cmpxchg);
 7526 %}
 7527 
 7528 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 7529   predicate(n->as_LoadStore()->result_not_used());
 7530   match(Set dummy (GetAndAddL mem add));
 7531   effect(KILL cr);
 7532   format %{ "addq_lock   $mem, $add" %}
 7533   ins_encode %{
 7534     __ lock();
 7535     __ addq($mem$$Address, $add$$constant);
 7536   %}
 7537   ins_pipe(pipe_cmpxchg);
 7538 %}
 7539 
 7540 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 7541   predicate(!n->as_LoadStore()->result_not_used());
 7542   match(Set newval (GetAndAddL mem newval));
 7543   effect(KILL cr);
 7544   format %{ "xaddq_lock  $mem, $newval" %}
 7545   ins_encode %{
 7546     __ lock();
 7547     __ xaddq($mem$$Address, $newval$$Register);
 7548   %}
 7549   ins_pipe(pipe_cmpxchg);
 7550 %}
 7551 
 7552 instruct xchgB( memory mem, rRegI newval) %{
 7553   match(Set newval (GetAndSetB mem newval));
 7554   format %{ "XCHGB  $newval,[$mem]" %}
 7555   ins_encode %{
 7556     __ xchgb($newval$$Register, $mem$$Address);
 7557   %}
 7558   ins_pipe( pipe_cmpxchg );
 7559 %}
 7560 
 7561 instruct xchgS( memory mem, rRegI newval) %{
 7562   match(Set newval (GetAndSetS mem newval));
 7563   format %{ "XCHGW  $newval,[$mem]" %}
 7564   ins_encode %{
 7565     __ xchgw($newval$$Register, $mem$$Address);
 7566   %}
 7567   ins_pipe( pipe_cmpxchg );
 7568 %}
 7569 
 7570 instruct xchgI( memory mem, rRegI newval) %{
 7571   match(Set newval (GetAndSetI mem newval));
 7572   format %{ "XCHGL  $newval,[$mem]" %}
 7573   ins_encode %{
 7574     __ xchgl($newval$$Register, $mem$$Address);
 7575   %}
 7576   ins_pipe( pipe_cmpxchg );
 7577 %}
 7578 
 7579 instruct xchgL( memory mem, rRegL newval) %{
 7580   match(Set newval (GetAndSetL mem newval));
 7581   format %{ "XCHGL  $newval,[$mem]" %}
 7582   ins_encode %{
 7583     __ xchgq($newval$$Register, $mem$$Address);
 7584   %}
 7585   ins_pipe( pipe_cmpxchg );
 7586 %}
 7587 
 7588 instruct xchgP( memory mem, rRegP newval) %{
 7589   match(Set newval (GetAndSetP mem newval));
 7590   predicate(n->as_LoadStore()->barrier_data() == 0);
 7591   format %{ "XCHGQ  $newval,[$mem]" %}
 7592   ins_encode %{
 7593     __ xchgq($newval$$Register, $mem$$Address);
 7594   %}
 7595   ins_pipe( pipe_cmpxchg );
 7596 %}
 7597 
 7598 instruct xchgN( memory mem, rRegN newval) %{
 7599   match(Set newval (GetAndSetN mem newval));
 7600   format %{ "XCHGL  $newval,$mem]" %}
 7601   ins_encode %{
 7602     __ xchgl($newval$$Register, $mem$$Address);
 7603   %}
 7604   ins_pipe( pipe_cmpxchg );
 7605 %}
 7606 
 7607 //----------Abs Instructions-------------------------------------------
 7608 
 7609 // Integer Absolute Instructions
 7610 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7611 %{
 7612   match(Set dst (AbsI src));
 7613   effect(TEMP dst, KILL cr);
 7614   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 7615             "subl    $dst, $src\n\t"
 7616             "cmovll  $dst, $src" %}
 7617   ins_encode %{
 7618     __ xorl($dst$$Register, $dst$$Register);
 7619     __ subl($dst$$Register, $src$$Register);
 7620     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 7621   %}
 7622 
 7623   ins_pipe(ialu_reg_reg);
 7624 %}
 7625 
 7626 // Long Absolute Instructions
 7627 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7628 %{
 7629   match(Set dst (AbsL src));
 7630   effect(TEMP dst, KILL cr);
 7631   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 7632             "subq    $dst, $src\n\t"
 7633             "cmovlq  $dst, $src" %}
 7634   ins_encode %{
 7635     __ xorl($dst$$Register, $dst$$Register);
 7636     __ subq($dst$$Register, $src$$Register);
 7637     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 7638   %}
 7639 
 7640   ins_pipe(ialu_reg_reg);
 7641 %}
 7642 
 7643 //----------Subtraction Instructions-------------------------------------------
 7644 
 7645 // Integer Subtraction Instructions
 7646 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7647 %{
 7648   match(Set dst (SubI dst src));
 7649   effect(KILL cr);
 7650   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7651 
 7652   format %{ "subl    $dst, $src\t# int" %}
 7653   ins_encode %{
 7654     __ subl($dst$$Register, $src$$Register);
 7655   %}
 7656   ins_pipe(ialu_reg_reg);
 7657 %}
 7658 
 7659 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7660 %{
 7661   match(Set dst (SubI dst (LoadI src)));
 7662   effect(KILL cr);
 7663   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7664 
 7665   ins_cost(150);
 7666   format %{ "subl    $dst, $src\t# int" %}
 7667   ins_encode %{
 7668     __ subl($dst$$Register, $src$$Address);
 7669   %}
 7670   ins_pipe(ialu_reg_mem);
 7671 %}
 7672 
 7673 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7674 %{
 7675   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 7676   effect(KILL cr);
 7677   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7678 
 7679   ins_cost(150);
 7680   format %{ "subl    $dst, $src\t# int" %}
 7681   ins_encode %{
 7682     __ subl($dst$$Address, $src$$Register);
 7683   %}
 7684   ins_pipe(ialu_mem_reg);
 7685 %}
 7686 
 7687 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7688 %{
 7689   match(Set dst (SubL dst src));
 7690   effect(KILL cr);
 7691   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7692 
 7693   format %{ "subq    $dst, $src\t# long" %}
 7694   ins_encode %{
 7695     __ subq($dst$$Register, $src$$Register);
 7696   %}
 7697   ins_pipe(ialu_reg_reg);
 7698 %}
 7699 
 7700 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7701 %{
 7702   match(Set dst (SubL dst (LoadL src)));
 7703   effect(KILL cr);
 7704   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7705 
 7706   ins_cost(150);
 7707   format %{ "subq    $dst, $src\t# long" %}
 7708   ins_encode %{
 7709     __ subq($dst$$Register, $src$$Address);
 7710   %}
 7711   ins_pipe(ialu_reg_mem);
 7712 %}
 7713 
 7714 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7715 %{
 7716   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 7717   effect(KILL cr);
 7718   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7719 
 7720   ins_cost(150);
 7721   format %{ "subq    $dst, $src\t# long" %}
 7722   ins_encode %{
 7723     __ subq($dst$$Address, $src$$Register);
 7724   %}
 7725   ins_pipe(ialu_mem_reg);
 7726 %}
 7727 
 7728 // Subtract from a pointer
 7729 // XXX hmpf???
 7730 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 7731 %{
 7732   match(Set dst (AddP dst (SubI zero src)));
 7733   effect(KILL cr);
 7734 
 7735   format %{ "subq    $dst, $src\t# ptr - int" %}
 7736   ins_encode %{
 7737     __ subq($dst$$Register, $src$$Register);
 7738   %}
 7739   ins_pipe(ialu_reg_reg);
 7740 %}
 7741 
 7742 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 7743 %{
 7744   match(Set dst (SubI zero dst));
 7745   effect(KILL cr);
 7746   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7747 
 7748   format %{ "negl    $dst\t# int" %}
 7749   ins_encode %{
 7750     __ negl($dst$$Register);
 7751   %}
 7752   ins_pipe(ialu_reg);
 7753 %}
 7754 
 7755 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 7756 %{
 7757   match(Set dst (NegI dst));
 7758   effect(KILL cr);
 7759   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7760 
 7761   format %{ "negl    $dst\t# int" %}
 7762   ins_encode %{
 7763     __ negl($dst$$Register);
 7764   %}
 7765   ins_pipe(ialu_reg);
 7766 %}
 7767 
 7768 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 7769 %{
 7770   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 7771   effect(KILL cr);
 7772   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7773 
 7774   format %{ "negl    $dst\t# int" %}
 7775   ins_encode %{
 7776     __ negl($dst$$Address);
 7777   %}
 7778   ins_pipe(ialu_reg);
 7779 %}
 7780 
 7781 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 7782 %{
 7783   match(Set dst (SubL zero dst));
 7784   effect(KILL cr);
 7785   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7786 
 7787   format %{ "negq    $dst\t# long" %}
 7788   ins_encode %{
 7789     __ negq($dst$$Register);
 7790   %}
 7791   ins_pipe(ialu_reg);
 7792 %}
 7793 
 7794 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 7795 %{
 7796   match(Set dst (NegL dst));
 7797   effect(KILL cr);
 7798   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7799 
 7800   format %{ "negq    $dst\t# int" %}
 7801   ins_encode %{
 7802     __ negq($dst$$Register);
 7803   %}
 7804   ins_pipe(ialu_reg);
 7805 %}
 7806 
 7807 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 7808 %{
 7809   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 7810   effect(KILL cr);
 7811   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7812 
 7813   format %{ "negq    $dst\t# long" %}
 7814   ins_encode %{
 7815     __ negq($dst$$Address);
 7816   %}
 7817   ins_pipe(ialu_reg);
 7818 %}
 7819 
 7820 //----------Multiplication/Division Instructions-------------------------------
 7821 // Integer Multiplication Instructions
 7822 // Multiply Register
 7823 
 7824 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7825 %{
 7826   match(Set dst (MulI dst src));
 7827   effect(KILL cr);
 7828 
 7829   ins_cost(300);
 7830   format %{ "imull   $dst, $src\t# int" %}
 7831   ins_encode %{
 7832     __ imull($dst$$Register, $src$$Register);
 7833   %}
 7834   ins_pipe(ialu_reg_reg_alu0);
 7835 %}
 7836 
 7837 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 7838 %{
 7839   match(Set dst (MulI src imm));
 7840   effect(KILL cr);
 7841 
 7842   ins_cost(300);
 7843   format %{ "imull   $dst, $src, $imm\t# int" %}
 7844   ins_encode %{
 7845     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 7846   %}
 7847   ins_pipe(ialu_reg_reg_alu0);
 7848 %}
 7849 
 7850 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 7851 %{
 7852   match(Set dst (MulI dst (LoadI src)));
 7853   effect(KILL cr);
 7854 
 7855   ins_cost(350);
 7856   format %{ "imull   $dst, $src\t# int" %}
 7857   ins_encode %{
 7858     __ imull($dst$$Register, $src$$Address);
 7859   %}
 7860   ins_pipe(ialu_reg_mem_alu0);
 7861 %}
 7862 
 7863 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 7864 %{
 7865   match(Set dst (MulI (LoadI src) imm));
 7866   effect(KILL cr);
 7867 
 7868   ins_cost(300);
 7869   format %{ "imull   $dst, $src, $imm\t# int" %}
 7870   ins_encode %{
 7871     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 7872   %}
 7873   ins_pipe(ialu_reg_mem_alu0);
 7874 %}
 7875 
 7876 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 7877 %{
 7878   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 7879   effect(KILL cr, KILL src2);
 7880 
 7881   expand %{ mulI_rReg(dst, src1, cr);
 7882            mulI_rReg(src2, src3, cr);
 7883            addI_rReg(dst, src2, cr); %}
 7884 %}
 7885 
 7886 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7887 %{
 7888   match(Set dst (MulL dst src));
 7889   effect(KILL cr);
 7890 
 7891   ins_cost(300);
 7892   format %{ "imulq   $dst, $src\t# long" %}
 7893   ins_encode %{
 7894     __ imulq($dst$$Register, $src$$Register);
 7895   %}
 7896   ins_pipe(ialu_reg_reg_alu0);
 7897 %}
 7898 
 7899 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 7900 %{
 7901   match(Set dst (MulL src imm));
 7902   effect(KILL cr);
 7903 
 7904   ins_cost(300);
 7905   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7906   ins_encode %{
 7907     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 7908   %}
 7909   ins_pipe(ialu_reg_reg_alu0);
 7910 %}
 7911 
 7912 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 7913 %{
 7914   match(Set dst (MulL dst (LoadL src)));
 7915   effect(KILL cr);
 7916 
 7917   ins_cost(350);
 7918   format %{ "imulq   $dst, $src\t# long" %}
 7919   ins_encode %{
 7920     __ imulq($dst$$Register, $src$$Address);
 7921   %}
 7922   ins_pipe(ialu_reg_mem_alu0);
 7923 %}
 7924 
 7925 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 7926 %{
 7927   match(Set dst (MulL (LoadL src) imm));
 7928   effect(KILL cr);
 7929 
 7930   ins_cost(300);
 7931   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7932   ins_encode %{
 7933     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 7934   %}
 7935   ins_pipe(ialu_reg_mem_alu0);
 7936 %}
 7937 
 7938 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7939 %{
 7940   match(Set dst (MulHiL src rax));
 7941   effect(USE_KILL rax, KILL cr);
 7942 
 7943   ins_cost(300);
 7944   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 7945   ins_encode %{
 7946     __ imulq($src$$Register);
 7947   %}
 7948   ins_pipe(ialu_reg_reg_alu0);
 7949 %}
 7950 
 7951 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7952 %{
 7953   match(Set dst (UMulHiL src rax));
 7954   effect(USE_KILL rax, KILL cr);
 7955 
 7956   ins_cost(300);
 7957   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 7958   ins_encode %{
 7959     __ mulq($src$$Register);
 7960   %}
 7961   ins_pipe(ialu_reg_reg_alu0);
 7962 %}
 7963 
 7964 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7965                    rFlagsReg cr)
 7966 %{
 7967   match(Set rax (DivI rax div));
 7968   effect(KILL rdx, KILL cr);
 7969 
 7970   ins_cost(30*100+10*100); // XXX
 7971   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7972             "jne,s   normal\n\t"
 7973             "xorl    rdx, rdx\n\t"
 7974             "cmpl    $div, -1\n\t"
 7975             "je,s    done\n"
 7976     "normal: cdql\n\t"
 7977             "idivl   $div\n"
 7978     "done:"        %}
 7979   ins_encode(cdql_enc(div));
 7980   ins_pipe(ialu_reg_reg_alu0);
 7981 %}
 7982 
 7983 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7984                    rFlagsReg cr)
 7985 %{
 7986   match(Set rax (DivL rax div));
 7987   effect(KILL rdx, KILL cr);
 7988 
 7989   ins_cost(30*100+10*100); // XXX
 7990   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7991             "cmpq    rax, rdx\n\t"
 7992             "jne,s   normal\n\t"
 7993             "xorl    rdx, rdx\n\t"
 7994             "cmpq    $div, -1\n\t"
 7995             "je,s    done\n"
 7996     "normal: cdqq\n\t"
 7997             "idivq   $div\n"
 7998     "done:"        %}
 7999   ins_encode(cdqq_enc(div));
 8000   ins_pipe(ialu_reg_reg_alu0);
 8001 %}
 8002 
 8003 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8004 %{
 8005   match(Set rax (UDivI rax div));
 8006   effect(KILL rdx, KILL cr);
 8007 
 8008   ins_cost(300);
 8009   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8010   ins_encode %{
 8011     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8012   %}
 8013   ins_pipe(ialu_reg_reg_alu0);
 8014 %}
 8015 
 8016 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8017 %{
 8018   match(Set rax (UDivL rax div));
 8019   effect(KILL rdx, KILL cr);
 8020 
 8021   ins_cost(300);
 8022   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8023   ins_encode %{
 8024      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8025   %}
 8026   ins_pipe(ialu_reg_reg_alu0);
 8027 %}
 8028 
 8029 // Integer DIVMOD with Register, both quotient and mod results
 8030 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8031                              rFlagsReg cr)
 8032 %{
 8033   match(DivModI rax div);
 8034   effect(KILL cr);
 8035 
 8036   ins_cost(30*100+10*100); // XXX
 8037   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8038             "jne,s   normal\n\t"
 8039             "xorl    rdx, rdx\n\t"
 8040             "cmpl    $div, -1\n\t"
 8041             "je,s    done\n"
 8042     "normal: cdql\n\t"
 8043             "idivl   $div\n"
 8044     "done:"        %}
 8045   ins_encode(cdql_enc(div));
 8046   ins_pipe(pipe_slow);
 8047 %}
 8048 
 8049 // Long DIVMOD with Register, both quotient and mod results
 8050 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8051                              rFlagsReg cr)
 8052 %{
 8053   match(DivModL rax div);
 8054   effect(KILL cr);
 8055 
 8056   ins_cost(30*100+10*100); // XXX
 8057   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8058             "cmpq    rax, rdx\n\t"
 8059             "jne,s   normal\n\t"
 8060             "xorl    rdx, rdx\n\t"
 8061             "cmpq    $div, -1\n\t"
 8062             "je,s    done\n"
 8063     "normal: cdqq\n\t"
 8064             "idivq   $div\n"
 8065     "done:"        %}
 8066   ins_encode(cdqq_enc(div));
 8067   ins_pipe(pipe_slow);
 8068 %}
 8069 
 8070 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8071 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8072                               no_rax_rdx_RegI div, rFlagsReg cr)
 8073 %{
 8074   match(UDivModI rax div);
 8075   effect(TEMP tmp, KILL cr);
 8076 
 8077   ins_cost(300);
 8078   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8079             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8080           %}
 8081   ins_encode %{
 8082     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8083   %}
 8084   ins_pipe(pipe_slow);
 8085 %}
 8086 
 8087 // Unsigned long DIVMOD with Register, both quotient and mod results
 8088 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8089                               no_rax_rdx_RegL div, rFlagsReg cr)
 8090 %{
 8091   match(UDivModL rax div);
 8092   effect(TEMP tmp, KILL cr);
 8093 
 8094   ins_cost(300);
 8095   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8096             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8097           %}
 8098   ins_encode %{
 8099     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8100   %}
 8101   ins_pipe(pipe_slow);
 8102 %}
 8103 
 8104 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8105                    rFlagsReg cr)
 8106 %{
 8107   match(Set rdx (ModI rax div));
 8108   effect(KILL rax, KILL cr);
 8109 
 8110   ins_cost(300); // XXX
 8111   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8112             "jne,s   normal\n\t"
 8113             "xorl    rdx, rdx\n\t"
 8114             "cmpl    $div, -1\n\t"
 8115             "je,s    done\n"
 8116     "normal: cdql\n\t"
 8117             "idivl   $div\n"
 8118     "done:"        %}
 8119   ins_encode(cdql_enc(div));
 8120   ins_pipe(ialu_reg_reg_alu0);
 8121 %}
 8122 
 8123 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8124                    rFlagsReg cr)
 8125 %{
 8126   match(Set rdx (ModL rax div));
 8127   effect(KILL rax, KILL cr);
 8128 
 8129   ins_cost(300); // XXX
 8130   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8131             "cmpq    rax, rdx\n\t"
 8132             "jne,s   normal\n\t"
 8133             "xorl    rdx, rdx\n\t"
 8134             "cmpq    $div, -1\n\t"
 8135             "je,s    done\n"
 8136     "normal: cdqq\n\t"
 8137             "idivq   $div\n"
 8138     "done:"        %}
 8139   ins_encode(cdqq_enc(div));
 8140   ins_pipe(ialu_reg_reg_alu0);
 8141 %}
 8142 
 8143 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8144 %{
 8145   match(Set rdx (UModI rax div));
 8146   effect(KILL rax, KILL cr);
 8147 
 8148   ins_cost(300);
 8149   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8150   ins_encode %{
 8151     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8152   %}
 8153   ins_pipe(ialu_reg_reg_alu0);
 8154 %}
 8155 
 8156 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8157 %{
 8158   match(Set rdx (UModL rax div));
 8159   effect(KILL rax, KILL cr);
 8160 
 8161   ins_cost(300);
 8162   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8163   ins_encode %{
 8164     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8165   %}
 8166   ins_pipe(ialu_reg_reg_alu0);
 8167 %}
 8168 
 8169 // Integer Shift Instructions
 8170 // Shift Left by one, two, three
 8171 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8172 %{
 8173   match(Set dst (LShiftI dst shift));
 8174   effect(KILL cr);
 8175 
 8176   format %{ "sall    $dst, $shift" %}
 8177   ins_encode %{
 8178     __ sall($dst$$Register, $shift$$constant);
 8179   %}
 8180   ins_pipe(ialu_reg);
 8181 %}
 8182 
 8183 // Shift Left by 8-bit immediate
 8184 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8185 %{
 8186   match(Set dst (LShiftI dst shift));
 8187   effect(KILL cr);
 8188 
 8189   format %{ "sall    $dst, $shift" %}
 8190   ins_encode %{
 8191     __ sall($dst$$Register, $shift$$constant);
 8192   %}
 8193   ins_pipe(ialu_reg);
 8194 %}
 8195 
 8196 // Shift Left by 8-bit immediate
 8197 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8198 %{
 8199   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8200   effect(KILL cr);
 8201 
 8202   format %{ "sall    $dst, $shift" %}
 8203   ins_encode %{
 8204     __ sall($dst$$Address, $shift$$constant);
 8205   %}
 8206   ins_pipe(ialu_mem_imm);
 8207 %}
 8208 
 8209 // Shift Left by variable
 8210 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8211 %{
 8212   predicate(!VM_Version::supports_bmi2());
 8213   match(Set dst (LShiftI dst shift));
 8214   effect(KILL cr);
 8215 
 8216   format %{ "sall    $dst, $shift" %}
 8217   ins_encode %{
 8218     __ sall($dst$$Register);
 8219   %}
 8220   ins_pipe(ialu_reg_reg);
 8221 %}
 8222 
 8223 // Shift Left by variable
 8224 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8225 %{
 8226   predicate(!VM_Version::supports_bmi2());
 8227   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8228   effect(KILL cr);
 8229 
 8230   format %{ "sall    $dst, $shift" %}
 8231   ins_encode %{
 8232     __ sall($dst$$Address);
 8233   %}
 8234   ins_pipe(ialu_mem_reg);
 8235 %}
 8236 
 8237 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8238 %{
 8239   predicate(VM_Version::supports_bmi2());
 8240   match(Set dst (LShiftI src shift));
 8241 
 8242   format %{ "shlxl   $dst, $src, $shift" %}
 8243   ins_encode %{
 8244     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 8245   %}
 8246   ins_pipe(ialu_reg_reg);
 8247 %}
 8248 
 8249 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8250 %{
 8251   predicate(VM_Version::supports_bmi2());
 8252   match(Set dst (LShiftI (LoadI src) shift));
 8253   ins_cost(175);
 8254   format %{ "shlxl   $dst, $src, $shift" %}
 8255   ins_encode %{
 8256     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 8257   %}
 8258   ins_pipe(ialu_reg_mem);
 8259 %}
 8260 
 8261 // Arithmetic Shift Right by 8-bit immediate
 8262 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8263 %{
 8264   match(Set dst (RShiftI dst shift));
 8265   effect(KILL cr);
 8266 
 8267   format %{ "sarl    $dst, $shift" %}
 8268   ins_encode %{
 8269     __ sarl($dst$$Register, $shift$$constant);
 8270   %}
 8271   ins_pipe(ialu_mem_imm);
 8272 %}
 8273 
 8274 // Arithmetic Shift Right by 8-bit immediate
 8275 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8276 %{
 8277   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8278   effect(KILL cr);
 8279 
 8280   format %{ "sarl    $dst, $shift" %}
 8281   ins_encode %{
 8282     __ sarl($dst$$Address, $shift$$constant);
 8283   %}
 8284   ins_pipe(ialu_mem_imm);
 8285 %}
 8286 
 8287 // Arithmetic Shift Right by variable
 8288 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8289 %{
 8290   predicate(!VM_Version::supports_bmi2());
 8291   match(Set dst (RShiftI dst shift));
 8292   effect(KILL cr);
 8293 
 8294   format %{ "sarl    $dst, $shift" %}
 8295   ins_encode %{
 8296     __ sarl($dst$$Register);
 8297   %}
 8298   ins_pipe(ialu_reg_reg);
 8299 %}
 8300 
 8301 // Arithmetic Shift Right by variable
 8302 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8303 %{
 8304   predicate(!VM_Version::supports_bmi2());
 8305   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8306   effect(KILL cr);
 8307 
 8308   format %{ "sarl    $dst, $shift" %}
 8309   ins_encode %{
 8310     __ sarl($dst$$Address);
 8311   %}
 8312   ins_pipe(ialu_mem_reg);
 8313 %}
 8314 
 8315 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8316 %{
 8317   predicate(VM_Version::supports_bmi2());
 8318   match(Set dst (RShiftI src shift));
 8319 
 8320   format %{ "sarxl   $dst, $src, $shift" %}
 8321   ins_encode %{
 8322     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 8323   %}
 8324   ins_pipe(ialu_reg_reg);
 8325 %}
 8326 
 8327 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8328 %{
 8329   predicate(VM_Version::supports_bmi2());
 8330   match(Set dst (RShiftI (LoadI src) shift));
 8331   ins_cost(175);
 8332   format %{ "sarxl   $dst, $src, $shift" %}
 8333   ins_encode %{
 8334     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 8335   %}
 8336   ins_pipe(ialu_reg_mem);
 8337 %}
 8338 
 8339 // Logical Shift Right by 8-bit immediate
 8340 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8341 %{
 8342   match(Set dst (URShiftI dst shift));
 8343   effect(KILL cr);
 8344 
 8345   format %{ "shrl    $dst, $shift" %}
 8346   ins_encode %{
 8347     __ shrl($dst$$Register, $shift$$constant);
 8348   %}
 8349   ins_pipe(ialu_reg);
 8350 %}
 8351 
 8352 // Logical Shift Right by 8-bit immediate
 8353 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8354 %{
 8355   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8356   effect(KILL cr);
 8357 
 8358   format %{ "shrl    $dst, $shift" %}
 8359   ins_encode %{
 8360     __ shrl($dst$$Address, $shift$$constant);
 8361   %}
 8362   ins_pipe(ialu_mem_imm);
 8363 %}
 8364 
 8365 // Logical Shift Right by variable
 8366 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8367 %{
 8368   predicate(!VM_Version::supports_bmi2());
 8369   match(Set dst (URShiftI dst shift));
 8370   effect(KILL cr);
 8371 
 8372   format %{ "shrl    $dst, $shift" %}
 8373   ins_encode %{
 8374     __ shrl($dst$$Register);
 8375   %}
 8376   ins_pipe(ialu_reg_reg);
 8377 %}
 8378 
 8379 // Logical Shift Right by variable
 8380 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8381 %{
 8382   predicate(!VM_Version::supports_bmi2());
 8383   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8384   effect(KILL cr);
 8385 
 8386   format %{ "shrl    $dst, $shift" %}
 8387   ins_encode %{
 8388     __ shrl($dst$$Address);
 8389   %}
 8390   ins_pipe(ialu_mem_reg);
 8391 %}
 8392 
 8393 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8394 %{
 8395   predicate(VM_Version::supports_bmi2());
 8396   match(Set dst (URShiftI src shift));
 8397 
 8398   format %{ "shrxl   $dst, $src, $shift" %}
 8399   ins_encode %{
 8400     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 8401   %}
 8402   ins_pipe(ialu_reg_reg);
 8403 %}
 8404 
 8405 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8406 %{
 8407   predicate(VM_Version::supports_bmi2());
 8408   match(Set dst (URShiftI (LoadI src) shift));
 8409   ins_cost(175);
 8410   format %{ "shrxl   $dst, $src, $shift" %}
 8411   ins_encode %{
 8412     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 8413   %}
 8414   ins_pipe(ialu_reg_mem);
 8415 %}
 8416 
 8417 // Long Shift Instructions
 8418 // Shift Left by one, two, three
 8419 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 8420 %{
 8421   match(Set dst (LShiftL dst shift));
 8422   effect(KILL cr);
 8423 
 8424   format %{ "salq    $dst, $shift" %}
 8425   ins_encode %{
 8426     __ salq($dst$$Register, $shift$$constant);
 8427   %}
 8428   ins_pipe(ialu_reg);
 8429 %}
 8430 
 8431 // Shift Left by 8-bit immediate
 8432 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8433 %{
 8434   match(Set dst (LShiftL dst shift));
 8435   effect(KILL cr);
 8436 
 8437   format %{ "salq    $dst, $shift" %}
 8438   ins_encode %{
 8439     __ salq($dst$$Register, $shift$$constant);
 8440   %}
 8441   ins_pipe(ialu_reg);
 8442 %}
 8443 
 8444 // Shift Left by 8-bit immediate
 8445 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8446 %{
 8447   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8448   effect(KILL cr);
 8449 
 8450   format %{ "salq    $dst, $shift" %}
 8451   ins_encode %{
 8452     __ salq($dst$$Address, $shift$$constant);
 8453   %}
 8454   ins_pipe(ialu_mem_imm);
 8455 %}
 8456 
 8457 // Shift Left by variable
 8458 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8459 %{
 8460   predicate(!VM_Version::supports_bmi2());
 8461   match(Set dst (LShiftL dst shift));
 8462   effect(KILL cr);
 8463 
 8464   format %{ "salq    $dst, $shift" %}
 8465   ins_encode %{
 8466     __ salq($dst$$Register);
 8467   %}
 8468   ins_pipe(ialu_reg_reg);
 8469 %}
 8470 
 8471 // Shift Left by variable
 8472 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8473 %{
 8474   predicate(!VM_Version::supports_bmi2());
 8475   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8476   effect(KILL cr);
 8477 
 8478   format %{ "salq    $dst, $shift" %}
 8479   ins_encode %{
 8480     __ salq($dst$$Address);
 8481   %}
 8482   ins_pipe(ialu_mem_reg);
 8483 %}
 8484 
 8485 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8486 %{
 8487   predicate(VM_Version::supports_bmi2());
 8488   match(Set dst (LShiftL src shift));
 8489 
 8490   format %{ "shlxq   $dst, $src, $shift" %}
 8491   ins_encode %{
 8492     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 8493   %}
 8494   ins_pipe(ialu_reg_reg);
 8495 %}
 8496 
 8497 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8498 %{
 8499   predicate(VM_Version::supports_bmi2());
 8500   match(Set dst (LShiftL (LoadL src) shift));
 8501   ins_cost(175);
 8502   format %{ "shlxq   $dst, $src, $shift" %}
 8503   ins_encode %{
 8504     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 8505   %}
 8506   ins_pipe(ialu_reg_mem);
 8507 %}
 8508 
 8509 // Arithmetic Shift Right by 8-bit immediate
 8510 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 8511 %{
 8512   match(Set dst (RShiftL dst shift));
 8513   effect(KILL cr);
 8514 
 8515   format %{ "sarq    $dst, $shift" %}
 8516   ins_encode %{
 8517     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 8518   %}
 8519   ins_pipe(ialu_mem_imm);
 8520 %}
 8521 
 8522 // Arithmetic Shift Right by 8-bit immediate
 8523 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 8524 %{
 8525   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8526   effect(KILL cr);
 8527 
 8528   format %{ "sarq    $dst, $shift" %}
 8529   ins_encode %{
 8530     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 8531   %}
 8532   ins_pipe(ialu_mem_imm);
 8533 %}
 8534 
 8535 // Arithmetic Shift Right by variable
 8536 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8537 %{
 8538   predicate(!VM_Version::supports_bmi2());
 8539   match(Set dst (RShiftL dst shift));
 8540   effect(KILL cr);
 8541 
 8542   format %{ "sarq    $dst, $shift" %}
 8543   ins_encode %{
 8544     __ sarq($dst$$Register);
 8545   %}
 8546   ins_pipe(ialu_reg_reg);
 8547 %}
 8548 
 8549 // Arithmetic Shift Right by variable
 8550 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8551 %{
 8552   predicate(!VM_Version::supports_bmi2());
 8553   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8554   effect(KILL cr);
 8555 
 8556   format %{ "sarq    $dst, $shift" %}
 8557   ins_encode %{
 8558     __ sarq($dst$$Address);
 8559   %}
 8560   ins_pipe(ialu_mem_reg);
 8561 %}
 8562 
 8563 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8564 %{
 8565   predicate(VM_Version::supports_bmi2());
 8566   match(Set dst (RShiftL src shift));
 8567 
 8568   format %{ "sarxq   $dst, $src, $shift" %}
 8569   ins_encode %{
 8570     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 8571   %}
 8572   ins_pipe(ialu_reg_reg);
 8573 %}
 8574 
 8575 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8576 %{
 8577   predicate(VM_Version::supports_bmi2());
 8578   match(Set dst (RShiftL (LoadL src) shift));
 8579   ins_cost(175);
 8580   format %{ "sarxq   $dst, $src, $shift" %}
 8581   ins_encode %{
 8582     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 8583   %}
 8584   ins_pipe(ialu_reg_mem);
 8585 %}
 8586 
 8587 // Logical Shift Right by 8-bit immediate
 8588 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8589 %{
 8590   match(Set dst (URShiftL dst shift));
 8591   effect(KILL cr);
 8592 
 8593   format %{ "shrq    $dst, $shift" %}
 8594   ins_encode %{
 8595     __ shrq($dst$$Register, $shift$$constant);
 8596   %}
 8597   ins_pipe(ialu_reg);
 8598 %}
 8599 
 8600 // Logical Shift Right by 8-bit immediate
 8601 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8602 %{
 8603   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8604   effect(KILL cr);
 8605 
 8606   format %{ "shrq    $dst, $shift" %}
 8607   ins_encode %{
 8608     __ shrq($dst$$Address, $shift$$constant);
 8609   %}
 8610   ins_pipe(ialu_mem_imm);
 8611 %}
 8612 
 8613 // Logical Shift Right by variable
 8614 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8615 %{
 8616   predicate(!VM_Version::supports_bmi2());
 8617   match(Set dst (URShiftL dst shift));
 8618   effect(KILL cr);
 8619 
 8620   format %{ "shrq    $dst, $shift" %}
 8621   ins_encode %{
 8622     __ shrq($dst$$Register);
 8623   %}
 8624   ins_pipe(ialu_reg_reg);
 8625 %}
 8626 
 8627 // Logical Shift Right by variable
 8628 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8629 %{
 8630   predicate(!VM_Version::supports_bmi2());
 8631   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8632   effect(KILL cr);
 8633 
 8634   format %{ "shrq    $dst, $shift" %}
 8635   ins_encode %{
 8636     __ shrq($dst$$Address);
 8637   %}
 8638   ins_pipe(ialu_mem_reg);
 8639 %}
 8640 
 8641 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8642 %{
 8643   predicate(VM_Version::supports_bmi2());
 8644   match(Set dst (URShiftL src shift));
 8645 
 8646   format %{ "shrxq   $dst, $src, $shift" %}
 8647   ins_encode %{
 8648     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 8649   %}
 8650   ins_pipe(ialu_reg_reg);
 8651 %}
 8652 
 8653 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8654 %{
 8655   predicate(VM_Version::supports_bmi2());
 8656   match(Set dst (URShiftL (LoadL src) shift));
 8657   ins_cost(175);
 8658   format %{ "shrxq   $dst, $src, $shift" %}
 8659   ins_encode %{
 8660     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 8661   %}
 8662   ins_pipe(ialu_reg_mem);
 8663 %}
 8664 
 8665 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 8666 // This idiom is used by the compiler for the i2b bytecode.
 8667 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 8668 %{
 8669   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 8670 
 8671   format %{ "movsbl  $dst, $src\t# i2b" %}
 8672   ins_encode %{
 8673     __ movsbl($dst$$Register, $src$$Register);
 8674   %}
 8675   ins_pipe(ialu_reg_reg);
 8676 %}
 8677 
 8678 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 8679 // This idiom is used by the compiler the i2s bytecode.
 8680 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 8681 %{
 8682   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 8683 
 8684   format %{ "movswl  $dst, $src\t# i2s" %}
 8685   ins_encode %{
 8686     __ movswl($dst$$Register, $src$$Register);
 8687   %}
 8688   ins_pipe(ialu_reg_reg);
 8689 %}
 8690 
 8691 // ROL/ROR instructions
 8692 
 8693 // Rotate left by constant.
 8694 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8695 %{
 8696   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8697   match(Set dst (RotateLeft dst shift));
 8698   effect(KILL cr);
 8699   format %{ "roll    $dst, $shift" %}
 8700   ins_encode %{
 8701     __ roll($dst$$Register, $shift$$constant);
 8702   %}
 8703   ins_pipe(ialu_reg);
 8704 %}
 8705 
 8706 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 8707 %{
 8708   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8709   match(Set dst (RotateLeft src shift));
 8710   format %{ "rolxl   $dst, $src, $shift" %}
 8711   ins_encode %{
 8712     int shift = 32 - ($shift$$constant & 31);
 8713     __ rorxl($dst$$Register, $src$$Register, shift);
 8714   %}
 8715   ins_pipe(ialu_reg_reg);
 8716 %}
 8717 
 8718 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8719 %{
 8720   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8721   match(Set dst (RotateLeft (LoadI src) shift));
 8722   ins_cost(175);
 8723   format %{ "rolxl   $dst, $src, $shift" %}
 8724   ins_encode %{
 8725     int shift = 32 - ($shift$$constant & 31);
 8726     __ rorxl($dst$$Register, $src$$Address, shift);
 8727   %}
 8728   ins_pipe(ialu_reg_mem);
 8729 %}
 8730 
 8731 // Rotate Left by variable
 8732 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8733 %{
 8734   predicate(n->bottom_type()->basic_type() == T_INT);
 8735   match(Set dst (RotateLeft dst shift));
 8736   effect(KILL cr);
 8737   format %{ "roll    $dst, $shift" %}
 8738   ins_encode %{
 8739     __ roll($dst$$Register);
 8740   %}
 8741   ins_pipe(ialu_reg_reg);
 8742 %}
 8743 
 8744 // Rotate Right by constant.
 8745 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8746 %{
 8747   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8748   match(Set dst (RotateRight dst shift));
 8749   effect(KILL cr);
 8750   format %{ "rorl    $dst, $shift" %}
 8751   ins_encode %{
 8752     __ rorl($dst$$Register, $shift$$constant);
 8753   %}
 8754   ins_pipe(ialu_reg);
 8755 %}
 8756 
 8757 // Rotate Right by constant.
 8758 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 8759 %{
 8760   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8761   match(Set dst (RotateRight src shift));
 8762   format %{ "rorxl   $dst, $src, $shift" %}
 8763   ins_encode %{
 8764     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 8765   %}
 8766   ins_pipe(ialu_reg_reg);
 8767 %}
 8768 
 8769 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8770 %{
 8771   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8772   match(Set dst (RotateRight (LoadI src) shift));
 8773   ins_cost(175);
 8774   format %{ "rorxl   $dst, $src, $shift" %}
 8775   ins_encode %{
 8776     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 8777   %}
 8778   ins_pipe(ialu_reg_mem);
 8779 %}
 8780 
 8781 // Rotate Right by variable
 8782 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8783 %{
 8784   predicate(n->bottom_type()->basic_type() == T_INT);
 8785   match(Set dst (RotateRight dst shift));
 8786   effect(KILL cr);
 8787   format %{ "rorl    $dst, $shift" %}
 8788   ins_encode %{
 8789     __ rorl($dst$$Register);
 8790   %}
 8791   ins_pipe(ialu_reg_reg);
 8792 %}
 8793 
 8794 // Rotate Left by constant.
 8795 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8796 %{
 8797   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8798   match(Set dst (RotateLeft dst shift));
 8799   effect(KILL cr);
 8800   format %{ "rolq    $dst, $shift" %}
 8801   ins_encode %{
 8802     __ rolq($dst$$Register, $shift$$constant);
 8803   %}
 8804   ins_pipe(ialu_reg);
 8805 %}
 8806 
 8807 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 8808 %{
 8809   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8810   match(Set dst (RotateLeft src shift));
 8811   format %{ "rolxq   $dst, $src, $shift" %}
 8812   ins_encode %{
 8813     int shift = 64 - ($shift$$constant & 63);
 8814     __ rorxq($dst$$Register, $src$$Register, shift);
 8815   %}
 8816   ins_pipe(ialu_reg_reg);
 8817 %}
 8818 
 8819 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8820 %{
 8821   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8822   match(Set dst (RotateLeft (LoadL src) shift));
 8823   ins_cost(175);
 8824   format %{ "rolxq   $dst, $src, $shift" %}
 8825   ins_encode %{
 8826     int shift = 64 - ($shift$$constant & 63);
 8827     __ rorxq($dst$$Register, $src$$Address, shift);
 8828   %}
 8829   ins_pipe(ialu_reg_mem);
 8830 %}
 8831 
 8832 // Rotate Left by variable
 8833 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8834 %{
 8835   predicate(n->bottom_type()->basic_type() == T_LONG);
 8836   match(Set dst (RotateLeft dst shift));
 8837   effect(KILL cr);
 8838   format %{ "rolq    $dst, $shift" %}
 8839   ins_encode %{
 8840     __ rolq($dst$$Register);
 8841   %}
 8842   ins_pipe(ialu_reg_reg);
 8843 %}
 8844 
 8845 // Rotate Right by constant.
 8846 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8847 %{
 8848   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8849   match(Set dst (RotateRight dst shift));
 8850   effect(KILL cr);
 8851   format %{ "rorq    $dst, $shift" %}
 8852   ins_encode %{
 8853     __ rorq($dst$$Register, $shift$$constant);
 8854   %}
 8855   ins_pipe(ialu_reg);
 8856 %}
 8857 
 8858 // Rotate Right by constant
 8859 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 8860 %{
 8861   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8862   match(Set dst (RotateRight src shift));
 8863   format %{ "rorxq   $dst, $src, $shift" %}
 8864   ins_encode %{
 8865     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 8866   %}
 8867   ins_pipe(ialu_reg_reg);
 8868 %}
 8869 
 8870 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8871 %{
 8872   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8873   match(Set dst (RotateRight (LoadL src) shift));
 8874   ins_cost(175);
 8875   format %{ "rorxq   $dst, $src, $shift" %}
 8876   ins_encode %{
 8877     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 8878   %}
 8879   ins_pipe(ialu_reg_mem);
 8880 %}
 8881 
 8882 // Rotate Right by variable
 8883 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8884 %{
 8885   predicate(n->bottom_type()->basic_type() == T_LONG);
 8886   match(Set dst (RotateRight dst shift));
 8887   effect(KILL cr);
 8888   format %{ "rorq    $dst, $shift" %}
 8889   ins_encode %{
 8890     __ rorq($dst$$Register);
 8891   %}
 8892   ins_pipe(ialu_reg_reg);
 8893 %}
 8894 
 8895 //----------------------------- CompressBits/ExpandBits ------------------------
 8896 
 8897 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8898   predicate(n->bottom_type()->isa_long());
 8899   match(Set dst (CompressBits src mask));
 8900   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8901   ins_encode %{
 8902     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 8903   %}
 8904   ins_pipe( pipe_slow );
 8905 %}
 8906 
 8907 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8908   predicate(n->bottom_type()->isa_long());
 8909   match(Set dst (ExpandBits src mask));
 8910   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8911   ins_encode %{
 8912     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 8913   %}
 8914   ins_pipe( pipe_slow );
 8915 %}
 8916 
 8917 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8918   predicate(n->bottom_type()->isa_long());
 8919   match(Set dst (CompressBits src (LoadL mask)));
 8920   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8921   ins_encode %{
 8922     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 8923   %}
 8924   ins_pipe( pipe_slow );
 8925 %}
 8926 
 8927 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8928   predicate(n->bottom_type()->isa_long());
 8929   match(Set dst (ExpandBits src (LoadL mask)));
 8930   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8931   ins_encode %{
 8932     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 8933   %}
 8934   ins_pipe( pipe_slow );
 8935 %}
 8936 
 8937 
 8938 // Logical Instructions
 8939 
 8940 // Integer Logical Instructions
 8941 
 8942 // And Instructions
 8943 // And Register with Register
 8944 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8945 %{
 8946   match(Set dst (AndI dst src));
 8947   effect(KILL cr);
 8948   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8949 
 8950   format %{ "andl    $dst, $src\t# int" %}
 8951   ins_encode %{
 8952     __ andl($dst$$Register, $src$$Register);
 8953   %}
 8954   ins_pipe(ialu_reg_reg);
 8955 %}
 8956 
 8957 // And Register with Immediate 255
 8958 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 8959 %{
 8960   match(Set dst (AndI src mask));
 8961 
 8962   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 8963   ins_encode %{
 8964     __ movzbl($dst$$Register, $src$$Register);
 8965   %}
 8966   ins_pipe(ialu_reg);
 8967 %}
 8968 
 8969 // And Register with Immediate 255 and promote to long
 8970 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 8971 %{
 8972   match(Set dst (ConvI2L (AndI src mask)));
 8973 
 8974   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 8975   ins_encode %{
 8976     __ movzbl($dst$$Register, $src$$Register);
 8977   %}
 8978   ins_pipe(ialu_reg);
 8979 %}
 8980 
 8981 // And Register with Immediate 65535
 8982 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 8983 %{
 8984   match(Set dst (AndI src mask));
 8985 
 8986   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 8987   ins_encode %{
 8988     __ movzwl($dst$$Register, $src$$Register);
 8989   %}
 8990   ins_pipe(ialu_reg);
 8991 %}
 8992 
 8993 // And Register with Immediate 65535 and promote to long
 8994 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 8995 %{
 8996   match(Set dst (ConvI2L (AndI src mask)));
 8997 
 8998   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 8999   ins_encode %{
 9000     __ movzwl($dst$$Register, $src$$Register);
 9001   %}
 9002   ins_pipe(ialu_reg);
 9003 %}
 9004 
 9005 // Can skip int2long conversions after AND with small bitmask
 9006 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9007 %{
 9008   predicate(VM_Version::supports_bmi2());
 9009   ins_cost(125);
 9010   effect(TEMP tmp, KILL cr);
 9011   match(Set dst (ConvI2L (AndI src mask)));
 9012   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9013   ins_encode %{
 9014     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9015     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9016   %}
 9017   ins_pipe(ialu_reg_reg);
 9018 %}
 9019 
 9020 // And Register with Immediate
 9021 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9022 %{
 9023   match(Set dst (AndI dst src));
 9024   effect(KILL cr);
 9025   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9026 
 9027   format %{ "andl    $dst, $src\t# int" %}
 9028   ins_encode %{
 9029     __ andl($dst$$Register, $src$$constant);
 9030   %}
 9031   ins_pipe(ialu_reg);
 9032 %}
 9033 
 9034 // And Register with Memory
 9035 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9036 %{
 9037   match(Set dst (AndI dst (LoadI src)));
 9038   effect(KILL cr);
 9039   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9040 
 9041   ins_cost(150);
 9042   format %{ "andl    $dst, $src\t# int" %}
 9043   ins_encode %{
 9044     __ andl($dst$$Register, $src$$Address);
 9045   %}
 9046   ins_pipe(ialu_reg_mem);
 9047 %}
 9048 
 9049 // And Memory with Register
 9050 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9051 %{
 9052   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9053   effect(KILL cr);
 9054   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9055 
 9056   ins_cost(150);
 9057   format %{ "andb    $dst, $src\t# byte" %}
 9058   ins_encode %{
 9059     __ andb($dst$$Address, $src$$Register);
 9060   %}
 9061   ins_pipe(ialu_mem_reg);
 9062 %}
 9063 
 9064 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9065 %{
 9066   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9067   effect(KILL cr);
 9068   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9069 
 9070   ins_cost(150);
 9071   format %{ "andl    $dst, $src\t# int" %}
 9072   ins_encode %{
 9073     __ andl($dst$$Address, $src$$Register);
 9074   %}
 9075   ins_pipe(ialu_mem_reg);
 9076 %}
 9077 
 9078 // And Memory with Immediate
 9079 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9080 %{
 9081   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9082   effect(KILL cr);
 9083   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9084 
 9085   ins_cost(125);
 9086   format %{ "andl    $dst, $src\t# int" %}
 9087   ins_encode %{
 9088     __ andl($dst$$Address, $src$$constant);
 9089   %}
 9090   ins_pipe(ialu_mem_imm);
 9091 %}
 9092 
 9093 // BMI1 instructions
 9094 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9095   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9096   predicate(UseBMI1Instructions);
 9097   effect(KILL cr);
 9098   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9099 
 9100   ins_cost(125);
 9101   format %{ "andnl  $dst, $src1, $src2" %}
 9102 
 9103   ins_encode %{
 9104     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9105   %}
 9106   ins_pipe(ialu_reg_mem);
 9107 %}
 9108 
 9109 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9110   match(Set dst (AndI (XorI src1 minus_1) src2));
 9111   predicate(UseBMI1Instructions);
 9112   effect(KILL cr);
 9113   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9114 
 9115   format %{ "andnl  $dst, $src1, $src2" %}
 9116 
 9117   ins_encode %{
 9118     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9119   %}
 9120   ins_pipe(ialu_reg);
 9121 %}
 9122 
 9123 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9124   match(Set dst (AndI (SubI imm_zero src) src));
 9125   predicate(UseBMI1Instructions);
 9126   effect(KILL cr);
 9127   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9128 
 9129   format %{ "blsil  $dst, $src" %}
 9130 
 9131   ins_encode %{
 9132     __ blsil($dst$$Register, $src$$Register);
 9133   %}
 9134   ins_pipe(ialu_reg);
 9135 %}
 9136 
 9137 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9138   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9139   predicate(UseBMI1Instructions);
 9140   effect(KILL cr);
 9141   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9142 
 9143   ins_cost(125);
 9144   format %{ "blsil  $dst, $src" %}
 9145 
 9146   ins_encode %{
 9147     __ blsil($dst$$Register, $src$$Address);
 9148   %}
 9149   ins_pipe(ialu_reg_mem);
 9150 %}
 9151 
 9152 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9153 %{
 9154   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9155   predicate(UseBMI1Instructions);
 9156   effect(KILL cr);
 9157   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9158 
 9159   ins_cost(125);
 9160   format %{ "blsmskl $dst, $src" %}
 9161 
 9162   ins_encode %{
 9163     __ blsmskl($dst$$Register, $src$$Address);
 9164   %}
 9165   ins_pipe(ialu_reg_mem);
 9166 %}
 9167 
 9168 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9169 %{
 9170   match(Set dst (XorI (AddI src minus_1) src));
 9171   predicate(UseBMI1Instructions);
 9172   effect(KILL cr);
 9173   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9174 
 9175   format %{ "blsmskl $dst, $src" %}
 9176 
 9177   ins_encode %{
 9178     __ blsmskl($dst$$Register, $src$$Register);
 9179   %}
 9180 
 9181   ins_pipe(ialu_reg);
 9182 %}
 9183 
 9184 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9185 %{
 9186   match(Set dst (AndI (AddI src minus_1) src) );
 9187   predicate(UseBMI1Instructions);
 9188   effect(KILL cr);
 9189   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9190 
 9191   format %{ "blsrl  $dst, $src" %}
 9192 
 9193   ins_encode %{
 9194     __ blsrl($dst$$Register, $src$$Register);
 9195   %}
 9196 
 9197   ins_pipe(ialu_reg_mem);
 9198 %}
 9199 
 9200 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9201 %{
 9202   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9203   predicate(UseBMI1Instructions);
 9204   effect(KILL cr);
 9205   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9206 
 9207   ins_cost(125);
 9208   format %{ "blsrl  $dst, $src" %}
 9209 
 9210   ins_encode %{
 9211     __ blsrl($dst$$Register, $src$$Address);
 9212   %}
 9213 
 9214   ins_pipe(ialu_reg);
 9215 %}
 9216 
 9217 // Or Instructions
 9218 // Or Register with Register
 9219 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9220 %{
 9221   match(Set dst (OrI dst src));
 9222   effect(KILL cr);
 9223   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9224 
 9225   format %{ "orl     $dst, $src\t# int" %}
 9226   ins_encode %{
 9227     __ orl($dst$$Register, $src$$Register);
 9228   %}
 9229   ins_pipe(ialu_reg_reg);
 9230 %}
 9231 
 9232 // Or Register with Immediate
 9233 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9234 %{
 9235   match(Set dst (OrI dst src));
 9236   effect(KILL cr);
 9237   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9238 
 9239   format %{ "orl     $dst, $src\t# int" %}
 9240   ins_encode %{
 9241     __ orl($dst$$Register, $src$$constant);
 9242   %}
 9243   ins_pipe(ialu_reg);
 9244 %}
 9245 
 9246 // Or Register with Memory
 9247 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9248 %{
 9249   match(Set dst (OrI dst (LoadI src)));
 9250   effect(KILL cr);
 9251   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9252 
 9253   ins_cost(150);
 9254   format %{ "orl     $dst, $src\t# int" %}
 9255   ins_encode %{
 9256     __ orl($dst$$Register, $src$$Address);
 9257   %}
 9258   ins_pipe(ialu_reg_mem);
 9259 %}
 9260 
 9261 // Or Memory with Register
 9262 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9263 %{
 9264   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9265   effect(KILL cr);
 9266   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9267 
 9268   ins_cost(150);
 9269   format %{ "orb    $dst, $src\t# byte" %}
 9270   ins_encode %{
 9271     __ orb($dst$$Address, $src$$Register);
 9272   %}
 9273   ins_pipe(ialu_mem_reg);
 9274 %}
 9275 
 9276 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9277 %{
 9278   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9279   effect(KILL cr);
 9280   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9281 
 9282   ins_cost(150);
 9283   format %{ "orl     $dst, $src\t# int" %}
 9284   ins_encode %{
 9285     __ orl($dst$$Address, $src$$Register);
 9286   %}
 9287   ins_pipe(ialu_mem_reg);
 9288 %}
 9289 
 9290 // Or Memory with Immediate
 9291 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9292 %{
 9293   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9294   effect(KILL cr);
 9295   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9296 
 9297   ins_cost(125);
 9298   format %{ "orl     $dst, $src\t# int" %}
 9299   ins_encode %{
 9300     __ orl($dst$$Address, $src$$constant);
 9301   %}
 9302   ins_pipe(ialu_mem_imm);
 9303 %}
 9304 
 9305 // Xor Instructions
 9306 // Xor Register with Register
 9307 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9308 %{
 9309   match(Set dst (XorI dst src));
 9310   effect(KILL cr);
 9311   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9312 
 9313   format %{ "xorl    $dst, $src\t# int" %}
 9314   ins_encode %{
 9315     __ xorl($dst$$Register, $src$$Register);
 9316   %}
 9317   ins_pipe(ialu_reg_reg);
 9318 %}
 9319 
 9320 // Xor Register with Immediate -1
 9321 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9322   match(Set dst (XorI dst imm));
 9323 
 9324   format %{ "not    $dst" %}
 9325   ins_encode %{
 9326      __ notl($dst$$Register);
 9327   %}
 9328   ins_pipe(ialu_reg);
 9329 %}
 9330 
 9331 // Xor Register with Immediate
 9332 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9333 %{
 9334   match(Set dst (XorI dst src));
 9335   effect(KILL cr);
 9336   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9337 
 9338   format %{ "xorl    $dst, $src\t# int" %}
 9339   ins_encode %{
 9340     __ xorl($dst$$Register, $src$$constant);
 9341   %}
 9342   ins_pipe(ialu_reg);
 9343 %}
 9344 
 9345 // Xor Register with Memory
 9346 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9347 %{
 9348   match(Set dst (XorI dst (LoadI src)));
 9349   effect(KILL cr);
 9350   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9351 
 9352   ins_cost(150);
 9353   format %{ "xorl    $dst, $src\t# int" %}
 9354   ins_encode %{
 9355     __ xorl($dst$$Register, $src$$Address);
 9356   %}
 9357   ins_pipe(ialu_reg_mem);
 9358 %}
 9359 
 9360 // Xor Memory with Register
 9361 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9362 %{
 9363   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9364   effect(KILL cr);
 9365   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9366 
 9367   ins_cost(150);
 9368   format %{ "xorb    $dst, $src\t# byte" %}
 9369   ins_encode %{
 9370     __ xorb($dst$$Address, $src$$Register);
 9371   %}
 9372   ins_pipe(ialu_mem_reg);
 9373 %}
 9374 
 9375 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9376 %{
 9377   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9378   effect(KILL cr);
 9379   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9380 
 9381   ins_cost(150);
 9382   format %{ "xorl    $dst, $src\t# int" %}
 9383   ins_encode %{
 9384     __ xorl($dst$$Address, $src$$Register);
 9385   %}
 9386   ins_pipe(ialu_mem_reg);
 9387 %}
 9388 
 9389 // Xor Memory with Immediate
 9390 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9391 %{
 9392   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9393   effect(KILL cr);
 9394   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9395 
 9396   ins_cost(125);
 9397   format %{ "xorl    $dst, $src\t# int" %}
 9398   ins_encode %{
 9399     __ xorl($dst$$Address, $src$$constant);
 9400   %}
 9401   ins_pipe(ialu_mem_imm);
 9402 %}
 9403 
 9404 
 9405 // Long Logical Instructions
 9406 
 9407 // And Instructions
 9408 // And Register with Register
 9409 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9410 %{
 9411   match(Set dst (AndL dst src));
 9412   effect(KILL cr);
 9413   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9414 
 9415   format %{ "andq    $dst, $src\t# long" %}
 9416   ins_encode %{
 9417     __ andq($dst$$Register, $src$$Register);
 9418   %}
 9419   ins_pipe(ialu_reg_reg);
 9420 %}
 9421 
 9422 // And Register with Immediate 255
 9423 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
 9424 %{
 9425   match(Set dst (AndL src mask));
 9426 
 9427   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
 9428   ins_encode %{
 9429     // movzbl zeroes out the upper 32-bit and does not need REX.W
 9430     __ movzbl($dst$$Register, $src$$Register);
 9431   %}
 9432   ins_pipe(ialu_reg);
 9433 %}
 9434 
 9435 // And Register with Immediate 65535
 9436 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
 9437 %{
 9438   match(Set dst (AndL src mask));
 9439 
 9440   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
 9441   ins_encode %{
 9442     // movzwl zeroes out the upper 32-bit and does not need REX.W
 9443     __ movzwl($dst$$Register, $src$$Register);
 9444   %}
 9445   ins_pipe(ialu_reg);
 9446 %}
 9447 
 9448 // And Register with Immediate
 9449 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9450 %{
 9451   match(Set dst (AndL dst src));
 9452   effect(KILL cr);
 9453   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9454 
 9455   format %{ "andq    $dst, $src\t# long" %}
 9456   ins_encode %{
 9457     __ andq($dst$$Register, $src$$constant);
 9458   %}
 9459   ins_pipe(ialu_reg);
 9460 %}
 9461 
 9462 // And Register with Memory
 9463 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9464 %{
 9465   match(Set dst (AndL dst (LoadL src)));
 9466   effect(KILL cr);
 9467   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9468 
 9469   ins_cost(150);
 9470   format %{ "andq    $dst, $src\t# long" %}
 9471   ins_encode %{
 9472     __ andq($dst$$Register, $src$$Address);
 9473   %}
 9474   ins_pipe(ialu_reg_mem);
 9475 %}
 9476 
 9477 // And Memory with Register
 9478 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9479 %{
 9480   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9481   effect(KILL cr);
 9482   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9483 
 9484   ins_cost(150);
 9485   format %{ "andq    $dst, $src\t# long" %}
 9486   ins_encode %{
 9487     __ andq($dst$$Address, $src$$Register);
 9488   %}
 9489   ins_pipe(ialu_mem_reg);
 9490 %}
 9491 
 9492 // And Memory with Immediate
 9493 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9494 %{
 9495   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9496   effect(KILL cr);
 9497   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9498 
 9499   ins_cost(125);
 9500   format %{ "andq    $dst, $src\t# long" %}
 9501   ins_encode %{
 9502     __ andq($dst$$Address, $src$$constant);
 9503   %}
 9504   ins_pipe(ialu_mem_imm);
 9505 %}
 9506 
 9507 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9508 %{
 9509   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9510   // because AND/OR works well enough for 8/32-bit values.
 9511   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9512 
 9513   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9514   effect(KILL cr);
 9515 
 9516   ins_cost(125);
 9517   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9518   ins_encode %{
 9519     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9520   %}
 9521   ins_pipe(ialu_mem_imm);
 9522 %}
 9523 
 9524 // BMI1 instructions
 9525 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9526   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9527   predicate(UseBMI1Instructions);
 9528   effect(KILL cr);
 9529   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9530 
 9531   ins_cost(125);
 9532   format %{ "andnq  $dst, $src1, $src2" %}
 9533 
 9534   ins_encode %{
 9535     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9536   %}
 9537   ins_pipe(ialu_reg_mem);
 9538 %}
 9539 
 9540 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9541   match(Set dst (AndL (XorL src1 minus_1) src2));
 9542   predicate(UseBMI1Instructions);
 9543   effect(KILL cr);
 9544   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9545 
 9546   format %{ "andnq  $dst, $src1, $src2" %}
 9547 
 9548   ins_encode %{
 9549   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9550   %}
 9551   ins_pipe(ialu_reg_mem);
 9552 %}
 9553 
 9554 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9555   match(Set dst (AndL (SubL imm_zero src) src));
 9556   predicate(UseBMI1Instructions);
 9557   effect(KILL cr);
 9558   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9559 
 9560   format %{ "blsiq  $dst, $src" %}
 9561 
 9562   ins_encode %{
 9563     __ blsiq($dst$$Register, $src$$Register);
 9564   %}
 9565   ins_pipe(ialu_reg);
 9566 %}
 9567 
 9568 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9569   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9570   predicate(UseBMI1Instructions);
 9571   effect(KILL cr);
 9572   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9573 
 9574   ins_cost(125);
 9575   format %{ "blsiq  $dst, $src" %}
 9576 
 9577   ins_encode %{
 9578     __ blsiq($dst$$Register, $src$$Address);
 9579   %}
 9580   ins_pipe(ialu_reg_mem);
 9581 %}
 9582 
 9583 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9584 %{
 9585   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9586   predicate(UseBMI1Instructions);
 9587   effect(KILL cr);
 9588   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9589 
 9590   ins_cost(125);
 9591   format %{ "blsmskq $dst, $src" %}
 9592 
 9593   ins_encode %{
 9594     __ blsmskq($dst$$Register, $src$$Address);
 9595   %}
 9596   ins_pipe(ialu_reg_mem);
 9597 %}
 9598 
 9599 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9600 %{
 9601   match(Set dst (XorL (AddL src minus_1) src));
 9602   predicate(UseBMI1Instructions);
 9603   effect(KILL cr);
 9604   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9605 
 9606   format %{ "blsmskq $dst, $src" %}
 9607 
 9608   ins_encode %{
 9609     __ blsmskq($dst$$Register, $src$$Register);
 9610   %}
 9611 
 9612   ins_pipe(ialu_reg);
 9613 %}
 9614 
 9615 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9616 %{
 9617   match(Set dst (AndL (AddL src minus_1) src) );
 9618   predicate(UseBMI1Instructions);
 9619   effect(KILL cr);
 9620   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9621 
 9622   format %{ "blsrq  $dst, $src" %}
 9623 
 9624   ins_encode %{
 9625     __ blsrq($dst$$Register, $src$$Register);
 9626   %}
 9627 
 9628   ins_pipe(ialu_reg);
 9629 %}
 9630 
 9631 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9632 %{
 9633   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9634   predicate(UseBMI1Instructions);
 9635   effect(KILL cr);
 9636   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9637 
 9638   ins_cost(125);
 9639   format %{ "blsrq  $dst, $src" %}
 9640 
 9641   ins_encode %{
 9642     __ blsrq($dst$$Register, $src$$Address);
 9643   %}
 9644 
 9645   ins_pipe(ialu_reg);
 9646 %}
 9647 
 9648 // Or Instructions
 9649 // Or Register with Register
 9650 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9651 %{
 9652   match(Set dst (OrL dst src));
 9653   effect(KILL cr);
 9654   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9655 
 9656   format %{ "orq     $dst, $src\t# long" %}
 9657   ins_encode %{
 9658     __ orq($dst$$Register, $src$$Register);
 9659   %}
 9660   ins_pipe(ialu_reg_reg);
 9661 %}
 9662 
 9663 // Use any_RegP to match R15 (TLS register) without spilling.
 9664 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 9665   match(Set dst (OrL dst (CastP2X src)));
 9666   effect(KILL cr);
 9667   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9668 
 9669   format %{ "orq     $dst, $src\t# long" %}
 9670   ins_encode %{
 9671     __ orq($dst$$Register, $src$$Register);
 9672   %}
 9673   ins_pipe(ialu_reg_reg);
 9674 %}
 9675 
 9676 
 9677 // Or Register with Immediate
 9678 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9679 %{
 9680   match(Set dst (OrL dst src));
 9681   effect(KILL cr);
 9682   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9683 
 9684   format %{ "orq     $dst, $src\t# long" %}
 9685   ins_encode %{
 9686     __ orq($dst$$Register, $src$$constant);
 9687   %}
 9688   ins_pipe(ialu_reg);
 9689 %}
 9690 
 9691 // Or Register with Memory
 9692 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9693 %{
 9694   match(Set dst (OrL dst (LoadL src)));
 9695   effect(KILL cr);
 9696   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9697 
 9698   ins_cost(150);
 9699   format %{ "orq     $dst, $src\t# long" %}
 9700   ins_encode %{
 9701     __ orq($dst$$Register, $src$$Address);
 9702   %}
 9703   ins_pipe(ialu_reg_mem);
 9704 %}
 9705 
 9706 // Or Memory with Register
 9707 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9708 %{
 9709   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9710   effect(KILL cr);
 9711   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9712 
 9713   ins_cost(150);
 9714   format %{ "orq     $dst, $src\t# long" %}
 9715   ins_encode %{
 9716     __ orq($dst$$Address, $src$$Register);
 9717   %}
 9718   ins_pipe(ialu_mem_reg);
 9719 %}
 9720 
 9721 // Or Memory with Immediate
 9722 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9723 %{
 9724   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9725   effect(KILL cr);
 9726   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9727 
 9728   ins_cost(125);
 9729   format %{ "orq     $dst, $src\t# long" %}
 9730   ins_encode %{
 9731     __ orq($dst$$Address, $src$$constant);
 9732   %}
 9733   ins_pipe(ialu_mem_imm);
 9734 %}
 9735 
 9736 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
 9737 %{
 9738   // con should be a pure 64-bit power of 2 immediate
 9739   // because AND/OR works well enough for 8/32-bit values.
 9740   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
 9741 
 9742   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
 9743   effect(KILL cr);
 9744 
 9745   ins_cost(125);
 9746   format %{ "btsq    $dst, log2($con)\t# long" %}
 9747   ins_encode %{
 9748     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
 9749   %}
 9750   ins_pipe(ialu_mem_imm);
 9751 %}
 9752 
 9753 // Xor Instructions
 9754 // Xor Register with Register
 9755 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9756 %{
 9757   match(Set dst (XorL dst src));
 9758   effect(KILL cr);
 9759   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9760 
 9761   format %{ "xorq    $dst, $src\t# long" %}
 9762   ins_encode %{
 9763     __ xorq($dst$$Register, $src$$Register);
 9764   %}
 9765   ins_pipe(ialu_reg_reg);
 9766 %}
 9767 
 9768 // Xor Register with Immediate -1
 9769 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 9770   match(Set dst (XorL dst imm));
 9771 
 9772   format %{ "notq   $dst" %}
 9773   ins_encode %{
 9774      __ notq($dst$$Register);
 9775   %}
 9776   ins_pipe(ialu_reg);
 9777 %}
 9778 
 9779 // Xor Register with Immediate
 9780 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9781 %{
 9782   match(Set dst (XorL dst src));
 9783   effect(KILL cr);
 9784   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9785 
 9786   format %{ "xorq    $dst, $src\t# long" %}
 9787   ins_encode %{
 9788     __ xorq($dst$$Register, $src$$constant);
 9789   %}
 9790   ins_pipe(ialu_reg);
 9791 %}
 9792 
 9793 // Xor Register with Memory
 9794 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9795 %{
 9796   match(Set dst (XorL dst (LoadL src)));
 9797   effect(KILL cr);
 9798   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9799 
 9800   ins_cost(150);
 9801   format %{ "xorq    $dst, $src\t# long" %}
 9802   ins_encode %{
 9803     __ xorq($dst$$Register, $src$$Address);
 9804   %}
 9805   ins_pipe(ialu_reg_mem);
 9806 %}
 9807 
 9808 // Xor Memory with Register
 9809 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9810 %{
 9811   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9812   effect(KILL cr);
 9813   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9814 
 9815   ins_cost(150);
 9816   format %{ "xorq    $dst, $src\t# long" %}
 9817   ins_encode %{
 9818     __ xorq($dst$$Address, $src$$Register);
 9819   %}
 9820   ins_pipe(ialu_mem_reg);
 9821 %}
 9822 
 9823 // Xor Memory with Immediate
 9824 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9825 %{
 9826   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9827   effect(KILL cr);
 9828   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9829 
 9830   ins_cost(125);
 9831   format %{ "xorq    $dst, $src\t# long" %}
 9832   ins_encode %{
 9833     __ xorq($dst$$Address, $src$$constant);
 9834   %}
 9835   ins_pipe(ialu_mem_imm);
 9836 %}
 9837 
 9838 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 9839 %{
 9840   match(Set dst (CmpLTMask p q));
 9841   effect(KILL cr);
 9842 
 9843   ins_cost(400);
 9844   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 9845             "setlt   $dst\n\t"
 9846             "movzbl  $dst, $dst\n\t"
 9847             "negl    $dst" %}
 9848   ins_encode %{
 9849     __ cmpl($p$$Register, $q$$Register);
 9850     __ setb(Assembler::less, $dst$$Register);
 9851     __ movzbl($dst$$Register, $dst$$Register);
 9852     __ negl($dst$$Register);
 9853   %}
 9854   ins_pipe(pipe_slow);
 9855 %}
 9856 
 9857 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
 9858 %{
 9859   match(Set dst (CmpLTMask dst zero));
 9860   effect(KILL cr);
 9861 
 9862   ins_cost(100);
 9863   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 9864   ins_encode %{
 9865     __ sarl($dst$$Register, 31);
 9866   %}
 9867   ins_pipe(ialu_reg);
 9868 %}
 9869 
 9870 /* Better to save a register than avoid a branch */
 9871 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9872 %{
 9873   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 9874   effect(KILL cr);
 9875   ins_cost(300);
 9876   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
 9877             "jge     done\n\t"
 9878             "addl    $p,$y\n"
 9879             "done:   " %}
 9880   ins_encode %{
 9881     Register Rp = $p$$Register;
 9882     Register Rq = $q$$Register;
 9883     Register Ry = $y$$Register;
 9884     Label done;
 9885     __ subl(Rp, Rq);
 9886     __ jccb(Assembler::greaterEqual, done);
 9887     __ addl(Rp, Ry);
 9888     __ bind(done);
 9889   %}
 9890   ins_pipe(pipe_cmplt);
 9891 %}
 9892 
 9893 /* Better to save a register than avoid a branch */
 9894 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9895 %{
 9896   match(Set y (AndI (CmpLTMask p q) y));
 9897   effect(KILL cr);
 9898 
 9899   ins_cost(300);
 9900 
 9901   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
 9902             "jlt     done\n\t"
 9903             "xorl    $y, $y\n"
 9904             "done:   " %}
 9905   ins_encode %{
 9906     Register Rp = $p$$Register;
 9907     Register Rq = $q$$Register;
 9908     Register Ry = $y$$Register;
 9909     Label done;
 9910     __ cmpl(Rp, Rq);
 9911     __ jccb(Assembler::less, done);
 9912     __ xorl(Ry, Ry);
 9913     __ bind(done);
 9914   %}
 9915   ins_pipe(pipe_cmplt);
 9916 %}
 9917 
 9918 
 9919 //---------- FP Instructions------------------------------------------------
 9920 
 9921 // Really expensive, avoid
 9922 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 9923 %{
 9924   match(Set cr (CmpF src1 src2));
 9925 
 9926   ins_cost(500);
 9927   format %{ "ucomiss $src1, $src2\n\t"
 9928             "jnp,s   exit\n\t"
 9929             "pushfq\t# saw NaN, set CF\n\t"
 9930             "andq    [rsp], #0xffffff2b\n\t"
 9931             "popfq\n"
 9932     "exit:" %}
 9933   ins_encode %{
 9934     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9935     emit_cmpfp_fixup(_masm);
 9936   %}
 9937   ins_pipe(pipe_slow);
 9938 %}
 9939 
 9940 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 9941   match(Set cr (CmpF src1 src2));
 9942 
 9943   ins_cost(100);
 9944   format %{ "ucomiss $src1, $src2" %}
 9945   ins_encode %{
 9946     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9947   %}
 9948   ins_pipe(pipe_slow);
 9949 %}
 9950 
 9951 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 9952   match(Set cr (CmpF src1 (LoadF src2)));
 9953 
 9954   ins_cost(100);
 9955   format %{ "ucomiss $src1, $src2" %}
 9956   ins_encode %{
 9957     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9958   %}
 9959   ins_pipe(pipe_slow);
 9960 %}
 9961 
 9962 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
 9963   match(Set cr (CmpF src con));
 9964   ins_cost(100);
 9965   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
 9966   ins_encode %{
 9967     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9968   %}
 9969   ins_pipe(pipe_slow);
 9970 %}
 9971 
 9972 // Really expensive, avoid
 9973 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 9974 %{
 9975   match(Set cr (CmpD src1 src2));
 9976 
 9977   ins_cost(500);
 9978   format %{ "ucomisd $src1, $src2\n\t"
 9979             "jnp,s   exit\n\t"
 9980             "pushfq\t# saw NaN, set CF\n\t"
 9981             "andq    [rsp], #0xffffff2b\n\t"
 9982             "popfq\n"
 9983     "exit:" %}
 9984   ins_encode %{
 9985     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9986     emit_cmpfp_fixup(_masm);
 9987   %}
 9988   ins_pipe(pipe_slow);
 9989 %}
 9990 
 9991 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 9992   match(Set cr (CmpD src1 src2));
 9993 
 9994   ins_cost(100);
 9995   format %{ "ucomisd $src1, $src2 test" %}
 9996   ins_encode %{
 9997     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9998   %}
 9999   ins_pipe(pipe_slow);
10000 %}
10001 
10002 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10003   match(Set cr (CmpD src1 (LoadD src2)));
10004 
10005   ins_cost(100);
10006   format %{ "ucomisd $src1, $src2" %}
10007   ins_encode %{
10008     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10009   %}
10010   ins_pipe(pipe_slow);
10011 %}
10012 
10013 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10014   match(Set cr (CmpD src con));
10015   ins_cost(100);
10016   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10017   ins_encode %{
10018     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10019   %}
10020   ins_pipe(pipe_slow);
10021 %}
10022 
10023 // Compare into -1,0,1
10024 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10025 %{
10026   match(Set dst (CmpF3 src1 src2));
10027   effect(KILL cr);
10028 
10029   ins_cost(275);
10030   format %{ "ucomiss $src1, $src2\n\t"
10031             "movl    $dst, #-1\n\t"
10032             "jp,s    done\n\t"
10033             "jb,s    done\n\t"
10034             "setne   $dst\n\t"
10035             "movzbl  $dst, $dst\n"
10036     "done:" %}
10037   ins_encode %{
10038     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10039     emit_cmpfp3(_masm, $dst$$Register);
10040   %}
10041   ins_pipe(pipe_slow);
10042 %}
10043 
10044 // Compare into -1,0,1
10045 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10046 %{
10047   match(Set dst (CmpF3 src1 (LoadF src2)));
10048   effect(KILL cr);
10049 
10050   ins_cost(275);
10051   format %{ "ucomiss $src1, $src2\n\t"
10052             "movl    $dst, #-1\n\t"
10053             "jp,s    done\n\t"
10054             "jb,s    done\n\t"
10055             "setne   $dst\n\t"
10056             "movzbl  $dst, $dst\n"
10057     "done:" %}
10058   ins_encode %{
10059     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10060     emit_cmpfp3(_masm, $dst$$Register);
10061   %}
10062   ins_pipe(pipe_slow);
10063 %}
10064 
10065 // Compare into -1,0,1
10066 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10067   match(Set dst (CmpF3 src con));
10068   effect(KILL cr);
10069 
10070   ins_cost(275);
10071   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10072             "movl    $dst, #-1\n\t"
10073             "jp,s    done\n\t"
10074             "jb,s    done\n\t"
10075             "setne   $dst\n\t"
10076             "movzbl  $dst, $dst\n"
10077     "done:" %}
10078   ins_encode %{
10079     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10080     emit_cmpfp3(_masm, $dst$$Register);
10081   %}
10082   ins_pipe(pipe_slow);
10083 %}
10084 
10085 // Compare into -1,0,1
10086 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10087 %{
10088   match(Set dst (CmpD3 src1 src2));
10089   effect(KILL cr);
10090 
10091   ins_cost(275);
10092   format %{ "ucomisd $src1, $src2\n\t"
10093             "movl    $dst, #-1\n\t"
10094             "jp,s    done\n\t"
10095             "jb,s    done\n\t"
10096             "setne   $dst\n\t"
10097             "movzbl  $dst, $dst\n"
10098     "done:" %}
10099   ins_encode %{
10100     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10101     emit_cmpfp3(_masm, $dst$$Register);
10102   %}
10103   ins_pipe(pipe_slow);
10104 %}
10105 
10106 // Compare into -1,0,1
10107 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10108 %{
10109   match(Set dst (CmpD3 src1 (LoadD src2)));
10110   effect(KILL cr);
10111 
10112   ins_cost(275);
10113   format %{ "ucomisd $src1, $src2\n\t"
10114             "movl    $dst, #-1\n\t"
10115             "jp,s    done\n\t"
10116             "jb,s    done\n\t"
10117             "setne   $dst\n\t"
10118             "movzbl  $dst, $dst\n"
10119     "done:" %}
10120   ins_encode %{
10121     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10122     emit_cmpfp3(_masm, $dst$$Register);
10123   %}
10124   ins_pipe(pipe_slow);
10125 %}
10126 
10127 // Compare into -1,0,1
10128 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10129   match(Set dst (CmpD3 src con));
10130   effect(KILL cr);
10131 
10132   ins_cost(275);
10133   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10134             "movl    $dst, #-1\n\t"
10135             "jp,s    done\n\t"
10136             "jb,s    done\n\t"
10137             "setne   $dst\n\t"
10138             "movzbl  $dst, $dst\n"
10139     "done:" %}
10140   ins_encode %{
10141     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10142     emit_cmpfp3(_masm, $dst$$Register);
10143   %}
10144   ins_pipe(pipe_slow);
10145 %}
10146 
10147 //----------Arithmetic Conversion Instructions---------------------------------
10148 
10149 instruct convF2D_reg_reg(regD dst, regF src)
10150 %{
10151   match(Set dst (ConvF2D src));
10152   effect(TEMP dst);
10153   format %{ "cvtss2sd $dst, $src" %}
10154   ins_encode %{
10155     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10156   %}
10157   ins_pipe(pipe_slow); // XXX
10158 %}
10159 
10160 instruct convF2D_reg_mem(regD dst, memory src)
10161 %{
10162   match(Set dst (ConvF2D (LoadF src)));
10163 
10164   format %{ "cvtss2sd $dst, $src" %}
10165   ins_encode %{
10166     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10167   %}
10168   ins_pipe(pipe_slow); // XXX
10169 %}
10170 
10171 instruct convD2F_reg_reg(regF dst, regD src)
10172 %{
10173   match(Set dst (ConvD2F src));
10174   effect(TEMP dst);
10175   format %{ "cvtsd2ss $dst, $src" %}
10176   ins_encode %{
10177     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10178   %}
10179   ins_pipe(pipe_slow); // XXX
10180 %}
10181 
10182 instruct convD2F_reg_mem(regF dst, memory src)
10183 %{
10184   match(Set dst (ConvD2F (LoadD src)));
10185 
10186   format %{ "cvtsd2ss $dst, $src" %}
10187   ins_encode %{
10188     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10189   %}
10190   ins_pipe(pipe_slow); // XXX
10191 %}
10192 
10193 // XXX do mem variants
10194 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10195 %{
10196   match(Set dst (ConvF2I src));
10197   effect(KILL cr);
10198   format %{ "convert_f2i $dst, $src" %}
10199   ins_encode %{
10200     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10201   %}
10202   ins_pipe(pipe_slow);
10203 %}
10204 
10205 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10206 %{
10207   match(Set dst (ConvF2L src));
10208   effect(KILL cr);
10209   format %{ "convert_f2l $dst, $src"%}
10210   ins_encode %{
10211     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10212   %}
10213   ins_pipe(pipe_slow);
10214 %}
10215 
10216 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10217 %{
10218   match(Set dst (ConvD2I src));
10219   effect(KILL cr);
10220   format %{ "convert_d2i $dst, $src"%}
10221   ins_encode %{
10222     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10223   %}
10224   ins_pipe(pipe_slow);
10225 %}
10226 
10227 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10228 %{
10229   match(Set dst (ConvD2L src));
10230   effect(KILL cr);
10231   format %{ "convert_d2l $dst, $src"%}
10232   ins_encode %{
10233     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10234   %}
10235   ins_pipe(pipe_slow);
10236 %}
10237 
10238 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10239 %{
10240   match(Set dst (RoundD src));
10241   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10242   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10243   ins_encode %{
10244     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10245   %}
10246   ins_pipe(pipe_slow);
10247 %}
10248 
10249 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10250 %{
10251   match(Set dst (RoundF src));
10252   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10253   format %{ "round_float $dst,$src" %}
10254   ins_encode %{
10255     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10256   %}
10257   ins_pipe(pipe_slow);
10258 %}
10259 
10260 instruct convI2F_reg_reg(regF dst, rRegI src)
10261 %{
10262   predicate(!UseXmmI2F);
10263   match(Set dst (ConvI2F src));
10264 
10265   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10266   ins_encode %{
10267     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10268   %}
10269   ins_pipe(pipe_slow); // XXX
10270 %}
10271 
10272 instruct convI2F_reg_mem(regF dst, memory src)
10273 %{
10274   match(Set dst (ConvI2F (LoadI src)));
10275 
10276   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10277   ins_encode %{
10278     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10279   %}
10280   ins_pipe(pipe_slow); // XXX
10281 %}
10282 
10283 instruct convI2D_reg_reg(regD dst, rRegI src)
10284 %{
10285   predicate(!UseXmmI2D);
10286   match(Set dst (ConvI2D src));
10287 
10288   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10289   ins_encode %{
10290     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10291   %}
10292   ins_pipe(pipe_slow); // XXX
10293 %}
10294 
10295 instruct convI2D_reg_mem(regD dst, memory src)
10296 %{
10297   match(Set dst (ConvI2D (LoadI src)));
10298 
10299   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10300   ins_encode %{
10301     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10302   %}
10303   ins_pipe(pipe_slow); // XXX
10304 %}
10305 
10306 instruct convXI2F_reg(regF dst, rRegI src)
10307 %{
10308   predicate(UseXmmI2F);
10309   match(Set dst (ConvI2F src));
10310 
10311   format %{ "movdl $dst, $src\n\t"
10312             "cvtdq2psl $dst, $dst\t# i2f" %}
10313   ins_encode %{
10314     __ movdl($dst$$XMMRegister, $src$$Register);
10315     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10316   %}
10317   ins_pipe(pipe_slow); // XXX
10318 %}
10319 
10320 instruct convXI2D_reg(regD dst, rRegI src)
10321 %{
10322   predicate(UseXmmI2D);
10323   match(Set dst (ConvI2D src));
10324 
10325   format %{ "movdl $dst, $src\n\t"
10326             "cvtdq2pdl $dst, $dst\t# i2d" %}
10327   ins_encode %{
10328     __ movdl($dst$$XMMRegister, $src$$Register);
10329     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10330   %}
10331   ins_pipe(pipe_slow); // XXX
10332 %}
10333 
10334 instruct convL2F_reg_reg(regF dst, rRegL src)
10335 %{
10336   match(Set dst (ConvL2F src));
10337 
10338   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10339   ins_encode %{
10340     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10341   %}
10342   ins_pipe(pipe_slow); // XXX
10343 %}
10344 
10345 instruct convL2F_reg_mem(regF dst, memory src)
10346 %{
10347   match(Set dst (ConvL2F (LoadL src)));
10348 
10349   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10350   ins_encode %{
10351     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10352   %}
10353   ins_pipe(pipe_slow); // XXX
10354 %}
10355 
10356 instruct convL2D_reg_reg(regD dst, rRegL src)
10357 %{
10358   match(Set dst (ConvL2D src));
10359 
10360   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10361   ins_encode %{
10362     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10363   %}
10364   ins_pipe(pipe_slow); // XXX
10365 %}
10366 
10367 instruct convL2D_reg_mem(regD dst, memory src)
10368 %{
10369   match(Set dst (ConvL2D (LoadL src)));
10370 
10371   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10372   ins_encode %{
10373     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10374   %}
10375   ins_pipe(pipe_slow); // XXX
10376 %}
10377 
10378 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10379 %{
10380   match(Set dst (ConvI2L src));
10381 
10382   ins_cost(125);
10383   format %{ "movslq  $dst, $src\t# i2l" %}
10384   ins_encode %{
10385     __ movslq($dst$$Register, $src$$Register);
10386   %}
10387   ins_pipe(ialu_reg_reg);
10388 %}
10389 
10390 // Zero-extend convert int to long
10391 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10392 %{
10393   match(Set dst (AndL (ConvI2L src) mask));
10394 
10395   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10396   ins_encode %{
10397     if ($dst$$reg != $src$$reg) {
10398       __ movl($dst$$Register, $src$$Register);
10399     }
10400   %}
10401   ins_pipe(ialu_reg_reg);
10402 %}
10403 
10404 // Zero-extend convert int to long
10405 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10406 %{
10407   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10408 
10409   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10410   ins_encode %{
10411     __ movl($dst$$Register, $src$$Address);
10412   %}
10413   ins_pipe(ialu_reg_mem);
10414 %}
10415 
10416 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10417 %{
10418   match(Set dst (AndL src mask));
10419 
10420   format %{ "movl    $dst, $src\t# zero-extend long" %}
10421   ins_encode %{
10422     __ movl($dst$$Register, $src$$Register);
10423   %}
10424   ins_pipe(ialu_reg_reg);
10425 %}
10426 
10427 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10428 %{
10429   match(Set dst (ConvL2I src));
10430 
10431   format %{ "movl    $dst, $src\t# l2i" %}
10432   ins_encode %{
10433     __ movl($dst$$Register, $src$$Register);
10434   %}
10435   ins_pipe(ialu_reg_reg);
10436 %}
10437 
10438 
10439 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10440   match(Set dst (MoveF2I src));
10441   effect(DEF dst, USE src);
10442 
10443   ins_cost(125);
10444   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10445   ins_encode %{
10446     __ movl($dst$$Register, Address(rsp, $src$$disp));
10447   %}
10448   ins_pipe(ialu_reg_mem);
10449 %}
10450 
10451 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10452   match(Set dst (MoveI2F src));
10453   effect(DEF dst, USE src);
10454 
10455   ins_cost(125);
10456   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10457   ins_encode %{
10458     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10459   %}
10460   ins_pipe(pipe_slow);
10461 %}
10462 
10463 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10464   match(Set dst (MoveD2L src));
10465   effect(DEF dst, USE src);
10466 
10467   ins_cost(125);
10468   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10469   ins_encode %{
10470     __ movq($dst$$Register, Address(rsp, $src$$disp));
10471   %}
10472   ins_pipe(ialu_reg_mem);
10473 %}
10474 
10475 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10476   predicate(!UseXmmLoadAndClearUpper);
10477   match(Set dst (MoveL2D src));
10478   effect(DEF dst, USE src);
10479 
10480   ins_cost(125);
10481   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10482   ins_encode %{
10483     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10484   %}
10485   ins_pipe(pipe_slow);
10486 %}
10487 
10488 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10489   predicate(UseXmmLoadAndClearUpper);
10490   match(Set dst (MoveL2D src));
10491   effect(DEF dst, USE src);
10492 
10493   ins_cost(125);
10494   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10495   ins_encode %{
10496     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10497   %}
10498   ins_pipe(pipe_slow);
10499 %}
10500 
10501 
10502 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10503   match(Set dst (MoveF2I src));
10504   effect(DEF dst, USE src);
10505 
10506   ins_cost(95); // XXX
10507   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10508   ins_encode %{
10509     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10510   %}
10511   ins_pipe(pipe_slow);
10512 %}
10513 
10514 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10515   match(Set dst (MoveI2F src));
10516   effect(DEF dst, USE src);
10517 
10518   ins_cost(100);
10519   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10520   ins_encode %{
10521     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10522   %}
10523   ins_pipe( ialu_mem_reg );
10524 %}
10525 
10526 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10527   match(Set dst (MoveD2L src));
10528   effect(DEF dst, USE src);
10529 
10530   ins_cost(95); // XXX
10531   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10532   ins_encode %{
10533     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10534   %}
10535   ins_pipe(pipe_slow);
10536 %}
10537 
10538 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10539   match(Set dst (MoveL2D src));
10540   effect(DEF dst, USE src);
10541 
10542   ins_cost(100);
10543   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10544   ins_encode %{
10545     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10546   %}
10547   ins_pipe(ialu_mem_reg);
10548 %}
10549 
10550 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10551   match(Set dst (MoveF2I src));
10552   effect(DEF dst, USE src);
10553   ins_cost(85);
10554   format %{ "movd    $dst,$src\t# MoveF2I" %}
10555   ins_encode %{
10556     __ movdl($dst$$Register, $src$$XMMRegister);
10557   %}
10558   ins_pipe( pipe_slow );
10559 %}
10560 
10561 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10562   match(Set dst (MoveD2L src));
10563   effect(DEF dst, USE src);
10564   ins_cost(85);
10565   format %{ "movd    $dst,$src\t# MoveD2L" %}
10566   ins_encode %{
10567     __ movdq($dst$$Register, $src$$XMMRegister);
10568   %}
10569   ins_pipe( pipe_slow );
10570 %}
10571 
10572 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10573   match(Set dst (MoveI2F src));
10574   effect(DEF dst, USE src);
10575   ins_cost(100);
10576   format %{ "movd    $dst,$src\t# MoveI2F" %}
10577   ins_encode %{
10578     __ movdl($dst$$XMMRegister, $src$$Register);
10579   %}
10580   ins_pipe( pipe_slow );
10581 %}
10582 
10583 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10584   match(Set dst (MoveL2D src));
10585   effect(DEF dst, USE src);
10586   ins_cost(100);
10587   format %{ "movd    $dst,$src\t# MoveL2D" %}
10588   ins_encode %{
10589      __ movdq($dst$$XMMRegister, $src$$Register);
10590   %}
10591   ins_pipe( pipe_slow );
10592 %}
10593 
10594 // Fast clearing of an array
10595 // Small ClearArray non-AVX512.
10596 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10597                   Universe dummy, rFlagsReg cr)
10598 %{
10599   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
10600   match(Set dummy (ClearArray cnt base));
10601   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10602 
10603   format %{ $$template
10604     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10605     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10606     $$emit$$"jg      LARGE\n\t"
10607     $$emit$$"dec     rcx\n\t"
10608     $$emit$$"js      DONE\t# Zero length\n\t"
10609     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10610     $$emit$$"dec     rcx\n\t"
10611     $$emit$$"jge     LOOP\n\t"
10612     $$emit$$"jmp     DONE\n\t"
10613     $$emit$$"# LARGE:\n\t"
10614     if (UseFastStosb) {
10615        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10616        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10617     } else if (UseXMMForObjInit) {
10618        $$emit$$"mov     rdi,rax\n\t"
10619        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10620        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10621        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10622        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10623        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10624        $$emit$$"add     0x40,rax\n\t"
10625        $$emit$$"# L_zero_64_bytes:\n\t"
10626        $$emit$$"sub     0x8,rcx\n\t"
10627        $$emit$$"jge     L_loop\n\t"
10628        $$emit$$"add     0x4,rcx\n\t"
10629        $$emit$$"jl      L_tail\n\t"
10630        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10631        $$emit$$"add     0x20,rax\n\t"
10632        $$emit$$"sub     0x4,rcx\n\t"
10633        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10634        $$emit$$"add     0x4,rcx\n\t"
10635        $$emit$$"jle     L_end\n\t"
10636        $$emit$$"dec     rcx\n\t"
10637        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10638        $$emit$$"vmovq   xmm0,(rax)\n\t"
10639        $$emit$$"add     0x8,rax\n\t"
10640        $$emit$$"dec     rcx\n\t"
10641        $$emit$$"jge     L_sloop\n\t"
10642        $$emit$$"# L_end:\n\t"
10643     } else {
10644        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10645     }
10646     $$emit$$"# DONE"
10647   %}
10648   ins_encode %{
10649     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10650                  $tmp$$XMMRegister, false, knoreg);
10651   %}
10652   ins_pipe(pipe_slow);
10653 %}
10654 
10655 // Small ClearArray AVX512 non-constant length.
10656 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10657                        Universe dummy, rFlagsReg cr)
10658 %{
10659   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
10660   match(Set dummy (ClearArray cnt base));
10661   ins_cost(125);
10662   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10663 
10664   format %{ $$template
10665     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10666     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10667     $$emit$$"jg      LARGE\n\t"
10668     $$emit$$"dec     rcx\n\t"
10669     $$emit$$"js      DONE\t# Zero length\n\t"
10670     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10671     $$emit$$"dec     rcx\n\t"
10672     $$emit$$"jge     LOOP\n\t"
10673     $$emit$$"jmp     DONE\n\t"
10674     $$emit$$"# LARGE:\n\t"
10675     if (UseFastStosb) {
10676        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10677        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10678     } else if (UseXMMForObjInit) {
10679        $$emit$$"mov     rdi,rax\n\t"
10680        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10681        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10682        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10683        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10684        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10685        $$emit$$"add     0x40,rax\n\t"
10686        $$emit$$"# L_zero_64_bytes:\n\t"
10687        $$emit$$"sub     0x8,rcx\n\t"
10688        $$emit$$"jge     L_loop\n\t"
10689        $$emit$$"add     0x4,rcx\n\t"
10690        $$emit$$"jl      L_tail\n\t"
10691        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10692        $$emit$$"add     0x20,rax\n\t"
10693        $$emit$$"sub     0x4,rcx\n\t"
10694        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10695        $$emit$$"add     0x4,rcx\n\t"
10696        $$emit$$"jle     L_end\n\t"
10697        $$emit$$"dec     rcx\n\t"
10698        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10699        $$emit$$"vmovq   xmm0,(rax)\n\t"
10700        $$emit$$"add     0x8,rax\n\t"
10701        $$emit$$"dec     rcx\n\t"
10702        $$emit$$"jge     L_sloop\n\t"
10703        $$emit$$"# L_end:\n\t"
10704     } else {
10705        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10706     }
10707     $$emit$$"# DONE"
10708   %}
10709   ins_encode %{
10710     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10711                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
10712   %}
10713   ins_pipe(pipe_slow);
10714 %}
10715 
10716 // Large ClearArray non-AVX512.
10717 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10718                         Universe dummy, rFlagsReg cr)
10719 %{
10720   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
10721   match(Set dummy (ClearArray cnt base));
10722   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10723 
10724   format %{ $$template
10725     if (UseFastStosb) {
10726        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10727        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10728        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10729     } else if (UseXMMForObjInit) {
10730        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10731        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10732        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10733        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10734        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10735        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10736        $$emit$$"add     0x40,rax\n\t"
10737        $$emit$$"# L_zero_64_bytes:\n\t"
10738        $$emit$$"sub     0x8,rcx\n\t"
10739        $$emit$$"jge     L_loop\n\t"
10740        $$emit$$"add     0x4,rcx\n\t"
10741        $$emit$$"jl      L_tail\n\t"
10742        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10743        $$emit$$"add     0x20,rax\n\t"
10744        $$emit$$"sub     0x4,rcx\n\t"
10745        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10746        $$emit$$"add     0x4,rcx\n\t"
10747        $$emit$$"jle     L_end\n\t"
10748        $$emit$$"dec     rcx\n\t"
10749        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10750        $$emit$$"vmovq   xmm0,(rax)\n\t"
10751        $$emit$$"add     0x8,rax\n\t"
10752        $$emit$$"dec     rcx\n\t"
10753        $$emit$$"jge     L_sloop\n\t"
10754        $$emit$$"# L_end:\n\t"
10755     } else {
10756        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10757        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10758     }
10759   %}
10760   ins_encode %{
10761     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10762                  $tmp$$XMMRegister, true, knoreg);
10763   %}
10764   ins_pipe(pipe_slow);
10765 %}
10766 
10767 // Large ClearArray AVX512.
10768 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
10769                              Universe dummy, rFlagsReg cr)
10770 %{
10771   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
10772   match(Set dummy (ClearArray cnt base));
10773   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
10774 
10775   format %{ $$template
10776     if (UseFastStosb) {
10777        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10778        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10779        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10780     } else if (UseXMMForObjInit) {
10781        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10782        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10783        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10784        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10785        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10786        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10787        $$emit$$"add     0x40,rax\n\t"
10788        $$emit$$"# L_zero_64_bytes:\n\t"
10789        $$emit$$"sub     0x8,rcx\n\t"
10790        $$emit$$"jge     L_loop\n\t"
10791        $$emit$$"add     0x4,rcx\n\t"
10792        $$emit$$"jl      L_tail\n\t"
10793        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10794        $$emit$$"add     0x20,rax\n\t"
10795        $$emit$$"sub     0x4,rcx\n\t"
10796        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10797        $$emit$$"add     0x4,rcx\n\t"
10798        $$emit$$"jle     L_end\n\t"
10799        $$emit$$"dec     rcx\n\t"
10800        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10801        $$emit$$"vmovq   xmm0,(rax)\n\t"
10802        $$emit$$"add     0x8,rax\n\t"
10803        $$emit$$"dec     rcx\n\t"
10804        $$emit$$"jge     L_sloop\n\t"
10805        $$emit$$"# L_end:\n\t"
10806     } else {
10807        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10808        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10809     }
10810   %}
10811   ins_encode %{
10812     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10813                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
10814   %}
10815   ins_pipe(pipe_slow);
10816 %}
10817 
10818 // Small ClearArray AVX512 constant length.
10819 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
10820 %{
10821   predicate(!((ClearArrayNode*)n)->is_large() &&
10822               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
10823   match(Set dummy (ClearArray cnt base));
10824   ins_cost(100);
10825   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
10826   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
10827   ins_encode %{
10828    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10829   %}
10830   ins_pipe(pipe_slow);
10831 %}
10832 
10833 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10834                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10835 %{
10836   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10837   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10838   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10839 
10840   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10841   ins_encode %{
10842     __ string_compare($str1$$Register, $str2$$Register,
10843                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10844                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
10845   %}
10846   ins_pipe( pipe_slow );
10847 %}
10848 
10849 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10850                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10851 %{
10852   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10853   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10854   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10855 
10856   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10857   ins_encode %{
10858     __ string_compare($str1$$Register, $str2$$Register,
10859                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10860                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
10861   %}
10862   ins_pipe( pipe_slow );
10863 %}
10864 
10865 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10866                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10867 %{
10868   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10869   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10870   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10871 
10872   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10873   ins_encode %{
10874     __ string_compare($str1$$Register, $str2$$Register,
10875                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10876                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
10877   %}
10878   ins_pipe( pipe_slow );
10879 %}
10880 
10881 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10882                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10883 %{
10884   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10885   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10886   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10887 
10888   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10889   ins_encode %{
10890     __ string_compare($str1$$Register, $str2$$Register,
10891                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10892                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
10893   %}
10894   ins_pipe( pipe_slow );
10895 %}
10896 
10897 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10898                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10899 %{
10900   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10901   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10902   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10903 
10904   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10905   ins_encode %{
10906     __ string_compare($str1$$Register, $str2$$Register,
10907                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10908                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
10909   %}
10910   ins_pipe( pipe_slow );
10911 %}
10912 
10913 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10914                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10915 %{
10916   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10917   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10918   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10919 
10920   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10921   ins_encode %{
10922     __ string_compare($str1$$Register, $str2$$Register,
10923                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10924                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
10925   %}
10926   ins_pipe( pipe_slow );
10927 %}
10928 
10929 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10930                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
10931 %{
10932   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10933   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10934   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10935 
10936   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10937   ins_encode %{
10938     __ string_compare($str2$$Register, $str1$$Register,
10939                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10940                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
10941   %}
10942   ins_pipe( pipe_slow );
10943 %}
10944 
10945 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10946                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
10947 %{
10948   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10949   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10950   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10951 
10952   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10953   ins_encode %{
10954     __ string_compare($str2$$Register, $str1$$Register,
10955                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10956                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
10957   %}
10958   ins_pipe( pipe_slow );
10959 %}
10960 
10961 // fast search of substring with known size.
10962 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10963                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10964 %{
10965   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10966   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10967   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10968 
10969   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10970   ins_encode %{
10971     int icnt2 = (int)$int_cnt2$$constant;
10972     if (icnt2 >= 16) {
10973       // IndexOf for constant substrings with size >= 16 elements
10974       // which don't need to be loaded through stack.
10975       __ string_indexofC8($str1$$Register, $str2$$Register,
10976                           $cnt1$$Register, $cnt2$$Register,
10977                           icnt2, $result$$Register,
10978                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10979     } else {
10980       // Small strings are loaded through stack if they cross page boundary.
10981       __ string_indexof($str1$$Register, $str2$$Register,
10982                         $cnt1$$Register, $cnt2$$Register,
10983                         icnt2, $result$$Register,
10984                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10985     }
10986   %}
10987   ins_pipe( pipe_slow );
10988 %}
10989 
10990 // fast search of substring with known size.
10991 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10992                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10993 %{
10994   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10995   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10996   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10997 
10998   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
10999   ins_encode %{
11000     int icnt2 = (int)$int_cnt2$$constant;
11001     if (icnt2 >= 8) {
11002       // IndexOf for constant substrings with size >= 8 elements
11003       // which don't need to be loaded through stack.
11004       __ string_indexofC8($str1$$Register, $str2$$Register,
11005                           $cnt1$$Register, $cnt2$$Register,
11006                           icnt2, $result$$Register,
11007                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11008     } else {
11009       // Small strings are loaded through stack if they cross page boundary.
11010       __ string_indexof($str1$$Register, $str2$$Register,
11011                         $cnt1$$Register, $cnt2$$Register,
11012                         icnt2, $result$$Register,
11013                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11014     }
11015   %}
11016   ins_pipe( pipe_slow );
11017 %}
11018 
11019 // fast search of substring with known size.
11020 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11021                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11022 %{
11023   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11024   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11025   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11026 
11027   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11028   ins_encode %{
11029     int icnt2 = (int)$int_cnt2$$constant;
11030     if (icnt2 >= 8) {
11031       // IndexOf for constant substrings with size >= 8 elements
11032       // which don't need to be loaded through stack.
11033       __ string_indexofC8($str1$$Register, $str2$$Register,
11034                           $cnt1$$Register, $cnt2$$Register,
11035                           icnt2, $result$$Register,
11036                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11037     } else {
11038       // Small strings are loaded through stack if they cross page boundary.
11039       __ string_indexof($str1$$Register, $str2$$Register,
11040                         $cnt1$$Register, $cnt2$$Register,
11041                         icnt2, $result$$Register,
11042                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11043     }
11044   %}
11045   ins_pipe( pipe_slow );
11046 %}
11047 
11048 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11049                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11050 %{
11051   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11052   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11053   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11054 
11055   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11056   ins_encode %{
11057     __ string_indexof($str1$$Register, $str2$$Register,
11058                       $cnt1$$Register, $cnt2$$Register,
11059                       (-1), $result$$Register,
11060                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11061   %}
11062   ins_pipe( pipe_slow );
11063 %}
11064 
11065 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11066                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11067 %{
11068   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11069   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11070   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11071 
11072   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11073   ins_encode %{
11074     __ string_indexof($str1$$Register, $str2$$Register,
11075                       $cnt1$$Register, $cnt2$$Register,
11076                       (-1), $result$$Register,
11077                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11078   %}
11079   ins_pipe( pipe_slow );
11080 %}
11081 
11082 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11083                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11084 %{
11085   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11086   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11087   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11088 
11089   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11090   ins_encode %{
11091     __ string_indexof($str1$$Register, $str2$$Register,
11092                       $cnt1$$Register, $cnt2$$Register,
11093                       (-1), $result$$Register,
11094                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11095   %}
11096   ins_pipe( pipe_slow );
11097 %}
11098 
11099 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11100                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11101 %{
11102   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11103   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11104   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11105   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11106   ins_encode %{
11107     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11108                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11109   %}
11110   ins_pipe( pipe_slow );
11111 %}
11112 
11113 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11114                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11115 %{
11116   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11117   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11118   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11119   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11120   ins_encode %{
11121     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11122                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11123   %}
11124   ins_pipe( pipe_slow );
11125 %}
11126 
11127 // fast string equals
11128 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11129                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11130 %{
11131   predicate(!VM_Version::supports_avx512vlbw());
11132   match(Set result (StrEquals (Binary str1 str2) cnt));
11133   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11134 
11135   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11136   ins_encode %{
11137     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11138                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11139                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11140   %}
11141   ins_pipe( pipe_slow );
11142 %}
11143 
11144 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11145                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11146 %{
11147   predicate(VM_Version::supports_avx512vlbw());
11148   match(Set result (StrEquals (Binary str1 str2) cnt));
11149   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11150 
11151   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11152   ins_encode %{
11153     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11154                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11155                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11156   %}
11157   ins_pipe( pipe_slow );
11158 %}
11159 
11160 // fast array equals
11161 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11162                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11163 %{
11164   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11165   match(Set result (AryEq ary1 ary2));
11166   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11167 
11168   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11169   ins_encode %{
11170     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11171                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11172                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11173   %}
11174   ins_pipe( pipe_slow );
11175 %}
11176 
11177 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11178                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11179 %{
11180   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11181   match(Set result (AryEq ary1 ary2));
11182   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11183 
11184   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11185   ins_encode %{
11186     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11187                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11188                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11189   %}
11190   ins_pipe( pipe_slow );
11191 %}
11192 
11193 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11194                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11195 %{
11196   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11197   match(Set result (AryEq ary1 ary2));
11198   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11199 
11200   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11201   ins_encode %{
11202     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11203                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11204                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11205   %}
11206   ins_pipe( pipe_slow );
11207 %}
11208 
11209 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11210                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11211 %{
11212   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11213   match(Set result (AryEq ary1 ary2));
11214   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11215 
11216   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11217   ins_encode %{
11218     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11219                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11220                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11221   %}
11222   ins_pipe( pipe_slow );
11223 %}
11224 
11225 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11226                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11227                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11228                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11229                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11230 %{
11231   predicate(UseAVX >= 2);
11232   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11233   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11234          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11235          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11236          USE basic_type, KILL cr);
11237 
11238   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
11239   ins_encode %{
11240     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11241                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11242                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11243                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11244                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11245                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11246                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11247   %}
11248   ins_pipe( pipe_slow );
11249 %}
11250 
11251 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11252                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11253 %{
11254   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11255   match(Set result (CountPositives ary1 len));
11256   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11257 
11258   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11259   ins_encode %{
11260     __ count_positives($ary1$$Register, $len$$Register,
11261                        $result$$Register, $tmp3$$Register,
11262                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11263   %}
11264   ins_pipe( pipe_slow );
11265 %}
11266 
11267 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11268                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11269 %{
11270   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11271   match(Set result (CountPositives ary1 len));
11272   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11273 
11274   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11275   ins_encode %{
11276     __ count_positives($ary1$$Register, $len$$Register,
11277                        $result$$Register, $tmp3$$Register,
11278                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11279   %}
11280   ins_pipe( pipe_slow );
11281 %}
11282 
11283 // fast char[] to byte[] compression
11284 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11285                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11286   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11287   match(Set result (StrCompressedCopy src (Binary dst len)));
11288   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11289          USE_KILL len, KILL tmp5, KILL cr);
11290 
11291   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11292   ins_encode %{
11293     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11294                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11295                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11296                            knoreg, knoreg);
11297   %}
11298   ins_pipe( pipe_slow );
11299 %}
11300 
11301 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11302                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11303   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11304   match(Set result (StrCompressedCopy src (Binary dst len)));
11305   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11306          USE_KILL len, KILL tmp5, KILL cr);
11307 
11308   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11309   ins_encode %{
11310     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11311                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11312                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11313                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11314   %}
11315   ins_pipe( pipe_slow );
11316 %}
11317 // fast byte[] to char[] inflation
11318 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11319                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11320   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11321   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11322   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11323 
11324   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11325   ins_encode %{
11326     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11327                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11328   %}
11329   ins_pipe( pipe_slow );
11330 %}
11331 
11332 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11333                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11334   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11335   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11336   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11337 
11338   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11339   ins_encode %{
11340     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11341                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11342   %}
11343   ins_pipe( pipe_slow );
11344 %}
11345 
11346 // encode char[] to byte[] in ISO_8859_1
11347 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11348                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11349                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11350   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11351   match(Set result (EncodeISOArray src (Binary dst len)));
11352   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11353 
11354   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11355   ins_encode %{
11356     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11357                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11358                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11359   %}
11360   ins_pipe( pipe_slow );
11361 %}
11362 
11363 // encode char[] to byte[] in ASCII
11364 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11365                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11366                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11367   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11368   match(Set result (EncodeISOArray src (Binary dst len)));
11369   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11370 
11371   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11372   ins_encode %{
11373     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11374                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11375                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11376   %}
11377   ins_pipe( pipe_slow );
11378 %}
11379 
11380 //----------Overflow Math Instructions-----------------------------------------
11381 
11382 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11383 %{
11384   match(Set cr (OverflowAddI op1 op2));
11385   effect(DEF cr, USE_KILL op1, USE op2);
11386 
11387   format %{ "addl    $op1, $op2\t# overflow check int" %}
11388 
11389   ins_encode %{
11390     __ addl($op1$$Register, $op2$$Register);
11391   %}
11392   ins_pipe(ialu_reg_reg);
11393 %}
11394 
11395 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11396 %{
11397   match(Set cr (OverflowAddI op1 op2));
11398   effect(DEF cr, USE_KILL op1, USE op2);
11399 
11400   format %{ "addl    $op1, $op2\t# overflow check int" %}
11401 
11402   ins_encode %{
11403     __ addl($op1$$Register, $op2$$constant);
11404   %}
11405   ins_pipe(ialu_reg_reg);
11406 %}
11407 
11408 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11409 %{
11410   match(Set cr (OverflowAddL op1 op2));
11411   effect(DEF cr, USE_KILL op1, USE op2);
11412 
11413   format %{ "addq    $op1, $op2\t# overflow check long" %}
11414   ins_encode %{
11415     __ addq($op1$$Register, $op2$$Register);
11416   %}
11417   ins_pipe(ialu_reg_reg);
11418 %}
11419 
11420 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11421 %{
11422   match(Set cr (OverflowAddL op1 op2));
11423   effect(DEF cr, USE_KILL op1, USE op2);
11424 
11425   format %{ "addq    $op1, $op2\t# overflow check long" %}
11426   ins_encode %{
11427     __ addq($op1$$Register, $op2$$constant);
11428   %}
11429   ins_pipe(ialu_reg_reg);
11430 %}
11431 
11432 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11433 %{
11434   match(Set cr (OverflowSubI op1 op2));
11435 
11436   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11437   ins_encode %{
11438     __ cmpl($op1$$Register, $op2$$Register);
11439   %}
11440   ins_pipe(ialu_reg_reg);
11441 %}
11442 
11443 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11444 %{
11445   match(Set cr (OverflowSubI op1 op2));
11446 
11447   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11448   ins_encode %{
11449     __ cmpl($op1$$Register, $op2$$constant);
11450   %}
11451   ins_pipe(ialu_reg_reg);
11452 %}
11453 
11454 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11455 %{
11456   match(Set cr (OverflowSubL op1 op2));
11457 
11458   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11459   ins_encode %{
11460     __ cmpq($op1$$Register, $op2$$Register);
11461   %}
11462   ins_pipe(ialu_reg_reg);
11463 %}
11464 
11465 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11466 %{
11467   match(Set cr (OverflowSubL op1 op2));
11468 
11469   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11470   ins_encode %{
11471     __ cmpq($op1$$Register, $op2$$constant);
11472   %}
11473   ins_pipe(ialu_reg_reg);
11474 %}
11475 
11476 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11477 %{
11478   match(Set cr (OverflowSubI zero op2));
11479   effect(DEF cr, USE_KILL op2);
11480 
11481   format %{ "negl    $op2\t# overflow check int" %}
11482   ins_encode %{
11483     __ negl($op2$$Register);
11484   %}
11485   ins_pipe(ialu_reg_reg);
11486 %}
11487 
11488 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11489 %{
11490   match(Set cr (OverflowSubL zero op2));
11491   effect(DEF cr, USE_KILL op2);
11492 
11493   format %{ "negq    $op2\t# overflow check long" %}
11494   ins_encode %{
11495     __ negq($op2$$Register);
11496   %}
11497   ins_pipe(ialu_reg_reg);
11498 %}
11499 
11500 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11501 %{
11502   match(Set cr (OverflowMulI op1 op2));
11503   effect(DEF cr, USE_KILL op1, USE op2);
11504 
11505   format %{ "imull    $op1, $op2\t# overflow check int" %}
11506   ins_encode %{
11507     __ imull($op1$$Register, $op2$$Register);
11508   %}
11509   ins_pipe(ialu_reg_reg_alu0);
11510 %}
11511 
11512 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11513 %{
11514   match(Set cr (OverflowMulI op1 op2));
11515   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11516 
11517   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11518   ins_encode %{
11519     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11520   %}
11521   ins_pipe(ialu_reg_reg_alu0);
11522 %}
11523 
11524 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11525 %{
11526   match(Set cr (OverflowMulL op1 op2));
11527   effect(DEF cr, USE_KILL op1, USE op2);
11528 
11529   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11530   ins_encode %{
11531     __ imulq($op1$$Register, $op2$$Register);
11532   %}
11533   ins_pipe(ialu_reg_reg_alu0);
11534 %}
11535 
11536 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11537 %{
11538   match(Set cr (OverflowMulL op1 op2));
11539   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11540 
11541   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11542   ins_encode %{
11543     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11544   %}
11545   ins_pipe(ialu_reg_reg_alu0);
11546 %}
11547 
11548 
11549 //----------Control Flow Instructions------------------------------------------
11550 // Signed compare Instructions
11551 
11552 // XXX more variants!!
11553 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11554 %{
11555   match(Set cr (CmpI op1 op2));
11556   effect(DEF cr, USE op1, USE op2);
11557 
11558   format %{ "cmpl    $op1, $op2" %}
11559   ins_encode %{
11560     __ cmpl($op1$$Register, $op2$$Register);
11561   %}
11562   ins_pipe(ialu_cr_reg_reg);
11563 %}
11564 
11565 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11566 %{
11567   match(Set cr (CmpI op1 op2));
11568 
11569   format %{ "cmpl    $op1, $op2" %}
11570   ins_encode %{
11571     __ cmpl($op1$$Register, $op2$$constant);
11572   %}
11573   ins_pipe(ialu_cr_reg_imm);
11574 %}
11575 
11576 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11577 %{
11578   match(Set cr (CmpI op1 (LoadI op2)));
11579 
11580   ins_cost(500); // XXX
11581   format %{ "cmpl    $op1, $op2" %}
11582   ins_encode %{
11583     __ cmpl($op1$$Register, $op2$$Address);
11584   %}
11585   ins_pipe(ialu_cr_reg_mem);
11586 %}
11587 
11588 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11589 %{
11590   match(Set cr (CmpI src zero));
11591 
11592   format %{ "testl   $src, $src" %}
11593   ins_encode %{
11594     __ testl($src$$Register, $src$$Register);
11595   %}
11596   ins_pipe(ialu_cr_reg_imm);
11597 %}
11598 
11599 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11600 %{
11601   match(Set cr (CmpI (AndI src con) zero));
11602 
11603   format %{ "testl   $src, $con" %}
11604   ins_encode %{
11605     __ testl($src$$Register, $con$$constant);
11606   %}
11607   ins_pipe(ialu_cr_reg_imm);
11608 %}
11609 
11610 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11611 %{
11612   match(Set cr (CmpI (AndI src1 src2) zero));
11613 
11614   format %{ "testl   $src1, $src2" %}
11615   ins_encode %{
11616     __ testl($src1$$Register, $src2$$Register);
11617   %}
11618   ins_pipe(ialu_cr_reg_imm);
11619 %}
11620 
11621 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11622 %{
11623   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11624 
11625   format %{ "testl   $src, $mem" %}
11626   ins_encode %{
11627     __ testl($src$$Register, $mem$$Address);
11628   %}
11629   ins_pipe(ialu_cr_reg_mem);
11630 %}
11631 
11632 // Unsigned compare Instructions; really, same as signed except they
11633 // produce an rFlagsRegU instead of rFlagsReg.
11634 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11635 %{
11636   match(Set cr (CmpU op1 op2));
11637 
11638   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11639   ins_encode %{
11640     __ cmpl($op1$$Register, $op2$$Register);
11641   %}
11642   ins_pipe(ialu_cr_reg_reg);
11643 %}
11644 
11645 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11646 %{
11647   match(Set cr (CmpU op1 op2));
11648 
11649   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11650   ins_encode %{
11651     __ cmpl($op1$$Register, $op2$$constant);
11652   %}
11653   ins_pipe(ialu_cr_reg_imm);
11654 %}
11655 
11656 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11657 %{
11658   match(Set cr (CmpU op1 (LoadI op2)));
11659 
11660   ins_cost(500); // XXX
11661   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11662   ins_encode %{
11663     __ cmpl($op1$$Register, $op2$$Address);
11664   %}
11665   ins_pipe(ialu_cr_reg_mem);
11666 %}
11667 
11668 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11669 %{
11670   match(Set cr (CmpU src zero));
11671 
11672   format %{ "testl   $src, $src\t# unsigned" %}
11673   ins_encode %{
11674     __ testl($src$$Register, $src$$Register);
11675   %}
11676   ins_pipe(ialu_cr_reg_imm);
11677 %}
11678 
11679 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11680 %{
11681   match(Set cr (CmpP op1 op2));
11682 
11683   format %{ "cmpq    $op1, $op2\t# ptr" %}
11684   ins_encode %{
11685     __ cmpq($op1$$Register, $op2$$Register);
11686   %}
11687   ins_pipe(ialu_cr_reg_reg);
11688 %}
11689 
11690 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11691 %{
11692   match(Set cr (CmpP op1 (LoadP op2)));
11693   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11694 
11695   ins_cost(500); // XXX
11696   format %{ "cmpq    $op1, $op2\t# ptr" %}
11697   ins_encode %{
11698     __ cmpq($op1$$Register, $op2$$Address);
11699   %}
11700   ins_pipe(ialu_cr_reg_mem);
11701 %}
11702 
11703 // XXX this is generalized by compP_rReg_mem???
11704 // Compare raw pointer (used in out-of-heap check).
11705 // Only works because non-oop pointers must be raw pointers
11706 // and raw pointers have no anti-dependencies.
11707 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11708 %{
11709   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11710             n->in(2)->as_Load()->barrier_data() == 0);
11711   match(Set cr (CmpP op1 (LoadP op2)));
11712 
11713   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11714   ins_encode %{
11715     __ cmpq($op1$$Register, $op2$$Address);
11716   %}
11717   ins_pipe(ialu_cr_reg_mem);
11718 %}
11719 
11720 // This will generate a signed flags result. This should be OK since
11721 // any compare to a zero should be eq/neq.
11722 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11723 %{
11724   match(Set cr (CmpP src zero));
11725 
11726   format %{ "testq   $src, $src\t# ptr" %}
11727   ins_encode %{
11728     __ testq($src$$Register, $src$$Register);
11729   %}
11730   ins_pipe(ialu_cr_reg_imm);
11731 %}
11732 
11733 // This will generate a signed flags result. This should be OK since
11734 // any compare to a zero should be eq/neq.
11735 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11736 %{
11737   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
11738             n->in(1)->as_Load()->barrier_data() == 0);
11739   match(Set cr (CmpP (LoadP op) zero));
11740 
11741   ins_cost(500); // XXX
11742   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11743   ins_encode %{
11744     __ testq($op$$Address, 0xFFFFFFFF);
11745   %}
11746   ins_pipe(ialu_cr_reg_imm);
11747 %}
11748 
11749 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11750 %{
11751   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
11752             n->in(1)->as_Load()->barrier_data() == 0);
11753   match(Set cr (CmpP (LoadP mem) zero));
11754 
11755   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11756   ins_encode %{
11757     __ cmpq(r12, $mem$$Address);
11758   %}
11759   ins_pipe(ialu_cr_reg_mem);
11760 %}
11761 
11762 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11763 %{
11764   match(Set cr (CmpN op1 op2));
11765 
11766   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11767   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11768   ins_pipe(ialu_cr_reg_reg);
11769 %}
11770 
11771 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11772 %{
11773   match(Set cr (CmpN src (LoadN mem)));
11774 
11775   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11776   ins_encode %{
11777     __ cmpl($src$$Register, $mem$$Address);
11778   %}
11779   ins_pipe(ialu_cr_reg_mem);
11780 %}
11781 
11782 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11783   match(Set cr (CmpN op1 op2));
11784 
11785   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11786   ins_encode %{
11787     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11788   %}
11789   ins_pipe(ialu_cr_reg_imm);
11790 %}
11791 
11792 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11793 %{
11794   match(Set cr (CmpN src (LoadN mem)));
11795 
11796   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11797   ins_encode %{
11798     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11799   %}
11800   ins_pipe(ialu_cr_reg_mem);
11801 %}
11802 
11803 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11804   match(Set cr (CmpN op1 op2));
11805 
11806   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11807   ins_encode %{
11808     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11809   %}
11810   ins_pipe(ialu_cr_reg_imm);
11811 %}
11812 
11813 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11814 %{
11815   match(Set cr (CmpN src (LoadNKlass mem)));
11816 
11817   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11818   ins_encode %{
11819     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11820   %}
11821   ins_pipe(ialu_cr_reg_mem);
11822 %}
11823 
11824 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11825   match(Set cr (CmpN src zero));
11826 
11827   format %{ "testl   $src, $src\t# compressed ptr" %}
11828   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11829   ins_pipe(ialu_cr_reg_imm);
11830 %}
11831 
11832 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11833 %{
11834   predicate(CompressedOops::base() != NULL);
11835   match(Set cr (CmpN (LoadN mem) zero));
11836 
11837   ins_cost(500); // XXX
11838   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11839   ins_encode %{
11840     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11841   %}
11842   ins_pipe(ialu_cr_reg_mem);
11843 %}
11844 
11845 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11846 %{
11847   predicate(CompressedOops::base() == NULL);
11848   match(Set cr (CmpN (LoadN mem) zero));
11849 
11850   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11851   ins_encode %{
11852     __ cmpl(r12, $mem$$Address);
11853   %}
11854   ins_pipe(ialu_cr_reg_mem);
11855 %}
11856 
11857 // Yanked all unsigned pointer compare operations.
11858 // Pointer compares are done with CmpP which is already unsigned.
11859 
11860 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11861 %{
11862   match(Set cr (CmpL op1 op2));
11863 
11864   format %{ "cmpq    $op1, $op2" %}
11865   ins_encode %{
11866     __ cmpq($op1$$Register, $op2$$Register);
11867   %}
11868   ins_pipe(ialu_cr_reg_reg);
11869 %}
11870 
11871 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11872 %{
11873   match(Set cr (CmpL op1 op2));
11874 
11875   format %{ "cmpq    $op1, $op2" %}
11876   ins_encode %{
11877     __ cmpq($op1$$Register, $op2$$constant);
11878   %}
11879   ins_pipe(ialu_cr_reg_imm);
11880 %}
11881 
11882 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11883 %{
11884   match(Set cr (CmpL op1 (LoadL op2)));
11885 
11886   format %{ "cmpq    $op1, $op2" %}
11887   ins_encode %{
11888     __ cmpq($op1$$Register, $op2$$Address);
11889   %}
11890   ins_pipe(ialu_cr_reg_mem);
11891 %}
11892 
11893 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11894 %{
11895   match(Set cr (CmpL src zero));
11896 
11897   format %{ "testq   $src, $src" %}
11898   ins_encode %{
11899     __ testq($src$$Register, $src$$Register);
11900   %}
11901   ins_pipe(ialu_cr_reg_imm);
11902 %}
11903 
11904 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11905 %{
11906   match(Set cr (CmpL (AndL src con) zero));
11907 
11908   format %{ "testq   $src, $con\t# long" %}
11909   ins_encode %{
11910     __ testq($src$$Register, $con$$constant);
11911   %}
11912   ins_pipe(ialu_cr_reg_imm);
11913 %}
11914 
11915 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
11916 %{
11917   match(Set cr (CmpL (AndL src1 src2) zero));
11918 
11919   format %{ "testq   $src1, $src2\t# long" %}
11920   ins_encode %{
11921     __ testq($src1$$Register, $src2$$Register);
11922   %}
11923   ins_pipe(ialu_cr_reg_imm);
11924 %}
11925 
11926 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11927 %{
11928   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11929 
11930   format %{ "testq   $src, $mem" %}
11931   ins_encode %{
11932     __ testq($src$$Register, $mem$$Address);
11933   %}
11934   ins_pipe(ialu_cr_reg_mem);
11935 %}
11936 
11937 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11938 %{
11939   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11940 
11941   format %{ "testq   $src, $mem" %}
11942   ins_encode %{
11943     __ testq($src$$Register, $mem$$Address);
11944   %}
11945   ins_pipe(ialu_cr_reg_mem);
11946 %}
11947 
11948 // Manifest a CmpU result in an integer register.  Very painful.
11949 // This is the test to avoid.
11950 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
11951 %{
11952   match(Set dst (CmpU3 src1 src2));
11953   effect(KILL flags);
11954 
11955   ins_cost(275); // XXX
11956   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
11957             "movl    $dst, -1\n\t"
11958             "jb,u    done\n\t"
11959             "setne   $dst\n\t"
11960             "movzbl  $dst, $dst\n\t"
11961     "done:" %}
11962   ins_encode %{
11963     Label done;
11964     __ cmpl($src1$$Register, $src2$$Register);
11965     __ movl($dst$$Register, -1);
11966     __ jccb(Assembler::below, done);
11967     __ setb(Assembler::notZero, $dst$$Register);
11968     __ movzbl($dst$$Register, $dst$$Register);
11969     __ bind(done);
11970   %}
11971   ins_pipe(pipe_slow);
11972 %}
11973 
11974 // Manifest a CmpL result in an integer register.  Very painful.
11975 // This is the test to avoid.
11976 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11977 %{
11978   match(Set dst (CmpL3 src1 src2));
11979   effect(KILL flags);
11980 
11981   ins_cost(275); // XXX
11982   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11983             "movl    $dst, -1\n\t"
11984             "jl,s    done\n\t"
11985             "setne   $dst\n\t"
11986             "movzbl  $dst, $dst\n\t"
11987     "done:" %}
11988   ins_encode %{
11989     Label done;
11990     __ cmpq($src1$$Register, $src2$$Register);
11991     __ movl($dst$$Register, -1);
11992     __ jccb(Assembler::less, done);
11993     __ setb(Assembler::notZero, $dst$$Register);
11994     __ movzbl($dst$$Register, $dst$$Register);
11995     __ bind(done);
11996   %}
11997   ins_pipe(pipe_slow);
11998 %}
11999 
12000 // Manifest a CmpUL result in an integer register.  Very painful.
12001 // This is the test to avoid.
12002 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12003 %{
12004   match(Set dst (CmpUL3 src1 src2));
12005   effect(KILL flags);
12006 
12007   ins_cost(275); // XXX
12008   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12009             "movl    $dst, -1\n\t"
12010             "jb,u    done\n\t"
12011             "setne   $dst\n\t"
12012             "movzbl  $dst, $dst\n\t"
12013     "done:" %}
12014   ins_encode %{
12015     Label done;
12016     __ cmpq($src1$$Register, $src2$$Register);
12017     __ movl($dst$$Register, -1);
12018     __ jccb(Assembler::below, done);
12019     __ setb(Assembler::notZero, $dst$$Register);
12020     __ movzbl($dst$$Register, $dst$$Register);
12021     __ bind(done);
12022   %}
12023   ins_pipe(pipe_slow);
12024 %}
12025 
12026 // Unsigned long compare Instructions; really, same as signed long except they
12027 // produce an rFlagsRegU instead of rFlagsReg.
12028 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12029 %{
12030   match(Set cr (CmpUL op1 op2));
12031 
12032   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12033   ins_encode %{
12034     __ cmpq($op1$$Register, $op2$$Register);
12035   %}
12036   ins_pipe(ialu_cr_reg_reg);
12037 %}
12038 
12039 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12040 %{
12041   match(Set cr (CmpUL op1 op2));
12042 
12043   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12044   ins_encode %{
12045     __ cmpq($op1$$Register, $op2$$constant);
12046   %}
12047   ins_pipe(ialu_cr_reg_imm);
12048 %}
12049 
12050 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12051 %{
12052   match(Set cr (CmpUL op1 (LoadL op2)));
12053 
12054   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12055   ins_encode %{
12056     __ cmpq($op1$$Register, $op2$$Address);
12057   %}
12058   ins_pipe(ialu_cr_reg_mem);
12059 %}
12060 
12061 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12062 %{
12063   match(Set cr (CmpUL src zero));
12064 
12065   format %{ "testq   $src, $src\t# unsigned" %}
12066   ins_encode %{
12067     __ testq($src$$Register, $src$$Register);
12068   %}
12069   ins_pipe(ialu_cr_reg_imm);
12070 %}
12071 
12072 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12073 %{
12074   match(Set cr (CmpI (LoadB mem) imm));
12075 
12076   ins_cost(125);
12077   format %{ "cmpb    $mem, $imm" %}
12078   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12079   ins_pipe(ialu_cr_reg_mem);
12080 %}
12081 
12082 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12083 %{
12084   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12085 
12086   ins_cost(125);
12087   format %{ "testb   $mem, $imm\t# ubyte" %}
12088   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12089   ins_pipe(ialu_cr_reg_mem);
12090 %}
12091 
12092 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12093 %{
12094   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12095 
12096   ins_cost(125);
12097   format %{ "testb   $mem, $imm\t# byte" %}
12098   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12099   ins_pipe(ialu_cr_reg_mem);
12100 %}
12101 
12102 //----------Max and Min--------------------------------------------------------
12103 // Min Instructions
12104 
12105 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12106 %{
12107   effect(USE_DEF dst, USE src, USE cr);
12108 
12109   format %{ "cmovlgt $dst, $src\t# min" %}
12110   ins_encode %{
12111     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12112   %}
12113   ins_pipe(pipe_cmov_reg);
12114 %}
12115 
12116 
12117 instruct minI_rReg(rRegI dst, rRegI src)
12118 %{
12119   match(Set dst (MinI dst src));
12120 
12121   ins_cost(200);
12122   expand %{
12123     rFlagsReg cr;
12124     compI_rReg(cr, dst, src);
12125     cmovI_reg_g(dst, src, cr);
12126   %}
12127 %}
12128 
12129 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12130 %{
12131   effect(USE_DEF dst, USE src, USE cr);
12132 
12133   format %{ "cmovllt $dst, $src\t# max" %}
12134   ins_encode %{
12135     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12136   %}
12137   ins_pipe(pipe_cmov_reg);
12138 %}
12139 
12140 
12141 instruct maxI_rReg(rRegI dst, rRegI src)
12142 %{
12143   match(Set dst (MaxI dst src));
12144 
12145   ins_cost(200);
12146   expand %{
12147     rFlagsReg cr;
12148     compI_rReg(cr, dst, src);
12149     cmovI_reg_l(dst, src, cr);
12150   %}
12151 %}
12152 
12153 // ============================================================================
12154 // Branch Instructions
12155 
12156 // Jump Direct - Label defines a relative address from JMP+1
12157 instruct jmpDir(label labl)
12158 %{
12159   match(Goto);
12160   effect(USE labl);
12161 
12162   ins_cost(300);
12163   format %{ "jmp     $labl" %}
12164   size(5);
12165   ins_encode %{
12166     Label* L = $labl$$label;
12167     __ jmp(*L, false); // Always long jump
12168   %}
12169   ins_pipe(pipe_jmp);
12170 %}
12171 
12172 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12173 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12174 %{
12175   match(If cop cr);
12176   effect(USE labl);
12177 
12178   ins_cost(300);
12179   format %{ "j$cop     $labl" %}
12180   size(6);
12181   ins_encode %{
12182     Label* L = $labl$$label;
12183     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12184   %}
12185   ins_pipe(pipe_jcc);
12186 %}
12187 
12188 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12189 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12190 %{
12191   match(CountedLoopEnd cop cr);
12192   effect(USE labl);
12193 
12194   ins_cost(300);
12195   format %{ "j$cop     $labl\t# loop end" %}
12196   size(6);
12197   ins_encode %{
12198     Label* L = $labl$$label;
12199     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12200   %}
12201   ins_pipe(pipe_jcc);
12202 %}
12203 
12204 // Jump Direct Conditional - using unsigned comparison
12205 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12206   match(If cop cmp);
12207   effect(USE labl);
12208 
12209   ins_cost(300);
12210   format %{ "j$cop,u   $labl" %}
12211   size(6);
12212   ins_encode %{
12213     Label* L = $labl$$label;
12214     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12215   %}
12216   ins_pipe(pipe_jcc);
12217 %}
12218 
12219 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12220   match(If cop cmp);
12221   effect(USE labl);
12222 
12223   ins_cost(200);
12224   format %{ "j$cop,u   $labl" %}
12225   size(6);
12226   ins_encode %{
12227     Label* L = $labl$$label;
12228     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12229   %}
12230   ins_pipe(pipe_jcc);
12231 %}
12232 
12233 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12234   match(If cop cmp);
12235   effect(USE labl);
12236 
12237   ins_cost(200);
12238   format %{ $$template
12239     if ($cop$$cmpcode == Assembler::notEqual) {
12240       $$emit$$"jp,u    $labl\n\t"
12241       $$emit$$"j$cop,u   $labl"
12242     } else {
12243       $$emit$$"jp,u    done\n\t"
12244       $$emit$$"j$cop,u   $labl\n\t"
12245       $$emit$$"done:"
12246     }
12247   %}
12248   ins_encode %{
12249     Label* l = $labl$$label;
12250     if ($cop$$cmpcode == Assembler::notEqual) {
12251       __ jcc(Assembler::parity, *l, false);
12252       __ jcc(Assembler::notEqual, *l, false);
12253     } else if ($cop$$cmpcode == Assembler::equal) {
12254       Label done;
12255       __ jccb(Assembler::parity, done);
12256       __ jcc(Assembler::equal, *l, false);
12257       __ bind(done);
12258     } else {
12259        ShouldNotReachHere();
12260     }
12261   %}
12262   ins_pipe(pipe_jcc);
12263 %}
12264 
12265 // ============================================================================
12266 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12267 // superklass array for an instance of the superklass.  Set a hidden
12268 // internal cache on a hit (cache is checked with exposed code in
12269 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12270 // encoding ALSO sets flags.
12271 
12272 instruct partialSubtypeCheck(rdi_RegP result,
12273                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12274                              rFlagsReg cr)
12275 %{
12276   match(Set result (PartialSubtypeCheck sub super));
12277   effect(KILL rcx, KILL cr);
12278 
12279   ins_cost(1100);  // slightly larger than the next version
12280   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12281             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12282             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12283             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12284             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12285             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12286             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12287     "miss:\t" %}
12288 
12289   opcode(0x1); // Force a XOR of RDI
12290   ins_encode(enc_PartialSubtypeCheck());
12291   ins_pipe(pipe_slow);
12292 %}
12293 
12294 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12295                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12296                                      immP0 zero,
12297                                      rdi_RegP result)
12298 %{
12299   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12300   effect(KILL rcx, KILL result);
12301 
12302   ins_cost(1000);
12303   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12304             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12305             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12306             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12307             "jne,s   miss\t\t# Missed: flags nz\n\t"
12308             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12309     "miss:\t" %}
12310 
12311   opcode(0x0); // No need to XOR RDI
12312   ins_encode(enc_PartialSubtypeCheck());
12313   ins_pipe(pipe_slow);
12314 %}
12315 
12316 // ============================================================================
12317 // Branch Instructions -- short offset versions
12318 //
12319 // These instructions are used to replace jumps of a long offset (the default
12320 // match) with jumps of a shorter offset.  These instructions are all tagged
12321 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12322 // match rules in general matching.  Instead, the ADLC generates a conversion
12323 // method in the MachNode which can be used to do in-place replacement of the
12324 // long variant with the shorter variant.  The compiler will determine if a
12325 // branch can be taken by the is_short_branch_offset() predicate in the machine
12326 // specific code section of the file.
12327 
12328 // Jump Direct - Label defines a relative address from JMP+1
12329 instruct jmpDir_short(label labl) %{
12330   match(Goto);
12331   effect(USE labl);
12332 
12333   ins_cost(300);
12334   format %{ "jmp,s   $labl" %}
12335   size(2);
12336   ins_encode %{
12337     Label* L = $labl$$label;
12338     __ jmpb(*L);
12339   %}
12340   ins_pipe(pipe_jmp);
12341   ins_short_branch(1);
12342 %}
12343 
12344 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12345 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12346   match(If cop cr);
12347   effect(USE labl);
12348 
12349   ins_cost(300);
12350   format %{ "j$cop,s   $labl" %}
12351   size(2);
12352   ins_encode %{
12353     Label* L = $labl$$label;
12354     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12355   %}
12356   ins_pipe(pipe_jcc);
12357   ins_short_branch(1);
12358 %}
12359 
12360 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12361 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12362   match(CountedLoopEnd cop cr);
12363   effect(USE labl);
12364 
12365   ins_cost(300);
12366   format %{ "j$cop,s   $labl\t# loop end" %}
12367   size(2);
12368   ins_encode %{
12369     Label* L = $labl$$label;
12370     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12371   %}
12372   ins_pipe(pipe_jcc);
12373   ins_short_branch(1);
12374 %}
12375 
12376 // Jump Direct Conditional - using unsigned comparison
12377 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12378   match(If cop cmp);
12379   effect(USE labl);
12380 
12381   ins_cost(300);
12382   format %{ "j$cop,us  $labl" %}
12383   size(2);
12384   ins_encode %{
12385     Label* L = $labl$$label;
12386     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12387   %}
12388   ins_pipe(pipe_jcc);
12389   ins_short_branch(1);
12390 %}
12391 
12392 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12393   match(If cop cmp);
12394   effect(USE labl);
12395 
12396   ins_cost(300);
12397   format %{ "j$cop,us  $labl" %}
12398   size(2);
12399   ins_encode %{
12400     Label* L = $labl$$label;
12401     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12402   %}
12403   ins_pipe(pipe_jcc);
12404   ins_short_branch(1);
12405 %}
12406 
12407 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12408   match(If cop cmp);
12409   effect(USE labl);
12410 
12411   ins_cost(300);
12412   format %{ $$template
12413     if ($cop$$cmpcode == Assembler::notEqual) {
12414       $$emit$$"jp,u,s  $labl\n\t"
12415       $$emit$$"j$cop,u,s  $labl"
12416     } else {
12417       $$emit$$"jp,u,s  done\n\t"
12418       $$emit$$"j$cop,u,s  $labl\n\t"
12419       $$emit$$"done:"
12420     }
12421   %}
12422   size(4);
12423   ins_encode %{
12424     Label* l = $labl$$label;
12425     if ($cop$$cmpcode == Assembler::notEqual) {
12426       __ jccb(Assembler::parity, *l);
12427       __ jccb(Assembler::notEqual, *l);
12428     } else if ($cop$$cmpcode == Assembler::equal) {
12429       Label done;
12430       __ jccb(Assembler::parity, done);
12431       __ jccb(Assembler::equal, *l);
12432       __ bind(done);
12433     } else {
12434        ShouldNotReachHere();
12435     }
12436   %}
12437   ins_pipe(pipe_jcc);
12438   ins_short_branch(1);
12439 %}
12440 
12441 // ============================================================================
12442 // inlined locking and unlocking
12443 
12444 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12445   predicate(Compile::current()->use_rtm());
12446   match(Set cr (FastLock object box));
12447   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12448   ins_cost(300);
12449   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12450   ins_encode %{
12451     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12452                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
12453                  _rtm_counters, _stack_rtm_counters,
12454                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12455                  true, ra_->C->profile_rtm());
12456   %}
12457   ins_pipe(pipe_slow);
12458 %}
12459 
12460 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12461   predicate(!Compile::current()->use_rtm());
12462   match(Set cr (FastLock object box));
12463   effect(TEMP tmp, TEMP scr, USE_KILL box);
12464   ins_cost(300);
12465   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12466   ins_encode %{
12467     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12468                  $scr$$Register, noreg, noreg, r15_thread, nullptr, nullptr, nullptr, false, false);
12469   %}
12470   ins_pipe(pipe_slow);
12471 %}
12472 
12473 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12474   match(Set cr (FastUnlock object box));
12475   effect(TEMP tmp, USE_KILL box);
12476   ins_cost(300);
12477   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12478   ins_encode %{
12479     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12480   %}
12481   ins_pipe(pipe_slow);
12482 %}
12483 
12484 
12485 // ============================================================================
12486 // Safepoint Instructions
12487 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12488 %{
12489   match(SafePoint poll);
12490   effect(KILL cr, USE poll);
12491 
12492   format %{ "testl   rax, [$poll]\t"
12493             "# Safepoint: poll for GC" %}
12494   ins_cost(125);
12495   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12496   ins_encode %{
12497     __ relocate(relocInfo::poll_type);
12498     address pre_pc = __ pc();
12499     __ testl(rax, Address($poll$$Register, 0));
12500     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12501   %}
12502   ins_pipe(ialu_reg_mem);
12503 %}
12504 
12505 instruct mask_all_evexL(kReg dst, rRegL src) %{
12506   match(Set dst (MaskAll src));
12507   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12508   ins_encode %{
12509     int mask_len = Matcher::vector_length(this);
12510     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12511   %}
12512   ins_pipe( pipe_slow );
12513 %}
12514 
12515 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12516   predicate(Matcher::vector_length(n) > 32);
12517   match(Set dst (MaskAll src));
12518   effect(TEMP tmp);
12519   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12520   ins_encode %{
12521     int mask_len = Matcher::vector_length(this);
12522     __ movslq($tmp$$Register, $src$$Register);
12523     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12524   %}
12525   ins_pipe( pipe_slow );
12526 %}
12527 
12528 // ============================================================================
12529 // Procedure Call/Return Instructions
12530 // Call Java Static Instruction
12531 // Note: If this code changes, the corresponding ret_addr_offset() and
12532 //       compute_padding() functions will have to be adjusted.
12533 instruct CallStaticJavaDirect(method meth) %{
12534   match(CallStaticJava);
12535   effect(USE meth);
12536 
12537   ins_cost(300);
12538   format %{ "call,static " %}
12539   opcode(0xE8); /* E8 cd */
12540   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12541   ins_pipe(pipe_slow);
12542   ins_alignment(4);
12543 %}
12544 
12545 // Call Java Dynamic Instruction
12546 // Note: If this code changes, the corresponding ret_addr_offset() and
12547 //       compute_padding() functions will have to be adjusted.
12548 instruct CallDynamicJavaDirect(method meth)
12549 %{
12550   match(CallDynamicJava);
12551   effect(USE meth);
12552 
12553   ins_cost(300);
12554   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12555             "call,dynamic " %}
12556   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12557   ins_pipe(pipe_slow);
12558   ins_alignment(4);
12559 %}
12560 
12561 // Call Runtime Instruction
12562 instruct CallRuntimeDirect(method meth)
12563 %{
12564   match(CallRuntime);
12565   effect(USE meth);
12566 
12567   ins_cost(300);
12568   format %{ "call,runtime " %}
12569   ins_encode(clear_avx, Java_To_Runtime(meth));
12570   ins_pipe(pipe_slow);
12571 %}
12572 
12573 // Call runtime without safepoint
12574 instruct CallLeafDirect(method meth)
12575 %{
12576   match(CallLeaf);
12577   effect(USE meth);
12578 
12579   ins_cost(300);
12580   format %{ "call_leaf,runtime " %}
12581   ins_encode(clear_avx, Java_To_Runtime(meth));
12582   ins_pipe(pipe_slow);
12583 %}
12584 
12585 // Call runtime without safepoint and with vector arguments
12586 instruct CallLeafDirectVector(method meth)
12587 %{
12588   match(CallLeafVector);
12589   effect(USE meth);
12590 
12591   ins_cost(300);
12592   format %{ "call_leaf,vector " %}
12593   ins_encode(Java_To_Runtime(meth));
12594   ins_pipe(pipe_slow);
12595 %}
12596 
12597 // Call runtime without safepoint
12598 instruct CallLeafNoFPDirect(method meth)
12599 %{
12600   match(CallLeafNoFP);
12601   effect(USE meth);
12602 
12603   ins_cost(300);
12604   format %{ "call_leaf_nofp,runtime " %}
12605   ins_encode(clear_avx, Java_To_Runtime(meth));
12606   ins_pipe(pipe_slow);
12607 %}
12608 
12609 // Return Instruction
12610 // Remove the return address & jump to it.
12611 // Notice: We always emit a nop after a ret to make sure there is room
12612 // for safepoint patching
12613 instruct Ret()
12614 %{
12615   match(Return);
12616 
12617   format %{ "ret" %}
12618   ins_encode %{
12619     __ ret(0);
12620   %}
12621   ins_pipe(pipe_jmp);
12622 %}
12623 
12624 // Tail Call; Jump from runtime stub to Java code.
12625 // Also known as an 'interprocedural jump'.
12626 // Target of jump will eventually return to caller.
12627 // TailJump below removes the return address.
12628 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12629 %{
12630   match(TailCall jump_target method_ptr);
12631 
12632   ins_cost(300);
12633   format %{ "jmp     $jump_target\t# rbx holds method" %}
12634   ins_encode %{
12635     __ jmp($jump_target$$Register);
12636   %}
12637   ins_pipe(pipe_jmp);
12638 %}
12639 
12640 // Tail Jump; remove the return address; jump to target.
12641 // TailCall above leaves the return address around.
12642 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12643 %{
12644   match(TailJump jump_target ex_oop);
12645 
12646   ins_cost(300);
12647   format %{ "popq    rdx\t# pop return address\n\t"
12648             "jmp     $jump_target" %}
12649   ins_encode %{
12650     __ popq(as_Register(RDX_enc));
12651     __ jmp($jump_target$$Register);
12652   %}
12653   ins_pipe(pipe_jmp);
12654 %}
12655 
12656 // Create exception oop: created by stack-crawling runtime code.
12657 // Created exception is now available to this handler, and is setup
12658 // just prior to jumping to this handler.  No code emitted.
12659 instruct CreateException(rax_RegP ex_oop)
12660 %{
12661   match(Set ex_oop (CreateEx));
12662 
12663   size(0);
12664   // use the following format syntax
12665   format %{ "# exception oop is in rax; no code emitted" %}
12666   ins_encode();
12667   ins_pipe(empty);
12668 %}
12669 
12670 // Rethrow exception:
12671 // The exception oop will come in the first argument position.
12672 // Then JUMP (not call) to the rethrow stub code.
12673 instruct RethrowException()
12674 %{
12675   match(Rethrow);
12676 
12677   // use the following format syntax
12678   format %{ "jmp     rethrow_stub" %}
12679   ins_encode %{
12680     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12681   %}
12682   ins_pipe(pipe_jmp);
12683 %}
12684 
12685 // ============================================================================
12686 // This name is KNOWN by the ADLC and cannot be changed.
12687 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12688 // for this guy.
12689 instruct tlsLoadP(r15_RegP dst) %{
12690   match(Set dst (ThreadLocal));
12691   effect(DEF dst);
12692 
12693   size(0);
12694   format %{ "# TLS is in R15" %}
12695   ins_encode( /*empty encoding*/ );
12696   ins_pipe(ialu_reg_reg);
12697 %}
12698 
12699 
12700 //----------PEEPHOLE RULES-----------------------------------------------------
12701 // These must follow all instruction definitions as they use the names
12702 // defined in the instructions definitions.
12703 //
12704 // peeppredicate ( rule_predicate );
12705 // // the predicate unless which the peephole rule will be ignored
12706 //
12707 // peepmatch ( root_instr_name [preceding_instruction]* );
12708 //
12709 // peepprocedure ( procedure_name );
12710 // // provide a procedure name to perform the optimization, the procedure should
12711 // // reside in the architecture dependent peephole file, the method has the
12712 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12713 // // with the arguments being the basic block, the current node index inside the
12714 // // block, the register allocator, the functions upon invoked return a new node
12715 // // defined in peepreplace, and the rules of the nodes appearing in the
12716 // // corresponding peepmatch, the function return true if successful, else
12717 // // return false
12718 //
12719 // peepconstraint %{
12720 // (instruction_number.operand_name relational_op instruction_number.operand_name
12721 //  [, ...] );
12722 // // instruction numbers are zero-based using left to right order in peepmatch
12723 //
12724 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12725 // // provide an instruction_number.operand_name for each operand that appears
12726 // // in the replacement instruction's match rule
12727 //
12728 // ---------VM FLAGS---------------------------------------------------------
12729 //
12730 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12731 //
12732 // Each peephole rule is given an identifying number starting with zero and
12733 // increasing by one in the order seen by the parser.  An individual peephole
12734 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12735 // on the command-line.
12736 //
12737 // ---------CURRENT LIMITATIONS----------------------------------------------
12738 //
12739 // Only transformations inside a basic block (do we need more for peephole)
12740 //
12741 // ---------EXAMPLE----------------------------------------------------------
12742 //
12743 // // pertinent parts of existing instructions in architecture description
12744 // instruct movI(rRegI dst, rRegI src)
12745 // %{
12746 //   match(Set dst (CopyI src));
12747 // %}
12748 //
12749 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12750 // %{
12751 //   match(Set dst (AddI dst src));
12752 //   effect(KILL cr);
12753 // %}
12754 //
12755 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12756 // %{
12757 //   match(Set dst (AddI dst src));
12758 // %}
12759 //
12760 // 1. Simple replacement
12761 // - Only match adjacent instructions in same basic block
12762 // - Only equality constraints
12763 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
12764 // - Only one replacement instruction
12765 //
12766 // // Change (inc mov) to lea
12767 // peephole %{
12768 //   // lea should only be emitted when beneficial
12769 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12770 //   // increment preceded by register-register move
12771 //   peepmatch ( incI_rReg movI );
12772 //   // require that the destination register of the increment
12773 //   // match the destination register of the move
12774 //   peepconstraint ( 0.dst == 1.dst );
12775 //   // construct a replacement instruction that sets
12776 //   // the destination to ( move's source register + one )
12777 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12778 // %}
12779 //
12780 // 2. Procedural replacement
12781 // - More flexible finding relevent nodes
12782 // - More flexible constraints
12783 // - More flexible transformations
12784 // - May utilise architecture-dependent API more effectively
12785 // - Currently only one replacement instruction due to adlc parsing capabilities
12786 //
12787 // // Change (inc mov) to lea
12788 // peephole %{
12789 //   // lea should only be emitted when beneficial
12790 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
12791 //   // the rule numbers of these nodes inside are passed into the function below
12792 //   peepmatch ( incI_rReg movI );
12793 //   // the method that takes the responsibility of transformation
12794 //   peepprocedure ( inc_mov_to_lea );
12795 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
12796 //   // node is passed into the function above
12797 //   peepreplace ( leaI_rReg_immI() );
12798 // %}
12799 
12800 // These instructions is not matched by the matcher but used by the peephole
12801 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
12802 %{
12803   predicate(false);
12804   match(Set dst (AddI src1 src2));
12805   format %{ "leal    $dst, [$src1 + $src2]" %}
12806   ins_encode %{
12807     Register dst = $dst$$Register;
12808     Register src1 = $src1$$Register;
12809     Register src2 = $src2$$Register;
12810     if (src1 != rbp && src1 != r13) {
12811       __ leal(dst, Address(src1, src2, Address::times_1));
12812     } else {
12813       assert(src2 != rbp && src2 != r13, "");
12814       __ leal(dst, Address(src2, src1, Address::times_1));
12815     }
12816   %}
12817   ins_pipe(ialu_reg_reg);
12818 %}
12819 
12820 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
12821 %{
12822   predicate(false);
12823   match(Set dst (AddI src1 src2));
12824   format %{ "leal    $dst, [$src1 + $src2]" %}
12825   ins_encode %{
12826     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
12827   %}
12828   ins_pipe(ialu_reg_reg);
12829 %}
12830 
12831 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
12832 %{
12833   predicate(false);
12834   match(Set dst (LShiftI src shift));
12835   format %{ "leal    $dst, [$src << $shift]" %}
12836   ins_encode %{
12837     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12838     Register src = $src$$Register;
12839     if (scale == Address::times_2 && src != rbp && src != r13) {
12840       __ leal($dst$$Register, Address(src, src, Address::times_1));
12841     } else {
12842       __ leal($dst$$Register, Address(noreg, src, scale));
12843     }
12844   %}
12845   ins_pipe(ialu_reg_reg);
12846 %}
12847 
12848 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
12849 %{
12850   predicate(false);
12851   match(Set dst (AddL src1 src2));
12852   format %{ "leaq    $dst, [$src1 + $src2]" %}
12853   ins_encode %{
12854     Register dst = $dst$$Register;
12855     Register src1 = $src1$$Register;
12856     Register src2 = $src2$$Register;
12857     if (src1 != rbp && src1 != r13) {
12858       __ leaq(dst, Address(src1, src2, Address::times_1));
12859     } else {
12860       assert(src2 != rbp && src2 != r13, "");
12861       __ leaq(dst, Address(src2, src1, Address::times_1));
12862     }
12863   %}
12864   ins_pipe(ialu_reg_reg);
12865 %}
12866 
12867 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
12868 %{
12869   predicate(false);
12870   match(Set dst (AddL src1 src2));
12871   format %{ "leaq    $dst, [$src1 + $src2]" %}
12872   ins_encode %{
12873     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
12874   %}
12875   ins_pipe(ialu_reg_reg);
12876 %}
12877 
12878 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
12879 %{
12880   predicate(false);
12881   match(Set dst (LShiftL src shift));
12882   format %{ "leaq    $dst, [$src << $shift]" %}
12883   ins_encode %{
12884     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
12885     Register src = $src$$Register;
12886     if (scale == Address::times_2 && src != rbp && src != r13) {
12887       __ leaq($dst$$Register, Address(src, src, Address::times_1));
12888     } else {
12889       __ leaq($dst$$Register, Address(noreg, src, scale));
12890     }
12891   %}
12892   ins_pipe(ialu_reg_reg);
12893 %}
12894 
12895 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
12896 // sal}) with lea instructions. The {add, sal} rules are beneficial in
12897 // processors with at least partial ALU support for lea
12898 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
12899 // beneficial for processors with full ALU support
12900 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
12901 
12902 peephole
12903 %{
12904   peeppredicate(VM_Version::supports_fast_2op_lea());
12905   peepmatch (addI_rReg);
12906   peepprocedure (lea_coalesce_reg);
12907   peepreplace (leaI_rReg_rReg_peep());
12908 %}
12909 
12910 peephole
12911 %{
12912   peeppredicate(VM_Version::supports_fast_2op_lea());
12913   peepmatch (addI_rReg_imm);
12914   peepprocedure (lea_coalesce_imm);
12915   peepreplace (leaI_rReg_immI_peep());
12916 %}
12917 
12918 peephole
12919 %{
12920   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12921                 VM_Version::is_intel_cascade_lake());
12922   peepmatch (incI_rReg);
12923   peepprocedure (lea_coalesce_imm);
12924   peepreplace (leaI_rReg_immI_peep());
12925 %}
12926 
12927 peephole
12928 %{
12929   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12930                 VM_Version::is_intel_cascade_lake());
12931   peepmatch (decI_rReg);
12932   peepprocedure (lea_coalesce_imm);
12933   peepreplace (leaI_rReg_immI_peep());
12934 %}
12935 
12936 peephole
12937 %{
12938   peeppredicate(VM_Version::supports_fast_2op_lea());
12939   peepmatch (salI_rReg_immI2);
12940   peepprocedure (lea_coalesce_imm);
12941   peepreplace (leaI_rReg_immI2_peep());
12942 %}
12943 
12944 peephole
12945 %{
12946   peeppredicate(VM_Version::supports_fast_2op_lea());
12947   peepmatch (addL_rReg);
12948   peepprocedure (lea_coalesce_reg);
12949   peepreplace (leaL_rReg_rReg_peep());
12950 %}
12951 
12952 peephole
12953 %{
12954   peeppredicate(VM_Version::supports_fast_2op_lea());
12955   peepmatch (addL_rReg_imm);
12956   peepprocedure (lea_coalesce_imm);
12957   peepreplace (leaL_rReg_immL32_peep());
12958 %}
12959 
12960 peephole
12961 %{
12962   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12963                 VM_Version::is_intel_cascade_lake());
12964   peepmatch (incL_rReg);
12965   peepprocedure (lea_coalesce_imm);
12966   peepreplace (leaL_rReg_immL32_peep());
12967 %}
12968 
12969 peephole
12970 %{
12971   peeppredicate(VM_Version::supports_fast_3op_lea() ||
12972                 VM_Version::is_intel_cascade_lake());
12973   peepmatch (decL_rReg);
12974   peepprocedure (lea_coalesce_imm);
12975   peepreplace (leaL_rReg_immL32_peep());
12976 %}
12977 
12978 peephole
12979 %{
12980   peeppredicate(VM_Version::supports_fast_2op_lea());
12981   peepmatch (salL_rReg_immI2);
12982   peepprocedure (lea_coalesce_imm);
12983   peepreplace (leaL_rReg_immI2_peep());
12984 %}
12985 
12986 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
12987 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
12988 
12989 //int variant
12990 peephole
12991 %{
12992   peepmatch (testI_reg);
12993   peepprocedure (test_may_remove);
12994 %}
12995 
12996 //long variant
12997 peephole
12998 %{
12999   peepmatch (testL_reg);
13000   peepprocedure (test_may_remove);
13001 %}
13002 
13003 
13004 //----------SMARTSPILL RULES---------------------------------------------------
13005 // These must follow all instruction definitions as they use the names
13006 // defined in the instructions definitions.