1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // archtecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all int registers (excluding RSP)
  245 reg_class int_reg %{
  246   return _INT_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RAX, RDX, and RSP)
  250 reg_class int_no_rax_rdx_reg %{
  251   return _INT_NO_RAX_RDX_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RCX and RSP)
  255 reg_class int_no_rcx_reg %{
  256   return _INT_NO_RCX_REG_mask;
  257 %}
  258 
  259 // Singleton class for RAX pointer register
  260 reg_class ptr_rax_reg(RAX, RAX_H);
  261 
  262 // Singleton class for RBX pointer register
  263 reg_class ptr_rbx_reg(RBX, RBX_H);
  264 
  265 // Singleton class for RSI pointer register
  266 reg_class ptr_rsi_reg(RSI, RSI_H);
  267 
  268 // Singleton class for RBP pointer register
  269 reg_class ptr_rbp_reg(RBP, RBP_H);
  270 
  271 // Singleton class for RDI pointer register
  272 reg_class ptr_rdi_reg(RDI, RDI_H);
  273 
  274 // Singleton class for stack pointer
  275 reg_class ptr_rsp_reg(RSP, RSP_H);
  276 
  277 // Singleton class for TLS pointer
  278 reg_class ptr_r15_reg(R15, R15_H);
  279 
  280 // Singleton class for RAX long register
  281 reg_class long_rax_reg(RAX, RAX_H);
  282 
  283 // Singleton class for RCX long register
  284 reg_class long_rcx_reg(RCX, RCX_H);
  285 
  286 // Singleton class for RDX long register
  287 reg_class long_rdx_reg(RDX, RDX_H);
  288 
  289 // Singleton class for RAX int register
  290 reg_class int_rax_reg(RAX);
  291 
  292 // Singleton class for RBX int register
  293 reg_class int_rbx_reg(RBX);
  294 
  295 // Singleton class for RCX int register
  296 reg_class int_rcx_reg(RCX);
  297 
  298 // Singleton class for RCX int register
  299 reg_class int_rdx_reg(RDX);
  300 
  301 // Singleton class for RCX int register
  302 reg_class int_rdi_reg(RDI);
  303 
  304 // Singleton class for instruction pointer
  305 // reg_class ip_reg(RIP);
  306 
  307 %}
  308 
  309 //----------SOURCE BLOCK-------------------------------------------------------
  310 // This is a block of C++ code which provides values, functions, and
  311 // definitions necessary in the rest of the architecture description
  312 source_hpp %{
  313 
  314 extern RegMask _ANY_REG_mask;
  315 extern RegMask _PTR_REG_mask;
  316 extern RegMask _PTR_REG_NO_RBP_mask;
  317 extern RegMask _PTR_NO_RAX_REG_mask;
  318 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  319 extern RegMask _LONG_REG_mask;
  320 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  321 extern RegMask _LONG_NO_RCX_REG_mask;
  322 extern RegMask _INT_REG_mask;
  323 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  324 extern RegMask _INT_NO_RCX_REG_mask;
  325 
  326 extern RegMask _STACK_OR_PTR_REG_mask;
  327 extern RegMask _STACK_OR_LONG_REG_mask;
  328 extern RegMask _STACK_OR_INT_REG_mask;
  329 
  330 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  331 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  332 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  333 
  334 %}
  335 
  336 source %{
  337 #define   RELOC_IMM64    Assembler::imm_operand
  338 #define   RELOC_DISP32   Assembler::disp32_operand
  339 
  340 #define __ _masm.
  341 
  342 RegMask _ANY_REG_mask;
  343 RegMask _PTR_REG_mask;
  344 RegMask _PTR_REG_NO_RBP_mask;
  345 RegMask _PTR_NO_RAX_REG_mask;
  346 RegMask _PTR_NO_RAX_RBX_REG_mask;
  347 RegMask _LONG_REG_mask;
  348 RegMask _LONG_NO_RAX_RDX_REG_mask;
  349 RegMask _LONG_NO_RCX_REG_mask;
  350 RegMask _INT_REG_mask;
  351 RegMask _INT_NO_RAX_RDX_REG_mask;
  352 RegMask _INT_NO_RCX_REG_mask;
  353 RegMask _STACK_OR_PTR_REG_mask;
  354 RegMask _STACK_OR_LONG_REG_mask;
  355 RegMask _STACK_OR_INT_REG_mask;
  356 
  357 static bool need_r12_heapbase() {
  358   return UseCompressedOops;
  359 }
  360 
  361 void reg_mask_init() {
  362   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  363   // We derive a number of subsets from it.
  364   _ANY_REG_mask = _ALL_REG_mask;
  365 
  366   if (PreserveFramePointer) {
  367     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  368     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  369   }
  370   if (need_r12_heapbase()) {
  371     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  372     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  373   }
  374 
  375   _PTR_REG_mask = _ANY_REG_mask;
  376   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  377   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  378   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  379   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  380 
  381   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  382   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  383 
  384   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  385   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  386   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  387 
  388   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  389   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  390   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  391 
  392   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  393   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  394   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  395 
  396   _LONG_REG_mask = _PTR_REG_mask;
  397   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  398   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  399 
  400   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  401   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  402   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  403   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  404   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  405 
  406   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  407   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  408   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  409 
  410   _INT_REG_mask = _ALL_INT_REG_mask;
  411   if (PreserveFramePointer) {
  412     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  413   }
  414   if (need_r12_heapbase()) {
  415     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  416   }
  417 
  418   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  419   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  420 
  421   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  422   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  423   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  424 
  425   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  426   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  427 
  428   if (Matcher::has_predicated_vectors()) {
  429     // Post-loop multi-versioning expects mask to be present in K1 register, till the time
  430     // its fixed, RA should not be allocting K1 register, this shall prevent any accidental
  431     // curruption of value held in K1 register.
  432     if (PostLoopMultiversioning) {
  433       const_cast<RegMask*>(&_VECTMASK_REG_mask)->Remove(OptoReg::as_OptoReg(k1->as_VMReg()));
  434       const_cast<RegMask*>(&_VECTMASK_REG_mask)->Remove(OptoReg::as_OptoReg(k1->as_VMReg()->next()));
  435     }
  436   }
  437 }
  438 
  439 static bool generate_vzeroupper(Compile* C) {
  440   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  441 }
  442 
  443 static int clear_avx_size() {
  444   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  445 }
  446 
  447 // !!!!! Special hack to get all types of calls to specify the byte offset
  448 //       from the start of the call to the point where the return address
  449 //       will point.
  450 int MachCallStaticJavaNode::ret_addr_offset()
  451 {
  452   int offset = 5; // 5 bytes from start of call to where return address points
  453   offset += clear_avx_size();
  454   return offset;
  455 }
  456 
  457 int MachCallDynamicJavaNode::ret_addr_offset()
  458 {
  459   int offset = 15; // 15 bytes from start of call to where return address points
  460   offset += clear_avx_size();
  461   return offset;
  462 }
  463 
  464 int MachCallRuntimeNode::ret_addr_offset() {
  465   int offset = 13; // movq r10,#addr; callq (r10)
  466   if (this->ideal_Opcode() != Op_CallLeafVector) {
  467     offset += clear_avx_size();
  468   }
  469   return offset;
  470 }
  471 
  472 int MachCallNativeNode::ret_addr_offset() {
  473   int offset = 13; // movq r10,#addr; callq (r10)
  474   offset += clear_avx_size();
  475   return offset;
  476 }
  477 //
  478 // Compute padding required for nodes which need alignment
  479 //
  480 
  481 // The address of the call instruction needs to be 4-byte aligned to
  482 // ensure that it does not span a cache line so that it can be patched.
  483 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  484 {
  485   current_offset += clear_avx_size(); // skip vzeroupper
  486   current_offset += 1; // skip call opcode byte
  487   return align_up(current_offset, alignment_required()) - current_offset;
  488 }
  489 
  490 // The address of the call instruction needs to be 4-byte aligned to
  491 // ensure that it does not span a cache line so that it can be patched.
  492 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  493 {
  494   current_offset += clear_avx_size(); // skip vzeroupper
  495   current_offset += 11; // skip movq instruction + call opcode byte
  496   return align_up(current_offset, alignment_required()) - current_offset;
  497 }
  498 
  499 // EMIT_RM()
  500 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  501   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  502   cbuf.insts()->emit_int8(c);
  503 }
  504 
  505 // EMIT_CC()
  506 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  507   unsigned char c = (unsigned char) (f1 | f2);
  508   cbuf.insts()->emit_int8(c);
  509 }
  510 
  511 // EMIT_OPCODE()
  512 void emit_opcode(CodeBuffer &cbuf, int code) {
  513   cbuf.insts()->emit_int8((unsigned char) code);
  514 }
  515 
  516 // EMIT_OPCODE() w/ relocation information
  517 void emit_opcode(CodeBuffer &cbuf,
  518                  int code, relocInfo::relocType reloc, int offset, int format)
  519 {
  520   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  521   emit_opcode(cbuf, code);
  522 }
  523 
  524 // EMIT_D8()
  525 void emit_d8(CodeBuffer &cbuf, int d8) {
  526   cbuf.insts()->emit_int8((unsigned char) d8);
  527 }
  528 
  529 // EMIT_D16()
  530 void emit_d16(CodeBuffer &cbuf, int d16) {
  531   cbuf.insts()->emit_int16(d16);
  532 }
  533 
  534 // EMIT_D32()
  535 void emit_d32(CodeBuffer &cbuf, int d32) {
  536   cbuf.insts()->emit_int32(d32);
  537 }
  538 
  539 // EMIT_D64()
  540 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  541   cbuf.insts()->emit_int64(d64);
  542 }
  543 
  544 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  545 void emit_d32_reloc(CodeBuffer& cbuf,
  546                     int d32,
  547                     relocInfo::relocType reloc,
  548                     int format)
  549 {
  550   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  551   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  552   cbuf.insts()->emit_int32(d32);
  553 }
  554 
  555 // emit 32 bit value and construct relocation entry from RelocationHolder
  556 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  557 #ifdef ASSERT
  558   if (rspec.reloc()->type() == relocInfo::oop_type &&
  559       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  560     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  561     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  562   }
  563 #endif
  564   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  565   cbuf.insts()->emit_int32(d32);
  566 }
  567 
  568 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  569   address next_ip = cbuf.insts_end() + 4;
  570   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  571                  external_word_Relocation::spec(addr),
  572                  RELOC_DISP32);
  573 }
  574 
  575 
  576 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  577 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  578   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  579   cbuf.insts()->emit_int64(d64);
  580 }
  581 
  582 // emit 64 bit value and construct relocation entry from RelocationHolder
  583 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  584 #ifdef ASSERT
  585   if (rspec.reloc()->type() == relocInfo::oop_type &&
  586       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  587     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  588     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  589   }
  590 #endif
  591   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  592   cbuf.insts()->emit_int64(d64);
  593 }
  594 
  595 // Access stack slot for load or store
  596 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  597 {
  598   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  599   if (-0x80 <= disp && disp < 0x80) {
  600     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  601     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  602     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  603   } else {
  604     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  605     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  606     emit_d32(cbuf, disp);     // Displacement // R/M byte
  607   }
  608 }
  609 
  610    // rRegI ereg, memory mem) %{    // emit_reg_mem
  611 void encode_RegMem(CodeBuffer &cbuf,
  612                    int reg,
  613                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  614 {
  615   assert(disp_reloc == relocInfo::none, "cannot have disp");
  616   int regenc = reg & 7;
  617   int baseenc = base & 7;
  618   int indexenc = index & 7;
  619 
  620   // There is no index & no scale, use form without SIB byte
  621   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  622     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  623     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  624       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  625     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  626       // If 8-bit displacement, mode 0x1
  627       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  628       emit_d8(cbuf, disp);
  629     } else {
  630       // If 32-bit displacement
  631       if (base == -1) { // Special flag for absolute address
  632         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  633         if (disp_reloc != relocInfo::none) {
  634           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  635         } else {
  636           emit_d32(cbuf, disp);
  637         }
  638       } else {
  639         // Normal base + offset
  640         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  641         if (disp_reloc != relocInfo::none) {
  642           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  643         } else {
  644           emit_d32(cbuf, disp);
  645         }
  646       }
  647     }
  648   } else {
  649     // Else, encode with the SIB byte
  650     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  651     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  652       // If no displacement
  653       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  654       emit_rm(cbuf, scale, indexenc, baseenc);
  655     } else {
  656       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  657         // If 8-bit displacement, mode 0x1
  658         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  659         emit_rm(cbuf, scale, indexenc, baseenc);
  660         emit_d8(cbuf, disp);
  661       } else {
  662         // If 32-bit displacement
  663         if (base == 0x04 ) {
  664           emit_rm(cbuf, 0x2, regenc, 0x4);
  665           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  666         } else {
  667           emit_rm(cbuf, 0x2, regenc, 0x4);
  668           emit_rm(cbuf, scale, indexenc, baseenc); // *
  669         }
  670         if (disp_reloc != relocInfo::none) {
  671           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  672         } else {
  673           emit_d32(cbuf, disp);
  674         }
  675       }
  676     }
  677   }
  678 }
  679 
  680 // This could be in MacroAssembler but it's fairly C2 specific
  681 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  682   Label exit;
  683   __ jccb(Assembler::noParity, exit);
  684   __ pushf();
  685   //
  686   // comiss/ucomiss instructions set ZF,PF,CF flags and
  687   // zero OF,AF,SF for NaN values.
  688   // Fixup flags by zeroing ZF,PF so that compare of NaN
  689   // values returns 'less than' result (CF is set).
  690   // Leave the rest of flags unchanged.
  691   //
  692   //    7 6 5 4 3 2 1 0
  693   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  694   //    0 0 1 0 1 0 1 1   (0x2B)
  695   //
  696   __ andq(Address(rsp, 0), 0xffffff2b);
  697   __ popf();
  698   __ bind(exit);
  699 }
  700 
  701 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  702   Label done;
  703   __ movl(dst, -1);
  704   __ jcc(Assembler::parity, done);
  705   __ jcc(Assembler::below, done);
  706   __ setb(Assembler::notEqual, dst);
  707   __ movzbl(dst, dst);
  708   __ bind(done);
  709 }
  710 
  711 // Math.min()    # Math.max()
  712 // --------------------------
  713 // ucomis[s/d]   #
  714 // ja   -> b     # a
  715 // jp   -> NaN   # NaN
  716 // jb   -> a     # b
  717 // je            #
  718 // |-jz -> a | b # a & b
  719 // |    -> a     #
  720 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  721                      XMMRegister a, XMMRegister b,
  722                      XMMRegister xmmt, Register rt,
  723                      bool min, bool single) {
  724 
  725   Label nan, zero, below, above, done;
  726 
  727   if (single)
  728     __ ucomiss(a, b);
  729   else
  730     __ ucomisd(a, b);
  731 
  732   if (dst->encoding() != (min ? b : a)->encoding())
  733     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  734   else
  735     __ jccb(Assembler::above, done);
  736 
  737   __ jccb(Assembler::parity, nan);  // PF=1
  738   __ jccb(Assembler::below, below); // CF=1
  739 
  740   // equal
  741   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  742   if (single) {
  743     __ ucomiss(a, xmmt);
  744     __ jccb(Assembler::equal, zero);
  745 
  746     __ movflt(dst, a);
  747     __ jmp(done);
  748   }
  749   else {
  750     __ ucomisd(a, xmmt);
  751     __ jccb(Assembler::equal, zero);
  752 
  753     __ movdbl(dst, a);
  754     __ jmp(done);
  755   }
  756 
  757   __ bind(zero);
  758   if (min)
  759     __ vpor(dst, a, b, Assembler::AVX_128bit);
  760   else
  761     __ vpand(dst, a, b, Assembler::AVX_128bit);
  762 
  763   __ jmp(done);
  764 
  765   __ bind(above);
  766   if (single)
  767     __ movflt(dst, min ? b : a);
  768   else
  769     __ movdbl(dst, min ? b : a);
  770 
  771   __ jmp(done);
  772 
  773   __ bind(nan);
  774   if (single) {
  775     __ movl(rt, 0x7fc00000); // Float.NaN
  776     __ movdl(dst, rt);
  777   }
  778   else {
  779     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  780     __ movdq(dst, rt);
  781   }
  782   __ jmp(done);
  783 
  784   __ bind(below);
  785   if (single)
  786     __ movflt(dst, min ? a : b);
  787   else
  788     __ movdbl(dst, min ? a : b);
  789 
  790   __ bind(done);
  791 }
  792 
  793 //=============================================================================
  794 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  795 
  796 int ConstantTable::calculate_table_base_offset() const {
  797   return 0;  // absolute addressing, no offset
  798 }
  799 
  800 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  801 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  802   ShouldNotReachHere();
  803 }
  804 
  805 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  806   // Empty encoding
  807 }
  808 
  809 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  810   return 0;
  811 }
  812 
  813 #ifndef PRODUCT
  814 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  815   st->print("# MachConstantBaseNode (empty encoding)");
  816 }
  817 #endif
  818 
  819 
  820 //=============================================================================
  821 #ifndef PRODUCT
  822 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  823   Compile* C = ra_->C;
  824 
  825   int framesize = C->output()->frame_size_in_bytes();
  826   int bangsize = C->output()->bang_size_in_bytes();
  827   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  828   // Remove wordSize for return addr which is already pushed.
  829   framesize -= wordSize;
  830 
  831   if (C->output()->need_stack_bang(bangsize)) {
  832     framesize -= wordSize;
  833     st->print("# stack bang (%d bytes)", bangsize);
  834     st->print("\n\t");
  835     st->print("pushq   rbp\t# Save rbp");
  836     if (PreserveFramePointer) {
  837         st->print("\n\t");
  838         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  839     }
  840     if (framesize) {
  841       st->print("\n\t");
  842       st->print("subq    rsp, #%d\t# Create frame",framesize);
  843     }
  844   } else {
  845     st->print("subq    rsp, #%d\t# Create frame",framesize);
  846     st->print("\n\t");
  847     framesize -= wordSize;
  848     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  849     if (PreserveFramePointer) {
  850       st->print("\n\t");
  851       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  852       if (framesize > 0) {
  853         st->print("\n\t");
  854         st->print("addq    rbp, #%d", framesize);
  855       }
  856     }
  857   }
  858 
  859   if (VerifyStackAtCalls) {
  860     st->print("\n\t");
  861     framesize -= wordSize;
  862     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  863 #ifdef ASSERT
  864     st->print("\n\t");
  865     st->print("# stack alignment check");
  866 #endif
  867   }
  868   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  869     st->print("\n\t");
  870     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  871     st->print("\n\t");
  872     st->print("je      fast_entry\t");
  873     st->print("\n\t");
  874     st->print("call    #nmethod_entry_barrier_stub\t");
  875     st->print("\n\tfast_entry:");
  876   }
  877   st->cr();
  878 }
  879 #endif
  880 
  881 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  882   Compile* C = ra_->C;
  883   MacroAssembler _masm(&cbuf);
  884 
  885   int framesize = C->output()->frame_size_in_bytes();
  886   int bangsize = C->output()->bang_size_in_bytes();
  887 
  888   if (C->clinit_barrier_on_entry()) {
  889     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  890     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  891 
  892     Label L_skip_barrier;
  893     Register klass = rscratch1;
  894 
  895     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  896     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  897 
  898     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  899 
  900     __ bind(L_skip_barrier);
  901   }
  902 
  903   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  904 
  905   C->output()->set_frame_complete(cbuf.insts_size());
  906 
  907   if (C->has_mach_constant_base_node()) {
  908     // NOTE: We set the table base offset here because users might be
  909     // emitted before MachConstantBaseNode.
  910     ConstantTable& constant_table = C->output()->constant_table();
  911     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  912   }
  913 }
  914 
  915 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  916 {
  917   return MachNode::size(ra_); // too many variables; just compute it
  918                               // the hard way
  919 }
  920 
  921 int MachPrologNode::reloc() const
  922 {
  923   return 0; // a large enough number
  924 }
  925 
  926 //=============================================================================
  927 #ifndef PRODUCT
  928 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  929 {
  930   Compile* C = ra_->C;
  931   if (generate_vzeroupper(C)) {
  932     st->print("vzeroupper");
  933     st->cr(); st->print("\t");
  934   }
  935 
  936   int framesize = C->output()->frame_size_in_bytes();
  937   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  938   // Remove word for return adr already pushed
  939   // and RBP
  940   framesize -= 2*wordSize;
  941 
  942   if (framesize) {
  943     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  944     st->print("\t");
  945   }
  946 
  947   st->print_cr("popq    rbp");
  948   if (do_polling() && C->is_method_compilation()) {
  949     st->print("\t");
  950     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  951                  "ja       #safepoint_stub\t"
  952                  "# Safepoint: poll for GC");
  953   }
  954 }
  955 #endif
  956 
  957 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  958 {
  959   Compile* C = ra_->C;
  960   MacroAssembler _masm(&cbuf);
  961 
  962   if (generate_vzeroupper(C)) {
  963     // Clear upper bits of YMM registers when current compiled code uses
  964     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  965     __ vzeroupper();
  966   }
  967 
  968   int framesize = C->output()->frame_size_in_bytes();
  969   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  970   // Remove word for return adr already pushed
  971   // and RBP
  972   framesize -= 2*wordSize;
  973 
  974   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  975 
  976   if (framesize) {
  977     emit_opcode(cbuf, Assembler::REX_W);
  978     if (framesize < 0x80) {
  979       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  980       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  981       emit_d8(cbuf, framesize);
  982     } else {
  983       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  984       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  985       emit_d32(cbuf, framesize);
  986     }
  987   }
  988 
  989   // popq rbp
  990   emit_opcode(cbuf, 0x58 | RBP_enc);
  991 
  992   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  993     __ reserved_stack_check();
  994   }
  995 
  996   if (do_polling() && C->is_method_compilation()) {
  997     MacroAssembler _masm(&cbuf);
  998     Label dummy_label;
  999     Label* code_stub = &dummy_label;
 1000     if (!C->output()->in_scratch_emit_size()) {
 1001       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1002       C->output()->add_stub(stub);
 1003       code_stub = &stub->entry();
 1004     }
 1005     __ relocate(relocInfo::poll_return_type);
 1006     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1007   }
 1008 }
 1009 
 1010 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1011 {
 1012   return MachNode::size(ra_); // too many variables; just compute it
 1013                               // the hard way
 1014 }
 1015 
 1016 int MachEpilogNode::reloc() const
 1017 {
 1018   return 2; // a large enough number
 1019 }
 1020 
 1021 const Pipeline* MachEpilogNode::pipeline() const
 1022 {
 1023   return MachNode::pipeline_class();
 1024 }
 1025 
 1026 //=============================================================================
 1027 
 1028 enum RC {
 1029   rc_bad,
 1030   rc_int,
 1031   rc_kreg,
 1032   rc_float,
 1033   rc_stack
 1034 };
 1035 
 1036 static enum RC rc_class(OptoReg::Name reg)
 1037 {
 1038   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1039 
 1040   if (OptoReg::is_stack(reg)) return rc_stack;
 1041 
 1042   VMReg r = OptoReg::as_VMReg(reg);
 1043 
 1044   if (r->is_Register()) return rc_int;
 1045 
 1046   if (r->is_KRegister()) return rc_kreg;
 1047 
 1048   assert(r->is_XMMRegister(), "must be");
 1049   return rc_float;
 1050 }
 1051 
 1052 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1053 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1054                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1055 
 1056 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1057                      int stack_offset, int reg, uint ireg, outputStream* st);
 1058 
 1059 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1060                                       int dst_offset, uint ireg, outputStream* st) {
 1061   if (cbuf) {
 1062     MacroAssembler _masm(cbuf);
 1063     switch (ireg) {
 1064     case Op_VecS:
 1065       __ movq(Address(rsp, -8), rax);
 1066       __ movl(rax, Address(rsp, src_offset));
 1067       __ movl(Address(rsp, dst_offset), rax);
 1068       __ movq(rax, Address(rsp, -8));
 1069       break;
 1070     case Op_VecD:
 1071       __ pushq(Address(rsp, src_offset));
 1072       __ popq (Address(rsp, dst_offset));
 1073       break;
 1074     case Op_VecX:
 1075       __ pushq(Address(rsp, src_offset));
 1076       __ popq (Address(rsp, dst_offset));
 1077       __ pushq(Address(rsp, src_offset+8));
 1078       __ popq (Address(rsp, dst_offset+8));
 1079       break;
 1080     case Op_VecY:
 1081       __ vmovdqu(Address(rsp, -32), xmm0);
 1082       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1083       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1084       __ vmovdqu(xmm0, Address(rsp, -32));
 1085       break;
 1086     case Op_VecZ:
 1087       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1088       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1089       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1090       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1091       break;
 1092     default:
 1093       ShouldNotReachHere();
 1094     }
 1095 #ifndef PRODUCT
 1096   } else {
 1097     switch (ireg) {
 1098     case Op_VecS:
 1099       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1100                 "movl    rax, [rsp + #%d]\n\t"
 1101                 "movl    [rsp + #%d], rax\n\t"
 1102                 "movq    rax, [rsp - #8]",
 1103                 src_offset, dst_offset);
 1104       break;
 1105     case Op_VecD:
 1106       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1107                 "popq    [rsp + #%d]",
 1108                 src_offset, dst_offset);
 1109       break;
 1110      case Op_VecX:
 1111       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1112                 "popq    [rsp + #%d]\n\t"
 1113                 "pushq   [rsp + #%d]\n\t"
 1114                 "popq    [rsp + #%d]",
 1115                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1116       break;
 1117     case Op_VecY:
 1118       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1119                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1120                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1121                 "vmovdqu xmm0, [rsp - #32]",
 1122                 src_offset, dst_offset);
 1123       break;
 1124     case Op_VecZ:
 1125       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1126                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1127                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1128                 "vmovdqu xmm0, [rsp - #64]",
 1129                 src_offset, dst_offset);
 1130       break;
 1131     default:
 1132       ShouldNotReachHere();
 1133     }
 1134 #endif
 1135   }
 1136 }
 1137 
 1138 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1139                                        PhaseRegAlloc* ra_,
 1140                                        bool do_size,
 1141                                        outputStream* st) const {
 1142   assert(cbuf != NULL || st  != NULL, "sanity");
 1143   // Get registers to move
 1144   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1145   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1146   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1147   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1148 
 1149   enum RC src_second_rc = rc_class(src_second);
 1150   enum RC src_first_rc = rc_class(src_first);
 1151   enum RC dst_second_rc = rc_class(dst_second);
 1152   enum RC dst_first_rc = rc_class(dst_first);
 1153 
 1154   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1155          "must move at least 1 register" );
 1156 
 1157   if (src_first == dst_first && src_second == dst_second) {
 1158     // Self copy, no move
 1159     return 0;
 1160   }
 1161   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1162     uint ireg = ideal_reg();
 1163     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1164     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1165     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1166       // mem -> mem
 1167       int src_offset = ra_->reg2offset(src_first);
 1168       int dst_offset = ra_->reg2offset(dst_first);
 1169       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1170     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1171       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1172     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1173       int stack_offset = ra_->reg2offset(dst_first);
 1174       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1175     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1176       int stack_offset = ra_->reg2offset(src_first);
 1177       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1178     } else {
 1179       ShouldNotReachHere();
 1180     }
 1181     return 0;
 1182   }
 1183   if (src_first_rc == rc_stack) {
 1184     // mem ->
 1185     if (dst_first_rc == rc_stack) {
 1186       // mem -> mem
 1187       assert(src_second != dst_first, "overlap");
 1188       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1189           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1190         // 64-bit
 1191         int src_offset = ra_->reg2offset(src_first);
 1192         int dst_offset = ra_->reg2offset(dst_first);
 1193         if (cbuf) {
 1194           MacroAssembler _masm(cbuf);
 1195           __ pushq(Address(rsp, src_offset));
 1196           __ popq (Address(rsp, dst_offset));
 1197 #ifndef PRODUCT
 1198         } else {
 1199           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1200                     "popq    [rsp + #%d]",
 1201                      src_offset, dst_offset);
 1202 #endif
 1203         }
 1204       } else {
 1205         // 32-bit
 1206         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1207         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1208         // No pushl/popl, so:
 1209         int src_offset = ra_->reg2offset(src_first);
 1210         int dst_offset = ra_->reg2offset(dst_first);
 1211         if (cbuf) {
 1212           MacroAssembler _masm(cbuf);
 1213           __ movq(Address(rsp, -8), rax);
 1214           __ movl(rax, Address(rsp, src_offset));
 1215           __ movl(Address(rsp, dst_offset), rax);
 1216           __ movq(rax, Address(rsp, -8));
 1217 #ifndef PRODUCT
 1218         } else {
 1219           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1220                     "movl    rax, [rsp + #%d]\n\t"
 1221                     "movl    [rsp + #%d], rax\n\t"
 1222                     "movq    rax, [rsp - #8]",
 1223                      src_offset, dst_offset);
 1224 #endif
 1225         }
 1226       }
 1227       return 0;
 1228     } else if (dst_first_rc == rc_int) {
 1229       // mem -> gpr
 1230       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1231           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1232         // 64-bit
 1233         int offset = ra_->reg2offset(src_first);
 1234         if (cbuf) {
 1235           MacroAssembler _masm(cbuf);
 1236           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1237 #ifndef PRODUCT
 1238         } else {
 1239           st->print("movq    %s, [rsp + #%d]\t# spill",
 1240                      Matcher::regName[dst_first],
 1241                      offset);
 1242 #endif
 1243         }
 1244       } else {
 1245         // 32-bit
 1246         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1247         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1248         int offset = ra_->reg2offset(src_first);
 1249         if (cbuf) {
 1250           MacroAssembler _masm(cbuf);
 1251           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1252 #ifndef PRODUCT
 1253         } else {
 1254           st->print("movl    %s, [rsp + #%d]\t# spill",
 1255                      Matcher::regName[dst_first],
 1256                      offset);
 1257 #endif
 1258         }
 1259       }
 1260       return 0;
 1261     } else if (dst_first_rc == rc_float) {
 1262       // mem-> xmm
 1263       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1264           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1265         // 64-bit
 1266         int offset = ra_->reg2offset(src_first);
 1267         if (cbuf) {
 1268           MacroAssembler _masm(cbuf);
 1269           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1270 #ifndef PRODUCT
 1271         } else {
 1272           st->print("%s  %s, [rsp + #%d]\t# spill",
 1273                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1274                      Matcher::regName[dst_first],
 1275                      offset);
 1276 #endif
 1277         }
 1278       } else {
 1279         // 32-bit
 1280         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1281         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1282         int offset = ra_->reg2offset(src_first);
 1283         if (cbuf) {
 1284           MacroAssembler _masm(cbuf);
 1285           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1286 #ifndef PRODUCT
 1287         } else {
 1288           st->print("movss   %s, [rsp + #%d]\t# spill",
 1289                      Matcher::regName[dst_first],
 1290                      offset);
 1291 #endif
 1292         }
 1293       }
 1294       return 0;
 1295     } else if (dst_first_rc == rc_kreg) {
 1296       // mem -> kreg
 1297       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1298           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1299         // 64-bit
 1300         int offset = ra_->reg2offset(src_first);
 1301         if (cbuf) {
 1302           MacroAssembler _masm(cbuf);
 1303           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1304 #ifndef PRODUCT
 1305         } else {
 1306           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1307                      Matcher::regName[dst_first],
 1308                      offset);
 1309 #endif
 1310         }
 1311       }
 1312       return 0;
 1313     }
 1314   } else if (src_first_rc == rc_int) {
 1315     // gpr ->
 1316     if (dst_first_rc == rc_stack) {
 1317       // gpr -> mem
 1318       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1319           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1320         // 64-bit
 1321         int offset = ra_->reg2offset(dst_first);
 1322         if (cbuf) {
 1323           MacroAssembler _masm(cbuf);
 1324           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1325 #ifndef PRODUCT
 1326         } else {
 1327           st->print("movq    [rsp + #%d], %s\t# spill",
 1328                      offset,
 1329                      Matcher::regName[src_first]);
 1330 #endif
 1331         }
 1332       } else {
 1333         // 32-bit
 1334         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1335         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1336         int offset = ra_->reg2offset(dst_first);
 1337         if (cbuf) {
 1338           MacroAssembler _masm(cbuf);
 1339           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1340 #ifndef PRODUCT
 1341         } else {
 1342           st->print("movl    [rsp + #%d], %s\t# spill",
 1343                      offset,
 1344                      Matcher::regName[src_first]);
 1345 #endif
 1346         }
 1347       }
 1348       return 0;
 1349     } else if (dst_first_rc == rc_int) {
 1350       // gpr -> gpr
 1351       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1352           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1353         // 64-bit
 1354         if (cbuf) {
 1355           MacroAssembler _masm(cbuf);
 1356           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1357                   as_Register(Matcher::_regEncode[src_first]));
 1358 #ifndef PRODUCT
 1359         } else {
 1360           st->print("movq    %s, %s\t# spill",
 1361                      Matcher::regName[dst_first],
 1362                      Matcher::regName[src_first]);
 1363 #endif
 1364         }
 1365         return 0;
 1366       } else {
 1367         // 32-bit
 1368         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1369         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1370         if (cbuf) {
 1371           MacroAssembler _masm(cbuf);
 1372           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1373                   as_Register(Matcher::_regEncode[src_first]));
 1374 #ifndef PRODUCT
 1375         } else {
 1376           st->print("movl    %s, %s\t# spill",
 1377                      Matcher::regName[dst_first],
 1378                      Matcher::regName[src_first]);
 1379 #endif
 1380         }
 1381         return 0;
 1382       }
 1383     } else if (dst_first_rc == rc_float) {
 1384       // gpr -> xmm
 1385       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1386           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1387         // 64-bit
 1388         if (cbuf) {
 1389           MacroAssembler _masm(cbuf);
 1390           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1391 #ifndef PRODUCT
 1392         } else {
 1393           st->print("movdq   %s, %s\t# spill",
 1394                      Matcher::regName[dst_first],
 1395                      Matcher::regName[src_first]);
 1396 #endif
 1397         }
 1398       } else {
 1399         // 32-bit
 1400         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1401         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1402         if (cbuf) {
 1403           MacroAssembler _masm(cbuf);
 1404           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1405 #ifndef PRODUCT
 1406         } else {
 1407           st->print("movdl   %s, %s\t# spill",
 1408                      Matcher::regName[dst_first],
 1409                      Matcher::regName[src_first]);
 1410 #endif
 1411         }
 1412       }
 1413       return 0;
 1414     } else if (dst_first_rc == rc_kreg) {
 1415       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1416           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1417         // 64-bit
 1418         if (cbuf) {
 1419           MacroAssembler _masm(cbuf);
 1420           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1421   #ifndef PRODUCT
 1422         } else {
 1423            st->print("kmovq   %s, %s\t# spill",
 1424                        Matcher::regName[dst_first],
 1425                        Matcher::regName[src_first]);
 1426   #endif
 1427         }
 1428       }
 1429       Unimplemented();
 1430       return 0;
 1431     }
 1432   } else if (src_first_rc == rc_float) {
 1433     // xmm ->
 1434     if (dst_first_rc == rc_stack) {
 1435       // xmm -> mem
 1436       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1437           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1438         // 64-bit
 1439         int offset = ra_->reg2offset(dst_first);
 1440         if (cbuf) {
 1441           MacroAssembler _masm(cbuf);
 1442           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1443 #ifndef PRODUCT
 1444         } else {
 1445           st->print("movsd   [rsp + #%d], %s\t# spill",
 1446                      offset,
 1447                      Matcher::regName[src_first]);
 1448 #endif
 1449         }
 1450       } else {
 1451         // 32-bit
 1452         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1453         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1454         int offset = ra_->reg2offset(dst_first);
 1455         if (cbuf) {
 1456           MacroAssembler _masm(cbuf);
 1457           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1458 #ifndef PRODUCT
 1459         } else {
 1460           st->print("movss   [rsp + #%d], %s\t# spill",
 1461                      offset,
 1462                      Matcher::regName[src_first]);
 1463 #endif
 1464         }
 1465       }
 1466       return 0;
 1467     } else if (dst_first_rc == rc_int) {
 1468       // xmm -> gpr
 1469       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1470           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1471         // 64-bit
 1472         if (cbuf) {
 1473           MacroAssembler _masm(cbuf);
 1474           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1475 #ifndef PRODUCT
 1476         } else {
 1477           st->print("movdq   %s, %s\t# spill",
 1478                      Matcher::regName[dst_first],
 1479                      Matcher::regName[src_first]);
 1480 #endif
 1481         }
 1482       } else {
 1483         // 32-bit
 1484         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1485         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1486         if (cbuf) {
 1487           MacroAssembler _masm(cbuf);
 1488           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1489 #ifndef PRODUCT
 1490         } else {
 1491           st->print("movdl   %s, %s\t# spill",
 1492                      Matcher::regName[dst_first],
 1493                      Matcher::regName[src_first]);
 1494 #endif
 1495         }
 1496       }
 1497       return 0;
 1498     } else if (dst_first_rc == rc_float) {
 1499       // xmm -> xmm
 1500       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1501           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1502         // 64-bit
 1503         if (cbuf) {
 1504           MacroAssembler _masm(cbuf);
 1505           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1506 #ifndef PRODUCT
 1507         } else {
 1508           st->print("%s  %s, %s\t# spill",
 1509                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1510                      Matcher::regName[dst_first],
 1511                      Matcher::regName[src_first]);
 1512 #endif
 1513         }
 1514       } else {
 1515         // 32-bit
 1516         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1517         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1518         if (cbuf) {
 1519           MacroAssembler _masm(cbuf);
 1520           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1521 #ifndef PRODUCT
 1522         } else {
 1523           st->print("%s  %s, %s\t# spill",
 1524                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1525                      Matcher::regName[dst_first],
 1526                      Matcher::regName[src_first]);
 1527 #endif
 1528         }
 1529       }
 1530       return 0;
 1531     } else if (dst_first_rc == rc_kreg) {
 1532       assert(false, "Illegal spilling");
 1533       return 0;
 1534     }
 1535   } else if (src_first_rc == rc_kreg) {
 1536     if (dst_first_rc == rc_stack) {
 1537       // mem -> kreg
 1538       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1539           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1540         // 64-bit
 1541         int offset = ra_->reg2offset(dst_first);
 1542         if (cbuf) {
 1543           MacroAssembler _masm(cbuf);
 1544           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1545 #ifndef PRODUCT
 1546         } else {
 1547           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1548                      offset,
 1549                      Matcher::regName[src_first]);
 1550 #endif
 1551         }
 1552       }
 1553       return 0;
 1554     } else if (dst_first_rc == rc_int) {
 1555       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1556           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1557         // 64-bit
 1558         if (cbuf) {
 1559           MacroAssembler _masm(cbuf);
 1560           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1561 #ifndef PRODUCT
 1562         } else {
 1563          st->print("kmovq   %s, %s\t# spill",
 1564                      Matcher::regName[dst_first],
 1565                      Matcher::regName[src_first]);
 1566 #endif
 1567         }
 1568       }
 1569       Unimplemented();
 1570       return 0;
 1571     } else if (dst_first_rc == rc_kreg) {
 1572       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1573           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1574         // 64-bit
 1575         if (cbuf) {
 1576           MacroAssembler _masm(cbuf);
 1577           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1578 #ifndef PRODUCT
 1579         } else {
 1580          st->print("kmovq   %s, %s\t# spill",
 1581                      Matcher::regName[dst_first],
 1582                      Matcher::regName[src_first]);
 1583 #endif
 1584         }
 1585       }
 1586       return 0;
 1587     } else if (dst_first_rc == rc_float) {
 1588       assert(false, "Illegal spill");
 1589       return 0;
 1590     }
 1591   }
 1592 
 1593   assert(0," foo ");
 1594   Unimplemented();
 1595   return 0;
 1596 }
 1597 
 1598 #ifndef PRODUCT
 1599 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1600   implementation(NULL, ra_, false, st);
 1601 }
 1602 #endif
 1603 
 1604 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1605   implementation(&cbuf, ra_, false, NULL);
 1606 }
 1607 
 1608 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1609   return MachNode::size(ra_);
 1610 }
 1611 
 1612 //=============================================================================
 1613 #ifndef PRODUCT
 1614 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1615 {
 1616   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1617   int reg = ra_->get_reg_first(this);
 1618   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1619             Matcher::regName[reg], offset);
 1620 }
 1621 #endif
 1622 
 1623 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1624 {
 1625   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1626   int reg = ra_->get_encode(this);
 1627   if (offset >= 0x80) {
 1628     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1629     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1630     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1631     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1632     emit_d32(cbuf, offset);
 1633   } else {
 1634     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1635     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1636     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1637     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1638     emit_d8(cbuf, offset);
 1639   }
 1640 }
 1641 
 1642 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1643 {
 1644   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1645   return (offset < 0x80) ? 5 : 8; // REX
 1646 }
 1647 
 1648 //=============================================================================
 1649 #ifndef PRODUCT
 1650 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1651 {
 1652   if (UseCompressedClassPointers) {
 1653     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1654     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1655     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1656   } else {
 1657     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1658                  "# Inline cache check");
 1659   }
 1660   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1661   st->print_cr("\tnop\t# nops to align entry point");
 1662 }
 1663 #endif
 1664 
 1665 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1666 {
 1667   MacroAssembler masm(&cbuf);
 1668   uint insts_size = cbuf.insts_size();
 1669   if (UseCompressedClassPointers) {
 1670     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1671     masm.cmpptr(rax, rscratch1);
 1672   } else {
 1673     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1674   }
 1675 
 1676   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1677 
 1678   /* WARNING these NOPs are critical so that verified entry point is properly
 1679      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1680   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1681   if (OptoBreakpoint) {
 1682     // Leave space for int3
 1683     nops_cnt -= 1;
 1684   }
 1685   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1686   if (nops_cnt > 0)
 1687     masm.nop(nops_cnt);
 1688 }
 1689 
 1690 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1691 {
 1692   return MachNode::size(ra_); // too many variables; just compute it
 1693                               // the hard way
 1694 }
 1695 
 1696 
 1697 //=============================================================================
 1698 
 1699 const bool Matcher::supports_vector_calling_convention(void) {
 1700   if (EnableVectorSupport && UseVectorStubs) {
 1701     return true;
 1702   }
 1703   return false;
 1704 }
 1705 
 1706 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1707   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1708   int lo = XMM0_num;
 1709   int hi = XMM0b_num;
 1710   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1711   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1712   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1713   return OptoRegPair(hi, lo);
 1714 }
 1715 
 1716 // Is this branch offset short enough that a short branch can be used?
 1717 //
 1718 // NOTE: If the platform does not provide any short branch variants, then
 1719 //       this method should return false for offset 0.
 1720 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1721   // The passed offset is relative to address of the branch.
 1722   // On 86 a branch displacement is calculated relative to address
 1723   // of a next instruction.
 1724   offset -= br_size;
 1725 
 1726   // the short version of jmpConUCF2 contains multiple branches,
 1727   // making the reach slightly less
 1728   if (rule == jmpConUCF2_rule)
 1729     return (-126 <= offset && offset <= 125);
 1730   return (-128 <= offset && offset <= 127);
 1731 }
 1732 
 1733 // Return whether or not this register is ever used as an argument.
 1734 // This function is used on startup to build the trampoline stubs in
 1735 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1736 // call in the trampoline, and arguments in those registers not be
 1737 // available to the callee.
 1738 bool Matcher::can_be_java_arg(int reg)
 1739 {
 1740   return
 1741     reg ==  RDI_num || reg == RDI_H_num ||
 1742     reg ==  RSI_num || reg == RSI_H_num ||
 1743     reg ==  RDX_num || reg == RDX_H_num ||
 1744     reg ==  RCX_num || reg == RCX_H_num ||
 1745     reg ==   R8_num || reg ==  R8_H_num ||
 1746     reg ==   R9_num || reg ==  R9_H_num ||
 1747     reg ==  R12_num || reg == R12_H_num ||
 1748     reg == XMM0_num || reg == XMM0b_num ||
 1749     reg == XMM1_num || reg == XMM1b_num ||
 1750     reg == XMM2_num || reg == XMM2b_num ||
 1751     reg == XMM3_num || reg == XMM3b_num ||
 1752     reg == XMM4_num || reg == XMM4b_num ||
 1753     reg == XMM5_num || reg == XMM5b_num ||
 1754     reg == XMM6_num || reg == XMM6b_num ||
 1755     reg == XMM7_num || reg == XMM7b_num;
 1756 }
 1757 
 1758 bool Matcher::is_spillable_arg(int reg)
 1759 {
 1760   return can_be_java_arg(reg);
 1761 }
 1762 
 1763 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1764   // In 64 bit mode a code which use multiply when
 1765   // devisor is constant is faster than hardware
 1766   // DIV instruction (it uses MulHiL).
 1767   return false;
 1768 }
 1769 
 1770 // Register for DIVI projection of divmodI
 1771 RegMask Matcher::divI_proj_mask() {
 1772   return INT_RAX_REG_mask();
 1773 }
 1774 
 1775 // Register for MODI projection of divmodI
 1776 RegMask Matcher::modI_proj_mask() {
 1777   return INT_RDX_REG_mask();
 1778 }
 1779 
 1780 // Register for DIVL projection of divmodL
 1781 RegMask Matcher::divL_proj_mask() {
 1782   return LONG_RAX_REG_mask();
 1783 }
 1784 
 1785 // Register for MODL projection of divmodL
 1786 RegMask Matcher::modL_proj_mask() {
 1787   return LONG_RDX_REG_mask();
 1788 }
 1789 
 1790 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1791 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1792     return NO_REG_mask();
 1793 }
 1794 
 1795 %}
 1796 
 1797 //----------ENCODING BLOCK-----------------------------------------------------
 1798 // This block specifies the encoding classes used by the compiler to
 1799 // output byte streams.  Encoding classes are parameterized macros
 1800 // used by Machine Instruction Nodes in order to generate the bit
 1801 // encoding of the instruction.  Operands specify their base encoding
 1802 // interface with the interface keyword.  There are currently
 1803 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1804 // COND_INTER.  REG_INTER causes an operand to generate a function
 1805 // which returns its register number when queried.  CONST_INTER causes
 1806 // an operand to generate a function which returns the value of the
 1807 // constant when queried.  MEMORY_INTER causes an operand to generate
 1808 // four functions which return the Base Register, the Index Register,
 1809 // the Scale Value, and the Offset Value of the operand when queried.
 1810 // COND_INTER causes an operand to generate six functions which return
 1811 // the encoding code (ie - encoding bits for the instruction)
 1812 // associated with each basic boolean condition for a conditional
 1813 // instruction.
 1814 //
 1815 // Instructions specify two basic values for encoding.  Again, a
 1816 // function is available to check if the constant displacement is an
 1817 // oop. They use the ins_encode keyword to specify their encoding
 1818 // classes (which must be a sequence of enc_class names, and their
 1819 // parameters, specified in the encoding block), and they use the
 1820 // opcode keyword to specify, in order, their primary, secondary, and
 1821 // tertiary opcode.  Only the opcode sections which a particular
 1822 // instruction needs for encoding need to be specified.
 1823 encode %{
 1824   // Build emit functions for each basic byte or larger field in the
 1825   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1826   // from C++ code in the enc_class source block.  Emit functions will
 1827   // live in the main source block for now.  In future, we can
 1828   // generalize this by adding a syntax that specifies the sizes of
 1829   // fields in an order, so that the adlc can build the emit functions
 1830   // automagically
 1831 
 1832   // Emit primary opcode
 1833   enc_class OpcP
 1834   %{
 1835     emit_opcode(cbuf, $primary);
 1836   %}
 1837 
 1838   // Emit secondary opcode
 1839   enc_class OpcS
 1840   %{
 1841     emit_opcode(cbuf, $secondary);
 1842   %}
 1843 
 1844   // Emit tertiary opcode
 1845   enc_class OpcT
 1846   %{
 1847     emit_opcode(cbuf, $tertiary);
 1848   %}
 1849 
 1850   // Emit opcode directly
 1851   enc_class Opcode(immI d8)
 1852   %{
 1853     emit_opcode(cbuf, $d8$$constant);
 1854   %}
 1855 
 1856   // Emit size prefix
 1857   enc_class SizePrefix
 1858   %{
 1859     emit_opcode(cbuf, 0x66);
 1860   %}
 1861 
 1862   enc_class reg(rRegI reg)
 1863   %{
 1864     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1865   %}
 1866 
 1867   enc_class reg_reg(rRegI dst, rRegI src)
 1868   %{
 1869     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1870   %}
 1871 
 1872   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1873   %{
 1874     emit_opcode(cbuf, $opcode$$constant);
 1875     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1876   %}
 1877 
 1878   enc_class cdql_enc(no_rax_rdx_RegI div)
 1879   %{
 1880     // Full implementation of Java idiv and irem; checks for
 1881     // special case as described in JVM spec., p.243 & p.271.
 1882     //
 1883     //         normal case                           special case
 1884     //
 1885     // input : rax: dividend                         min_int
 1886     //         reg: divisor                          -1
 1887     //
 1888     // output: rax: quotient  (= rax idiv reg)       min_int
 1889     //         rdx: remainder (= rax irem reg)       0
 1890     //
 1891     //  Code sequnce:
 1892     //
 1893     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1894     //    5:   75 07/08                jne    e <normal>
 1895     //    7:   33 d2                   xor    %edx,%edx
 1896     //  [div >= 8 -> offset + 1]
 1897     //  [REX_B]
 1898     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1899     //    c:   74 03/04                je     11 <done>
 1900     // 000000000000000e <normal>:
 1901     //    e:   99                      cltd
 1902     //  [div >= 8 -> offset + 1]
 1903     //  [REX_B]
 1904     //    f:   f7 f9                   idiv   $div
 1905     // 0000000000000011 <done>:
 1906     MacroAssembler _masm(&cbuf);
 1907     Label normal;
 1908     Label done;
 1909 
 1910     // cmp    $0x80000000,%eax
 1911     __ cmp(as_Register(RAX_enc), 0x80000000);
 1912 
 1913     // jne    e <normal>
 1914     __ jccb(Assembler::notEqual, normal);
 1915 
 1916     // xor    %edx,%edx
 1917     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1918 
 1919     // cmp    $0xffffffffffffffff,%ecx
 1920     __ cmpl($div$$Register, -1);
 1921 
 1922     // je     11 <done>
 1923     __ jccb(Assembler::equal, done);
 1924 
 1925     // <normal>
 1926     // cltd
 1927     __ bind(normal);
 1928     __ cdql();
 1929 
 1930     // idivl
 1931     // <done>
 1932     __ idivl($div$$Register);
 1933     __ bind(done);
 1934   %}
 1935 
 1936   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1937   %{
 1938     // Full implementation of Java ldiv and lrem; checks for
 1939     // special case as described in JVM spec., p.243 & p.271.
 1940     //
 1941     //         normal case                           special case
 1942     //
 1943     // input : rax: dividend                         min_long
 1944     //         reg: divisor                          -1
 1945     //
 1946     // output: rax: quotient  (= rax idiv reg)       min_long
 1947     //         rdx: remainder (= rax irem reg)       0
 1948     //
 1949     //  Code sequnce:
 1950     //
 1951     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1952     //    7:   00 00 80
 1953     //    a:   48 39 d0                cmp    %rdx,%rax
 1954     //    d:   75 08                   jne    17 <normal>
 1955     //    f:   33 d2                   xor    %edx,%edx
 1956     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1957     //   15:   74 05                   je     1c <done>
 1958     // 0000000000000017 <normal>:
 1959     //   17:   48 99                   cqto
 1960     //   19:   48 f7 f9                idiv   $div
 1961     // 000000000000001c <done>:
 1962     MacroAssembler _masm(&cbuf);
 1963     Label normal;
 1964     Label done;
 1965 
 1966     // mov    $0x8000000000000000,%rdx
 1967     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1968 
 1969     // cmp    %rdx,%rax
 1970     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1971 
 1972     // jne    17 <normal>
 1973     __ jccb(Assembler::notEqual, normal);
 1974 
 1975     // xor    %edx,%edx
 1976     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1977 
 1978     // cmp    $0xffffffffffffffff,$div
 1979     __ cmpq($div$$Register, -1);
 1980 
 1981     // je     1e <done>
 1982     __ jccb(Assembler::equal, done);
 1983 
 1984     // <normal>
 1985     // cqto
 1986     __ bind(normal);
 1987     __ cdqq();
 1988 
 1989     // idivq (note: must be emitted by the user of this rule)
 1990     // <done>
 1991     __ idivq($div$$Register);
 1992     __ bind(done);
 1993   %}
 1994 
 1995   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 1996   enc_class OpcSE(immI imm)
 1997   %{
 1998     // Emit primary opcode and set sign-extend bit
 1999     // Check for 8-bit immediate, and set sign extend bit in opcode
 2000     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2001       emit_opcode(cbuf, $primary | 0x02);
 2002     } else {
 2003       // 32-bit immediate
 2004       emit_opcode(cbuf, $primary);
 2005     }
 2006   %}
 2007 
 2008   enc_class OpcSErm(rRegI dst, immI imm)
 2009   %{
 2010     // OpcSEr/m
 2011     int dstenc = $dst$$reg;
 2012     if (dstenc >= 8) {
 2013       emit_opcode(cbuf, Assembler::REX_B);
 2014       dstenc -= 8;
 2015     }
 2016     // Emit primary opcode and set sign-extend bit
 2017     // Check for 8-bit immediate, and set sign extend bit in opcode
 2018     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2019       emit_opcode(cbuf, $primary | 0x02);
 2020     } else {
 2021       // 32-bit immediate
 2022       emit_opcode(cbuf, $primary);
 2023     }
 2024     // Emit r/m byte with secondary opcode, after primary opcode.
 2025     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2026   %}
 2027 
 2028   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2029   %{
 2030     // OpcSEr/m
 2031     int dstenc = $dst$$reg;
 2032     if (dstenc < 8) {
 2033       emit_opcode(cbuf, Assembler::REX_W);
 2034     } else {
 2035       emit_opcode(cbuf, Assembler::REX_WB);
 2036       dstenc -= 8;
 2037     }
 2038     // Emit primary opcode and set sign-extend bit
 2039     // Check for 8-bit immediate, and set sign extend bit in opcode
 2040     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2041       emit_opcode(cbuf, $primary | 0x02);
 2042     } else {
 2043       // 32-bit immediate
 2044       emit_opcode(cbuf, $primary);
 2045     }
 2046     // Emit r/m byte with secondary opcode, after primary opcode.
 2047     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2048   %}
 2049 
 2050   enc_class Con8or32(immI imm)
 2051   %{
 2052     // Check for 8-bit immediate, and set sign extend bit in opcode
 2053     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2054       $$$emit8$imm$$constant;
 2055     } else {
 2056       // 32-bit immediate
 2057       $$$emit32$imm$$constant;
 2058     }
 2059   %}
 2060 
 2061   enc_class opc2_reg(rRegI dst)
 2062   %{
 2063     // BSWAP
 2064     emit_cc(cbuf, $secondary, $dst$$reg);
 2065   %}
 2066 
 2067   enc_class opc3_reg(rRegI dst)
 2068   %{
 2069     // BSWAP
 2070     emit_cc(cbuf, $tertiary, $dst$$reg);
 2071   %}
 2072 
 2073   enc_class reg_opc(rRegI div)
 2074   %{
 2075     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2076     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2077   %}
 2078 
 2079   enc_class enc_cmov(cmpOp cop)
 2080   %{
 2081     // CMOV
 2082     $$$emit8$primary;
 2083     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2084   %}
 2085 
 2086   enc_class enc_PartialSubtypeCheck()
 2087   %{
 2088     Register Rrdi = as_Register(RDI_enc); // result register
 2089     Register Rrax = as_Register(RAX_enc); // super class
 2090     Register Rrcx = as_Register(RCX_enc); // killed
 2091     Register Rrsi = as_Register(RSI_enc); // sub class
 2092     Label miss;
 2093     const bool set_cond_codes = true;
 2094 
 2095     MacroAssembler _masm(&cbuf);
 2096     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2097                                      NULL, &miss,
 2098                                      /*set_cond_codes:*/ true);
 2099     if ($primary) {
 2100       __ xorptr(Rrdi, Rrdi);
 2101     }
 2102     __ bind(miss);
 2103   %}
 2104 
 2105   enc_class clear_avx %{
 2106     debug_only(int off0 = cbuf.insts_size());
 2107     if (generate_vzeroupper(Compile::current())) {
 2108       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2109       // Clear upper bits of YMM registers when current compiled code uses
 2110       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2111       MacroAssembler _masm(&cbuf);
 2112       __ vzeroupper();
 2113     }
 2114     debug_only(int off1 = cbuf.insts_size());
 2115     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2116   %}
 2117 
 2118   enc_class Java_To_Runtime(method meth) %{
 2119     // No relocation needed
 2120     MacroAssembler _masm(&cbuf);
 2121     __ mov64(r10, (int64_t) $meth$$method);
 2122     __ call(r10);
 2123   %}
 2124 
 2125   enc_class Java_To_Interpreter(method meth)
 2126   %{
 2127     // CALL Java_To_Interpreter
 2128     // This is the instruction starting address for relocation info.
 2129     cbuf.set_insts_mark();
 2130     $$$emit8$primary;
 2131     // CALL directly to the runtime
 2132     emit_d32_reloc(cbuf,
 2133                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2134                    runtime_call_Relocation::spec(),
 2135                    RELOC_DISP32);
 2136   %}
 2137 
 2138   enc_class Java_Static_Call(method meth)
 2139   %{
 2140     // JAVA STATIC CALL
 2141     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2142     // determine who we intended to call.
 2143     cbuf.set_insts_mark();
 2144     $$$emit8$primary;
 2145 
 2146     if (!_method) {
 2147       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2148                      runtime_call_Relocation::spec(),
 2149                      RELOC_DISP32);
 2150     } else {
 2151       int method_index = resolved_method_index(cbuf);
 2152       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2153                                                   : static_call_Relocation::spec(method_index);
 2154       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2155                      rspec, RELOC_DISP32);
 2156       // Emit stubs for static call.
 2157       address mark = cbuf.insts_mark();
 2158       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2159       if (stub == NULL) {
 2160         ciEnv::current()->record_failure("CodeCache is full");
 2161         return;
 2162       }
 2163     }
 2164   %}
 2165 
 2166   enc_class Java_Dynamic_Call(method meth) %{
 2167     MacroAssembler _masm(&cbuf);
 2168     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2169   %}
 2170 
 2171   enc_class Java_Compiled_Call(method meth)
 2172   %{
 2173     // JAVA COMPILED CALL
 2174     int disp = in_bytes(Method:: from_compiled_offset());
 2175 
 2176     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
 2177     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
 2178 
 2179     // callq *disp(%rax)
 2180     cbuf.set_insts_mark();
 2181     $$$emit8$primary;
 2182     if (disp < 0x80) {
 2183       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
 2184       emit_d8(cbuf, disp); // Displacement
 2185     } else {
 2186       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
 2187       emit_d32(cbuf, disp); // Displacement
 2188     }
 2189   %}
 2190 
 2191   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2192   %{
 2193     // SAL, SAR, SHR
 2194     int dstenc = $dst$$reg;
 2195     if (dstenc >= 8) {
 2196       emit_opcode(cbuf, Assembler::REX_B);
 2197       dstenc -= 8;
 2198     }
 2199     $$$emit8$primary;
 2200     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2201     $$$emit8$shift$$constant;
 2202   %}
 2203 
 2204   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2205   %{
 2206     // SAL, SAR, SHR
 2207     int dstenc = $dst$$reg;
 2208     if (dstenc < 8) {
 2209       emit_opcode(cbuf, Assembler::REX_W);
 2210     } else {
 2211       emit_opcode(cbuf, Assembler::REX_WB);
 2212       dstenc -= 8;
 2213     }
 2214     $$$emit8$primary;
 2215     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2216     $$$emit8$shift$$constant;
 2217   %}
 2218 
 2219   enc_class load_immI(rRegI dst, immI src)
 2220   %{
 2221     int dstenc = $dst$$reg;
 2222     if (dstenc >= 8) {
 2223       emit_opcode(cbuf, Assembler::REX_B);
 2224       dstenc -= 8;
 2225     }
 2226     emit_opcode(cbuf, 0xB8 | dstenc);
 2227     $$$emit32$src$$constant;
 2228   %}
 2229 
 2230   enc_class load_immL(rRegL dst, immL src)
 2231   %{
 2232     int dstenc = $dst$$reg;
 2233     if (dstenc < 8) {
 2234       emit_opcode(cbuf, Assembler::REX_W);
 2235     } else {
 2236       emit_opcode(cbuf, Assembler::REX_WB);
 2237       dstenc -= 8;
 2238     }
 2239     emit_opcode(cbuf, 0xB8 | dstenc);
 2240     emit_d64(cbuf, $src$$constant);
 2241   %}
 2242 
 2243   enc_class load_immUL32(rRegL dst, immUL32 src)
 2244   %{
 2245     // same as load_immI, but this time we care about zeroes in the high word
 2246     int dstenc = $dst$$reg;
 2247     if (dstenc >= 8) {
 2248       emit_opcode(cbuf, Assembler::REX_B);
 2249       dstenc -= 8;
 2250     }
 2251     emit_opcode(cbuf, 0xB8 | dstenc);
 2252     $$$emit32$src$$constant;
 2253   %}
 2254 
 2255   enc_class load_immL32(rRegL dst, immL32 src)
 2256   %{
 2257     int dstenc = $dst$$reg;
 2258     if (dstenc < 8) {
 2259       emit_opcode(cbuf, Assembler::REX_W);
 2260     } else {
 2261       emit_opcode(cbuf, Assembler::REX_WB);
 2262       dstenc -= 8;
 2263     }
 2264     emit_opcode(cbuf, 0xC7);
 2265     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2266     $$$emit32$src$$constant;
 2267   %}
 2268 
 2269   enc_class load_immP31(rRegP dst, immP32 src)
 2270   %{
 2271     // same as load_immI, but this time we care about zeroes in the high word
 2272     int dstenc = $dst$$reg;
 2273     if (dstenc >= 8) {
 2274       emit_opcode(cbuf, Assembler::REX_B);
 2275       dstenc -= 8;
 2276     }
 2277     emit_opcode(cbuf, 0xB8 | dstenc);
 2278     $$$emit32$src$$constant;
 2279   %}
 2280 
 2281   enc_class load_immP(rRegP dst, immP src)
 2282   %{
 2283     int dstenc = $dst$$reg;
 2284     if (dstenc < 8) {
 2285       emit_opcode(cbuf, Assembler::REX_W);
 2286     } else {
 2287       emit_opcode(cbuf, Assembler::REX_WB);
 2288       dstenc -= 8;
 2289     }
 2290     emit_opcode(cbuf, 0xB8 | dstenc);
 2291     // This next line should be generated from ADLC
 2292     if ($src->constant_reloc() != relocInfo::none) {
 2293       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2294     } else {
 2295       emit_d64(cbuf, $src$$constant);
 2296     }
 2297   %}
 2298 
 2299   enc_class Con32(immI src)
 2300   %{
 2301     // Output immediate
 2302     $$$emit32$src$$constant;
 2303   %}
 2304 
 2305   enc_class Con32F_as_bits(immF src)
 2306   %{
 2307     // Output Float immediate bits
 2308     jfloat jf = $src$$constant;
 2309     jint jf_as_bits = jint_cast(jf);
 2310     emit_d32(cbuf, jf_as_bits);
 2311   %}
 2312 
 2313   enc_class Con16(immI src)
 2314   %{
 2315     // Output immediate
 2316     $$$emit16$src$$constant;
 2317   %}
 2318 
 2319   // How is this different from Con32??? XXX
 2320   enc_class Con_d32(immI src)
 2321   %{
 2322     emit_d32(cbuf,$src$$constant);
 2323   %}
 2324 
 2325   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2326     // Output immediate memory reference
 2327     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2328     emit_d32(cbuf, 0x00);
 2329   %}
 2330 
 2331   enc_class lock_prefix()
 2332   %{
 2333     emit_opcode(cbuf, 0xF0); // lock
 2334   %}
 2335 
 2336   enc_class REX_mem(memory mem)
 2337   %{
 2338     if ($mem$$base >= 8) {
 2339       if ($mem$$index < 8) {
 2340         emit_opcode(cbuf, Assembler::REX_B);
 2341       } else {
 2342         emit_opcode(cbuf, Assembler::REX_XB);
 2343       }
 2344     } else {
 2345       if ($mem$$index >= 8) {
 2346         emit_opcode(cbuf, Assembler::REX_X);
 2347       }
 2348     }
 2349   %}
 2350 
 2351   enc_class REX_mem_wide(memory mem)
 2352   %{
 2353     if ($mem$$base >= 8) {
 2354       if ($mem$$index < 8) {
 2355         emit_opcode(cbuf, Assembler::REX_WB);
 2356       } else {
 2357         emit_opcode(cbuf, Assembler::REX_WXB);
 2358       }
 2359     } else {
 2360       if ($mem$$index < 8) {
 2361         emit_opcode(cbuf, Assembler::REX_W);
 2362       } else {
 2363         emit_opcode(cbuf, Assembler::REX_WX);
 2364       }
 2365     }
 2366   %}
 2367 
 2368   // for byte regs
 2369   enc_class REX_breg(rRegI reg)
 2370   %{
 2371     if ($reg$$reg >= 4) {
 2372       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2373     }
 2374   %}
 2375 
 2376   // for byte regs
 2377   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2378   %{
 2379     if ($dst$$reg < 8) {
 2380       if ($src$$reg >= 4) {
 2381         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2382       }
 2383     } else {
 2384       if ($src$$reg < 8) {
 2385         emit_opcode(cbuf, Assembler::REX_R);
 2386       } else {
 2387         emit_opcode(cbuf, Assembler::REX_RB);
 2388       }
 2389     }
 2390   %}
 2391 
 2392   // for byte regs
 2393   enc_class REX_breg_mem(rRegI reg, memory mem)
 2394   %{
 2395     if ($reg$$reg < 8) {
 2396       if ($mem$$base < 8) {
 2397         if ($mem$$index >= 8) {
 2398           emit_opcode(cbuf, Assembler::REX_X);
 2399         } else if ($reg$$reg >= 4) {
 2400           emit_opcode(cbuf, Assembler::REX);
 2401         }
 2402       } else {
 2403         if ($mem$$index < 8) {
 2404           emit_opcode(cbuf, Assembler::REX_B);
 2405         } else {
 2406           emit_opcode(cbuf, Assembler::REX_XB);
 2407         }
 2408       }
 2409     } else {
 2410       if ($mem$$base < 8) {
 2411         if ($mem$$index < 8) {
 2412           emit_opcode(cbuf, Assembler::REX_R);
 2413         } else {
 2414           emit_opcode(cbuf, Assembler::REX_RX);
 2415         }
 2416       } else {
 2417         if ($mem$$index < 8) {
 2418           emit_opcode(cbuf, Assembler::REX_RB);
 2419         } else {
 2420           emit_opcode(cbuf, Assembler::REX_RXB);
 2421         }
 2422       }
 2423     }
 2424   %}
 2425 
 2426   enc_class REX_reg(rRegI reg)
 2427   %{
 2428     if ($reg$$reg >= 8) {
 2429       emit_opcode(cbuf, Assembler::REX_B);
 2430     }
 2431   %}
 2432 
 2433   enc_class REX_reg_wide(rRegI reg)
 2434   %{
 2435     if ($reg$$reg < 8) {
 2436       emit_opcode(cbuf, Assembler::REX_W);
 2437     } else {
 2438       emit_opcode(cbuf, Assembler::REX_WB);
 2439     }
 2440   %}
 2441 
 2442   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2443   %{
 2444     if ($dst$$reg < 8) {
 2445       if ($src$$reg >= 8) {
 2446         emit_opcode(cbuf, Assembler::REX_B);
 2447       }
 2448     } else {
 2449       if ($src$$reg < 8) {
 2450         emit_opcode(cbuf, Assembler::REX_R);
 2451       } else {
 2452         emit_opcode(cbuf, Assembler::REX_RB);
 2453       }
 2454     }
 2455   %}
 2456 
 2457   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2458   %{
 2459     if ($dst$$reg < 8) {
 2460       if ($src$$reg < 8) {
 2461         emit_opcode(cbuf, Assembler::REX_W);
 2462       } else {
 2463         emit_opcode(cbuf, Assembler::REX_WB);
 2464       }
 2465     } else {
 2466       if ($src$$reg < 8) {
 2467         emit_opcode(cbuf, Assembler::REX_WR);
 2468       } else {
 2469         emit_opcode(cbuf, Assembler::REX_WRB);
 2470       }
 2471     }
 2472   %}
 2473 
 2474   enc_class REX_reg_mem(rRegI reg, memory mem)
 2475   %{
 2476     if ($reg$$reg < 8) {
 2477       if ($mem$$base < 8) {
 2478         if ($mem$$index >= 8) {
 2479           emit_opcode(cbuf, Assembler::REX_X);
 2480         }
 2481       } else {
 2482         if ($mem$$index < 8) {
 2483           emit_opcode(cbuf, Assembler::REX_B);
 2484         } else {
 2485           emit_opcode(cbuf, Assembler::REX_XB);
 2486         }
 2487       }
 2488     } else {
 2489       if ($mem$$base < 8) {
 2490         if ($mem$$index < 8) {
 2491           emit_opcode(cbuf, Assembler::REX_R);
 2492         } else {
 2493           emit_opcode(cbuf, Assembler::REX_RX);
 2494         }
 2495       } else {
 2496         if ($mem$$index < 8) {
 2497           emit_opcode(cbuf, Assembler::REX_RB);
 2498         } else {
 2499           emit_opcode(cbuf, Assembler::REX_RXB);
 2500         }
 2501       }
 2502     }
 2503   %}
 2504 
 2505   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2506   %{
 2507     if ($reg$$reg < 8) {
 2508       if ($mem$$base < 8) {
 2509         if ($mem$$index < 8) {
 2510           emit_opcode(cbuf, Assembler::REX_W);
 2511         } else {
 2512           emit_opcode(cbuf, Assembler::REX_WX);
 2513         }
 2514       } else {
 2515         if ($mem$$index < 8) {
 2516           emit_opcode(cbuf, Assembler::REX_WB);
 2517         } else {
 2518           emit_opcode(cbuf, Assembler::REX_WXB);
 2519         }
 2520       }
 2521     } else {
 2522       if ($mem$$base < 8) {
 2523         if ($mem$$index < 8) {
 2524           emit_opcode(cbuf, Assembler::REX_WR);
 2525         } else {
 2526           emit_opcode(cbuf, Assembler::REX_WRX);
 2527         }
 2528       } else {
 2529         if ($mem$$index < 8) {
 2530           emit_opcode(cbuf, Assembler::REX_WRB);
 2531         } else {
 2532           emit_opcode(cbuf, Assembler::REX_WRXB);
 2533         }
 2534       }
 2535     }
 2536   %}
 2537 
 2538   enc_class reg_mem(rRegI ereg, memory mem)
 2539   %{
 2540     // High registers handle in encode_RegMem
 2541     int reg = $ereg$$reg;
 2542     int base = $mem$$base;
 2543     int index = $mem$$index;
 2544     int scale = $mem$$scale;
 2545     int disp = $mem$$disp;
 2546     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2547 
 2548     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2549   %}
 2550 
 2551   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2552   %{
 2553     int rm_byte_opcode = $rm_opcode$$constant;
 2554 
 2555     // High registers handle in encode_RegMem
 2556     int base = $mem$$base;
 2557     int index = $mem$$index;
 2558     int scale = $mem$$scale;
 2559     int displace = $mem$$disp;
 2560 
 2561     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2562                                             // working with static
 2563                                             // globals
 2564     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2565                   disp_reloc);
 2566   %}
 2567 
 2568   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2569   %{
 2570     int reg_encoding = $dst$$reg;
 2571     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2572     int index        = 0x04;            // 0x04 indicates no index
 2573     int scale        = 0x00;            // 0x00 indicates no scale
 2574     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2575     relocInfo::relocType disp_reloc = relocInfo::none;
 2576     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2577                   disp_reloc);
 2578   %}
 2579 
 2580   enc_class neg_reg(rRegI dst)
 2581   %{
 2582     int dstenc = $dst$$reg;
 2583     if (dstenc >= 8) {
 2584       emit_opcode(cbuf, Assembler::REX_B);
 2585       dstenc -= 8;
 2586     }
 2587     // NEG $dst
 2588     emit_opcode(cbuf, 0xF7);
 2589     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2590   %}
 2591 
 2592   enc_class neg_reg_wide(rRegI dst)
 2593   %{
 2594     int dstenc = $dst$$reg;
 2595     if (dstenc < 8) {
 2596       emit_opcode(cbuf, Assembler::REX_W);
 2597     } else {
 2598       emit_opcode(cbuf, Assembler::REX_WB);
 2599       dstenc -= 8;
 2600     }
 2601     // NEG $dst
 2602     emit_opcode(cbuf, 0xF7);
 2603     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2604   %}
 2605 
 2606   enc_class setLT_reg(rRegI dst)
 2607   %{
 2608     int dstenc = $dst$$reg;
 2609     if (dstenc >= 8) {
 2610       emit_opcode(cbuf, Assembler::REX_B);
 2611       dstenc -= 8;
 2612     } else if (dstenc >= 4) {
 2613       emit_opcode(cbuf, Assembler::REX);
 2614     }
 2615     // SETLT $dst
 2616     emit_opcode(cbuf, 0x0F);
 2617     emit_opcode(cbuf, 0x9C);
 2618     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2619   %}
 2620 
 2621   enc_class setNZ_reg(rRegI dst)
 2622   %{
 2623     int dstenc = $dst$$reg;
 2624     if (dstenc >= 8) {
 2625       emit_opcode(cbuf, Assembler::REX_B);
 2626       dstenc -= 8;
 2627     } else if (dstenc >= 4) {
 2628       emit_opcode(cbuf, Assembler::REX);
 2629     }
 2630     // SETNZ $dst
 2631     emit_opcode(cbuf, 0x0F);
 2632     emit_opcode(cbuf, 0x95);
 2633     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2634   %}
 2635 
 2636 
 2637   // Compare the lonogs and set -1, 0, or 1 into dst
 2638   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2639   %{
 2640     int src1enc = $src1$$reg;
 2641     int src2enc = $src2$$reg;
 2642     int dstenc = $dst$$reg;
 2643 
 2644     // cmpq $src1, $src2
 2645     if (src1enc < 8) {
 2646       if (src2enc < 8) {
 2647         emit_opcode(cbuf, Assembler::REX_W);
 2648       } else {
 2649         emit_opcode(cbuf, Assembler::REX_WB);
 2650       }
 2651     } else {
 2652       if (src2enc < 8) {
 2653         emit_opcode(cbuf, Assembler::REX_WR);
 2654       } else {
 2655         emit_opcode(cbuf, Assembler::REX_WRB);
 2656       }
 2657     }
 2658     emit_opcode(cbuf, 0x3B);
 2659     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2660 
 2661     // movl $dst, -1
 2662     if (dstenc >= 8) {
 2663       emit_opcode(cbuf, Assembler::REX_B);
 2664     }
 2665     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2666     emit_d32(cbuf, -1);
 2667 
 2668     // jl,s done
 2669     emit_opcode(cbuf, 0x7C);
 2670     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2671 
 2672     // setne $dst
 2673     if (dstenc >= 4) {
 2674       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2675     }
 2676     emit_opcode(cbuf, 0x0F);
 2677     emit_opcode(cbuf, 0x95);
 2678     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2679 
 2680     // movzbl $dst, $dst
 2681     if (dstenc >= 4) {
 2682       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2683     }
 2684     emit_opcode(cbuf, 0x0F);
 2685     emit_opcode(cbuf, 0xB6);
 2686     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2687   %}
 2688 
 2689   enc_class Push_ResultXD(regD dst) %{
 2690     MacroAssembler _masm(&cbuf);
 2691     __ fstp_d(Address(rsp, 0));
 2692     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2693     __ addptr(rsp, 8);
 2694   %}
 2695 
 2696   enc_class Push_SrcXD(regD src) %{
 2697     MacroAssembler _masm(&cbuf);
 2698     __ subptr(rsp, 8);
 2699     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2700     __ fld_d(Address(rsp, 0));
 2701   %}
 2702 
 2703 
 2704   enc_class enc_rethrow()
 2705   %{
 2706     cbuf.set_insts_mark();
 2707     emit_opcode(cbuf, 0xE9); // jmp entry
 2708     emit_d32_reloc(cbuf,
 2709                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2710                    runtime_call_Relocation::spec(),
 2711                    RELOC_DISP32);
 2712   %}
 2713 
 2714 %}
 2715 
 2716 
 2717 
 2718 //----------FRAME--------------------------------------------------------------
 2719 // Definition of frame structure and management information.
 2720 //
 2721 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2722 //                             |   (to get allocators register number
 2723 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2724 //  r   CALLER     |        |
 2725 //  o     |        +--------+      pad to even-align allocators stack-slot
 2726 //  w     V        |  pad0  |        numbers; owned by CALLER
 2727 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2728 //  h     ^        |   in   |  5
 2729 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2730 //  |     |        |        |  3
 2731 //  |     |        +--------+
 2732 //  V     |        | old out|      Empty on Intel, window on Sparc
 2733 //        |    old |preserve|      Must be even aligned.
 2734 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2735 //        |        |   in   |  3   area for Intel ret address
 2736 //     Owned by    |preserve|      Empty on Sparc.
 2737 //       SELF      +--------+
 2738 //        |        |  pad2  |  2   pad to align old SP
 2739 //        |        +--------+  1
 2740 //        |        | locks  |  0
 2741 //        |        +--------+----> OptoReg::stack0(), even aligned
 2742 //        |        |  pad1  | 11   pad to align new SP
 2743 //        |        +--------+
 2744 //        |        |        | 10
 2745 //        |        | spills |  9   spills
 2746 //        V        |        |  8   (pad0 slot for callee)
 2747 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2748 //        ^        |  out   |  7
 2749 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2750 //     Owned by    +--------+
 2751 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2752 //        |    new |preserve|      Must be even-aligned.
 2753 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2754 //        |        |        |
 2755 //
 2756 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2757 //         known from SELF's arguments and the Java calling convention.
 2758 //         Region 6-7 is determined per call site.
 2759 // Note 2: If the calling convention leaves holes in the incoming argument
 2760 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2761 //         are owned by the CALLEE.  Holes should not be nessecary in the
 2762 //         incoming area, as the Java calling convention is completely under
 2763 //         the control of the AD file.  Doubles can be sorted and packed to
 2764 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 2765 //         varargs C calling conventions.
 2766 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2767 //         even aligned with pad0 as needed.
 2768 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2769 //         region 6-11 is even aligned; it may be padded out more so that
 2770 //         the region from SP to FP meets the minimum stack alignment.
 2771 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2772 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2773 //         SP meets the minimum alignment.
 2774 
 2775 frame
 2776 %{
 2777   // These three registers define part of the calling convention
 2778   // between compiled code and the interpreter.
 2779   inline_cache_reg(RAX);                // Inline Cache Register
 2780 
 2781   // Optional: name the operand used by cisc-spilling to access
 2782   // [stack_pointer + offset]
 2783   cisc_spilling_operand_name(indOffset32);
 2784 
 2785   // Number of stack slots consumed by locking an object
 2786   sync_stack_slots(2);
 2787 
 2788   // Compiled code's Frame Pointer
 2789   frame_pointer(RSP);
 2790 
 2791   // Interpreter stores its frame pointer in a register which is
 2792   // stored to the stack by I2CAdaptors.
 2793   // I2CAdaptors convert from interpreted java to compiled java.
 2794   interpreter_frame_pointer(RBP);
 2795 
 2796   // Stack alignment requirement
 2797   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2798 
 2799   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2800   // for calls to C.  Supports the var-args backing area for register parms.
 2801   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2802 
 2803   // The after-PROLOG location of the return address.  Location of
 2804   // return address specifies a type (REG or STACK) and a number
 2805   // representing the register number (i.e. - use a register name) or
 2806   // stack slot.
 2807   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2808   // Otherwise, it is above the locks and verification slot and alignment word
 2809   return_addr(STACK - 2 +
 2810               align_up((Compile::current()->in_preserve_stack_slots() +
 2811                         Compile::current()->fixed_slots()),
 2812                        stack_alignment_in_slots()));
 2813 
 2814   // Location of compiled Java return values.  Same as C for now.
 2815   return_value
 2816   %{
 2817     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2818            "only return normal values");
 2819 
 2820     static const int lo[Op_RegL + 1] = {
 2821       0,
 2822       0,
 2823       RAX_num,  // Op_RegN
 2824       RAX_num,  // Op_RegI
 2825       RAX_num,  // Op_RegP
 2826       XMM0_num, // Op_RegF
 2827       XMM0_num, // Op_RegD
 2828       RAX_num   // Op_RegL
 2829     };
 2830     static const int hi[Op_RegL + 1] = {
 2831       0,
 2832       0,
 2833       OptoReg::Bad, // Op_RegN
 2834       OptoReg::Bad, // Op_RegI
 2835       RAX_H_num,    // Op_RegP
 2836       OptoReg::Bad, // Op_RegF
 2837       XMM0b_num,    // Op_RegD
 2838       RAX_H_num     // Op_RegL
 2839     };
 2840     // Excluded flags and vector registers.
 2841     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2842     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2843   %}
 2844 %}
 2845 
 2846 //----------ATTRIBUTES---------------------------------------------------------
 2847 //----------Operand Attributes-------------------------------------------------
 2848 op_attrib op_cost(0);        // Required cost attribute
 2849 
 2850 //----------Instruction Attributes---------------------------------------------
 2851 ins_attrib ins_cost(100);       // Required cost attribute
 2852 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2853 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2854                                 // a non-matching short branch variant
 2855                                 // of some long branch?
 2856 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2857                                 // be a power of 2) specifies the
 2858                                 // alignment that some part of the
 2859                                 // instruction (not necessarily the
 2860                                 // start) requires.  If > 1, a
 2861                                 // compute_padding() function must be
 2862                                 // provided for the instruction
 2863 
 2864 //----------OPERANDS-----------------------------------------------------------
 2865 // Operand definitions must precede instruction definitions for correct parsing
 2866 // in the ADLC because operands constitute user defined types which are used in
 2867 // instruction definitions.
 2868 
 2869 //----------Simple Operands----------------------------------------------------
 2870 // Immediate Operands
 2871 // Integer Immediate
 2872 operand immI()
 2873 %{
 2874   match(ConI);
 2875 
 2876   op_cost(10);
 2877   format %{ %}
 2878   interface(CONST_INTER);
 2879 %}
 2880 
 2881 // Constant for test vs zero
 2882 operand immI_0()
 2883 %{
 2884   predicate(n->get_int() == 0);
 2885   match(ConI);
 2886 
 2887   op_cost(0);
 2888   format %{ %}
 2889   interface(CONST_INTER);
 2890 %}
 2891 
 2892 // Constant for increment
 2893 operand immI_1()
 2894 %{
 2895   predicate(n->get_int() == 1);
 2896   match(ConI);
 2897 
 2898   op_cost(0);
 2899   format %{ %}
 2900   interface(CONST_INTER);
 2901 %}
 2902 
 2903 // Constant for decrement
 2904 operand immI_M1()
 2905 %{
 2906   predicate(n->get_int() == -1);
 2907   match(ConI);
 2908 
 2909   op_cost(0);
 2910   format %{ %}
 2911   interface(CONST_INTER);
 2912 %}
 2913 
 2914 operand immI_2()
 2915 %{
 2916   predicate(n->get_int() == 2);
 2917   match(ConI);
 2918 
 2919   op_cost(0);
 2920   format %{ %}
 2921   interface(CONST_INTER);
 2922 %}
 2923 
 2924 operand immI_4()
 2925 %{
 2926   predicate(n->get_int() == 4);
 2927   match(ConI);
 2928 
 2929   op_cost(0);
 2930   format %{ %}
 2931   interface(CONST_INTER);
 2932 %}
 2933 
 2934 operand immI_8()
 2935 %{
 2936   predicate(n->get_int() == 8);
 2937   match(ConI);
 2938 
 2939   op_cost(0);
 2940   format %{ %}
 2941   interface(CONST_INTER);
 2942 %}
 2943 
 2944 // Valid scale values for addressing modes
 2945 operand immI2()
 2946 %{
 2947   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2948   match(ConI);
 2949 
 2950   format %{ %}
 2951   interface(CONST_INTER);
 2952 %}
 2953 
 2954 operand immU7()
 2955 %{
 2956   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2957   match(ConI);
 2958 
 2959   op_cost(5);
 2960   format %{ %}
 2961   interface(CONST_INTER);
 2962 %}
 2963 
 2964 operand immI8()
 2965 %{
 2966   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2967   match(ConI);
 2968 
 2969   op_cost(5);
 2970   format %{ %}
 2971   interface(CONST_INTER);
 2972 %}
 2973 
 2974 operand immU8()
 2975 %{
 2976   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2977   match(ConI);
 2978 
 2979   op_cost(5);
 2980   format %{ %}
 2981   interface(CONST_INTER);
 2982 %}
 2983 
 2984 operand immI16()
 2985 %{
 2986   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2987   match(ConI);
 2988 
 2989   op_cost(10);
 2990   format %{ %}
 2991   interface(CONST_INTER);
 2992 %}
 2993 
 2994 // Int Immediate non-negative
 2995 operand immU31()
 2996 %{
 2997   predicate(n->get_int() >= 0);
 2998   match(ConI);
 2999 
 3000   op_cost(0);
 3001   format %{ %}
 3002   interface(CONST_INTER);
 3003 %}
 3004 
 3005 // Constant for long shifts
 3006 operand immI_32()
 3007 %{
 3008   predicate( n->get_int() == 32 );
 3009   match(ConI);
 3010 
 3011   op_cost(0);
 3012   format %{ %}
 3013   interface(CONST_INTER);
 3014 %}
 3015 
 3016 // Constant for long shifts
 3017 operand immI_64()
 3018 %{
 3019   predicate( n->get_int() == 64 );
 3020   match(ConI);
 3021 
 3022   op_cost(0);
 3023   format %{ %}
 3024   interface(CONST_INTER);
 3025 %}
 3026 
 3027 // Pointer Immediate
 3028 operand immP()
 3029 %{
 3030   match(ConP);
 3031 
 3032   op_cost(10);
 3033   format %{ %}
 3034   interface(CONST_INTER);
 3035 %}
 3036 
 3037 // NULL Pointer Immediate
 3038 operand immP0()
 3039 %{
 3040   predicate(n->get_ptr() == 0);
 3041   match(ConP);
 3042 
 3043   op_cost(5);
 3044   format %{ %}
 3045   interface(CONST_INTER);
 3046 %}
 3047 
 3048 // Pointer Immediate
 3049 operand immN() %{
 3050   match(ConN);
 3051 
 3052   op_cost(10);
 3053   format %{ %}
 3054   interface(CONST_INTER);
 3055 %}
 3056 
 3057 operand immNKlass() %{
 3058   match(ConNKlass);
 3059 
 3060   op_cost(10);
 3061   format %{ %}
 3062   interface(CONST_INTER);
 3063 %}
 3064 
 3065 // NULL Pointer Immediate
 3066 operand immN0() %{
 3067   predicate(n->get_narrowcon() == 0);
 3068   match(ConN);
 3069 
 3070   op_cost(5);
 3071   format %{ %}
 3072   interface(CONST_INTER);
 3073 %}
 3074 
 3075 operand immP31()
 3076 %{
 3077   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3078             && (n->get_ptr() >> 31) == 0);
 3079   match(ConP);
 3080 
 3081   op_cost(5);
 3082   format %{ %}
 3083   interface(CONST_INTER);
 3084 %}
 3085 
 3086 
 3087 // Long Immediate
 3088 operand immL()
 3089 %{
 3090   match(ConL);
 3091 
 3092   op_cost(20);
 3093   format %{ %}
 3094   interface(CONST_INTER);
 3095 %}
 3096 
 3097 // Long Immediate 8-bit
 3098 operand immL8()
 3099 %{
 3100   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3101   match(ConL);
 3102 
 3103   op_cost(5);
 3104   format %{ %}
 3105   interface(CONST_INTER);
 3106 %}
 3107 
 3108 // Long Immediate 32-bit unsigned
 3109 operand immUL32()
 3110 %{
 3111   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3112   match(ConL);
 3113 
 3114   op_cost(10);
 3115   format %{ %}
 3116   interface(CONST_INTER);
 3117 %}
 3118 
 3119 // Long Immediate 32-bit signed
 3120 operand immL32()
 3121 %{
 3122   predicate(n->get_long() == (int) (n->get_long()));
 3123   match(ConL);
 3124 
 3125   op_cost(15);
 3126   format %{ %}
 3127   interface(CONST_INTER);
 3128 %}
 3129 
 3130 operand immL_Pow2()
 3131 %{
 3132   predicate(is_power_of_2((julong)n->get_long()));
 3133   match(ConL);
 3134 
 3135   op_cost(15);
 3136   format %{ %}
 3137   interface(CONST_INTER);
 3138 %}
 3139 
 3140 operand immL_NotPow2()
 3141 %{
 3142   predicate(is_power_of_2((julong)~n->get_long()));
 3143   match(ConL);
 3144 
 3145   op_cost(15);
 3146   format %{ %}
 3147   interface(CONST_INTER);
 3148 %}
 3149 
 3150 // Long Immediate zero
 3151 operand immL0()
 3152 %{
 3153   predicate(n->get_long() == 0L);
 3154   match(ConL);
 3155 
 3156   op_cost(10);
 3157   format %{ %}
 3158   interface(CONST_INTER);
 3159 %}
 3160 
 3161 // Constant for increment
 3162 operand immL1()
 3163 %{
 3164   predicate(n->get_long() == 1);
 3165   match(ConL);
 3166 
 3167   format %{ %}
 3168   interface(CONST_INTER);
 3169 %}
 3170 
 3171 // Constant for decrement
 3172 operand immL_M1()
 3173 %{
 3174   predicate(n->get_long() == -1);
 3175   match(ConL);
 3176 
 3177   format %{ %}
 3178   interface(CONST_INTER);
 3179 %}
 3180 
 3181 // Long Immediate: the value 10
 3182 operand immL10()
 3183 %{
 3184   predicate(n->get_long() == 10);
 3185   match(ConL);
 3186 
 3187   format %{ %}
 3188   interface(CONST_INTER);
 3189 %}
 3190 
 3191 // Long immediate from 0 to 127.
 3192 // Used for a shorter form of long mul by 10.
 3193 operand immL_127()
 3194 %{
 3195   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3196   match(ConL);
 3197 
 3198   op_cost(10);
 3199   format %{ %}
 3200   interface(CONST_INTER);
 3201 %}
 3202 
 3203 // Long Immediate: low 32-bit mask
 3204 operand immL_32bits()
 3205 %{
 3206   predicate(n->get_long() == 0xFFFFFFFFL);
 3207   match(ConL);
 3208   op_cost(20);
 3209 
 3210   format %{ %}
 3211   interface(CONST_INTER);
 3212 %}
 3213 
 3214 // Int Immediate: 2^n-1, postive
 3215 operand immI_Pow2M1()
 3216 %{
 3217   predicate((n->get_int() > 0)
 3218             && is_power_of_2(n->get_int() + 1));
 3219   match(ConI);
 3220 
 3221   op_cost(20);
 3222   format %{ %}
 3223   interface(CONST_INTER);
 3224 %}
 3225 
 3226 // Float Immediate zero
 3227 operand immF0()
 3228 %{
 3229   predicate(jint_cast(n->getf()) == 0);
 3230   match(ConF);
 3231 
 3232   op_cost(5);
 3233   format %{ %}
 3234   interface(CONST_INTER);
 3235 %}
 3236 
 3237 // Float Immediate
 3238 operand immF()
 3239 %{
 3240   match(ConF);
 3241 
 3242   op_cost(15);
 3243   format %{ %}
 3244   interface(CONST_INTER);
 3245 %}
 3246 
 3247 // Double Immediate zero
 3248 operand immD0()
 3249 %{
 3250   predicate(jlong_cast(n->getd()) == 0);
 3251   match(ConD);
 3252 
 3253   op_cost(5);
 3254   format %{ %}
 3255   interface(CONST_INTER);
 3256 %}
 3257 
 3258 // Double Immediate
 3259 operand immD()
 3260 %{
 3261   match(ConD);
 3262 
 3263   op_cost(15);
 3264   format %{ %}
 3265   interface(CONST_INTER);
 3266 %}
 3267 
 3268 // Immediates for special shifts (sign extend)
 3269 
 3270 // Constants for increment
 3271 operand immI_16()
 3272 %{
 3273   predicate(n->get_int() == 16);
 3274   match(ConI);
 3275 
 3276   format %{ %}
 3277   interface(CONST_INTER);
 3278 %}
 3279 
 3280 operand immI_24()
 3281 %{
 3282   predicate(n->get_int() == 24);
 3283   match(ConI);
 3284 
 3285   format %{ %}
 3286   interface(CONST_INTER);
 3287 %}
 3288 
 3289 // Constant for byte-wide masking
 3290 operand immI_255()
 3291 %{
 3292   predicate(n->get_int() == 255);
 3293   match(ConI);
 3294 
 3295   format %{ %}
 3296   interface(CONST_INTER);
 3297 %}
 3298 
 3299 // Constant for short-wide masking
 3300 operand immI_65535()
 3301 %{
 3302   predicate(n->get_int() == 65535);
 3303   match(ConI);
 3304 
 3305   format %{ %}
 3306   interface(CONST_INTER);
 3307 %}
 3308 
 3309 // Constant for byte-wide masking
 3310 operand immL_255()
 3311 %{
 3312   predicate(n->get_long() == 255);
 3313   match(ConL);
 3314 
 3315   format %{ %}
 3316   interface(CONST_INTER);
 3317 %}
 3318 
 3319 // Constant for short-wide masking
 3320 operand immL_65535()
 3321 %{
 3322   predicate(n->get_long() == 65535);
 3323   match(ConL);
 3324 
 3325   format %{ %}
 3326   interface(CONST_INTER);
 3327 %}
 3328 
 3329 operand kReg()
 3330 %{
 3331   constraint(ALLOC_IN_RC(vectmask_reg));
 3332   match(RegVectMask);
 3333   format %{%}
 3334   interface(REG_INTER);
 3335 %}
 3336 
 3337 operand kReg_K1()
 3338 %{
 3339   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3340   match(RegVectMask);
 3341   format %{%}
 3342   interface(REG_INTER);
 3343 %}
 3344 
 3345 operand kReg_K2()
 3346 %{
 3347   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3348   match(RegVectMask);
 3349   format %{%}
 3350   interface(REG_INTER);
 3351 %}
 3352 
 3353 // Special Registers
 3354 operand kReg_K3()
 3355 %{
 3356   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3357   match(RegVectMask);
 3358   format %{%}
 3359   interface(REG_INTER);
 3360 %}
 3361 
 3362 operand kReg_K4()
 3363 %{
 3364   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3365   match(RegVectMask);
 3366   format %{%}
 3367   interface(REG_INTER);
 3368 %}
 3369 
 3370 operand kReg_K5()
 3371 %{
 3372   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3373   match(RegVectMask);
 3374   format %{%}
 3375   interface(REG_INTER);
 3376 %}
 3377 
 3378 operand kReg_K6()
 3379 %{
 3380   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3381   match(RegVectMask);
 3382   format %{%}
 3383   interface(REG_INTER);
 3384 %}
 3385 
 3386 // Special Registers
 3387 operand kReg_K7()
 3388 %{
 3389   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3390   match(RegVectMask);
 3391   format %{%}
 3392   interface(REG_INTER);
 3393 %}
 3394 
 3395 // Register Operands
 3396 // Integer Register
 3397 operand rRegI()
 3398 %{
 3399   constraint(ALLOC_IN_RC(int_reg));
 3400   match(RegI);
 3401 
 3402   match(rax_RegI);
 3403   match(rbx_RegI);
 3404   match(rcx_RegI);
 3405   match(rdx_RegI);
 3406   match(rdi_RegI);
 3407 
 3408   format %{ %}
 3409   interface(REG_INTER);
 3410 %}
 3411 
 3412 // Special Registers
 3413 operand rax_RegI()
 3414 %{
 3415   constraint(ALLOC_IN_RC(int_rax_reg));
 3416   match(RegI);
 3417   match(rRegI);
 3418 
 3419   format %{ "RAX" %}
 3420   interface(REG_INTER);
 3421 %}
 3422 
 3423 // Special Registers
 3424 operand rbx_RegI()
 3425 %{
 3426   constraint(ALLOC_IN_RC(int_rbx_reg));
 3427   match(RegI);
 3428   match(rRegI);
 3429 
 3430   format %{ "RBX" %}
 3431   interface(REG_INTER);
 3432 %}
 3433 
 3434 operand rcx_RegI()
 3435 %{
 3436   constraint(ALLOC_IN_RC(int_rcx_reg));
 3437   match(RegI);
 3438   match(rRegI);
 3439 
 3440   format %{ "RCX" %}
 3441   interface(REG_INTER);
 3442 %}
 3443 
 3444 operand rdx_RegI()
 3445 %{
 3446   constraint(ALLOC_IN_RC(int_rdx_reg));
 3447   match(RegI);
 3448   match(rRegI);
 3449 
 3450   format %{ "RDX" %}
 3451   interface(REG_INTER);
 3452 %}
 3453 
 3454 operand rdi_RegI()
 3455 %{
 3456   constraint(ALLOC_IN_RC(int_rdi_reg));
 3457   match(RegI);
 3458   match(rRegI);
 3459 
 3460   format %{ "RDI" %}
 3461   interface(REG_INTER);
 3462 %}
 3463 
 3464 operand no_rax_rdx_RegI()
 3465 %{
 3466   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3467   match(RegI);
 3468   match(rbx_RegI);
 3469   match(rcx_RegI);
 3470   match(rdi_RegI);
 3471 
 3472   format %{ %}
 3473   interface(REG_INTER);
 3474 %}
 3475 
 3476 // Pointer Register
 3477 operand any_RegP()
 3478 %{
 3479   constraint(ALLOC_IN_RC(any_reg));
 3480   match(RegP);
 3481   match(rax_RegP);
 3482   match(rbx_RegP);
 3483   match(rdi_RegP);
 3484   match(rsi_RegP);
 3485   match(rbp_RegP);
 3486   match(r15_RegP);
 3487   match(rRegP);
 3488 
 3489   format %{ %}
 3490   interface(REG_INTER);
 3491 %}
 3492 
 3493 operand rRegP()
 3494 %{
 3495   constraint(ALLOC_IN_RC(ptr_reg));
 3496   match(RegP);
 3497   match(rax_RegP);
 3498   match(rbx_RegP);
 3499   match(rdi_RegP);
 3500   match(rsi_RegP);
 3501   match(rbp_RegP);  // See Q&A below about
 3502   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3503 
 3504   format %{ %}
 3505   interface(REG_INTER);
 3506 %}
 3507 
 3508 operand rRegN() %{
 3509   constraint(ALLOC_IN_RC(int_reg));
 3510   match(RegN);
 3511 
 3512   format %{ %}
 3513   interface(REG_INTER);
 3514 %}
 3515 
 3516 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3517 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3518 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3519 // The output of an instruction is controlled by the allocator, which respects
 3520 // register class masks, not match rules.  Unless an instruction mentions
 3521 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3522 // by the allocator as an input.
 3523 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3524 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3525 // result, RBP is not included in the output of the instruction either.
 3526 
 3527 operand no_rax_RegP()
 3528 %{
 3529   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3530   match(RegP);
 3531   match(rbx_RegP);
 3532   match(rsi_RegP);
 3533   match(rdi_RegP);
 3534 
 3535   format %{ %}
 3536   interface(REG_INTER);
 3537 %}
 3538 
 3539 // This operand is not allowed to use RBP even if
 3540 // RBP is not used to hold the frame pointer.
 3541 operand no_rbp_RegP()
 3542 %{
 3543   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3544   match(RegP);
 3545   match(rbx_RegP);
 3546   match(rsi_RegP);
 3547   match(rdi_RegP);
 3548 
 3549   format %{ %}
 3550   interface(REG_INTER);
 3551 %}
 3552 
 3553 operand no_rax_rbx_RegP()
 3554 %{
 3555   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3556   match(RegP);
 3557   match(rsi_RegP);
 3558   match(rdi_RegP);
 3559 
 3560   format %{ %}
 3561   interface(REG_INTER);
 3562 %}
 3563 
 3564 // Special Registers
 3565 // Return a pointer value
 3566 operand rax_RegP()
 3567 %{
 3568   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3569   match(RegP);
 3570   match(rRegP);
 3571 
 3572   format %{ %}
 3573   interface(REG_INTER);
 3574 %}
 3575 
 3576 // Special Registers
 3577 // Return a compressed pointer value
 3578 operand rax_RegN()
 3579 %{
 3580   constraint(ALLOC_IN_RC(int_rax_reg));
 3581   match(RegN);
 3582   match(rRegN);
 3583 
 3584   format %{ %}
 3585   interface(REG_INTER);
 3586 %}
 3587 
 3588 // Used in AtomicAdd
 3589 operand rbx_RegP()
 3590 %{
 3591   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3592   match(RegP);
 3593   match(rRegP);
 3594 
 3595   format %{ %}
 3596   interface(REG_INTER);
 3597 %}
 3598 
 3599 operand rsi_RegP()
 3600 %{
 3601   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3602   match(RegP);
 3603   match(rRegP);
 3604 
 3605   format %{ %}
 3606   interface(REG_INTER);
 3607 %}
 3608 
 3609 operand rbp_RegP()
 3610 %{
 3611   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3612   match(RegP);
 3613   match(rRegP);
 3614 
 3615   format %{ %}
 3616   interface(REG_INTER);
 3617 %}
 3618 
 3619 // Used in rep stosq
 3620 operand rdi_RegP()
 3621 %{
 3622   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3623   match(RegP);
 3624   match(rRegP);
 3625 
 3626   format %{ %}
 3627   interface(REG_INTER);
 3628 %}
 3629 
 3630 operand r15_RegP()
 3631 %{
 3632   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3633   match(RegP);
 3634   match(rRegP);
 3635 
 3636   format %{ %}
 3637   interface(REG_INTER);
 3638 %}
 3639 
 3640 operand rRegL()
 3641 %{
 3642   constraint(ALLOC_IN_RC(long_reg));
 3643   match(RegL);
 3644   match(rax_RegL);
 3645   match(rdx_RegL);
 3646 
 3647   format %{ %}
 3648   interface(REG_INTER);
 3649 %}
 3650 
 3651 // Special Registers
 3652 operand no_rax_rdx_RegL()
 3653 %{
 3654   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3655   match(RegL);
 3656   match(rRegL);
 3657 
 3658   format %{ %}
 3659   interface(REG_INTER);
 3660 %}
 3661 
 3662 operand no_rax_RegL()
 3663 %{
 3664   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3665   match(RegL);
 3666   match(rRegL);
 3667   match(rdx_RegL);
 3668 
 3669   format %{ %}
 3670   interface(REG_INTER);
 3671 %}
 3672 
 3673 operand rax_RegL()
 3674 %{
 3675   constraint(ALLOC_IN_RC(long_rax_reg));
 3676   match(RegL);
 3677   match(rRegL);
 3678 
 3679   format %{ "RAX" %}
 3680   interface(REG_INTER);
 3681 %}
 3682 
 3683 operand rcx_RegL()
 3684 %{
 3685   constraint(ALLOC_IN_RC(long_rcx_reg));
 3686   match(RegL);
 3687   match(rRegL);
 3688 
 3689   format %{ %}
 3690   interface(REG_INTER);
 3691 %}
 3692 
 3693 operand rdx_RegL()
 3694 %{
 3695   constraint(ALLOC_IN_RC(long_rdx_reg));
 3696   match(RegL);
 3697   match(rRegL);
 3698 
 3699   format %{ %}
 3700   interface(REG_INTER);
 3701 %}
 3702 
 3703 // Flags register, used as output of compare instructions
 3704 operand rFlagsReg()
 3705 %{
 3706   constraint(ALLOC_IN_RC(int_flags));
 3707   match(RegFlags);
 3708 
 3709   format %{ "RFLAGS" %}
 3710   interface(REG_INTER);
 3711 %}
 3712 
 3713 // Flags register, used as output of FLOATING POINT compare instructions
 3714 operand rFlagsRegU()
 3715 %{
 3716   constraint(ALLOC_IN_RC(int_flags));
 3717   match(RegFlags);
 3718 
 3719   format %{ "RFLAGS_U" %}
 3720   interface(REG_INTER);
 3721 %}
 3722 
 3723 operand rFlagsRegUCF() %{
 3724   constraint(ALLOC_IN_RC(int_flags));
 3725   match(RegFlags);
 3726   predicate(false);
 3727 
 3728   format %{ "RFLAGS_U_CF" %}
 3729   interface(REG_INTER);
 3730 %}
 3731 
 3732 // Float register operands
 3733 operand regF() %{
 3734    constraint(ALLOC_IN_RC(float_reg));
 3735    match(RegF);
 3736 
 3737    format %{ %}
 3738    interface(REG_INTER);
 3739 %}
 3740 
 3741 // Float register operands
 3742 operand legRegF() %{
 3743    constraint(ALLOC_IN_RC(float_reg_legacy));
 3744    match(RegF);
 3745 
 3746    format %{ %}
 3747    interface(REG_INTER);
 3748 %}
 3749 
 3750 // Float register operands
 3751 operand vlRegF() %{
 3752    constraint(ALLOC_IN_RC(float_reg_vl));
 3753    match(RegF);
 3754 
 3755    format %{ %}
 3756    interface(REG_INTER);
 3757 %}
 3758 
 3759 // Double register operands
 3760 operand regD() %{
 3761    constraint(ALLOC_IN_RC(double_reg));
 3762    match(RegD);
 3763 
 3764    format %{ %}
 3765    interface(REG_INTER);
 3766 %}
 3767 
 3768 // Double register operands
 3769 operand legRegD() %{
 3770    constraint(ALLOC_IN_RC(double_reg_legacy));
 3771    match(RegD);
 3772 
 3773    format %{ %}
 3774    interface(REG_INTER);
 3775 %}
 3776 
 3777 // Double register operands
 3778 operand vlRegD() %{
 3779    constraint(ALLOC_IN_RC(double_reg_vl));
 3780    match(RegD);
 3781 
 3782    format %{ %}
 3783    interface(REG_INTER);
 3784 %}
 3785 
 3786 //----------Memory Operands----------------------------------------------------
 3787 // Direct Memory Operand
 3788 // operand direct(immP addr)
 3789 // %{
 3790 //   match(addr);
 3791 
 3792 //   format %{ "[$addr]" %}
 3793 //   interface(MEMORY_INTER) %{
 3794 //     base(0xFFFFFFFF);
 3795 //     index(0x4);
 3796 //     scale(0x0);
 3797 //     disp($addr);
 3798 //   %}
 3799 // %}
 3800 
 3801 // Indirect Memory Operand
 3802 operand indirect(any_RegP reg)
 3803 %{
 3804   constraint(ALLOC_IN_RC(ptr_reg));
 3805   match(reg);
 3806 
 3807   format %{ "[$reg]" %}
 3808   interface(MEMORY_INTER) %{
 3809     base($reg);
 3810     index(0x4);
 3811     scale(0x0);
 3812     disp(0x0);
 3813   %}
 3814 %}
 3815 
 3816 // Indirect Memory Plus Short Offset Operand
 3817 operand indOffset8(any_RegP reg, immL8 off)
 3818 %{
 3819   constraint(ALLOC_IN_RC(ptr_reg));
 3820   match(AddP reg off);
 3821 
 3822   format %{ "[$reg + $off (8-bit)]" %}
 3823   interface(MEMORY_INTER) %{
 3824     base($reg);
 3825     index(0x4);
 3826     scale(0x0);
 3827     disp($off);
 3828   %}
 3829 %}
 3830 
 3831 // Indirect Memory Plus Long Offset Operand
 3832 operand indOffset32(any_RegP reg, immL32 off)
 3833 %{
 3834   constraint(ALLOC_IN_RC(ptr_reg));
 3835   match(AddP reg off);
 3836 
 3837   format %{ "[$reg + $off (32-bit)]" %}
 3838   interface(MEMORY_INTER) %{
 3839     base($reg);
 3840     index(0x4);
 3841     scale(0x0);
 3842     disp($off);
 3843   %}
 3844 %}
 3845 
 3846 // Indirect Memory Plus Index Register Plus Offset Operand
 3847 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3848 %{
 3849   constraint(ALLOC_IN_RC(ptr_reg));
 3850   match(AddP (AddP reg lreg) off);
 3851 
 3852   op_cost(10);
 3853   format %{"[$reg + $off + $lreg]" %}
 3854   interface(MEMORY_INTER) %{
 3855     base($reg);
 3856     index($lreg);
 3857     scale(0x0);
 3858     disp($off);
 3859   %}
 3860 %}
 3861 
 3862 // Indirect Memory Plus Index Register Plus Offset Operand
 3863 operand indIndex(any_RegP reg, rRegL lreg)
 3864 %{
 3865   constraint(ALLOC_IN_RC(ptr_reg));
 3866   match(AddP reg lreg);
 3867 
 3868   op_cost(10);
 3869   format %{"[$reg + $lreg]" %}
 3870   interface(MEMORY_INTER) %{
 3871     base($reg);
 3872     index($lreg);
 3873     scale(0x0);
 3874     disp(0x0);
 3875   %}
 3876 %}
 3877 
 3878 // Indirect Memory Times Scale Plus Index Register
 3879 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3880 %{
 3881   constraint(ALLOC_IN_RC(ptr_reg));
 3882   match(AddP reg (LShiftL lreg scale));
 3883 
 3884   op_cost(10);
 3885   format %{"[$reg + $lreg << $scale]" %}
 3886   interface(MEMORY_INTER) %{
 3887     base($reg);
 3888     index($lreg);
 3889     scale($scale);
 3890     disp(0x0);
 3891   %}
 3892 %}
 3893 
 3894 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3895 %{
 3896   constraint(ALLOC_IN_RC(ptr_reg));
 3897   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3898   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3899 
 3900   op_cost(10);
 3901   format %{"[$reg + pos $idx << $scale]" %}
 3902   interface(MEMORY_INTER) %{
 3903     base($reg);
 3904     index($idx);
 3905     scale($scale);
 3906     disp(0x0);
 3907   %}
 3908 %}
 3909 
 3910 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3911 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3912 %{
 3913   constraint(ALLOC_IN_RC(ptr_reg));
 3914   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3915 
 3916   op_cost(10);
 3917   format %{"[$reg + $off + $lreg << $scale]" %}
 3918   interface(MEMORY_INTER) %{
 3919     base($reg);
 3920     index($lreg);
 3921     scale($scale);
 3922     disp($off);
 3923   %}
 3924 %}
 3925 
 3926 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3927 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3928 %{
 3929   constraint(ALLOC_IN_RC(ptr_reg));
 3930   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3931   match(AddP (AddP reg (ConvI2L idx)) off);
 3932 
 3933   op_cost(10);
 3934   format %{"[$reg + $off + $idx]" %}
 3935   interface(MEMORY_INTER) %{
 3936     base($reg);
 3937     index($idx);
 3938     scale(0x0);
 3939     disp($off);
 3940   %}
 3941 %}
 3942 
 3943 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3944 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3945 %{
 3946   constraint(ALLOC_IN_RC(ptr_reg));
 3947   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3948   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3949 
 3950   op_cost(10);
 3951   format %{"[$reg + $off + $idx << $scale]" %}
 3952   interface(MEMORY_INTER) %{
 3953     base($reg);
 3954     index($idx);
 3955     scale($scale);
 3956     disp($off);
 3957   %}
 3958 %}
 3959 
 3960 // Indirect Narrow Oop Plus Offset Operand
 3961 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3962 // we can't free r12 even with CompressedOops::base() == NULL.
 3963 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3964   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3965   constraint(ALLOC_IN_RC(ptr_reg));
 3966   match(AddP (DecodeN reg) off);
 3967 
 3968   op_cost(10);
 3969   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3970   interface(MEMORY_INTER) %{
 3971     base(0xc); // R12
 3972     index($reg);
 3973     scale(0x3);
 3974     disp($off);
 3975   %}
 3976 %}
 3977 
 3978 // Indirect Memory Operand
 3979 operand indirectNarrow(rRegN reg)
 3980 %{
 3981   predicate(CompressedOops::shift() == 0);
 3982   constraint(ALLOC_IN_RC(ptr_reg));
 3983   match(DecodeN reg);
 3984 
 3985   format %{ "[$reg]" %}
 3986   interface(MEMORY_INTER) %{
 3987     base($reg);
 3988     index(0x4);
 3989     scale(0x0);
 3990     disp(0x0);
 3991   %}
 3992 %}
 3993 
 3994 // Indirect Memory Plus Short Offset Operand
 3995 operand indOffset8Narrow(rRegN reg, immL8 off)
 3996 %{
 3997   predicate(CompressedOops::shift() == 0);
 3998   constraint(ALLOC_IN_RC(ptr_reg));
 3999   match(AddP (DecodeN reg) off);
 4000 
 4001   format %{ "[$reg + $off (8-bit)]" %}
 4002   interface(MEMORY_INTER) %{
 4003     base($reg);
 4004     index(0x4);
 4005     scale(0x0);
 4006     disp($off);
 4007   %}
 4008 %}
 4009 
 4010 // Indirect Memory Plus Long Offset Operand
 4011 operand indOffset32Narrow(rRegN reg, immL32 off)
 4012 %{
 4013   predicate(CompressedOops::shift() == 0);
 4014   constraint(ALLOC_IN_RC(ptr_reg));
 4015   match(AddP (DecodeN reg) off);
 4016 
 4017   format %{ "[$reg + $off (32-bit)]" %}
 4018   interface(MEMORY_INTER) %{
 4019     base($reg);
 4020     index(0x4);
 4021     scale(0x0);
 4022     disp($off);
 4023   %}
 4024 %}
 4025 
 4026 // Indirect Memory Plus Index Register Plus Offset Operand
 4027 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4028 %{
 4029   predicate(CompressedOops::shift() == 0);
 4030   constraint(ALLOC_IN_RC(ptr_reg));
 4031   match(AddP (AddP (DecodeN reg) lreg) off);
 4032 
 4033   op_cost(10);
 4034   format %{"[$reg + $off + $lreg]" %}
 4035   interface(MEMORY_INTER) %{
 4036     base($reg);
 4037     index($lreg);
 4038     scale(0x0);
 4039     disp($off);
 4040   %}
 4041 %}
 4042 
 4043 // Indirect Memory Plus Index Register Plus Offset Operand
 4044 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4045 %{
 4046   predicate(CompressedOops::shift() == 0);
 4047   constraint(ALLOC_IN_RC(ptr_reg));
 4048   match(AddP (DecodeN reg) lreg);
 4049 
 4050   op_cost(10);
 4051   format %{"[$reg + $lreg]" %}
 4052   interface(MEMORY_INTER) %{
 4053     base($reg);
 4054     index($lreg);
 4055     scale(0x0);
 4056     disp(0x0);
 4057   %}
 4058 %}
 4059 
 4060 // Indirect Memory Times Scale Plus Index Register
 4061 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4062 %{
 4063   predicate(CompressedOops::shift() == 0);
 4064   constraint(ALLOC_IN_RC(ptr_reg));
 4065   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4066 
 4067   op_cost(10);
 4068   format %{"[$reg + $lreg << $scale]" %}
 4069   interface(MEMORY_INTER) %{
 4070     base($reg);
 4071     index($lreg);
 4072     scale($scale);
 4073     disp(0x0);
 4074   %}
 4075 %}
 4076 
 4077 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4078 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4079 %{
 4080   predicate(CompressedOops::shift() == 0);
 4081   constraint(ALLOC_IN_RC(ptr_reg));
 4082   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4083 
 4084   op_cost(10);
 4085   format %{"[$reg + $off + $lreg << $scale]" %}
 4086   interface(MEMORY_INTER) %{
 4087     base($reg);
 4088     index($lreg);
 4089     scale($scale);
 4090     disp($off);
 4091   %}
 4092 %}
 4093 
 4094 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4095 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4096 %{
 4097   constraint(ALLOC_IN_RC(ptr_reg));
 4098   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4099   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4100 
 4101   op_cost(10);
 4102   format %{"[$reg + $off + $idx]" %}
 4103   interface(MEMORY_INTER) %{
 4104     base($reg);
 4105     index($idx);
 4106     scale(0x0);
 4107     disp($off);
 4108   %}
 4109 %}
 4110 
 4111 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4112 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4113 %{
 4114   constraint(ALLOC_IN_RC(ptr_reg));
 4115   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4116   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4117 
 4118   op_cost(10);
 4119   format %{"[$reg + $off + $idx << $scale]" %}
 4120   interface(MEMORY_INTER) %{
 4121     base($reg);
 4122     index($idx);
 4123     scale($scale);
 4124     disp($off);
 4125   %}
 4126 %}
 4127 
 4128 //----------Special Memory Operands--------------------------------------------
 4129 // Stack Slot Operand - This operand is used for loading and storing temporary
 4130 //                      values on the stack where a match requires a value to
 4131 //                      flow through memory.
 4132 operand stackSlotP(sRegP reg)
 4133 %{
 4134   constraint(ALLOC_IN_RC(stack_slots));
 4135   // No match rule because this operand is only generated in matching
 4136 
 4137   format %{ "[$reg]" %}
 4138   interface(MEMORY_INTER) %{
 4139     base(0x4);   // RSP
 4140     index(0x4);  // No Index
 4141     scale(0x0);  // No Scale
 4142     disp($reg);  // Stack Offset
 4143   %}
 4144 %}
 4145 
 4146 operand stackSlotI(sRegI reg)
 4147 %{
 4148   constraint(ALLOC_IN_RC(stack_slots));
 4149   // No match rule because this operand is only generated in matching
 4150 
 4151   format %{ "[$reg]" %}
 4152   interface(MEMORY_INTER) %{
 4153     base(0x4);   // RSP
 4154     index(0x4);  // No Index
 4155     scale(0x0);  // No Scale
 4156     disp($reg);  // Stack Offset
 4157   %}
 4158 %}
 4159 
 4160 operand stackSlotF(sRegF reg)
 4161 %{
 4162   constraint(ALLOC_IN_RC(stack_slots));
 4163   // No match rule because this operand is only generated in matching
 4164 
 4165   format %{ "[$reg]" %}
 4166   interface(MEMORY_INTER) %{
 4167     base(0x4);   // RSP
 4168     index(0x4);  // No Index
 4169     scale(0x0);  // No Scale
 4170     disp($reg);  // Stack Offset
 4171   %}
 4172 %}
 4173 
 4174 operand stackSlotD(sRegD reg)
 4175 %{
 4176   constraint(ALLOC_IN_RC(stack_slots));
 4177   // No match rule because this operand is only generated in matching
 4178 
 4179   format %{ "[$reg]" %}
 4180   interface(MEMORY_INTER) %{
 4181     base(0x4);   // RSP
 4182     index(0x4);  // No Index
 4183     scale(0x0);  // No Scale
 4184     disp($reg);  // Stack Offset
 4185   %}
 4186 %}
 4187 operand stackSlotL(sRegL reg)
 4188 %{
 4189   constraint(ALLOC_IN_RC(stack_slots));
 4190   // No match rule because this operand is only generated in matching
 4191 
 4192   format %{ "[$reg]" %}
 4193   interface(MEMORY_INTER) %{
 4194     base(0x4);   // RSP
 4195     index(0x4);  // No Index
 4196     scale(0x0);  // No Scale
 4197     disp($reg);  // Stack Offset
 4198   %}
 4199 %}
 4200 
 4201 //----------Conditional Branch Operands----------------------------------------
 4202 // Comparison Op  - This is the operation of the comparison, and is limited to
 4203 //                  the following set of codes:
 4204 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4205 //
 4206 // Other attributes of the comparison, such as unsignedness, are specified
 4207 // by the comparison instruction that sets a condition code flags register.
 4208 // That result is represented by a flags operand whose subtype is appropriate
 4209 // to the unsignedness (etc.) of the comparison.
 4210 //
 4211 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4212 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4213 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4214 
 4215 // Comparision Code
 4216 operand cmpOp()
 4217 %{
 4218   match(Bool);
 4219 
 4220   format %{ "" %}
 4221   interface(COND_INTER) %{
 4222     equal(0x4, "e");
 4223     not_equal(0x5, "ne");
 4224     less(0xC, "l");
 4225     greater_equal(0xD, "ge");
 4226     less_equal(0xE, "le");
 4227     greater(0xF, "g");
 4228     overflow(0x0, "o");
 4229     no_overflow(0x1, "no");
 4230   %}
 4231 %}
 4232 
 4233 // Comparison Code, unsigned compare.  Used by FP also, with
 4234 // C2 (unordered) turned into GT or LT already.  The other bits
 4235 // C0 and C3 are turned into Carry & Zero flags.
 4236 operand cmpOpU()
 4237 %{
 4238   match(Bool);
 4239 
 4240   format %{ "" %}
 4241   interface(COND_INTER) %{
 4242     equal(0x4, "e");
 4243     not_equal(0x5, "ne");
 4244     less(0x2, "b");
 4245     greater_equal(0x3, "nb");
 4246     less_equal(0x6, "be");
 4247     greater(0x7, "nbe");
 4248     overflow(0x0, "o");
 4249     no_overflow(0x1, "no");
 4250   %}
 4251 %}
 4252 
 4253 
 4254 // Floating comparisons that don't require any fixup for the unordered case
 4255 operand cmpOpUCF() %{
 4256   match(Bool);
 4257   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4258             n->as_Bool()->_test._test == BoolTest::ge ||
 4259             n->as_Bool()->_test._test == BoolTest::le ||
 4260             n->as_Bool()->_test._test == BoolTest::gt);
 4261   format %{ "" %}
 4262   interface(COND_INTER) %{
 4263     equal(0x4, "e");
 4264     not_equal(0x5, "ne");
 4265     less(0x2, "b");
 4266     greater_equal(0x3, "nb");
 4267     less_equal(0x6, "be");
 4268     greater(0x7, "nbe");
 4269     overflow(0x0, "o");
 4270     no_overflow(0x1, "no");
 4271   %}
 4272 %}
 4273 
 4274 
 4275 // Floating comparisons that can be fixed up with extra conditional jumps
 4276 operand cmpOpUCF2() %{
 4277   match(Bool);
 4278   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
 4279             n->as_Bool()->_test._test == BoolTest::eq);
 4280   format %{ "" %}
 4281   interface(COND_INTER) %{
 4282     equal(0x4, "e");
 4283     not_equal(0x5, "ne");
 4284     less(0x2, "b");
 4285     greater_equal(0x3, "nb");
 4286     less_equal(0x6, "be");
 4287     greater(0x7, "nbe");
 4288     overflow(0x0, "o");
 4289     no_overflow(0x1, "no");
 4290   %}
 4291 %}
 4292 
 4293 //----------OPERAND CLASSES----------------------------------------------------
 4294 // Operand Classes are groups of operands that are used as to simplify
 4295 // instruction definitions by not requiring the AD writer to specify separate
 4296 // instructions for every form of operand when the instruction accepts
 4297 // multiple operand types with the same basic encoding and format.  The classic
 4298 // case of this is memory operands.
 4299 
 4300 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4301                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4302                indCompressedOopOffset,
 4303                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4304                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4305                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4306 
 4307 //----------PIPELINE-----------------------------------------------------------
 4308 // Rules which define the behavior of the target architectures pipeline.
 4309 pipeline %{
 4310 
 4311 //----------ATTRIBUTES---------------------------------------------------------
 4312 attributes %{
 4313   variable_size_instructions;        // Fixed size instructions
 4314   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4315   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4316   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4317   instruction_fetch_units = 1;       // of 16 bytes
 4318 
 4319   // List of nop instructions
 4320   nops( MachNop );
 4321 %}
 4322 
 4323 //----------RESOURCES----------------------------------------------------------
 4324 // Resources are the functional units available to the machine
 4325 
 4326 // Generic P2/P3 pipeline
 4327 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4328 // 3 instructions decoded per cycle.
 4329 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4330 // 3 ALU op, only ALU0 handles mul instructions.
 4331 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4332            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4333            BR, FPU,
 4334            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4335 
 4336 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4337 // Pipeline Description specifies the stages in the machine's pipeline
 4338 
 4339 // Generic P2/P3 pipeline
 4340 pipe_desc(S0, S1, S2, S3, S4, S5);
 4341 
 4342 //----------PIPELINE CLASSES---------------------------------------------------
 4343 // Pipeline Classes describe the stages in which input and output are
 4344 // referenced by the hardware pipeline.
 4345 
 4346 // Naming convention: ialu or fpu
 4347 // Then: _reg
 4348 // Then: _reg if there is a 2nd register
 4349 // Then: _long if it's a pair of instructions implementing a long
 4350 // Then: _fat if it requires the big decoder
 4351 //   Or: _mem if it requires the big decoder and a memory unit.
 4352 
 4353 // Integer ALU reg operation
 4354 pipe_class ialu_reg(rRegI dst)
 4355 %{
 4356     single_instruction;
 4357     dst    : S4(write);
 4358     dst    : S3(read);
 4359     DECODE : S0;        // any decoder
 4360     ALU    : S3;        // any alu
 4361 %}
 4362 
 4363 // Long ALU reg operation
 4364 pipe_class ialu_reg_long(rRegL dst)
 4365 %{
 4366     instruction_count(2);
 4367     dst    : S4(write);
 4368     dst    : S3(read);
 4369     DECODE : S0(2);     // any 2 decoders
 4370     ALU    : S3(2);     // both alus
 4371 %}
 4372 
 4373 // Integer ALU reg operation using big decoder
 4374 pipe_class ialu_reg_fat(rRegI dst)
 4375 %{
 4376     single_instruction;
 4377     dst    : S4(write);
 4378     dst    : S3(read);
 4379     D0     : S0;        // big decoder only
 4380     ALU    : S3;        // any alu
 4381 %}
 4382 
 4383 // Integer ALU reg-reg operation
 4384 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4385 %{
 4386     single_instruction;
 4387     dst    : S4(write);
 4388     src    : S3(read);
 4389     DECODE : S0;        // any decoder
 4390     ALU    : S3;        // any alu
 4391 %}
 4392 
 4393 // Integer ALU reg-reg operation
 4394 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4395 %{
 4396     single_instruction;
 4397     dst    : S4(write);
 4398     src    : S3(read);
 4399     D0     : S0;        // big decoder only
 4400     ALU    : S3;        // any alu
 4401 %}
 4402 
 4403 // Integer ALU reg-mem operation
 4404 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4405 %{
 4406     single_instruction;
 4407     dst    : S5(write);
 4408     mem    : S3(read);
 4409     D0     : S0;        // big decoder only
 4410     ALU    : S4;        // any alu
 4411     MEM    : S3;        // any mem
 4412 %}
 4413 
 4414 // Integer mem operation (prefetch)
 4415 pipe_class ialu_mem(memory mem)
 4416 %{
 4417     single_instruction;
 4418     mem    : S3(read);
 4419     D0     : S0;        // big decoder only
 4420     MEM    : S3;        // any mem
 4421 %}
 4422 
 4423 // Integer Store to Memory
 4424 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4425 %{
 4426     single_instruction;
 4427     mem    : S3(read);
 4428     src    : S5(read);
 4429     D0     : S0;        // big decoder only
 4430     ALU    : S4;        // any alu
 4431     MEM    : S3;
 4432 %}
 4433 
 4434 // // Long Store to Memory
 4435 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4436 // %{
 4437 //     instruction_count(2);
 4438 //     mem    : S3(read);
 4439 //     src    : S5(read);
 4440 //     D0     : S0(2);          // big decoder only; twice
 4441 //     ALU    : S4(2);     // any 2 alus
 4442 //     MEM    : S3(2);  // Both mems
 4443 // %}
 4444 
 4445 // Integer Store to Memory
 4446 pipe_class ialu_mem_imm(memory mem)
 4447 %{
 4448     single_instruction;
 4449     mem    : S3(read);
 4450     D0     : S0;        // big decoder only
 4451     ALU    : S4;        // any alu
 4452     MEM    : S3;
 4453 %}
 4454 
 4455 // Integer ALU0 reg-reg operation
 4456 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4457 %{
 4458     single_instruction;
 4459     dst    : S4(write);
 4460     src    : S3(read);
 4461     D0     : S0;        // Big decoder only
 4462     ALU0   : S3;        // only alu0
 4463 %}
 4464 
 4465 // Integer ALU0 reg-mem operation
 4466 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4467 %{
 4468     single_instruction;
 4469     dst    : S5(write);
 4470     mem    : S3(read);
 4471     D0     : S0;        // big decoder only
 4472     ALU0   : S4;        // ALU0 only
 4473     MEM    : S3;        // any mem
 4474 %}
 4475 
 4476 // Integer ALU reg-reg operation
 4477 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4478 %{
 4479     single_instruction;
 4480     cr     : S4(write);
 4481     src1   : S3(read);
 4482     src2   : S3(read);
 4483     DECODE : S0;        // any decoder
 4484     ALU    : S3;        // any alu
 4485 %}
 4486 
 4487 // Integer ALU reg-imm operation
 4488 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4489 %{
 4490     single_instruction;
 4491     cr     : S4(write);
 4492     src1   : S3(read);
 4493     DECODE : S0;        // any decoder
 4494     ALU    : S3;        // any alu
 4495 %}
 4496 
 4497 // Integer ALU reg-mem operation
 4498 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4499 %{
 4500     single_instruction;
 4501     cr     : S4(write);
 4502     src1   : S3(read);
 4503     src2   : S3(read);
 4504     D0     : S0;        // big decoder only
 4505     ALU    : S4;        // any alu
 4506     MEM    : S3;
 4507 %}
 4508 
 4509 // Conditional move reg-reg
 4510 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4511 %{
 4512     instruction_count(4);
 4513     y      : S4(read);
 4514     q      : S3(read);
 4515     p      : S3(read);
 4516     DECODE : S0(4);     // any decoder
 4517 %}
 4518 
 4519 // Conditional move reg-reg
 4520 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4521 %{
 4522     single_instruction;
 4523     dst    : S4(write);
 4524     src    : S3(read);
 4525     cr     : S3(read);
 4526     DECODE : S0;        // any decoder
 4527 %}
 4528 
 4529 // Conditional move reg-mem
 4530 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4531 %{
 4532     single_instruction;
 4533     dst    : S4(write);
 4534     src    : S3(read);
 4535     cr     : S3(read);
 4536     DECODE : S0;        // any decoder
 4537     MEM    : S3;
 4538 %}
 4539 
 4540 // Conditional move reg-reg long
 4541 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4542 %{
 4543     single_instruction;
 4544     dst    : S4(write);
 4545     src    : S3(read);
 4546     cr     : S3(read);
 4547     DECODE : S0(2);     // any 2 decoders
 4548 %}
 4549 
 4550 // XXX
 4551 // // Conditional move double reg-reg
 4552 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4553 // %{
 4554 //     single_instruction;
 4555 //     dst    : S4(write);
 4556 //     src    : S3(read);
 4557 //     cr     : S3(read);
 4558 //     DECODE : S0;     // any decoder
 4559 // %}
 4560 
 4561 // Float reg-reg operation
 4562 pipe_class fpu_reg(regD dst)
 4563 %{
 4564     instruction_count(2);
 4565     dst    : S3(read);
 4566     DECODE : S0(2);     // any 2 decoders
 4567     FPU    : S3;
 4568 %}
 4569 
 4570 // Float reg-reg operation
 4571 pipe_class fpu_reg_reg(regD dst, regD src)
 4572 %{
 4573     instruction_count(2);
 4574     dst    : S4(write);
 4575     src    : S3(read);
 4576     DECODE : S0(2);     // any 2 decoders
 4577     FPU    : S3;
 4578 %}
 4579 
 4580 // Float reg-reg operation
 4581 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4582 %{
 4583     instruction_count(3);
 4584     dst    : S4(write);
 4585     src1   : S3(read);
 4586     src2   : S3(read);
 4587     DECODE : S0(3);     // any 3 decoders
 4588     FPU    : S3(2);
 4589 %}
 4590 
 4591 // Float reg-reg operation
 4592 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4593 %{
 4594     instruction_count(4);
 4595     dst    : S4(write);
 4596     src1   : S3(read);
 4597     src2   : S3(read);
 4598     src3   : S3(read);
 4599     DECODE : S0(4);     // any 3 decoders
 4600     FPU    : S3(2);
 4601 %}
 4602 
 4603 // Float reg-reg operation
 4604 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4605 %{
 4606     instruction_count(4);
 4607     dst    : S4(write);
 4608     src1   : S3(read);
 4609     src2   : S3(read);
 4610     src3   : S3(read);
 4611     DECODE : S1(3);     // any 3 decoders
 4612     D0     : S0;        // Big decoder only
 4613     FPU    : S3(2);
 4614     MEM    : S3;
 4615 %}
 4616 
 4617 // Float reg-mem operation
 4618 pipe_class fpu_reg_mem(regD dst, memory mem)
 4619 %{
 4620     instruction_count(2);
 4621     dst    : S5(write);
 4622     mem    : S3(read);
 4623     D0     : S0;        // big decoder only
 4624     DECODE : S1;        // any decoder for FPU POP
 4625     FPU    : S4;
 4626     MEM    : S3;        // any mem
 4627 %}
 4628 
 4629 // Float reg-mem operation
 4630 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4631 %{
 4632     instruction_count(3);
 4633     dst    : S5(write);
 4634     src1   : S3(read);
 4635     mem    : S3(read);
 4636     D0     : S0;        // big decoder only
 4637     DECODE : S1(2);     // any decoder for FPU POP
 4638     FPU    : S4;
 4639     MEM    : S3;        // any mem
 4640 %}
 4641 
 4642 // Float mem-reg operation
 4643 pipe_class fpu_mem_reg(memory mem, regD src)
 4644 %{
 4645     instruction_count(2);
 4646     src    : S5(read);
 4647     mem    : S3(read);
 4648     DECODE : S0;        // any decoder for FPU PUSH
 4649     D0     : S1;        // big decoder only
 4650     FPU    : S4;
 4651     MEM    : S3;        // any mem
 4652 %}
 4653 
 4654 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4655 %{
 4656     instruction_count(3);
 4657     src1   : S3(read);
 4658     src2   : S3(read);
 4659     mem    : S3(read);
 4660     DECODE : S0(2);     // any decoder for FPU PUSH
 4661     D0     : S1;        // big decoder only
 4662     FPU    : S4;
 4663     MEM    : S3;        // any mem
 4664 %}
 4665 
 4666 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4667 %{
 4668     instruction_count(3);
 4669     src1   : S3(read);
 4670     src2   : S3(read);
 4671     mem    : S4(read);
 4672     DECODE : S0;        // any decoder for FPU PUSH
 4673     D0     : S0(2);     // big decoder only
 4674     FPU    : S4;
 4675     MEM    : S3(2);     // any mem
 4676 %}
 4677 
 4678 pipe_class fpu_mem_mem(memory dst, memory src1)
 4679 %{
 4680     instruction_count(2);
 4681     src1   : S3(read);
 4682     dst    : S4(read);
 4683     D0     : S0(2);     // big decoder only
 4684     MEM    : S3(2);     // any mem
 4685 %}
 4686 
 4687 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4688 %{
 4689     instruction_count(3);
 4690     src1   : S3(read);
 4691     src2   : S3(read);
 4692     dst    : S4(read);
 4693     D0     : S0(3);     // big decoder only
 4694     FPU    : S4;
 4695     MEM    : S3(3);     // any mem
 4696 %}
 4697 
 4698 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4699 %{
 4700     instruction_count(3);
 4701     src1   : S4(read);
 4702     mem    : S4(read);
 4703     DECODE : S0;        // any decoder for FPU PUSH
 4704     D0     : S0(2);     // big decoder only
 4705     FPU    : S4;
 4706     MEM    : S3(2);     // any mem
 4707 %}
 4708 
 4709 // Float load constant
 4710 pipe_class fpu_reg_con(regD dst)
 4711 %{
 4712     instruction_count(2);
 4713     dst    : S5(write);
 4714     D0     : S0;        // big decoder only for the load
 4715     DECODE : S1;        // any decoder for FPU POP
 4716     FPU    : S4;
 4717     MEM    : S3;        // any mem
 4718 %}
 4719 
 4720 // Float load constant
 4721 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4722 %{
 4723     instruction_count(3);
 4724     dst    : S5(write);
 4725     src    : S3(read);
 4726     D0     : S0;        // big decoder only for the load
 4727     DECODE : S1(2);     // any decoder for FPU POP
 4728     FPU    : S4;
 4729     MEM    : S3;        // any mem
 4730 %}
 4731 
 4732 // UnConditional branch
 4733 pipe_class pipe_jmp(label labl)
 4734 %{
 4735     single_instruction;
 4736     BR   : S3;
 4737 %}
 4738 
 4739 // Conditional branch
 4740 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4741 %{
 4742     single_instruction;
 4743     cr    : S1(read);
 4744     BR    : S3;
 4745 %}
 4746 
 4747 // Allocation idiom
 4748 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4749 %{
 4750     instruction_count(1); force_serialization;
 4751     fixed_latency(6);
 4752     heap_ptr : S3(read);
 4753     DECODE   : S0(3);
 4754     D0       : S2;
 4755     MEM      : S3;
 4756     ALU      : S3(2);
 4757     dst      : S5(write);
 4758     BR       : S5;
 4759 %}
 4760 
 4761 // Generic big/slow expanded idiom
 4762 pipe_class pipe_slow()
 4763 %{
 4764     instruction_count(10); multiple_bundles; force_serialization;
 4765     fixed_latency(100);
 4766     D0  : S0(2);
 4767     MEM : S3(2);
 4768 %}
 4769 
 4770 // The real do-nothing guy
 4771 pipe_class empty()
 4772 %{
 4773     instruction_count(0);
 4774 %}
 4775 
 4776 // Define the class for the Nop node
 4777 define
 4778 %{
 4779    MachNop = empty;
 4780 %}
 4781 
 4782 %}
 4783 
 4784 //----------INSTRUCTIONS-------------------------------------------------------
 4785 //
 4786 // match      -- States which machine-independent subtree may be replaced
 4787 //               by this instruction.
 4788 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4789 //               selection to identify a minimum cost tree of machine
 4790 //               instructions that matches a tree of machine-independent
 4791 //               instructions.
 4792 // format     -- A string providing the disassembly for this instruction.
 4793 //               The value of an instruction's operand may be inserted
 4794 //               by referring to it with a '$' prefix.
 4795 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4796 //               to within an encode class as $primary, $secondary, and $tertiary
 4797 //               rrspectively.  The primary opcode is commonly used to
 4798 //               indicate the type of machine instruction, while secondary
 4799 //               and tertiary are often used for prefix options or addressing
 4800 //               modes.
 4801 // ins_encode -- A list of encode classes with parameters. The encode class
 4802 //               name must have been defined in an 'enc_class' specification
 4803 //               in the encode section of the architecture description.
 4804 
 4805 //----------Load/Store/Move Instructions---------------------------------------
 4806 //----------Load Instructions--------------------------------------------------
 4807 
 4808 // Load Byte (8 bit signed)
 4809 instruct loadB(rRegI dst, memory mem)
 4810 %{
 4811   match(Set dst (LoadB mem));
 4812 
 4813   ins_cost(125);
 4814   format %{ "movsbl  $dst, $mem\t# byte" %}
 4815 
 4816   ins_encode %{
 4817     __ movsbl($dst$$Register, $mem$$Address);
 4818   %}
 4819 
 4820   ins_pipe(ialu_reg_mem);
 4821 %}
 4822 
 4823 // Load Byte (8 bit signed) into Long Register
 4824 instruct loadB2L(rRegL dst, memory mem)
 4825 %{
 4826   match(Set dst (ConvI2L (LoadB mem)));
 4827 
 4828   ins_cost(125);
 4829   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4830 
 4831   ins_encode %{
 4832     __ movsbq($dst$$Register, $mem$$Address);
 4833   %}
 4834 
 4835   ins_pipe(ialu_reg_mem);
 4836 %}
 4837 
 4838 // Load Unsigned Byte (8 bit UNsigned)
 4839 instruct loadUB(rRegI dst, memory mem)
 4840 %{
 4841   match(Set dst (LoadUB mem));
 4842 
 4843   ins_cost(125);
 4844   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4845 
 4846   ins_encode %{
 4847     __ movzbl($dst$$Register, $mem$$Address);
 4848   %}
 4849 
 4850   ins_pipe(ialu_reg_mem);
 4851 %}
 4852 
 4853 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4854 instruct loadUB2L(rRegL dst, memory mem)
 4855 %{
 4856   match(Set dst (ConvI2L (LoadUB mem)));
 4857 
 4858   ins_cost(125);
 4859   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4860 
 4861   ins_encode %{
 4862     __ movzbq($dst$$Register, $mem$$Address);
 4863   %}
 4864 
 4865   ins_pipe(ialu_reg_mem);
 4866 %}
 4867 
 4868 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4869 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4870   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4871   effect(KILL cr);
 4872 
 4873   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4874             "andl    $dst, right_n_bits($mask, 8)" %}
 4875   ins_encode %{
 4876     Register Rdst = $dst$$Register;
 4877     __ movzbq(Rdst, $mem$$Address);
 4878     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4879   %}
 4880   ins_pipe(ialu_reg_mem);
 4881 %}
 4882 
 4883 // Load Short (16 bit signed)
 4884 instruct loadS(rRegI dst, memory mem)
 4885 %{
 4886   match(Set dst (LoadS mem));
 4887 
 4888   ins_cost(125);
 4889   format %{ "movswl $dst, $mem\t# short" %}
 4890 
 4891   ins_encode %{
 4892     __ movswl($dst$$Register, $mem$$Address);
 4893   %}
 4894 
 4895   ins_pipe(ialu_reg_mem);
 4896 %}
 4897 
 4898 // Load Short (16 bit signed) to Byte (8 bit signed)
 4899 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4900   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4901 
 4902   ins_cost(125);
 4903   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4904   ins_encode %{
 4905     __ movsbl($dst$$Register, $mem$$Address);
 4906   %}
 4907   ins_pipe(ialu_reg_mem);
 4908 %}
 4909 
 4910 // Load Short (16 bit signed) into Long Register
 4911 instruct loadS2L(rRegL dst, memory mem)
 4912 %{
 4913   match(Set dst (ConvI2L (LoadS mem)));
 4914 
 4915   ins_cost(125);
 4916   format %{ "movswq $dst, $mem\t# short -> long" %}
 4917 
 4918   ins_encode %{
 4919     __ movswq($dst$$Register, $mem$$Address);
 4920   %}
 4921 
 4922   ins_pipe(ialu_reg_mem);
 4923 %}
 4924 
 4925 // Load Unsigned Short/Char (16 bit UNsigned)
 4926 instruct loadUS(rRegI dst, memory mem)
 4927 %{
 4928   match(Set dst (LoadUS mem));
 4929 
 4930   ins_cost(125);
 4931   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4932 
 4933   ins_encode %{
 4934     __ movzwl($dst$$Register, $mem$$Address);
 4935   %}
 4936 
 4937   ins_pipe(ialu_reg_mem);
 4938 %}
 4939 
 4940 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4941 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4942   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4943 
 4944   ins_cost(125);
 4945   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4946   ins_encode %{
 4947     __ movsbl($dst$$Register, $mem$$Address);
 4948   %}
 4949   ins_pipe(ialu_reg_mem);
 4950 %}
 4951 
 4952 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4953 instruct loadUS2L(rRegL dst, memory mem)
 4954 %{
 4955   match(Set dst (ConvI2L (LoadUS mem)));
 4956 
 4957   ins_cost(125);
 4958   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4959 
 4960   ins_encode %{
 4961     __ movzwq($dst$$Register, $mem$$Address);
 4962   %}
 4963 
 4964   ins_pipe(ialu_reg_mem);
 4965 %}
 4966 
 4967 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4968 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4969   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4970 
 4971   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4972   ins_encode %{
 4973     __ movzbq($dst$$Register, $mem$$Address);
 4974   %}
 4975   ins_pipe(ialu_reg_mem);
 4976 %}
 4977 
 4978 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4979 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4980   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4981   effect(KILL cr);
 4982 
 4983   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4984             "andl    $dst, right_n_bits($mask, 16)" %}
 4985   ins_encode %{
 4986     Register Rdst = $dst$$Register;
 4987     __ movzwq(Rdst, $mem$$Address);
 4988     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4989   %}
 4990   ins_pipe(ialu_reg_mem);
 4991 %}
 4992 
 4993 // Load Integer
 4994 instruct loadI(rRegI dst, memory mem)
 4995 %{
 4996   match(Set dst (LoadI mem));
 4997 
 4998   ins_cost(125);
 4999   format %{ "movl    $dst, $mem\t# int" %}
 5000 
 5001   ins_encode %{
 5002     __ movl($dst$$Register, $mem$$Address);
 5003   %}
 5004 
 5005   ins_pipe(ialu_reg_mem);
 5006 %}
 5007 
 5008 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5009 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5010   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5011 
 5012   ins_cost(125);
 5013   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5014   ins_encode %{
 5015     __ movsbl($dst$$Register, $mem$$Address);
 5016   %}
 5017   ins_pipe(ialu_reg_mem);
 5018 %}
 5019 
 5020 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5021 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5022   match(Set dst (AndI (LoadI mem) mask));
 5023 
 5024   ins_cost(125);
 5025   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5026   ins_encode %{
 5027     __ movzbl($dst$$Register, $mem$$Address);
 5028   %}
 5029   ins_pipe(ialu_reg_mem);
 5030 %}
 5031 
 5032 // Load Integer (32 bit signed) to Short (16 bit signed)
 5033 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5034   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5035 
 5036   ins_cost(125);
 5037   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5038   ins_encode %{
 5039     __ movswl($dst$$Register, $mem$$Address);
 5040   %}
 5041   ins_pipe(ialu_reg_mem);
 5042 %}
 5043 
 5044 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5045 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5046   match(Set dst (AndI (LoadI mem) mask));
 5047 
 5048   ins_cost(125);
 5049   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5050   ins_encode %{
 5051     __ movzwl($dst$$Register, $mem$$Address);
 5052   %}
 5053   ins_pipe(ialu_reg_mem);
 5054 %}
 5055 
 5056 // Load Integer into Long Register
 5057 instruct loadI2L(rRegL dst, memory mem)
 5058 %{
 5059   match(Set dst (ConvI2L (LoadI mem)));
 5060 
 5061   ins_cost(125);
 5062   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5063 
 5064   ins_encode %{
 5065     __ movslq($dst$$Register, $mem$$Address);
 5066   %}
 5067 
 5068   ins_pipe(ialu_reg_mem);
 5069 %}
 5070 
 5071 // Load Integer with mask 0xFF into Long Register
 5072 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5073   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5074 
 5075   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5076   ins_encode %{
 5077     __ movzbq($dst$$Register, $mem$$Address);
 5078   %}
 5079   ins_pipe(ialu_reg_mem);
 5080 %}
 5081 
 5082 // Load Integer with mask 0xFFFF into Long Register
 5083 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5084   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5085 
 5086   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5087   ins_encode %{
 5088     __ movzwq($dst$$Register, $mem$$Address);
 5089   %}
 5090   ins_pipe(ialu_reg_mem);
 5091 %}
 5092 
 5093 // Load Integer with a 31-bit mask into Long Register
 5094 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5095   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5096   effect(KILL cr);
 5097 
 5098   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5099             "andl    $dst, $mask" %}
 5100   ins_encode %{
 5101     Register Rdst = $dst$$Register;
 5102     __ movl(Rdst, $mem$$Address);
 5103     __ andl(Rdst, $mask$$constant);
 5104   %}
 5105   ins_pipe(ialu_reg_mem);
 5106 %}
 5107 
 5108 // Load Unsigned Integer into Long Register
 5109 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5110 %{
 5111   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5112 
 5113   ins_cost(125);
 5114   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5115 
 5116   ins_encode %{
 5117     __ movl($dst$$Register, $mem$$Address);
 5118   %}
 5119 
 5120   ins_pipe(ialu_reg_mem);
 5121 %}
 5122 
 5123 // Load Long
 5124 instruct loadL(rRegL dst, memory mem)
 5125 %{
 5126   match(Set dst (LoadL mem));
 5127 
 5128   ins_cost(125);
 5129   format %{ "movq    $dst, $mem\t# long" %}
 5130 
 5131   ins_encode %{
 5132     __ movq($dst$$Register, $mem$$Address);
 5133   %}
 5134 
 5135   ins_pipe(ialu_reg_mem); // XXX
 5136 %}
 5137 
 5138 // Load Range
 5139 instruct loadRange(rRegI dst, memory mem)
 5140 %{
 5141   match(Set dst (LoadRange mem));
 5142 
 5143   ins_cost(125); // XXX
 5144   format %{ "movl    $dst, $mem\t# range" %}
 5145   ins_encode %{
 5146     __ movl($dst$$Register, $mem$$Address);
 5147   %}
 5148   ins_pipe(ialu_reg_mem);
 5149 %}
 5150 
 5151 // Load Pointer
 5152 instruct loadP(rRegP dst, memory mem)
 5153 %{
 5154   match(Set dst (LoadP mem));
 5155   predicate(n->as_Load()->barrier_data() == 0);
 5156 
 5157   ins_cost(125); // XXX
 5158   format %{ "movq    $dst, $mem\t# ptr" %}
 5159   ins_encode %{
 5160     __ movq($dst$$Register, $mem$$Address);
 5161   %}
 5162   ins_pipe(ialu_reg_mem); // XXX
 5163 %}
 5164 
 5165 // Load Compressed Pointer
 5166 instruct loadN(rRegN dst, memory mem)
 5167 %{
 5168    match(Set dst (LoadN mem));
 5169 
 5170    ins_cost(125); // XXX
 5171    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5172    ins_encode %{
 5173      __ movl($dst$$Register, $mem$$Address);
 5174    %}
 5175    ins_pipe(ialu_reg_mem); // XXX
 5176 %}
 5177 
 5178 
 5179 // Load Klass Pointer
 5180 instruct loadKlass(rRegP dst, memory mem)
 5181 %{
 5182   match(Set dst (LoadKlass mem));
 5183 
 5184   ins_cost(125); // XXX
 5185   format %{ "movq    $dst, $mem\t# class" %}
 5186   ins_encode %{
 5187     __ movq($dst$$Register, $mem$$Address);
 5188   %}
 5189   ins_pipe(ialu_reg_mem); // XXX
 5190 %}
 5191 
 5192 // Load narrow Klass Pointer
 5193 instruct loadNKlass(rRegN dst, memory mem)
 5194 %{
 5195   predicate(!UseCompactObjectHeaders);
 5196   match(Set dst (LoadNKlass mem));
 5197 
 5198   ins_cost(125); // XXX
 5199   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5200   ins_encode %{
 5201     __ movl($dst$$Register, $mem$$Address);
 5202   %}
 5203   ins_pipe(ialu_reg_mem); // XXX
 5204 %}
 5205 
 5206 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 5207 %{
 5208   predicate(UseCompactObjectHeaders);
 5209   match(Set dst (LoadNKlass mem));
 5210   effect(KILL cr);
 5211   ins_cost(125); // XXX
 5212   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5213   ins_encode %{
 5214     Register index = $mem$$index != 4 ? $mem$$index$$Register : noreg;
 5215     Address::ScaleFactor sf = (index != noreg) ? static_cast<Address::ScaleFactor>($mem$$scale) : Address::no_scale;
 5216     __ load_nklass_compact_c2($dst$$Register, $mem$$base$$Register, index, sf, $mem$$disp);
 5217   %}
 5218   ins_pipe(pipe_slow); // XXX
 5219 %}
 5220 
 5221 // Load Float
 5222 instruct loadF(regF dst, memory mem)
 5223 %{
 5224   match(Set dst (LoadF mem));
 5225 
 5226   ins_cost(145); // XXX
 5227   format %{ "movss   $dst, $mem\t# float" %}
 5228   ins_encode %{
 5229     __ movflt($dst$$XMMRegister, $mem$$Address);
 5230   %}
 5231   ins_pipe(pipe_slow); // XXX
 5232 %}
 5233 
 5234 // Load Float
 5235 instruct MoveF2VL(vlRegF dst, regF src) %{
 5236   match(Set dst src);
 5237   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 5238   ins_encode %{
 5239     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 5240   %}
 5241   ins_pipe( fpu_reg_reg );
 5242 %}
 5243 
 5244 // Load Float
 5245 instruct MoveF2LEG(legRegF dst, regF src) %{
 5246   match(Set dst src);
 5247   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 5248   ins_encode %{
 5249     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 5250   %}
 5251   ins_pipe( fpu_reg_reg );
 5252 %}
 5253 
 5254 // Load Float
 5255 instruct MoveVL2F(regF dst, vlRegF src) %{
 5256   match(Set dst src);
 5257   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 5258   ins_encode %{
 5259     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 5260   %}
 5261   ins_pipe( fpu_reg_reg );
 5262 %}
 5263 
 5264 // Load Float
 5265 instruct MoveLEG2F(regF dst, legRegF src) %{
 5266   match(Set dst src);
 5267   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 5268   ins_encode %{
 5269     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 5270   %}
 5271   ins_pipe( fpu_reg_reg );
 5272 %}
 5273 
 5274 // Load Double
 5275 instruct loadD_partial(regD dst, memory mem)
 5276 %{
 5277   predicate(!UseXmmLoadAndClearUpper);
 5278   match(Set dst (LoadD mem));
 5279 
 5280   ins_cost(145); // XXX
 5281   format %{ "movlpd  $dst, $mem\t# double" %}
 5282   ins_encode %{
 5283     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5284   %}
 5285   ins_pipe(pipe_slow); // XXX
 5286 %}
 5287 
 5288 instruct loadD(regD dst, memory mem)
 5289 %{
 5290   predicate(UseXmmLoadAndClearUpper);
 5291   match(Set dst (LoadD mem));
 5292 
 5293   ins_cost(145); // XXX
 5294   format %{ "movsd   $dst, $mem\t# double" %}
 5295   ins_encode %{
 5296     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5297   %}
 5298   ins_pipe(pipe_slow); // XXX
 5299 %}
 5300 
 5301 // Load Double
 5302 instruct MoveD2VL(vlRegD dst, regD src) %{
 5303   match(Set dst src);
 5304   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 5305   ins_encode %{
 5306     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 5307   %}
 5308   ins_pipe( fpu_reg_reg );
 5309 %}
 5310 
 5311 // Load Double
 5312 instruct MoveD2LEG(legRegD dst, regD src) %{
 5313   match(Set dst src);
 5314   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 5315   ins_encode %{
 5316     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 5317   %}
 5318   ins_pipe( fpu_reg_reg );
 5319 %}
 5320 
 5321 // Load Double
 5322 instruct MoveVL2D(regD dst, vlRegD src) %{
 5323   match(Set dst src);
 5324   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 5325   ins_encode %{
 5326     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 5327   %}
 5328   ins_pipe( fpu_reg_reg );
 5329 %}
 5330 
 5331 // Load Double
 5332 instruct MoveLEG2D(regD dst, legRegD src) %{
 5333   match(Set dst src);
 5334   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 5335   ins_encode %{
 5336     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 5337   %}
 5338   ins_pipe( fpu_reg_reg );
 5339 %}
 5340 
 5341 // Following pseudo code describes the algorithm for max[FD]:
 5342 // Min algorithm is on similar lines
 5343 //  btmp = (b < +0.0) ? a : b
 5344 //  atmp = (b < +0.0) ? b : a
 5345 //  Tmp  = Max_Float(atmp , btmp)
 5346 //  Res  = (atmp == NaN) ? atmp : Tmp
 5347 
 5348 // max = java.lang.Math.max(float a, float b)
 5349 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5350   predicate(UseAVX > 0 && !n->is_reduction());
 5351   match(Set dst (MaxF a b));
 5352   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5353   format %{
 5354      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5355      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5356      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5357      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5358      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5359   %}
 5360   ins_encode %{
 5361     int vector_len = Assembler::AVX_128bit;
 5362     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5363     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5364     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5365     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5366     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5367  %}
 5368   ins_pipe( pipe_slow );
 5369 %}
 5370 
 5371 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5372   predicate(UseAVX > 0 && n->is_reduction());
 5373   match(Set dst (MaxF a b));
 5374   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5375 
 5376   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5377   ins_encode %{
 5378     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5379                     false /*min*/, true /*single*/);
 5380   %}
 5381   ins_pipe( pipe_slow );
 5382 %}
 5383 
 5384 // max = java.lang.Math.max(double a, double b)
 5385 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5386   predicate(UseAVX > 0 && !n->is_reduction());
 5387   match(Set dst (MaxD a b));
 5388   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5389   format %{
 5390      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5391      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5392      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5393      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5394      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5395   %}
 5396   ins_encode %{
 5397     int vector_len = Assembler::AVX_128bit;
 5398     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5399     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5400     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5401     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5402     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5403   %}
 5404   ins_pipe( pipe_slow );
 5405 %}
 5406 
 5407 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5408   predicate(UseAVX > 0 && n->is_reduction());
 5409   match(Set dst (MaxD a b));
 5410   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5411 
 5412   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5413   ins_encode %{
 5414     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5415                     false /*min*/, false /*single*/);
 5416   %}
 5417   ins_pipe( pipe_slow );
 5418 %}
 5419 
 5420 // min = java.lang.Math.min(float a, float b)
 5421 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5422   predicate(UseAVX > 0 && !n->is_reduction());
 5423   match(Set dst (MinF a b));
 5424   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5425   format %{
 5426      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5427      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5428      "vminss           $tmp,$atmp,$btmp           \n\t"
 5429      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5430      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5431   %}
 5432   ins_encode %{
 5433     int vector_len = Assembler::AVX_128bit;
 5434     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5435     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5436     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5437     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5438     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5439   %}
 5440   ins_pipe( pipe_slow );
 5441 %}
 5442 
 5443 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5444   predicate(UseAVX > 0 && n->is_reduction());
 5445   match(Set dst (MinF a b));
 5446   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5447 
 5448   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5449   ins_encode %{
 5450     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5451                     true /*min*/, true /*single*/);
 5452   %}
 5453   ins_pipe( pipe_slow );
 5454 %}
 5455 
 5456 // min = java.lang.Math.min(double a, double b)
 5457 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5458   predicate(UseAVX > 0 && !n->is_reduction());
 5459   match(Set dst (MinD a b));
 5460   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5461   format %{
 5462      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5463      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5464      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5465      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5466      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5467   %}
 5468   ins_encode %{
 5469     int vector_len = Assembler::AVX_128bit;
 5470     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5471     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5472     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5473     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5474     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5475   %}
 5476   ins_pipe( pipe_slow );
 5477 %}
 5478 
 5479 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5480   predicate(UseAVX > 0 && n->is_reduction());
 5481   match(Set dst (MinD a b));
 5482   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5483 
 5484   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5485   ins_encode %{
 5486     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5487                     true /*min*/, false /*single*/);
 5488   %}
 5489   ins_pipe( pipe_slow );
 5490 %}
 5491 
 5492 // Load Effective Address
 5493 instruct leaP8(rRegP dst, indOffset8 mem)
 5494 %{
 5495   match(Set dst mem);
 5496 
 5497   ins_cost(110); // XXX
 5498   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5499   ins_encode %{
 5500     __ leaq($dst$$Register, $mem$$Address);
 5501   %}
 5502   ins_pipe(ialu_reg_reg_fat);
 5503 %}
 5504 
 5505 instruct leaP32(rRegP dst, indOffset32 mem)
 5506 %{
 5507   match(Set dst mem);
 5508 
 5509   ins_cost(110);
 5510   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5511   ins_encode %{
 5512     __ leaq($dst$$Register, $mem$$Address);
 5513   %}
 5514   ins_pipe(ialu_reg_reg_fat);
 5515 %}
 5516 
 5517 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5518 %{
 5519   match(Set dst mem);
 5520 
 5521   ins_cost(110);
 5522   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5523   ins_encode %{
 5524     __ leaq($dst$$Register, $mem$$Address);
 5525   %}
 5526   ins_pipe(ialu_reg_reg_fat);
 5527 %}
 5528 
 5529 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5530 %{
 5531   match(Set dst mem);
 5532 
 5533   ins_cost(110);
 5534   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5535   ins_encode %{
 5536     __ leaq($dst$$Register, $mem$$Address);
 5537   %}
 5538   ins_pipe(ialu_reg_reg_fat);
 5539 %}
 5540 
 5541 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5542 %{
 5543   match(Set dst mem);
 5544 
 5545   ins_cost(110);
 5546   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5547   ins_encode %{
 5548     __ leaq($dst$$Register, $mem$$Address);
 5549   %}
 5550   ins_pipe(ialu_reg_reg_fat);
 5551 %}
 5552 
 5553 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5554 %{
 5555   match(Set dst mem);
 5556 
 5557   ins_cost(110);
 5558   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5559   ins_encode %{
 5560     __ leaq($dst$$Register, $mem$$Address);
 5561   %}
 5562   ins_pipe(ialu_reg_reg_fat);
 5563 %}
 5564 
 5565 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5566 %{
 5567   match(Set dst mem);
 5568 
 5569   ins_cost(110);
 5570   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5571   ins_encode %{
 5572     __ leaq($dst$$Register, $mem$$Address);
 5573   %}
 5574   ins_pipe(ialu_reg_reg_fat);
 5575 %}
 5576 
 5577 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5578 %{
 5579   match(Set dst mem);
 5580 
 5581   ins_cost(110);
 5582   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5583   ins_encode %{
 5584     __ leaq($dst$$Register, $mem$$Address);
 5585   %}
 5586   ins_pipe(ialu_reg_reg_fat);
 5587 %}
 5588 
 5589 // Load Effective Address which uses Narrow (32-bits) oop
 5590 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5591 %{
 5592   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5593   match(Set dst mem);
 5594 
 5595   ins_cost(110);
 5596   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5597   ins_encode %{
 5598     __ leaq($dst$$Register, $mem$$Address);
 5599   %}
 5600   ins_pipe(ialu_reg_reg_fat);
 5601 %}
 5602 
 5603 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5604 %{
 5605   predicate(CompressedOops::shift() == 0);
 5606   match(Set dst mem);
 5607 
 5608   ins_cost(110); // XXX
 5609   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5610   ins_encode %{
 5611     __ leaq($dst$$Register, $mem$$Address);
 5612   %}
 5613   ins_pipe(ialu_reg_reg_fat);
 5614 %}
 5615 
 5616 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5617 %{
 5618   predicate(CompressedOops::shift() == 0);
 5619   match(Set dst mem);
 5620 
 5621   ins_cost(110);
 5622   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5623   ins_encode %{
 5624     __ leaq($dst$$Register, $mem$$Address);
 5625   %}
 5626   ins_pipe(ialu_reg_reg_fat);
 5627 %}
 5628 
 5629 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5630 %{
 5631   predicate(CompressedOops::shift() == 0);
 5632   match(Set dst mem);
 5633 
 5634   ins_cost(110);
 5635   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5636   ins_encode %{
 5637     __ leaq($dst$$Register, $mem$$Address);
 5638   %}
 5639   ins_pipe(ialu_reg_reg_fat);
 5640 %}
 5641 
 5642 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5643 %{
 5644   predicate(CompressedOops::shift() == 0);
 5645   match(Set dst mem);
 5646 
 5647   ins_cost(110);
 5648   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5649   ins_encode %{
 5650     __ leaq($dst$$Register, $mem$$Address);
 5651   %}
 5652   ins_pipe(ialu_reg_reg_fat);
 5653 %}
 5654 
 5655 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5656 %{
 5657   predicate(CompressedOops::shift() == 0);
 5658   match(Set dst mem);
 5659 
 5660   ins_cost(110);
 5661   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5662   ins_encode %{
 5663     __ leaq($dst$$Register, $mem$$Address);
 5664   %}
 5665   ins_pipe(ialu_reg_reg_fat);
 5666 %}
 5667 
 5668 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5669 %{
 5670   predicate(CompressedOops::shift() == 0);
 5671   match(Set dst mem);
 5672 
 5673   ins_cost(110);
 5674   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5675   ins_encode %{
 5676     __ leaq($dst$$Register, $mem$$Address);
 5677   %}
 5678   ins_pipe(ialu_reg_reg_fat);
 5679 %}
 5680 
 5681 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5682 %{
 5683   predicate(CompressedOops::shift() == 0);
 5684   match(Set dst mem);
 5685 
 5686   ins_cost(110);
 5687   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5688   ins_encode %{
 5689     __ leaq($dst$$Register, $mem$$Address);
 5690   %}
 5691   ins_pipe(ialu_reg_reg_fat);
 5692 %}
 5693 
 5694 instruct loadConI(rRegI dst, immI src)
 5695 %{
 5696   match(Set dst src);
 5697 
 5698   format %{ "movl    $dst, $src\t# int" %}
 5699   ins_encode %{
 5700     __ movl($dst$$Register, $src$$constant);
 5701   %}
 5702   ins_pipe(ialu_reg_fat); // XXX
 5703 %}
 5704 
 5705 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5706 %{
 5707   match(Set dst src);
 5708   effect(KILL cr);
 5709 
 5710   ins_cost(50);
 5711   format %{ "xorl    $dst, $dst\t# int" %}
 5712   ins_encode %{
 5713     __ xorl($dst$$Register, $dst$$Register);
 5714   %}
 5715   ins_pipe(ialu_reg);
 5716 %}
 5717 
 5718 instruct loadConL(rRegL dst, immL src)
 5719 %{
 5720   match(Set dst src);
 5721 
 5722   ins_cost(150);
 5723   format %{ "movq    $dst, $src\t# long" %}
 5724   ins_encode %{
 5725     __ mov64($dst$$Register, $src$$constant);
 5726   %}
 5727   ins_pipe(ialu_reg);
 5728 %}
 5729 
 5730 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5731 %{
 5732   match(Set dst src);
 5733   effect(KILL cr);
 5734 
 5735   ins_cost(50);
 5736   format %{ "xorl    $dst, $dst\t# long" %}
 5737   ins_encode %{
 5738     __ xorl($dst$$Register, $dst$$Register);
 5739   %}
 5740   ins_pipe(ialu_reg); // XXX
 5741 %}
 5742 
 5743 instruct loadConUL32(rRegL dst, immUL32 src)
 5744 %{
 5745   match(Set dst src);
 5746 
 5747   ins_cost(60);
 5748   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5749   ins_encode %{
 5750     __ movl($dst$$Register, $src$$constant);
 5751   %}
 5752   ins_pipe(ialu_reg);
 5753 %}
 5754 
 5755 instruct loadConL32(rRegL dst, immL32 src)
 5756 %{
 5757   match(Set dst src);
 5758 
 5759   ins_cost(70);
 5760   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5761   ins_encode %{
 5762     __ movq($dst$$Register, $src$$constant);
 5763   %}
 5764   ins_pipe(ialu_reg);
 5765 %}
 5766 
 5767 instruct loadConP(rRegP dst, immP con) %{
 5768   match(Set dst con);
 5769 
 5770   format %{ "movq    $dst, $con\t# ptr" %}
 5771   ins_encode %{
 5772     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5773   %}
 5774   ins_pipe(ialu_reg_fat); // XXX
 5775 %}
 5776 
 5777 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5778 %{
 5779   match(Set dst src);
 5780   effect(KILL cr);
 5781 
 5782   ins_cost(50);
 5783   format %{ "xorl    $dst, $dst\t# ptr" %}
 5784   ins_encode %{
 5785     __ xorl($dst$$Register, $dst$$Register);
 5786   %}
 5787   ins_pipe(ialu_reg);
 5788 %}
 5789 
 5790 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5791 %{
 5792   match(Set dst src);
 5793   effect(KILL cr);
 5794 
 5795   ins_cost(60);
 5796   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5797   ins_encode %{
 5798     __ movl($dst$$Register, $src$$constant);
 5799   %}
 5800   ins_pipe(ialu_reg);
 5801 %}
 5802 
 5803 instruct loadConF(regF dst, immF con) %{
 5804   match(Set dst con);
 5805   ins_cost(125);
 5806   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5807   ins_encode %{
 5808     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5809   %}
 5810   ins_pipe(pipe_slow);
 5811 %}
 5812 
 5813 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5814   match(Set dst src);
 5815   effect(KILL cr);
 5816   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5817   ins_encode %{
 5818     __ xorq($dst$$Register, $dst$$Register);
 5819   %}
 5820   ins_pipe(ialu_reg);
 5821 %}
 5822 
 5823 instruct loadConN(rRegN dst, immN src) %{
 5824   match(Set dst src);
 5825 
 5826   ins_cost(125);
 5827   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5828   ins_encode %{
 5829     address con = (address)$src$$constant;
 5830     if (con == NULL) {
 5831       ShouldNotReachHere();
 5832     } else {
 5833       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5834     }
 5835   %}
 5836   ins_pipe(ialu_reg_fat); // XXX
 5837 %}
 5838 
 5839 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5840   match(Set dst src);
 5841 
 5842   ins_cost(125);
 5843   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5844   ins_encode %{
 5845     address con = (address)$src$$constant;
 5846     if (con == NULL) {
 5847       ShouldNotReachHere();
 5848     } else {
 5849       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5850     }
 5851   %}
 5852   ins_pipe(ialu_reg_fat); // XXX
 5853 %}
 5854 
 5855 instruct loadConF0(regF dst, immF0 src)
 5856 %{
 5857   match(Set dst src);
 5858   ins_cost(100);
 5859 
 5860   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5861   ins_encode %{
 5862     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5863   %}
 5864   ins_pipe(pipe_slow);
 5865 %}
 5866 
 5867 // Use the same format since predicate() can not be used here.
 5868 instruct loadConD(regD dst, immD con) %{
 5869   match(Set dst con);
 5870   ins_cost(125);
 5871   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5872   ins_encode %{
 5873     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5874   %}
 5875   ins_pipe(pipe_slow);
 5876 %}
 5877 
 5878 instruct loadConD0(regD dst, immD0 src)
 5879 %{
 5880   match(Set dst src);
 5881   ins_cost(100);
 5882 
 5883   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5884   ins_encode %{
 5885     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5886   %}
 5887   ins_pipe(pipe_slow);
 5888 %}
 5889 
 5890 instruct loadSSI(rRegI dst, stackSlotI src)
 5891 %{
 5892   match(Set dst src);
 5893 
 5894   ins_cost(125);
 5895   format %{ "movl    $dst, $src\t# int stk" %}
 5896   opcode(0x8B);
 5897   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5898   ins_pipe(ialu_reg_mem);
 5899 %}
 5900 
 5901 instruct loadSSL(rRegL dst, stackSlotL src)
 5902 %{
 5903   match(Set dst src);
 5904 
 5905   ins_cost(125);
 5906   format %{ "movq    $dst, $src\t# long stk" %}
 5907   opcode(0x8B);
 5908   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5909   ins_pipe(ialu_reg_mem);
 5910 %}
 5911 
 5912 instruct loadSSP(rRegP dst, stackSlotP src)
 5913 %{
 5914   match(Set dst src);
 5915 
 5916   ins_cost(125);
 5917   format %{ "movq    $dst, $src\t# ptr stk" %}
 5918   opcode(0x8B);
 5919   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5920   ins_pipe(ialu_reg_mem);
 5921 %}
 5922 
 5923 instruct loadSSF(regF dst, stackSlotF src)
 5924 %{
 5925   match(Set dst src);
 5926 
 5927   ins_cost(125);
 5928   format %{ "movss   $dst, $src\t# float stk" %}
 5929   ins_encode %{
 5930     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5931   %}
 5932   ins_pipe(pipe_slow); // XXX
 5933 %}
 5934 
 5935 // Use the same format since predicate() can not be used here.
 5936 instruct loadSSD(regD dst, stackSlotD src)
 5937 %{
 5938   match(Set dst src);
 5939 
 5940   ins_cost(125);
 5941   format %{ "movsd   $dst, $src\t# double stk" %}
 5942   ins_encode  %{
 5943     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5944   %}
 5945   ins_pipe(pipe_slow); // XXX
 5946 %}
 5947 
 5948 // Prefetch instructions for allocation.
 5949 // Must be safe to execute with invalid address (cannot fault).
 5950 
 5951 instruct prefetchAlloc( memory mem ) %{
 5952   predicate(AllocatePrefetchInstr==3);
 5953   match(PrefetchAllocation mem);
 5954   ins_cost(125);
 5955 
 5956   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5957   ins_encode %{
 5958     __ prefetchw($mem$$Address);
 5959   %}
 5960   ins_pipe(ialu_mem);
 5961 %}
 5962 
 5963 instruct prefetchAllocNTA( memory mem ) %{
 5964   predicate(AllocatePrefetchInstr==0);
 5965   match(PrefetchAllocation mem);
 5966   ins_cost(125);
 5967 
 5968   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5969   ins_encode %{
 5970     __ prefetchnta($mem$$Address);
 5971   %}
 5972   ins_pipe(ialu_mem);
 5973 %}
 5974 
 5975 instruct prefetchAllocT0( memory mem ) %{
 5976   predicate(AllocatePrefetchInstr==1);
 5977   match(PrefetchAllocation mem);
 5978   ins_cost(125);
 5979 
 5980   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5981   ins_encode %{
 5982     __ prefetcht0($mem$$Address);
 5983   %}
 5984   ins_pipe(ialu_mem);
 5985 %}
 5986 
 5987 instruct prefetchAllocT2( memory mem ) %{
 5988   predicate(AllocatePrefetchInstr==2);
 5989   match(PrefetchAllocation mem);
 5990   ins_cost(125);
 5991 
 5992   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5993   ins_encode %{
 5994     __ prefetcht2($mem$$Address);
 5995   %}
 5996   ins_pipe(ialu_mem);
 5997 %}
 5998 
 5999 //----------Store Instructions-------------------------------------------------
 6000 
 6001 // Store Byte
 6002 instruct storeB(memory mem, rRegI src)
 6003 %{
 6004   match(Set mem (StoreB mem src));
 6005 
 6006   ins_cost(125); // XXX
 6007   format %{ "movb    $mem, $src\t# byte" %}
 6008   ins_encode %{
 6009     __ movb($mem$$Address, $src$$Register);
 6010   %}
 6011   ins_pipe(ialu_mem_reg);
 6012 %}
 6013 
 6014 // Store Char/Short
 6015 instruct storeC(memory mem, rRegI src)
 6016 %{
 6017   match(Set mem (StoreC mem src));
 6018 
 6019   ins_cost(125); // XXX
 6020   format %{ "movw    $mem, $src\t# char/short" %}
 6021   ins_encode %{
 6022     __ movw($mem$$Address, $src$$Register);
 6023   %}
 6024   ins_pipe(ialu_mem_reg);
 6025 %}
 6026 
 6027 // Store Integer
 6028 instruct storeI(memory mem, rRegI src)
 6029 %{
 6030   match(Set mem (StoreI mem src));
 6031 
 6032   ins_cost(125); // XXX
 6033   format %{ "movl    $mem, $src\t# int" %}
 6034   ins_encode %{
 6035     __ movl($mem$$Address, $src$$Register);
 6036   %}
 6037   ins_pipe(ialu_mem_reg);
 6038 %}
 6039 
 6040 // Store Long
 6041 instruct storeL(memory mem, rRegL src)
 6042 %{
 6043   match(Set mem (StoreL mem src));
 6044 
 6045   ins_cost(125); // XXX
 6046   format %{ "movq    $mem, $src\t# long" %}
 6047   ins_encode %{
 6048     __ movq($mem$$Address, $src$$Register);
 6049   %}
 6050   ins_pipe(ialu_mem_reg); // XXX
 6051 %}
 6052 
 6053 // Store Pointer
 6054 instruct storeP(memory mem, any_RegP src)
 6055 %{
 6056   match(Set mem (StoreP mem src));
 6057 
 6058   ins_cost(125); // XXX
 6059   format %{ "movq    $mem, $src\t# ptr" %}
 6060   ins_encode %{
 6061     __ movq($mem$$Address, $src$$Register);
 6062   %}
 6063   ins_pipe(ialu_mem_reg);
 6064 %}
 6065 
 6066 instruct storeImmP0(memory mem, immP0 zero)
 6067 %{
 6068   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6069   match(Set mem (StoreP mem zero));
 6070 
 6071   ins_cost(125); // XXX
 6072   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6073   ins_encode %{
 6074     __ movq($mem$$Address, r12);
 6075   %}
 6076   ins_pipe(ialu_mem_reg);
 6077 %}
 6078 
 6079 // Store NULL Pointer, mark word, or other simple pointer constant.
 6080 instruct storeImmP(memory mem, immP31 src)
 6081 %{
 6082   match(Set mem (StoreP mem src));
 6083 
 6084   ins_cost(150); // XXX
 6085   format %{ "movq    $mem, $src\t# ptr" %}
 6086   ins_encode %{
 6087     __ movq($mem$$Address, $src$$constant);
 6088   %}
 6089   ins_pipe(ialu_mem_imm);
 6090 %}
 6091 
 6092 // Store Compressed Pointer
 6093 instruct storeN(memory mem, rRegN src)
 6094 %{
 6095   match(Set mem (StoreN mem src));
 6096 
 6097   ins_cost(125); // XXX
 6098   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6099   ins_encode %{
 6100     __ movl($mem$$Address, $src$$Register);
 6101   %}
 6102   ins_pipe(ialu_mem_reg);
 6103 %}
 6104 
 6105 instruct storeNKlass(memory mem, rRegN src)
 6106 %{
 6107   match(Set mem (StoreNKlass mem src));
 6108 
 6109   ins_cost(125); // XXX
 6110   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6111   ins_encode %{
 6112     __ movl($mem$$Address, $src$$Register);
 6113   %}
 6114   ins_pipe(ialu_mem_reg);
 6115 %}
 6116 
 6117 instruct storeImmN0(memory mem, immN0 zero)
 6118 %{
 6119   predicate(CompressedOops::base() == NULL);
 6120   match(Set mem (StoreN mem zero));
 6121 
 6122   ins_cost(125); // XXX
 6123   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6124   ins_encode %{
 6125     __ movl($mem$$Address, r12);
 6126   %}
 6127   ins_pipe(ialu_mem_reg);
 6128 %}
 6129 
 6130 instruct storeImmN(memory mem, immN src)
 6131 %{
 6132   match(Set mem (StoreN mem src));
 6133 
 6134   ins_cost(150); // XXX
 6135   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6136   ins_encode %{
 6137     address con = (address)$src$$constant;
 6138     if (con == NULL) {
 6139       __ movl($mem$$Address, (int32_t)0);
 6140     } else {
 6141       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6142     }
 6143   %}
 6144   ins_pipe(ialu_mem_imm);
 6145 %}
 6146 
 6147 instruct storeImmNKlass(memory mem, immNKlass src)
 6148 %{
 6149   match(Set mem (StoreNKlass mem src));
 6150 
 6151   ins_cost(150); // XXX
 6152   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6153   ins_encode %{
 6154     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6155   %}
 6156   ins_pipe(ialu_mem_imm);
 6157 %}
 6158 
 6159 // Store Integer Immediate
 6160 instruct storeImmI0(memory mem, immI_0 zero)
 6161 %{
 6162   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6163   match(Set mem (StoreI mem zero));
 6164 
 6165   ins_cost(125); // XXX
 6166   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6167   ins_encode %{
 6168     __ movl($mem$$Address, r12);
 6169   %}
 6170   ins_pipe(ialu_mem_reg);
 6171 %}
 6172 
 6173 instruct storeImmI(memory mem, immI src)
 6174 %{
 6175   match(Set mem (StoreI mem src));
 6176 
 6177   ins_cost(150);
 6178   format %{ "movl    $mem, $src\t# int" %}
 6179   ins_encode %{
 6180     __ movl($mem$$Address, $src$$constant);
 6181   %}
 6182   ins_pipe(ialu_mem_imm);
 6183 %}
 6184 
 6185 // Store Long Immediate
 6186 instruct storeImmL0(memory mem, immL0 zero)
 6187 %{
 6188   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6189   match(Set mem (StoreL mem zero));
 6190 
 6191   ins_cost(125); // XXX
 6192   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6193   ins_encode %{
 6194     __ movq($mem$$Address, r12);
 6195   %}
 6196   ins_pipe(ialu_mem_reg);
 6197 %}
 6198 
 6199 instruct storeImmL(memory mem, immL32 src)
 6200 %{
 6201   match(Set mem (StoreL mem src));
 6202 
 6203   ins_cost(150);
 6204   format %{ "movq    $mem, $src\t# long" %}
 6205   ins_encode %{
 6206     __ movq($mem$$Address, $src$$constant);
 6207   %}
 6208   ins_pipe(ialu_mem_imm);
 6209 %}
 6210 
 6211 // Store Short/Char Immediate
 6212 instruct storeImmC0(memory mem, immI_0 zero)
 6213 %{
 6214   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6215   match(Set mem (StoreC mem zero));
 6216 
 6217   ins_cost(125); // XXX
 6218   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6219   ins_encode %{
 6220     __ movw($mem$$Address, r12);
 6221   %}
 6222   ins_pipe(ialu_mem_reg);
 6223 %}
 6224 
 6225 instruct storeImmI16(memory mem, immI16 src)
 6226 %{
 6227   predicate(UseStoreImmI16);
 6228   match(Set mem (StoreC mem src));
 6229 
 6230   ins_cost(150);
 6231   format %{ "movw    $mem, $src\t# short/char" %}
 6232   ins_encode %{
 6233     __ movw($mem$$Address, $src$$constant);
 6234   %}
 6235   ins_pipe(ialu_mem_imm);
 6236 %}
 6237 
 6238 // Store Byte Immediate
 6239 instruct storeImmB0(memory mem, immI_0 zero)
 6240 %{
 6241   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6242   match(Set mem (StoreB mem zero));
 6243 
 6244   ins_cost(125); // XXX
 6245   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6246   ins_encode %{
 6247     __ movb($mem$$Address, r12);
 6248   %}
 6249   ins_pipe(ialu_mem_reg);
 6250 %}
 6251 
 6252 instruct storeImmB(memory mem, immI8 src)
 6253 %{
 6254   match(Set mem (StoreB mem src));
 6255 
 6256   ins_cost(150); // XXX
 6257   format %{ "movb    $mem, $src\t# byte" %}
 6258   ins_encode %{
 6259     __ movb($mem$$Address, $src$$constant);
 6260   %}
 6261   ins_pipe(ialu_mem_imm);
 6262 %}
 6263 
 6264 // Store CMS card-mark Immediate
 6265 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6266 %{
 6267   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6268   match(Set mem (StoreCM mem zero));
 6269 
 6270   ins_cost(125); // XXX
 6271   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6272   ins_encode %{
 6273     __ movb($mem$$Address, r12);
 6274   %}
 6275   ins_pipe(ialu_mem_reg);
 6276 %}
 6277 
 6278 instruct storeImmCM0(memory mem, immI_0 src)
 6279 %{
 6280   match(Set mem (StoreCM mem src));
 6281 
 6282   ins_cost(150); // XXX
 6283   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6284   ins_encode %{
 6285     __ movb($mem$$Address, $src$$constant);
 6286   %}
 6287   ins_pipe(ialu_mem_imm);
 6288 %}
 6289 
 6290 // Store Float
 6291 instruct storeF(memory mem, regF src)
 6292 %{
 6293   match(Set mem (StoreF mem src));
 6294 
 6295   ins_cost(95); // XXX
 6296   format %{ "movss   $mem, $src\t# float" %}
 6297   ins_encode %{
 6298     __ movflt($mem$$Address, $src$$XMMRegister);
 6299   %}
 6300   ins_pipe(pipe_slow); // XXX
 6301 %}
 6302 
 6303 // Store immediate Float value (it is faster than store from XMM register)
 6304 instruct storeF0(memory mem, immF0 zero)
 6305 %{
 6306   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6307   match(Set mem (StoreF mem zero));
 6308 
 6309   ins_cost(25); // XXX
 6310   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6311   ins_encode %{
 6312     __ movl($mem$$Address, r12);
 6313   %}
 6314   ins_pipe(ialu_mem_reg);
 6315 %}
 6316 
 6317 instruct storeF_imm(memory mem, immF src)
 6318 %{
 6319   match(Set mem (StoreF mem src));
 6320 
 6321   ins_cost(50);
 6322   format %{ "movl    $mem, $src\t# float" %}
 6323   ins_encode %{
 6324     __ movl($mem$$Address, jint_cast($src$$constant));
 6325   %}
 6326   ins_pipe(ialu_mem_imm);
 6327 %}
 6328 
 6329 // Store Double
 6330 instruct storeD(memory mem, regD src)
 6331 %{
 6332   match(Set mem (StoreD mem src));
 6333 
 6334   ins_cost(95); // XXX
 6335   format %{ "movsd   $mem, $src\t# double" %}
 6336   ins_encode %{
 6337     __ movdbl($mem$$Address, $src$$XMMRegister);
 6338   %}
 6339   ins_pipe(pipe_slow); // XXX
 6340 %}
 6341 
 6342 // Store immediate double 0.0 (it is faster than store from XMM register)
 6343 instruct storeD0_imm(memory mem, immD0 src)
 6344 %{
 6345   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6346   match(Set mem (StoreD mem src));
 6347 
 6348   ins_cost(50);
 6349   format %{ "movq    $mem, $src\t# double 0." %}
 6350   ins_encode %{
 6351     __ movq($mem$$Address, $src$$constant);
 6352   %}
 6353   ins_pipe(ialu_mem_imm);
 6354 %}
 6355 
 6356 instruct storeD0(memory mem, immD0 zero)
 6357 %{
 6358   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6359   match(Set mem (StoreD mem zero));
 6360 
 6361   ins_cost(25); // XXX
 6362   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6363   ins_encode %{
 6364     __ movq($mem$$Address, r12);
 6365   %}
 6366   ins_pipe(ialu_mem_reg);
 6367 %}
 6368 
 6369 instruct storeSSI(stackSlotI dst, rRegI src)
 6370 %{
 6371   match(Set dst src);
 6372 
 6373   ins_cost(100);
 6374   format %{ "movl    $dst, $src\t# int stk" %}
 6375   opcode(0x89);
 6376   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6377   ins_pipe( ialu_mem_reg );
 6378 %}
 6379 
 6380 instruct storeSSL(stackSlotL dst, rRegL src)
 6381 %{
 6382   match(Set dst src);
 6383 
 6384   ins_cost(100);
 6385   format %{ "movq    $dst, $src\t# long stk" %}
 6386   opcode(0x89);
 6387   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6388   ins_pipe(ialu_mem_reg);
 6389 %}
 6390 
 6391 instruct storeSSP(stackSlotP dst, rRegP src)
 6392 %{
 6393   match(Set dst src);
 6394 
 6395   ins_cost(100);
 6396   format %{ "movq    $dst, $src\t# ptr stk" %}
 6397   opcode(0x89);
 6398   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6399   ins_pipe(ialu_mem_reg);
 6400 %}
 6401 
 6402 instruct storeSSF(stackSlotF dst, regF src)
 6403 %{
 6404   match(Set dst src);
 6405 
 6406   ins_cost(95); // XXX
 6407   format %{ "movss   $dst, $src\t# float stk" %}
 6408   ins_encode %{
 6409     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6410   %}
 6411   ins_pipe(pipe_slow); // XXX
 6412 %}
 6413 
 6414 instruct storeSSD(stackSlotD dst, regD src)
 6415 %{
 6416   match(Set dst src);
 6417 
 6418   ins_cost(95); // XXX
 6419   format %{ "movsd   $dst, $src\t# double stk" %}
 6420   ins_encode %{
 6421     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6422   %}
 6423   ins_pipe(pipe_slow); // XXX
 6424 %}
 6425 
 6426 instruct cacheWB(indirect addr)
 6427 %{
 6428   predicate(VM_Version::supports_data_cache_line_flush());
 6429   match(CacheWB addr);
 6430 
 6431   ins_cost(100);
 6432   format %{"cache wb $addr" %}
 6433   ins_encode %{
 6434     assert($addr->index_position() < 0, "should be");
 6435     assert($addr$$disp == 0, "should be");
 6436     __ cache_wb(Address($addr$$base$$Register, 0));
 6437   %}
 6438   ins_pipe(pipe_slow); // XXX
 6439 %}
 6440 
 6441 instruct cacheWBPreSync()
 6442 %{
 6443   predicate(VM_Version::supports_data_cache_line_flush());
 6444   match(CacheWBPreSync);
 6445 
 6446   ins_cost(100);
 6447   format %{"cache wb presync" %}
 6448   ins_encode %{
 6449     __ cache_wbsync(true);
 6450   %}
 6451   ins_pipe(pipe_slow); // XXX
 6452 %}
 6453 
 6454 instruct cacheWBPostSync()
 6455 %{
 6456   predicate(VM_Version::supports_data_cache_line_flush());
 6457   match(CacheWBPostSync);
 6458 
 6459   ins_cost(100);
 6460   format %{"cache wb postsync" %}
 6461   ins_encode %{
 6462     __ cache_wbsync(false);
 6463   %}
 6464   ins_pipe(pipe_slow); // XXX
 6465 %}
 6466 
 6467 //----------BSWAP Instructions-------------------------------------------------
 6468 instruct bytes_reverse_int(rRegI dst) %{
 6469   match(Set dst (ReverseBytesI dst));
 6470 
 6471   format %{ "bswapl  $dst" %}
 6472   ins_encode %{
 6473     __ bswapl($dst$$Register);
 6474   %}
 6475   ins_pipe( ialu_reg );
 6476 %}
 6477 
 6478 instruct bytes_reverse_long(rRegL dst) %{
 6479   match(Set dst (ReverseBytesL dst));
 6480 
 6481   format %{ "bswapq  $dst" %}
 6482   ins_encode %{
 6483     __ bswapq($dst$$Register);
 6484   %}
 6485   ins_pipe( ialu_reg);
 6486 %}
 6487 
 6488 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6489   match(Set dst (ReverseBytesUS dst));
 6490   effect(KILL cr);
 6491 
 6492   format %{ "bswapl  $dst\n\t"
 6493             "shrl    $dst,16\n\t" %}
 6494   ins_encode %{
 6495     __ bswapl($dst$$Register);
 6496     __ shrl($dst$$Register, 16);
 6497   %}
 6498   ins_pipe( ialu_reg );
 6499 %}
 6500 
 6501 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6502   match(Set dst (ReverseBytesS dst));
 6503   effect(KILL cr);
 6504 
 6505   format %{ "bswapl  $dst\n\t"
 6506             "sar     $dst,16\n\t" %}
 6507   ins_encode %{
 6508     __ bswapl($dst$$Register);
 6509     __ sarl($dst$$Register, 16);
 6510   %}
 6511   ins_pipe( ialu_reg );
 6512 %}
 6513 
 6514 //---------- Zeros Count Instructions ------------------------------------------
 6515 
 6516 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6517   predicate(UseCountLeadingZerosInstruction);
 6518   match(Set dst (CountLeadingZerosI src));
 6519   effect(KILL cr);
 6520 
 6521   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6522   ins_encode %{
 6523     __ lzcntl($dst$$Register, $src$$Register);
 6524   %}
 6525   ins_pipe(ialu_reg);
 6526 %}
 6527 
 6528 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6529   predicate(!UseCountLeadingZerosInstruction);
 6530   match(Set dst (CountLeadingZerosI src));
 6531   effect(KILL cr);
 6532 
 6533   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6534             "jnz     skip\n\t"
 6535             "movl    $dst, -1\n"
 6536       "skip:\n\t"
 6537             "negl    $dst\n\t"
 6538             "addl    $dst, 31" %}
 6539   ins_encode %{
 6540     Register Rdst = $dst$$Register;
 6541     Register Rsrc = $src$$Register;
 6542     Label skip;
 6543     __ bsrl(Rdst, Rsrc);
 6544     __ jccb(Assembler::notZero, skip);
 6545     __ movl(Rdst, -1);
 6546     __ bind(skip);
 6547     __ negl(Rdst);
 6548     __ addl(Rdst, BitsPerInt - 1);
 6549   %}
 6550   ins_pipe(ialu_reg);
 6551 %}
 6552 
 6553 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6554   predicate(UseCountLeadingZerosInstruction);
 6555   match(Set dst (CountLeadingZerosL src));
 6556   effect(KILL cr);
 6557 
 6558   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6559   ins_encode %{
 6560     __ lzcntq($dst$$Register, $src$$Register);
 6561   %}
 6562   ins_pipe(ialu_reg);
 6563 %}
 6564 
 6565 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6566   predicate(!UseCountLeadingZerosInstruction);
 6567   match(Set dst (CountLeadingZerosL src));
 6568   effect(KILL cr);
 6569 
 6570   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6571             "jnz     skip\n\t"
 6572             "movl    $dst, -1\n"
 6573       "skip:\n\t"
 6574             "negl    $dst\n\t"
 6575             "addl    $dst, 63" %}
 6576   ins_encode %{
 6577     Register Rdst = $dst$$Register;
 6578     Register Rsrc = $src$$Register;
 6579     Label skip;
 6580     __ bsrq(Rdst, Rsrc);
 6581     __ jccb(Assembler::notZero, skip);
 6582     __ movl(Rdst, -1);
 6583     __ bind(skip);
 6584     __ negl(Rdst);
 6585     __ addl(Rdst, BitsPerLong - 1);
 6586   %}
 6587   ins_pipe(ialu_reg);
 6588 %}
 6589 
 6590 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6591   predicate(UseCountTrailingZerosInstruction);
 6592   match(Set dst (CountTrailingZerosI src));
 6593   effect(KILL cr);
 6594 
 6595   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6596   ins_encode %{
 6597     __ tzcntl($dst$$Register, $src$$Register);
 6598   %}
 6599   ins_pipe(ialu_reg);
 6600 %}
 6601 
 6602 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6603   predicate(!UseCountTrailingZerosInstruction);
 6604   match(Set dst (CountTrailingZerosI src));
 6605   effect(KILL cr);
 6606 
 6607   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6608             "jnz     done\n\t"
 6609             "movl    $dst, 32\n"
 6610       "done:" %}
 6611   ins_encode %{
 6612     Register Rdst = $dst$$Register;
 6613     Label done;
 6614     __ bsfl(Rdst, $src$$Register);
 6615     __ jccb(Assembler::notZero, done);
 6616     __ movl(Rdst, BitsPerInt);
 6617     __ bind(done);
 6618   %}
 6619   ins_pipe(ialu_reg);
 6620 %}
 6621 
 6622 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6623   predicate(UseCountTrailingZerosInstruction);
 6624   match(Set dst (CountTrailingZerosL src));
 6625   effect(KILL cr);
 6626 
 6627   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6628   ins_encode %{
 6629     __ tzcntq($dst$$Register, $src$$Register);
 6630   %}
 6631   ins_pipe(ialu_reg);
 6632 %}
 6633 
 6634 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6635   predicate(!UseCountTrailingZerosInstruction);
 6636   match(Set dst (CountTrailingZerosL src));
 6637   effect(KILL cr);
 6638 
 6639   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6640             "jnz     done\n\t"
 6641             "movl    $dst, 64\n"
 6642       "done:" %}
 6643   ins_encode %{
 6644     Register Rdst = $dst$$Register;
 6645     Label done;
 6646     __ bsfq(Rdst, $src$$Register);
 6647     __ jccb(Assembler::notZero, done);
 6648     __ movl(Rdst, BitsPerLong);
 6649     __ bind(done);
 6650   %}
 6651   ins_pipe(ialu_reg);
 6652 %}
 6653 
 6654 
 6655 //---------- Population Count Instructions -------------------------------------
 6656 
 6657 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6658   predicate(UsePopCountInstruction);
 6659   match(Set dst (PopCountI src));
 6660   effect(KILL cr);
 6661 
 6662   format %{ "popcnt  $dst, $src" %}
 6663   ins_encode %{
 6664     __ popcntl($dst$$Register, $src$$Register);
 6665   %}
 6666   ins_pipe(ialu_reg);
 6667 %}
 6668 
 6669 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6670   predicate(UsePopCountInstruction);
 6671   match(Set dst (PopCountI (LoadI mem)));
 6672   effect(KILL cr);
 6673 
 6674   format %{ "popcnt  $dst, $mem" %}
 6675   ins_encode %{
 6676     __ popcntl($dst$$Register, $mem$$Address);
 6677   %}
 6678   ins_pipe(ialu_reg);
 6679 %}
 6680 
 6681 // Note: Long.bitCount(long) returns an int.
 6682 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6683   predicate(UsePopCountInstruction);
 6684   match(Set dst (PopCountL src));
 6685   effect(KILL cr);
 6686 
 6687   format %{ "popcnt  $dst, $src" %}
 6688   ins_encode %{
 6689     __ popcntq($dst$$Register, $src$$Register);
 6690   %}
 6691   ins_pipe(ialu_reg);
 6692 %}
 6693 
 6694 // Note: Long.bitCount(long) returns an int.
 6695 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6696   predicate(UsePopCountInstruction);
 6697   match(Set dst (PopCountL (LoadL mem)));
 6698   effect(KILL cr);
 6699 
 6700   format %{ "popcnt  $dst, $mem" %}
 6701   ins_encode %{
 6702     __ popcntq($dst$$Register, $mem$$Address);
 6703   %}
 6704   ins_pipe(ialu_reg);
 6705 %}
 6706 
 6707 
 6708 //----------MemBar Instructions-----------------------------------------------
 6709 // Memory barrier flavors
 6710 
 6711 instruct membar_acquire()
 6712 %{
 6713   match(MemBarAcquire);
 6714   match(LoadFence);
 6715   ins_cost(0);
 6716 
 6717   size(0);
 6718   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6719   ins_encode();
 6720   ins_pipe(empty);
 6721 %}
 6722 
 6723 instruct membar_acquire_lock()
 6724 %{
 6725   match(MemBarAcquireLock);
 6726   ins_cost(0);
 6727 
 6728   size(0);
 6729   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6730   ins_encode();
 6731   ins_pipe(empty);
 6732 %}
 6733 
 6734 instruct membar_release()
 6735 %{
 6736   match(MemBarRelease);
 6737   match(StoreFence);
 6738   ins_cost(0);
 6739 
 6740   size(0);
 6741   format %{ "MEMBAR-release ! (empty encoding)" %}
 6742   ins_encode();
 6743   ins_pipe(empty);
 6744 %}
 6745 
 6746 instruct membar_release_lock()
 6747 %{
 6748   match(MemBarReleaseLock);
 6749   ins_cost(0);
 6750 
 6751   size(0);
 6752   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6753   ins_encode();
 6754   ins_pipe(empty);
 6755 %}
 6756 
 6757 instruct membar_volatile(rFlagsReg cr) %{
 6758   match(MemBarVolatile);
 6759   effect(KILL cr);
 6760   ins_cost(400);
 6761 
 6762   format %{
 6763     $$template
 6764     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6765   %}
 6766   ins_encode %{
 6767     __ membar(Assembler::StoreLoad);
 6768   %}
 6769   ins_pipe(pipe_slow);
 6770 %}
 6771 
 6772 instruct unnecessary_membar_volatile()
 6773 %{
 6774   match(MemBarVolatile);
 6775   predicate(Matcher::post_store_load_barrier(n));
 6776   ins_cost(0);
 6777 
 6778   size(0);
 6779   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6780   ins_encode();
 6781   ins_pipe(empty);
 6782 %}
 6783 
 6784 instruct membar_storestore() %{
 6785   match(MemBarStoreStore);
 6786   match(StoreStoreFence);
 6787   ins_cost(0);
 6788 
 6789   size(0);
 6790   format %{ "MEMBAR-storestore (empty encoding)" %}
 6791   ins_encode( );
 6792   ins_pipe(empty);
 6793 %}
 6794 
 6795 //----------Move Instructions--------------------------------------------------
 6796 
 6797 instruct castX2P(rRegP dst, rRegL src)
 6798 %{
 6799   match(Set dst (CastX2P src));
 6800 
 6801   format %{ "movq    $dst, $src\t# long->ptr" %}
 6802   ins_encode %{
 6803     if ($dst$$reg != $src$$reg) {
 6804       __ movptr($dst$$Register, $src$$Register);
 6805     }
 6806   %}
 6807   ins_pipe(ialu_reg_reg); // XXX
 6808 %}
 6809 
 6810 instruct castP2X(rRegL dst, rRegP src)
 6811 %{
 6812   match(Set dst (CastP2X src));
 6813 
 6814   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6815   ins_encode %{
 6816     if ($dst$$reg != $src$$reg) {
 6817       __ movptr($dst$$Register, $src$$Register);
 6818     }
 6819   %}
 6820   ins_pipe(ialu_reg_reg); // XXX
 6821 %}
 6822 
 6823 // Convert oop into int for vectors alignment masking
 6824 instruct convP2I(rRegI dst, rRegP src)
 6825 %{
 6826   match(Set dst (ConvL2I (CastP2X src)));
 6827 
 6828   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6829   ins_encode %{
 6830     __ movl($dst$$Register, $src$$Register);
 6831   %}
 6832   ins_pipe(ialu_reg_reg); // XXX
 6833 %}
 6834 
 6835 // Convert compressed oop into int for vectors alignment masking
 6836 // in case of 32bit oops (heap < 4Gb).
 6837 instruct convN2I(rRegI dst, rRegN src)
 6838 %{
 6839   predicate(CompressedOops::shift() == 0);
 6840   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6841 
 6842   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6843   ins_encode %{
 6844     __ movl($dst$$Register, $src$$Register);
 6845   %}
 6846   ins_pipe(ialu_reg_reg); // XXX
 6847 %}
 6848 
 6849 // Convert oop pointer into compressed form
 6850 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6851   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6852   match(Set dst (EncodeP src));
 6853   effect(KILL cr);
 6854   format %{ "encode_heap_oop $dst,$src" %}
 6855   ins_encode %{
 6856     Register s = $src$$Register;
 6857     Register d = $dst$$Register;
 6858     if (s != d) {
 6859       __ movq(d, s);
 6860     }
 6861     __ encode_heap_oop(d);
 6862   %}
 6863   ins_pipe(ialu_reg_long);
 6864 %}
 6865 
 6866 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6867   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6868   match(Set dst (EncodeP src));
 6869   effect(KILL cr);
 6870   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6871   ins_encode %{
 6872     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6873   %}
 6874   ins_pipe(ialu_reg_long);
 6875 %}
 6876 
 6877 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6878   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6879             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6880   match(Set dst (DecodeN src));
 6881   effect(KILL cr);
 6882   format %{ "decode_heap_oop $dst,$src" %}
 6883   ins_encode %{
 6884     Register s = $src$$Register;
 6885     Register d = $dst$$Register;
 6886     if (s != d) {
 6887       __ movq(d, s);
 6888     }
 6889     __ decode_heap_oop(d);
 6890   %}
 6891   ins_pipe(ialu_reg_long);
 6892 %}
 6893 
 6894 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6895   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6896             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6897   match(Set dst (DecodeN src));
 6898   effect(KILL cr);
 6899   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6900   ins_encode %{
 6901     Register s = $src$$Register;
 6902     Register d = $dst$$Register;
 6903     if (s != d) {
 6904       __ decode_heap_oop_not_null(d, s);
 6905     } else {
 6906       __ decode_heap_oop_not_null(d);
 6907     }
 6908   %}
 6909   ins_pipe(ialu_reg_long);
 6910 %}
 6911 
 6912 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6913   match(Set dst (EncodePKlass src));
 6914   effect(TEMP dst, KILL cr);
 6915   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6916   ins_encode %{
 6917     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6918   %}
 6919   ins_pipe(ialu_reg_long);
 6920 %}
 6921 
 6922 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6923   match(Set dst (DecodeNKlass src));
 6924   effect(TEMP dst, KILL cr);
 6925   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6926   ins_encode %{
 6927     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6928   %}
 6929   ins_pipe(ialu_reg_long);
 6930 %}
 6931 
 6932 //----------Conditional Move---------------------------------------------------
 6933 // Jump
 6934 // dummy instruction for generating temp registers
 6935 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6936   match(Jump (LShiftL switch_val shift));
 6937   ins_cost(350);
 6938   predicate(false);
 6939   effect(TEMP dest);
 6940 
 6941   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6942             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6943   ins_encode %{
 6944     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6945     // to do that and the compiler is using that register as one it can allocate.
 6946     // So we build it all by hand.
 6947     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6948     // ArrayAddress dispatch(table, index);
 6949     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6950     __ lea($dest$$Register, $constantaddress);
 6951     __ jmp(dispatch);
 6952   %}
 6953   ins_pipe(pipe_jmp);
 6954 %}
 6955 
 6956 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6957   match(Jump (AddL (LShiftL switch_val shift) offset));
 6958   ins_cost(350);
 6959   effect(TEMP dest);
 6960 
 6961   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6962             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6963   ins_encode %{
 6964     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6965     // to do that and the compiler is using that register as one it can allocate.
 6966     // So we build it all by hand.
 6967     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6968     // ArrayAddress dispatch(table, index);
 6969     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6970     __ lea($dest$$Register, $constantaddress);
 6971     __ jmp(dispatch);
 6972   %}
 6973   ins_pipe(pipe_jmp);
 6974 %}
 6975 
 6976 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6977   match(Jump switch_val);
 6978   ins_cost(350);
 6979   effect(TEMP dest);
 6980 
 6981   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6982             "jmp     [$dest + $switch_val]\n\t" %}
 6983   ins_encode %{
 6984     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6985     // to do that and the compiler is using that register as one it can allocate.
 6986     // So we build it all by hand.
 6987     // Address index(noreg, switch_reg, Address::times_1);
 6988     // ArrayAddress dispatch(table, index);
 6989     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6990     __ lea($dest$$Register, $constantaddress);
 6991     __ jmp(dispatch);
 6992   %}
 6993   ins_pipe(pipe_jmp);
 6994 %}
 6995 
 6996 // Conditional move
 6997 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6998 %{
 6999   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7000 
 7001   ins_cost(200); // XXX
 7002   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7003   ins_encode %{
 7004     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7005   %}
 7006   ins_pipe(pipe_cmov_reg);
 7007 %}
 7008 
 7009 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7010   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7011 
 7012   ins_cost(200); // XXX
 7013   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7014   ins_encode %{
 7015     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7016   %}
 7017   ins_pipe(pipe_cmov_reg);
 7018 %}
 7019 
 7020 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7021   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7022   ins_cost(200);
 7023   expand %{
 7024     cmovI_regU(cop, cr, dst, src);
 7025   %}
 7026 %}
 7027 
 7028 // Conditional move
 7029 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7030   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7031 
 7032   ins_cost(250); // XXX
 7033   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7034   ins_encode %{
 7035     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7036   %}
 7037   ins_pipe(pipe_cmov_mem);
 7038 %}
 7039 
 7040 // Conditional move
 7041 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7042 %{
 7043   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7044 
 7045   ins_cost(250); // XXX
 7046   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7047   ins_encode %{
 7048     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7049   %}
 7050   ins_pipe(pipe_cmov_mem);
 7051 %}
 7052 
 7053 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7054   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7055   ins_cost(250);
 7056   expand %{
 7057     cmovI_memU(cop, cr, dst, src);
 7058   %}
 7059 %}
 7060 
 7061 // Conditional move
 7062 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7063 %{
 7064   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7065 
 7066   ins_cost(200); // XXX
 7067   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7068   ins_encode %{
 7069     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7070   %}
 7071   ins_pipe(pipe_cmov_reg);
 7072 %}
 7073 
 7074 // Conditional move
 7075 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7076 %{
 7077   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7078 
 7079   ins_cost(200); // XXX
 7080   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7081   ins_encode %{
 7082     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7083   %}
 7084   ins_pipe(pipe_cmov_reg);
 7085 %}
 7086 
 7087 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7088   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7089   ins_cost(200);
 7090   expand %{
 7091     cmovN_regU(cop, cr, dst, src);
 7092   %}
 7093 %}
 7094 
 7095 // Conditional move
 7096 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7097 %{
 7098   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7099 
 7100   ins_cost(200); // XXX
 7101   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7102   ins_encode %{
 7103     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7104   %}
 7105   ins_pipe(pipe_cmov_reg);  // XXX
 7106 %}
 7107 
 7108 // Conditional move
 7109 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7110 %{
 7111   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7112 
 7113   ins_cost(200); // XXX
 7114   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7115   ins_encode %{
 7116     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7117   %}
 7118   ins_pipe(pipe_cmov_reg); // XXX
 7119 %}
 7120 
 7121 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7122   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7123   ins_cost(200);
 7124   expand %{
 7125     cmovP_regU(cop, cr, dst, src);
 7126   %}
 7127 %}
 7128 
 7129 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7130 // correctly meets the two pointer arguments; one is an incoming
 7131 // register but the other is a memory operand.  ALSO appears to
 7132 // be buggy with implicit null checks.
 7133 //
 7134 //// Conditional move
 7135 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7136 //%{
 7137 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7138 //  ins_cost(250);
 7139 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7140 //  opcode(0x0F,0x40);
 7141 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7142 //  ins_pipe( pipe_cmov_mem );
 7143 //%}
 7144 //
 7145 //// Conditional move
 7146 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7147 //%{
 7148 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7149 //  ins_cost(250);
 7150 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7151 //  opcode(0x0F,0x40);
 7152 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7153 //  ins_pipe( pipe_cmov_mem );
 7154 //%}
 7155 
 7156 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7157 %{
 7158   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7159 
 7160   ins_cost(200); // XXX
 7161   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7162   ins_encode %{
 7163     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7164   %}
 7165   ins_pipe(pipe_cmov_reg);  // XXX
 7166 %}
 7167 
 7168 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7169 %{
 7170   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7171 
 7172   ins_cost(200); // XXX
 7173   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7174   ins_encode %{
 7175     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7176   %}
 7177   ins_pipe(pipe_cmov_mem);  // XXX
 7178 %}
 7179 
 7180 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7181 %{
 7182   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7183 
 7184   ins_cost(200); // XXX
 7185   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7186   ins_encode %{
 7187     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7188   %}
 7189   ins_pipe(pipe_cmov_reg); // XXX
 7190 %}
 7191 
 7192 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7193   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7194   ins_cost(200);
 7195   expand %{
 7196     cmovL_regU(cop, cr, dst, src);
 7197   %}
 7198 %}
 7199 
 7200 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7201 %{
 7202   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7203 
 7204   ins_cost(200); // XXX
 7205   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7206   ins_encode %{
 7207     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7208   %}
 7209   ins_pipe(pipe_cmov_mem); // XXX
 7210 %}
 7211 
 7212 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7213   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7214   ins_cost(200);
 7215   expand %{
 7216     cmovL_memU(cop, cr, dst, src);
 7217   %}
 7218 %}
 7219 
 7220 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7221 %{
 7222   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7223 
 7224   ins_cost(200); // XXX
 7225   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7226             "movss     $dst, $src\n"
 7227     "skip:" %}
 7228   ins_encode %{
 7229     Label Lskip;
 7230     // Invert sense of branch from sense of CMOV
 7231     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7232     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7233     __ bind(Lskip);
 7234   %}
 7235   ins_pipe(pipe_slow);
 7236 %}
 7237 
 7238 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7239 // %{
 7240 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7241 
 7242 //   ins_cost(200); // XXX
 7243 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7244 //             "movss     $dst, $src\n"
 7245 //     "skip:" %}
 7246 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7247 //   ins_pipe(pipe_slow);
 7248 // %}
 7249 
 7250 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7251 %{
 7252   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7253 
 7254   ins_cost(200); // XXX
 7255   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7256             "movss     $dst, $src\n"
 7257     "skip:" %}
 7258   ins_encode %{
 7259     Label Lskip;
 7260     // Invert sense of branch from sense of CMOV
 7261     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7262     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7263     __ bind(Lskip);
 7264   %}
 7265   ins_pipe(pipe_slow);
 7266 %}
 7267 
 7268 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7269   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7270   ins_cost(200);
 7271   expand %{
 7272     cmovF_regU(cop, cr, dst, src);
 7273   %}
 7274 %}
 7275 
 7276 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7277 %{
 7278   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7279 
 7280   ins_cost(200); // XXX
 7281   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7282             "movsd     $dst, $src\n"
 7283     "skip:" %}
 7284   ins_encode %{
 7285     Label Lskip;
 7286     // Invert sense of branch from sense of CMOV
 7287     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7288     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7289     __ bind(Lskip);
 7290   %}
 7291   ins_pipe(pipe_slow);
 7292 %}
 7293 
 7294 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7295 %{
 7296   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7297 
 7298   ins_cost(200); // XXX
 7299   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7300             "movsd     $dst, $src\n"
 7301     "skip:" %}
 7302   ins_encode %{
 7303     Label Lskip;
 7304     // Invert sense of branch from sense of CMOV
 7305     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7306     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7307     __ bind(Lskip);
 7308   %}
 7309   ins_pipe(pipe_slow);
 7310 %}
 7311 
 7312 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7313   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7314   ins_cost(200);
 7315   expand %{
 7316     cmovD_regU(cop, cr, dst, src);
 7317   %}
 7318 %}
 7319 
 7320 //----------Arithmetic Instructions--------------------------------------------
 7321 //----------Addition Instructions----------------------------------------------
 7322 
 7323 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7324 %{
 7325   match(Set dst (AddI dst src));
 7326   effect(KILL cr);
 7327 
 7328   format %{ "addl    $dst, $src\t# int" %}
 7329   ins_encode %{
 7330     __ addl($dst$$Register, $src$$Register);
 7331   %}
 7332   ins_pipe(ialu_reg_reg);
 7333 %}
 7334 
 7335 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7336 %{
 7337   match(Set dst (AddI dst src));
 7338   effect(KILL cr);
 7339 
 7340   format %{ "addl    $dst, $src\t# int" %}
 7341   ins_encode %{
 7342     __ addl($dst$$Register, $src$$constant);
 7343   %}
 7344   ins_pipe( ialu_reg );
 7345 %}
 7346 
 7347 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7348 %{
 7349   match(Set dst (AddI dst (LoadI src)));
 7350   effect(KILL cr);
 7351 
 7352   ins_cost(125); // XXX
 7353   format %{ "addl    $dst, $src\t# int" %}
 7354   ins_encode %{
 7355     __ addl($dst$$Register, $src$$Address);
 7356   %}
 7357   ins_pipe(ialu_reg_mem);
 7358 %}
 7359 
 7360 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7361 %{
 7362   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7363   effect(KILL cr);
 7364 
 7365   ins_cost(150); // XXX
 7366   format %{ "addl    $dst, $src\t# int" %}
 7367   ins_encode %{
 7368     __ addl($dst$$Address, $src$$Register);
 7369   %}
 7370   ins_pipe(ialu_mem_reg);
 7371 %}
 7372 
 7373 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7374 %{
 7375   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7376   effect(KILL cr);
 7377 
 7378   ins_cost(125); // XXX
 7379   format %{ "addl    $dst, $src\t# int" %}
 7380   ins_encode %{
 7381     __ addl($dst$$Address, $src$$constant);
 7382   %}
 7383   ins_pipe(ialu_mem_imm);
 7384 %}
 7385 
 7386 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7387 %{
 7388   predicate(UseIncDec);
 7389   match(Set dst (AddI dst src));
 7390   effect(KILL cr);
 7391 
 7392   format %{ "incl    $dst\t# int" %}
 7393   ins_encode %{
 7394     __ incrementl($dst$$Register);
 7395   %}
 7396   ins_pipe(ialu_reg);
 7397 %}
 7398 
 7399 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7400 %{
 7401   predicate(UseIncDec);
 7402   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7403   effect(KILL cr);
 7404 
 7405   ins_cost(125); // XXX
 7406   format %{ "incl    $dst\t# int" %}
 7407   ins_encode %{
 7408     __ incrementl($dst$$Address);
 7409   %}
 7410   ins_pipe(ialu_mem_imm);
 7411 %}
 7412 
 7413 // XXX why does that use AddI
 7414 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7415 %{
 7416   predicate(UseIncDec);
 7417   match(Set dst (AddI dst src));
 7418   effect(KILL cr);
 7419 
 7420   format %{ "decl    $dst\t# int" %}
 7421   ins_encode %{
 7422     __ decrementl($dst$$Register);
 7423   %}
 7424   ins_pipe(ialu_reg);
 7425 %}
 7426 
 7427 // XXX why does that use AddI
 7428 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7429 %{
 7430   predicate(UseIncDec);
 7431   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7432   effect(KILL cr);
 7433 
 7434   ins_cost(125); // XXX
 7435   format %{ "decl    $dst\t# int" %}
 7436   ins_encode %{
 7437     __ decrementl($dst$$Address);
 7438   %}
 7439   ins_pipe(ialu_mem_imm);
 7440 %}
 7441 
 7442 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
 7443 %{
 7444   match(Set dst (AddI src0 src1));
 7445 
 7446   ins_cost(110);
 7447   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
 7448   ins_encode %{
 7449     __ leal($dst$$Register, Address($src0$$Register, $src1$$constant));
 7450   %}
 7451   ins_pipe(ialu_reg_reg);
 7452 %}
 7453 
 7454 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7455 %{
 7456   match(Set dst (AddL dst src));
 7457   effect(KILL cr);
 7458 
 7459   format %{ "addq    $dst, $src\t# long" %}
 7460   ins_encode %{
 7461     __ addq($dst$$Register, $src$$Register);
 7462   %}
 7463   ins_pipe(ialu_reg_reg);
 7464 %}
 7465 
 7466 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7467 %{
 7468   match(Set dst (AddL dst src));
 7469   effect(KILL cr);
 7470 
 7471   format %{ "addq    $dst, $src\t# long" %}
 7472   ins_encode %{
 7473     __ addq($dst$$Register, $src$$constant);
 7474   %}
 7475   ins_pipe( ialu_reg );
 7476 %}
 7477 
 7478 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7479 %{
 7480   match(Set dst (AddL dst (LoadL src)));
 7481   effect(KILL cr);
 7482 
 7483   ins_cost(125); // XXX
 7484   format %{ "addq    $dst, $src\t# long" %}
 7485   ins_encode %{
 7486     __ addq($dst$$Register, $src$$Address);
 7487   %}
 7488   ins_pipe(ialu_reg_mem);
 7489 %}
 7490 
 7491 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7492 %{
 7493   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7494   effect(KILL cr);
 7495 
 7496   ins_cost(150); // XXX
 7497   format %{ "addq    $dst, $src\t# long" %}
 7498   ins_encode %{
 7499     __ addq($dst$$Address, $src$$Register);
 7500   %}
 7501   ins_pipe(ialu_mem_reg);
 7502 %}
 7503 
 7504 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7505 %{
 7506   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7507   effect(KILL cr);
 7508 
 7509   ins_cost(125); // XXX
 7510   format %{ "addq    $dst, $src\t# long" %}
 7511   ins_encode %{
 7512     __ addq($dst$$Address, $src$$constant);
 7513   %}
 7514   ins_pipe(ialu_mem_imm);
 7515 %}
 7516 
 7517 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7518 %{
 7519   predicate(UseIncDec);
 7520   match(Set dst (AddL dst src));
 7521   effect(KILL cr);
 7522 
 7523   format %{ "incq    $dst\t# long" %}
 7524   ins_encode %{
 7525     __ incrementq($dst$$Register);
 7526   %}
 7527   ins_pipe(ialu_reg);
 7528 %}
 7529 
 7530 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7531 %{
 7532   predicate(UseIncDec);
 7533   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7534   effect(KILL cr);
 7535 
 7536   ins_cost(125); // XXX
 7537   format %{ "incq    $dst\t# long" %}
 7538   ins_encode %{
 7539     __ incrementq($dst$$Address);
 7540   %}
 7541   ins_pipe(ialu_mem_imm);
 7542 %}
 7543 
 7544 // XXX why does that use AddL
 7545 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7546 %{
 7547   predicate(UseIncDec);
 7548   match(Set dst (AddL dst src));
 7549   effect(KILL cr);
 7550 
 7551   format %{ "decq    $dst\t# long" %}
 7552   ins_encode %{
 7553     __ decrementq($dst$$Register);
 7554   %}
 7555   ins_pipe(ialu_reg);
 7556 %}
 7557 
 7558 // XXX why does that use AddL
 7559 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7560 %{
 7561   predicate(UseIncDec);
 7562   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7563   effect(KILL cr);
 7564 
 7565   ins_cost(125); // XXX
 7566   format %{ "decq    $dst\t# long" %}
 7567   ins_encode %{
 7568     __ decrementq($dst$$Address);
 7569   %}
 7570   ins_pipe(ialu_mem_imm);
 7571 %}
 7572 
 7573 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
 7574 %{
 7575   match(Set dst (AddL src0 src1));
 7576 
 7577   ins_cost(110);
 7578   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
 7579   ins_encode %{
 7580     __ leaq($dst$$Register, Address($src0$$Register, $src1$$constant));
 7581   %}
 7582   ins_pipe(ialu_reg_reg);
 7583 %}
 7584 
 7585 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7586 %{
 7587   match(Set dst (AddP dst src));
 7588   effect(KILL cr);
 7589 
 7590   format %{ "addq    $dst, $src\t# ptr" %}
 7591   ins_encode %{
 7592     __ addq($dst$$Register, $src$$Register);
 7593   %}
 7594   ins_pipe(ialu_reg_reg);
 7595 %}
 7596 
 7597 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7598 %{
 7599   match(Set dst (AddP dst src));
 7600   effect(KILL cr);
 7601 
 7602   format %{ "addq    $dst, $src\t# ptr" %}
 7603   ins_encode %{
 7604     __ addq($dst$$Register, $src$$constant);
 7605   %}
 7606   ins_pipe( ialu_reg );
 7607 %}
 7608 
 7609 // XXX addP mem ops ????
 7610 
 7611 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
 7612 %{
 7613   match(Set dst (AddP src0 src1));
 7614 
 7615   ins_cost(110);
 7616   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
 7617   ins_encode %{
 7618     __ leaq($dst$$Register, Address($src0$$Register, $src1$$constant));
 7619   %}
 7620   ins_pipe(ialu_reg_reg);
 7621 %}
 7622 
 7623 instruct checkCastPP(rRegP dst)
 7624 %{
 7625   match(Set dst (CheckCastPP dst));
 7626 
 7627   size(0);
 7628   format %{ "# checkcastPP of $dst" %}
 7629   ins_encode(/* empty encoding */);
 7630   ins_pipe(empty);
 7631 %}
 7632 
 7633 instruct castPP(rRegP dst)
 7634 %{
 7635   match(Set dst (CastPP dst));
 7636 
 7637   size(0);
 7638   format %{ "# castPP of $dst" %}
 7639   ins_encode(/* empty encoding */);
 7640   ins_pipe(empty);
 7641 %}
 7642 
 7643 instruct castII(rRegI dst)
 7644 %{
 7645   match(Set dst (CastII dst));
 7646 
 7647   size(0);
 7648   format %{ "# castII of $dst" %}
 7649   ins_encode(/* empty encoding */);
 7650   ins_cost(0);
 7651   ins_pipe(empty);
 7652 %}
 7653 
 7654 instruct castLL(rRegL dst)
 7655 %{
 7656   match(Set dst (CastLL dst));
 7657 
 7658   size(0);
 7659   format %{ "# castLL of $dst" %}
 7660   ins_encode(/* empty encoding */);
 7661   ins_cost(0);
 7662   ins_pipe(empty);
 7663 %}
 7664 
 7665 instruct castFF(regF dst)
 7666 %{
 7667   match(Set dst (CastFF dst));
 7668 
 7669   size(0);
 7670   format %{ "# castFF of $dst" %}
 7671   ins_encode(/* empty encoding */);
 7672   ins_cost(0);
 7673   ins_pipe(empty);
 7674 %}
 7675 
 7676 instruct castDD(regD dst)
 7677 %{
 7678   match(Set dst (CastDD dst));
 7679 
 7680   size(0);
 7681   format %{ "# castDD of $dst" %}
 7682   ins_encode(/* empty encoding */);
 7683   ins_cost(0);
 7684   ins_pipe(empty);
 7685 %}
 7686 
 7687 // LoadP-locked same as a regular LoadP when used with compare-swap
 7688 instruct loadPLocked(rRegP dst, memory mem)
 7689 %{
 7690   match(Set dst (LoadPLocked mem));
 7691 
 7692   ins_cost(125); // XXX
 7693   format %{ "movq    $dst, $mem\t# ptr locked" %}
 7694   ins_encode %{
 7695     __ movq($dst$$Register, $mem$$Address);
 7696   %}
 7697   ins_pipe(ialu_reg_mem); // XXX
 7698 %}
 7699 
 7700 // Conditional-store of the updated heap-top.
 7701 // Used during allocation of the shared heap.
 7702 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
 7703 
 7704 instruct storePConditional(memory heap_top_ptr,
 7705                            rax_RegP oldval, rRegP newval,
 7706                            rFlagsReg cr)
 7707 %{
 7708   predicate(n->as_LoadStore()->barrier_data() == 0);
 7709   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 7710 
 7711   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
 7712             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
 7713   ins_encode %{
 7714     __ lock();
 7715     __ cmpxchgq($newval$$Register, $heap_top_ptr$$Address);
 7716   %}
 7717   ins_pipe(pipe_cmpxchg);
 7718 %}
 7719 
 7720 // Conditional-store of an int value.
 7721 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7722 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
 7723 %{
 7724   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 7725   effect(KILL oldval);
 7726 
 7727   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7728   opcode(0x0F, 0xB1);
 7729   ins_encode(lock_prefix,
 7730              REX_reg_mem(newval, mem),
 7731              OpcP, OpcS,
 7732              reg_mem(newval, mem));
 7733   ins_pipe(pipe_cmpxchg);
 7734 %}
 7735 
 7736 // Conditional-store of a long value.
 7737 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7738 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
 7739 %{
 7740   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 7741   effect(KILL oldval);
 7742 
 7743   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7744   ins_encode %{
 7745     __ lock();
 7746     __ cmpxchgq($newval$$Register, $mem$$Address);
 7747   %}
 7748   ins_pipe(pipe_cmpxchg);
 7749 %}
 7750 
 7751 
 7752 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7753 instruct compareAndSwapP(rRegI res,
 7754                          memory mem_ptr,
 7755                          rax_RegP oldval, rRegP newval,
 7756                          rFlagsReg cr)
 7757 %{
 7758   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7759   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7760   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7761   effect(KILL cr, KILL oldval);
 7762 
 7763   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7764             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7765             "sete    $res\n\t"
 7766             "movzbl  $res, $res" %}
 7767   ins_encode %{
 7768     __ lock();
 7769     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7770     __ sete($res$$Register);
 7771     __ movzbl($res$$Register, $res$$Register);
 7772   %}
 7773   ins_pipe( pipe_cmpxchg );
 7774 %}
 7775 
 7776 instruct compareAndSwapL(rRegI res,
 7777                          memory mem_ptr,
 7778                          rax_RegL oldval, rRegL newval,
 7779                          rFlagsReg cr)
 7780 %{
 7781   predicate(VM_Version::supports_cx8());
 7782   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7783   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7784   effect(KILL cr, KILL oldval);
 7785 
 7786   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7787             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7788             "sete    $res\n\t"
 7789             "movzbl  $res, $res" %}
 7790   ins_encode %{
 7791     __ lock();
 7792     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7793     __ sete($res$$Register);
 7794     __ movzbl($res$$Register, $res$$Register);
 7795   %}
 7796   ins_pipe( pipe_cmpxchg );
 7797 %}
 7798 
 7799 instruct compareAndSwapI(rRegI res,
 7800                          memory mem_ptr,
 7801                          rax_RegI oldval, rRegI newval,
 7802                          rFlagsReg cr)
 7803 %{
 7804   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7805   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7806   effect(KILL cr, KILL oldval);
 7807 
 7808   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7809             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7810             "sete    $res\n\t"
 7811             "movzbl  $res, $res" %}
 7812   ins_encode %{
 7813     __ lock();
 7814     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7815     __ sete($res$$Register);
 7816     __ movzbl($res$$Register, $res$$Register);
 7817   %}
 7818   ins_pipe( pipe_cmpxchg );
 7819 %}
 7820 
 7821 instruct compareAndSwapB(rRegI res,
 7822                          memory mem_ptr,
 7823                          rax_RegI oldval, rRegI newval,
 7824                          rFlagsReg cr)
 7825 %{
 7826   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7827   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7828   effect(KILL cr, KILL oldval);
 7829 
 7830   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7831             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7832             "sete    $res\n\t"
 7833             "movzbl  $res, $res" %}
 7834   ins_encode %{
 7835     __ lock();
 7836     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7837     __ sete($res$$Register);
 7838     __ movzbl($res$$Register, $res$$Register);
 7839   %}
 7840   ins_pipe( pipe_cmpxchg );
 7841 %}
 7842 
 7843 instruct compareAndSwapS(rRegI res,
 7844                          memory mem_ptr,
 7845                          rax_RegI oldval, rRegI newval,
 7846                          rFlagsReg cr)
 7847 %{
 7848   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7849   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7850   effect(KILL cr, KILL oldval);
 7851 
 7852   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7853             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7854             "sete    $res\n\t"
 7855             "movzbl  $res, $res" %}
 7856   ins_encode %{
 7857     __ lock();
 7858     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7859     __ sete($res$$Register);
 7860     __ movzbl($res$$Register, $res$$Register);
 7861   %}
 7862   ins_pipe( pipe_cmpxchg );
 7863 %}
 7864 
 7865 instruct compareAndSwapN(rRegI res,
 7866                           memory mem_ptr,
 7867                           rax_RegN oldval, rRegN newval,
 7868                           rFlagsReg cr) %{
 7869   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7870   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7871   effect(KILL cr, KILL oldval);
 7872 
 7873   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7874             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7875             "sete    $res\n\t"
 7876             "movzbl  $res, $res" %}
 7877   ins_encode %{
 7878     __ lock();
 7879     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7880     __ sete($res$$Register);
 7881     __ movzbl($res$$Register, $res$$Register);
 7882   %}
 7883   ins_pipe( pipe_cmpxchg );
 7884 %}
 7885 
 7886 instruct compareAndExchangeB(
 7887                          memory mem_ptr,
 7888                          rax_RegI oldval, rRegI newval,
 7889                          rFlagsReg cr)
 7890 %{
 7891   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7892   effect(KILL cr);
 7893 
 7894   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7895             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7896   ins_encode %{
 7897     __ lock();
 7898     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7899   %}
 7900   ins_pipe( pipe_cmpxchg );
 7901 %}
 7902 
 7903 instruct compareAndExchangeS(
 7904                          memory mem_ptr,
 7905                          rax_RegI oldval, rRegI newval,
 7906                          rFlagsReg cr)
 7907 %{
 7908   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7909   effect(KILL cr);
 7910 
 7911   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7912             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7913   ins_encode %{
 7914     __ lock();
 7915     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7916   %}
 7917   ins_pipe( pipe_cmpxchg );
 7918 %}
 7919 
 7920 instruct compareAndExchangeI(
 7921                          memory mem_ptr,
 7922                          rax_RegI oldval, rRegI newval,
 7923                          rFlagsReg cr)
 7924 %{
 7925   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7926   effect(KILL cr);
 7927 
 7928   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7929             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7930   ins_encode %{
 7931     __ lock();
 7932     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7933   %}
 7934   ins_pipe( pipe_cmpxchg );
 7935 %}
 7936 
 7937 instruct compareAndExchangeL(
 7938                          memory mem_ptr,
 7939                          rax_RegL oldval, rRegL newval,
 7940                          rFlagsReg cr)
 7941 %{
 7942   predicate(VM_Version::supports_cx8());
 7943   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7944   effect(KILL cr);
 7945 
 7946   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7947             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7948   ins_encode %{
 7949     __ lock();
 7950     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7951   %}
 7952   ins_pipe( pipe_cmpxchg );
 7953 %}
 7954 
 7955 instruct compareAndExchangeN(
 7956                           memory mem_ptr,
 7957                           rax_RegN oldval, rRegN newval,
 7958                           rFlagsReg cr) %{
 7959   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7960   effect(KILL cr);
 7961 
 7962   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7963             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7964   ins_encode %{
 7965     __ lock();
 7966     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7967   %}
 7968   ins_pipe( pipe_cmpxchg );
 7969 %}
 7970 
 7971 instruct compareAndExchangeP(
 7972                          memory mem_ptr,
 7973                          rax_RegP oldval, rRegP newval,
 7974                          rFlagsReg cr)
 7975 %{
 7976   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7977   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7978   effect(KILL cr);
 7979 
 7980   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7981             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7982   ins_encode %{
 7983     __ lock();
 7984     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7985   %}
 7986   ins_pipe( pipe_cmpxchg );
 7987 %}
 7988 
 7989 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7990   predicate(n->as_LoadStore()->result_not_used());
 7991   match(Set dummy (GetAndAddB mem add));
 7992   effect(KILL cr);
 7993   format %{ "ADDB  [$mem],$add" %}
 7994   ins_encode %{
 7995     __ lock();
 7996     __ addb($mem$$Address, $add$$constant);
 7997   %}
 7998   ins_pipe( pipe_cmpxchg );
 7999 %}
 8000 
 8001 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8002   match(Set newval (GetAndAddB mem newval));
 8003   effect(KILL cr);
 8004   format %{ "XADDB  [$mem],$newval" %}
 8005   ins_encode %{
 8006     __ lock();
 8007     __ xaddb($mem$$Address, $newval$$Register);
 8008   %}
 8009   ins_pipe( pipe_cmpxchg );
 8010 %}
 8011 
 8012 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8013   predicate(n->as_LoadStore()->result_not_used());
 8014   match(Set dummy (GetAndAddS mem add));
 8015   effect(KILL cr);
 8016   format %{ "ADDW  [$mem],$add" %}
 8017   ins_encode %{
 8018     __ lock();
 8019     __ addw($mem$$Address, $add$$constant);
 8020   %}
 8021   ins_pipe( pipe_cmpxchg );
 8022 %}
 8023 
 8024 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8025   match(Set newval (GetAndAddS mem newval));
 8026   effect(KILL cr);
 8027   format %{ "XADDW  [$mem],$newval" %}
 8028   ins_encode %{
 8029     __ lock();
 8030     __ xaddw($mem$$Address, $newval$$Register);
 8031   %}
 8032   ins_pipe( pipe_cmpxchg );
 8033 %}
 8034 
 8035 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8036   predicate(n->as_LoadStore()->result_not_used());
 8037   match(Set dummy (GetAndAddI mem add));
 8038   effect(KILL cr);
 8039   format %{ "ADDL  [$mem],$add" %}
 8040   ins_encode %{
 8041     __ lock();
 8042     __ addl($mem$$Address, $add$$constant);
 8043   %}
 8044   ins_pipe( pipe_cmpxchg );
 8045 %}
 8046 
 8047 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8048   match(Set newval (GetAndAddI mem newval));
 8049   effect(KILL cr);
 8050   format %{ "XADDL  [$mem],$newval" %}
 8051   ins_encode %{
 8052     __ lock();
 8053     __ xaddl($mem$$Address, $newval$$Register);
 8054   %}
 8055   ins_pipe( pipe_cmpxchg );
 8056 %}
 8057 
 8058 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8059   predicate(n->as_LoadStore()->result_not_used());
 8060   match(Set dummy (GetAndAddL mem add));
 8061   effect(KILL cr);
 8062   format %{ "ADDQ  [$mem],$add" %}
 8063   ins_encode %{
 8064     __ lock();
 8065     __ addq($mem$$Address, $add$$constant);
 8066   %}
 8067   ins_pipe( pipe_cmpxchg );
 8068 %}
 8069 
 8070 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8071   match(Set newval (GetAndAddL mem newval));
 8072   effect(KILL cr);
 8073   format %{ "XADDQ  [$mem],$newval" %}
 8074   ins_encode %{
 8075     __ lock();
 8076     __ xaddq($mem$$Address, $newval$$Register);
 8077   %}
 8078   ins_pipe( pipe_cmpxchg );
 8079 %}
 8080 
 8081 instruct xchgB( memory mem, rRegI newval) %{
 8082   match(Set newval (GetAndSetB mem newval));
 8083   format %{ "XCHGB  $newval,[$mem]" %}
 8084   ins_encode %{
 8085     __ xchgb($newval$$Register, $mem$$Address);
 8086   %}
 8087   ins_pipe( pipe_cmpxchg );
 8088 %}
 8089 
 8090 instruct xchgS( memory mem, rRegI newval) %{
 8091   match(Set newval (GetAndSetS mem newval));
 8092   format %{ "XCHGW  $newval,[$mem]" %}
 8093   ins_encode %{
 8094     __ xchgw($newval$$Register, $mem$$Address);
 8095   %}
 8096   ins_pipe( pipe_cmpxchg );
 8097 %}
 8098 
 8099 instruct xchgI( memory mem, rRegI newval) %{
 8100   match(Set newval (GetAndSetI mem newval));
 8101   format %{ "XCHGL  $newval,[$mem]" %}
 8102   ins_encode %{
 8103     __ xchgl($newval$$Register, $mem$$Address);
 8104   %}
 8105   ins_pipe( pipe_cmpxchg );
 8106 %}
 8107 
 8108 instruct xchgL( memory mem, rRegL newval) %{
 8109   match(Set newval (GetAndSetL mem newval));
 8110   format %{ "XCHGL  $newval,[$mem]" %}
 8111   ins_encode %{
 8112     __ xchgq($newval$$Register, $mem$$Address);
 8113   %}
 8114   ins_pipe( pipe_cmpxchg );
 8115 %}
 8116 
 8117 instruct xchgP( memory mem, rRegP newval) %{
 8118   match(Set newval (GetAndSetP mem newval));
 8119   predicate(n->as_LoadStore()->barrier_data() == 0);
 8120   format %{ "XCHGQ  $newval,[$mem]" %}
 8121   ins_encode %{
 8122     __ xchgq($newval$$Register, $mem$$Address);
 8123   %}
 8124   ins_pipe( pipe_cmpxchg );
 8125 %}
 8126 
 8127 instruct xchgN( memory mem, rRegN newval) %{
 8128   match(Set newval (GetAndSetN mem newval));
 8129   format %{ "XCHGL  $newval,$mem]" %}
 8130   ins_encode %{
 8131     __ xchgl($newval$$Register, $mem$$Address);
 8132   %}
 8133   ins_pipe( pipe_cmpxchg );
 8134 %}
 8135 
 8136 //----------Abs Instructions-------------------------------------------
 8137 
 8138 // Integer Absolute Instructions
 8139 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8140 %{
 8141   match(Set dst (AbsI src));
 8142   effect(TEMP dst, TEMP tmp, KILL cr);
 8143   format %{ "movl $tmp, $src\n\t"
 8144             "sarl $tmp, 31\n\t"
 8145             "movl $dst, $src\n\t"
 8146             "xorl $dst, $tmp\n\t"
 8147             "subl $dst, $tmp\n"
 8148           %}
 8149   ins_encode %{
 8150     __ movl($tmp$$Register, $src$$Register);
 8151     __ sarl($tmp$$Register, 31);
 8152     __ movl($dst$$Register, $src$$Register);
 8153     __ xorl($dst$$Register, $tmp$$Register);
 8154     __ subl($dst$$Register, $tmp$$Register);
 8155   %}
 8156 
 8157   ins_pipe(ialu_reg_reg);
 8158 %}
 8159 
 8160 // Long Absolute Instructions
 8161 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8162 %{
 8163   match(Set dst (AbsL src));
 8164   effect(TEMP dst, TEMP tmp, KILL cr);
 8165   format %{ "movq $tmp, $src\n\t"
 8166             "sarq $tmp, 63\n\t"
 8167             "movq $dst, $src\n\t"
 8168             "xorq $dst, $tmp\n\t"
 8169             "subq $dst, $tmp\n"
 8170           %}
 8171   ins_encode %{
 8172     __ movq($tmp$$Register, $src$$Register);
 8173     __ sarq($tmp$$Register, 63);
 8174     __ movq($dst$$Register, $src$$Register);
 8175     __ xorq($dst$$Register, $tmp$$Register);
 8176     __ subq($dst$$Register, $tmp$$Register);
 8177   %}
 8178 
 8179   ins_pipe(ialu_reg_reg);
 8180 %}
 8181 
 8182 //----------Subtraction Instructions-------------------------------------------
 8183 
 8184 // Integer Subtraction Instructions
 8185 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8186 %{
 8187   match(Set dst (SubI dst src));
 8188   effect(KILL cr);
 8189 
 8190   format %{ "subl    $dst, $src\t# int" %}
 8191   ins_encode %{
 8192     __ subl($dst$$Register, $src$$Register);
 8193   %}
 8194   ins_pipe(ialu_reg_reg);
 8195 %}
 8196 
 8197 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8198 %{
 8199   match(Set dst (SubI dst src));
 8200   effect(KILL cr);
 8201 
 8202   format %{ "subl    $dst, $src\t# int" %}
 8203   ins_encode %{
 8204     __ subl($dst$$Register, $src$$constant);
 8205   %}
 8206   ins_pipe(ialu_reg);
 8207 %}
 8208 
 8209 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8210 %{
 8211   match(Set dst (SubI dst (LoadI src)));
 8212   effect(KILL cr);
 8213 
 8214   ins_cost(125);
 8215   format %{ "subl    $dst, $src\t# int" %}
 8216   ins_encode %{
 8217     __ subl($dst$$Register, $src$$Address);
 8218   %}
 8219   ins_pipe(ialu_reg_mem);
 8220 %}
 8221 
 8222 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8223 %{
 8224   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8225   effect(KILL cr);
 8226 
 8227   ins_cost(150);
 8228   format %{ "subl    $dst, $src\t# int" %}
 8229   ins_encode %{
 8230     __ subl($dst$$Address, $src$$Register);
 8231   %}
 8232   ins_pipe(ialu_mem_reg);
 8233 %}
 8234 
 8235 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8236 %{
 8237   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8238   effect(KILL cr);
 8239 
 8240   ins_cost(125); // XXX
 8241   format %{ "subl    $dst, $src\t# int" %}
 8242   ins_encode %{
 8243     __ subl($dst$$Address, $src$$constant);
 8244   %}
 8245   ins_pipe(ialu_mem_imm);
 8246 %}
 8247 
 8248 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8249 %{
 8250   match(Set dst (SubL dst src));
 8251   effect(KILL cr);
 8252 
 8253   format %{ "subq    $dst, $src\t# long" %}
 8254   ins_encode %{
 8255     __ subq($dst$$Register, $src$$Register);
 8256   %}
 8257   ins_pipe(ialu_reg_reg);
 8258 %}
 8259 
 8260 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8261 %{
 8262   match(Set dst (SubL dst src));
 8263   effect(KILL cr);
 8264 
 8265   format %{ "subq    $dst, $src\t# long" %}
 8266   ins_encode %{
 8267     __ subq($dst$$Register, $src$$constant);
 8268   %}
 8269   ins_pipe(ialu_reg);
 8270 %}
 8271 
 8272 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8273 %{
 8274   match(Set dst (SubL dst (LoadL src)));
 8275   effect(KILL cr);
 8276 
 8277   ins_cost(125);
 8278   format %{ "subq    $dst, $src\t# long" %}
 8279   ins_encode %{
 8280     __ subq($dst$$Register, $src$$Address);
 8281   %}
 8282   ins_pipe(ialu_reg_mem);
 8283 %}
 8284 
 8285 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8286 %{
 8287   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8288   effect(KILL cr);
 8289 
 8290   ins_cost(150);
 8291   format %{ "subq    $dst, $src\t# long" %}
 8292   ins_encode %{
 8293     __ subq($dst$$Address, $src$$Register);
 8294   %}
 8295   ins_pipe(ialu_mem_reg);
 8296 %}
 8297 
 8298 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8299 %{
 8300   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8301   effect(KILL cr);
 8302 
 8303   ins_cost(125); // XXX
 8304   format %{ "subq    $dst, $src\t# long" %}
 8305   ins_encode %{
 8306     __ subq($dst$$Address, $src$$constant);
 8307   %}
 8308   ins_pipe(ialu_mem_imm);
 8309 %}
 8310 
 8311 // Subtract from a pointer
 8312 // XXX hmpf???
 8313 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8314 %{
 8315   match(Set dst (AddP dst (SubI zero src)));
 8316   effect(KILL cr);
 8317 
 8318   format %{ "subq    $dst, $src\t# ptr - int" %}
 8319   opcode(0x2B);
 8320   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8321   ins_pipe(ialu_reg_reg);
 8322 %}
 8323 
 8324 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8325 %{
 8326   match(Set dst (SubI zero dst));
 8327   effect(KILL cr);
 8328 
 8329   format %{ "negl    $dst\t# int" %}
 8330   ins_encode %{
 8331     __ negl($dst$$Register);
 8332   %}
 8333   ins_pipe(ialu_reg);
 8334 %}
 8335 
 8336 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8337 %{
 8338   match(Set dst (NegI dst));
 8339   effect(KILL cr);
 8340 
 8341   format %{ "negl    $dst\t# int" %}
 8342   ins_encode %{
 8343     __ negl($dst$$Register);
 8344   %}
 8345   ins_pipe(ialu_reg);
 8346 %}
 8347 
 8348 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8349 %{
 8350   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8351   effect(KILL cr);
 8352 
 8353   format %{ "negl    $dst\t# int" %}
 8354   ins_encode %{
 8355     __ negl($dst$$Address);
 8356   %}
 8357   ins_pipe(ialu_reg);
 8358 %}
 8359 
 8360 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8361 %{
 8362   match(Set dst (SubL zero dst));
 8363   effect(KILL cr);
 8364 
 8365   format %{ "negq    $dst\t# long" %}
 8366   ins_encode %{
 8367     __ negq($dst$$Register);
 8368   %}
 8369   ins_pipe(ialu_reg);
 8370 %}
 8371 
 8372 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8373 %{
 8374   match(Set dst (NegL dst));
 8375   effect(KILL cr);
 8376 
 8377   format %{ "negq    $dst\t# int" %}
 8378   ins_encode %{
 8379     __ negq($dst$$Register);
 8380   %}
 8381   ins_pipe(ialu_reg);
 8382 %}
 8383 
 8384 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8385 %{
 8386   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8387   effect(KILL cr);
 8388 
 8389   format %{ "negq    $dst\t# long" %}
 8390   ins_encode %{
 8391     __ negq($dst$$Address);
 8392   %}
 8393   ins_pipe(ialu_reg);
 8394 %}
 8395 
 8396 //----------Multiplication/Division Instructions-------------------------------
 8397 // Integer Multiplication Instructions
 8398 // Multiply Register
 8399 
 8400 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8401 %{
 8402   match(Set dst (MulI dst src));
 8403   effect(KILL cr);
 8404 
 8405   ins_cost(300);
 8406   format %{ "imull   $dst, $src\t# int" %}
 8407   ins_encode %{
 8408     __ imull($dst$$Register, $src$$Register);
 8409   %}
 8410   ins_pipe(ialu_reg_reg_alu0);
 8411 %}
 8412 
 8413 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8414 %{
 8415   match(Set dst (MulI src imm));
 8416   effect(KILL cr);
 8417 
 8418   ins_cost(300);
 8419   format %{ "imull   $dst, $src, $imm\t# int" %}
 8420   ins_encode %{
 8421     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8422   %}
 8423   ins_pipe(ialu_reg_reg_alu0);
 8424 %}
 8425 
 8426 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8427 %{
 8428   match(Set dst (MulI dst (LoadI src)));
 8429   effect(KILL cr);
 8430 
 8431   ins_cost(350);
 8432   format %{ "imull   $dst, $src\t# int" %}
 8433   ins_encode %{
 8434     __ imull($dst$$Register, $src$$Address);
 8435   %}
 8436   ins_pipe(ialu_reg_mem_alu0);
 8437 %}
 8438 
 8439 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8440 %{
 8441   match(Set dst (MulI (LoadI src) imm));
 8442   effect(KILL cr);
 8443 
 8444   ins_cost(300);
 8445   format %{ "imull   $dst, $src, $imm\t# int" %}
 8446   ins_encode %{
 8447     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8448   %}
 8449   ins_pipe(ialu_reg_mem_alu0);
 8450 %}
 8451 
 8452 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8453 %{
 8454   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8455   effect(KILL cr, KILL src2);
 8456 
 8457   expand %{ mulI_rReg(dst, src1, cr);
 8458            mulI_rReg(src2, src3, cr);
 8459            addI_rReg(dst, src2, cr); %}
 8460 %}
 8461 
 8462 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8463 %{
 8464   match(Set dst (MulL dst src));
 8465   effect(KILL cr);
 8466 
 8467   ins_cost(300);
 8468   format %{ "imulq   $dst, $src\t# long" %}
 8469   ins_encode %{
 8470     __ imulq($dst$$Register, $src$$Register);
 8471   %}
 8472   ins_pipe(ialu_reg_reg_alu0);
 8473 %}
 8474 
 8475 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8476 %{
 8477   match(Set dst (MulL src imm));
 8478   effect(KILL cr);
 8479 
 8480   ins_cost(300);
 8481   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8482   ins_encode %{
 8483     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8484   %}
 8485   ins_pipe(ialu_reg_reg_alu0);
 8486 %}
 8487 
 8488 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8489 %{
 8490   match(Set dst (MulL dst (LoadL src)));
 8491   effect(KILL cr);
 8492 
 8493   ins_cost(350);
 8494   format %{ "imulq   $dst, $src\t# long" %}
 8495   ins_encode %{
 8496     __ imulq($dst$$Register, $src$$Address);
 8497   %}
 8498   ins_pipe(ialu_reg_mem_alu0);
 8499 %}
 8500 
 8501 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8502 %{
 8503   match(Set dst (MulL (LoadL src) imm));
 8504   effect(KILL cr);
 8505 
 8506   ins_cost(300);
 8507   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8508   ins_encode %{
 8509     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8510   %}
 8511   ins_pipe(ialu_reg_mem_alu0);
 8512 %}
 8513 
 8514 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8515 %{
 8516   match(Set dst (MulHiL src rax));
 8517   effect(USE_KILL rax, KILL cr);
 8518 
 8519   ins_cost(300);
 8520   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8521   ins_encode %{
 8522     __ imulq($src$$Register);
 8523   %}
 8524   ins_pipe(ialu_reg_reg_alu0);
 8525 %}
 8526 
 8527 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8528                    rFlagsReg cr)
 8529 %{
 8530   match(Set rax (DivI rax div));
 8531   effect(KILL rdx, KILL cr);
 8532 
 8533   ins_cost(30*100+10*100); // XXX
 8534   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8535             "jne,s   normal\n\t"
 8536             "xorl    rdx, rdx\n\t"
 8537             "cmpl    $div, -1\n\t"
 8538             "je,s    done\n"
 8539     "normal: cdql\n\t"
 8540             "idivl   $div\n"
 8541     "done:"        %}
 8542   ins_encode(cdql_enc(div));
 8543   ins_pipe(ialu_reg_reg_alu0);
 8544 %}
 8545 
 8546 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8547                    rFlagsReg cr)
 8548 %{
 8549   match(Set rax (DivL rax div));
 8550   effect(KILL rdx, KILL cr);
 8551 
 8552   ins_cost(30*100+10*100); // XXX
 8553   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8554             "cmpq    rax, rdx\n\t"
 8555             "jne,s   normal\n\t"
 8556             "xorl    rdx, rdx\n\t"
 8557             "cmpq    $div, -1\n\t"
 8558             "je,s    done\n"
 8559     "normal: cdqq\n\t"
 8560             "idivq   $div\n"
 8561     "done:"        %}
 8562   ins_encode(cdqq_enc(div));
 8563   ins_pipe(ialu_reg_reg_alu0);
 8564 %}
 8565 
 8566 // Integer DIVMOD with Register, both quotient and mod results
 8567 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8568                              rFlagsReg cr)
 8569 %{
 8570   match(DivModI rax div);
 8571   effect(KILL cr);
 8572 
 8573   ins_cost(30*100+10*100); // XXX
 8574   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8575             "jne,s   normal\n\t"
 8576             "xorl    rdx, rdx\n\t"
 8577             "cmpl    $div, -1\n\t"
 8578             "je,s    done\n"
 8579     "normal: cdql\n\t"
 8580             "idivl   $div\n"
 8581     "done:"        %}
 8582   ins_encode(cdql_enc(div));
 8583   ins_pipe(pipe_slow);
 8584 %}
 8585 
 8586 // Long DIVMOD with Register, both quotient and mod results
 8587 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8588                              rFlagsReg cr)
 8589 %{
 8590   match(DivModL rax div);
 8591   effect(KILL cr);
 8592 
 8593   ins_cost(30*100+10*100); // XXX
 8594   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8595             "cmpq    rax, rdx\n\t"
 8596             "jne,s   normal\n\t"
 8597             "xorl    rdx, rdx\n\t"
 8598             "cmpq    $div, -1\n\t"
 8599             "je,s    done\n"
 8600     "normal: cdqq\n\t"
 8601             "idivq   $div\n"
 8602     "done:"        %}
 8603   ins_encode(cdqq_enc(div));
 8604   ins_pipe(pipe_slow);
 8605 %}
 8606 
 8607 //----------- DivL-By-Constant-Expansions--------------------------------------
 8608 // DivI cases are handled by the compiler
 8609 
 8610 // Magic constant, reciprocal of 10
 8611 instruct loadConL_0x6666666666666667(rRegL dst)
 8612 %{
 8613   effect(DEF dst);
 8614 
 8615   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
 8616   ins_encode(load_immL(dst, 0x6666666666666667));
 8617   ins_pipe(ialu_reg);
 8618 %}
 8619 
 8620 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8621 %{
 8622   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
 8623 
 8624   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
 8625   ins_encode %{
 8626     __ imulq($src$$Register);
 8627   %}
 8628   ins_pipe(ialu_reg_reg_alu0);
 8629 %}
 8630 
 8631 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
 8632 %{
 8633   effect(USE_DEF dst, KILL cr);
 8634 
 8635   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
 8636   ins_encode %{
 8637     __ sarq($dst$$Register, 63);
 8638   %}
 8639   ins_pipe(ialu_reg);
 8640 %}
 8641 
 8642 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
 8643 %{
 8644   effect(USE_DEF dst, KILL cr);
 8645 
 8646   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
 8647   ins_encode %{
 8648     __ sarq($dst$$Register, 2);
 8649   %}
 8650   ins_pipe(ialu_reg);
 8651 %}
 8652 
 8653 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
 8654 %{
 8655   match(Set dst (DivL src div));
 8656 
 8657   ins_cost((5+8)*100);
 8658   expand %{
 8659     rax_RegL rax;                     // Killed temp
 8660     rFlagsReg cr;                     // Killed
 8661     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
 8662     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
 8663     sarL_rReg_63(src, cr);            // sarq  src, 63
 8664     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
 8665     subL_rReg(dst, src, cr);          // subl  rdx, src
 8666   %}
 8667 %}
 8668 
 8669 //-----------------------------------------------------------------------------
 8670 
 8671 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8672                    rFlagsReg cr)
 8673 %{
 8674   match(Set rdx (ModI rax div));
 8675   effect(KILL rax, KILL cr);
 8676 
 8677   ins_cost(300); // XXX
 8678   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8679             "jne,s   normal\n\t"
 8680             "xorl    rdx, rdx\n\t"
 8681             "cmpl    $div, -1\n\t"
 8682             "je,s    done\n"
 8683     "normal: cdql\n\t"
 8684             "idivl   $div\n"
 8685     "done:"        %}
 8686   ins_encode(cdql_enc(div));
 8687   ins_pipe(ialu_reg_reg_alu0);
 8688 %}
 8689 
 8690 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8691                    rFlagsReg cr)
 8692 %{
 8693   match(Set rdx (ModL rax div));
 8694   effect(KILL rax, KILL cr);
 8695 
 8696   ins_cost(300); // XXX
 8697   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8698             "cmpq    rax, rdx\n\t"
 8699             "jne,s   normal\n\t"
 8700             "xorl    rdx, rdx\n\t"
 8701             "cmpq    $div, -1\n\t"
 8702             "je,s    done\n"
 8703     "normal: cdqq\n\t"
 8704             "idivq   $div\n"
 8705     "done:"        %}
 8706   ins_encode(cdqq_enc(div));
 8707   ins_pipe(ialu_reg_reg_alu0);
 8708 %}
 8709 
 8710 // Integer Shift Instructions
 8711 // Shift Left by one
 8712 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8713 %{
 8714   match(Set dst (LShiftI dst shift));
 8715   effect(KILL cr);
 8716 
 8717   format %{ "sall    $dst, $shift" %}
 8718   ins_encode %{
 8719     __ sall($dst$$Register, $shift$$constant);
 8720   %}
 8721   ins_pipe(ialu_reg);
 8722 %}
 8723 
 8724 // Shift Left by one
 8725 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8726 %{
 8727   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8728   effect(KILL cr);
 8729 
 8730   format %{ "sall    $dst, $shift\t" %}
 8731   ins_encode %{
 8732     __ sall($dst$$Address, $shift$$constant);
 8733   %}
 8734   ins_pipe(ialu_mem_imm);
 8735 %}
 8736 
 8737 // Shift Left by 8-bit immediate
 8738 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8739 %{
 8740   match(Set dst (LShiftI dst shift));
 8741   effect(KILL cr);
 8742 
 8743   format %{ "sall    $dst, $shift" %}
 8744   ins_encode %{
 8745     __ sall($dst$$Register, $shift$$constant);
 8746   %}
 8747   ins_pipe(ialu_reg);
 8748 %}
 8749 
 8750 // Shift Left by 8-bit immediate
 8751 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8752 %{
 8753   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8754   effect(KILL cr);
 8755 
 8756   format %{ "sall    $dst, $shift" %}
 8757   ins_encode %{
 8758     __ sall($dst$$Address, $shift$$constant);
 8759   %}
 8760   ins_pipe(ialu_mem_imm);
 8761 %}
 8762 
 8763 // Shift Left by variable
 8764 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8765 %{
 8766   match(Set dst (LShiftI dst shift));
 8767   effect(KILL cr);
 8768 
 8769   format %{ "sall    $dst, $shift" %}
 8770   ins_encode %{
 8771     __ sall($dst$$Register);
 8772   %}
 8773   ins_pipe(ialu_reg_reg);
 8774 %}
 8775 
 8776 // Shift Left by variable
 8777 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8778 %{
 8779   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8780   effect(KILL cr);
 8781 
 8782   format %{ "sall    $dst, $shift" %}
 8783   ins_encode %{
 8784     __ sall($dst$$Address);
 8785   %}
 8786   ins_pipe(ialu_mem_reg);
 8787 %}
 8788 
 8789 // Arithmetic shift right by one
 8790 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8791 %{
 8792   match(Set dst (RShiftI dst shift));
 8793   effect(KILL cr);
 8794 
 8795   format %{ "sarl    $dst, $shift" %}
 8796   ins_encode %{
 8797     __ sarl($dst$$Register, $shift$$constant);
 8798   %}
 8799   ins_pipe(ialu_reg);
 8800 %}
 8801 
 8802 // Arithmetic shift right by one
 8803 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8804 %{
 8805   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8806   effect(KILL cr);
 8807 
 8808   format %{ "sarl    $dst, $shift" %}
 8809   ins_encode %{
 8810     __ sarl($dst$$Address, $shift$$constant);
 8811   %}
 8812   ins_pipe(ialu_mem_imm);
 8813 %}
 8814 
 8815 // Arithmetic Shift Right by 8-bit immediate
 8816 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8817 %{
 8818   match(Set dst (RShiftI dst shift));
 8819   effect(KILL cr);
 8820 
 8821   format %{ "sarl    $dst, $shift" %}
 8822   ins_encode %{
 8823     __ sarl($dst$$Register, $shift$$constant);
 8824   %}
 8825   ins_pipe(ialu_mem_imm);
 8826 %}
 8827 
 8828 // Arithmetic Shift Right by 8-bit immediate
 8829 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8830 %{
 8831   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8832   effect(KILL cr);
 8833 
 8834   format %{ "sarl    $dst, $shift" %}
 8835   ins_encode %{
 8836     __ sarl($dst$$Address, $shift$$constant);
 8837   %}
 8838   ins_pipe(ialu_mem_imm);
 8839 %}
 8840 
 8841 // Arithmetic Shift Right by variable
 8842 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8843 %{
 8844   match(Set dst (RShiftI dst shift));
 8845   effect(KILL cr);
 8846   format %{ "sarl    $dst, $shift" %}
 8847   ins_encode %{
 8848     __ sarl($dst$$Register);
 8849   %}
 8850   ins_pipe(ialu_reg_reg);
 8851 %}
 8852 
 8853 // Arithmetic Shift Right by variable
 8854 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8855 %{
 8856   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8857   effect(KILL cr);
 8858 
 8859   format %{ "sarl    $dst, $shift" %}
 8860   ins_encode %{
 8861     __ sarl($dst$$Address);
 8862   %}
 8863   ins_pipe(ialu_mem_reg);
 8864 %}
 8865 
 8866 // Logical shift right by one
 8867 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8868 %{
 8869   match(Set dst (URShiftI dst shift));
 8870   effect(KILL cr);
 8871 
 8872   format %{ "shrl    $dst, $shift" %}
 8873   ins_encode %{
 8874     __ shrl($dst$$Register, $shift$$constant);
 8875   %}
 8876   ins_pipe(ialu_reg);
 8877 %}
 8878 
 8879 // Logical shift right by one
 8880 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8881 %{
 8882   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8883   effect(KILL cr);
 8884 
 8885   format %{ "shrl    $dst, $shift" %}
 8886   ins_encode %{
 8887     __ shrl($dst$$Address, $shift$$constant);
 8888   %}
 8889   ins_pipe(ialu_mem_imm);
 8890 %}
 8891 
 8892 // Logical Shift Right by 8-bit immediate
 8893 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8894 %{
 8895   match(Set dst (URShiftI dst shift));
 8896   effect(KILL cr);
 8897 
 8898   format %{ "shrl    $dst, $shift" %}
 8899   ins_encode %{
 8900     __ shrl($dst$$Register, $shift$$constant);
 8901   %}
 8902   ins_pipe(ialu_reg);
 8903 %}
 8904 
 8905 // Logical Shift Right by 8-bit immediate
 8906 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8907 %{
 8908   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8909   effect(KILL cr);
 8910 
 8911   format %{ "shrl    $dst, $shift" %}
 8912   ins_encode %{
 8913     __ shrl($dst$$Address, $shift$$constant);
 8914   %}
 8915   ins_pipe(ialu_mem_imm);
 8916 %}
 8917 
 8918 // Logical Shift Right by variable
 8919 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8920 %{
 8921   match(Set dst (URShiftI dst shift));
 8922   effect(KILL cr);
 8923 
 8924   format %{ "shrl    $dst, $shift" %}
 8925   ins_encode %{
 8926     __ shrl($dst$$Register);
 8927   %}
 8928   ins_pipe(ialu_reg_reg);
 8929 %}
 8930 
 8931 // Logical Shift Right by variable
 8932 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8933 %{
 8934   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8935   effect(KILL cr);
 8936 
 8937   format %{ "shrl    $dst, $shift" %}
 8938   ins_encode %{
 8939     __ shrl($dst$$Address);
 8940   %}
 8941   ins_pipe(ialu_mem_reg);
 8942 %}
 8943 
 8944 // Long Shift Instructions
 8945 // Shift Left by one
 8946 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 8947 %{
 8948   match(Set dst (LShiftL dst shift));
 8949   effect(KILL cr);
 8950 
 8951   format %{ "salq    $dst, $shift" %}
 8952   ins_encode %{
 8953     __ salq($dst$$Register, $shift$$constant);
 8954   %}
 8955   ins_pipe(ialu_reg);
 8956 %}
 8957 
 8958 // Shift Left by one
 8959 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8960 %{
 8961   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8962   effect(KILL cr);
 8963 
 8964   format %{ "salq    $dst, $shift" %}
 8965   ins_encode %{
 8966     __ salq($dst$$Address, $shift$$constant);
 8967   %}
 8968   ins_pipe(ialu_mem_imm);
 8969 %}
 8970 
 8971 // Shift Left by 8-bit immediate
 8972 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8973 %{
 8974   match(Set dst (LShiftL dst shift));
 8975   effect(KILL cr);
 8976 
 8977   format %{ "salq    $dst, $shift" %}
 8978   ins_encode %{
 8979     __ salq($dst$$Register, $shift$$constant);
 8980   %}
 8981   ins_pipe(ialu_reg);
 8982 %}
 8983 
 8984 // Shift Left by 8-bit immediate
 8985 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8986 %{
 8987   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8988   effect(KILL cr);
 8989 
 8990   format %{ "salq    $dst, $shift" %}
 8991   ins_encode %{
 8992     __ salq($dst$$Address, $shift$$constant);
 8993   %}
 8994   ins_pipe(ialu_mem_imm);
 8995 %}
 8996 
 8997 // Shift Left by variable
 8998 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8999 %{
 9000   match(Set dst (LShiftL dst shift));
 9001   effect(KILL cr);
 9002 
 9003   format %{ "salq    $dst, $shift" %}
 9004   ins_encode %{
 9005     __ salq($dst$$Register);
 9006   %}
 9007   ins_pipe(ialu_reg_reg);
 9008 %}
 9009 
 9010 // Shift Left by variable
 9011 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9012 %{
 9013   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9014   effect(KILL cr);
 9015 
 9016   format %{ "salq    $dst, $shift" %}
 9017   ins_encode %{
 9018     __ salq($dst$$Address);
 9019   %}
 9020   ins_pipe(ialu_mem_reg);
 9021 %}
 9022 
 9023 // Arithmetic shift right by one
 9024 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9025 %{
 9026   match(Set dst (RShiftL dst shift));
 9027   effect(KILL cr);
 9028 
 9029   format %{ "sarq    $dst, $shift" %}
 9030   ins_encode %{
 9031     __ sarq($dst$$Register, $shift$$constant);
 9032   %}
 9033   ins_pipe(ialu_reg);
 9034 %}
 9035 
 9036 // Arithmetic shift right by one
 9037 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9038 %{
 9039   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9040   effect(KILL cr);
 9041 
 9042   format %{ "sarq    $dst, $shift" %}
 9043   ins_encode %{
 9044     __ sarq($dst$$Address, $shift$$constant);
 9045   %}
 9046   ins_pipe(ialu_mem_imm);
 9047 %}
 9048 
 9049 // Arithmetic Shift Right by 8-bit immediate
 9050 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9051 %{
 9052   match(Set dst (RShiftL dst shift));
 9053   effect(KILL cr);
 9054 
 9055   format %{ "sarq    $dst, $shift" %}
 9056   ins_encode %{
 9057     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9058   %}
 9059   ins_pipe(ialu_mem_imm);
 9060 %}
 9061 
 9062 // Arithmetic Shift Right by 8-bit immediate
 9063 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9064 %{
 9065   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9066   effect(KILL cr);
 9067 
 9068   format %{ "sarq    $dst, $shift" %}
 9069   ins_encode %{
 9070     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9071   %}
 9072   ins_pipe(ialu_mem_imm);
 9073 %}
 9074 
 9075 // Arithmetic Shift Right by variable
 9076 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9077 %{
 9078   match(Set dst (RShiftL dst shift));
 9079   effect(KILL cr);
 9080 
 9081   format %{ "sarq    $dst, $shift" %}
 9082   ins_encode %{
 9083     __ sarq($dst$$Register);
 9084   %}
 9085   ins_pipe(ialu_reg_reg);
 9086 %}
 9087 
 9088 // Arithmetic Shift Right by variable
 9089 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9090 %{
 9091   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9092   effect(KILL cr);
 9093 
 9094   format %{ "sarq    $dst, $shift" %}
 9095   ins_encode %{
 9096     __ sarq($dst$$Address);
 9097   %}
 9098   ins_pipe(ialu_mem_reg);
 9099 %}
 9100 
 9101 // Logical shift right by one
 9102 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9103 %{
 9104   match(Set dst (URShiftL dst shift));
 9105   effect(KILL cr);
 9106 
 9107   format %{ "shrq    $dst, $shift" %}
 9108   ins_encode %{
 9109     __ shrq($dst$$Register, $shift$$constant);
 9110   %}
 9111   ins_pipe(ialu_reg);
 9112 %}
 9113 
 9114 // Logical shift right by one
 9115 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9116 %{
 9117   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9118   effect(KILL cr);
 9119 
 9120   format %{ "shrq    $dst, $shift" %}
 9121   ins_encode %{
 9122     __ shrq($dst$$Address, $shift$$constant);
 9123   %}
 9124   ins_pipe(ialu_mem_imm);
 9125 %}
 9126 
 9127 // Logical Shift Right by 8-bit immediate
 9128 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9129 %{
 9130   match(Set dst (URShiftL dst shift));
 9131   effect(KILL cr);
 9132 
 9133   format %{ "shrq    $dst, $shift" %}
 9134   ins_encode %{
 9135     __ shrq($dst$$Register, $shift$$constant);
 9136   %}
 9137   ins_pipe(ialu_reg);
 9138 %}
 9139 
 9140 // Logical Shift Right by 8-bit immediate
 9141 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9142 %{
 9143   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9144   effect(KILL cr);
 9145 
 9146   format %{ "shrq    $dst, $shift" %}
 9147   ins_encode %{
 9148     __ shrq($dst$$Address, $shift$$constant);
 9149   %}
 9150   ins_pipe(ialu_mem_imm);
 9151 %}
 9152 
 9153 // Logical Shift Right by variable
 9154 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9155 %{
 9156   match(Set dst (URShiftL dst shift));
 9157   effect(KILL cr);
 9158 
 9159   format %{ "shrq    $dst, $shift" %}
 9160   ins_encode %{
 9161     __ shrq($dst$$Register);
 9162   %}
 9163   ins_pipe(ialu_reg_reg);
 9164 %}
 9165 
 9166 // Logical Shift Right by variable
 9167 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9168 %{
 9169   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9170   effect(KILL cr);
 9171 
 9172   format %{ "shrq    $dst, $shift" %}
 9173   ins_encode %{
 9174     __ shrq($dst$$Address);
 9175   %}
 9176   ins_pipe(ialu_mem_reg);
 9177 %}
 9178 
 9179 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9180 // This idiom is used by the compiler for the i2b bytecode.
 9181 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9182 %{
 9183   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9184 
 9185   format %{ "movsbl  $dst, $src\t# i2b" %}
 9186   ins_encode %{
 9187     __ movsbl($dst$$Register, $src$$Register);
 9188   %}
 9189   ins_pipe(ialu_reg_reg);
 9190 %}
 9191 
 9192 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9193 // This idiom is used by the compiler the i2s bytecode.
 9194 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9195 %{
 9196   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9197 
 9198   format %{ "movswl  $dst, $src\t# i2s" %}
 9199   ins_encode %{
 9200     __ movswl($dst$$Register, $src$$Register);
 9201   %}
 9202   ins_pipe(ialu_reg_reg);
 9203 %}
 9204 
 9205 // ROL/ROR instructions
 9206 
 9207 // Rotate left by constant.
 9208 instruct rolI_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9209 %{
 9210   predicate(n->bottom_type()->basic_type() == T_INT);
 9211   match(Set dst (RotateLeft dst shift));
 9212   effect(KILL cr);
 9213   format %{ "roll    $dst, $shift" %}
 9214   ins_encode %{
 9215     __ roll($dst$$Register, $shift$$constant);
 9216   %}
 9217   ins_pipe(ialu_reg);
 9218 %}
 9219 
 9220 // Rotate Left by variable
 9221 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9222 %{
 9223   predicate(n->bottom_type()->basic_type() == T_INT);
 9224   match(Set dst (RotateLeft dst shift));
 9225   effect(KILL cr);
 9226   format %{ "roll    $dst, $shift" %}
 9227   ins_encode %{
 9228     __ roll($dst$$Register);
 9229   %}
 9230   ins_pipe(ialu_reg_reg);
 9231 %}
 9232 
 9233 // Rotate Right by constant.
 9234 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9235 %{
 9236   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9237   match(Set dst (RotateRight dst shift));
 9238   effect(KILL cr);
 9239   format %{ "rorl    $dst, $shift" %}
 9240   ins_encode %{
 9241     __ rorl($dst$$Register, $shift$$constant);
 9242   %}
 9243   ins_pipe(ialu_reg);
 9244 %}
 9245 
 9246 // Rotate Right by constant.
 9247 instruct rorI_immI8(rRegI dst, immI8 shift)
 9248 %{
 9249   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9250   match(Set dst (RotateRight dst shift));
 9251   format %{ "rorxd     $dst, $shift" %}
 9252   ins_encode %{
 9253     __ rorxd($dst$$Register, $dst$$Register, $shift$$constant);
 9254   %}
 9255   ins_pipe(ialu_reg_reg);
 9256 %}
 9257 
 9258 // Rotate Right by variable
 9259 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9260 %{
 9261   predicate(n->bottom_type()->basic_type() == T_INT);
 9262   match(Set dst (RotateRight dst shift));
 9263   effect(KILL cr);
 9264   format %{ "rorl    $dst, $shift" %}
 9265   ins_encode %{
 9266     __ rorl($dst$$Register);
 9267   %}
 9268   ins_pipe(ialu_reg_reg);
 9269 %}
 9270 
 9271 
 9272 // Rotate Left by constant.
 9273 instruct rolL_immI8(rRegL dst, immI8 shift, rFlagsReg cr)
 9274 %{
 9275   predicate(n->bottom_type()->basic_type() == T_LONG);
 9276   match(Set dst (RotateLeft dst shift));
 9277   effect(KILL cr);
 9278   format %{ "rolq    $dst, $shift" %}
 9279   ins_encode %{
 9280     __ rolq($dst$$Register, $shift$$constant);
 9281   %}
 9282   ins_pipe(ialu_reg);
 9283 %}
 9284 
 9285 // Rotate Left by variable
 9286 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9287 %{
 9288   predicate(n->bottom_type()->basic_type() == T_LONG);
 9289   match(Set dst (RotateLeft dst shift));
 9290   effect(KILL cr);
 9291   format %{ "rolq    $dst, $shift" %}
 9292   ins_encode %{
 9293     __ rolq($dst$$Register);
 9294   %}
 9295   ins_pipe(ialu_reg_reg);
 9296 %}
 9297 
 9298 
 9299 // Rotate Right by constant.
 9300 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9301 %{
 9302   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9303   match(Set dst (RotateRight dst shift));
 9304   effect(KILL cr);
 9305   format %{ "rorq    $dst, $shift" %}
 9306   ins_encode %{
 9307     __ rorq($dst$$Register, $shift$$constant);
 9308   %}
 9309   ins_pipe(ialu_reg);
 9310 %}
 9311 
 9312 
 9313 // Rotate Right by constant
 9314 instruct rorL_immI8(rRegL dst, immI8 shift)
 9315 %{
 9316   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9317   match(Set dst (RotateRight dst shift));
 9318   format %{ "rorxq    $dst, $shift" %}
 9319   ins_encode %{
 9320     __ rorxq($dst$$Register, $dst$$Register, $shift$$constant);
 9321   %}
 9322   ins_pipe(ialu_reg_reg);
 9323 %}
 9324 
 9325 // Rotate Right by variable
 9326 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9327 %{
 9328   predicate(n->bottom_type()->basic_type() == T_LONG);
 9329   match(Set dst (RotateRight dst shift));
 9330   effect(KILL cr);
 9331   format %{ "rorq    $dst, $shift" %}
 9332   ins_encode %{
 9333     __ rorq($dst$$Register);
 9334   %}
 9335   ins_pipe(ialu_reg_reg);
 9336 %}
 9337 
 9338 
 9339 // Logical Instructions
 9340 
 9341 // Integer Logical Instructions
 9342 
 9343 // And Instructions
 9344 // And Register with Register
 9345 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9346 %{
 9347   match(Set dst (AndI dst src));
 9348   effect(KILL cr);
 9349 
 9350   format %{ "andl    $dst, $src\t# int" %}
 9351   ins_encode %{
 9352     __ andl($dst$$Register, $src$$Register);
 9353   %}
 9354   ins_pipe(ialu_reg_reg);
 9355 %}
 9356 
 9357 // And Register with Immediate 255
 9358 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
 9359 %{
 9360   match(Set dst (AndI dst src));
 9361 
 9362   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
 9363   ins_encode %{
 9364     __ movzbl($dst$$Register, $dst$$Register);
 9365   %}
 9366   ins_pipe(ialu_reg);
 9367 %}
 9368 
 9369 // And Register with Immediate 255 and promote to long
 9370 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9371 %{
 9372   match(Set dst (ConvI2L (AndI src mask)));
 9373 
 9374   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9375   ins_encode %{
 9376     __ movzbl($dst$$Register, $src$$Register);
 9377   %}
 9378   ins_pipe(ialu_reg);
 9379 %}
 9380 
 9381 // And Register with Immediate 65535
 9382 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
 9383 %{
 9384   match(Set dst (AndI dst src));
 9385 
 9386   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
 9387   ins_encode %{
 9388     __ movzwl($dst$$Register, $dst$$Register);
 9389   %}
 9390   ins_pipe(ialu_reg);
 9391 %}
 9392 
 9393 // And Register with Immediate 65535 and promote to long
 9394 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9395 %{
 9396   match(Set dst (ConvI2L (AndI src mask)));
 9397 
 9398   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9399   ins_encode %{
 9400     __ movzwl($dst$$Register, $src$$Register);
 9401   %}
 9402   ins_pipe(ialu_reg);
 9403 %}
 9404 
 9405 // Can skip int2long conversions after AND with small bitmask
 9406 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9407 %{
 9408   predicate(VM_Version::supports_bmi2());
 9409   ins_cost(125);
 9410   effect(TEMP tmp, KILL cr);
 9411   match(Set dst (ConvI2L (AndI src mask)));
 9412   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9413   ins_encode %{
 9414     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9415     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9416   %}
 9417   ins_pipe(ialu_reg_reg);
 9418 %}
 9419 
 9420 // And Register with Immediate
 9421 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9422 %{
 9423   match(Set dst (AndI dst src));
 9424   effect(KILL cr);
 9425 
 9426   format %{ "andl    $dst, $src\t# int" %}
 9427   ins_encode %{
 9428     __ andl($dst$$Register, $src$$constant);
 9429   %}
 9430   ins_pipe(ialu_reg);
 9431 %}
 9432 
 9433 // And Register with Memory
 9434 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9435 %{
 9436   match(Set dst (AndI dst (LoadI src)));
 9437   effect(KILL cr);
 9438 
 9439   ins_cost(125);
 9440   format %{ "andl    $dst, $src\t# int" %}
 9441   ins_encode %{
 9442     __ andl($dst$$Register, $src$$Address);
 9443   %}
 9444   ins_pipe(ialu_reg_mem);
 9445 %}
 9446 
 9447 // And Memory with Register
 9448 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9449 %{
 9450   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9451   effect(KILL cr);
 9452 
 9453   ins_cost(150);
 9454   format %{ "andb    $dst, $src\t# byte" %}
 9455   ins_encode %{
 9456     __ andb($dst$$Address, $src$$Register);
 9457   %}
 9458   ins_pipe(ialu_mem_reg);
 9459 %}
 9460 
 9461 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9462 %{
 9463   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9464   effect(KILL cr);
 9465 
 9466   ins_cost(150);
 9467   format %{ "andl    $dst, $src\t# int" %}
 9468   ins_encode %{
 9469     __ andl($dst$$Address, $src$$Register);
 9470   %}
 9471   ins_pipe(ialu_mem_reg);
 9472 %}
 9473 
 9474 // And Memory with Immediate
 9475 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9476 %{
 9477   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9478   effect(KILL cr);
 9479 
 9480   ins_cost(125);
 9481   format %{ "andl    $dst, $src\t# int" %}
 9482   ins_encode %{
 9483     __ andl($dst$$Address, $src$$constant);
 9484   %}
 9485   ins_pipe(ialu_mem_imm);
 9486 %}
 9487 
 9488 // BMI1 instructions
 9489 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9490   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9491   predicate(UseBMI1Instructions);
 9492   effect(KILL cr);
 9493 
 9494   ins_cost(125);
 9495   format %{ "andnl  $dst, $src1, $src2" %}
 9496 
 9497   ins_encode %{
 9498     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9499   %}
 9500   ins_pipe(ialu_reg_mem);
 9501 %}
 9502 
 9503 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9504   match(Set dst (AndI (XorI src1 minus_1) src2));
 9505   predicate(UseBMI1Instructions);
 9506   effect(KILL cr);
 9507 
 9508   format %{ "andnl  $dst, $src1, $src2" %}
 9509 
 9510   ins_encode %{
 9511     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9512   %}
 9513   ins_pipe(ialu_reg);
 9514 %}
 9515 
 9516 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9517   match(Set dst (AndI (SubI imm_zero src) src));
 9518   predicate(UseBMI1Instructions);
 9519   effect(KILL cr);
 9520 
 9521   format %{ "blsil  $dst, $src" %}
 9522 
 9523   ins_encode %{
 9524     __ blsil($dst$$Register, $src$$Register);
 9525   %}
 9526   ins_pipe(ialu_reg);
 9527 %}
 9528 
 9529 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9530   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9531   predicate(UseBMI1Instructions);
 9532   effect(KILL cr);
 9533 
 9534   ins_cost(125);
 9535   format %{ "blsil  $dst, $src" %}
 9536 
 9537   ins_encode %{
 9538     __ blsil($dst$$Register, $src$$Address);
 9539   %}
 9540   ins_pipe(ialu_reg_mem);
 9541 %}
 9542 
 9543 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9544 %{
 9545   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9546   predicate(UseBMI1Instructions);
 9547   effect(KILL cr);
 9548 
 9549   ins_cost(125);
 9550   format %{ "blsmskl $dst, $src" %}
 9551 
 9552   ins_encode %{
 9553     __ blsmskl($dst$$Register, $src$$Address);
 9554   %}
 9555   ins_pipe(ialu_reg_mem);
 9556 %}
 9557 
 9558 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9559 %{
 9560   match(Set dst (XorI (AddI src minus_1) src));
 9561   predicate(UseBMI1Instructions);
 9562   effect(KILL cr);
 9563 
 9564   format %{ "blsmskl $dst, $src" %}
 9565 
 9566   ins_encode %{
 9567     __ blsmskl($dst$$Register, $src$$Register);
 9568   %}
 9569 
 9570   ins_pipe(ialu_reg);
 9571 %}
 9572 
 9573 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9574 %{
 9575   match(Set dst (AndI (AddI src minus_1) src) );
 9576   predicate(UseBMI1Instructions);
 9577   effect(KILL cr);
 9578 
 9579   format %{ "blsrl  $dst, $src" %}
 9580 
 9581   ins_encode %{
 9582     __ blsrl($dst$$Register, $src$$Register);
 9583   %}
 9584 
 9585   ins_pipe(ialu_reg_mem);
 9586 %}
 9587 
 9588 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9589 %{
 9590   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9591   predicate(UseBMI1Instructions);
 9592   effect(KILL cr);
 9593 
 9594   ins_cost(125);
 9595   format %{ "blsrl  $dst, $src" %}
 9596 
 9597   ins_encode %{
 9598     __ blsrl($dst$$Register, $src$$Address);
 9599   %}
 9600 
 9601   ins_pipe(ialu_reg);
 9602 %}
 9603 
 9604 // Or Instructions
 9605 // Or Register with Register
 9606 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9607 %{
 9608   match(Set dst (OrI dst src));
 9609   effect(KILL cr);
 9610 
 9611   format %{ "orl     $dst, $src\t# int" %}
 9612   ins_encode %{
 9613     __ orl($dst$$Register, $src$$Register);
 9614   %}
 9615   ins_pipe(ialu_reg_reg);
 9616 %}
 9617 
 9618 // Or Register with Immediate
 9619 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9620 %{
 9621   match(Set dst (OrI dst src));
 9622   effect(KILL cr);
 9623 
 9624   format %{ "orl     $dst, $src\t# int" %}
 9625   ins_encode %{
 9626     __ orl($dst$$Register, $src$$constant);
 9627   %}
 9628   ins_pipe(ialu_reg);
 9629 %}
 9630 
 9631 // Or Register with Memory
 9632 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9633 %{
 9634   match(Set dst (OrI dst (LoadI src)));
 9635   effect(KILL cr);
 9636 
 9637   ins_cost(125);
 9638   format %{ "orl     $dst, $src\t# int" %}
 9639   ins_encode %{
 9640     __ orl($dst$$Register, $src$$Address);
 9641   %}
 9642   ins_pipe(ialu_reg_mem);
 9643 %}
 9644 
 9645 // Or Memory with Register
 9646 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9647 %{
 9648   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9649   effect(KILL cr);
 9650 
 9651   ins_cost(150);
 9652   format %{ "orb    $dst, $src\t# byte" %}
 9653   ins_encode %{
 9654     __ orb($dst$$Address, $src$$Register);
 9655   %}
 9656   ins_pipe(ialu_mem_reg);
 9657 %}
 9658 
 9659 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9660 %{
 9661   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9662   effect(KILL cr);
 9663 
 9664   ins_cost(150);
 9665   format %{ "orl     $dst, $src\t# int" %}
 9666   ins_encode %{
 9667     __ orl($dst$$Address, $src$$Register);
 9668   %}
 9669   ins_pipe(ialu_mem_reg);
 9670 %}
 9671 
 9672 // Or Memory with Immediate
 9673 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9674 %{
 9675   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9676   effect(KILL cr);
 9677 
 9678   ins_cost(125);
 9679   format %{ "orl     $dst, $src\t# int" %}
 9680   ins_encode %{
 9681     __ orl($dst$$Address, $src$$constant);
 9682   %}
 9683   ins_pipe(ialu_mem_imm);
 9684 %}
 9685 
 9686 // Xor Instructions
 9687 // Xor Register with Register
 9688 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9689 %{
 9690   match(Set dst (XorI dst src));
 9691   effect(KILL cr);
 9692 
 9693   format %{ "xorl    $dst, $src\t# int" %}
 9694   ins_encode %{
 9695     __ xorl($dst$$Register, $src$$Register);
 9696   %}
 9697   ins_pipe(ialu_reg_reg);
 9698 %}
 9699 
 9700 // Xor Register with Immediate -1
 9701 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9702   match(Set dst (XorI dst imm));
 9703 
 9704   format %{ "not    $dst" %}
 9705   ins_encode %{
 9706      __ notl($dst$$Register);
 9707   %}
 9708   ins_pipe(ialu_reg);
 9709 %}
 9710 
 9711 // Xor Register with Immediate
 9712 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9713 %{
 9714   match(Set dst (XorI dst src));
 9715   effect(KILL cr);
 9716 
 9717   format %{ "xorl    $dst, $src\t# int" %}
 9718   ins_encode %{
 9719     __ xorl($dst$$Register, $src$$constant);
 9720   %}
 9721   ins_pipe(ialu_reg);
 9722 %}
 9723 
 9724 // Xor Register with Memory
 9725 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9726 %{
 9727   match(Set dst (XorI dst (LoadI src)));
 9728   effect(KILL cr);
 9729 
 9730   ins_cost(125);
 9731   format %{ "xorl    $dst, $src\t# int" %}
 9732   ins_encode %{
 9733     __ xorl($dst$$Register, $src$$Address);
 9734   %}
 9735   ins_pipe(ialu_reg_mem);
 9736 %}
 9737 
 9738 // Xor Memory with Register
 9739 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9740 %{
 9741   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9742   effect(KILL cr);
 9743 
 9744   ins_cost(150);
 9745   format %{ "xorb    $dst, $src\t# byte" %}
 9746   ins_encode %{
 9747     __ xorb($dst$$Address, $src$$Register);
 9748   %}
 9749   ins_pipe(ialu_mem_reg);
 9750 %}
 9751 
 9752 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9753 %{
 9754   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9755   effect(KILL cr);
 9756 
 9757   ins_cost(150);
 9758   format %{ "xorl    $dst, $src\t# int" %}
 9759   ins_encode %{
 9760     __ xorl($dst$$Address, $src$$Register);
 9761   %}
 9762   ins_pipe(ialu_mem_reg);
 9763 %}
 9764 
 9765 // Xor Memory with Immediate
 9766 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9767 %{
 9768   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9769   effect(KILL cr);
 9770 
 9771   ins_cost(125);
 9772   format %{ "xorl    $dst, $src\t# int" %}
 9773   ins_encode %{
 9774     __ xorl($dst$$Address, $src$$constant);
 9775   %}
 9776   ins_pipe(ialu_mem_imm);
 9777 %}
 9778 
 9779 
 9780 // Long Logical Instructions
 9781 
 9782 // And Instructions
 9783 // And Register with Register
 9784 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9785 %{
 9786   match(Set dst (AndL dst src));
 9787   effect(KILL cr);
 9788 
 9789   format %{ "andq    $dst, $src\t# long" %}
 9790   ins_encode %{
 9791     __ andq($dst$$Register, $src$$Register);
 9792   %}
 9793   ins_pipe(ialu_reg_reg);
 9794 %}
 9795 
 9796 // And Register with Immediate 255
 9797 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
 9798 %{
 9799   match(Set dst (AndL dst src));
 9800 
 9801   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
 9802   ins_encode %{
 9803     __ movzbq($dst$$Register, $dst$$Register);
 9804   %}
 9805   ins_pipe(ialu_reg);
 9806 %}
 9807 
 9808 // And Register with Immediate 65535
 9809 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
 9810 %{
 9811   match(Set dst (AndL dst src));
 9812 
 9813   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
 9814   ins_encode %{
 9815     __ movzwq($dst$$Register, $dst$$Register);
 9816   %}
 9817   ins_pipe(ialu_reg);
 9818 %}
 9819 
 9820 // And Register with Immediate
 9821 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9822 %{
 9823   match(Set dst (AndL dst src));
 9824   effect(KILL cr);
 9825 
 9826   format %{ "andq    $dst, $src\t# long" %}
 9827   ins_encode %{
 9828     __ andq($dst$$Register, $src$$constant);
 9829   %}
 9830   ins_pipe(ialu_reg);
 9831 %}
 9832 
 9833 // And Register with Memory
 9834 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9835 %{
 9836   match(Set dst (AndL dst (LoadL src)));
 9837   effect(KILL cr);
 9838 
 9839   ins_cost(125);
 9840   format %{ "andq    $dst, $src\t# long" %}
 9841   ins_encode %{
 9842     __ andq($dst$$Register, $src$$Address);
 9843   %}
 9844   ins_pipe(ialu_reg_mem);
 9845 %}
 9846 
 9847 // And Memory with Register
 9848 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9849 %{
 9850   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9851   effect(KILL cr);
 9852 
 9853   ins_cost(150);
 9854   format %{ "andq    $dst, $src\t# long" %}
 9855   ins_encode %{
 9856     __ andq($dst$$Address, $src$$Register);
 9857   %}
 9858   ins_pipe(ialu_mem_reg);
 9859 %}
 9860 
 9861 // And Memory with Immediate
 9862 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9863 %{
 9864   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9865   effect(KILL cr);
 9866 
 9867   ins_cost(125);
 9868   format %{ "andq    $dst, $src\t# long" %}
 9869   ins_encode %{
 9870     __ andq($dst$$Address, $src$$constant);
 9871   %}
 9872   ins_pipe(ialu_mem_imm);
 9873 %}
 9874 
 9875 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9876 %{
 9877   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9878   // because AND/OR works well enough for 8/32-bit values.
 9879   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9880 
 9881   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9882   effect(KILL cr);
 9883 
 9884   ins_cost(125);
 9885   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9886   ins_encode %{
 9887     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9888   %}
 9889   ins_pipe(ialu_mem_imm);
 9890 %}
 9891 
 9892 // BMI1 instructions
 9893 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9894   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9895   predicate(UseBMI1Instructions);
 9896   effect(KILL cr);
 9897 
 9898   ins_cost(125);
 9899   format %{ "andnq  $dst, $src1, $src2" %}
 9900 
 9901   ins_encode %{
 9902     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9903   %}
 9904   ins_pipe(ialu_reg_mem);
 9905 %}
 9906 
 9907 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9908   match(Set dst (AndL (XorL src1 minus_1) src2));
 9909   predicate(UseBMI1Instructions);
 9910   effect(KILL cr);
 9911 
 9912   format %{ "andnq  $dst, $src1, $src2" %}
 9913 
 9914   ins_encode %{
 9915   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9916   %}
 9917   ins_pipe(ialu_reg_mem);
 9918 %}
 9919 
 9920 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9921   match(Set dst (AndL (SubL imm_zero src) src));
 9922   predicate(UseBMI1Instructions);
 9923   effect(KILL cr);
 9924 
 9925   format %{ "blsiq  $dst, $src" %}
 9926 
 9927   ins_encode %{
 9928     __ blsiq($dst$$Register, $src$$Register);
 9929   %}
 9930   ins_pipe(ialu_reg);
 9931 %}
 9932 
 9933 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9934   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9935   predicate(UseBMI1Instructions);
 9936   effect(KILL cr);
 9937 
 9938   ins_cost(125);
 9939   format %{ "blsiq  $dst, $src" %}
 9940 
 9941   ins_encode %{
 9942     __ blsiq($dst$$Register, $src$$Address);
 9943   %}
 9944   ins_pipe(ialu_reg_mem);
 9945 %}
 9946 
 9947 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9948 %{
 9949   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9950   predicate(UseBMI1Instructions);
 9951   effect(KILL cr);
 9952 
 9953   ins_cost(125);
 9954   format %{ "blsmskq $dst, $src" %}
 9955 
 9956   ins_encode %{
 9957     __ blsmskq($dst$$Register, $src$$Address);
 9958   %}
 9959   ins_pipe(ialu_reg_mem);
 9960 %}
 9961 
 9962 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9963 %{
 9964   match(Set dst (XorL (AddL src minus_1) src));
 9965   predicate(UseBMI1Instructions);
 9966   effect(KILL cr);
 9967 
 9968   format %{ "blsmskq $dst, $src" %}
 9969 
 9970   ins_encode %{
 9971     __ blsmskq($dst$$Register, $src$$Register);
 9972   %}
 9973 
 9974   ins_pipe(ialu_reg);
 9975 %}
 9976 
 9977 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9978 %{
 9979   match(Set dst (AndL (AddL src minus_1) src) );
 9980   predicate(UseBMI1Instructions);
 9981   effect(KILL cr);
 9982 
 9983   format %{ "blsrq  $dst, $src" %}
 9984 
 9985   ins_encode %{
 9986     __ blsrq($dst$$Register, $src$$Register);
 9987   %}
 9988 
 9989   ins_pipe(ialu_reg);
 9990 %}
 9991 
 9992 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9993 %{
 9994   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9995   predicate(UseBMI1Instructions);
 9996   effect(KILL cr);
 9997 
 9998   ins_cost(125);
 9999   format %{ "blsrq  $dst, $src" %}
10000 
10001   ins_encode %{
10002     __ blsrq($dst$$Register, $src$$Address);
10003   %}
10004 
10005   ins_pipe(ialu_reg);
10006 %}
10007 
10008 // Or Instructions
10009 // Or Register with Register
10010 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10011 %{
10012   match(Set dst (OrL dst src));
10013   effect(KILL cr);
10014 
10015   format %{ "orq     $dst, $src\t# long" %}
10016   ins_encode %{
10017     __ orq($dst$$Register, $src$$Register);
10018   %}
10019   ins_pipe(ialu_reg_reg);
10020 %}
10021 
10022 // Use any_RegP to match R15 (TLS register) without spilling.
10023 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10024   match(Set dst (OrL dst (CastP2X src)));
10025   effect(KILL cr);
10026 
10027   format %{ "orq     $dst, $src\t# long" %}
10028   ins_encode %{
10029     __ orq($dst$$Register, $src$$Register);
10030   %}
10031   ins_pipe(ialu_reg_reg);
10032 %}
10033 
10034 
10035 // Or Register with Immediate
10036 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10037 %{
10038   match(Set dst (OrL dst src));
10039   effect(KILL cr);
10040 
10041   format %{ "orq     $dst, $src\t# long" %}
10042   ins_encode %{
10043     __ orq($dst$$Register, $src$$constant);
10044   %}
10045   ins_pipe(ialu_reg);
10046 %}
10047 
10048 // Or Register with Memory
10049 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10050 %{
10051   match(Set dst (OrL dst (LoadL src)));
10052   effect(KILL cr);
10053 
10054   ins_cost(125);
10055   format %{ "orq     $dst, $src\t# long" %}
10056   ins_encode %{
10057     __ orq($dst$$Register, $src$$Address);
10058   %}
10059   ins_pipe(ialu_reg_mem);
10060 %}
10061 
10062 // Or Memory with Register
10063 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10064 %{
10065   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10066   effect(KILL cr);
10067 
10068   ins_cost(150);
10069   format %{ "orq     $dst, $src\t# long" %}
10070   ins_encode %{
10071     __ orq($dst$$Address, $src$$Register);
10072   %}
10073   ins_pipe(ialu_mem_reg);
10074 %}
10075 
10076 // Or Memory with Immediate
10077 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10078 %{
10079   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10080   effect(KILL cr);
10081 
10082   ins_cost(125);
10083   format %{ "orq     $dst, $src\t# long" %}
10084   ins_encode %{
10085     __ orq($dst$$Address, $src$$constant);
10086   %}
10087   ins_pipe(ialu_mem_imm);
10088 %}
10089 
10090 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10091 %{
10092   // con should be a pure 64-bit power of 2 immediate
10093   // because AND/OR works well enough for 8/32-bit values.
10094   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10095 
10096   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10097   effect(KILL cr);
10098 
10099   ins_cost(125);
10100   format %{ "btsq    $dst, log2($con)\t# long" %}
10101   ins_encode %{
10102     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10103   %}
10104   ins_pipe(ialu_mem_imm);
10105 %}
10106 
10107 // Xor Instructions
10108 // Xor Register with Register
10109 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10110 %{
10111   match(Set dst (XorL dst src));
10112   effect(KILL cr);
10113 
10114   format %{ "xorq    $dst, $src\t# long" %}
10115   ins_encode %{
10116     __ xorq($dst$$Register, $src$$Register);
10117   %}
10118   ins_pipe(ialu_reg_reg);
10119 %}
10120 
10121 // Xor Register with Immediate -1
10122 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10123   match(Set dst (XorL dst imm));
10124 
10125   format %{ "notq   $dst" %}
10126   ins_encode %{
10127      __ notq($dst$$Register);
10128   %}
10129   ins_pipe(ialu_reg);
10130 %}
10131 
10132 // Xor Register with Immediate
10133 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10134 %{
10135   match(Set dst (XorL dst src));
10136   effect(KILL cr);
10137 
10138   format %{ "xorq    $dst, $src\t# long" %}
10139   ins_encode %{
10140     __ xorq($dst$$Register, $src$$constant);
10141   %}
10142   ins_pipe(ialu_reg);
10143 %}
10144 
10145 // Xor Register with Memory
10146 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10147 %{
10148   match(Set dst (XorL dst (LoadL src)));
10149   effect(KILL cr);
10150 
10151   ins_cost(125);
10152   format %{ "xorq    $dst, $src\t# long" %}
10153   ins_encode %{
10154     __ xorq($dst$$Register, $src$$Address);
10155   %}
10156   ins_pipe(ialu_reg_mem);
10157 %}
10158 
10159 // Xor Memory with Register
10160 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10161 %{
10162   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10163   effect(KILL cr);
10164 
10165   ins_cost(150);
10166   format %{ "xorq    $dst, $src\t# long" %}
10167   ins_encode %{
10168     __ xorq($dst$$Address, $src$$Register);
10169   %}
10170   ins_pipe(ialu_mem_reg);
10171 %}
10172 
10173 // Xor Memory with Immediate
10174 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10175 %{
10176   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10177   effect(KILL cr);
10178 
10179   ins_cost(125);
10180   format %{ "xorq    $dst, $src\t# long" %}
10181   ins_encode %{
10182     __ xorq($dst$$Address, $src$$constant);
10183   %}
10184   ins_pipe(ialu_mem_imm);
10185 %}
10186 
10187 // Convert Int to Boolean
10188 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10189 %{
10190   match(Set dst (Conv2B src));
10191   effect(KILL cr);
10192 
10193   format %{ "testl   $src, $src\t# ci2b\n\t"
10194             "setnz   $dst\n\t"
10195             "movzbl  $dst, $dst" %}
10196   ins_encode %{
10197     __ testl($src$$Register, $src$$Register);
10198     __ set_byte_if_not_zero($dst$$Register);
10199     __ movzbl($dst$$Register, $dst$$Register);
10200   %}
10201   ins_pipe(pipe_slow); // XXX
10202 %}
10203 
10204 // Convert Pointer to Boolean
10205 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10206 %{
10207   match(Set dst (Conv2B src));
10208   effect(KILL cr);
10209 
10210   format %{ "testq   $src, $src\t# cp2b\n\t"
10211             "setnz   $dst\n\t"
10212             "movzbl  $dst, $dst" %}
10213   ins_encode %{
10214     __ testq($src$$Register, $src$$Register);
10215     __ set_byte_if_not_zero($dst$$Register);
10216     __ movzbl($dst$$Register, $dst$$Register);
10217   %}
10218   ins_pipe(pipe_slow); // XXX
10219 %}
10220 
10221 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10222 %{
10223   match(Set dst (CmpLTMask p q));
10224   effect(KILL cr);
10225 
10226   ins_cost(400);
10227   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10228             "setlt   $dst\n\t"
10229             "movzbl  $dst, $dst\n\t"
10230             "negl    $dst" %}
10231   ins_encode %{
10232     __ cmpl($p$$Register, $q$$Register);
10233     __ setl($dst$$Register);
10234     __ movzbl($dst$$Register, $dst$$Register);
10235     __ negl($dst$$Register);
10236   %}
10237   ins_pipe(pipe_slow);
10238 %}
10239 
10240 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10241 %{
10242   match(Set dst (CmpLTMask dst zero));
10243   effect(KILL cr);
10244 
10245   ins_cost(100);
10246   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10247   ins_encode %{
10248     __ sarl($dst$$Register, 31);
10249   %}
10250   ins_pipe(ialu_reg);
10251 %}
10252 
10253 /* Better to save a register than avoid a branch */
10254 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10255 %{
10256   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10257   effect(KILL cr);
10258   ins_cost(300);
10259   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10260             "jge     done\n\t"
10261             "addl    $p,$y\n"
10262             "done:   " %}
10263   ins_encode %{
10264     Register Rp = $p$$Register;
10265     Register Rq = $q$$Register;
10266     Register Ry = $y$$Register;
10267     Label done;
10268     __ subl(Rp, Rq);
10269     __ jccb(Assembler::greaterEqual, done);
10270     __ addl(Rp, Ry);
10271     __ bind(done);
10272   %}
10273   ins_pipe(pipe_cmplt);
10274 %}
10275 
10276 /* Better to save a register than avoid a branch */
10277 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10278 %{
10279   match(Set y (AndI (CmpLTMask p q) y));
10280   effect(KILL cr);
10281 
10282   ins_cost(300);
10283 
10284   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10285             "jlt     done\n\t"
10286             "xorl    $y, $y\n"
10287             "done:   " %}
10288   ins_encode %{
10289     Register Rp = $p$$Register;
10290     Register Rq = $q$$Register;
10291     Register Ry = $y$$Register;
10292     Label done;
10293     __ cmpl(Rp, Rq);
10294     __ jccb(Assembler::less, done);
10295     __ xorl(Ry, Ry);
10296     __ bind(done);
10297   %}
10298   ins_pipe(pipe_cmplt);
10299 %}
10300 
10301 
10302 //---------- FP Instructions------------------------------------------------
10303 
10304 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10305 %{
10306   match(Set cr (CmpF src1 src2));
10307 
10308   ins_cost(145);
10309   format %{ "ucomiss $src1, $src2\n\t"
10310             "jnp,s   exit\n\t"
10311             "pushfq\t# saw NaN, set CF\n\t"
10312             "andq    [rsp], #0xffffff2b\n\t"
10313             "popfq\n"
10314     "exit:" %}
10315   ins_encode %{
10316     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10317     emit_cmpfp_fixup(_masm);
10318   %}
10319   ins_pipe(pipe_slow);
10320 %}
10321 
10322 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10323   match(Set cr (CmpF src1 src2));
10324 
10325   ins_cost(100);
10326   format %{ "ucomiss $src1, $src2" %}
10327   ins_encode %{
10328     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10329   %}
10330   ins_pipe(pipe_slow);
10331 %}
10332 
10333 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10334 %{
10335   match(Set cr (CmpF src1 (LoadF src2)));
10336 
10337   ins_cost(145);
10338   format %{ "ucomiss $src1, $src2\n\t"
10339             "jnp,s   exit\n\t"
10340             "pushfq\t# saw NaN, set CF\n\t"
10341             "andq    [rsp], #0xffffff2b\n\t"
10342             "popfq\n"
10343     "exit:" %}
10344   ins_encode %{
10345     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10346     emit_cmpfp_fixup(_masm);
10347   %}
10348   ins_pipe(pipe_slow);
10349 %}
10350 
10351 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10352   match(Set cr (CmpF src1 (LoadF src2)));
10353 
10354   ins_cost(100);
10355   format %{ "ucomiss $src1, $src2" %}
10356   ins_encode %{
10357     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10358   %}
10359   ins_pipe(pipe_slow);
10360 %}
10361 
10362 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10363   match(Set cr (CmpF src con));
10364 
10365   ins_cost(145);
10366   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10367             "jnp,s   exit\n\t"
10368             "pushfq\t# saw NaN, set CF\n\t"
10369             "andq    [rsp], #0xffffff2b\n\t"
10370             "popfq\n"
10371     "exit:" %}
10372   ins_encode %{
10373     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10374     emit_cmpfp_fixup(_masm);
10375   %}
10376   ins_pipe(pipe_slow);
10377 %}
10378 
10379 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10380   match(Set cr (CmpF src con));
10381   ins_cost(100);
10382   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10383   ins_encode %{
10384     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10385   %}
10386   ins_pipe(pipe_slow);
10387 %}
10388 
10389 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10390 %{
10391   match(Set cr (CmpD src1 src2));
10392 
10393   ins_cost(145);
10394   format %{ "ucomisd $src1, $src2\n\t"
10395             "jnp,s   exit\n\t"
10396             "pushfq\t# saw NaN, set CF\n\t"
10397             "andq    [rsp], #0xffffff2b\n\t"
10398             "popfq\n"
10399     "exit:" %}
10400   ins_encode %{
10401     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10402     emit_cmpfp_fixup(_masm);
10403   %}
10404   ins_pipe(pipe_slow);
10405 %}
10406 
10407 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10408   match(Set cr (CmpD src1 src2));
10409 
10410   ins_cost(100);
10411   format %{ "ucomisd $src1, $src2 test" %}
10412   ins_encode %{
10413     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10414   %}
10415   ins_pipe(pipe_slow);
10416 %}
10417 
10418 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10419 %{
10420   match(Set cr (CmpD src1 (LoadD src2)));
10421 
10422   ins_cost(145);
10423   format %{ "ucomisd $src1, $src2\n\t"
10424             "jnp,s   exit\n\t"
10425             "pushfq\t# saw NaN, set CF\n\t"
10426             "andq    [rsp], #0xffffff2b\n\t"
10427             "popfq\n"
10428     "exit:" %}
10429   ins_encode %{
10430     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10431     emit_cmpfp_fixup(_masm);
10432   %}
10433   ins_pipe(pipe_slow);
10434 %}
10435 
10436 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10437   match(Set cr (CmpD src1 (LoadD src2)));
10438 
10439   ins_cost(100);
10440   format %{ "ucomisd $src1, $src2" %}
10441   ins_encode %{
10442     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10443   %}
10444   ins_pipe(pipe_slow);
10445 %}
10446 
10447 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10448   match(Set cr (CmpD src con));
10449 
10450   ins_cost(145);
10451   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10452             "jnp,s   exit\n\t"
10453             "pushfq\t# saw NaN, set CF\n\t"
10454             "andq    [rsp], #0xffffff2b\n\t"
10455             "popfq\n"
10456     "exit:" %}
10457   ins_encode %{
10458     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10459     emit_cmpfp_fixup(_masm);
10460   %}
10461   ins_pipe(pipe_slow);
10462 %}
10463 
10464 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10465   match(Set cr (CmpD src con));
10466   ins_cost(100);
10467   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10468   ins_encode %{
10469     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10470   %}
10471   ins_pipe(pipe_slow);
10472 %}
10473 
10474 // Compare into -1,0,1
10475 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10476 %{
10477   match(Set dst (CmpF3 src1 src2));
10478   effect(KILL cr);
10479 
10480   ins_cost(275);
10481   format %{ "ucomiss $src1, $src2\n\t"
10482             "movl    $dst, #-1\n\t"
10483             "jp,s    done\n\t"
10484             "jb,s    done\n\t"
10485             "setne   $dst\n\t"
10486             "movzbl  $dst, $dst\n"
10487     "done:" %}
10488   ins_encode %{
10489     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10490     emit_cmpfp3(_masm, $dst$$Register);
10491   %}
10492   ins_pipe(pipe_slow);
10493 %}
10494 
10495 // Compare into -1,0,1
10496 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10497 %{
10498   match(Set dst (CmpF3 src1 (LoadF src2)));
10499   effect(KILL cr);
10500 
10501   ins_cost(275);
10502   format %{ "ucomiss $src1, $src2\n\t"
10503             "movl    $dst, #-1\n\t"
10504             "jp,s    done\n\t"
10505             "jb,s    done\n\t"
10506             "setne   $dst\n\t"
10507             "movzbl  $dst, $dst\n"
10508     "done:" %}
10509   ins_encode %{
10510     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10511     emit_cmpfp3(_masm, $dst$$Register);
10512   %}
10513   ins_pipe(pipe_slow);
10514 %}
10515 
10516 // Compare into -1,0,1
10517 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10518   match(Set dst (CmpF3 src con));
10519   effect(KILL cr);
10520 
10521   ins_cost(275);
10522   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10523             "movl    $dst, #-1\n\t"
10524             "jp,s    done\n\t"
10525             "jb,s    done\n\t"
10526             "setne   $dst\n\t"
10527             "movzbl  $dst, $dst\n"
10528     "done:" %}
10529   ins_encode %{
10530     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10531     emit_cmpfp3(_masm, $dst$$Register);
10532   %}
10533   ins_pipe(pipe_slow);
10534 %}
10535 
10536 // Compare into -1,0,1
10537 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10538 %{
10539   match(Set dst (CmpD3 src1 src2));
10540   effect(KILL cr);
10541 
10542   ins_cost(275);
10543   format %{ "ucomisd $src1, $src2\n\t"
10544             "movl    $dst, #-1\n\t"
10545             "jp,s    done\n\t"
10546             "jb,s    done\n\t"
10547             "setne   $dst\n\t"
10548             "movzbl  $dst, $dst\n"
10549     "done:" %}
10550   ins_encode %{
10551     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10552     emit_cmpfp3(_masm, $dst$$Register);
10553   %}
10554   ins_pipe(pipe_slow);
10555 %}
10556 
10557 // Compare into -1,0,1
10558 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10559 %{
10560   match(Set dst (CmpD3 src1 (LoadD src2)));
10561   effect(KILL cr);
10562 
10563   ins_cost(275);
10564   format %{ "ucomisd $src1, $src2\n\t"
10565             "movl    $dst, #-1\n\t"
10566             "jp,s    done\n\t"
10567             "jb,s    done\n\t"
10568             "setne   $dst\n\t"
10569             "movzbl  $dst, $dst\n"
10570     "done:" %}
10571   ins_encode %{
10572     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10573     emit_cmpfp3(_masm, $dst$$Register);
10574   %}
10575   ins_pipe(pipe_slow);
10576 %}
10577 
10578 // Compare into -1,0,1
10579 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10580   match(Set dst (CmpD3 src con));
10581   effect(KILL cr);
10582 
10583   ins_cost(275);
10584   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10585             "movl    $dst, #-1\n\t"
10586             "jp,s    done\n\t"
10587             "jb,s    done\n\t"
10588             "setne   $dst\n\t"
10589             "movzbl  $dst, $dst\n"
10590     "done:" %}
10591   ins_encode %{
10592     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10593     emit_cmpfp3(_masm, $dst$$Register);
10594   %}
10595   ins_pipe(pipe_slow);
10596 %}
10597 
10598 //----------Arithmetic Conversion Instructions---------------------------------
10599 
10600 instruct convF2D_reg_reg(regD dst, regF src)
10601 %{
10602   match(Set dst (ConvF2D src));
10603 
10604   format %{ "cvtss2sd $dst, $src" %}
10605   ins_encode %{
10606     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10607   %}
10608   ins_pipe(pipe_slow); // XXX
10609 %}
10610 
10611 instruct convF2D_reg_mem(regD dst, memory src)
10612 %{
10613   match(Set dst (ConvF2D (LoadF src)));
10614 
10615   format %{ "cvtss2sd $dst, $src" %}
10616   ins_encode %{
10617     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10618   %}
10619   ins_pipe(pipe_slow); // XXX
10620 %}
10621 
10622 instruct convD2F_reg_reg(regF dst, regD src)
10623 %{
10624   match(Set dst (ConvD2F src));
10625 
10626   format %{ "cvtsd2ss $dst, $src" %}
10627   ins_encode %{
10628     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10629   %}
10630   ins_pipe(pipe_slow); // XXX
10631 %}
10632 
10633 instruct convD2F_reg_mem(regF dst, memory src)
10634 %{
10635   match(Set dst (ConvD2F (LoadD src)));
10636 
10637   format %{ "cvtsd2ss $dst, $src" %}
10638   ins_encode %{
10639     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10640   %}
10641   ins_pipe(pipe_slow); // XXX
10642 %}
10643 
10644 // XXX do mem variants
10645 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10646 %{
10647   match(Set dst (ConvF2I src));
10648   effect(KILL cr);
10649   format %{ "convert_f2i $dst,$src" %}
10650   ins_encode %{
10651     __ convert_f2i($dst$$Register, $src$$XMMRegister);
10652   %}
10653   ins_pipe(pipe_slow);
10654 %}
10655 
10656 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10657 %{
10658   match(Set dst (ConvF2L src));
10659   effect(KILL cr);
10660   format %{ "convert_f2l $dst,$src"%}
10661   ins_encode %{
10662     __ convert_f2l($dst$$Register, $src$$XMMRegister);
10663   %}
10664   ins_pipe(pipe_slow);
10665 %}
10666 
10667 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10668 %{
10669   match(Set dst (ConvD2I src));
10670   effect(KILL cr);
10671   format %{ "convert_d2i $dst,$src"%}
10672   ins_encode %{
10673     __ convert_d2i($dst$$Register, $src$$XMMRegister);
10674   %}
10675   ins_pipe(pipe_slow);
10676 %}
10677 
10678 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10679 %{
10680   match(Set dst (ConvD2L src));
10681   effect(KILL cr);
10682   format %{ "convert_d2l $dst,$src"%}
10683   ins_encode %{
10684     __ convert_d2l($dst$$Register, $src$$XMMRegister);
10685   %}
10686   ins_pipe(pipe_slow);
10687 %}
10688 
10689 instruct convI2F_reg_reg(regF dst, rRegI src)
10690 %{
10691   predicate(!UseXmmI2F);
10692   match(Set dst (ConvI2F src));
10693 
10694   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10695   ins_encode %{
10696     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10697   %}
10698   ins_pipe(pipe_slow); // XXX
10699 %}
10700 
10701 instruct convI2F_reg_mem(regF dst, memory src)
10702 %{
10703   match(Set dst (ConvI2F (LoadI src)));
10704 
10705   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10706   ins_encode %{
10707     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10708   %}
10709   ins_pipe(pipe_slow); // XXX
10710 %}
10711 
10712 instruct convI2D_reg_reg(regD dst, rRegI src)
10713 %{
10714   predicate(!UseXmmI2D);
10715   match(Set dst (ConvI2D src));
10716 
10717   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10718   ins_encode %{
10719     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10720   %}
10721   ins_pipe(pipe_slow); // XXX
10722 %}
10723 
10724 instruct convI2D_reg_mem(regD dst, memory src)
10725 %{
10726   match(Set dst (ConvI2D (LoadI src)));
10727 
10728   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10729   ins_encode %{
10730     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10731   %}
10732   ins_pipe(pipe_slow); // XXX
10733 %}
10734 
10735 instruct convXI2F_reg(regF dst, rRegI src)
10736 %{
10737   predicate(UseXmmI2F);
10738   match(Set dst (ConvI2F src));
10739 
10740   format %{ "movdl $dst, $src\n\t"
10741             "cvtdq2psl $dst, $dst\t# i2f" %}
10742   ins_encode %{
10743     __ movdl($dst$$XMMRegister, $src$$Register);
10744     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10745   %}
10746   ins_pipe(pipe_slow); // XXX
10747 %}
10748 
10749 instruct convXI2D_reg(regD dst, rRegI src)
10750 %{
10751   predicate(UseXmmI2D);
10752   match(Set dst (ConvI2D src));
10753 
10754   format %{ "movdl $dst, $src\n\t"
10755             "cvtdq2pdl $dst, $dst\t# i2d" %}
10756   ins_encode %{
10757     __ movdl($dst$$XMMRegister, $src$$Register);
10758     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10759   %}
10760   ins_pipe(pipe_slow); // XXX
10761 %}
10762 
10763 instruct convL2F_reg_reg(regF dst, rRegL src)
10764 %{
10765   match(Set dst (ConvL2F src));
10766 
10767   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10768   ins_encode %{
10769     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10770   %}
10771   ins_pipe(pipe_slow); // XXX
10772 %}
10773 
10774 instruct convL2F_reg_mem(regF dst, memory src)
10775 %{
10776   match(Set dst (ConvL2F (LoadL src)));
10777 
10778   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10779   ins_encode %{
10780     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10781   %}
10782   ins_pipe(pipe_slow); // XXX
10783 %}
10784 
10785 instruct convL2D_reg_reg(regD dst, rRegL src)
10786 %{
10787   match(Set dst (ConvL2D src));
10788 
10789   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10790   ins_encode %{
10791     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10792   %}
10793   ins_pipe(pipe_slow); // XXX
10794 %}
10795 
10796 instruct convL2D_reg_mem(regD dst, memory src)
10797 %{
10798   match(Set dst (ConvL2D (LoadL src)));
10799 
10800   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10801   ins_encode %{
10802     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10803   %}
10804   ins_pipe(pipe_slow); // XXX
10805 %}
10806 
10807 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10808 %{
10809   match(Set dst (ConvI2L src));
10810 
10811   ins_cost(125);
10812   format %{ "movslq  $dst, $src\t# i2l" %}
10813   ins_encode %{
10814     __ movslq($dst$$Register, $src$$Register);
10815   %}
10816   ins_pipe(ialu_reg_reg);
10817 %}
10818 
10819 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10820 // %{
10821 //   match(Set dst (ConvI2L src));
10822 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10823 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10824 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10825 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10826 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10827 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10828 
10829 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10830 //   ins_encode(enc_copy(dst, src));
10831 // //   opcode(0x63); // needs REX.W
10832 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10833 //   ins_pipe(ialu_reg_reg);
10834 // %}
10835 
10836 // Zero-extend convert int to long
10837 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10838 %{
10839   match(Set dst (AndL (ConvI2L src) mask));
10840 
10841   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10842   ins_encode %{
10843     if ($dst$$reg != $src$$reg) {
10844       __ movl($dst$$Register, $src$$Register);
10845     }
10846   %}
10847   ins_pipe(ialu_reg_reg);
10848 %}
10849 
10850 // Zero-extend convert int to long
10851 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10852 %{
10853   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10854 
10855   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10856   ins_encode %{
10857     __ movl($dst$$Register, $src$$Address);
10858   %}
10859   ins_pipe(ialu_reg_mem);
10860 %}
10861 
10862 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10863 %{
10864   match(Set dst (AndL src mask));
10865 
10866   format %{ "movl    $dst, $src\t# zero-extend long" %}
10867   ins_encode %{
10868     __ movl($dst$$Register, $src$$Register);
10869   %}
10870   ins_pipe(ialu_reg_reg);
10871 %}
10872 
10873 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10874 %{
10875   match(Set dst (ConvL2I src));
10876 
10877   format %{ "movl    $dst, $src\t# l2i" %}
10878   ins_encode %{
10879     __ movl($dst$$Register, $src$$Register);
10880   %}
10881   ins_pipe(ialu_reg_reg);
10882 %}
10883 
10884 
10885 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10886   match(Set dst (MoveF2I src));
10887   effect(DEF dst, USE src);
10888 
10889   ins_cost(125);
10890   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10891   ins_encode %{
10892     __ movl($dst$$Register, Address(rsp, $src$$disp));
10893   %}
10894   ins_pipe(ialu_reg_mem);
10895 %}
10896 
10897 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10898   match(Set dst (MoveI2F src));
10899   effect(DEF dst, USE src);
10900 
10901   ins_cost(125);
10902   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10903   ins_encode %{
10904     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10905   %}
10906   ins_pipe(pipe_slow);
10907 %}
10908 
10909 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10910   match(Set dst (MoveD2L src));
10911   effect(DEF dst, USE src);
10912 
10913   ins_cost(125);
10914   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10915   ins_encode %{
10916     __ movq($dst$$Register, Address(rsp, $src$$disp));
10917   %}
10918   ins_pipe(ialu_reg_mem);
10919 %}
10920 
10921 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10922   predicate(!UseXmmLoadAndClearUpper);
10923   match(Set dst (MoveL2D src));
10924   effect(DEF dst, USE src);
10925 
10926   ins_cost(125);
10927   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10928   ins_encode %{
10929     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10930   %}
10931   ins_pipe(pipe_slow);
10932 %}
10933 
10934 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10935   predicate(UseXmmLoadAndClearUpper);
10936   match(Set dst (MoveL2D src));
10937   effect(DEF dst, USE src);
10938 
10939   ins_cost(125);
10940   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10941   ins_encode %{
10942     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10943   %}
10944   ins_pipe(pipe_slow);
10945 %}
10946 
10947 
10948 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10949   match(Set dst (MoveF2I src));
10950   effect(DEF dst, USE src);
10951 
10952   ins_cost(95); // XXX
10953   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10954   ins_encode %{
10955     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10956   %}
10957   ins_pipe(pipe_slow);
10958 %}
10959 
10960 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10961   match(Set dst (MoveI2F src));
10962   effect(DEF dst, USE src);
10963 
10964   ins_cost(100);
10965   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10966   ins_encode %{
10967     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10968   %}
10969   ins_pipe( ialu_mem_reg );
10970 %}
10971 
10972 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10973   match(Set dst (MoveD2L src));
10974   effect(DEF dst, USE src);
10975 
10976   ins_cost(95); // XXX
10977   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10978   ins_encode %{
10979     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10980   %}
10981   ins_pipe(pipe_slow);
10982 %}
10983 
10984 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10985   match(Set dst (MoveL2D src));
10986   effect(DEF dst, USE src);
10987 
10988   ins_cost(100);
10989   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10990   ins_encode %{
10991     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10992   %}
10993   ins_pipe(ialu_mem_reg);
10994 %}
10995 
10996 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10997   match(Set dst (MoveF2I src));
10998   effect(DEF dst, USE src);
10999   ins_cost(85);
11000   format %{ "movd    $dst,$src\t# MoveF2I" %}
11001   ins_encode %{
11002     __ movdl($dst$$Register, $src$$XMMRegister);
11003   %}
11004   ins_pipe( pipe_slow );
11005 %}
11006 
11007 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11008   match(Set dst (MoveD2L src));
11009   effect(DEF dst, USE src);
11010   ins_cost(85);
11011   format %{ "movd    $dst,$src\t# MoveD2L" %}
11012   ins_encode %{
11013     __ movdq($dst$$Register, $src$$XMMRegister);
11014   %}
11015   ins_pipe( pipe_slow );
11016 %}
11017 
11018 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11019   match(Set dst (MoveI2F src));
11020   effect(DEF dst, USE src);
11021   ins_cost(100);
11022   format %{ "movd    $dst,$src\t# MoveI2F" %}
11023   ins_encode %{
11024     __ movdl($dst$$XMMRegister, $src$$Register);
11025   %}
11026   ins_pipe( pipe_slow );
11027 %}
11028 
11029 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11030   match(Set dst (MoveL2D src));
11031   effect(DEF dst, USE src);
11032   ins_cost(100);
11033   format %{ "movd    $dst,$src\t# MoveL2D" %}
11034   ins_encode %{
11035      __ movdq($dst$$XMMRegister, $src$$Register);
11036   %}
11037   ins_pipe( pipe_slow );
11038 %}
11039 
11040 // Fast clearing of an array
11041 // Small ClearArray non-AVX512.
11042 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11043                   Universe dummy, rFlagsReg cr)
11044 %{
11045   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11046   match(Set dummy (ClearArray cnt base));
11047   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11048 
11049   format %{ $$template
11050     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11051     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11052     $$emit$$"jg      LARGE\n\t"
11053     $$emit$$"dec     rcx\n\t"
11054     $$emit$$"js      DONE\t# Zero length\n\t"
11055     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11056     $$emit$$"dec     rcx\n\t"
11057     $$emit$$"jge     LOOP\n\t"
11058     $$emit$$"jmp     DONE\n\t"
11059     $$emit$$"# LARGE:\n\t"
11060     if (UseFastStosb) {
11061        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11062        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11063     } else if (UseXMMForObjInit) {
11064        $$emit$$"mov     rdi,rax\n\t"
11065        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11066        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11067        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11068        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11069        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11070        $$emit$$"add     0x40,rax\n\t"
11071        $$emit$$"# L_zero_64_bytes:\n\t"
11072        $$emit$$"sub     0x8,rcx\n\t"
11073        $$emit$$"jge     L_loop\n\t"
11074        $$emit$$"add     0x4,rcx\n\t"
11075        $$emit$$"jl      L_tail\n\t"
11076        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11077        $$emit$$"add     0x20,rax\n\t"
11078        $$emit$$"sub     0x4,rcx\n\t"
11079        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11080        $$emit$$"add     0x4,rcx\n\t"
11081        $$emit$$"jle     L_end\n\t"
11082        $$emit$$"dec     rcx\n\t"
11083        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11084        $$emit$$"vmovq   xmm0,(rax)\n\t"
11085        $$emit$$"add     0x8,rax\n\t"
11086        $$emit$$"dec     rcx\n\t"
11087        $$emit$$"jge     L_sloop\n\t"
11088        $$emit$$"# L_end:\n\t"
11089     } else {
11090        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11091     }
11092     $$emit$$"# DONE"
11093   %}
11094   ins_encode %{
11095     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11096                  $tmp$$XMMRegister, false, knoreg);
11097   %}
11098   ins_pipe(pipe_slow);
11099 %}
11100 
11101 // Small ClearArray AVX512 non-constant length.
11102 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11103                        Universe dummy, rFlagsReg cr)
11104 %{
11105   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11106   match(Set dummy (ClearArray cnt base));
11107   ins_cost(125);
11108   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11109 
11110   format %{ $$template
11111     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11112     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11113     $$emit$$"jg      LARGE\n\t"
11114     $$emit$$"dec     rcx\n\t"
11115     $$emit$$"js      DONE\t# Zero length\n\t"
11116     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11117     $$emit$$"dec     rcx\n\t"
11118     $$emit$$"jge     LOOP\n\t"
11119     $$emit$$"jmp     DONE\n\t"
11120     $$emit$$"# LARGE:\n\t"
11121     if (UseFastStosb) {
11122        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11123        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11124     } else if (UseXMMForObjInit) {
11125        $$emit$$"mov     rdi,rax\n\t"
11126        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11127        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11128        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11129        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11130        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11131        $$emit$$"add     0x40,rax\n\t"
11132        $$emit$$"# L_zero_64_bytes:\n\t"
11133        $$emit$$"sub     0x8,rcx\n\t"
11134        $$emit$$"jge     L_loop\n\t"
11135        $$emit$$"add     0x4,rcx\n\t"
11136        $$emit$$"jl      L_tail\n\t"
11137        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11138        $$emit$$"add     0x20,rax\n\t"
11139        $$emit$$"sub     0x4,rcx\n\t"
11140        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11141        $$emit$$"add     0x4,rcx\n\t"
11142        $$emit$$"jle     L_end\n\t"
11143        $$emit$$"dec     rcx\n\t"
11144        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11145        $$emit$$"vmovq   xmm0,(rax)\n\t"
11146        $$emit$$"add     0x8,rax\n\t"
11147        $$emit$$"dec     rcx\n\t"
11148        $$emit$$"jge     L_sloop\n\t"
11149        $$emit$$"# L_end:\n\t"
11150     } else {
11151        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11152     }
11153     $$emit$$"# DONE"
11154   %}
11155   ins_encode %{
11156     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11157                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11158   %}
11159   ins_pipe(pipe_slow);
11160 %}
11161 
11162 // Large ClearArray non-AVX512.
11163 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11164                         Universe dummy, rFlagsReg cr)
11165 %{
11166   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11167   match(Set dummy (ClearArray cnt base));
11168   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11169 
11170   format %{ $$template
11171     if (UseFastStosb) {
11172        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11173        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11174        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11175     } else if (UseXMMForObjInit) {
11176        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11177        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11178        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11179        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11180        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11181        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11182        $$emit$$"add     0x40,rax\n\t"
11183        $$emit$$"# L_zero_64_bytes:\n\t"
11184        $$emit$$"sub     0x8,rcx\n\t"
11185        $$emit$$"jge     L_loop\n\t"
11186        $$emit$$"add     0x4,rcx\n\t"
11187        $$emit$$"jl      L_tail\n\t"
11188        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11189        $$emit$$"add     0x20,rax\n\t"
11190        $$emit$$"sub     0x4,rcx\n\t"
11191        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11192        $$emit$$"add     0x4,rcx\n\t"
11193        $$emit$$"jle     L_end\n\t"
11194        $$emit$$"dec     rcx\n\t"
11195        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11196        $$emit$$"vmovq   xmm0,(rax)\n\t"
11197        $$emit$$"add     0x8,rax\n\t"
11198        $$emit$$"dec     rcx\n\t"
11199        $$emit$$"jge     L_sloop\n\t"
11200        $$emit$$"# L_end:\n\t"
11201     } else {
11202        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11203        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11204     }
11205   %}
11206   ins_encode %{
11207     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11208                  $tmp$$XMMRegister, true, knoreg);
11209   %}
11210   ins_pipe(pipe_slow);
11211 %}
11212 
11213 // Large ClearArray AVX512.
11214 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11215                              Universe dummy, rFlagsReg cr)
11216 %{
11217   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11218   match(Set dummy (ClearArray cnt base));
11219   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11220 
11221   format %{ $$template
11222     if (UseFastStosb) {
11223        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11224        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11225        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11226     } else if (UseXMMForObjInit) {
11227        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11228        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11229        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11230        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11231        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11232        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11233        $$emit$$"add     0x40,rax\n\t"
11234        $$emit$$"# L_zero_64_bytes:\n\t"
11235        $$emit$$"sub     0x8,rcx\n\t"
11236        $$emit$$"jge     L_loop\n\t"
11237        $$emit$$"add     0x4,rcx\n\t"
11238        $$emit$$"jl      L_tail\n\t"
11239        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11240        $$emit$$"add     0x20,rax\n\t"
11241        $$emit$$"sub     0x4,rcx\n\t"
11242        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11243        $$emit$$"add     0x4,rcx\n\t"
11244        $$emit$$"jle     L_end\n\t"
11245        $$emit$$"dec     rcx\n\t"
11246        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11247        $$emit$$"vmovq   xmm0,(rax)\n\t"
11248        $$emit$$"add     0x8,rax\n\t"
11249        $$emit$$"dec     rcx\n\t"
11250        $$emit$$"jge     L_sloop\n\t"
11251        $$emit$$"# L_end:\n\t"
11252     } else {
11253        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11254        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11255     }
11256   %}
11257   ins_encode %{
11258     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11259                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11260   %}
11261   ins_pipe(pipe_slow);
11262 %}
11263 
11264 // Small ClearArray AVX512 constant length.
11265 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11266 %{
11267   predicate(!((ClearArrayNode*)n)->is_large() &&
11268               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11269   match(Set dummy (ClearArray cnt base));
11270   ins_cost(100);
11271   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11272   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11273   ins_encode %{
11274    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11275   %}
11276   ins_pipe(pipe_slow);
11277 %}
11278 
11279 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11280                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11281 %{
11282   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11283   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11284   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11285 
11286   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11287   ins_encode %{
11288     __ string_compare($str1$$Register, $str2$$Register,
11289                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11290                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11291   %}
11292   ins_pipe( pipe_slow );
11293 %}
11294 
11295 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11296                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11297 %{
11298   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11299   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11300   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11301 
11302   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11303   ins_encode %{
11304     __ string_compare($str1$$Register, $str2$$Register,
11305                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11306                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11307   %}
11308   ins_pipe( pipe_slow );
11309 %}
11310 
11311 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11312                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11313 %{
11314   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11315   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11316   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11317 
11318   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11319   ins_encode %{
11320     __ string_compare($str1$$Register, $str2$$Register,
11321                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11322                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11323   %}
11324   ins_pipe( pipe_slow );
11325 %}
11326 
11327 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11328                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11329 %{
11330   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11331   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11332   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11333 
11334   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11335   ins_encode %{
11336     __ string_compare($str1$$Register, $str2$$Register,
11337                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11338                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11339   %}
11340   ins_pipe( pipe_slow );
11341 %}
11342 
11343 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11344                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11345 %{
11346   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11347   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11348   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11349 
11350   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11351   ins_encode %{
11352     __ string_compare($str1$$Register, $str2$$Register,
11353                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11354                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11355   %}
11356   ins_pipe( pipe_slow );
11357 %}
11358 
11359 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11360                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11361 %{
11362   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11363   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11364   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11365 
11366   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11367   ins_encode %{
11368     __ string_compare($str1$$Register, $str2$$Register,
11369                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11370                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11371   %}
11372   ins_pipe( pipe_slow );
11373 %}
11374 
11375 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11376                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11377 %{
11378   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11379   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11380   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11381 
11382   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11383   ins_encode %{
11384     __ string_compare($str2$$Register, $str1$$Register,
11385                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11386                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11387   %}
11388   ins_pipe( pipe_slow );
11389 %}
11390 
11391 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11392                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11393 %{
11394   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11395   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11396   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11397 
11398   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11399   ins_encode %{
11400     __ string_compare($str2$$Register, $str1$$Register,
11401                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11402                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11403   %}
11404   ins_pipe( pipe_slow );
11405 %}
11406 
11407 // fast search of substring with known size.
11408 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11409                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11410 %{
11411   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11412   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11413   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11414 
11415   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11416   ins_encode %{
11417     int icnt2 = (int)$int_cnt2$$constant;
11418     if (icnt2 >= 16) {
11419       // IndexOf for constant substrings with size >= 16 elements
11420       // which don't need to be loaded through stack.
11421       __ string_indexofC8($str1$$Register, $str2$$Register,
11422                           $cnt1$$Register, $cnt2$$Register,
11423                           icnt2, $result$$Register,
11424                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11425     } else {
11426       // Small strings are loaded through stack if they cross page boundary.
11427       __ string_indexof($str1$$Register, $str2$$Register,
11428                         $cnt1$$Register, $cnt2$$Register,
11429                         icnt2, $result$$Register,
11430                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11431     }
11432   %}
11433   ins_pipe( pipe_slow );
11434 %}
11435 
11436 // fast search of substring with known size.
11437 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11438                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11439 %{
11440   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11441   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11442   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11443 
11444   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11445   ins_encode %{
11446     int icnt2 = (int)$int_cnt2$$constant;
11447     if (icnt2 >= 8) {
11448       // IndexOf for constant substrings with size >= 8 elements
11449       // which don't need to be loaded through stack.
11450       __ string_indexofC8($str1$$Register, $str2$$Register,
11451                           $cnt1$$Register, $cnt2$$Register,
11452                           icnt2, $result$$Register,
11453                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11454     } else {
11455       // Small strings are loaded through stack if they cross page boundary.
11456       __ string_indexof($str1$$Register, $str2$$Register,
11457                         $cnt1$$Register, $cnt2$$Register,
11458                         icnt2, $result$$Register,
11459                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11460     }
11461   %}
11462   ins_pipe( pipe_slow );
11463 %}
11464 
11465 // fast search of substring with known size.
11466 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11467                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11468 %{
11469   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11470   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11471   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11472 
11473   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11474   ins_encode %{
11475     int icnt2 = (int)$int_cnt2$$constant;
11476     if (icnt2 >= 8) {
11477       // IndexOf for constant substrings with size >= 8 elements
11478       // which don't need to be loaded through stack.
11479       __ string_indexofC8($str1$$Register, $str2$$Register,
11480                           $cnt1$$Register, $cnt2$$Register,
11481                           icnt2, $result$$Register,
11482                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11483     } else {
11484       // Small strings are loaded through stack if they cross page boundary.
11485       __ string_indexof($str1$$Register, $str2$$Register,
11486                         $cnt1$$Register, $cnt2$$Register,
11487                         icnt2, $result$$Register,
11488                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11489     }
11490   %}
11491   ins_pipe( pipe_slow );
11492 %}
11493 
11494 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11495                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11496 %{
11497   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11498   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11499   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11500 
11501   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11502   ins_encode %{
11503     __ string_indexof($str1$$Register, $str2$$Register,
11504                       $cnt1$$Register, $cnt2$$Register,
11505                       (-1), $result$$Register,
11506                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11507   %}
11508   ins_pipe( pipe_slow );
11509 %}
11510 
11511 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11512                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11513 %{
11514   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11515   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11516   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11517 
11518   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11519   ins_encode %{
11520     __ string_indexof($str1$$Register, $str2$$Register,
11521                       $cnt1$$Register, $cnt2$$Register,
11522                       (-1), $result$$Register,
11523                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11524   %}
11525   ins_pipe( pipe_slow );
11526 %}
11527 
11528 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11529                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11530 %{
11531   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11532   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11533   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11534 
11535   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11536   ins_encode %{
11537     __ string_indexof($str1$$Register, $str2$$Register,
11538                       $cnt1$$Register, $cnt2$$Register,
11539                       (-1), $result$$Register,
11540                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11541   %}
11542   ins_pipe( pipe_slow );
11543 %}
11544 
11545 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11546                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11547 %{
11548   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11549   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11550   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11551   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11552   ins_encode %{
11553     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11554                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11555   %}
11556   ins_pipe( pipe_slow );
11557 %}
11558 
11559 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11560                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11561 %{
11562   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11563   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11564   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11565   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11566   ins_encode %{
11567     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11568                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11569   %}
11570   ins_pipe( pipe_slow );
11571 %}
11572 
11573 // fast string equals
11574 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11575                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11576 %{
11577   predicate(!VM_Version::supports_avx512vlbw());
11578   match(Set result (StrEquals (Binary str1 str2) cnt));
11579   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11580 
11581   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11582   ins_encode %{
11583     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11584                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11585                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11586   %}
11587   ins_pipe( pipe_slow );
11588 %}
11589 
11590 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11591                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11592 %{
11593   predicate(VM_Version::supports_avx512vlbw());
11594   match(Set result (StrEquals (Binary str1 str2) cnt));
11595   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11596 
11597   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11598   ins_encode %{
11599     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11600                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11601                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11602   %}
11603   ins_pipe( pipe_slow );
11604 %}
11605 
11606 // fast array equals
11607 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11608                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11609 %{
11610   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11611   match(Set result (AryEq ary1 ary2));
11612   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11613 
11614   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11615   ins_encode %{
11616     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11617                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11618                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11619   %}
11620   ins_pipe( pipe_slow );
11621 %}
11622 
11623 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11624                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11625 %{
11626   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11627   match(Set result (AryEq ary1 ary2));
11628   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11629 
11630   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11631   ins_encode %{
11632     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11633                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11634                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11635   %}
11636   ins_pipe( pipe_slow );
11637 %}
11638 
11639 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11640                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11641 %{
11642   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11643   match(Set result (AryEq ary1 ary2));
11644   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11645 
11646   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11647   ins_encode %{
11648     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11649                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11650                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11651   %}
11652   ins_pipe( pipe_slow );
11653 %}
11654 
11655 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11656                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11657 %{
11658   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11659   match(Set result (AryEq ary1 ary2));
11660   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11661 
11662   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11663   ins_encode %{
11664     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11665                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11666                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11667   %}
11668   ins_pipe( pipe_slow );
11669 %}
11670 
11671 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11672                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11673 %{
11674   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11675   match(Set result (HasNegatives ary1 len));
11676   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11677 
11678   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11679   ins_encode %{
11680     __ has_negatives($ary1$$Register, $len$$Register,
11681                      $result$$Register, $tmp3$$Register,
11682                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11683   %}
11684   ins_pipe( pipe_slow );
11685 %}
11686 
11687 instruct has_negatives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11688                             legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11689 %{
11690   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11691   match(Set result (HasNegatives ary1 len));
11692   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11693 
11694   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11695   ins_encode %{
11696     __ has_negatives($ary1$$Register, $len$$Register,
11697                      $result$$Register, $tmp3$$Register,
11698                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11699   %}
11700   ins_pipe( pipe_slow );
11701 %}
11702 
11703 // fast char[] to byte[] compression
11704 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11705                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11706   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11707   match(Set result (StrCompressedCopy src (Binary dst len)));
11708   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11709          USE_KILL len, KILL tmp5, KILL cr);
11710 
11711   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11712   ins_encode %{
11713     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11714                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11715                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11716                            knoreg, knoreg);
11717   %}
11718   ins_pipe( pipe_slow );
11719 %}
11720 
11721 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11722                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11723   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11724   match(Set result (StrCompressedCopy src (Binary dst len)));
11725   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11726          USE_KILL len, KILL tmp5, KILL cr);
11727 
11728   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11729   ins_encode %{
11730     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11731                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11732                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11733                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11734   %}
11735   ins_pipe( pipe_slow );
11736 %}
11737 // fast byte[] to char[] inflation
11738 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11739                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11740   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11741   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11742   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11743 
11744   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11745   ins_encode %{
11746     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11747                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11748   %}
11749   ins_pipe( pipe_slow );
11750 %}
11751 
11752 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11753                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11754   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11755   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11756   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11757 
11758   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11759   ins_encode %{
11760     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11761                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11762   %}
11763   ins_pipe( pipe_slow );
11764 %}
11765 
11766 // encode char[] to byte[] in ISO_8859_1
11767 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11768                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11769                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11770   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11771   match(Set result (EncodeISOArray src (Binary dst len)));
11772   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11773 
11774   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11775   ins_encode %{
11776     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11777                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11778                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11779   %}
11780   ins_pipe( pipe_slow );
11781 %}
11782 
11783 // encode char[] to byte[] in ASCII
11784 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11785                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11786                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11787   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11788   match(Set result (EncodeISOArray src (Binary dst len)));
11789   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11790 
11791   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11792   ins_encode %{
11793     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11794                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11795                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11796   %}
11797   ins_pipe( pipe_slow );
11798 %}
11799 
11800 //----------Overflow Math Instructions-----------------------------------------
11801 
11802 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11803 %{
11804   match(Set cr (OverflowAddI op1 op2));
11805   effect(DEF cr, USE_KILL op1, USE op2);
11806 
11807   format %{ "addl    $op1, $op2\t# overflow check int" %}
11808 
11809   ins_encode %{
11810     __ addl($op1$$Register, $op2$$Register);
11811   %}
11812   ins_pipe(ialu_reg_reg);
11813 %}
11814 
11815 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11816 %{
11817   match(Set cr (OverflowAddI op1 op2));
11818   effect(DEF cr, USE_KILL op1, USE op2);
11819 
11820   format %{ "addl    $op1, $op2\t# overflow check int" %}
11821 
11822   ins_encode %{
11823     __ addl($op1$$Register, $op2$$constant);
11824   %}
11825   ins_pipe(ialu_reg_reg);
11826 %}
11827 
11828 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11829 %{
11830   match(Set cr (OverflowAddL op1 op2));
11831   effect(DEF cr, USE_KILL op1, USE op2);
11832 
11833   format %{ "addq    $op1, $op2\t# overflow check long" %}
11834   ins_encode %{
11835     __ addq($op1$$Register, $op2$$Register);
11836   %}
11837   ins_pipe(ialu_reg_reg);
11838 %}
11839 
11840 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11841 %{
11842   match(Set cr (OverflowAddL op1 op2));
11843   effect(DEF cr, USE_KILL op1, USE op2);
11844 
11845   format %{ "addq    $op1, $op2\t# overflow check long" %}
11846   ins_encode %{
11847     __ addq($op1$$Register, $op2$$constant);
11848   %}
11849   ins_pipe(ialu_reg_reg);
11850 %}
11851 
11852 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11853 %{
11854   match(Set cr (OverflowSubI op1 op2));
11855 
11856   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11857   ins_encode %{
11858     __ cmpl($op1$$Register, $op2$$Register);
11859   %}
11860   ins_pipe(ialu_reg_reg);
11861 %}
11862 
11863 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11864 %{
11865   match(Set cr (OverflowSubI op1 op2));
11866 
11867   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11868   ins_encode %{
11869     __ cmpl($op1$$Register, $op2$$constant);
11870   %}
11871   ins_pipe(ialu_reg_reg);
11872 %}
11873 
11874 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11875 %{
11876   match(Set cr (OverflowSubL op1 op2));
11877 
11878   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11879   ins_encode %{
11880     __ cmpq($op1$$Register, $op2$$Register);
11881   %}
11882   ins_pipe(ialu_reg_reg);
11883 %}
11884 
11885 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11886 %{
11887   match(Set cr (OverflowSubL op1 op2));
11888 
11889   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11890   ins_encode %{
11891     __ cmpq($op1$$Register, $op2$$constant);
11892   %}
11893   ins_pipe(ialu_reg_reg);
11894 %}
11895 
11896 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11897 %{
11898   match(Set cr (OverflowSubI zero op2));
11899   effect(DEF cr, USE_KILL op2);
11900 
11901   format %{ "negl    $op2\t# overflow check int" %}
11902   ins_encode %{
11903     __ negl($op2$$Register);
11904   %}
11905   ins_pipe(ialu_reg_reg);
11906 %}
11907 
11908 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11909 %{
11910   match(Set cr (OverflowSubL zero op2));
11911   effect(DEF cr, USE_KILL op2);
11912 
11913   format %{ "negq    $op2\t# overflow check long" %}
11914   ins_encode %{
11915     __ negq($op2$$Register);
11916   %}
11917   ins_pipe(ialu_reg_reg);
11918 %}
11919 
11920 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11921 %{
11922   match(Set cr (OverflowMulI op1 op2));
11923   effect(DEF cr, USE_KILL op1, USE op2);
11924 
11925   format %{ "imull    $op1, $op2\t# overflow check int" %}
11926   ins_encode %{
11927     __ imull($op1$$Register, $op2$$Register);
11928   %}
11929   ins_pipe(ialu_reg_reg_alu0);
11930 %}
11931 
11932 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11933 %{
11934   match(Set cr (OverflowMulI op1 op2));
11935   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11936 
11937   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11938   ins_encode %{
11939     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11940   %}
11941   ins_pipe(ialu_reg_reg_alu0);
11942 %}
11943 
11944 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11945 %{
11946   match(Set cr (OverflowMulL op1 op2));
11947   effect(DEF cr, USE_KILL op1, USE op2);
11948 
11949   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11950   ins_encode %{
11951     __ imulq($op1$$Register, $op2$$Register);
11952   %}
11953   ins_pipe(ialu_reg_reg_alu0);
11954 %}
11955 
11956 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11957 %{
11958   match(Set cr (OverflowMulL op1 op2));
11959   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11960 
11961   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11962   ins_encode %{
11963     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11964   %}
11965   ins_pipe(ialu_reg_reg_alu0);
11966 %}
11967 
11968 
11969 //----------Control Flow Instructions------------------------------------------
11970 // Signed compare Instructions
11971 
11972 // XXX more variants!!
11973 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11974 %{
11975   match(Set cr (CmpI op1 op2));
11976   effect(DEF cr, USE op1, USE op2);
11977 
11978   format %{ "cmpl    $op1, $op2" %}
11979   ins_encode %{
11980     __ cmpl($op1$$Register, $op2$$Register);
11981   %}
11982   ins_pipe(ialu_cr_reg_reg);
11983 %}
11984 
11985 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11986 %{
11987   match(Set cr (CmpI op1 op2));
11988 
11989   format %{ "cmpl    $op1, $op2" %}
11990   ins_encode %{
11991     __ cmpl($op1$$Register, $op2$$constant);
11992   %}
11993   ins_pipe(ialu_cr_reg_imm);
11994 %}
11995 
11996 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11997 %{
11998   match(Set cr (CmpI op1 (LoadI op2)));
11999 
12000   ins_cost(500); // XXX
12001   format %{ "cmpl    $op1, $op2" %}
12002   ins_encode %{
12003     __ cmpl($op1$$Register, $op2$$Address);
12004   %}
12005   ins_pipe(ialu_cr_reg_mem);
12006 %}
12007 
12008 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12009 %{
12010   match(Set cr (CmpI src zero));
12011 
12012   format %{ "testl   $src, $src" %}
12013   ins_encode %{
12014     __ testl($src$$Register, $src$$Register);
12015   %}
12016   ins_pipe(ialu_cr_reg_imm);
12017 %}
12018 
12019 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12020 %{
12021   match(Set cr (CmpI (AndI src con) zero));
12022 
12023   format %{ "testl   $src, $con" %}
12024   ins_encode %{
12025     __ testl($src$$Register, $con$$constant);
12026   %}
12027   ins_pipe(ialu_cr_reg_imm);
12028 %}
12029 
12030 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12031 %{
12032   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12033 
12034   format %{ "testl   $src, $mem" %}
12035   ins_encode %{
12036     __ testl($src$$Register, $mem$$Address);
12037   %}
12038   ins_pipe(ialu_cr_reg_mem);
12039 %}
12040 
12041 // Unsigned compare Instructions; really, same as signed except they
12042 // produce an rFlagsRegU instead of rFlagsReg.
12043 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12044 %{
12045   match(Set cr (CmpU op1 op2));
12046 
12047   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12048   ins_encode %{
12049     __ cmpl($op1$$Register, $op2$$Register);
12050   %}
12051   ins_pipe(ialu_cr_reg_reg);
12052 %}
12053 
12054 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12055 %{
12056   match(Set cr (CmpU op1 op2));
12057 
12058   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12059   ins_encode %{
12060     __ cmpl($op1$$Register, $op2$$constant);
12061   %}
12062   ins_pipe(ialu_cr_reg_imm);
12063 %}
12064 
12065 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12066 %{
12067   match(Set cr (CmpU op1 (LoadI op2)));
12068 
12069   ins_cost(500); // XXX
12070   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12071   ins_encode %{
12072     __ cmpl($op1$$Register, $op2$$Address);
12073   %}
12074   ins_pipe(ialu_cr_reg_mem);
12075 %}
12076 
12077 // // // Cisc-spilled version of cmpU_rReg
12078 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12079 // //%{
12080 // //  match(Set cr (CmpU (LoadI op1) op2));
12081 // //
12082 // //  format %{ "CMPu   $op1,$op2" %}
12083 // //  ins_cost(500);
12084 // //  opcode(0x39);  /* Opcode 39 /r */
12085 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12086 // //%}
12087 
12088 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12089 %{
12090   match(Set cr (CmpU src zero));
12091 
12092   format %{ "testl   $src, $src\t# unsigned" %}
12093   ins_encode %{
12094     __ testl($src$$Register, $src$$Register);
12095   %}
12096   ins_pipe(ialu_cr_reg_imm);
12097 %}
12098 
12099 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12100 %{
12101   match(Set cr (CmpP op1 op2));
12102 
12103   format %{ "cmpq    $op1, $op2\t# ptr" %}
12104   ins_encode %{
12105     __ cmpq($op1$$Register, $op2$$Register);
12106   %}
12107   ins_pipe(ialu_cr_reg_reg);
12108 %}
12109 
12110 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12111 %{
12112   match(Set cr (CmpP op1 (LoadP op2)));
12113   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12114 
12115   ins_cost(500); // XXX
12116   format %{ "cmpq    $op1, $op2\t# ptr" %}
12117   ins_encode %{
12118     __ cmpq($op1$$Register, $op2$$Address);
12119   %}
12120   ins_pipe(ialu_cr_reg_mem);
12121 %}
12122 
12123 // // // Cisc-spilled version of cmpP_rReg
12124 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12125 // //%{
12126 // //  match(Set cr (CmpP (LoadP op1) op2));
12127 // //
12128 // //  format %{ "CMPu   $op1,$op2" %}
12129 // //  ins_cost(500);
12130 // //  opcode(0x39);  /* Opcode 39 /r */
12131 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12132 // //%}
12133 
12134 // XXX this is generalized by compP_rReg_mem???
12135 // Compare raw pointer (used in out-of-heap check).
12136 // Only works because non-oop pointers must be raw pointers
12137 // and raw pointers have no anti-dependencies.
12138 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12139 %{
12140   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12141             n->in(2)->as_Load()->barrier_data() == 0);
12142   match(Set cr (CmpP op1 (LoadP op2)));
12143 
12144   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12145   ins_encode %{
12146     __ cmpq($op1$$Register, $op2$$Address);
12147   %}
12148   ins_pipe(ialu_cr_reg_mem);
12149 %}
12150 
12151 // This will generate a signed flags result. This should be OK since
12152 // any compare to a zero should be eq/neq.
12153 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12154 %{
12155   match(Set cr (CmpP src zero));
12156 
12157   format %{ "testq   $src, $src\t# ptr" %}
12158   ins_encode %{
12159     __ testq($src$$Register, $src$$Register);
12160   %}
12161   ins_pipe(ialu_cr_reg_imm);
12162 %}
12163 
12164 // This will generate a signed flags result. This should be OK since
12165 // any compare to a zero should be eq/neq.
12166 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12167 %{
12168   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12169             n->in(1)->as_Load()->barrier_data() == 0);
12170   match(Set cr (CmpP (LoadP op) zero));
12171 
12172   ins_cost(500); // XXX
12173   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12174   ins_encode %{
12175     __ testq($op$$Address, 0xFFFFFFFF);
12176   %}
12177   ins_pipe(ialu_cr_reg_imm);
12178 %}
12179 
12180 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12181 %{
12182   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12183             n->in(1)->as_Load()->barrier_data() == 0);
12184   match(Set cr (CmpP (LoadP mem) zero));
12185 
12186   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12187   ins_encode %{
12188     __ cmpq(r12, $mem$$Address);
12189   %}
12190   ins_pipe(ialu_cr_reg_mem);
12191 %}
12192 
12193 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12194 %{
12195   match(Set cr (CmpN op1 op2));
12196 
12197   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12198   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12199   ins_pipe(ialu_cr_reg_reg);
12200 %}
12201 
12202 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12203 %{
12204   match(Set cr (CmpN src (LoadN mem)));
12205 
12206   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12207   ins_encode %{
12208     __ cmpl($src$$Register, $mem$$Address);
12209   %}
12210   ins_pipe(ialu_cr_reg_mem);
12211 %}
12212 
12213 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12214   match(Set cr (CmpN op1 op2));
12215 
12216   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12217   ins_encode %{
12218     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12219   %}
12220   ins_pipe(ialu_cr_reg_imm);
12221 %}
12222 
12223 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12224 %{
12225   match(Set cr (CmpN src (LoadN mem)));
12226 
12227   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12228   ins_encode %{
12229     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12230   %}
12231   ins_pipe(ialu_cr_reg_mem);
12232 %}
12233 
12234 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12235   match(Set cr (CmpN op1 op2));
12236 
12237   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12238   ins_encode %{
12239     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12240   %}
12241   ins_pipe(ialu_cr_reg_imm);
12242 %}
12243 
12244 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12245 %{
12246   predicate(!UseCompactObjectHeaders);
12247   match(Set cr (CmpN src (LoadNKlass mem)));
12248 
12249   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12250   ins_encode %{
12251     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12252   %}
12253   ins_pipe(ialu_cr_reg_mem);
12254 %}
12255 
12256 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12257   match(Set cr (CmpN src zero));
12258 
12259   format %{ "testl   $src, $src\t# compressed ptr" %}
12260   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12261   ins_pipe(ialu_cr_reg_imm);
12262 %}
12263 
12264 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12265 %{
12266   predicate(CompressedOops::base() != NULL);
12267   match(Set cr (CmpN (LoadN mem) zero));
12268 
12269   ins_cost(500); // XXX
12270   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12271   ins_encode %{
12272     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12273   %}
12274   ins_pipe(ialu_cr_reg_mem);
12275 %}
12276 
12277 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12278 %{
12279   predicate(CompressedOops::base() == NULL);
12280   match(Set cr (CmpN (LoadN mem) zero));
12281 
12282   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12283   ins_encode %{
12284     __ cmpl(r12, $mem$$Address);
12285   %}
12286   ins_pipe(ialu_cr_reg_mem);
12287 %}
12288 
12289 // Yanked all unsigned pointer compare operations.
12290 // Pointer compares are done with CmpP which is already unsigned.
12291 
12292 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12293 %{
12294   match(Set cr (CmpL op1 op2));
12295 
12296   format %{ "cmpq    $op1, $op2" %}
12297   ins_encode %{
12298     __ cmpq($op1$$Register, $op2$$Register);
12299   %}
12300   ins_pipe(ialu_cr_reg_reg);
12301 %}
12302 
12303 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12304 %{
12305   match(Set cr (CmpL op1 op2));
12306 
12307   format %{ "cmpq    $op1, $op2" %}
12308   ins_encode %{
12309     __ cmpq($op1$$Register, $op2$$constant);
12310   %}
12311   ins_pipe(ialu_cr_reg_imm);
12312 %}
12313 
12314 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12315 %{
12316   match(Set cr (CmpL op1 (LoadL op2)));
12317 
12318   format %{ "cmpq    $op1, $op2" %}
12319   ins_encode %{
12320     __ cmpq($op1$$Register, $op2$$Address);
12321   %}
12322   ins_pipe(ialu_cr_reg_mem);
12323 %}
12324 
12325 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12326 %{
12327   match(Set cr (CmpL src zero));
12328 
12329   format %{ "testq   $src, $src" %}
12330   ins_encode %{
12331     __ testq($src$$Register, $src$$Register);
12332   %}
12333   ins_pipe(ialu_cr_reg_imm);
12334 %}
12335 
12336 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12337 %{
12338   match(Set cr (CmpL (AndL src con) zero));
12339 
12340   format %{ "testq   $src, $con\t# long" %}
12341   ins_encode %{
12342     __ testq($src$$Register, $con$$constant);
12343   %}
12344   ins_pipe(ialu_cr_reg_imm);
12345 %}
12346 
12347 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12348 %{
12349   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12350 
12351   format %{ "testq   $src, $mem" %}
12352   ins_encode %{
12353     __ testq($src$$Register, $mem$$Address);
12354   %}
12355   ins_pipe(ialu_cr_reg_mem);
12356 %}
12357 
12358 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12359 %{
12360   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12361 
12362   format %{ "testq   $src, $mem" %}
12363   ins_encode %{
12364     __ testq($src$$Register, $mem$$Address);
12365   %}
12366   ins_pipe(ialu_cr_reg_mem);
12367 %}
12368 
12369 // Manifest a CmpL result in an integer register.  Very painful.
12370 // This is the test to avoid.
12371 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12372 %{
12373   match(Set dst (CmpL3 src1 src2));
12374   effect(KILL flags);
12375 
12376   ins_cost(275); // XXX
12377   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12378             "movl    $dst, -1\n\t"
12379             "jl,s    done\n\t"
12380             "setne   $dst\n\t"
12381             "movzbl  $dst, $dst\n\t"
12382     "done:" %}
12383   ins_encode %{
12384     Label done;
12385     __ cmpq($src1$$Register, $src2$$Register);
12386     __ movl($dst$$Register, -1);
12387     __ jccb(Assembler::less, done);
12388     __ setne($dst$$Register);
12389     __ movzbl($dst$$Register, $dst$$Register);
12390     __ bind(done);
12391   %}
12392   ins_pipe(pipe_slow);
12393 %}
12394 
12395 // Unsigned long compare Instructions; really, same as signed long except they
12396 // produce an rFlagsRegU instead of rFlagsReg.
12397 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12398 %{
12399   match(Set cr (CmpUL op1 op2));
12400 
12401   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12402   ins_encode %{
12403     __ cmpq($op1$$Register, $op2$$Register);
12404   %}
12405   ins_pipe(ialu_cr_reg_reg);
12406 %}
12407 
12408 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12409 %{
12410   match(Set cr (CmpUL op1 op2));
12411 
12412   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12413   ins_encode %{
12414     __ cmpq($op1$$Register, $op2$$constant);
12415   %}
12416   ins_pipe(ialu_cr_reg_imm);
12417 %}
12418 
12419 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12420 %{
12421   match(Set cr (CmpUL op1 (LoadL op2)));
12422 
12423   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12424   ins_encode %{
12425     __ cmpq($op1$$Register, $op2$$Address);
12426   %}
12427   ins_pipe(ialu_cr_reg_mem);
12428 %}
12429 
12430 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12431 %{
12432   match(Set cr (CmpUL src zero));
12433 
12434   format %{ "testq   $src, $src\t# unsigned" %}
12435   ins_encode %{
12436     __ testq($src$$Register, $src$$Register);
12437   %}
12438   ins_pipe(ialu_cr_reg_imm);
12439 %}
12440 
12441 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12442 %{
12443   match(Set cr (CmpI (LoadB mem) imm));
12444 
12445   ins_cost(125);
12446   format %{ "cmpb    $mem, $imm" %}
12447   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12448   ins_pipe(ialu_cr_reg_mem);
12449 %}
12450 
12451 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12452 %{
12453   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12454 
12455   ins_cost(125);
12456   format %{ "testb   $mem, $imm\t# ubyte" %}
12457   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12458   ins_pipe(ialu_cr_reg_mem);
12459 %}
12460 
12461 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12462 %{
12463   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12464 
12465   ins_cost(125);
12466   format %{ "testb   $mem, $imm\t# byte" %}
12467   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12468   ins_pipe(ialu_cr_reg_mem);
12469 %}
12470 
12471 //----------Max and Min--------------------------------------------------------
12472 // Min Instructions
12473 
12474 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12475 %{
12476   effect(USE_DEF dst, USE src, USE cr);
12477 
12478   format %{ "cmovlgt $dst, $src\t# min" %}
12479   ins_encode %{
12480     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12481   %}
12482   ins_pipe(pipe_cmov_reg);
12483 %}
12484 
12485 
12486 instruct minI_rReg(rRegI dst, rRegI src)
12487 %{
12488   match(Set dst (MinI dst src));
12489 
12490   ins_cost(200);
12491   expand %{
12492     rFlagsReg cr;
12493     compI_rReg(cr, dst, src);
12494     cmovI_reg_g(dst, src, cr);
12495   %}
12496 %}
12497 
12498 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12499 %{
12500   effect(USE_DEF dst, USE src, USE cr);
12501 
12502   format %{ "cmovllt $dst, $src\t# max" %}
12503   ins_encode %{
12504     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12505   %}
12506   ins_pipe(pipe_cmov_reg);
12507 %}
12508 
12509 
12510 instruct maxI_rReg(rRegI dst, rRegI src)
12511 %{
12512   match(Set dst (MaxI dst src));
12513 
12514   ins_cost(200);
12515   expand %{
12516     rFlagsReg cr;
12517     compI_rReg(cr, dst, src);
12518     cmovI_reg_l(dst, src, cr);
12519   %}
12520 %}
12521 
12522 // ============================================================================
12523 // Branch Instructions
12524 
12525 // Jump Direct - Label defines a relative address from JMP+1
12526 instruct jmpDir(label labl)
12527 %{
12528   match(Goto);
12529   effect(USE labl);
12530 
12531   ins_cost(300);
12532   format %{ "jmp     $labl" %}
12533   size(5);
12534   ins_encode %{
12535     Label* L = $labl$$label;
12536     __ jmp(*L, false); // Always long jump
12537   %}
12538   ins_pipe(pipe_jmp);
12539 %}
12540 
12541 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12542 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12543 %{
12544   match(If cop cr);
12545   effect(USE labl);
12546 
12547   ins_cost(300);
12548   format %{ "j$cop     $labl" %}
12549   size(6);
12550   ins_encode %{
12551     Label* L = $labl$$label;
12552     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12553   %}
12554   ins_pipe(pipe_jcc);
12555 %}
12556 
12557 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12558 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12559 %{
12560   predicate(!n->has_vector_mask_set());
12561   match(CountedLoopEnd cop cr);
12562   effect(USE labl);
12563 
12564   ins_cost(300);
12565   format %{ "j$cop     $labl\t# loop end" %}
12566   size(6);
12567   ins_encode %{
12568     Label* L = $labl$$label;
12569     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12570   %}
12571   ins_pipe(pipe_jcc);
12572 %}
12573 
12574 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12575 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12576   predicate(!n->has_vector_mask_set());
12577   match(CountedLoopEnd cop cmp);
12578   effect(USE labl);
12579 
12580   ins_cost(300);
12581   format %{ "j$cop,u   $labl\t# loop end" %}
12582   size(6);
12583   ins_encode %{
12584     Label* L = $labl$$label;
12585     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12586   %}
12587   ins_pipe(pipe_jcc);
12588 %}
12589 
12590 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12591   predicate(!n->has_vector_mask_set());
12592   match(CountedLoopEnd cop cmp);
12593   effect(USE labl);
12594 
12595   ins_cost(200);
12596   format %{ "j$cop,u   $labl\t# loop end" %}
12597   size(6);
12598   ins_encode %{
12599     Label* L = $labl$$label;
12600     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12601   %}
12602   ins_pipe(pipe_jcc);
12603 %}
12604 
12605 // mask version
12606 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12607 // Bounded mask operand used in following patten is needed for
12608 // post-loop multiversioning.
12609 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, kReg_K1 ktmp, rFlagsReg cr, label labl)
12610 %{
12611   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12612   match(CountedLoopEnd cop cr);
12613   effect(USE labl, TEMP ktmp);
12614 
12615   ins_cost(400);
12616   format %{ "j$cop     $labl\t# loop end\n\t"
12617             "restorevectmask \t# vector mask restore for loops" %}
12618   size(10);
12619   ins_encode %{
12620     Label* L = $labl$$label;
12621     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12622     __ restorevectmask($ktmp$$KRegister);
12623   %}
12624   ins_pipe(pipe_jcc);
12625 %}
12626 
12627 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12628 // Bounded mask operand used in following patten is needed for
12629 // post-loop multiversioning.
12630 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, kReg_K1 ktmp, rFlagsRegU cmp, label labl) %{
12631   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12632   match(CountedLoopEnd cop cmp);
12633   effect(USE labl, TEMP ktmp);
12634 
12635   ins_cost(400);
12636   format %{ "j$cop,u   $labl\t# loop end\n\t"
12637             "restorevectmask \t# vector mask restore for loops" %}
12638   size(10);
12639   ins_encode %{
12640     Label* L = $labl$$label;
12641     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12642     __ restorevectmask($ktmp$$KRegister);
12643   %}
12644   ins_pipe(pipe_jcc);
12645 %}
12646 
12647 // Bounded mask operand used in following patten is needed for
12648 // post-loop multiversioning.
12649 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, kReg_K1 ktmp, rFlagsRegUCF cmp, label labl) %{
12650   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12651   match(CountedLoopEnd cop cmp);
12652   effect(USE labl, TEMP ktmp);
12653 
12654   ins_cost(300);
12655   format %{ "j$cop,u   $labl\t# loop end\n\t"
12656             "restorevectmask \t# vector mask restore for loops" %}
12657   size(10);
12658   ins_encode %{
12659     Label* L = $labl$$label;
12660     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12661     __ restorevectmask($ktmp$$KRegister);
12662   %}
12663   ins_pipe(pipe_jcc);
12664 %}
12665 
12666 // Jump Direct Conditional - using unsigned comparison
12667 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12668   match(If cop cmp);
12669   effect(USE labl);
12670 
12671   ins_cost(300);
12672   format %{ "j$cop,u   $labl" %}
12673   size(6);
12674   ins_encode %{
12675     Label* L = $labl$$label;
12676     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12677   %}
12678   ins_pipe(pipe_jcc);
12679 %}
12680 
12681 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12682   match(If cop cmp);
12683   effect(USE labl);
12684 
12685   ins_cost(200);
12686   format %{ "j$cop,u   $labl" %}
12687   size(6);
12688   ins_encode %{
12689     Label* L = $labl$$label;
12690     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12691   %}
12692   ins_pipe(pipe_jcc);
12693 %}
12694 
12695 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12696   match(If cop cmp);
12697   effect(USE labl);
12698 
12699   ins_cost(200);
12700   format %{ $$template
12701     if ($cop$$cmpcode == Assembler::notEqual) {
12702       $$emit$$"jp,u    $labl\n\t"
12703       $$emit$$"j$cop,u   $labl"
12704     } else {
12705       $$emit$$"jp,u    done\n\t"
12706       $$emit$$"j$cop,u   $labl\n\t"
12707       $$emit$$"done:"
12708     }
12709   %}
12710   ins_encode %{
12711     Label* l = $labl$$label;
12712     if ($cop$$cmpcode == Assembler::notEqual) {
12713       __ jcc(Assembler::parity, *l, false);
12714       __ jcc(Assembler::notEqual, *l, false);
12715     } else if ($cop$$cmpcode == Assembler::equal) {
12716       Label done;
12717       __ jccb(Assembler::parity, done);
12718       __ jcc(Assembler::equal, *l, false);
12719       __ bind(done);
12720     } else {
12721        ShouldNotReachHere();
12722     }
12723   %}
12724   ins_pipe(pipe_jcc);
12725 %}
12726 
12727 // ============================================================================
12728 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12729 // superklass array for an instance of the superklass.  Set a hidden
12730 // internal cache on a hit (cache is checked with exposed code in
12731 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12732 // encoding ALSO sets flags.
12733 
12734 instruct partialSubtypeCheck(rdi_RegP result,
12735                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12736                              rFlagsReg cr)
12737 %{
12738   match(Set result (PartialSubtypeCheck sub super));
12739   effect(KILL rcx, KILL cr);
12740 
12741   ins_cost(1100);  // slightly larger than the next version
12742   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12743             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12744             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12745             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12746             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12747             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12748             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12749     "miss:\t" %}
12750 
12751   opcode(0x1); // Force a XOR of RDI
12752   ins_encode(enc_PartialSubtypeCheck());
12753   ins_pipe(pipe_slow);
12754 %}
12755 
12756 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12757                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12758                                      immP0 zero,
12759                                      rdi_RegP result)
12760 %{
12761   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12762   effect(KILL rcx, KILL result);
12763 
12764   ins_cost(1000);
12765   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12766             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12767             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12768             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12769             "jne,s   miss\t\t# Missed: flags nz\n\t"
12770             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12771     "miss:\t" %}
12772 
12773   opcode(0x0); // No need to XOR RDI
12774   ins_encode(enc_PartialSubtypeCheck());
12775   ins_pipe(pipe_slow);
12776 %}
12777 
12778 // ============================================================================
12779 // Branch Instructions -- short offset versions
12780 //
12781 // These instructions are used to replace jumps of a long offset (the default
12782 // match) with jumps of a shorter offset.  These instructions are all tagged
12783 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12784 // match rules in general matching.  Instead, the ADLC generates a conversion
12785 // method in the MachNode which can be used to do in-place replacement of the
12786 // long variant with the shorter variant.  The compiler will determine if a
12787 // branch can be taken by the is_short_branch_offset() predicate in the machine
12788 // specific code section of the file.
12789 
12790 // Jump Direct - Label defines a relative address from JMP+1
12791 instruct jmpDir_short(label labl) %{
12792   match(Goto);
12793   effect(USE labl);
12794 
12795   ins_cost(300);
12796   format %{ "jmp,s   $labl" %}
12797   size(2);
12798   ins_encode %{
12799     Label* L = $labl$$label;
12800     __ jmpb(*L);
12801   %}
12802   ins_pipe(pipe_jmp);
12803   ins_short_branch(1);
12804 %}
12805 
12806 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12807 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12808   match(If cop cr);
12809   effect(USE labl);
12810 
12811   ins_cost(300);
12812   format %{ "j$cop,s   $labl" %}
12813   size(2);
12814   ins_encode %{
12815     Label* L = $labl$$label;
12816     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12817   %}
12818   ins_pipe(pipe_jcc);
12819   ins_short_branch(1);
12820 %}
12821 
12822 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12823 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12824   match(CountedLoopEnd cop cr);
12825   effect(USE labl);
12826 
12827   ins_cost(300);
12828   format %{ "j$cop,s   $labl\t# loop end" %}
12829   size(2);
12830   ins_encode %{
12831     Label* L = $labl$$label;
12832     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12833   %}
12834   ins_pipe(pipe_jcc);
12835   ins_short_branch(1);
12836 %}
12837 
12838 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12839 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12840   match(CountedLoopEnd cop cmp);
12841   effect(USE labl);
12842 
12843   ins_cost(300);
12844   format %{ "j$cop,us  $labl\t# loop end" %}
12845   size(2);
12846   ins_encode %{
12847     Label* L = $labl$$label;
12848     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12849   %}
12850   ins_pipe(pipe_jcc);
12851   ins_short_branch(1);
12852 %}
12853 
12854 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12855   match(CountedLoopEnd cop cmp);
12856   effect(USE labl);
12857 
12858   ins_cost(300);
12859   format %{ "j$cop,us  $labl\t# loop end" %}
12860   size(2);
12861   ins_encode %{
12862     Label* L = $labl$$label;
12863     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12864   %}
12865   ins_pipe(pipe_jcc);
12866   ins_short_branch(1);
12867 %}
12868 
12869 // Jump Direct Conditional - using unsigned comparison
12870 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12871   match(If cop cmp);
12872   effect(USE labl);
12873 
12874   ins_cost(300);
12875   format %{ "j$cop,us  $labl" %}
12876   size(2);
12877   ins_encode %{
12878     Label* L = $labl$$label;
12879     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12880   %}
12881   ins_pipe(pipe_jcc);
12882   ins_short_branch(1);
12883 %}
12884 
12885 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12886   match(If cop cmp);
12887   effect(USE labl);
12888 
12889   ins_cost(300);
12890   format %{ "j$cop,us  $labl" %}
12891   size(2);
12892   ins_encode %{
12893     Label* L = $labl$$label;
12894     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12895   %}
12896   ins_pipe(pipe_jcc);
12897   ins_short_branch(1);
12898 %}
12899 
12900 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12901   match(If cop cmp);
12902   effect(USE labl);
12903 
12904   ins_cost(300);
12905   format %{ $$template
12906     if ($cop$$cmpcode == Assembler::notEqual) {
12907       $$emit$$"jp,u,s  $labl\n\t"
12908       $$emit$$"j$cop,u,s  $labl"
12909     } else {
12910       $$emit$$"jp,u,s  done\n\t"
12911       $$emit$$"j$cop,u,s  $labl\n\t"
12912       $$emit$$"done:"
12913     }
12914   %}
12915   size(4);
12916   ins_encode %{
12917     Label* l = $labl$$label;
12918     if ($cop$$cmpcode == Assembler::notEqual) {
12919       __ jccb(Assembler::parity, *l);
12920       __ jccb(Assembler::notEqual, *l);
12921     } else if ($cop$$cmpcode == Assembler::equal) {
12922       Label done;
12923       __ jccb(Assembler::parity, done);
12924       __ jccb(Assembler::equal, *l);
12925       __ bind(done);
12926     } else {
12927        ShouldNotReachHere();
12928     }
12929   %}
12930   ins_pipe(pipe_jcc);
12931   ins_short_branch(1);
12932 %}
12933 
12934 // ============================================================================
12935 // inlined locking and unlocking
12936 
12937 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12938   predicate(Compile::current()->use_rtm());
12939   match(Set cr (FastLock object box));
12940   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12941   ins_cost(300);
12942   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12943   ins_encode %{
12944     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12945                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
12946                  _counters, _rtm_counters, _stack_rtm_counters,
12947                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12948                  true, ra_->C->profile_rtm());
12949   %}
12950   ins_pipe(pipe_slow);
12951 %}
12952 
12953 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
12954   predicate(!Compile::current()->use_rtm());
12955   match(Set cr (FastLock object box));
12956   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
12957   ins_cost(300);
12958   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12959   ins_encode %{
12960     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12961                  $scr$$Register, $cx1$$Register, noreg, r15_thread, _counters, NULL, NULL, NULL, false, false);
12962   %}
12963   ins_pipe(pipe_slow);
12964 %}
12965 
12966 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12967   match(Set cr (FastUnlock object box));
12968   effect(TEMP tmp, USE_KILL box);
12969   ins_cost(300);
12970   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12971   ins_encode %{
12972     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12973   %}
12974   ins_pipe(pipe_slow);
12975 %}
12976 
12977 
12978 // ============================================================================
12979 // Safepoint Instructions
12980 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12981 %{
12982   match(SafePoint poll);
12983   effect(KILL cr, USE poll);
12984 
12985   format %{ "testl   rax, [$poll]\t"
12986             "# Safepoint: poll for GC" %}
12987   ins_cost(125);
12988   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12989   ins_encode %{
12990     __ relocate(relocInfo::poll_type);
12991     address pre_pc = __ pc();
12992     __ testl(rax, Address($poll$$Register, 0));
12993     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12994   %}
12995   ins_pipe(ialu_reg_mem);
12996 %}
12997 
12998 // ============================================================================
12999 // Procedure Call/Return Instructions
13000 // Call Java Static Instruction
13001 // Note: If this code changes, the corresponding ret_addr_offset() and
13002 //       compute_padding() functions will have to be adjusted.
13003 instruct CallStaticJavaDirect(method meth) %{
13004   match(CallStaticJava);
13005   effect(USE meth);
13006 
13007   ins_cost(300);
13008   format %{ "call,static " %}
13009   opcode(0xE8); /* E8 cd */
13010   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13011   ins_pipe(pipe_slow);
13012   ins_alignment(4);
13013 %}
13014 
13015 // Call Java Dynamic Instruction
13016 // Note: If this code changes, the corresponding ret_addr_offset() and
13017 //       compute_padding() functions will have to be adjusted.
13018 instruct CallDynamicJavaDirect(method meth)
13019 %{
13020   match(CallDynamicJava);
13021   effect(USE meth);
13022 
13023   ins_cost(300);
13024   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13025             "call,dynamic " %}
13026   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13027   ins_pipe(pipe_slow);
13028   ins_alignment(4);
13029 %}
13030 
13031 // Call Runtime Instruction
13032 instruct CallRuntimeDirect(method meth)
13033 %{
13034   match(CallRuntime);
13035   effect(USE meth);
13036 
13037   ins_cost(300);
13038   format %{ "call,runtime " %}
13039   ins_encode(clear_avx, Java_To_Runtime(meth));
13040   ins_pipe(pipe_slow);
13041 %}
13042 
13043 // Call runtime without safepoint
13044 instruct CallLeafDirect(method meth)
13045 %{
13046   match(CallLeaf);
13047   effect(USE meth);
13048 
13049   ins_cost(300);
13050   format %{ "call_leaf,runtime " %}
13051   ins_encode(clear_avx, Java_To_Runtime(meth));
13052   ins_pipe(pipe_slow);
13053 %}
13054 
13055 // Call runtime without safepoint and with vector arguments
13056 instruct CallLeafDirectVector(method meth)
13057 %{
13058   match(CallLeafVector);
13059   effect(USE meth);
13060 
13061   ins_cost(300);
13062   format %{ "call_leaf,vector " %}
13063   ins_encode(Java_To_Runtime(meth));
13064   ins_pipe(pipe_slow);
13065 %}
13066 
13067 //
13068 instruct CallNativeDirect(method meth)
13069 %{
13070   match(CallNative);
13071   effect(USE meth);
13072 
13073   ins_cost(300);
13074   format %{ "call_native " %}
13075   ins_encode(clear_avx, Java_To_Runtime(meth));
13076   ins_pipe(pipe_slow);
13077 %}
13078 
13079 // Call runtime without safepoint
13080 instruct CallLeafNoFPDirect(method meth)
13081 %{
13082   match(CallLeafNoFP);
13083   effect(USE meth);
13084 
13085   ins_cost(300);
13086   format %{ "call_leaf_nofp,runtime " %}
13087   ins_encode(clear_avx, Java_To_Runtime(meth));
13088   ins_pipe(pipe_slow);
13089 %}
13090 
13091 // Return Instruction
13092 // Remove the return address & jump to it.
13093 // Notice: We always emit a nop after a ret to make sure there is room
13094 // for safepoint patching
13095 instruct Ret()
13096 %{
13097   match(Return);
13098 
13099   format %{ "ret" %}
13100   ins_encode %{
13101     __ ret(0);
13102   %}
13103   ins_pipe(pipe_jmp);
13104 %}
13105 
13106 // Tail Call; Jump from runtime stub to Java code.
13107 // Also known as an 'interprocedural jump'.
13108 // Target of jump will eventually return to caller.
13109 // TailJump below removes the return address.
13110 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13111 %{
13112   match(TailCall jump_target method_ptr);
13113 
13114   ins_cost(300);
13115   format %{ "jmp     $jump_target\t# rbx holds method" %}
13116   ins_encode %{
13117     __ jmp($jump_target$$Register);
13118   %}
13119   ins_pipe(pipe_jmp);
13120 %}
13121 
13122 // Tail Jump; remove the return address; jump to target.
13123 // TailCall above leaves the return address around.
13124 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13125 %{
13126   match(TailJump jump_target ex_oop);
13127 
13128   ins_cost(300);
13129   format %{ "popq    rdx\t# pop return address\n\t"
13130             "jmp     $jump_target" %}
13131   ins_encode %{
13132     __ popq(as_Register(RDX_enc));
13133     __ jmp($jump_target$$Register);
13134   %}
13135   ins_pipe(pipe_jmp);
13136 %}
13137 
13138 // Create exception oop: created by stack-crawling runtime code.
13139 // Created exception is now available to this handler, and is setup
13140 // just prior to jumping to this handler.  No code emitted.
13141 instruct CreateException(rax_RegP ex_oop)
13142 %{
13143   match(Set ex_oop (CreateEx));
13144 
13145   size(0);
13146   // use the following format syntax
13147   format %{ "# exception oop is in rax; no code emitted" %}
13148   ins_encode();
13149   ins_pipe(empty);
13150 %}
13151 
13152 // Rethrow exception:
13153 // The exception oop will come in the first argument position.
13154 // Then JUMP (not call) to the rethrow stub code.
13155 instruct RethrowException()
13156 %{
13157   match(Rethrow);
13158 
13159   // use the following format syntax
13160   format %{ "jmp     rethrow_stub" %}
13161   ins_encode(enc_rethrow);
13162   ins_pipe(pipe_jmp);
13163 %}
13164 
13165 // ============================================================================
13166 // This name is KNOWN by the ADLC and cannot be changed.
13167 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13168 // for this guy.
13169 instruct tlsLoadP(r15_RegP dst) %{
13170   match(Set dst (ThreadLocal));
13171   effect(DEF dst);
13172 
13173   size(0);
13174   format %{ "# TLS is in R15" %}
13175   ins_encode( /*empty encoding*/ );
13176   ins_pipe(ialu_reg_reg);
13177 %}
13178 
13179 
13180 //----------PEEPHOLE RULES-----------------------------------------------------
13181 // These must follow all instruction definitions as they use the names
13182 // defined in the instructions definitions.
13183 //
13184 // peepmatch ( root_instr_name [preceding_instruction]* );
13185 //
13186 // peepconstraint %{
13187 // (instruction_number.operand_name relational_op instruction_number.operand_name
13188 //  [, ...] );
13189 // // instruction numbers are zero-based using left to right order in peepmatch
13190 //
13191 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13192 // // provide an instruction_number.operand_name for each operand that appears
13193 // // in the replacement instruction's match rule
13194 //
13195 // ---------VM FLAGS---------------------------------------------------------
13196 //
13197 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13198 //
13199 // Each peephole rule is given an identifying number starting with zero and
13200 // increasing by one in the order seen by the parser.  An individual peephole
13201 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13202 // on the command-line.
13203 //
13204 // ---------CURRENT LIMITATIONS----------------------------------------------
13205 //
13206 // Only match adjacent instructions in same basic block
13207 // Only equality constraints
13208 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13209 // Only one replacement instruction
13210 //
13211 // ---------EXAMPLE----------------------------------------------------------
13212 //
13213 // // pertinent parts of existing instructions in architecture description
13214 // instruct movI(rRegI dst, rRegI src)
13215 // %{
13216 //   match(Set dst (CopyI src));
13217 // %}
13218 //
13219 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13220 // %{
13221 //   match(Set dst (AddI dst src));
13222 //   effect(KILL cr);
13223 // %}
13224 //
13225 // // Change (inc mov) to lea
13226 // peephole %{
13227 //   // increment preceeded by register-register move
13228 //   peepmatch ( incI_rReg movI );
13229 //   // require that the destination register of the increment
13230 //   // match the destination register of the move
13231 //   peepconstraint ( 0.dst == 1.dst );
13232 //   // construct a replacement instruction that sets
13233 //   // the destination to ( move's source register + one )
13234 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13235 // %}
13236 //
13237 
13238 // Implementation no longer uses movX instructions since
13239 // machine-independent system no longer uses CopyX nodes.
13240 //
13241 // peephole
13242 // %{
13243 //   peepmatch (incI_rReg movI);
13244 //   peepconstraint (0.dst == 1.dst);
13245 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13246 // %}
13247 
13248 // peephole
13249 // %{
13250 //   peepmatch (decI_rReg movI);
13251 //   peepconstraint (0.dst == 1.dst);
13252 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13253 // %}
13254 
13255 // peephole
13256 // %{
13257 //   peepmatch (addI_rReg_imm movI);
13258 //   peepconstraint (0.dst == 1.dst);
13259 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13260 // %}
13261 
13262 // peephole
13263 // %{
13264 //   peepmatch (incL_rReg movL);
13265 //   peepconstraint (0.dst == 1.dst);
13266 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13267 // %}
13268 
13269 // peephole
13270 // %{
13271 //   peepmatch (decL_rReg movL);
13272 //   peepconstraint (0.dst == 1.dst);
13273 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13274 // %}
13275 
13276 // peephole
13277 // %{
13278 //   peepmatch (addL_rReg_imm movL);
13279 //   peepconstraint (0.dst == 1.dst);
13280 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13281 // %}
13282 
13283 // peephole
13284 // %{
13285 //   peepmatch (addP_rReg_imm movP);
13286 //   peepconstraint (0.dst == 1.dst);
13287 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13288 // %}
13289 
13290 // // Change load of spilled value to only a spill
13291 // instruct storeI(memory mem, rRegI src)
13292 // %{
13293 //   match(Set mem (StoreI mem src));
13294 // %}
13295 //
13296 // instruct loadI(rRegI dst, memory mem)
13297 // %{
13298 //   match(Set dst (LoadI mem));
13299 // %}
13300 //
13301 
13302 peephole
13303 %{
13304   peepmatch (loadI storeI);
13305   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13306   peepreplace (storeI(1.mem 1.mem 1.src));
13307 %}
13308 
13309 peephole
13310 %{
13311   peepmatch (loadL storeL);
13312   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13313   peepreplace (storeL(1.mem 1.mem 1.src));
13314 %}
13315 
13316 //----------SMARTSPILL RULES---------------------------------------------------
13317 // These must follow all instruction definitions as they use the names
13318 // defined in the instructions definitions.